I setup a promtheus server outside kubernetes cluster. My configuration has 2 scrape jobs for kubernetes-apiservers and kubernetes-nodes metrics. Target for kubernetes-apiservers endpoint is up, but with kubernetes-nodes, it shows error with "context deadline exceeded" .Please have look and give me advice.
global:
scrape_interval: 2m
scrape_timeout: 30s
scrape_configs:
- job_name: 'prometheus_master'
scrape_interval: 10s
static_configs:
- targets: ['localhost:9090']
- job_name: 'kubernetes-apiservers'
kubernetes_sd_configs:
- role: endpoints
kubeconfig_file: /etc/prometheus/dev-zznty9y19-kubeconfig
scheme: https
tls_config:
insecure_skip_verify: true
ca_file: /etc/prometheus/ca.crt
authorization:
# bearer_token_file: /etc/prometheus/dev-zznty9y19-kubeconfig
credentials_file: /etc/prometheus/token
relabel_configs:
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
action: keep
regex: default;kubernetes;https
- job_name: 'kubernetes-nodes'
kubernetes_sd_configs:
- role: node
# api_server: https://172.16.21.31
kubeconfig_file: /etc/prometheus/dev-zznty9y19-kubeconfig
scheme: https
tls_config:
insecure_skip_verify: true
ca_file: /etc/prometheus/ca.crt
authorization:
credentials_file: /etc/prometheus/token
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: 172.16.21.31:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics