diff --git a/codeblock/prometheus/kubernetes-pods.yaml b/codeblock/prometheus/kubernetes-pods.yaml new file mode 100644 index 0000000..b53cc44 --- /dev/null +++ b/codeblock/prometheus/kubernetes-pods.yaml @@ -0,0 +1,38 @@ +- job_name: "kubernetes-pods" + + kubernetes_sd_configs: + - role: pod + + relabel_configs: + # Example relabel to scrape only pods that have + # `prometheus.io/scrape: "true"` annotation. + - source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + action: keep + regex: true + + # Example relabel to customize metric path based on pod + # `prometheus.io/path: ` annotation. + - source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + action: replace + target_label: __metrics_path__ + regex: (.+) + + # Example relabel to scrape only single, desired port for the pod + # based on pod `prometheus.io/port: ` annotation. + - source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod diff --git a/codeblock/prometheus/kubernetes-service-endpoints.yaml b/codeblock/prometheus/kubernetes-service-endpoints.yaml new file mode 100644 index 0000000..7a4898a --- /dev/null +++ b/codeblock/prometheus/kubernetes-service-endpoints.yaml @@ -0,0 +1,46 @@ +- job_name: "kubernetes-service-endpoints" + + kubernetes_sd_configs: + - role: endpoints + + relabel_configs: + # prometheus relabel to scrape only endpoints that have + # `prometheus.io/scrape: "true"` annotation. + - source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + action: keep + regex: true + # + # prometheus relabel to customize metric path based on endpoints + # `prometheus.io/path: ` annotation. + - source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + action: replace + target_label: __metrics_path__ + regex: (.+) + # + # prometheus relabel to scrape only single, desired port for the service based + # on endpoints `prometheus.io/port: ` annotation. + - source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + # + # prometheus relabel to configure scrape scheme for all service scrape targets + # based on endpoints `prometheus.io/scheme: ` annotation. + - source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + action: replace + target_label: __scheme__ + regex: (https?) + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: service diff --git a/content/basics/monitoring/prometheus/annotation-discovery.md b/content/basics/monitoring/prometheus/annotation-discovery.md new file mode 100644 index 0000000..87c49de --- /dev/null +++ b/content/basics/monitoring/prometheus/annotation-discovery.md @@ -0,0 +1,81 @@ +# 基于 Pod 和 Service 注解的服务发现 + +## 背景 + +很多应用会为 Pod 或 Service 打上一些注解用于 Prometheus 的服务发现,如 `prometheus.io/scrape: "true"`,这种注解并不是 Prometheus 官方支持的,而是社区的习惯性用法,要使这种注解生效,还需结合 Prometheus 的采集配置,本文介绍具体的配置方法。 + +## 真实案例 + +### istio 指标采集 + +[istio](https://istio.io/) 使用了这种 Pod 注解,当 Pod 被自动注入 sidecar 的同时也会被自动注入以下注解: + +```yaml + prometheus.io/path: /stats/prometheus + prometheus.io/port: "15020" + prometheus.io/scrape: "true" +``` + +表示声明让 Prometheus 采集 Envoy Sidecar 暴露的 metrics,端口是 15020,路径是 `/stats/prometheus`。 + +除此之外,控制面组件 istiod 的 Pod 也会有类似注解: + +```yaml + prometheus.io/port: "15014" + prometheus.io/scrape: "true" +``` + +### Kubernetes Addon 指标采集 + +Kubenretes 源码仓库中的一些 addon 组件也使用了这种注解,有的是 Pod 注解,有的是 Service 注解。 +* [coredns](https://github.com/kubernetes/kubernetes/blob/release-1.30/cluster/addons/dns/coredns/coredns.yaml.base#L196-L197) 使用 Service 注解: + ```yaml showLineNumbers + apiVersion: v1 + kind: Service + metadata: + name: kube-dns + namespace: kube-system + annotations: + # highlight-start + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + # highlight-end + ``` +* [nodelocaldns](https://github.com/kubernetes/kubernetes/blob/release-1.30/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml#L125-L126) 使用 Pod 注解: + ```yaml showLineNumbers + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: node-local-dns + namespace: kube-system + labels: + k8s-app: node-local-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + selector: + matchLabels: + k8s-app: node-local-dns + template: + metadata: + labels: + k8s-app: node-local-dns + annotations: + # highlight-start + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + # highlight-end + ``` + +## Prometheus 采集配置 + +### 根据 Pod 注解动态采集 + + + +### 根据 Service 注解动态采集 + + diff --git a/content/basics/sidebars.ts b/content/basics/sidebars.ts index 960912e..6daed89 100644 --- a/content/basics/sidebars.ts +++ b/content/basics/sidebars.ts @@ -64,7 +64,19 @@ const sidebars: SidebarsConfig = { items: [ { type: 'category', - label: 'grafana', + label: 'Prometheus', + collapsed: true, + link: { + type: 'generated-index', + slug: '/monitoring/prometheus' + }, + items: [ + 'monitoring/prometheus/annotation-discovery', + ], + }, + { + type: 'category', + label: 'Grafana', collapsed: true, link: { type: 'generated-index',