From e79414726e51ef02a1bb8b09af3dd78ed0c4b9e9 Mon Sep 17 00:00:00 2001
From: Steven Sheehy <ssheehy@firescope.com>
Date: Sat, 20 Apr 2019 01:15:27 -0500
Subject: [PATCH] Add scrape config for control plane static pods
Signed-off-by: Steven Sheehy <ssheehy@firescope.com>
---
production/helm/loki-stack/Chart.yaml | 2 +-
production/helm/promtail/Chart.yaml | 2 +-
.../helm/promtail/templates/daemonset.yaml | 5 +-
production/helm/promtail/values.yaml | 48 +++++++++++++++++++
.../ksonnet/promtail/scrape_config.libsonnet | 34 ++++++++++---
tools/promtail.sh | 48 +++++++++++++++++++
6 files changed, 129 insertions(+), 10 deletions(-)
diff --git a/production/helm/loki-stack/Chart.yaml b/production/helm/loki-stack/Chart.yaml
index 86789f10..777922dc 100644
--- a/production/helm/loki-stack/Chart.yaml
+++ b/production/helm/loki-stack/Chart.yaml
@@ -1,5 +1,5 @@
name: loki-stack
-version: 0.7.2
+version: 0.8.0
appVersion: 0.0.1
kubeVersion: "^1.10.0-0"
description: "Loki: like Prometheus, but for logs."
diff --git a/production/helm/promtail/Chart.yaml b/production/helm/promtail/Chart.yaml
index 8f3ea952..806b1209 100644
--- a/production/helm/promtail/Chart.yaml
+++ b/production/helm/promtail/Chart.yaml
@@ -1,5 +1,5 @@
name: promtail
-version: 0.6.3
+version: 0.7.0
appVersion: 0.0.1
kubeVersion: "^1.10.0-0"
description: "Responsible for gathering logs and sending them to Loki"
diff --git a/production/helm/promtail/templates/daemonset.yaml b/production/helm/promtail/templates/daemonset.yaml
index f800bc3e..2d98b3fc 100644
--- a/production/helm/promtail/templates/daemonset.yaml
+++ b/production/helm/promtail/templates/daemonset.yaml
@@ -28,7 +28,10 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
- {{ toYaml .Values.podAnnotations | nindent 8 }}
+ checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+ {{- with .Values.podAnnotations }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
spec:
serviceAccountName: {{ template "promtail.serviceAccountName" . }}
{{- if .Values.priorityClassName }}
diff --git a/production/helm/promtail/values.yaml b/production/helm/promtail/values.yaml
index 2f5c0dcc..5d31ea2c 100644
--- a/production/helm/promtail/values.yaml
+++ b/production/helm/promtail/values.yaml
@@ -318,3 +318,51 @@ config:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
+ - entry_parser: '{{ .Values.entryParser }}'
+ job_name: kubernetes-pods-static
+ kubernetes_sd_configs:
+ - role: pod
+ relabel_configs:
+ - action: drop
+ regex: ^$
+ source_labels:
+ - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_pod_label_component
+ target_label: __service__
+ - source_labels:
+ - __meta_kubernetes_pod_node_name
+ target_label: __host__
+ - action: drop
+ regex: ^$
+ source_labels:
+ - __service__
+ - action: replace
+ replacement: $1
+ separator: /
+ source_labels:
+ - __meta_kubernetes_namespace
+ - __service__
+ target_label: job
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_namespace
+ target_label: namespace
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_pod_name
+ target_label: instance
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_pod_container_name
+ target_label: container_name
+ - action: labelmap
+ regex: __meta_kubernetes_pod_label_(.+)
+ - replacement: /var/log/pods/$1/*.log
+ separator: /
+ source_labels:
+ - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
+ - __meta_kubernetes_pod_container_name
+ target_label: __path__
+
diff --git a/production/ksonnet/promtail/scrape_config.libsonnet b/production/ksonnet/promtail/scrape_config.libsonnet
index 9f7fb327..35dddbcd 100644
--- a/production/ksonnet/promtail/scrape_config.libsonnet
+++ b/production/ksonnet/promtail/scrape_config.libsonnet
@@ -1,7 +1,7 @@
local config = import 'config.libsonnet';
config + {
- local gen_scrape_config(job_name) = {
+ local gen_scrape_config(job_name, pod_uid) = {
job_name: job_name,
entry_parser: $._config.promtail_config.entry_parser,
kubernetes_sd_configs: [{
@@ -61,7 +61,7 @@ config + {
// Kubernetes puts logs under subdirectories keyed pod UID and container_name.
{
- source_labels: ['__meta_kubernetes_pod_uid', '__meta_kubernetes_pod_container_name'],
+ source_labels: [pod_uid, '__meta_kubernetes_pod_container_name'],
target_label: '__path__',
separator: '/',
replacement: '/var/log/pods/$1/*.log',
@@ -72,7 +72,7 @@ config + {
promtail_config:: {
scrape_configs: [
// Scrape config to scrape any pods with a 'name' label.
- gen_scrape_config('kubernetes-pods-name') {
+ gen_scrape_config('kubernetes-pods-name', '__meta_kubernetes_pod_uid') {
prelabel_config:: [
// Use name label as __service__.
@@ -84,7 +84,7 @@ config + {
},
// Scrape config to scrape any pods with a 'app' label.
- gen_scrape_config('kubernetes-pods-app') {
+ gen_scrape_config('kubernetes-pods-app', '__meta_kubernetes_pod_uid') {
prelabel_config:: [
// Drop pods with a 'name' label. They will have already been added by
// the scrape_config that matches on the 'name' label
@@ -104,7 +104,7 @@ config + {
// Scrape config to scrape any pods with a direct controller (eg
// StatefulSets).
- gen_scrape_config('kubernetes-pods-direct-controllers') {
+ gen_scrape_config('kubernetes-pods-direct-controllers', '__meta_kubernetes_pod_uid') {
prelabel_config:: [
// Drop pods with a 'name' or 'app' label. They will have already been added by
// the scrape_config that matches above.
@@ -133,7 +133,7 @@ config + {
// Scrape config to scrape any pods with an indirect controller (eg
// Deployments).
- gen_scrape_config('kubernetes-pods-indirect-controller') {
+ gen_scrape_config('kubernetes-pods-indirect-controller', '__meta_kubernetes_pod_uid') {
prelabel_config:: [
// Drop pods with a 'name' or 'app' label. They will have already been added by
// the scrape_config that matches above.
@@ -160,6 +160,26 @@ config + {
},
]
},
+
+ // Scrape config to scrape any control plane static pods (e.g. kube-apiserver
+ // etcd, kube-controller-manager & kube-scheduler)
+ gen_scrape_config('kubernetes-pods-static', '__meta_kubernetes_pod_annotation_kubernetes_io_config_mirror') {
+ prelabel_config:: [
+ // Ignore pods that aren't mirror pods
+ {
+ action: 'drop',
+ source_labels: ['__meta_kubernetes_pod_annotation_kubernetes_io_config_mirror'],
+ regex: '^$',
+ },
+
+ // Static control plane pods usually have a component label that identifies them
+ {
+ action: 'replace',
+ source_labels: ['__meta_kubernetes_pod_label_component'],
+ target_label: '__service__',
+ },
+ ]
+ },
],
},
-}
\ No newline at end of file
+}
diff --git a/tools/promtail.sh b/tools/promtail.sh
index 635eb534..aae6c854 100755
--- a/tools/promtail.sh
+++ b/tools/promtail.sh
@@ -211,6 +211,54 @@ data:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
+ - entry_parser: <parser>
+ job_name: kubernetes-pods-static
+ kubernetes_sd_configs:
+ - role: pod
+ relabel_configs:
+ - action: drop
+ regex: ^$
+ source_labels:
+ - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_pod_label_component
+ target_label: __service__
+ - source_labels:
+ - __meta_kubernetes_pod_node_name
+ target_label: __host__
+ - action: drop
+ regex: ^$
+ source_labels:
+ - __service__
+ - action: replace
+ replacement: $1
+ separator: /
+ source_labels:
+ - __meta_kubernetes_namespace
+ - __service__
+ target_label: job
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_namespace
+ target_label: namespace
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_pod_name
+ target_label: instance
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_pod_container_name
+ target_label: container_name
+ - action: labelmap
+ regex: __meta_kubernetes_pod_label_(.+)
+ - replacement: /var/log/pods/$1/*.log
+ separator: /
+ source_labels:
+ - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
+ - __meta_kubernetes_pod_container_name
+ target_label: __path__
+
kind: ConfigMap
metadata:
name: promtail
--
GitLab