Use __meta_kubernetes_pod_controller_name to ensure we scrape all pods. (#154)

* Add a scrape config for pods without name or app labels.

Signed-off-by: Tom Wilkie <tom.wilkie@gmail.com>

* rebase with master and update to prefer name label but fallback to controller name

* Update scrape_config to use pod controller_name in the job

If a pod has a "name" label we use that in the job name. Otherwise,
- If the pod was created directly by a controller (statefulset,
  daemonset, job) then we use that controller_name.
- if the pod was created by controller created by a controller
  (deployment, cronJob) then we use the parent controller name.

- add a scrape_config.sh tool that generates the scrap_configs
  for promtail.sh and helm chart from the ksonnet lib.

* Reduce duplication in jsonnet.

Signed-off-by: Tom Wilkie <tom.wilkie@gmail.com>

* Fix rebase

Signed-off-by: Tom Wilkie <tom.wilkie@gmail.com>

* Tidy up promtail config imports.

Signed-off-by: Tom Wilkie <tom.wilkie@gmail.com>

production/helm/templates/promtail/configmap.yaml

@@ -11,24 +11,27 @@ metadata:
 data:
   promtail.yaml: |
     scrape_configs:
-      - job_name: kubernetes-pods
-        entry_parser: {{ .Values.promtail.entryParser }}
+      - entry_parser: '{{ .Values.promtail.entryParser }}'
+        job_name: kubernetes-pods-name
         kubernetes_sd_configs:
         - role: pod
         relabel_configs:
+        - source_labels:
+          - __meta_kubernetes_pod_label_name
+          target_label: __service__
         - source_labels:
           - __meta_kubernetes_pod_node_name
           target_label: __host__
         - action: drop
           regex: ^$
           source_labels:
-          - __meta_kubernetes_pod_label_name
+          - __service__
         - action: replace
           replacement: $1
           separator: /
           source_labels:
           - __meta_kubernetes_namespace
-          - __meta_kubernetes_pod_label_name
+          - __service__
           target_label: job
         - action: replace
           source_labels:
@@ -50,28 +53,137 @@ data:
           - __meta_kubernetes_pod_uid
           - __meta_kubernetes_pod_container_name
           target_label: __path__
-      - job_name: kubernetes-pods-app
-        entry_parser: {{ .Values.promtail.entryParser }}
+      - entry_parser: '{{ .Values.promtail.entryParser }}'
+        job_name: kubernetes-pods-app
         kubernetes_sd_configs:
         - role: pod
         relabel_configs:
+        - action: drop
+          regex: .+
+          source_labels:
+          - __meta_kubernetes_pod_label_name
+        - source_labels:
+          - __meta_kubernetes_pod_label_app
+          target_label: __service__
         - source_labels:
           - __meta_kubernetes_pod_node_name
           target_label: __host__
         - action: drop
           regex: ^$
           source_labels:
-          - __meta_kubernetes_pod_label_app
+          - __service__
+        - action: replace
+          replacement: $1
+          separator: /
+          source_labels:
+          - __meta_kubernetes_namespace
+          - __service__
+          target_label: job
+        - action: replace
+          source_labels:
+          - __meta_kubernetes_namespace
+          target_label: namespace
+        - action: replace
+          source_labels:
+          - __meta_kubernetes_pod_name
+          target_label: instance
+        - action: replace
+          source_labels:
+          - __meta_kubernetes_pod_container_name
+          target_label: container_name
+        - action: labelmap
+          regex: __meta_kubernetes_pod_label_(.+)
+        - replacement: /var/log/pods/$1/*.log
+          separator: /
+          source_labels:
+          - __meta_kubernetes_pod_uid
+          - __meta_kubernetes_pod_container_name
+          target_label: __path__
+      - entry_parser: '{{ .Values.promtail.entryParser }}'
+        job_name: kubernetes-pods-direct-controllers
+        kubernetes_sd_configs:
+        - role: pod
+        relabel_configs:
         - action: drop
           regex: .+
+          separator: ''
           source_labels:
           - __meta_kubernetes_pod_label_name
+          - __meta_kubernetes_pod_label_app
+        - action: drop
+          regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
+          source_labels:
+          - __meta_kubernetes_pod_controller_name
+        - source_labels:
+          - __meta_kubernetes_pod_controller_name
+          target_label: __service__
+        - source_labels:
+          - __meta_kubernetes_pod_node_name
+          target_label: __host__
+        - action: drop
+          regex: ^$
+          source_labels:
+          - __service__
         - action: replace
           replacement: $1
           separator: /
           source_labels:
           - __meta_kubernetes_namespace
+          - __service__
+          target_label: job
+        - action: replace
+          source_labels:
+          - __meta_kubernetes_namespace
+          target_label: namespace
+        - action: replace
+          source_labels:
+          - __meta_kubernetes_pod_name
+          target_label: instance
+        - action: replace
+          source_labels:
+          - __meta_kubernetes_pod_container_name
+          target_label: container_name
+        - action: labelmap
+          regex: __meta_kubernetes_pod_label_(.+)
+        - replacement: /var/log/pods/$1/*.log
+          separator: /
+          source_labels:
+          - __meta_kubernetes_pod_uid
+          - __meta_kubernetes_pod_container_name
+          target_label: __path__
+      - entry_parser: '{{ .Values.promtail.entryParser }}'
+        job_name: kubernetes-pods-indirect-controller
+        kubernetes_sd_configs:
+        - role: pod
+        relabel_configs:
+        - action: drop
+          regex: .+
+          separator: ''
+          source_labels:
+          - __meta_kubernetes_pod_label_name
           - __meta_kubernetes_pod_label_app
+        - action: keep
+          regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
+          source_labels:
+          - __meta_kubernetes_pod_controller_name
+        - action: replace
+          regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
+          source_labels:
+          - __meta_kubernetes_pod_controller_name
+          target_label: __service__
+        - source_labels:
+          - __meta_kubernetes_pod_node_name
+          target_label: __host__
+        - action: drop
+          regex: ^$
+          source_labels:
+          - __service__
+        - action: replace
+          replacement: $1
+          separator: /
+          source_labels:
+          - __meta_kubernetes_namespace
+          - __service__
           target_label: job
         - action: replace
           source_labels:

production/ksonnet/promtail/config.libsonnet

+{
+  _images+:: {
+    promtail: 'grafana/promtail:latest',
+  },
+
+  _config+:: {
+    prometheus_insecure_skip_verify: false,
+    promtail_config: {
+      username: '',
+      password: '',
+      scheme: 'https',
+      hostname: 'logs-us-west1.grafana.net',
+      container_root_path: '/var/lib/docker',
+      external_labels: {},
+    },
+
+    service_url:
+      if std.objectHas(self.promtail_config, 'username') then
+        '%(scheme)s://%(username)s:%(password)s@%(hostname)s/api/prom/push' % self.promtail_config
+      else
+        '%(scheme)s://%(hostname)s/api/prom/push' % self.promtail_config,
+  },
+}

production/ksonnet/promtail/promtail.libsonnet

 local k = import 'ksonnet-util/kausal.libsonnet';
+local config = import 'config.libsonnet';
+local scrape_config = import './scrape_config.libsonnet';
 
-k {
-  _images+:: {
-    promtail: 'grafana/promtail:latest',
-  },
-
-  _config+:: {
-    prometheus_insecure_skip_verify: false,
-    promtail_config: {
-      username: '',
-      password: '',
-      scheme: 'https',
-      hostname: 'logs-us-west1.grafana.net',
-      container_root_path: '/var/lib/docker',
-      external_labels: {},
-    },
-
-
-    service_url:
-      if std.objectHas(self.promtail_config, 'username') then
-        '%(scheme)s://%(username)s:%(password)s@%(hostname)s/api/prom/push' % self.promtail_config
-      else
-        '%(scheme)s://%(hostname)s/api/prom/push' % self.promtail_config,
-  },
-
+k + config + scrape_config {
   namespace:
     $.core.v1.namespace.new($._config.namespace),
 
@@ -37,152 +16,10 @@ k {
       policyRule.withVerbs(['get', 'list', 'watch']),
     ]),
 
-  promtail_config:: {
+  promtail_config+:: {
     client: {
       external_labels: $._config.promtail_config.external_labels,
     },
-    scrape_configs: [
-      {
-        job_name: 'kubernetes-pods',
-        kubernetes_sd_configs: [{
-          role: 'pod',
-        }],
-
-        relabel_configs: [
-          // Only scrape local pods; Promtail will drop targets with a __host__ label
-          // that does not match the current host name.
-          {
-            source_labels: ['__meta_kubernetes_pod_node_name'],
-            target_label: '__host__',
-          },
-
-          // Drop pods without a name label
-          {
-            source_labels: ['__meta_kubernetes_pod_label_name'],
-            action: 'drop',
-            regex: '^$',
-          },
-
-          // Rename jobs to be <namespace>/<name, from pod name label>
-          {
-            source_labels: ['__meta_kubernetes_namespace', '__meta_kubernetes_pod_label_name'],
-            action: 'replace',
-            separator: '/',
-            target_label: 'job',
-            replacement: '$1',
-          },
-
-          // But also include the namespace as a separate label, for routing alerts
-          {
-            source_labels: ['__meta_kubernetes_namespace'],
-            action: 'replace',
-            target_label: 'namespace',
-          },
-
-          // Rename instances to be the pod name
-          {
-            source_labels: ['__meta_kubernetes_pod_name'],
-            action: 'replace',
-            target_label: 'instance',
-          },
-
-          // Include container_name label
-          {
-            source_labels: ['__meta_kubernetes_pod_container_name'],
-            action: 'replace',
-            target_label: 'container_name',
-          },
-
-          // Also include all the other labels on the pod.
-          {
-            action: 'labelmap',
-            regex: '__meta_kubernetes_pod_label_(.+)',
-          },
-
-          // Kubernetes puts logs under subdirectories keyed pod UID and container_name.
-          {
-            source_labels: ['__meta_kubernetes_pod_uid', '__meta_kubernetes_pod_container_name'],
-            target_label: '__path__',
-            separator: '/',
-            replacement: '/var/log/pods/$1/*.log',
-          },
-        ],
-      },
-      {
-        job_name: 'kubernetes-pods-app',
-        kubernetes_sd_configs: [{
-          role: 'pod',
-        }],
-
-        relabel_configs: [
-          // Only scrape local pods; Promtail will drop targets with a __host__ label
-          // that does not match the current host name.
-          {
-            source_labels: ['__meta_kubernetes_pod_node_name'],
-            target_label: '__host__',
-          },
-
-          // Drop pods without a app label
-          {
-            source_labels: ['__meta_kubernetes_pod_label_app'],
-            action: 'drop',
-            regex: '^$',
-          },
-
-          // Drop pods with a 'name' and an 'app' label.  They will have already been added by
-          // the scrape_config that matches on the 'name' label
-          {
-            source_labels: ['__meta_kubernetes_pod_label_name'],
-            action: 'drop',
-            regex: '.+',
-          },
-
-          // Rename jobs to be <namespace>/<app, from pod app label>
-          {
-            source_labels: ['__meta_kubernetes_namespace', '__meta_kubernetes_pod_label_app'],
-            action: 'replace',
-            separator: '/',
-            target_label: 'job',
-            replacement: '$1',
-          },
-
-          // But also include the namespace as a separate label, for routing alerts
-          {
-            source_labels: ['__meta_kubernetes_namespace'],
-            action: 'replace',
-            target_label: 'namespace',
-          },
-
-          // Rename instances to be the pod name
-          {
-            source_labels: ['__meta_kubernetes_pod_name'],
-            action: 'replace',
-            target_label: 'instance',
-          },
-
-          // Include container_name label
-          {
-            source_labels: ['__meta_kubernetes_pod_container_name'],
-            action: 'replace',
-            target_label: 'container_name',
-          },
-
-          // Also include all the other labels on the pod.
-          {
-            action: 'labelmap',
-            regex: '__meta_kubernetes_pod_label_(.+)',
-          },
-
-          // Kubernetes puts logs under subdirectories keyed pod UID and container_name.
-          {
-            source_labels: ['__meta_kubernetes_pod_uid', '__meta_kubernetes_pod_container_name'],
-            target_label: '__path__',
-            separator: '/',
-            replacement: '/var/log/pods/$1/*.log',
-          },
-        ],
-      },
-    ],
   },
 
   local configMap = $.core.v1.configMap,

production/ksonnet/promtail/scrape_config.libsonnet

+local config = import 'config.libsonnet';
+
+config + {
+  local gen_scrape_config(job_name) = {
+    job_name: job_name,
+    entry_parser: $._config.promtail_config.entry_parser,
+    kubernetes_sd_configs: [{
+      role: 'pod',
+    }],
+
+    relabel_configs: self.prelabel_config + [
+      // Only scrape local pods; Promtail will drop targets with a __host__ label
+      // that does not match the current host name.
+      {
+        source_labels: ['__meta_kubernetes_pod_node_name'],
+        target_label: '__host__',
+      },
+
+      // Drop pods without a __service__ label.
+      {
+        source_labels: ['__service__'],
+        action: 'drop',
+        regex: '^$',
+      },
+
+      // Rename jobs to be <namespace>/<name, from pod name label>
+      {
+        source_labels: ['__meta_kubernetes_namespace', '__service__'],
+        action: 'replace',
+        separator: '/',
+        target_label: 'job',
+        replacement: '$1',
+      },
+
+      // But also include the namespace as a separate label, for routing alerts
+      {
+        source_labels: ['__meta_kubernetes_namespace'],
+        action: 'replace',
+        target_label: 'namespace',
+      },
+
+      // Rename instances to be the pod name
+      {
+        source_labels: ['__meta_kubernetes_pod_name'],
+        action: 'replace',
+        target_label: 'instance',
+      },
+
+      // Include container_name label
+      {
+        source_labels: ['__meta_kubernetes_pod_container_name'],
+        action: 'replace',
+        target_label: 'container_name',
+      },
+
+      // Also include all the other labels on the pod.
+      {
+        action: 'labelmap',
+        regex: '__meta_kubernetes_pod_label_(.+)',
+      },
+
+      // Kubernetes puts logs under subdirectories keyed pod UID and container_name.
+      {
+        source_labels: ['__meta_kubernetes_pod_uid', '__meta_kubernetes_pod_container_name'],
+        target_label: '__path__',
+        separator: '/',
+        replacement: '/var/log/pods/$1/*.log',
+      },
+    ],
+  },
+
+  promtail_config:: {
+    scrape_configs: [
+      // Scrape config to scrape any pods with a 'name' label.
+      gen_scrape_config('kubernetes-pods-name') {
+        prelabel_config:: [
+
+          // Use name label as __service__.
+          {
+            source_labels: ['__meta_kubernetes_pod_label_name'],
+            target_label: '__service__',
+          }
+        ],
+      },
+
+      // Scrape config to scrape any pods with a 'app' label.
+      gen_scrape_config('kubernetes-pods-app') {
+        prelabel_config:: [
+          // Drop pods with a 'name' label.  They will have already been added by
+          // the scrape_config that matches on the 'name' label
+          {
+            source_labels: ['__meta_kubernetes_pod_label_name'],
+            action: 'drop',
+            regex: '.+',
+          },
+
+          // Use app label as the __service__.
+          {
+            source_labels: ['__meta_kubernetes_pod_label_app'],
+            target_label: '__service__',
+          },
+        ],
+      },
+
+      // Scrape config to scrape any pods with a direct controller (eg
+      // StatefulSets).
+      gen_scrape_config('kubernetes-pods-direct-controllers') {
+        prelabel_config:: [
+          // Drop pods with a 'name' or 'app' label.  They will have already been added by
+          // the scrape_config that matches above.
+          {
+            source_labels: ['__meta_kubernetes_pod_label_name', '__meta_kubernetes_pod_label_app'],
+            separator: '',
+            action: 'drop',
+            regex: '.+',
+          },
+
+          // Drop pods with an indirect controller. eg Deployments create replicaSets
+          // which then create pods.
+          {
+            source_labels: ['__meta_kubernetes_pod_controller_name'],
+            action: 'drop',
+            regex: '^([0-9a-z-.]+)(-[0-9a-f]{8,10})$',
+          },
+
+          // Use controller name as __service__.
+          {
+            source_labels: ['__meta_kubernetes_pod_controller_name'],
+            target_label: '__service__',
+          },
+        ],
+      },
+
+      // Scrape config to scrape any pods with an indirect controller (eg
+      // Deployments).
+      gen_scrape_config('kubernetes-pods-indirect-controller') {
+        prelabel_config:: [
+          // Drop pods with a 'name' or 'app' label.  They will have already been added by
+          // the scrape_config that matches above.
+          {
+            source_labels: ['__meta_kubernetes_pod_label_name', '__meta_kubernetes_pod_label_app'],
+            separator: '',
+            action: 'drop',
+            regex: '.+',
+          },
+
+          // Drop pods not from an indirect controller. eg StatefulSets, DaemonSets
+          {
+            source_labels: ['__meta_kubernetes_pod_controller_name'],
+            regex: '^([0-9a-z-.]+)(-[0-9a-f]{8,10})$',
+            action: 'keep',
+          },
+
+          // put the indirect controller name into a temp label.
+          {
+            source_labels: ['__meta_kubernetes_pod_controller_name'],
+            action: 'replace',
+            regex: '^([0-9a-z-.]+)(-[0-9a-f]{8,10})$',
+            target_label: '__service__',
+          },
+        ]
+      },
+    ],
+  },
+}
\ No newline at end of file

tools/promtail.sh

@@ -16,23 +16,27 @@ apiVersion: v1
 data:
   promtail.yml: |
     scrape_configs:
-    - job_name: kubernetes-pods
+    - entry_parser: <parser>
+      job_name: kubernetes-pods-name
       kubernetes_sd_configs:
       - role: pod
       relabel_configs:
+      - source_labels:
+        - __meta_kubernetes_pod_label_name
+        target_label: __service__
       - source_labels:
         - __meta_kubernetes_pod_node_name
         target_label: __host__
       - action: drop
         regex: ^$
         source_labels:
-        - __meta_kubernetes_pod_label_name
+        - __service__
       - action: replace
         replacement: $1
         separator: /
         source_labels:
         - __meta_kubernetes_namespace
-        - __meta_kubernetes_pod_label_name
+        - __service__
         target_label: job
       - action: replace
         source_labels:
@@ -44,7 +48,7 @@ data:
         target_label: instance
       - action: replace
         source_labels:
-        - __meta_kubernetes_container_name
+        - __meta_kubernetes_pod_container_name
         target_label: container_name
       - action: labelmap
         regex: __meta_kubernetes_pod_label_(.+)
@@ -54,27 +58,137 @@ data:
         - __meta_kubernetes_pod_uid
         - __meta_kubernetes_pod_container_name
         target_label: __path__
-    - job_name: kubernetes-pods-app
+    - entry_parser: <parser>
+      job_name: kubernetes-pods-app
       kubernetes_sd_configs:
       - role: pod
       relabel_configs:
+      - action: drop
+        regex: .+
+        source_labels:
+        - __meta_kubernetes_pod_label_name
+      - source_labels:
+        - __meta_kubernetes_pod_label_app
+        target_label: __service__
       - source_labels:
         - __meta_kubernetes_pod_node_name
         target_label: __host__
       - action: drop
         regex: ^$
         source_labels:
-        - __meta_kubernetes_pod_label_app
+        - __service__
+      - action: replace
+        replacement: $1
+        separator: /
+        source_labels:
+        - __meta_kubernetes_namespace
+        - __service__
+        target_label: job
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_namespace
+        target_label: namespace
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_pod_name
+        target_label: instance
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_pod_container_name
+        target_label: container_name
+      - action: labelmap
+        regex: __meta_kubernetes_pod_label_(.+)
+      - replacement: /var/log/pods/$1/*.log
+        separator: /
+        source_labels:
+        - __meta_kubernetes_pod_uid
+        - __meta_kubernetes_pod_container_name
+        target_label: __path__
+    - entry_parser: <parser>
+      job_name: kubernetes-pods-direct-controllers
+      kubernetes_sd_configs:
+      - role: pod
+      relabel_configs:
       - action: drop
         regex: .+
+        separator: ''
         source_labels:
         - __meta_kubernetes_pod_label_name
+        - __meta_kubernetes_pod_label_app
+      - action: drop
+        regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
+        source_labels:
+        - __meta_kubernetes_pod_controller_name
+      - source_labels:
+        - __meta_kubernetes_pod_controller_name
+        target_label: __service__
+      - source_labels:
+        - __meta_kubernetes_pod_node_name
+        target_label: __host__
+      - action: drop
+        regex: ^$
+        source_labels:
+        - __service__
       - action: replace
         replacement: $1
         separator: /
         source_labels:
         - __meta_kubernetes_namespace
+        - __service__
+        target_label: job
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_namespace
+        target_label: namespace
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_pod_name
+        target_label: instance
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_pod_container_name
+        target_label: container_name
+      - action: labelmap
+        regex: __meta_kubernetes_pod_label_(.+)
+      - replacement: /var/log/pods/$1/*.log
+        separator: /
+        source_labels:
+        - __meta_kubernetes_pod_uid
+        - __meta_kubernetes_pod_container_name
+        target_label: __path__
+    - entry_parser: <parser>
+      job_name: kubernetes-pods-indirect-controller
+      kubernetes_sd_configs:
+      - role: pod
+      relabel_configs:
+      - action: drop
+        regex: .+
+        separator: ''
+        source_labels:
+        - __meta_kubernetes_pod_label_name
         - __meta_kubernetes_pod_label_app
+      - action: keep
+        regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
+        source_labels:
+        - __meta_kubernetes_pod_controller_name
+      - action: replace
+        regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
+        source_labels:
+        - __meta_kubernetes_pod_controller_name
+        target_label: __service__
+      - source_labels:
+        - __meta_kubernetes_pod_node_name
+        target_label: __host__
+      - action: drop
+        regex: ^$
+        source_labels:
+        - __service__
+      - action: replace
+        replacement: $1
+        separator: /
+        source_labels:
+        - __meta_kubernetes_namespace
+        - __service__
         target_label: job
       - action: replace
         source_labels:
@@ -86,7 +200,7 @@ data:
         target_label: instance
       - action: replace
         source_labels:
-        - __meta_kubernetes_container_name
+        - __meta_kubernetes_pod_container_name
         target_label: container_name
       - action: labelmap
         regex: __meta_kubernetes_pod_label_(.+)

tools/scrape_config.sh

+#!/bin/sh
+
+##########################################
+# Generate the scrape_config for the
+# promtail.sh script and the helm chart.
+#
+# The scrape_config is built from the
+# scrape_config defined in the promtail
+# ksonnet library.
+#########################################
+
+BASE=$(dirname $0)
+
+target=${1:-shell}
+
+case $target in
+    "shell")
+        (cd $BASE; jsonnet -e '((import "../production/ksonnet/promtail/scrape_config.libsonnet") + { _config:: { promtail_config: { entry_parser: "<parser>"}}}).promtail_config' | ytools 2>/dev/null)
+        ;;
+
+    "helm")
+        cat <<EOF
+{{- if .Values.promtail.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "promtail.fullname" . }}
+  labels:
+    app: {{ template "promtail.name" . }}
+    chart: {{ template "promtail.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  promtail.yaml: |
+    scrape_configs:
+EOF
+        (cd $BASE;
+          jsonnet -e '((import "../production/ksonnet/promtail/scrape_config.libsonnet") + { _config:: { promtail_config: { entry_parser: "{{ .Values.promtail.entryParser }}"}}}).promtail_config' \
+          | ytools 2>/dev/null \
+          | tail -n +3 \
+          | awk '{ print "      " $0 }' \
+        )
+        cat <<EOF
+{{- end }}
+EOF
+        ;;
+    *)
+        echo "unknown target. expected 'shell' or 'helm'"
+        exit 1
+esac