Add scrape config for control plane static pods

Signed-off-by: Steven Sheehy <ssheehy@firescope.com>

production/helm/loki-stack/Chart.yaml

 name: loki-stack
-version: 0.7.2
+version: 0.8.0
 appVersion: 0.0.1
 kubeVersion: "^1.10.0-0"
 description: "Loki: like Prometheus, but for logs."

production/helm/promtail/Chart.yaml

 name: promtail
-version: 0.6.3
+version: 0.7.0
 appVersion: 0.0.1
 kubeVersion: "^1.10.0-0"
 description: "Responsible for gathering logs and sending them to Loki"

production/helm/promtail/templates/daemonset.yaml

@@ -28,7 +28,10 @@ spec:
         {{- toYaml . | nindent 8 }}
         {{- end }}          
       annotations:
-        {{ toYaml .Values.podAnnotations | nindent 8 }}
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
     spec:
       serviceAccountName: {{ template "promtail.serviceAccountName" . }}
     {{- if .Values.priorityClassName }}

production/helm/promtail/values.yaml

@@ -318,3 +318,51 @@ config:
       - __meta_kubernetes_pod_uid
       - __meta_kubernetes_pod_container_name
       target_label: __path__
+  - entry_parser: '{{ .Values.entryParser }}'
+    job_name: kubernetes-pods-static
+    kubernetes_sd_configs:
+    - role: pod
+    relabel_configs:
+    - action: drop
+      regex: ^$
+      source_labels:
+      - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
+    - action: replace
+      source_labels:
+      - __meta_kubernetes_pod_label_component
+      target_label: __service__
+    - source_labels:
+      - __meta_kubernetes_pod_node_name
+      target_label: __host__
+    - action: drop
+      regex: ^$
+      source_labels:
+      - __service__
+    - action: replace
+      replacement: $1
+      separator: /
+      source_labels:
+      - __meta_kubernetes_namespace
+      - __service__
+      target_label: job
+    - action: replace
+      source_labels:
+      - __meta_kubernetes_namespace
+      target_label: namespace
+    - action: replace
+      source_labels:
+      - __meta_kubernetes_pod_name
+      target_label: instance
+    - action: replace
+      source_labels:
+      - __meta_kubernetes_pod_container_name
+      target_label: container_name
+    - action: labelmap
+      regex: __meta_kubernetes_pod_label_(.+)
+    - replacement: /var/log/pods/$1/*.log
+      separator: /
+      source_labels:
+      - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
+      - __meta_kubernetes_pod_container_name
+      target_label: __path__
+

production/ksonnet/promtail/scrape_config.libsonnet

 local config = import 'config.libsonnet';
 
 config + {
-  local gen_scrape_config(job_name) = {
+  local gen_scrape_config(job_name, pod_uid) = {
     job_name: job_name,
     entry_parser: $._config.promtail_config.entry_parser,
     kubernetes_sd_configs: [{
@@ -61,7 +61,7 @@ config + {
 
       // Kubernetes puts logs under subdirectories keyed pod UID and container_name.
       {
-        source_labels: ['__meta_kubernetes_pod_uid', '__meta_kubernetes_pod_container_name'],
+        source_labels: [pod_uid, '__meta_kubernetes_pod_container_name'],
         target_label: '__path__',
         separator: '/',
         replacement: '/var/log/pods/$1/*.log',
@@ -72,7 +72,7 @@ config + {
   promtail_config:: {
     scrape_configs: [
       // Scrape config to scrape any pods with a 'name' label.
-      gen_scrape_config('kubernetes-pods-name') {
+      gen_scrape_config('kubernetes-pods-name', '__meta_kubernetes_pod_uid') {
         prelabel_config:: [
 
           // Use name label as __service__.
@@ -84,7 +84,7 @@ config + {
       },
 
       // Scrape config to scrape any pods with a 'app' label.
-      gen_scrape_config('kubernetes-pods-app') {
+      gen_scrape_config('kubernetes-pods-app', '__meta_kubernetes_pod_uid') {
         prelabel_config:: [
           // Drop pods with a 'name' label.  They will have already been added by
           // the scrape_config that matches on the 'name' label
@@ -104,7 +104,7 @@ config + {
 
       // Scrape config to scrape any pods with a direct controller (eg
       // StatefulSets).
-      gen_scrape_config('kubernetes-pods-direct-controllers') {
+      gen_scrape_config('kubernetes-pods-direct-controllers', '__meta_kubernetes_pod_uid') {
         prelabel_config:: [
           // Drop pods with a 'name' or 'app' label.  They will have already been added by
           // the scrape_config that matches above.
@@ -133,7 +133,7 @@ config + {
 
       // Scrape config to scrape any pods with an indirect controller (eg
       // Deployments).
-      gen_scrape_config('kubernetes-pods-indirect-controller') {
+      gen_scrape_config('kubernetes-pods-indirect-controller', '__meta_kubernetes_pod_uid') {
         prelabel_config:: [
           // Drop pods with a 'name' or 'app' label.  They will have already been added by
           // the scrape_config that matches above.
@@ -160,6 +160,26 @@ config + {
           },
         ]
       },
+
+      // Scrape config to scrape any control plane static pods (e.g. kube-apiserver
+      // etcd, kube-controller-manager & kube-scheduler)
+      gen_scrape_config('kubernetes-pods-static', '__meta_kubernetes_pod_annotation_kubernetes_io_config_mirror') {
+        prelabel_config:: [
+          // Ignore pods that aren't mirror pods
+          {
+            action: 'drop',
+            source_labels: ['__meta_kubernetes_pod_annotation_kubernetes_io_config_mirror'],
+            regex: '^$',
+          },
+
+          // Static control plane pods usually have a component label that identifies them
+          {
+            action: 'replace',
+            source_labels: ['__meta_kubernetes_pod_label_component'],
+            target_label: '__service__',
+          },
+        ]
+      },
     ],
   },
-}
\ No newline at end of file
+}

tools/promtail.sh

@@ -211,6 +211,54 @@ data:
         - __meta_kubernetes_pod_uid
         - __meta_kubernetes_pod_container_name
         target_label: __path__
+    - entry_parser: <parser>
+      job_name: kubernetes-pods-static
+      kubernetes_sd_configs:
+      - role: pod
+      relabel_configs:
+      - action: drop
+        regex: ^$
+        source_labels:
+        - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_pod_label_component
+        target_label: __service__
+      - source_labels:
+        - __meta_kubernetes_pod_node_name
+        target_label: __host__
+      - action: drop
+        regex: ^$
+        source_labels:
+        - __service__
+      - action: replace
+        replacement: $1
+        separator: /
+        source_labels:
+        - __meta_kubernetes_namespace
+        - __service__
+        target_label: job
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_namespace
+        target_label: namespace
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_pod_name
+        target_label: instance
+      - action: replace
+        source_labels:
+        - __meta_kubernetes_pod_container_name
+        target_label: container_name
+      - action: labelmap
+        regex: __meta_kubernetes_pod_label_(.+)
+      - replacement: /var/log/pods/$1/*.log
+        separator: /
+        source_labels:
+        - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
+        - __meta_kubernetes_pod_container_name
+        target_label: __path__
+
 kind: ConfigMap
 metadata:
   name: promtail