diff --git a/production/helm/templates/promtail/configmap.yaml b/production/helm/templates/promtail/configmap.yaml
index e6c88b5e1fe407ab5f4e0b8ce45a69885b992a35..ec6f36db3ebdd4114e1238de3d66cbddb6525f07 100644
--- a/production/helm/templates/promtail/configmap.yaml
+++ b/production/helm/templates/promtail/configmap.yaml
@@ -11,24 +11,27 @@ metadata:
data:
promtail.yaml: |
scrape_configs:
- - job_name: kubernetes-pods
- entry_parser: {{ .Values.promtail.entryParser }}
+ - entry_parser: '{{ .Values.promtail.entryParser }}'
+ job_name: kubernetes-pods-name
kubernetes_sd_configs:
- role: pod
relabel_configs:
+ - source_labels:
+ - __meta_kubernetes_pod_label_name
+ target_label: __service__
- source_labels:
- __meta_kubernetes_pod_node_name
target_label: __host__
- action: drop
regex: ^$
source_labels:
- - __meta_kubernetes_pod_label_name
+ - __service__
- action: replace
replacement: $1
separator: /
source_labels:
- __meta_kubernetes_namespace
- - __meta_kubernetes_pod_label_name
+ - __service__
target_label: job
- action: replace
source_labels:
@@ -50,28 +53,137 @@ data:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
- - job_name: kubernetes-pods-app
- entry_parser: {{ .Values.promtail.entryParser }}
+ - entry_parser: '{{ .Values.promtail.entryParser }}'
+ job_name: kubernetes-pods-app
kubernetes_sd_configs:
- role: pod
relabel_configs:
+ - action: drop
+ regex: .+
+ source_labels:
+ - __meta_kubernetes_pod_label_name
+ - source_labels:
+ - __meta_kubernetes_pod_label_app
+ target_label: __service__
- source_labels:
- __meta_kubernetes_pod_node_name
target_label: __host__
- action: drop
regex: ^$
source_labels:
- - __meta_kubernetes_pod_label_app
+ - __service__
+ - action: replace
+ replacement: $1
+ separator: /
+ source_labels:
+ - __meta_kubernetes_namespace
+ - __service__
+ target_label: job
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_namespace
+ target_label: namespace
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_pod_name
+ target_label: instance
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_pod_container_name
+ target_label: container_name
+ - action: labelmap
+ regex: __meta_kubernetes_pod_label_(.+)
+ - replacement: /var/log/pods/$1/*.log
+ separator: /
+ source_labels:
+ - __meta_kubernetes_pod_uid
+ - __meta_kubernetes_pod_container_name
+ target_label: __path__
+ - entry_parser: '{{ .Values.promtail.entryParser }}'
+ job_name: kubernetes-pods-direct-controllers
+ kubernetes_sd_configs:
+ - role: pod
+ relabel_configs:
- action: drop
regex: .+
+ separator: ''
source_labels:
- __meta_kubernetes_pod_label_name
+ - __meta_kubernetes_pod_label_app
+ - action: drop
+ regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
+ source_labels:
+ - __meta_kubernetes_pod_controller_name
+ - source_labels:
+ - __meta_kubernetes_pod_controller_name
+ target_label: __service__
+ - source_labels:
+ - __meta_kubernetes_pod_node_name
+ target_label: __host__
+ - action: drop
+ regex: ^$
+ source_labels:
+ - __service__
- action: replace
replacement: $1
separator: /
source_labels:
- __meta_kubernetes_namespace
+ - __service__
+ target_label: job
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_namespace
+ target_label: namespace
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_pod_name
+ target_label: instance
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_pod_container_name
+ target_label: container_name
+ - action: labelmap
+ regex: __meta_kubernetes_pod_label_(.+)
+ - replacement: /var/log/pods/$1/*.log
+ separator: /
+ source_labels:
+ - __meta_kubernetes_pod_uid
+ - __meta_kubernetes_pod_container_name
+ target_label: __path__
+ - entry_parser: '{{ .Values.promtail.entryParser }}'
+ job_name: kubernetes-pods-indirect-controller
+ kubernetes_sd_configs:
+ - role: pod
+ relabel_configs:
+ - action: drop
+ regex: .+
+ separator: ''
+ source_labels:
+ - __meta_kubernetes_pod_label_name
- __meta_kubernetes_pod_label_app
+ - action: keep
+ regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
+ source_labels:
+ - __meta_kubernetes_pod_controller_name
+ - action: replace
+ regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
+ source_labels:
+ - __meta_kubernetes_pod_controller_name
+ target_label: __service__
+ - source_labels:
+ - __meta_kubernetes_pod_node_name
+ target_label: __host__
+ - action: drop
+ regex: ^$
+ source_labels:
+ - __service__
+ - action: replace
+ replacement: $1
+ separator: /
+ source_labels:
+ - __meta_kubernetes_namespace
+ - __service__
target_label: job
- action: replace
source_labels:
diff --git a/production/ksonnet/promtail/config.libsonnet b/production/ksonnet/promtail/config.libsonnet
new file mode 100644
index 0000000000000000000000000000000000000000..e4a3be040475537c6bff88bcc0cb051d38e89120
--- /dev/null
+++ b/production/ksonnet/promtail/config.libsonnet
@@ -0,0 +1,23 @@
+{
+ _images+:: {
+ promtail: 'grafana/promtail:latest',
+ },
+
+ _config+:: {
+ prometheus_insecure_skip_verify: false,
+ promtail_config: {
+ username: '',
+ password: '',
+ scheme: 'https',
+ hostname: 'logs-us-west1.grafana.net',
+ container_root_path: '/var/lib/docker',
+ external_labels: {},
+ },
+
+ service_url:
+ if std.objectHas(self.promtail_config, 'username') then
+ '%(scheme)s://%(username)s:%(password)s@%(hostname)s/api/prom/push' % self.promtail_config
+ else
+ '%(scheme)s://%(hostname)s/api/prom/push' % self.promtail_config,
+ },
+}
diff --git a/production/ksonnet/promtail/promtail.libsonnet b/production/ksonnet/promtail/promtail.libsonnet
index 7b2b87a7f7a85471d27009dd37fbfd3e281623a3..1653bc818e315881c8490e4c1ca8447295460776 100644
--- a/production/ksonnet/promtail/promtail.libsonnet
+++ b/production/ksonnet/promtail/promtail.libsonnet
@@ -1,29 +1,8 @@
local k = import 'ksonnet-util/kausal.libsonnet';
+local config = import 'config.libsonnet';
+local scrape_config = import './scrape_config.libsonnet';
-k {
- _images+:: {
- promtail: 'grafana/promtail:latest',
- },
-
- _config+:: {
- prometheus_insecure_skip_verify: false,
- promtail_config: {
- username: '',
- password: '',
- scheme: 'https',
- hostname: 'logs-us-west1.grafana.net',
- container_root_path: '/var/lib/docker',
- external_labels: {},
- },
-
-
- service_url:
- if std.objectHas(self.promtail_config, 'username') then
- '%(scheme)s://%(username)s:%(password)s@%(hostname)s/api/prom/push' % self.promtail_config
- else
- '%(scheme)s://%(hostname)s/api/prom/push' % self.promtail_config,
- },
-
+k + config + scrape_config {
namespace:
$.core.v1.namespace.new($._config.namespace),
@@ -37,152 +16,10 @@ k {
policyRule.withVerbs(['get', 'list', 'watch']),
]),
- promtail_config:: {
+ promtail_config+:: {
client: {
external_labels: $._config.promtail_config.external_labels,
},
- scrape_configs: [
- {
- job_name: 'kubernetes-pods',
- kubernetes_sd_configs: [{
- role: 'pod',
- }],
-
- relabel_configs: [
- // Only scrape local pods; Promtail will drop targets with a __host__ label
- // that does not match the current host name.
- {
- source_labels: ['__meta_kubernetes_pod_node_name'],
- target_label: '__host__',
- },
-
- // Drop pods without a name label
- {
- source_labels: ['__meta_kubernetes_pod_label_name'],
- action: 'drop',
- regex: '^$',
- },
-
- // Rename jobs to be <namespace>/<name, from pod name label>
- {
- source_labels: ['__meta_kubernetes_namespace', '__meta_kubernetes_pod_label_name'],
- action: 'replace',
- separator: '/',
- target_label: 'job',
- replacement: '$1',
- },
-
- // But also include the namespace as a separate label, for routing alerts
- {
- source_labels: ['__meta_kubernetes_namespace'],
- action: 'replace',
- target_label: 'namespace',
- },
-
- // Rename instances to be the pod name
- {
- source_labels: ['__meta_kubernetes_pod_name'],
- action: 'replace',
- target_label: 'instance',
- },
-
- // Include container_name label
- {
- source_labels: ['__meta_kubernetes_pod_container_name'],
- action: 'replace',
- target_label: 'container_name',
- },
-
- // Also include all the other labels on the pod.
- {
- action: 'labelmap',
- regex: '__meta_kubernetes_pod_label_(.+)',
- },
-
- // Kubernetes puts logs under subdirectories keyed pod UID and container_name.
- {
- source_labels: ['__meta_kubernetes_pod_uid', '__meta_kubernetes_pod_container_name'],
- target_label: '__path__',
- separator: '/',
- replacement: '/var/log/pods/$1/*.log',
- },
- ],
- },
- {
- job_name: 'kubernetes-pods-app',
- kubernetes_sd_configs: [{
- role: 'pod',
- }],
-
- relabel_configs: [
- // Only scrape local pods; Promtail will drop targets with a __host__ label
- // that does not match the current host name.
- {
- source_labels: ['__meta_kubernetes_pod_node_name'],
- target_label: '__host__',
- },
-
- // Drop pods without a app label
- {
- source_labels: ['__meta_kubernetes_pod_label_app'],
- action: 'drop',
- regex: '^$',
- },
-
- // Drop pods with a 'name' and an 'app' label. They will have already been added by
- // the scrape_config that matches on the 'name' label
- {
- source_labels: ['__meta_kubernetes_pod_label_name'],
- action: 'drop',
- regex: '.+',
- },
-
- // Rename jobs to be <namespace>/<app, from pod app label>
- {
- source_labels: ['__meta_kubernetes_namespace', '__meta_kubernetes_pod_label_app'],
- action: 'replace',
- separator: '/',
- target_label: 'job',
- replacement: '$1',
- },
-
- // But also include the namespace as a separate label, for routing alerts
- {
- source_labels: ['__meta_kubernetes_namespace'],
- action: 'replace',
- target_label: 'namespace',
- },
-
- // Rename instances to be the pod name
- {
- source_labels: ['__meta_kubernetes_pod_name'],
- action: 'replace',
- target_label: 'instance',
- },
-
- // Include container_name label
- {
- source_labels: ['__meta_kubernetes_pod_container_name'],
- action: 'replace',
- target_label: 'container_name',
- },
-
- // Also include all the other labels on the pod.
- {
- action: 'labelmap',
- regex: '__meta_kubernetes_pod_label_(.+)',
- },
-
- // Kubernetes puts logs under subdirectories keyed pod UID and container_name.
- {
- source_labels: ['__meta_kubernetes_pod_uid', '__meta_kubernetes_pod_container_name'],
- target_label: '__path__',
- separator: '/',
- replacement: '/var/log/pods/$1/*.log',
- },
- ],
- },
- ],
},
local configMap = $.core.v1.configMap,
diff --git a/production/ksonnet/promtail/scrape_config.libsonnet b/production/ksonnet/promtail/scrape_config.libsonnet
new file mode 100644
index 0000000000000000000000000000000000000000..9f7fb327c7180ff1f76616b07fa25252e06391c1
--- /dev/null
+++ b/production/ksonnet/promtail/scrape_config.libsonnet
@@ -0,0 +1,165 @@
+local config = import 'config.libsonnet';
+
+config + {
+ local gen_scrape_config(job_name) = {
+ job_name: job_name,
+ entry_parser: $._config.promtail_config.entry_parser,
+ kubernetes_sd_configs: [{
+ role: 'pod',
+ }],
+
+ relabel_configs: self.prelabel_config + [
+ // Only scrape local pods; Promtail will drop targets with a __host__ label
+ // that does not match the current host name.
+ {
+ source_labels: ['__meta_kubernetes_pod_node_name'],
+ target_label: '__host__',
+ },
+
+ // Drop pods without a __service__ label.
+ {
+ source_labels: ['__service__'],
+ action: 'drop',
+ regex: '^$',
+ },
+
+ // Rename jobs to be <namespace>/<name, from pod name label>
+ {
+ source_labels: ['__meta_kubernetes_namespace', '__service__'],
+ action: 'replace',
+ separator: '/',
+ target_label: 'job',
+ replacement: '$1',
+ },
+
+ // But also include the namespace as a separate label, for routing alerts
+ {
+ source_labels: ['__meta_kubernetes_namespace'],
+ action: 'replace',
+ target_label: 'namespace',
+ },
+
+ // Rename instances to be the pod name
+ {
+ source_labels: ['__meta_kubernetes_pod_name'],
+ action: 'replace',
+ target_label: 'instance',
+ },
+
+ // Include container_name label
+ {
+ source_labels: ['__meta_kubernetes_pod_container_name'],
+ action: 'replace',
+ target_label: 'container_name',
+ },
+
+ // Also include all the other labels on the pod.
+ {
+ action: 'labelmap',
+ regex: '__meta_kubernetes_pod_label_(.+)',
+ },
+
+ // Kubernetes puts logs under subdirectories keyed pod UID and container_name.
+ {
+ source_labels: ['__meta_kubernetes_pod_uid', '__meta_kubernetes_pod_container_name'],
+ target_label: '__path__',
+ separator: '/',
+ replacement: '/var/log/pods/$1/*.log',
+ },
+ ],
+ },
+
+ promtail_config:: {
+ scrape_configs: [
+ // Scrape config to scrape any pods with a 'name' label.
+ gen_scrape_config('kubernetes-pods-name') {
+ prelabel_config:: [
+
+ // Use name label as __service__.
+ {
+ source_labels: ['__meta_kubernetes_pod_label_name'],
+ target_label: '__service__',
+ }
+ ],
+ },
+
+ // Scrape config to scrape any pods with a 'app' label.
+ gen_scrape_config('kubernetes-pods-app') {
+ prelabel_config:: [
+ // Drop pods with a 'name' label. They will have already been added by
+ // the scrape_config that matches on the 'name' label
+ {
+ source_labels: ['__meta_kubernetes_pod_label_name'],
+ action: 'drop',
+ regex: '.+',
+ },
+
+ // Use app label as the __service__.
+ {
+ source_labels: ['__meta_kubernetes_pod_label_app'],
+ target_label: '__service__',
+ },
+ ],
+ },
+
+ // Scrape config to scrape any pods with a direct controller (eg
+ // StatefulSets).
+ gen_scrape_config('kubernetes-pods-direct-controllers') {
+ prelabel_config:: [
+ // Drop pods with a 'name' or 'app' label. They will have already been added by
+ // the scrape_config that matches above.
+ {
+ source_labels: ['__meta_kubernetes_pod_label_name', '__meta_kubernetes_pod_label_app'],
+ separator: '',
+ action: 'drop',
+ regex: '.+',
+ },
+
+ // Drop pods with an indirect controller. eg Deployments create replicaSets
+ // which then create pods.
+ {
+ source_labels: ['__meta_kubernetes_pod_controller_name'],
+ action: 'drop',
+ regex: '^([0-9a-z-.]+)(-[0-9a-f]{8,10})$',
+ },
+
+ // Use controller name as __service__.
+ {
+ source_labels: ['__meta_kubernetes_pod_controller_name'],
+ target_label: '__service__',
+ },
+ ],
+ },
+
+ // Scrape config to scrape any pods with an indirect controller (eg
+ // Deployments).
+ gen_scrape_config('kubernetes-pods-indirect-controller') {
+ prelabel_config:: [
+ // Drop pods with a 'name' or 'app' label. They will have already been added by
+ // the scrape_config that matches above.
+ {
+ source_labels: ['__meta_kubernetes_pod_label_name', '__meta_kubernetes_pod_label_app'],
+ separator: '',
+ action: 'drop',
+ regex: '.+',
+ },
+
+ // Drop pods not from an indirect controller. eg StatefulSets, DaemonSets
+ {
+ source_labels: ['__meta_kubernetes_pod_controller_name'],
+ regex: '^([0-9a-z-.]+)(-[0-9a-f]{8,10})$',
+ action: 'keep',
+ },
+
+ // put the indirect controller name into a temp label.
+ {
+ source_labels: ['__meta_kubernetes_pod_controller_name'],
+ action: 'replace',
+ regex: '^([0-9a-z-.]+)(-[0-9a-f]{8,10})$',
+ target_label: '__service__',
+ },
+ ]
+ },
+ ],
+ },
+}
\ No newline at end of file
diff --git a/tools/promtail.sh b/tools/promtail.sh
index 1fc87290c183f8ef70e99714892791bbc2045bb9..e01319e5b2bee2d1ce6646820610b2c155fbfcb8 100644
--- a/tools/promtail.sh
+++ b/tools/promtail.sh
@@ -16,23 +16,27 @@ apiVersion: v1
data:
promtail.yml: |
scrape_configs:
- - job_name: kubernetes-pods
+ - entry_parser: <parser>
+ job_name: kubernetes-pods-name
kubernetes_sd_configs:
- role: pod
relabel_configs:
+ - source_labels:
+ - __meta_kubernetes_pod_label_name
+ target_label: __service__
- source_labels:
- __meta_kubernetes_pod_node_name
target_label: __host__
- action: drop
regex: ^$
source_labels:
- - __meta_kubernetes_pod_label_name
+ - __service__
- action: replace
replacement: $1
separator: /
source_labels:
- __meta_kubernetes_namespace
- - __meta_kubernetes_pod_label_name
+ - __service__
target_label: job
- action: replace
source_labels:
@@ -44,7 +48,7 @@ data:
target_label: instance
- action: replace
source_labels:
- - __meta_kubernetes_container_name
+ - __meta_kubernetes_pod_container_name
target_label: container_name
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
@@ -54,27 +58,137 @@ data:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
- - job_name: kubernetes-pods-app
+ - entry_parser: <parser>
+ job_name: kubernetes-pods-app
kubernetes_sd_configs:
- role: pod
relabel_configs:
+ - action: drop
+ regex: .+
+ source_labels:
+ - __meta_kubernetes_pod_label_name
+ - source_labels:
+ - __meta_kubernetes_pod_label_app
+ target_label: __service__
- source_labels:
- __meta_kubernetes_pod_node_name
target_label: __host__
- action: drop
regex: ^$
source_labels:
- - __meta_kubernetes_pod_label_app
+ - __service__
+ - action: replace
+ replacement: $1
+ separator: /
+ source_labels:
+ - __meta_kubernetes_namespace
+ - __service__
+ target_label: job
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_namespace
+ target_label: namespace
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_pod_name
+ target_label: instance
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_pod_container_name
+ target_label: container_name
+ - action: labelmap
+ regex: __meta_kubernetes_pod_label_(.+)
+ - replacement: /var/log/pods/$1/*.log
+ separator: /
+ source_labels:
+ - __meta_kubernetes_pod_uid
+ - __meta_kubernetes_pod_container_name
+ target_label: __path__
+ - entry_parser: <parser>
+ job_name: kubernetes-pods-direct-controllers
+ kubernetes_sd_configs:
+ - role: pod
+ relabel_configs:
- action: drop
regex: .+
+ separator: ''
source_labels:
- __meta_kubernetes_pod_label_name
+ - __meta_kubernetes_pod_label_app
+ - action: drop
+ regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
+ source_labels:
+ - __meta_kubernetes_pod_controller_name
+ - source_labels:
+ - __meta_kubernetes_pod_controller_name
+ target_label: __service__
+ - source_labels:
+ - __meta_kubernetes_pod_node_name
+ target_label: __host__
+ - action: drop
+ regex: ^$
+ source_labels:
+ - __service__
- action: replace
replacement: $1
separator: /
source_labels:
- __meta_kubernetes_namespace
+ - __service__
+ target_label: job
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_namespace
+ target_label: namespace
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_pod_name
+ target_label: instance
+ - action: replace
+ source_labels:
+ - __meta_kubernetes_pod_container_name
+ target_label: container_name
+ - action: labelmap
+ regex: __meta_kubernetes_pod_label_(.+)
+ - replacement: /var/log/pods/$1/*.log
+ separator: /
+ source_labels:
+ - __meta_kubernetes_pod_uid
+ - __meta_kubernetes_pod_container_name
+ target_label: __path__
+ - entry_parser: <parser>
+ job_name: kubernetes-pods-indirect-controller
+ kubernetes_sd_configs:
+ - role: pod
+ relabel_configs:
+ - action: drop
+ regex: .+
+ separator: ''
+ source_labels:
+ - __meta_kubernetes_pod_label_name
- __meta_kubernetes_pod_label_app
+ - action: keep
+ regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
+ source_labels:
+ - __meta_kubernetes_pod_controller_name
+ - action: replace
+ regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
+ source_labels:
+ - __meta_kubernetes_pod_controller_name
+ target_label: __service__
+ - source_labels:
+ - __meta_kubernetes_pod_node_name
+ target_label: __host__
+ - action: drop
+ regex: ^$
+ source_labels:
+ - __service__
+ - action: replace
+ replacement: $1
+ separator: /
+ source_labels:
+ - __meta_kubernetes_namespace
+ - __service__
target_label: job
- action: replace
source_labels:
@@ -86,7 +200,7 @@ data:
target_label: instance
- action: replace
source_labels:
- - __meta_kubernetes_container_name
+ - __meta_kubernetes_pod_container_name
target_label: container_name
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
diff --git a/tools/scrape_config.sh b/tools/scrape_config.sh
new file mode 100755
index 0000000000000000000000000000000000000000..14a8bfc739b768381be8688cd356f9f4f15dcf26
--- /dev/null
+++ b/tools/scrape_config.sh
@@ -0,0 +1,50 @@
+#!/bin/sh
+
+##########################################
+# Generate the scrape_config for the
+# promtail.sh script and the helm chart.
+#
+# The scrape_config is built from the
+# scrape_config defined in the promtail
+# ksonnet library.
+#########################################
+
+BASE=$(dirname $0)
+
+target=${1:-shell}
+
+case $target in
+ "shell")
+ (cd $BASE; jsonnet -e '((import "../production/ksonnet/promtail/scrape_config.libsonnet") + { _config:: { promtail_config: { entry_parser: "<parser>"}}}).promtail_config' | ytools 2>/dev/null)
+ ;;
+
+ "helm")
+ cat <<EOF
+{{- if .Values.promtail.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ template "promtail.fullname" . }}
+ labels:
+ app: {{ template "promtail.name" . }}
+ chart: {{ template "promtail.chart" . }}
+ release: {{ .Release.Name }}
+ heritage: {{ .Release.Service }}
+data:
+ promtail.yaml: |
+ scrape_configs:
+EOF
+ (cd $BASE;
+ jsonnet -e '((import "../production/ksonnet/promtail/scrape_config.libsonnet") + { _config:: { promtail_config: { entry_parser: "{{ .Values.promtail.entryParser }}"}}}).promtail_config' \
+ | ytools 2>/dev/null \
+ | tail -n +3 \
+ | awk '{ print " " $0 }' \
+ )
+ cat <<EOF
+{{- end }}
+EOF
+ ;;
+ *)
+ echo "unknown target. expected 'shell' or 'helm'"
+ exit 1
+esac