From 2c4e30f2ff52e9b6ef968cb3a758eedc82902255 Mon Sep 17 00:00:00 2001
From: Edward Welch <edward.welch@grafana.com>
Date: Wed, 8 May 2019 11:26:21 -0400
Subject: [PATCH] Updating helm/ksonnet to use new pipeline config Updating
docs Rebase fixes
---
README.md | 2 +-
docs/promtail-examples.md | 22 +++++++++------
docs/promtail-setup.md | 6 ++---
docs/promtail.md | 11 ++++++++
docs/troubleshooting.md | 2 +-
pkg/promtail/promtail_test.go | 1 -
production/helm/promtail/values.yaml | 27 ++++++++++---------
production/ksonnet/promtail/config.libsonnet | 4 ++-
.../ksonnet/promtail/scrape_config.libsonnet | 2 +-
9 files changed, 48 insertions(+), 29 deletions(-)
diff --git a/README.md b/README.md
index 3b4cfb1d..5703f729 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ Once you have promtail, Loki, and Grafana running, continue with [our usage docs
- [API documentation](./docs/api.md) for alternative ways of getting logs into Loki.
- [Operations](./docs/operations.md) for important aspects of running Loki.
-- [Promtail](./docs/promtail-setup.md) on how to configure the agent that tails your logs.
+- [Promtail](./docs/promtail.md) is an agent which can tail your log files and push them to Loki.
- [Logcli](./docs/logcli.md) on how to query your logs without Grafana.
- [Troubleshooting](./docs/troubleshooting.md) for help around frequent error messages.
- [Usage](./docs/usage.md) for how to set up a Loki datasource in Grafana and query your logs.
diff --git a/docs/promtail-examples.md b/docs/promtail-examples.md
index dda607ad..47526922 100644
--- a/docs/promtail-examples.md
+++ b/docs/promtail-examples.md
@@ -1,7 +1,12 @@
-# promtail examples
-#### In this file you can see simple examples of configure promtail
+# Promtail Config Examples
-* This example of config promtail based on original docker [config](https://github.com/grafana/loki/blob/master/cmd/promtail/promtail-docker-config.yaml)
+## Pipeline Examples
+
+TODO Need pipeline examples
+
+## Simple Docker Config
+
+This example of config promtail based on original docker [config](https://github.com/grafana/loki/blob/master/cmd/promtail/promtail-docker-config.yaml)
and show how work with 2 and more sources:
Filename for example: my-docker-config.yaml
@@ -38,18 +43,19 @@ scrape_configs:
__path__: /srv/log/someone_service/*.log
```
-##### Description
-Scrape_config section of config.yaml contents are various jobs for parsing your logs on current host
+#### Description
+
+Scrape_config section of config.yaml contents contains various jobs for parsing your logs
-`job` and `host` these are tags on which you can filter parsed logs date on Grafana later
+`job` and `host` are examples of static labels added to all logs, labels are indexed by Loki and are used to help search logs.
`__path__` it is path to directory where stored your logs.
If you run promtail and this config.yaml in Docker container, don't forget use docker volumes for mapping real directories
with log to those folders in the container.
-* See next example of Dockerfile, who use our modified promtail config (my-docker-config.yaml)
-1) Create folder, for example `promtail`, then new folder build and in this filder conf and place there `my-docker-config.yaml`.
+#### Example Use
+1) Create folder, for example `promtail`, then new sub directory `build/conf` and place there `my-docker-config.yaml`.
2) Create new Dockerfile in root folder `promtail`, with contents
```
FROM grafana/promtail:latest
diff --git a/docs/promtail-setup.md b/docs/promtail-setup.md
index 3312bd7c..cd171c8c 100644
--- a/docs/promtail-setup.md
+++ b/docs/promtail-setup.md
@@ -1,9 +1,5 @@
# Promtail Setups
-## Design Documentation
-
-* [Extracting labels from logs](./design/labels.md)
-
## Daemonset method
Daemonset will deploy promtail on every node within the kubernetes cluster.
@@ -12,6 +8,8 @@ Daemonset deployment is great to collect all of the container logs within the
cluster. It is great solution for single tenant. All of the logs will send to a
single Loki server.
+Check the `production` folder for examples of a daemonset deployment for kubernetes using both helm and ksonnet.
+
### Example
```yaml
---Daemonset.yaml
diff --git a/docs/promtail.md b/docs/promtail.md
index 783dd61d..ff8b3bc9 100644
--- a/docs/promtail.md
+++ b/docs/promtail.md
@@ -1,3 +1,14 @@
+# Promtail
+
+* [Deployment Methods](./promtail-setup.md)
+* [Config and Usage Examples](./promtail-examples.md)
+* [Troubleshooting](./troubleshooting.md)
+
+
+## Design Documentation
+
+ * [Extracting labels from logs](./design/labels.md)
+
## Promtail and scrape_configs
Promtail is an agent which reads log files and sends streams of log data to
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index 5d09932f..59046ed9 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -46,7 +46,7 @@ The promtail configuration contains a `__path__` entry to a directory that promt
## Connecting to a promtail pod to troubleshoot
-Say you are missing logs from your nginx pod and want to investigate promtail.
+First check *Troubleshooting targets* section above, if that doesn't help answer your questions you can connect to the promtail pod to further investigate.
In your cluster if you are running promtail as a daemonset, you will have a promtail pod on each node, to figure out which promtail you want run:
diff --git a/pkg/promtail/promtail_test.go b/pkg/promtail/promtail_test.go
index 544818a0..31f5b2aa 100644
--- a/pkg/promtail/promtail_test.go
+++ b/pkg/promtail/promtail_test.go
@@ -26,7 +26,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/grafana/loki/pkg/logproto"
- "github.com/grafana/loki/pkg/parser"
"github.com/grafana/loki/pkg/promtail/api"
"github.com/grafana/loki/pkg/promtail/config"
"github.com/grafana/loki/pkg/promtail/scrape"
diff --git a/production/helm/promtail/values.yaml b/production/helm/promtail/values.yaml
index b2020dbd..285dbc64 100644
--- a/production/helm/promtail/values.yaml
+++ b/production/helm/promtail/values.yaml
@@ -6,8 +6,6 @@ annotations: {}
deploymentStrategy: RollingUpdate
-entryParser: docker
-
image:
repository: grafana/promtail
tag: latest
@@ -124,8 +122,9 @@ config:
# Period to resync directories being watched and files being tailed
sync_period: 10s
scrape_configs:
- - entry_parser: '{{ .Values.entryParser }}'
- job_name: kubernetes-pods-name
+ - job_name: kubernetes-pods-name
+ pipeline_stages:
+ - docker: {}
kubernetes_sd_configs:
- role: pod
relabel_configs:
@@ -166,8 +165,9 @@ config:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
- - entry_parser: '{{ .Values.entryParser }}'
- job_name: kubernetes-pods-app
+ - job_name: kubernetes-pods-app
+ pipeline_stages:
+ - docker: {}
kubernetes_sd_configs:
- role: pod
relabel_configs:
@@ -212,8 +212,9 @@ config:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
- - entry_parser: '{{ .Values.entryParser }}'
- job_name: kubernetes-pods-direct-controllers
+ - job_name: kubernetes-pods-direct-controllers
+ pipeline_stages:
+ - docker: {}
kubernetes_sd_configs:
- role: pod
relabel_configs:
@@ -264,8 +265,9 @@ config:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
- - entry_parser: '{{ .Values.entryParser }}'
- job_name: kubernetes-pods-indirect-controller
+ - job_name: kubernetes-pods-indirect-controller
+ pipeline_stages:
+ - docker: {}
kubernetes_sd_configs:
- role: pod
relabel_configs:
@@ -318,8 +320,9 @@ config:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
- - entry_parser: '{{ .Values.entryParser }}'
- job_name: kubernetes-pods-static
+ - job_name: kubernetes-pods-static
+ pipeline_stages:
+ - docker: {}
kubernetes_sd_configs:
- role: pod
relabel_configs:
diff --git a/production/ksonnet/promtail/config.libsonnet b/production/ksonnet/promtail/config.libsonnet
index 01ace73f..a3df83eb 100644
--- a/production/ksonnet/promtail/config.libsonnet
+++ b/production/ksonnet/promtail/config.libsonnet
@@ -14,7 +14,9 @@
external_labels: {},
}],
container_root_path: '/var/lib/docker',
- entry_parser: 'docker',
+ pipeline_stages: [{
+ docker: {},
+ }],
},
},
}
diff --git a/production/ksonnet/promtail/scrape_config.libsonnet b/production/ksonnet/promtail/scrape_config.libsonnet
index 81eba885..4ea41857 100644
--- a/production/ksonnet/promtail/scrape_config.libsonnet
+++ b/production/ksonnet/promtail/scrape_config.libsonnet
@@ -3,7 +3,7 @@ local config = import 'config.libsonnet';
config + {
local gen_scrape_config(job_name, pod_uid) = {
job_name: job_name,
- entry_parser: $._config.promtail_config.entry_parser,
+ pipeline_stages: $._config.promtail_config.pipeline_stages,
kubernetes_sd_configs: [{
role: 'pod',
}],
--
GitLab