From 1f84452119b39c56409ef439bd8de460787be8c9 Mon Sep 17 00:00:00 2001
From: Anthony Woods <awoods@grafana.com>
Date: Thu, 10 Jan 2019 03:20:31 +0800
Subject: [PATCH] allow promtail target path to be a glob match
Promtail now expects __path__ to be a glob. All files that match
the glob expression will be tailed. If new files are created that
match the glob, they will immediately be tailed.
---
cmd/promtail/promtail-docker-config.yaml | 2 +-
cmd/promtail/promtail-local-config.yaml | 2 +-
pkg/promtail/target.go | 59 ++++++++++++++-----
.../helm/templates/promtail/configmap.yaml | 4 +-
.../ksonnet/promtail/promtail.libsonnet | 6 +-
5 files changed, 50 insertions(+), 23 deletions(-)
diff --git a/cmd/promtail/promtail-docker-config.yaml b/cmd/promtail/promtail-docker-config.yaml
index 8bc5f24c..4c851c14 100644
--- a/cmd/promtail/promtail-docker-config.yaml
+++ b/cmd/promtail/promtail-docker-config.yaml
@@ -16,4 +16,4 @@ scrape_configs:
- localhost
labels:
job: varlogs
- __path__: /var/log
+ __path__: /var/log/*log
diff --git a/cmd/promtail/promtail-local-config.yaml b/cmd/promtail/promtail-local-config.yaml
index 11eb46e0..30b22435 100644
--- a/cmd/promtail/promtail-local-config.yaml
+++ b/cmd/promtail/promtail-local-config.yaml
@@ -16,4 +16,4 @@ scrape_configs:
- localhost
labels:
job: varlogs
- __path__: /var/log
+ __path__: /var/log/*log
diff --git a/pkg/promtail/target.go b/pkg/promtail/target.go
index 922eb6e0..fe0ff62b 100644
--- a/pkg/promtail/target.go
+++ b/pkg/promtail/target.go
@@ -1,7 +1,7 @@
package promtail
import (
- "io/ioutil"
+ "os"
"path/filepath"
"time"
@@ -48,14 +48,32 @@ type Target struct {
// NewTarget create a new Target.
func NewTarget(logger log.Logger, handler EntryHandler, positions *Positions, path string, labels model.LabelSet) (*Target, error) {
+ var err error
+ path, err = filepath.Abs(path)
+ if err != nil {
+ return nil, errors.Wrap(err, "filepath.Abs")
+ }
+ matches, err := filepath.Glob(path)
+ if err != nil {
+ return nil, errors.Wrap(err, "filepath.Glob")
+ }
+
watcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, errors.Wrap(err, "fsnotify.NewWatcher")
}
- if err := watcher.Add(path); err != nil {
- helpers.LogError("closing watcher", watcher.Close)
- return nil, errors.Wrap(err, "watcher.Add")
+ // get the current unique set of dirs to watch.
+ dirs := make(map[string]struct{})
+ for _, p := range matches {
+ dirs[filepath.Dir(p)] = struct{}{}
+ }
+ // watch each dir for any new files.
+ for dir := range dirs {
+ if err := watcher.Add(dir); err != nil {
+ helpers.LogError("closing watcher", watcher.Close)
+ return nil, errors.Wrap(err, "watcher.Add")
+ }
}
t := &Target{
@@ -68,23 +86,24 @@ func NewTarget(logger log.Logger, handler EntryHandler, positions *Positions, pa
tails: map[string]*tailer{},
}
- // Fist, we're going to add all the existing files
- fis, err := ioutil.ReadDir(t.path)
- if err != nil {
- return nil, errors.Wrap(err, "ioutil.ReadDir")
- }
- for _, fi := range fis {
+ // start tailing all of the matched files
+ for _, p := range matches {
+ fi, err := os.Stat(p)
+ if err != nil {
+ level.Error(t.logger).Log("msg", "failed to stat file", "error", err, "filename", p)
+ continue
+ }
if fi.IsDir() {
+ level.Debug(t.logger).Log("msg", "skipping matched dir", "filename", p)
continue
}
- tailer, err := newTailer(t.logger, t.handler, t.positions, filepath.Join(t.path, fi.Name()))
+ tailer, err := newTailer(t.logger, t.handler, t.positions, p)
if err != nil {
- level.Error(t.logger).Log("msg", "failed to tail file", "error", err)
+ level.Error(t.logger).Log("msg", "failed to tail file", "error", err, "filename", p)
continue
}
-
- t.tails[fi.Name()] = tailer
+ t.tails[p] = tailer
}
go t.run()
@@ -114,10 +133,18 @@ func (t *Target) run() {
level.Info(t.logger).Log("msg", "got 'create' for existing file", "filename", event.Name)
continue
}
-
+ matched, err := filepath.Match(t.path, event.Name)
+ if err != nil {
+ level.Error(t.logger).Log("msg", "failed to match file", "error", err, "filename", event.Name)
+ continue
+ }
+ if !matched {
+ level.Debug(t.logger).Log("msg", "new file does not match glob", "filename", event.Name)
+ continue
+ }
tailer, err := newTailer(t.logger, t.handler, t.positions, event.Name)
if err != nil {
- level.Error(t.logger).Log("msg", "failed to tail file", "error", err)
+ level.Error(t.logger).Log("msg", "failed to tail file", "error", err, "filename", event.Name)
continue
}
diff --git a/production/helm/templates/promtail/configmap.yaml b/production/helm/templates/promtail/configmap.yaml
index c7f9c2ce..e90695e9 100644
--- a/production/helm/templates/promtail/configmap.yaml
+++ b/production/helm/templates/promtail/configmap.yaml
@@ -36,7 +36,7 @@ data:
source_labels:
- __meta_kubernetes_pod_name
target_label: instance
- - replacement: /var/log/pods/$1
+ - replacement: /var/log/pods/$1/0.log
separator: /
source_labels:
- __meta_kubernetes_pod_uid
@@ -70,7 +70,7 @@ data:
target_label: instance
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- - replacement: /var/log/pods/$1
+ - replacement: /var/log/pods/$1/0.log
separator: /
source_labels:
- __meta_kubernetes_pod_uid
diff --git a/production/ksonnet/promtail/promtail.libsonnet b/production/ksonnet/promtail/promtail.libsonnet
index 0a1ee9f8..ef1653c3 100644
--- a/production/ksonnet/promtail/promtail.libsonnet
+++ b/production/ksonnet/promtail/promtail.libsonnet
@@ -11,7 +11,7 @@ k {
username: '',
password: '',
scheme: 'https',
- hostname: 'log-us.grafana.net',
+ hostname: 'logs-us-west1.grafana.net',
},
@@ -86,7 +86,7 @@ k {
source_labels: ['__meta_kubernetes_pod_uid', '__meta_kubernetes_pod_container_name'],
target_label: '__path__',
separator: '/',
- replacement: '/var/log/pods/$1',
+ replacement: '/var/log/pods/$1/0.log',
},
],
},
@@ -145,7 +145,7 @@ k {
source_labels: ['__meta_kubernetes_pod_uid', '__meta_kubernetes_pod_container_name'],
target_label: '__path__',
separator: '/',
- replacement: '/var/log/pods/$1',
+ replacement: '/var/log/pods/$1/0.log',
},
],
},
--
GitLab