diff --git a/cmd/promtail/promtail-docker-config.yaml b/cmd/promtail/promtail-docker-config.yaml index 8bc5f24ca7ebe241e4232549aea60e0f9109c85b..4c851c1499b7915d1e39e921d76cc8c44dbc9f16 100644 --- a/cmd/promtail/promtail-docker-config.yaml +++ b/cmd/promtail/promtail-docker-config.yaml @@ -16,4 +16,4 @@ scrape_configs: - localhost labels: job: varlogs - __path__: /var/log + __path__: /var/log/*log diff --git a/cmd/promtail/promtail-local-config.yaml b/cmd/promtail/promtail-local-config.yaml index 11eb46e08813c78c840cc1f553c97cc7a8ea71bf..30b22435b17d19846dedfde637853b6074c12094 100644 --- a/cmd/promtail/promtail-local-config.yaml +++ b/cmd/promtail/promtail-local-config.yaml @@ -16,4 +16,4 @@ scrape_configs: - localhost labels: job: varlogs - __path__: /var/log + __path__: /var/log/*log diff --git a/pkg/promtail/target.go b/pkg/promtail/target.go index 922eb6e07adc83c35f851d670ca8c25bfbe1daff..fe0ff62bb1ded2b6378c84e73cf679cb2f59db4a 100644 --- a/pkg/promtail/target.go +++ b/pkg/promtail/target.go @@ -1,7 +1,7 @@ package promtail import ( - "io/ioutil" + "os" "path/filepath" "time" @@ -48,14 +48,32 @@ type Target struct { // NewTarget create a new Target. func NewTarget(logger log.Logger, handler EntryHandler, positions *Positions, path string, labels model.LabelSet) (*Target, error) { + var err error + path, err = filepath.Abs(path) + if err != nil { + return nil, errors.Wrap(err, "filepath.Abs") + } + matches, err := filepath.Glob(path) + if err != nil { + return nil, errors.Wrap(err, "filepath.Glob") + } + watcher, err := fsnotify.NewWatcher() if err != nil { return nil, errors.Wrap(err, "fsnotify.NewWatcher") } - if err := watcher.Add(path); err != nil { - helpers.LogError("closing watcher", watcher.Close) - return nil, errors.Wrap(err, "watcher.Add") + // get the current unique set of dirs to watch. + dirs := make(map[string]struct{}) + for _, p := range matches { + dirs[filepath.Dir(p)] = struct{}{} + } + // watch each dir for any new files. + for dir := range dirs { + if err := watcher.Add(dir); err != nil { + helpers.LogError("closing watcher", watcher.Close) + return nil, errors.Wrap(err, "watcher.Add") + } } t := &Target{ @@ -68,23 +86,24 @@ func NewTarget(logger log.Logger, handler EntryHandler, positions *Positions, pa tails: map[string]*tailer{}, } - // Fist, we're going to add all the existing files - fis, err := ioutil.ReadDir(t.path) - if err != nil { - return nil, errors.Wrap(err, "ioutil.ReadDir") - } - for _, fi := range fis { + // start tailing all of the matched files + for _, p := range matches { + fi, err := os.Stat(p) + if err != nil { + level.Error(t.logger).Log("msg", "failed to stat file", "error", err, "filename", p) + continue + } if fi.IsDir() { + level.Debug(t.logger).Log("msg", "skipping matched dir", "filename", p) continue } - tailer, err := newTailer(t.logger, t.handler, t.positions, filepath.Join(t.path, fi.Name())) + tailer, err := newTailer(t.logger, t.handler, t.positions, p) if err != nil { - level.Error(t.logger).Log("msg", "failed to tail file", "error", err) + level.Error(t.logger).Log("msg", "failed to tail file", "error", err, "filename", p) continue } - - t.tails[fi.Name()] = tailer + t.tails[p] = tailer } go t.run() @@ -114,10 +133,18 @@ func (t *Target) run() { level.Info(t.logger).Log("msg", "got 'create' for existing file", "filename", event.Name) continue } - + matched, err := filepath.Match(t.path, event.Name) + if err != nil { + level.Error(t.logger).Log("msg", "failed to match file", "error", err, "filename", event.Name) + continue + } + if !matched { + level.Debug(t.logger).Log("msg", "new file does not match glob", "filename", event.Name) + continue + } tailer, err := newTailer(t.logger, t.handler, t.positions, event.Name) if err != nil { - level.Error(t.logger).Log("msg", "failed to tail file", "error", err) + level.Error(t.logger).Log("msg", "failed to tail file", "error", err, "filename", event.Name) continue } diff --git a/production/helm/templates/promtail/configmap.yaml b/production/helm/templates/promtail/configmap.yaml index c7f9c2ce023ac81a3e46557c0bef3d88ef2e17a1..e90695e93550dd5889401ae79feb2fb3fa3e57ad 100644 --- a/production/helm/templates/promtail/configmap.yaml +++ b/production/helm/templates/promtail/configmap.yaml @@ -36,7 +36,7 @@ data: source_labels: - __meta_kubernetes_pod_name target_label: instance - - replacement: /var/log/pods/$1 + - replacement: /var/log/pods/$1/0.log separator: / source_labels: - __meta_kubernetes_pod_uid @@ -70,7 +70,7 @@ data: target_label: instance - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1 + - replacement: /var/log/pods/$1/0.log separator: / source_labels: - __meta_kubernetes_pod_uid diff --git a/production/ksonnet/promtail/promtail.libsonnet b/production/ksonnet/promtail/promtail.libsonnet index 0a1ee9f8b8452c84efcd97c06fe19eb6c55c4551..ef1653c3b92dabe36dc3d60ea836609be681fb98 100644 --- a/production/ksonnet/promtail/promtail.libsonnet +++ b/production/ksonnet/promtail/promtail.libsonnet @@ -11,7 +11,7 @@ k { username: '', password: '', scheme: 'https', - hostname: 'log-us.grafana.net', + hostname: 'logs-us-west1.grafana.net', }, @@ -86,7 +86,7 @@ k { source_labels: ['__meta_kubernetes_pod_uid', '__meta_kubernetes_pod_container_name'], target_label: '__path__', separator: '/', - replacement: '/var/log/pods/$1', + replacement: '/var/log/pods/$1/0.log', }, ], }, @@ -145,7 +145,7 @@ k { source_labels: ['__meta_kubernetes_pod_uid', '__meta_kubernetes_pod_container_name'], target_label: '__path__', separator: '/', - replacement: '/var/log/pods/$1', + replacement: '/var/log/pods/$1/0.log', }, ], },