From 8c0b3d809916bfcabf43829b7088852323999db1 Mon Sep 17 00:00:00 2001
From: Steven Sheehy <ssheehy@firescope.com>
Date: Sat, 6 Apr 2019 01:30:25 -0500
Subject: [PATCH] Add a /ready endpoint to promtail
Signed-off-by: Steven Sheehy <ssheehy@firescope.com>
---
Makefile | 1 +
pkg/promtail/promtail.go | 10 ++++++++++
pkg/promtail/targets/filetarget.go | 5 +++++
pkg/promtail/targets/filetargetmanager.go | 18 ++++++++++++++++++
pkg/promtail/targets/manager.go | 11 +++++++++++
production/helm/loki-stack/Chart.yaml | 2 +-
production/helm/promtail/Chart.yaml | 2 +-
production/helm/promtail/values.yaml | 4 ++--
production/ksonnet/promtail/promtail.libsonnet | 8 ++++++++
tools/promtail.sh | 16 ++++++++++++++--
10 files changed, 71 insertions(+), 6 deletions(-)
diff --git a/Makefile b/Makefile
index 80a3a142..e7c27e0f 100644
--- a/Makefile
+++ b/Makefile
@@ -185,6 +185,7 @@ helm:
helm dependency build $$chart; \
helm package $$chart; \
done
+ rm -f production/helm/*/requirements.lock
helm-publish: helm
cp production/helm/README.md index.md
diff --git a/pkg/promtail/promtail.go b/pkg/promtail/promtail.go
index 99701662..ff9e6a82 100644
--- a/pkg/promtail/promtail.go
+++ b/pkg/promtail/promtail.go
@@ -1,6 +1,8 @@
package promtail
import (
+ "net/http"
+
"github.com/cortexproject/cortex/pkg/util"
"github.com/weaveworks/common/server"
@@ -40,6 +42,14 @@ func New(cfg config.Config) (*Promtail, error) {
return nil, err
}
+ server.HTTP.Path("/ready").Handler(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+ if tms.Ready() {
+ rw.WriteHeader(http.StatusNoContent)
+ } else {
+ rw.WriteHeader(http.StatusInternalServerError)
+ }
+ }))
+
return &Promtail{
client: client,
positions: positions,
diff --git a/pkg/promtail/targets/filetarget.go b/pkg/promtail/targets/filetarget.go
index 57eb5cd8..759b0527 100644
--- a/pkg/promtail/targets/filetarget.go
+++ b/pkg/promtail/targets/filetarget.go
@@ -106,6 +106,11 @@ func NewFileTarget(logger log.Logger, handler api.EntryHandler, positions *posit
return t, nil
}
+// Ready if at least one file is being tailed
+func (t *FileTarget) Ready() bool {
+ return len(t.tails) > 0
+}
+
// Stop the target.
func (t *FileTarget) Stop() {
close(t.quit)
diff --git a/pkg/promtail/targets/filetargetmanager.go b/pkg/promtail/targets/filetargetmanager.go
index 4ba4e7f6..91f9343d 100644
--- a/pkg/promtail/targets/filetargetmanager.go
+++ b/pkg/promtail/targets/filetargetmanager.go
@@ -100,6 +100,16 @@ func (tm *FileTargetManager) run() {
}
}
+// Ready if there's at least one file target
+func (tm *FileTargetManager) Ready() bool {
+ for _, s := range tm.syncers {
+ if s.ready() {
+ return true
+ }
+ }
+ return false
+}
+
// Stop the TargetManager.
func (tm *FileTargetManager) Stop() {
tm.quit()
@@ -194,6 +204,14 @@ func (s *syncer) newTarget(path string, labels model.LabelSet) (*FileTarget, err
return NewFileTarget(s.log, s.entryHandler, s.positions, path, labels, s.targetConfig)
}
+func (s *syncer) ready() bool {
+ for _, target := range s.targets {
+ if target.Ready() {
+ return true
+ }
+ }
+ return false
+}
func (s *syncer) stop() {
for key, target := range s.targets {
level.Info(s.log).Log("msg", "Removing target", "key", key)
diff --git a/pkg/promtail/targets/manager.go b/pkg/promtail/targets/manager.go
index 6108c34d..d281294a 100644
--- a/pkg/promtail/targets/manager.go
+++ b/pkg/promtail/targets/manager.go
@@ -10,6 +10,7 @@ import (
)
type targetManager interface {
+ Ready() bool
Stop()
}
@@ -47,6 +48,16 @@ func NewTargetManagers(
}
+// Ready if there's at least one ready FileTargetManager
+func (tm *TargetManagers) Ready() bool {
+ for _, t := range tm.targetManagers {
+ if t.Ready() {
+ return true
+ }
+ }
+ return false
+}
+
// Stop the TargetManagers.
func (tm *TargetManagers) Stop() {
for _, t := range tm.targetManagers {
diff --git a/production/helm/loki-stack/Chart.yaml b/production/helm/loki-stack/Chart.yaml
index 14761a85..b824c0ad 100644
--- a/production/helm/loki-stack/Chart.yaml
+++ b/production/helm/loki-stack/Chart.yaml
@@ -1,5 +1,5 @@
name: loki-stack
-version: 0.6.1
+version: 0.6.2
appVersion: 0.0.1
kubeVersion: "^1.10.0-0"
description: "Loki: like Prometheus, but for logs."
diff --git a/production/helm/promtail/Chart.yaml b/production/helm/promtail/Chart.yaml
index 1554c7e4..bc60af65 100644
--- a/production/helm/promtail/Chart.yaml
+++ b/production/helm/promtail/Chart.yaml
@@ -1,5 +1,5 @@
name: promtail
-version: 0.6.1
+version: 0.6.2
appVersion: 0.0.1
kubeVersion: "^1.10.0-0"
description: "Responsible for gathering logs and sending them to Loki"
diff --git a/production/helm/promtail/values.yaml b/production/helm/promtail/values.yaml
index e9f1539b..54ac463f 100644
--- a/production/helm/promtail/values.yaml
+++ b/production/helm/promtail/values.yaml
@@ -16,7 +16,7 @@ image:
livenessProbe:
failureThreshold: 5
httpGet:
- path: /metrics
+ path: /ready
port: http-metrics
initialDelaySeconds: 10
periodSeconds: 10
@@ -50,7 +50,7 @@ rbac:
readinessProbe:
failureThreshold: 5
httpGet:
- path: /metrics
+ path: /ready
port: http-metrics
initialDelaySeconds: 10
periodSeconds: 10
diff --git a/production/ksonnet/promtail/promtail.libsonnet b/production/ksonnet/promtail/promtail.libsonnet
index 1653bc81..4fc68f60 100644
--- a/production/ksonnet/promtail/promtail.libsonnet
+++ b/production/ksonnet/promtail/promtail.libsonnet
@@ -44,6 +44,14 @@ k + config + scrape_config {
container.withEnv([
container.envType.fromFieldPath('HOSTNAME', 'spec.nodeName'),
]) +
+ container.mixin.livenessProbe.httpGet.withPath('/ready') +
+ container.mixin.livenessProbe.httpGet.withPort(80) +
+ container.mixin.livenessProbe.withInitialDelaySeconds(10) +
+ container.mixin.livenessProbe.withTimeoutSeconds(1) +
+ container.mixin.readinessProbe.httpGet.withPath('/ready') +
+ container.mixin.readinessProbe.httpGet.withPort(80) +
+ container.mixin.readinessProbe.withInitialDelaySeconds(10) +
+ container.mixin.readinessProbe.withTimeoutSeconds(1) +
container.mixin.securityContext.withPrivileged(true) +
container.mixin.securityContext.withRunAsUser(0),
diff --git a/tools/promtail.sh b/tools/promtail.sh
index 3a196024..83ba6a7b 100755
--- a/tools/promtail.sh
+++ b/tools/promtail.sh
@@ -215,7 +215,7 @@ kind: ConfigMap
metadata:
name: promtail
---
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
kind: DaemonSet
metadata:
name: promtail
@@ -236,8 +236,20 @@ spec:
fieldRef:
fieldPath: spec.nodeName
image: grafana/promtail:latest
- imagePullPolicy: IfNotPresent
+ imagePullPolicy: Always
name: promtail
+ livenessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ scheme: HTTP
+ initialDelaySeconds: 10
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ scheme: HTTP
+ initialDelaySeconds: 10
ports:
- containerPort: 80
name: http-metrics
--
GitLab