diff --git a/Makefile b/Makefile
index 80a3a1427e508cff0c2566e957f19c9279d81878..e7c27e0f4189924c8026001cc96888c3b25200f9 100644
--- a/Makefile
+++ b/Makefile
@@ -185,6 +185,7 @@ helm:
helm dependency build $$chart; \
helm package $$chart; \
done
+ rm -f production/helm/*/requirements.lock
helm-publish: helm
cp production/helm/README.md index.md
diff --git a/pkg/promtail/promtail.go b/pkg/promtail/promtail.go
index 9970166290d6e5bb4ce4c57a919e07fb70dfeb82..ff9e6a82244c9bf8fe098a68f33b9d781884a94c 100644
--- a/pkg/promtail/promtail.go
+++ b/pkg/promtail/promtail.go
@@ -1,6 +1,8 @@
package promtail
import (
+ "net/http"
+
"github.com/cortexproject/cortex/pkg/util"
"github.com/weaveworks/common/server"
@@ -40,6 +42,14 @@ func New(cfg config.Config) (*Promtail, error) {
return nil, err
}
+ server.HTTP.Path("/ready").Handler(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+ if tms.Ready() {
+ rw.WriteHeader(http.StatusNoContent)
+ } else {
+ rw.WriteHeader(http.StatusInternalServerError)
+ }
+ }))
+
return &Promtail{
client: client,
positions: positions,
diff --git a/pkg/promtail/targets/filetarget.go b/pkg/promtail/targets/filetarget.go
index 57eb5cd804649a1bd76478e245d89678aa450f08..759b052739aa5b920dfd30d3cb9dc3a0ee18c7d6 100644
--- a/pkg/promtail/targets/filetarget.go
+++ b/pkg/promtail/targets/filetarget.go
@@ -106,6 +106,11 @@ func NewFileTarget(logger log.Logger, handler api.EntryHandler, positions *posit
return t, nil
}
+// Ready if at least one file is being tailed
+func (t *FileTarget) Ready() bool {
+ return len(t.tails) > 0
+}
+
// Stop the target.
func (t *FileTarget) Stop() {
close(t.quit)
diff --git a/pkg/promtail/targets/filetargetmanager.go b/pkg/promtail/targets/filetargetmanager.go
index 4ba4e7f6cc208fa0ef47c6a9bc3609d186f6aa9d..91f9343d136da725cd7b187bec374fb9c8a08bf6 100644
--- a/pkg/promtail/targets/filetargetmanager.go
+++ b/pkg/promtail/targets/filetargetmanager.go
@@ -100,6 +100,16 @@ func (tm *FileTargetManager) run() {
}
}
+// Ready if there's at least one file target
+func (tm *FileTargetManager) Ready() bool {
+ for _, s := range tm.syncers {
+ if s.ready() {
+ return true
+ }
+ }
+ return false
+}
+
// Stop the TargetManager.
func (tm *FileTargetManager) Stop() {
tm.quit()
@@ -194,6 +204,14 @@ func (s *syncer) newTarget(path string, labels model.LabelSet) (*FileTarget, err
return NewFileTarget(s.log, s.entryHandler, s.positions, path, labels, s.targetConfig)
}
+func (s *syncer) ready() bool {
+ for _, target := range s.targets {
+ if target.Ready() {
+ return true
+ }
+ }
+ return false
+}
func (s *syncer) stop() {
for key, target := range s.targets {
level.Info(s.log).Log("msg", "Removing target", "key", key)
diff --git a/pkg/promtail/targets/manager.go b/pkg/promtail/targets/manager.go
index 6108c34de9c4d9c6872eb2cd9f527aaae1fffd67..d281294ab62b1f51ac30e428c667a5805af28bfc 100644
--- a/pkg/promtail/targets/manager.go
+++ b/pkg/promtail/targets/manager.go
@@ -10,6 +10,7 @@ import (
)
type targetManager interface {
+ Ready() bool
Stop()
}
@@ -47,6 +48,16 @@ func NewTargetManagers(
}
+// Ready if there's at least one ready FileTargetManager
+func (tm *TargetManagers) Ready() bool {
+ for _, t := range tm.targetManagers {
+ if t.Ready() {
+ return true
+ }
+ }
+ return false
+}
+
// Stop the TargetManagers.
func (tm *TargetManagers) Stop() {
for _, t := range tm.targetManagers {
diff --git a/production/helm/loki-stack/Chart.yaml b/production/helm/loki-stack/Chart.yaml
index 14761a85ebc1003465b403f25475e7a0c45b9553..b824c0ad52363c2ed4401e070ae9f6d7b6748520 100644
--- a/production/helm/loki-stack/Chart.yaml
+++ b/production/helm/loki-stack/Chart.yaml
@@ -1,5 +1,5 @@
name: loki-stack
-version: 0.6.1
+version: 0.6.2
appVersion: 0.0.1
kubeVersion: "^1.10.0-0"
description: "Loki: like Prometheus, but for logs."
diff --git a/production/helm/promtail/Chart.yaml b/production/helm/promtail/Chart.yaml
index 1554c7e4ea3d5eb1c420e914a6297f586213fa15..bc60af652de358ab8fa216dcd74b6a9b0ddefe7c 100644
--- a/production/helm/promtail/Chart.yaml
+++ b/production/helm/promtail/Chart.yaml
@@ -1,5 +1,5 @@
name: promtail
-version: 0.6.1
+version: 0.6.2
appVersion: 0.0.1
kubeVersion: "^1.10.0-0"
description: "Responsible for gathering logs and sending them to Loki"
diff --git a/production/helm/promtail/values.yaml b/production/helm/promtail/values.yaml
index e9f1539b799fd56e36c229e7cc5f9d2620176895..54ac463f916c98d70ed2dad1fe922caa40671ef0 100644
--- a/production/helm/promtail/values.yaml
+++ b/production/helm/promtail/values.yaml
@@ -16,7 +16,7 @@ image:
livenessProbe:
failureThreshold: 5
httpGet:
- path: /metrics
+ path: /ready
port: http-metrics
initialDelaySeconds: 10
periodSeconds: 10
@@ -50,7 +50,7 @@ rbac:
readinessProbe:
failureThreshold: 5
httpGet:
- path: /metrics
+ path: /ready
port: http-metrics
initialDelaySeconds: 10
periodSeconds: 10
diff --git a/production/ksonnet/promtail/promtail.libsonnet b/production/ksonnet/promtail/promtail.libsonnet
index 1653bc818e315881c8490e4c1ca8447295460776..4fc68f60be093cda56b0be1f8b20a9d056d3e21a 100644
--- a/production/ksonnet/promtail/promtail.libsonnet
+++ b/production/ksonnet/promtail/promtail.libsonnet
@@ -44,6 +44,14 @@ k + config + scrape_config {
container.withEnv([
container.envType.fromFieldPath('HOSTNAME', 'spec.nodeName'),
]) +
+ container.mixin.livenessProbe.httpGet.withPath('/ready') +
+ container.mixin.livenessProbe.httpGet.withPort(80) +
+ container.mixin.livenessProbe.withInitialDelaySeconds(10) +
+ container.mixin.livenessProbe.withTimeoutSeconds(1) +
+ container.mixin.readinessProbe.httpGet.withPath('/ready') +
+ container.mixin.readinessProbe.httpGet.withPort(80) +
+ container.mixin.readinessProbe.withInitialDelaySeconds(10) +
+ container.mixin.readinessProbe.withTimeoutSeconds(1) +
container.mixin.securityContext.withPrivileged(true) +
container.mixin.securityContext.withRunAsUser(0),
diff --git a/tools/promtail.sh b/tools/promtail.sh
index 3a1960243c682489e051decf493c451eca69cdd0..83ba6a7bbedd8c80d23a3978c32c2242019bb161 100755
--- a/tools/promtail.sh
+++ b/tools/promtail.sh
@@ -215,7 +215,7 @@ kind: ConfigMap
metadata:
name: promtail
---
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
kind: DaemonSet
metadata:
name: promtail
@@ -236,8 +236,20 @@ spec:
fieldRef:
fieldPath: spec.nodeName
image: grafana/promtail:latest
- imagePullPolicy: IfNotPresent
+ imagePullPolicy: Always
name: promtail
+ livenessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ scheme: HTTP
+ initialDelaySeconds: 10
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: http-metrics
+ scheme: HTTP
+ initialDelaySeconds: 10
ports:
- containerPort: 80
name: http-metrics