From f7f09e2226d042f8978f3e9b2b2e62eeb6c289f0 Mon Sep 17 00:00:00 2001
From: Sandeep Sukhani <sandeep.d.sukhani@gmail.com>
Date: Thu, 30 May 2019 17:31:34 +0530
Subject: [PATCH] Limits: Reject entries based on age set in limits (#631)
* Reject entries based on age set in limits
---
cmd/loki/loki-local-config.yaml | 2 ++
pkg/distributor/distributor.go | 18 ++++++++++++++++++
production/helm/loki-stack/Chart.yaml | 2 +-
production/helm/loki/Chart.yaml | 2 +-
production/helm/loki/values.yaml | 2 ++
production/ksonnet/loki/config.libsonnet | 2 ++
6 files changed, 26 insertions(+), 2 deletions(-)
diff --git a/cmd/loki/loki-local-config.yaml b/cmd/loki/loki-local-config.yaml
index 4e6978c4..17e35113 100644
--- a/cmd/loki/loki-local-config.yaml
+++ b/cmd/loki/loki-local-config.yaml
@@ -31,6 +31,8 @@ storage_config:
limits_config:
enforce_metric_name: false
+ reject_old_samples: true
+ reject_old_samples_max_age: 168h
chunk_store_config:
max_look_back_period: 0
diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go
index a768c98b..a0063b88 100644
--- a/pkg/distributor/distributor.go
+++ b/pkg/distributor/distributor.go
@@ -5,6 +5,7 @@ import (
"flag"
"hash/fnv"
"sync/atomic"
+ "time"
cortex_client "github.com/cortexproject/cortex/pkg/ingester/client"
"github.com/cortexproject/cortex/pkg/ring"
@@ -21,6 +22,8 @@ import (
"github.com/grafana/loki/pkg/util"
)
+const metricName = "logs"
+
var (
ingesterAppends = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: "loki",
@@ -130,6 +133,21 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log
continue
}
+ entries := make([]logproto.Entry, 0, len(stream.Entries))
+ for _, entry := range stream.Entries {
+ if err := d.overrides.ValidateSample(userID, metricName, cortex_client.Sample{
+ TimestampMs: entry.Timestamp.UnixNano() / int64(time.Millisecond),
+ }); err != nil {
+ validationErr = err
+ continue
+ }
+ entries = append(entries, entry)
+ }
+
+ if len(entries) == 0 {
+ continue
+ }
+ stream.Entries = entries
keys = append(keys, tokenFor(userID, stream.Labels))
streams = append(streams, streamTracker{
stream: stream,
diff --git a/production/helm/loki-stack/Chart.yaml b/production/helm/loki-stack/Chart.yaml
index 1209a6ad..b4df444c 100644
--- a/production/helm/loki-stack/Chart.yaml
+++ b/production/helm/loki-stack/Chart.yaml
@@ -1,5 +1,5 @@
name: loki-stack
-version: 0.10.0
+version: 0.10.1
appVersion: 0.0.1
kubeVersion: "^1.10.0-0"
description: "Loki: like Prometheus, but for logs."
diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml
index d0ef9035..22ce5789 100644
--- a/production/helm/loki/Chart.yaml
+++ b/production/helm/loki/Chart.yaml
@@ -1,5 +1,5 @@
name: loki
-version: 0.9.0
+version: 0.9.1
appVersion: 0.0.1
kubeVersion: "^1.10.0-0"
description: "Loki: like Prometheus, but for logs."
diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml
index c1825e85..e871de7a 100644
--- a/production/helm/loki/values.yaml
+++ b/production/helm/loki/values.yaml
@@ -40,6 +40,8 @@ config:
# consistentreads: true
limits_config:
enforce_metric_name: false
+ reject_old_samples: true
+ reject_old_samples_max_age: 168h
schema_config:
configs:
- from: 2018-04-15
diff --git a/production/ksonnet/loki/config.libsonnet b/production/ksonnet/loki/config.libsonnet
index b1dd463c..7eaedd33 100644
--- a/production/ksonnet/loki/config.libsonnet
+++ b/production/ksonnet/loki/config.libsonnet
@@ -33,6 +33,8 @@
limits_config: {
enforce_metric_name: false,
+ reject_old_samples: true,
+ reject_old_samples_max_age: '168h',
},
ingester: {
--
GitLab