From ffe109384ba6124b1dc3011ea859797beb117aa8 Mon Sep 17 00:00:00 2001
From: Tom Wilkie <tom.wilkie@gmail.com>
Date: Tue, 29 Jan 2019 12:39:13 +0000
Subject: [PATCH] Tweak Loki Writes dashboards, add resource request and limits
to distributor and ingester ksonnet.
Signed-off-by: Tom Wilkie <tom.wilkie@gmail.com>
---
production/ksonnet/loki/distributor.libsonnet | 4 +++-
production/ksonnet/loki/ingester.libsonnet | 4 +++-
production/loki-mixin/dashboards.libsonnet | 18 +++++++++---------
3 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/production/ksonnet/loki/distributor.libsonnet b/production/ksonnet/loki/distributor.libsonnet
index ac767705..dabddc91 100644
--- a/production/ksonnet/loki/distributor.libsonnet
+++ b/production/ksonnet/loki/distributor.libsonnet
@@ -10,7 +10,9 @@
distributor_container::
container.new('distributor', $._images.distributor) +
container.withPorts($.util.defaultPorts) +
- container.withArgsMixin($.util.mapToFlags($.distributor_args)),
+ container.withArgsMixin($.util.mapToFlags($.distributor_args)) +
+ $.util.resourcesRequests('0.5', '100Mi') +
+ $.util.resourcesLimits('1', '200Mi'),
local deployment = $.apps.v1beta1.deployment,
diff --git a/production/ksonnet/loki/ingester.libsonnet b/production/ksonnet/loki/ingester.libsonnet
index 31860be2..2267209b 100644
--- a/production/ksonnet/loki/ingester.libsonnet
+++ b/production/ksonnet/loki/ingester.libsonnet
@@ -13,7 +13,9 @@
container.mixin.readinessProbe.httpGet.withPath('/ready') +
container.mixin.readinessProbe.httpGet.withPort(80) +
container.mixin.readinessProbe.withInitialDelaySeconds(15) +
- container.mixin.readinessProbe.withTimeoutSeconds(1),
+ container.mixin.readinessProbe.withTimeoutSeconds(1) +
+ $.util.resourcesRequests('1', '5Gi') +
+ $.util.resourcesLimits('2', '10Gi'),
local deployment = $.apps.v1beta1.deployment,
diff --git a/production/loki-mixin/dashboards.libsonnet b/production/loki-mixin/dashboards.libsonnet
index d2587cd2..81b62f05 100644
--- a/production/loki-mixin/dashboards.libsonnet
+++ b/production/loki-mixin/dashboards.libsonnet
@@ -5,44 +5,44 @@ local utils = import "mixin-utils/utils.libsonnet";
dashboards+: {
'loki-writes.json':
g.dashboard('Loki / Writes')
- .addTemplate('cluster', 'kube_pod_container_info{image=~".*loki.*"}', 'cluster')
- .addTemplate('namespace', 'kube_pod_container_info{image=~".*loki.*"}', 'namespace')
+ .addMultiTemplate('cluster', 'kube_pod_container_info{image=~".*loki.*"}', 'cluster')
+ .addMultiTemplate('namespace', 'kube_pod_container_info{image=~".*loki.*"}', 'namespace')
.addRow(
g.row('Frontend (cortex_gw)')
.addPanel(
g.panel('QPS') +
- g.qpsPanel('cortex_gw_request_duration_seconds_count{cluster="$cluster", job="$namespace/cortex-gw", route="cortex-write"}')
+ g.qpsPanel('cortex_gw_request_duration_seconds_count{cluster=~"$cluster", job=~"($namespace)/cortex-gw", route="cortex-write"}')
)
.addPanel(
g.panel('Latency') +
- utils.latencyRecordingRulePanel('cortex_gw_request_duration_seconds', [utils.selector.eq('job', '$namespace/cortex-gw'), utils.selector.eq('route', 'cortex-write')], extra_selectors=[utils.selector.eq('cluster', '$cluster')])
+ utils.latencyRecordingRulePanel('cortex_gw_request_duration_seconds', [utils.selector.re('job', '($namespace)/cortex-gw'), utils.selector.eq('route', 'cortex-write')], extra_selectors=[utils.selector.re('cluster', '$cluster')])
)
)
.addRow(
g.row('Distributor')
.addPanel(
g.panel('QPS') +
- g.qpsPanel('loki_request_duration_seconds_count{cluster="$cluster", job="$namespace/distributor", route="api_prom_push"}')
+ g.qpsPanel('loki_request_duration_seconds_count{cluster=~"($cluster)", job=~"($namespace)/distributor"}')
)
.addPanel(
g.panel('Latency') +
- utils.latencyRecordingRulePanel('loki_request_duration_seconds', [utils.selector.eq('job', '$namespace/distributor'), utils.selector.eq('route', 'api_prom_push')], extra_selectors=[utils.selector.eq('cluster', '$cluster')])
+ utils.latencyRecordingRulePanel('loki_request_duration_seconds', [utils.selector.re('job', '($namespace)/distributor')], extra_selectors=[utils.selector.re('cluster', '$cluster')])
)
)
.addRow(
g.row('Ingester')
.addPanel(
g.panel('QPS') +
- g.qpsPanel('loki_request_duration_seconds_count{cluster="$cluster", job="$namespace/ingester",route="/logproto.Pusher/Push"}')
+ g.qpsPanel('loki_request_duration_seconds_count{cluster=~"$cluster", job=~"($namespace)/ingester",route="/logproto.Pusher/Push"}')
)
.addPanel(
g.panel('Latency') +
- utils.latencyRecordingRulePanel('loki_request_duration_seconds', [utils.selector.eq('job', '$namespace/ingester'), utils.selector.eq('route', '/logproto.Pusher/Push')], extra_selectors=[utils.selector.eq('cluster', '$cluster')])
+ utils.latencyRecordingRulePanel('loki_request_duration_seconds', [utils.selector.re('job', '($namespace)/ingester'), utils.selector.eq('route', '/logproto.Pusher/Push')], extra_selectors=[utils.selector.re('cluster', '$cluster')])
)
),
'loki-reads.json':
- g.dashboard('loki / Reads')
+ g.dashboard('Loki / Reads')
.addTemplate('cluster', 'kube_pod_container_info{image=~".*loki.*"}', 'cluster')
.addTemplate('namespace', 'kube_pod_container_info{image=~".*loki.*"}', 'namespace')
.addRow(
--
GitLab