From 51bdd152374a2c2c9414181c118cc82f3dec6c1d Mon Sep 17 00:00:00 2001
From: Edward Welch <edward.welch@grafana.com>
Date: Fri, 31 May 2019 16:58:10 -0400
Subject: [PATCH] renaming `metric` stage to `metrics` as it defines multiple
metrics, similar to labels stage which is also plural. Adding a couple unit
tests to regex and json stage to act as examples
---
pkg/logentry/stages/json_test.go | 43 +++++++++++++++++++++++++++++
pkg/logentry/stages/metrics_test.go | 2 +-
pkg/logentry/stages/regex_test.go | 36 ++++++++++++++++++++++++
pkg/logentry/stages/stage.go | 2 +-
4 files changed, 81 insertions(+), 2 deletions(-)
diff --git a/pkg/logentry/stages/json_test.go b/pkg/logentry/stages/json_test.go
index 772a506c..55a4e19d 100644
--- a/pkg/logentry/stages/json_test.go
+++ b/pkg/logentry/stages/json_test.go
@@ -7,11 +7,54 @@ import (
"github.com/cortexproject/cortex/pkg/util"
"github.com/pkg/errors"
+ "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v2"
)
+var testJSONYaml = `
+pipeline_stages:
+- json:
+ expressions:
+ out: message
+ app:
+ nested:
+ duration:
+`
+
+var testJSONLogLine = `
+{
+ "time":"2012-11-01T22:08:41+00:00",
+ "app":"loki",
+ "component": ["parser","type"],
+ "level" : "WARN",
+ "nested" : {"child":"value"},
+ "duration" : 125,
+ "message" : "this is a log line"
+}
+`
+
+func TestPipeline_JSON(t *testing.T) {
+ expected := map[string]interface{}{
+ "out": "this is a log line",
+ "app": "loki",
+ "nested": "{\"child\":\"value\"}",
+ "duration": float64(125),
+ }
+
+ pl, err := NewPipeline(util.Logger, loadConfig(testJSONYaml), nil, prometheus.DefaultRegisterer)
+ if err != nil {
+ t.Fatal(err)
+ }
+ lbls := model.LabelSet{}
+ ts := time.Now()
+ entry := testJSONLogLine
+ extracted := map[string]interface{}{}
+ pl.Process(lbls, extracted, &ts, &entry)
+ assert.Equal(t, expected, extracted)
+}
+
var cfg = `json:
expressions:
key1: expression1
diff --git a/pkg/logentry/stages/metrics_test.go b/pkg/logentry/stages/metrics_test.go
index b404611e..9eb3c38f 100644
--- a/pkg/logentry/stages/metrics_test.go
+++ b/pkg/logentry/stages/metrics_test.go
@@ -19,7 +19,7 @@ pipeline_stages:
- json:
expressions:
app: app
-- metric:
+- metrics:
loki_count:
type: Counter
description: uhhhhhhh
diff --git a/pkg/logentry/stages/regex_test.go b/pkg/logentry/stages/regex_test.go
index 830bc015..be65fc21 100644
--- a/pkg/logentry/stages/regex_test.go
+++ b/pkg/logentry/stages/regex_test.go
@@ -7,11 +7,47 @@ import (
"github.com/cortexproject/cortex/pkg/util"
"github.com/pkg/errors"
+ "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v2"
)
+var testRegexYaml = `
+pipeline_stages:
+- regex:
+ expression: "^(?P<ip>\\S+) (?P<identd>\\S+) (?P<user>\\S+) \\[(?P<timestamp>[\\w:/]+\\s[+\\-]\\d{4})\\] \"(?P<action>\\S+)\\s?(?P<path>\\S+)?\\s?(?P<protocol>\\S+)?\" (?P<status>\\d{3}|-) (?P<size>\\d+|-)\\s?\"?(?P<referer>[^\"]*)\"?\\s?\"?(?P<useragent>[^\"]*)?\"?$"
+`
+
+var testRegexLogLine = `11.11.11.11 - frank [25/Jan/2000:14:00:01 -0500] "GET /1986.js HTTP/1.1" 200 932 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 GTB6"`
+
+func TestPipeline_Regex(t *testing.T) {
+ expected := map[string]interface{}{
+ "ip": "11.11.11.11",
+ "identd": "-",
+ "user": "frank",
+ "timestamp": "25/Jan/2000:14:00:01 -0500",
+ "action": "GET",
+ "path": "/1986.js",
+ "protocol": "HTTP/1.1",
+ "status": "200",
+ "size": "932",
+ "referer": "-",
+ "useragent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 GTB6",
+ }
+
+ pl, err := NewPipeline(util.Logger, loadConfig(testRegexYaml), nil, prometheus.DefaultRegisterer)
+ if err != nil {
+ t.Fatal(err)
+ }
+ lbls := model.LabelSet{}
+ ts := time.Now()
+ entry := testRegexLogLine
+ extracted := map[string]interface{}{}
+ pl.Process(lbls, extracted, &ts, &entry)
+ assert.Equal(t, expected, extracted)
+}
+
var regexCfg = `regex:
expression: "regexexpression"`
diff --git a/pkg/logentry/stages/stage.go b/pkg/logentry/stages/stage.go
index f5f2d7f5..44db7f06 100644
--- a/pkg/logentry/stages/stage.go
+++ b/pkg/logentry/stages/stage.go
@@ -12,7 +12,7 @@ import (
const (
StageTypeJSON = "json"
StageTypeRegex = "regex"
- StageTypeMetric = "metric"
+ StageTypeMetric = "metrics"
StageTypeLabel = "labels"
StageTypeTimestamp = "timestamp"
StageTypeOutput = "output"
--
GitLab