diff --git a/pkg/logentry/stages/json_test.go b/pkg/logentry/stages/json_test.go index 772a506c191ce8f4ac1e9e03b3a8c442d6b24bd0..55a4e19d7a485f98c7f5661bf472659fe3b15ba0 100644 --- a/pkg/logentry/stages/json_test.go +++ b/pkg/logentry/stages/json_test.go @@ -7,11 +7,54 @@ import ( "github.com/cortexproject/cortex/pkg/util" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/stretchr/testify/assert" "gopkg.in/yaml.v2" ) +var testJSONYaml = ` +pipeline_stages: +- json: + expressions: + out: message + app: + nested: + duration: +` + +var testJSONLogLine = ` +{ + "time":"2012-11-01T22:08:41+00:00", + "app":"loki", + "component": ["parser","type"], + "level" : "WARN", + "nested" : {"child":"value"}, + "duration" : 125, + "message" : "this is a log line" +} +` + +func TestPipeline_JSON(t *testing.T) { + expected := map[string]interface{}{ + "out": "this is a log line", + "app": "loki", + "nested": "{\"child\":\"value\"}", + "duration": float64(125), + } + + pl, err := NewPipeline(util.Logger, loadConfig(testJSONYaml), nil, prometheus.DefaultRegisterer) + if err != nil { + t.Fatal(err) + } + lbls := model.LabelSet{} + ts := time.Now() + entry := testJSONLogLine + extracted := map[string]interface{}{} + pl.Process(lbls, extracted, &ts, &entry) + assert.Equal(t, expected, extracted) +} + var cfg = `json: expressions: key1: expression1 diff --git a/pkg/logentry/stages/metrics_test.go b/pkg/logentry/stages/metrics_test.go index b404611ea600105fd36f15ff591b90af64241173..9eb3c38fa98ce355c1a26649c31fd1aa7d706f52 100644 --- a/pkg/logentry/stages/metrics_test.go +++ b/pkg/logentry/stages/metrics_test.go @@ -19,7 +19,7 @@ pipeline_stages: - json: expressions: app: app -- metric: +- metrics: loki_count: type: Counter description: uhhhhhhh diff --git a/pkg/logentry/stages/regex_test.go b/pkg/logentry/stages/regex_test.go index 830bc0159042c3e9755780d23a17278894d3fc9e..be65fc215777b30ad1cb907aff9630749664873a 100644 --- a/pkg/logentry/stages/regex_test.go +++ b/pkg/logentry/stages/regex_test.go @@ -7,11 +7,47 @@ import ( "github.com/cortexproject/cortex/pkg/util" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/stretchr/testify/assert" "gopkg.in/yaml.v2" ) +var testRegexYaml = ` +pipeline_stages: +- regex: + expression: "^(?P<ip>\\S+) (?P<identd>\\S+) (?P<user>\\S+) \\[(?P<timestamp>[\\w:/]+\\s[+\\-]\\d{4})\\] \"(?P<action>\\S+)\\s?(?P<path>\\S+)?\\s?(?P<protocol>\\S+)?\" (?P<status>\\d{3}|-) (?P<size>\\d+|-)\\s?\"?(?P<referer>[^\"]*)\"?\\s?\"?(?P<useragent>[^\"]*)?\"?$" +` + +var testRegexLogLine = `11.11.11.11 - frank [25/Jan/2000:14:00:01 -0500] "GET /1986.js HTTP/1.1" 200 932 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 GTB6"` + +func TestPipeline_Regex(t *testing.T) { + expected := map[string]interface{}{ + "ip": "11.11.11.11", + "identd": "-", + "user": "frank", + "timestamp": "25/Jan/2000:14:00:01 -0500", + "action": "GET", + "path": "/1986.js", + "protocol": "HTTP/1.1", + "status": "200", + "size": "932", + "referer": "-", + "useragent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 GTB6", + } + + pl, err := NewPipeline(util.Logger, loadConfig(testRegexYaml), nil, prometheus.DefaultRegisterer) + if err != nil { + t.Fatal(err) + } + lbls := model.LabelSet{} + ts := time.Now() + entry := testRegexLogLine + extracted := map[string]interface{}{} + pl.Process(lbls, extracted, &ts, &entry) + assert.Equal(t, expected, extracted) +} + var regexCfg = `regex: expression: "regexexpression"` diff --git a/pkg/logentry/stages/stage.go b/pkg/logentry/stages/stage.go index f5f2d7f52b16394e4d38391215755723848bcba9..44db7f06628d247b90e49d5dd922a9dfebca05b6 100644 --- a/pkg/logentry/stages/stage.go +++ b/pkg/logentry/stages/stage.go @@ -12,7 +12,7 @@ import ( const ( StageTypeJSON = "json" StageTypeRegex = "regex" - StageTypeMetric = "metric" + StageTypeMetric = "metrics" StageTypeLabel = "labels" StageTypeTimestamp = "timestamp" StageTypeOutput = "output"