renaming `metric` stage to `metrics` as it defines multiple metrics, similar...

renaming `metric` stage to `metrics` as it defines multiple metrics, similar to labels stage which is also plural.
Adding a couple unit tests to regex and json stage to act as examples

pkg/logentry/stages/json_test.go

@@ -7,11 +7,54 @@ import (
 
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v2"
 )
 
+var testJSONYaml = `
+pipeline_stages:
+- json:
+    expressions:
+      out:  message
+      app:
+      nested:
+      duration:
+`
+
+var testJSONLogLine = `
+{
+	"time":"2012-11-01T22:08:41+00:00",
+	"app":"loki",
+	"component": ["parser","type"],
+	"level" : "WARN",
+	"nested" : {"child":"value"},
+    "duration" : 125,
+	"message" : "this is a log line"
+}
+`
+
+func TestPipeline_JSON(t *testing.T) {
+	expected := map[string]interface{}{
+		"out":      "this is a log line",
+		"app":      "loki",
+		"nested":   "{\"child\":\"value\"}",
+		"duration": float64(125),
+	}
+
+	pl, err := NewPipeline(util.Logger, loadConfig(testJSONYaml), nil, prometheus.DefaultRegisterer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	lbls := model.LabelSet{}
+	ts := time.Now()
+	entry := testJSONLogLine
+	extracted := map[string]interface{}{}
+	pl.Process(lbls, extracted, &ts, &entry)
+	assert.Equal(t, expected, extracted)
+}
+
 var cfg = `json:
   expressions:
     key1: expression1

pkg/logentry/stages/metrics_test.go

@@ -19,7 +19,7 @@ pipeline_stages:
 - json:
     expressions:
       app: app
-- metric:
+- metrics:
     loki_count:
       type: Counter
       description: uhhhhhhh

pkg/logentry/stages/regex_test.go

@@ -7,11 +7,47 @@ import (
 
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v2"
 )
 
+var testRegexYaml = `
+pipeline_stages:
+- regex:
+    expression: "^(?P<ip>\\S+) (?P<identd>\\S+) (?P<user>\\S+) \\[(?P<timestamp>[\\w:/]+\\s[+\\-]\\d{4})\\] \"(?P<action>\\S+)\\s?(?P<path>\\S+)?\\s?(?P<protocol>\\S+)?\" (?P<status>\\d{3}|-) (?P<size>\\d+|-)\\s?\"?(?P<referer>[^\"]*)\"?\\s?\"?(?P<useragent>[^\"]*)?\"?$"
+`
+
+var testRegexLogLine = `11.11.11.11 - frank [25/Jan/2000:14:00:01 -0500] "GET /1986.js HTTP/1.1" 200 932 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 GTB6"`
+
+func TestPipeline_Regex(t *testing.T) {
+	expected := map[string]interface{}{
+		"ip":        "11.11.11.11",
+		"identd":    "-",
+		"user":      "frank",
+		"timestamp": "25/Jan/2000:14:00:01 -0500",
+		"action":    "GET",
+		"path":      "/1986.js",
+		"protocol":  "HTTP/1.1",
+		"status":    "200",
+		"size":      "932",
+		"referer":   "-",
+		"useragent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 GTB6",
+	}
+
+	pl, err := NewPipeline(util.Logger, loadConfig(testRegexYaml), nil, prometheus.DefaultRegisterer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	lbls := model.LabelSet{}
+	ts := time.Now()
+	entry := testRegexLogLine
+	extracted := map[string]interface{}{}
+	pl.Process(lbls, extracted, &ts, &entry)
+	assert.Equal(t, expected, extracted)
+}
+
 var regexCfg = `regex: 
   expression: "regexexpression"`
 

pkg/logentry/stages/stage.go

@@ -12,7 +12,7 @@ import (
 const (
 	StageTypeJSON      = "json"
 	StageTypeRegex     = "regex"
-	StageTypeMetric    = "metric"
+	StageTypeMetric    = "metrics"
 	StageTypeLabel     = "labels"
 	StageTypeTimestamp = "timestamp"
 	StageTypeOutput    = "output"