From 6185d543209a16772e8e84eb712e8f58d3fcf0c8 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Fri, 13 Sep 2019 08:53:59 -0400
Subject: [PATCH] Updated stream json objects to be more parse friendly (#1010)
* Added custom marshaller for Entry
Signed-off-by: Joe Elliott <number101010@gmail.com>
* Added comment and test
Signed-off-by: Joe Elliott <number101010@gmail.com>
* Fixed spelling error
* Added a custom marshaller to map entries => values
* Added support for labels
Signed-off-by: Joe Elliott <number101010@gmail.com>
* Changed test to only check to the microsecond level
Signed-off-by: Joe Elliott <number101010@gmail.com>
* Extended Entry tests
Signed-off-by: Joe Elliott <number101010@gmail.com>
* Swapped to nanosecs encoded as strings
* Cleaned up docs with new loki paths. Removed references to deprecated endpoints
* Added support for /loki/api/v1/push
Signed-off-by: Joe Elliott <number101010@gmail.com>
* Removed obsolete comment
Signed-off-by: Joe Elliott <number101010@gmail.com>
* Updated docs for new push endpoint
Signed-off-by: Joe Elliott <number101010@gmail.com>
* Ran gofmt on test file
Signed-off-by: Joe Elliott <number101010@gmail.com>
* Set GOGC=20 on test to avoid out of memory issue in circle ci
Signed-off-by: Joe Elliott <number101010@gmail.com>
---
Makefile | 2 +-
docs/loki/api.md | 122 ++++++++++++-----------------------
pkg/logproto/marshal.go | 35 ++++++++++
pkg/logproto/marshal_test.go | 85 ++++++++++++++++++++++++
pkg/loki/modules.go | 4 ++
5 files changed, 167 insertions(+), 81 deletions(-)
create mode 100644 pkg/logproto/marshal.go
create mode 100644 pkg/logproto/marshal_test.go
diff --git a/Makefile b/Makefile
index d20a2449..d687919b 100644
--- a/Makefile
+++ b/Makefile
@@ -214,7 +214,7 @@ lint:
########
test: all
- go test -p=6 ./...
+ GOGC=20 go test -p=6 ./...
#########
# Clean #
diff --git a/docs/loki/api.md b/docs/loki/api.md
index fe451d7f..9a4aa49f 100644
--- a/docs/loki/api.md
+++ b/docs/loki/api.md
@@ -2,7 +2,7 @@
The Loki server has the following API endpoints (_Note:_ Authentication is out of scope for this project):
-- `POST /api/prom/push`
+- `POST /loki/api/v1/push`
For sending log entries, expects a snappy compressed proto in the HTTP Body:
@@ -23,7 +23,7 @@ The Loki server has the following API endpoints (_Note:_ Authentication is out o
```
-- `GET /api/v1/query`
+- `GET /loki/api/v1/query`
For doing instant queries at a single point in time, accepts the following parameters in the query-string:
@@ -51,7 +51,7 @@ The Loki server has the following API endpoints (_Note:_ Authentication is out o
Examples:
```bash
- $ curl -G -s "http://localhost:3100/api/v1/query" --data-urlencode 'query=sum(rate({job="varlogs"}[10m])) by (level)' | jq
+ $ curl -G -s "http://localhost:3100/loki/api/v1/query" --data-urlencode 'query=sum(rate({job="varlogs"}[10m])) by (level)' | jq
{
"status" : "success",
"data": {
@@ -88,31 +88,31 @@ The Loki server has the following API endpoints (_Note:_ Authentication is out o
```
```bash
- curl -G -s "http://localhost:3100/api/v1/query" --data-urlencode 'query={job="varlogs"}' | jq
+ curl -G -s "http://localhost:3100/loki/api/v1/query" --data-urlencode 'query={job="varlogs"}' | jq
{
- "status" : "success",
- "data": {
- "resultType": "streams",
- "result": [
- {
- "labels": "{filename=\"/var/log/myproject.log\", job=\"varlogs\", level=\"info\"}",
- "entries": [
- {
- "ts": "2019-06-06T19:25:41.972739Z",
- "line": "foo"
- },
- {
- "ts": "2019-06-06T19:25:41.972722Z",
- "line": "bar"
- }
- ]
- }
+ "resultType": "streams",
+ "result": [
+ {
+ "stream": {
+ "filename": "/var/hostlog/syslog",
+ "job": "varlogs"
+ },
+ "values": [
+ [
+ "1568234281726420425",
+ "foo"
+ ],
+ [
+ "1568234269716526880",
+ "bar"
+ ]
]
}
+ ]
}
```
-- `GET /api/v1/query_range`
+- `GET /loki/api/v1/query_range`
For doing queries over a range of time, accepts the following parameters in the query-string:
@@ -142,7 +142,7 @@ The Loki server has the following API endpoints (_Note:_ Authentication is out o
Examples:
```bash
- $ curl -G -s "http://localhost:3100/api/v1/query_range" --data-urlencode 'query=sum(rate({job="varlogs"}[10m])) by (level)' --data-urlencode 'step=300' | jq
+ $ curl -G -s "http://localhost:3100/loki/api/v1/query_range" --data-urlencode 'query=sum(rate({job="varlogs"}[10m])) by (level)' --data-urlencode 'step=300' | jq
{
"status" : "success",
"data": {
@@ -192,69 +192,31 @@ The Loki server has the following API endpoints (_Note:_ Authentication is out o
```
```bash
- curl -G -s "http://localhost:3100/api/v1/query_range" --data-urlencode 'query={job="varlogs"}' | jq
- {
- "status" : "success",
- "data": {
- "resultType": "streams",
- "result": [
- {
- "labels": "{filename=\"/var/log/myproject.log\", job=\"varlogs\", level=\"info\"}",
- "entries": [
- {
- "ts": "2019-06-06T19:25:41.972739Z",
- "line": "foo"
- },
- {
- "ts": "2019-06-06T19:25:41.972722Z",
- "line": "bar"
- }
- ]
- }
- ]
- }
- }
- ```
-
-- `GET /api/prom/query`
-
- For doing queries, accepts the following parameters in the query-string:
-
- - `query`: a [logQL query](../querying.md) (eg: `{name=~"mysql.+"}` or `{name=~"mysql.+"} |= "error"`)
- - `limit`: max number of entries to return
- - `start`: the start time for the query, as a nanosecond Unix epoch (nanoseconds since 1970) or as RFC3339Nano (eg: "2006-01-02T15:04:05.999999999-07:00"). Default is always one hour ago.
- - `end`: the end time for the query, as a nanosecond Unix epoch (nanoseconds since 1970) or as RFC3339Nano (eg: "2006-01-02T15:04:05.999999999-07:00"). Default is current time.
- - `direction`: `forward` or `backward`, useful when specifying a limit. Default is backward.
- - `regexp`: a regex to filter the returned results
-
- Loki needs to query the index store in order to find log streams for particular labels and the store is spread out by time,
- so you need to specify the start and end labels accordingly. Querying a long time into the history will cause additional
- load to the index server and make the query slower.
-
- > This endpoint will be deprecated in the future you should use `api/v1/query_range` instead.
- > You can only query for logs, it doesn't accept [queries returning metrics](../querying.md#counting-logs).
-
- Responses looks like this:
-
- ```json
+ curl -G -s "http://localhost:3100/loki/api/v1/query_range" --data-urlencode 'query={job="varlogs"}' | jq
{
- "streams": [
+ "resultType": "streams",
+ "result": [
{
- "labels": "{instance=\"...\", job=\"...\", namespace=\"...\"}",
- "entries": [
- {
- "ts": "2018-06-27T05:20:28.699492635Z",
- "line": "..."
- },
- ...
+ "stream": {
+ "filename": "/var/hostlog/syslog",
+ "job": "varlogs"
+ },
+ "values": [
+ [
+ "1568234281726420425",
+ "foo"
+ ],
+ [
+ "1568234269716526880",
+ "bar"
+ ]
]
- },
- ...
+ }
]
}
```
-- `GET /api/prom/label`
+- `GET /loki/api/v1/label`
For doing label name queries, accepts the following parameters in the query-string:
@@ -273,7 +235,7 @@ The Loki server has the following API endpoints (_Note:_ Authentication is out o
}
```
-- `GET /api/prom/label/<name>/values`
+- `GET /loki/api/v1/label/<name>/values`
For doing label values queries, accepts the following parameters in the query-string:
diff --git a/pkg/logproto/marshal.go b/pkg/logproto/marshal.go
new file mode 100644
index 00000000..396f690b
--- /dev/null
+++ b/pkg/logproto/marshal.go
@@ -0,0 +1,35 @@
+package logproto
+
+import (
+ "encoding/json"
+ fmt "fmt"
+
+ "github.com/prometheus/prometheus/promql"
+)
+
+// MarshalJSON converts an Entry object to be prom compatible for http queries
+func (e *Entry) MarshalJSON() ([]byte, error) {
+ l, err := json.Marshal(e.Line)
+ if err != nil {
+ return nil, err
+ }
+ return []byte(fmt.Sprintf("[\"%d\",%s]", e.Timestamp.UnixNano(), l)), nil
+}
+
+// MarshalJSON converts a Stream object to be prom compatible for http queries
+func (s *Stream) MarshalJSON() ([]byte, error) {
+ parsedLabels, err := promql.ParseMetric(s.Labels)
+ if err != nil {
+ return nil, err
+ }
+ l, err := json.Marshal(parsedLabels)
+ if err != nil {
+ return nil, err
+ }
+ e, err := json.Marshal(s.Entries)
+ if err != nil {
+ return nil, err
+ }
+
+ return []byte(fmt.Sprintf("{\"stream\":%s,\"values\":%s}", l, e)), nil
+}
diff --git a/pkg/logproto/marshal_test.go b/pkg/logproto/marshal_test.go
new file mode 100644
index 00000000..9b793667
--- /dev/null
+++ b/pkg/logproto/marshal_test.go
@@ -0,0 +1,85 @@
+package logproto
+
+import (
+ "encoding/json"
+ fmt "fmt"
+ reflect "reflect"
+ "testing"
+ time "time"
+
+ "github.com/prometheus/prometheus/promql"
+ "github.com/stretchr/testify/require"
+)
+
+var (
+ entries = []Entry{
+ {
+ Timestamp: time.Now(),
+ Line: "testline",
+ },
+ {
+ Timestamp: time.Date(2019, 9, 10, 1, 1, 1, 1, time.UTC),
+ Line: "{}\"'!@$%&*^(_)(",
+ },
+ }
+ streams = []Stream{
+ {
+ Labels: "{}",
+ Entries: []Entry{},
+ },
+ {
+ Labels: "{name=\"value\",name1=\"value1\"}",
+ Entries: []Entry{},
+ },
+ }
+)
+
+func Test_EntryMarshalJSON(t *testing.T) {
+ var array []interface{}
+
+ for _, entry := range entries {
+
+ bytes, err := entry.MarshalJSON()
+ require.NoError(t, err)
+
+ err = json.Unmarshal(bytes, &array)
+ require.NoError(t, err)
+
+ timestamp, ok := array[0].(string)
+ require.True(t, ok)
+
+ line, ok := array[1].(string)
+ require.True(t, ok)
+
+ require.Equal(t, fmt.Sprint(entry.Timestamp.UnixNano()), timestamp, "Timestamps not equal ", array[0])
+ require.Equal(t, entry.Line, line, "Lines are not equal ", array[1])
+ }
+}
+
+func Test_StreamMarshalJSON(t *testing.T) {
+ actual := struct {
+ Labels map[string]string `json:"stream"`
+ Entries []Entry `json:"values"`
+ }{}
+
+ for _, expected := range streams {
+
+ bytes, err := expected.MarshalJSON()
+ require.NoError(t, err)
+
+ err = json.Unmarshal(bytes, &actual)
+ require.NoError(t, err)
+
+ // check labels
+ expectedLabels, err := promql.ParseMetric(expected.Labels)
+ require.NoError(t, err)
+
+ require.Equal(t, len(actual.Labels), len(expectedLabels))
+ for _, l := range expectedLabels {
+ require.Equal(t, l.Value, actual.Labels[l.Name])
+ }
+
+ // check entries
+ require.True(t, reflect.DeepEqual(actual.Entries, expected.Entries))
+ }
+}
diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go
index 7d5a9f24..17e043b0 100644
--- a/pkg/loki/modules.go
+++ b/pkg/loki/modules.go
@@ -127,9 +127,13 @@ func (t *Loki) initDistributor() (err error) {
}
t.server.HTTP.Path("/ready").Handler(http.HandlerFunc(t.distributor.ReadinessHandler))
+ t.server.HTTP.Handle("/loki/api/v1/push", middleware.Merge(
+ t.httpAuthMiddleware,
+ ).Wrap(http.HandlerFunc(t.distributor.PushHandler)))
t.server.HTTP.Handle("/api/prom/push", middleware.Merge(
t.httpAuthMiddleware,
).Wrap(http.HandlerFunc(t.distributor.PushHandler)))
+
return
}
--
GitLab