diff --git a/cmd/logcli/client.go b/cmd/logcli/client.go
index 3acaea6b65eeadec632ad5b0a781da090892e2c0..9e31353ee69b52b7606f594bded3942fad1dd59d 100644
--- a/cmd/logcli/client.go
+++ b/cmd/logcli/client.go
@@ -68,7 +68,7 @@ func doRequest(path string, out interface{}) error {
}
defer func() {
if err := resp.Body.Close(); err != nil {
- fmt.Println("error closing body", err)
+ log.Println("error closing body", err)
}
}()
@@ -92,7 +92,7 @@ func wsConnect(path string) (*websocket.Conn, error) {
} else if strings.HasPrefix(url, "http") {
url = strings.Replace(url, "http", "ws", 1)
}
- fmt.Println(url)
+ log.Println(url)
h := http.Header{"Authorization": {"Basic " + base64.StdEncoding.EncodeToString([]byte(*username+":"+*password))}}
c, resp, err := websocket.DefaultDialer.Dial(url, h)
diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go
index 596bdda6f0f0e1795bfd60297f35afb3aa96bfed..42861a2d1b2423e7ce873688cd9c164841840c97 100644
--- a/cmd/logcli/main.go
+++ b/cmd/logcli/main.go
@@ -14,14 +14,17 @@ var (
username = app.Flag("username", "Username for HTTP basic auth.").Default("").Envar("GRAFANA_USERNAME").String()
password = app.Flag("password", "Password for HTTP basic auth.").Default("").Envar("GRAFANA_PASSWORD").String()
- queryCmd = app.Command("query", "Run a LogQL query.")
- queryStr = queryCmd.Arg("query", "eg '{foo=\"bar\",baz=\"blip\"}'").Required().String()
- regexpStr = queryCmd.Arg("regex", "").String()
- limit = queryCmd.Flag("limit", "Limit on number of entries to print.").Default("30").Int()
- since = queryCmd.Flag("since", "Lookback window.").Default("1h").Duration()
- forward = queryCmd.Flag("forward", "Scan forwards through logs.").Default("false").Bool()
- tail = queryCmd.Flag("tail", "Tail the logs").Short('t').Default("false").Bool()
- noLabels = queryCmd.Flag("no-labels", "Do not print labels").Default("false").Bool()
+ queryCmd = app.Command("query", "Run a LogQL query.")
+ queryStr = queryCmd.Arg("query", "eg '{foo=\"bar\",baz=\"blip\"}'").Required().String()
+ regexpStr = queryCmd.Arg("regex", "").String()
+ limit = queryCmd.Flag("limit", "Limit on number of entries to print.").Default("30").Int()
+ since = queryCmd.Flag("since", "Lookback window.").Default("1h").Duration()
+ forward = queryCmd.Flag("forward", "Scan forwards through logs.").Default("false").Bool()
+ tail = queryCmd.Flag("tail", "Tail the logs").Short('t').Default("false").Bool()
+ noLabels = queryCmd.Flag("no-labels", "Do not print any labels").Default("false").Bool()
+ ignoreLabelsKey = queryCmd.Flag("exclude-label", "Exclude labels given the provided key during output.").Strings()
+ showLabelsKey = queryCmd.Flag("include-label", "Include labels given the provided key during output.").Strings()
+ fixedLabelsLen = queryCmd.Flag("labels-length", "Set a fixed padding to labels").Default("0").Int()
labelsCmd = app.Command("labels", "Find values for a given label.")
labelName = labelsCmd.Arg("label", "The name of the label.").HintAction(listLabels).String()
diff --git a/cmd/logcli/query.go b/cmd/logcli/query.go
index b28fadb6a866a0ab2fd51c54d309f8e495bc6587..f5b57162794772872c5470162a4b82d7373ebb32 100644
--- a/cmd/logcli/query.go
+++ b/cmd/logcli/query.go
@@ -1,14 +1,12 @@
package main
import (
- "fmt"
"log"
"strings"
"time"
"github.com/fatih/color"
"github.com/prometheus/prometheus/pkg/labels"
- "github.com/prometheus/prometheus/promql"
"github.com/grafana/loki/pkg/iter"
"github.com/grafana/loki/pkg/logproto"
@@ -21,9 +19,8 @@ func doQuery() {
}
var (
- i iter.EntryIterator
- common labels.Labels
- maxLabelsLen = 100
+ i iter.EntryIterator
+ common labels.Labels
)
end := time.Now()
@@ -43,24 +40,43 @@ func doQuery() {
labelsCache := func(labels string) labels.Labels {
return cache[labels]
}
+
common = commonLabels(lss)
- i = iter.NewQueryResponseIterator(resp, d)
+
+ // Remove the labels we want to show from common
+ if len(*showLabelsKey) > 0 {
+ common = common.MatchLabels(false, *showLabelsKey...)
+ }
if len(common) > 0 {
- fmt.Println("Common labels:", color.RedString(common.String()))
+ log.Println("Common labels:", color.RedString(common.String()))
+ }
+
+ if len(*ignoreLabelsKey) > 0 {
+ log.Println("Ignoring labels key:", color.RedString(strings.Join(*ignoreLabelsKey, ",")))
}
+ // Get the max size of labels
+ maxLabelsLen := *fixedLabelsLen
for _, ls := range cache {
ls = subtract(common, ls)
+ if len(*ignoreLabelsKey) > 0 {
+ ls = ls.MatchLabels(false, *ignoreLabelsKey...)
+ }
len := len(ls.String())
if maxLabelsLen < len {
maxLabelsLen = len
}
}
+ i = iter.NewQueryResponseIterator(resp, d)
+
for i.Next() {
ls := labelsCache(i.Labels())
ls = subtract(ls, common)
+ if len(*ignoreLabelsKey) > 0 {
+ ls = ls.MatchLabels(false, *ignoreLabelsKey...)
+ }
labels := ""
if !*noLabels {
@@ -74,95 +90,3 @@ func doQuery() {
log.Fatalf("Error from iterator: %v", err)
}
}
-
-func printLogEntry(ts time.Time, lbls string, line string) {
- fmt.Println(
- color.BlueString(ts.Format(time.RFC3339)),
- color.RedString(lbls),
- strings.TrimSpace(line),
- )
-}
-
-func padLabel(ls labels.Labels, maxLabelsLen int) string {
- labels := ls.String()
- if len(labels) < maxLabelsLen {
- labels += strings.Repeat(" ", maxLabelsLen-len(labels))
- }
- return labels
-}
-
-func mustParseLabels(labels string) labels.Labels {
- ls, err := promql.ParseMetric(labels)
- if err != nil {
- log.Fatalf("Failed to parse labels: %+v", err)
- }
- return ls
-}
-
-func parseLabels(resp *logproto.QueryResponse) (map[string]labels.Labels, []labels.Labels) {
- cache := make(map[string]labels.Labels, len(resp.Streams))
- lss := make([]labels.Labels, 0, len(resp.Streams))
- for _, stream := range resp.Streams {
- ls := mustParseLabels(stream.Labels)
- cache[stream.Labels] = ls
- lss = append(lss, ls)
- }
- return cache, lss
-}
-
-func commonLabels(lss []labels.Labels) labels.Labels {
- if len(lss) == 0 {
- return nil
- }
-
- result := lss[0]
- for i := 1; i < len(lss); i++ {
- result = intersect(result, lss[i])
- }
- return result
-}
-
-func intersect(a, b labels.Labels) labels.Labels {
- var result labels.Labels
- for i, j := 0, 0; i < len(a) && j < len(b); {
- k := strings.Compare(a[i].Name, b[j].Name)
- switch {
- case k == 0:
- if a[i].Value == b[j].Value {
- result = append(result, a[i])
- }
- i++
- j++
- case k < 0:
- i++
- case k > 0:
- j++
- }
- }
- return result
-}
-
-// subtract b from a
-func subtract(a, b labels.Labels) labels.Labels {
- var result labels.Labels
- i, j := 0, 0
- for i < len(a) && j < len(b) {
- k := strings.Compare(a[i].Name, b[j].Name)
- if k != 0 || a[i].Value != b[j].Value {
- result = append(result, a[i])
- }
- switch {
- case k == 0:
- i++
- j++
- case k < 0:
- i++
- case k > 0:
- j++
- }
- }
- for ; i < len(a); i++ {
- result = append(result, a[i])
- }
- return result
-}
diff --git a/cmd/logcli/query_test.go b/cmd/logcli/query_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..a8a3db231df977fd94df22d58c1310fa9da94558
--- /dev/null
+++ b/cmd/logcli/query_test.go
@@ -0,0 +1,130 @@
+package main
+
+import (
+ "reflect"
+ "testing"
+
+ "github.com/prometheus/prometheus/pkg/labels"
+)
+
+func Test_commonLabels(t *testing.T) {
+ type args struct {
+ lss []labels.Labels
+ }
+ tests := []struct {
+ name string
+ args args
+ want labels.Labels
+ }{
+ {
+ "Extract common labels source > target",
+ args{
+ []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{bar="foo", foo="foo", baz="baz"}`)},
+ },
+ mustParseLabels(`{bar="foo"}`),
+ },
+ {
+ "Extract common labels source > target",
+ args{
+ []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{bar="foo", foo="bar", baz="baz"}`)},
+ },
+ mustParseLabels(`{foo="bar", bar="foo"}`),
+ },
+ {
+ "Extract common labels source < target",
+ args{
+ []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{bar="foo"}`)},
+ },
+ mustParseLabels(`{bar="foo"}`),
+ },
+ {
+ "Extract common labels source < target no common",
+ args{
+ []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{fo="bar"}`)},
+ },
+ labels.Labels{},
+ },
+ {
+ "Extract common labels source = target no common",
+ args{
+ []labels.Labels{mustParseLabels(`{foo="bar"}`), mustParseLabels(`{fooo="bar"}`)},
+ },
+ labels.Labels{},
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := commonLabels(tt.args.lss); !reflect.DeepEqual(got, tt.want) {
+ t.Errorf("commonLabels() = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func Test_subtract(t *testing.T) {
+ type args struct {
+ a labels.Labels
+ b labels.Labels
+ }
+ tests := []struct {
+ name string
+ args args
+ want labels.Labels
+ }{
+ {
+ "Subtract labels source > target",
+ args{
+ mustParseLabels(`{foo="bar", bar="foo"}`),
+ mustParseLabels(`{bar="foo", foo="foo", baz="baz"}`),
+ },
+ mustParseLabels(`{foo="bar"}`),
+ },
+ {
+ "Subtract labels source < target",
+ args{
+ mustParseLabels(`{foo="bar", bar="foo"}`),
+ mustParseLabels(`{bar="foo"}`),
+ },
+ mustParseLabels(`{foo="bar"}`),
+ },
+ {
+ "Subtract labels source < target no sub",
+ args{
+ mustParseLabels(`{foo="bar", bar="foo"}`),
+ mustParseLabels(`{fo="bar"}`),
+ },
+ mustParseLabels(`{bar="foo", foo="bar"}`),
+ },
+ {
+ "Subtract labels source = target no sub",
+ args{
+ mustParseLabels(`{foo="bar"}`),
+ mustParseLabels(`{fiz="buz"}`),
+ },
+ mustParseLabels(`{foo="bar"}`),
+ },
+ {
+ "Subtract labels source > target no sub",
+ args{
+ mustParseLabels(`{foo="bar"}`),
+ mustParseLabels(`{fiz="buz", foo="baz"}`),
+ },
+ mustParseLabels(`{foo="bar"}`),
+ },
+ {
+ "Subtract labels source > target no sub",
+ args{
+ mustParseLabels(`{a="b", foo="bar", baz="baz", fizz="fizz"}`),
+ mustParseLabels(`{foo="bar", baz="baz", buzz="buzz", fizz="fizz"}`),
+ },
+ mustParseLabels(`{a="b"}`),
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := subtract(tt.args.a, tt.args.b); !reflect.DeepEqual(got, tt.want) {
+ t.Errorf("subtract() = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
diff --git a/cmd/logcli/tail.go b/cmd/logcli/tail.go
index 51e64bd221e0ec50c118787daa94566e47954746..27331aa58a1048f08003ff1df3fa0ae0f8c819a9 100644
--- a/cmd/logcli/tail.go
+++ b/cmd/logcli/tail.go
@@ -2,6 +2,9 @@ package main
import (
"log"
+ "strings"
+
+ "github.com/fatih/color"
"github.com/grafana/loki/pkg/logproto"
)
@@ -14,6 +17,14 @@ func tailQuery() {
stream := new(logproto.Stream)
+ if len(*ignoreLabelsKey) > 0 {
+ log.Println("Ingoring labels key:", color.RedString(strings.Join(*ignoreLabelsKey, ",")))
+ }
+
+ if len(*showLabelsKey) > 0 {
+ log.Println("Print only labels key:", color.RedString(strings.Join(*showLabelsKey, ",")))
+ }
+
for {
err := conn.ReadJSON(stream)
if err != nil {
@@ -23,7 +34,25 @@ func tailQuery() {
labels := ""
if !*noLabels {
- labels = stream.Labels
+
+ if len(*ignoreLabelsKey) > 0 || len(*showLabelsKey) > 0 {
+
+ ls := mustParseLabels(stream.GetLabels())
+
+ if len(*showLabelsKey) > 0 {
+ ls = ls.MatchLabels(true, *showLabelsKey...)
+ }
+
+ if len(*ignoreLabelsKey) > 0 {
+ ls = ls.MatchLabels(false, *ignoreLabelsKey...)
+ }
+
+ labels = ls.String()
+
+ } else {
+
+ labels = stream.Labels
+ }
}
for _, entry := range stream.Entries {
printLogEntry(entry.Timestamp, labels, entry.Line)
diff --git a/cmd/logcli/utils.go b/cmd/logcli/utils.go
new file mode 100644
index 0000000000000000000000000000000000000000..807c3a6184873d8d0e01b5e5d34e7459afaa4feb
--- /dev/null
+++ b/cmd/logcli/utils.go
@@ -0,0 +1,109 @@
+package main
+
+import (
+ "fmt"
+ "log"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/fatih/color"
+ "github.com/grafana/loki/pkg/logproto"
+ "github.com/prometheus/prometheus/pkg/labels"
+ "github.com/prometheus/prometheus/promql"
+)
+
+// print a log entry
+func printLogEntry(ts time.Time, lbls string, line string) {
+ fmt.Println(
+ color.BlueString(ts.Format(time.RFC3339)),
+ color.RedString(lbls),
+ strings.TrimSpace(line),
+ )
+}
+
+// add some padding after labels
+func padLabel(ls labels.Labels, maxLabelsLen int) string {
+ labels := ls.String()
+ if len(labels) < maxLabelsLen {
+ labels += strings.Repeat(" ", maxLabelsLen-len(labels))
+ }
+ return labels
+}
+
+// parse labels from string
+func mustParseLabels(labels string) labels.Labels {
+ ls, err := promql.ParseMetric(labels)
+ if err != nil {
+ log.Fatalf("Failed to parse labels: %+v", err)
+ }
+ return ls
+}
+
+// parse labels from response stream
+func parseLabels(resp *logproto.QueryResponse) (map[string]labels.Labels, []labels.Labels) {
+ cache := make(map[string]labels.Labels, len(resp.Streams))
+ lss := make([]labels.Labels, 0, len(resp.Streams))
+ for _, stream := range resp.Streams {
+ ls := mustParseLabels(stream.Labels)
+ cache[stream.Labels] = ls
+ lss = append(lss, ls)
+ }
+ return cache, lss
+}
+
+// return common labels between given lavels set
+func commonLabels(lss []labels.Labels) labels.Labels {
+ if len(lss) == 0 {
+ return nil
+ }
+
+ result := lss[0]
+ for i := 1; i < len(lss); i++ {
+ result = intersect(result, lss[i])
+ }
+ return result
+}
+
+// intersect two labels set
+func intersect(a, b labels.Labels) labels.Labels {
+
+ set := labels.Labels{}
+ ma := a.Map()
+ mb := b.Map()
+
+ for ka, va := range ma {
+ if vb, ok := mb[ka]; ok {
+ if vb == va {
+ set = append(set, labels.Label{
+ Name: ka,
+ Value: va,
+ })
+ }
+ }
+ }
+ sort.Sort(set)
+ return set
+}
+
+// subtract labels set b from labels set a
+func subtract(a, b labels.Labels) labels.Labels {
+
+ set := labels.Labels{}
+ ma := a.Map()
+ mb := b.Map()
+
+ for ka, va := range ma {
+ if vb, ok := mb[ka]; ok {
+ if vb == va {
+ continue
+ }
+ }
+ set = append(set, labels.Label{
+ Name: ka,
+ Value: va,
+ })
+ }
+ sort.Sort(set)
+ return set
+}
diff --git a/docs/logcli.md b/docs/logcli.md
index d7330a2826bbb96d337ba97f02c52f09ad7b1b98..b72727867bb6c62570e795fc63cb235f545ce098 100644
--- a/docs/logcli.md
+++ b/docs/logcli.md
@@ -81,15 +81,20 @@ usage: logcli query [<flags>] <query> [<regex>]
Run a LogQL query.
Flags:
- --help Show context-sensitive help (also try --help-long and --help-man).
- --addr="" Server address, need to specify.
- --username="" Username for HTTP basic auth.
- --password="" Password for HTTP basic auth.
- --limit=30 Limit on number of entries to print.
- --since=1h Lookback window.
- --forward Scan forwards through logs.
- -t, --tail Tail the logs
- --no-labels Do not print labels
+ --help Show context-sensitive help (also try --help-long and --help-man).
+ --addr="" Server address, need to specify.
+ --username="" Username for HTTP basic auth.
+ --password="" Password for HTTP basic auth.
+ --limit=30 Limit on number of entries to print.
+ --since=1h Lookback window.
+ --forward Scan forwards through logs.
+ -t, --tail Tail the logs
+ --no-labels Do not print any labels
+ --exclude-label=EXCLUDE-LABEL ...
+ Exclude labels given the provided key during output.
+ --include-label=INCLUDE-LABEL ...
+ Include labels given the provided key during output.
+ --labels-length=0 Set a fixed padding to labels
Args:
<query> eg '{foo="bar",baz="blip"}'