From f6d5ebf5136a291a89424e87e8f1d90d1416b0af Mon Sep 17 00:00:00 2001
From: Tom Wilkie <tom.wilkie@gmail.com>
Date: Sun, 25 Nov 2018 12:06:28 +0000
Subject: [PATCH] Document how to get promtail running locally.
Signed-off-by: Tom Wilkie <tom.wilkie@gmail.com>
---
.gitignore | 1 +
README.md | 88 ++++----------------
cmd/tempo/main.go | 2 +-
docs/api.md | 70 ++++++++++++++++
docs/promtail-local-config.yaml | 9 ++
docs/{local.yaml => tempo-local-config.yaml} | 0
pkg/promtail/config.go | 40 +--------
pkg/promtail/position.go | 2 +-
pkg/promtail/target.go | 9 +-
pkg/promtail/targetmanager.go | 5 +-
10 files changed, 110 insertions(+), 116 deletions(-)
create mode 100644 docs/api.md
create mode 100644 docs/promtail-local-config.yaml
rename docs/{local.yaml => tempo-local-config.yaml} (100%)
diff --git a/.gitignore b/.gitignore
index a3a41d54..b5c50e79 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,4 @@ mixin/vendor/
cmd/tempo/tempo
cmd/promtail/promtail
/tempo
+/promtail
diff --git a/README.md b/README.md
index 3b9fd8e6..eb22b059 100644
--- a/README.md
+++ b/README.md
@@ -8,11 +8,25 @@ not index the contents of the logs, but rather a set of labels for each log stea
## Run it locally
-Tempo can be run in a single host, no-dependencies mode using the following commands:
+Tempo can be run in a single host, no-dependencies mode using the following commands.
+
+Tempo consists of 3 components; `tempo` is the main server, responsible for storing
+logs and processing queries. `promtail` is the agent, responsible for gather logs
+and sending them to tempo and `grafana` as the UI.
+
+To run tempo, use the following commands:
```
$ go build ./cmd/tempo
-$ ./tempo -config.file=./docs/local.yaml
+$ ./tempo -config.file=./docs/tempo-local-config.yaml
+...
+```
+
+To run promtail, use the following commands:
+
+```
+$ go build ./cmd/promtail
+$ ./promtail -config.file=./docs/promtail-local-config.yaml -positions.file=./positions.yaml -client.url=http://localhost/api/prom/push
...
```
@@ -89,73 +103,3 @@ Args:
<query> eg '{foo="bar",baz="blip"}'
[<regex>]
```
-
-## API
-
-*nb* Authentication is out of scope for this project. You are expected to run an
-authenticating reverse proxy in front of our services, such as an Nginx with basic
-auth or a OAuth2 proxy.
-
-There are 4 API endpoints:
-
-- `POST /api/prom/push`
-
- For sending log entries, expects a snappy compresses proto in the HTTP Body.
-
-- `GET /api/prom/query`
-
- For doing queries, accepts the following paramters in the query-string:
- - `query`: a logQL query
- - `limit`: max number of entries to return
- - `start`: the start time for the query, as a nanosecond Unix epoch (nanoseconds since 1970)
- - `end`: the end time for the query, as a nanosecond Unix epoch (nanoseconds since 1970)
- - `direction`: `forward` or `backward`, useful when specifying a limit
- - `regexp`: a regex to filter the returned results, will eventually be rolled into the query language
-
- Responses looks like this:
- ```
- {
- "streams": [
- {
- "labels": "{instance=\"...\", job=\"...\", namespace=\"...\"}",
- "entries": [
- {
- "timestamp": "2018-06-27T05:20:28.699492635Z",
- "line": "..."
- },
- ...
- ]
- },
- ...
- ]
- }
- ```
-
-- `GET /api/prom/label`
-
- For retrieving the names of the labels one can query on.
-
- Responses looks like this:
- ```
- {
- "values": [
- "instance",
- "job",
- ...
- ]
- }
- ```
-
-- `GET /api/prom/label/<name>/values`
- For retrieving the label values one can query on.
-
- Responses looks like this:
- ```
- {
- "values": [
- "default",
- "cortex-ops",
- ...
- ]
- }
- ```
diff --git a/cmd/tempo/main.go b/cmd/tempo/main.go
index f980184b..8566f244 100644
--- a/cmd/tempo/main.go
+++ b/cmd/tempo/main.go
@@ -62,7 +62,7 @@ func readConfig(filename string, cfg *tempo.Config) error {
return errors.Wrap(err, "Error reading config file: %v")
}
- if err := yaml.Unmarshal(buf, &cfg); err != nil {
+ if err := yaml.UnmarshalStrict(buf, &cfg); err != nil {
return errors.Wrap(err, "Error reading config file: %v")
}
return nil
diff --git a/docs/api.md b/docs/api.md
new file mode 100644
index 00000000..439af9f2
--- /dev/null
+++ b/docs/api.md
@@ -0,0 +1,70 @@
+
+## API
+
+*nb* Authentication is out of scope for this project. You are expected to run an
+authenticating reverse proxy in front of our services, such as an Nginx with basic
+auth or a OAuth2 proxy.
+
+There are 4 API endpoints:
+
+- `POST /api/prom/push`
+
+ For sending log entries, expects a snappy compresses proto in the HTTP Body.
+
+- `GET /api/prom/query`
+
+ For doing queries, accepts the following paramters in the query-string:
+ - `query`: a logQL query
+ - `limit`: max number of entries to return
+ - `start`: the start time for the query, as a nanosecond Unix epoch (nanoseconds since 1970)
+ - `end`: the end time for the query, as a nanosecond Unix epoch (nanoseconds since 1970)
+ - `direction`: `forward` or `backward`, useful when specifying a limit
+ - `regexp`: a regex to filter the returned results, will eventually be rolled into the query language
+
+ Responses looks like this:
+ ```
+ {
+ "streams": [
+ {
+ "labels": "{instance=\"...\", job=\"...\", namespace=\"...\"}",
+ "entries": [
+ {
+ "timestamp": "2018-06-27T05:20:28.699492635Z",
+ "line": "..."
+ },
+ ...
+ ]
+ },
+ ...
+ ]
+ }
+ ```
+
+- `GET /api/prom/label`
+
+ For retrieving the names of the labels one can query on.
+
+ Responses looks like this:
+ ```
+ {
+ "values": [
+ "instance",
+ "job",
+ ...
+ ]
+ }
+ ```
+
+- `GET /api/prom/label/<name>/values`
+ For retrieving the label values one can query on.
+
+ Responses looks like this:
+ ```
+ {
+ "values": [
+ "default",
+ "cortex-ops",
+ ...
+ ]
+ }
+ ```
diff --git a/docs/promtail-local-config.yaml b/docs/promtail-local-config.yaml
new file mode 100644
index 00000000..a083d0ca
--- /dev/null
+++ b/docs/promtail-local-config.yaml
@@ -0,0 +1,9 @@
+scrape_configs:
+- job_name: system
+
+ static_configs:
+ - targets:
+ - localhost
+ labels:
+ job: system
+ __path__: /var/log/system.log
diff --git a/docs/local.yaml b/docs/tempo-local-config.yaml
similarity index 100%
rename from docs/local.yaml
rename to docs/tempo-local-config.yaml
diff --git a/pkg/promtail/config.go b/pkg/promtail/config.go
index 515afbb2..9c3f41c0 100644
--- a/pkg/promtail/config.go
+++ b/pkg/promtail/config.go
@@ -4,20 +4,17 @@ import (
"fmt"
"io/ioutil"
"path/filepath"
- "strings"
yaml "gopkg.in/yaml.v2"
"github.com/prometheus/prometheus/config"
sd_config "github.com/prometheus/prometheus/discovery/config"
+ "github.com/prometheus/prometheus/discovery/targetgroup"
)
// Config for promtail, describing what files to watch.
type Config struct {
ScrapeConfig []ScrapeConfig `yaml:"scrape_configs,omitempty"`
-
- // Catches all undefined fields and must be empty after parsing.
- XXX map[string]interface{} `yaml:",inline"`
}
// LoadConfig loads config from a file.
@@ -28,34 +25,19 @@ func LoadConfig(filename string) (*Config, error) {
}
var cfg Config
- if err := yaml.Unmarshal(buf, &cfg); err != nil {
+ if err := yaml.UnmarshalStrict(buf, &cfg); err != nil {
return nil, err
}
return &cfg, nil
}
-// UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
- type plain Config
- err := unmarshal((*plain)(c))
- if err != nil {
- return err
- }
- if err = checkOverflow(c.XXX, "config"); err != nil {
- return err
- }
- return nil
-}
-
// ScrapeConfig describes a job to scrape.
type ScrapeConfig struct {
JobName string `yaml:"job_name,omitempty"`
- ServiceDiscoveryConfig sd_config.ServiceDiscoveryConfig `yaml:",inline"`
RelabelConfigs []*config.RelabelConfig `yaml:"relabel_configs,omitempty"`
-
- // Catches all undefined fields and must be empty after parsing.
- XXX map[string]interface{} `yaml:",inline"`
+ ServiceDiscoveryConfig sd_config.ServiceDiscoveryConfig `yaml:",inline"`
+ StaticConfig targetgroup.Group `yaml:"static_config"`
}
// UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -65,22 +47,8 @@ func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
if err != nil {
return err
}
- if err = checkOverflow(c.XXX, "scrape_config"); err != nil {
- return err
- }
if len(c.JobName) == 0 {
return fmt.Errorf("job_name is empty")
}
return nil
}
-
-func checkOverflow(m map[string]interface{}, ctx string) error {
- if len(m) > 0 {
- var keys []string
- for k := range m {
- keys = append(keys, k)
- }
- return fmt.Errorf("unknown fields in %s: %s", ctx, strings.Join(keys, ", "))
- }
- return nil
-}
diff --git a/pkg/promtail/position.go b/pkg/promtail/position.go
index 3c911927..225de730 100644
--- a/pkg/promtail/position.go
+++ b/pkg/promtail/position.go
@@ -111,7 +111,7 @@ func readPositionsFile(filename string) (map[string]int64, error) {
}
var p positionsFile
- if err := yaml.Unmarshal(buf, &p); err != nil {
+ if err := yaml.UnmarshalStrict(buf, &p); err != nil {
return nil, err
}
diff --git a/pkg/promtail/target.go b/pkg/promtail/target.go
index 9f2cf5df..2ef0ce0e 100644
--- a/pkg/promtail/target.go
+++ b/pkg/promtail/target.go
@@ -8,6 +8,7 @@ import (
"github.com/grafana/tempo/pkg/helpers"
"github.com/hpcloud/tail"
+ "github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/model"
log "github.com/sirupsen/logrus"
@@ -41,16 +42,14 @@ type Target struct {
// NewTarget create a new Target.
func NewTarget(c *Client, positions *Positions, path string, labels model.LabelSet) (*Target, error) {
- log.Info("newTarget", labels)
-
watcher, err := fsnotify.NewWatcher()
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "fsnotify.NewWatcher")
}
if err := watcher.Add(path); err != nil {
helpers.LogError("closing watcher", watcher.Close)
- return nil, err
+ return nil, errors.Wrap(err, "watcher.Add")
}
t := &Target{
@@ -66,7 +65,7 @@ func NewTarget(c *Client, positions *Positions, path string, labels model.LabelS
// Fist, we're going to add all the existing files
fis, err := ioutil.ReadDir(t.path)
if err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "ioutil.ReadDir")
}
for _, fi := range fis {
if fi.IsDir() {
diff --git a/pkg/promtail/targetmanager.go b/pkg/promtail/targetmanager.go
index a261b3b4..c5b39bc2 100644
--- a/pkg/promtail/targetmanager.go
+++ b/pkg/promtail/targetmanager.go
@@ -61,7 +61,6 @@ func NewTargetManager(
}
config := map[string]sd_config.ServiceDiscoveryConfig{}
-
for _, cfg := range scrapeConfig {
s := &syncer{
log: logger,
@@ -75,6 +74,7 @@ func NewTargetManager(
}
go tm.run()
+ go tm.manager.Run()
return tm, tm.manager.ApplyConfig(config)
}
@@ -109,10 +109,13 @@ func (s *syncer) Sync(groups []*targetgroup.Group) {
for _, group := range groups {
for _, t := range group.Targets {
+ level.Debug(s.log).Log("msg", "new target", "labels", t)
+
labels := group.Labels.Merge(t)
labels = relabel.Process(labels, s.relabelConfig...)
// Drop empty targets (drop in relabeling).
if labels == nil {
+ level.Debug(s.log).Log("msg", "dropping target, no labels")
continue
}
--
GitLab