Skip to content

Commit

Permalink
promtail: restore build support for non-linux platforms
Browse files Browse the repository at this point in the history
Support for building promtail on non-Linux platforms (and Linux
platforms where CGO has been disabled) has been added.

This commit also reverts accidentally commited code in journaltarget.go
that panics if a timestamp occurs before the timestamp of the previous
received journal entry.
  • Loading branch information
rfratto committed Jul 10, 2019
1 parent 303bffa commit 837e102
Show file tree
Hide file tree
Showing 7 changed files with 121 additions and 78 deletions.
5 changes: 3 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ loki-build-image/$(UPTODATE): loki-build-image/*
# All the boiler plate for building golang follows:
SUDO := $(shell docker info >/dev/null 2>&1 || echo "sudo -E")
BUILD_IN_CONTAINER := true
CGO_ENABLED := 0
# RM is parameterized to allow CircleCI to run builds, as it
# currently disallows `docker run --rm`. This value is overridden
# in circle.yml
Expand Down Expand Up @@ -149,13 +150,13 @@ $(EXES) $(DEBUG_EXES) $(PROTO_GOS) $(YACC_GOS) lint test shell check-generated-f
else

$(DEBUG_EXES): loki-build-image/$(UPTODATE)
CGO_ENABLED=0 go build $(DEBUG_GO_FLAGS) -o $@ ./$(@D)
CGO_ENABLED=$(CGO_ENABLED) go build $(DEBUG_GO_FLAGS) -o $@ ./$(@D)
$(NETGO_CHECK)
# Copy the delve binary to make it easily available to put in the binary's container.
[ -f "/go/bin/dlv" ] && mv "/go/bin/dlv" $(@D)/dlv

$(EXES): loki-build-image/$(UPTODATE)
CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
CGO_ENABLED=$(CGO_ENABLED) go build $(GO_FLAGS) -o $@ ./$(@D)
$(NETGO_CHECK)

%.pb.go: loki-build-image/$(UPTODATE)
Expand Down
6 changes: 3 additions & 3 deletions build/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ ARG GOARCH="amd64"
COPY . /go/src/github.com/grafana/loki
WORKDIR /go/src/github.com/grafana/loki
RUN touch loki-build-image/.uptodate &&\
mkdir /build
mkdir /build

# production image
FROM golang as builder-production
ARG APP
RUN make BUILD_IN_CONTAINER=false cmd/${APP}/${APP} &&\
RUN make CGO_ENABLED=1 BUILD_IN_CONTAINER=false cmd/${APP}/${APP} &&\
mv cmd/${APP}/${APP} /build/${APP}

FROM scratch as production
Expand All @@ -28,7 +28,7 @@ COPY --from=builder-production /build/${APP} /usr/bin/${APP}
FROM golang as builder-debug
ARG APP
RUN go get github.com/go-delve/delve/cmd/dlv &&\
make BUILD_IN_CONTAINER=false cmd/promtail/promtail-debug &&\
make CGO_ENBALED=1 BUILD_IN_CONTAINER=false cmd/promtail/promtail-debug &&\
mv cmd/${APP}/${APP}-debug /build/app-debug &&\
mv cmd/${APP}/dlv /build/dlv

Expand Down
11 changes: 3 additions & 8 deletions pkg/promtail/targets/journaltarget.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
// +build linux,cgo

package targets

import (
Expand Down Expand Up @@ -53,7 +55,6 @@ type JournalTarget struct {

r journalReader
until chan time.Time
last uint64
}

// NewJournalTarget configures a new JournalTarget.
Expand Down Expand Up @@ -122,7 +123,7 @@ func journalTargetWithReader(

go func() {
err := t.r.Follow(until, ioutil.Discard)
if err != sdjournal.ErrExpired {
if err != nil && err != sdjournal.ErrExpired {
level.Error(t.logger).Log("msg", "received error during sdjournal follow", "err", err.Error())
}
}()
Expand All @@ -139,12 +140,6 @@ func (t *JournalTarget) formatter(entry *sdjournal.JournalEntry) (string, error)
return journalEmptyStr, nil
}
entryLabels := makeJournalFields(entry.Fields)
fmt.Printf("%s: %s\n", ts, msg)

if entry.RealtimeTimestamp < t.last {
panic("UNORDERED")
}
t.last = entry.RealtimeTimestamp

// Add constant labels
for k, v := range t.labels {
Expand Down
2 changes: 1 addition & 1 deletion pkg/promtail/targets/journaltarget_test.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// +build linux
// +build linux,cgo

package targets

Expand Down
78 changes: 15 additions & 63 deletions pkg/promtail/targets/journaltargetmanager.go
Original file line number Diff line number Diff line change
@@ -1,93 +1,45 @@
// +build !linux !cgo

package targets

import (
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/grafana/loki/pkg/logentry/stages"
"github.com/grafana/loki/pkg/promtail/api"
"github.com/grafana/loki/pkg/promtail/positions"
"github.com/grafana/loki/pkg/promtail/scrape"
"github.com/prometheus/client_golang/prometheus"
)

// JournalTargetManager manages a series of JournalTargets.
type JournalTargetManager struct {
logger log.Logger
targets map[string]*JournalTarget
}
type JournalTargetManager struct{}

// NewJournalTargetManager creates a new JournalTargetManager.
// NewJournalTargetManager returns nil as JournalTargets are not supported
// on this platform.
func NewJournalTargetManager(
logger log.Logger,
positions *positions.Positions,
client api.EntryHandler,
scrapeConfigs []scrape.Config,
) (*JournalTargetManager, error) {
tm := &JournalTargetManager{
logger: logger,
targets: make(map[string]*JournalTarget),
}

for _, cfg := range scrapeConfigs {
if cfg.JournalConfig == nil {
continue
}

registerer := prometheus.DefaultRegisterer
pipeline, err := stages.NewPipeline(log.With(logger, "component", "journal_pipeline"), cfg.PipelineStages, &cfg.JobName, registerer)
if err != nil {
return nil, err
}

t, err := NewJournalTarget(
logger,
pipeline.Wrap(client),
positions,
cfg.RelabelConfigs,
cfg.JournalConfig,
)
if err != nil {
return nil, err
}

tm.targets[cfg.JobName] = t
}

return tm, nil
level.Warn(logger).Log("msg", "WARNING!!! Journal target manager initialized on platform without Journal support!")
return &JournalTargetManager{}, nil
}

// Ready returns true if at least one JournalTarget is also ready.
// Ready always returns false for JournalTargetManager on non-Linux
// platforms.
func (tm *JournalTargetManager) Ready() bool {
for _, t := range tm.targets {
if t.Ready() {
return true
}
}
return false
}

// Stop stops the JournalTargetManager and all of its JournalTargets.
func (tm *JournalTargetManager) Stop() {
for _, t := range tm.targets {
if err := t.Stop(); err != nil {
level.Error(t.logger).Log("msg", "error stopping JournalTarget", "err", err.Error())
}
}
}
// Stop is a no-op on non-Linux platforms.
func (tm *JournalTargetManager) Stop() {}

// ActiveTargets returns the list of JournalTargets where journal data
// is being read. ActiveTargets is an alias to AllTargets as
// JournalTargets cannot be deactivated, only stopped.
// ActiveTargets always returns nil on non-Linux platforms.
func (tm *JournalTargetManager) ActiveTargets() map[string][]Target {
return tm.AllTargets()
return nil
}

// AllTargets returns the list of all targets where journal data
// is currently being read.
// AllTargets always returns nil on non-Linux platforms.
func (tm *JournalTargetManager) AllTargets() map[string][]Target {
result := make(map[string][]Target, len(tm.targets))
for k, v := range tm.targets {
result[k] = []Target{v}
}
return result
return nil
}
95 changes: 95 additions & 0 deletions pkg/promtail/targets/journaltargetmanager_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
// +build cgo

package targets

import (
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/grafana/loki/pkg/logentry/stages"
"github.com/grafana/loki/pkg/promtail/api"
"github.com/grafana/loki/pkg/promtail/positions"
"github.com/grafana/loki/pkg/promtail/scrape"
"github.com/prometheus/client_golang/prometheus"
)

// JournalTargetManager manages a series of JournalTargets.
type JournalTargetManager struct {
logger log.Logger
targets map[string]*JournalTarget
}

// NewJournalTargetManager creates a new JournalTargetManager.
func NewJournalTargetManager(
logger log.Logger,
positions *positions.Positions,
client api.EntryHandler,
scrapeConfigs []scrape.Config,
) (*JournalTargetManager, error) {
tm := &JournalTargetManager{
logger: logger,
targets: make(map[string]*JournalTarget),
}

for _, cfg := range scrapeConfigs {
if cfg.JournalConfig == nil {
continue
}

registerer := prometheus.DefaultRegisterer
pipeline, err := stages.NewPipeline(log.With(logger, "component", "journal_pipeline"), cfg.PipelineStages, &cfg.JobName, registerer)
if err != nil {
return nil, err
}

t, err := NewJournalTarget(
logger,
pipeline.Wrap(client),
positions,
cfg.RelabelConfigs,
cfg.JournalConfig,
)
if err != nil {
return nil, err
}

tm.targets[cfg.JobName] = t
}

return tm, nil
}

// Ready returns true if at least one JournalTarget is also ready.
func (tm *JournalTargetManager) Ready() bool {
for _, t := range tm.targets {
if t.Ready() {
return true
}
}
return false
}

// Stop stops the JournalTargetManager and all of its JournalTargets.
func (tm *JournalTargetManager) Stop() {
for _, t := range tm.targets {
if err := t.Stop(); err != nil {
level.Error(t.logger).Log("msg", "error stopping JournalTarget", "err", err.Error())
}
}
}

// ActiveTargets returns the list of JournalTargets where journal data
// is being read. ActiveTargets is an alias to AllTargets as
// JournalTargets cannot be deactivated, only stopped.
func (tm *JournalTargetManager) ActiveTargets() map[string][]Target {
return tm.AllTargets()
}

// AllTargets returns the list of all targets where journal data
// is currently being read.
func (tm *JournalTargetManager) AllTargets() map[string][]Target {
result := make(map[string][]Target, len(tm.targets))
for k, v := range tm.targets {
result[k] = []Target{v}
}
return result
}
2 changes: 1 addition & 1 deletion pkg/promtail/targets/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func (tm *TargetManagers) AllTargets() map[string][]Target {
return result
}

// Ready if there's at least one ready FileTargetManager
// Ready if there's at least one ready target manager.
func (tm *TargetManagers) Ready() bool {
for _, t := range tm.targetManagers {
if t.Ready() {
Expand Down

0 comments on commit 837e102

Please sign in to comment.