From 3e2a8d9b31bd8b7da3e89c69ef78802af0cec7e5 Mon Sep 17 00:00:00 2001 From: gpolaert Date: Mon, 3 Aug 2020 19:38:59 +0200 Subject: [PATCH] feat: Add new logger module - Pubstack Analytics Module (#1331) * Pubstack Analytics V1 (#11) * V1 Pubstack (#7) * feat: Add Pubstack Logger (#6) * first version of pubstack analytics * bypass viperconfig * commit #1 * gofmt * update configuration and make the tests pass * add readme on how to configure the adapter and update the network calls * update logging and fix intake url definition * feat: Pubstack Analytics Connector * fixing go mod * fix: bad behaviour on appending path to auction url * add buffering * support bootstyrap like configuration * implement route for all the objects * supports termination signal handling for goroutines * move readme to the correct location * wording * enable configuration reload + add tests * fix logs messages * fix tests * fix log line * conclude merge * merge * update go mod Co-authored-by: Amaury Ravanel * fix duplicated channel keys Co-authored-by: Amaury Ravanel * first pass - PR reviews * rename channel* -> eventChannel * dead code * Review (#10) * use json.Decoder * update documentation * use nil instead []byte("") * clean code * do not use http.DefaultClient * fix race condition (need validation) * separate the sender and buffer logics * refactor the default configuration * remove error counter * Review GP + AR * updating default config * add more logs * remove alias fields in json * fix json serializer * close event channels Co-authored-by: Amaury Ravanel * fix race condition * first pass (pr reviews) * refactor: store enabled modules into a dedicated struct * stop goroutine * test: improve coverage * PR Review * Revert "refactor: store enabled modules into a dedicated struct" This reverts commit f57d9d61680c74244effc39a5d96d6cbb2f19f7d. # Conflicts: # analytics/config/config_test.go Co-authored-by: Amaury Ravanel --- analytics/clients/http.go | 12 + analytics/config/config.go | 17 ++ analytics/config/config_test.go | 41 +++ analytics/pubstack/README.md | 28 ++ analytics/pubstack/config.go | 51 ++++ analytics/pubstack/config_test.go | 102 +++++++ .../pubstack/eventchannel/eventchannel.go | 137 +++++++++ .../eventchannel/eventchannel_test.go | 136 +++++++++ analytics/pubstack/eventchannel/sender.go | 45 +++ .../pubstack/eventchannel/sender_test.go | 40 +++ analytics/pubstack/helpers/json.go | 88 ++++++ analytics/pubstack/helpers/json_test.go | 61 ++++ .../pubstack/mocks/mock_openrtb_request.json | 64 ++++ .../pubstack/mocks/mock_openrtb_response.json | 91 ++++++ analytics/pubstack/pubstack_module.go | 273 ++++++++++++++++++ analytics/pubstack/pubstack_module_test.go | 186 ++++++++++++ config/config.go | 24 +- go.mod | 1 + go.sum | 2 + 19 files changed, 1398 insertions(+), 1 deletion(-) create mode 100644 analytics/clients/http.go create mode 100644 analytics/pubstack/README.md create mode 100644 analytics/pubstack/config.go create mode 100644 analytics/pubstack/config_test.go create mode 100644 analytics/pubstack/eventchannel/eventchannel.go create mode 100644 analytics/pubstack/eventchannel/eventchannel_test.go create mode 100644 analytics/pubstack/eventchannel/sender.go create mode 100644 analytics/pubstack/eventchannel/sender_test.go create mode 100644 analytics/pubstack/helpers/json.go create mode 100644 analytics/pubstack/helpers/json_test.go create mode 100644 analytics/pubstack/mocks/mock_openrtb_request.json create mode 100644 analytics/pubstack/mocks/mock_openrtb_response.json create mode 100644 analytics/pubstack/pubstack_module.go create mode 100644 analytics/pubstack/pubstack_module_test.go diff --git a/analytics/clients/http.go b/analytics/clients/http.go new file mode 100644 index 00000000000..bc7f863ebdd --- /dev/null +++ b/analytics/clients/http.go @@ -0,0 +1,12 @@ +package clients + +import ( + "net/http" +) + +var defaultHttpInstance = http.DefaultClient + +func GetDefaultHttpInstance() *http.Client { + // TODO 2020-06-22 @see https://github.com/prebid/prebid-server/pull/1331#discussion_r436110097 + return defaultHttpInstance +} diff --git a/analytics/config/config.go b/analytics/config/config.go index 7be7c8ecca3..7f7ded0ffc4 100644 --- a/analytics/config/config.go +++ b/analytics/config/config.go @@ -3,7 +3,9 @@ package config import ( "github.com/golang/glog" "github.com/prebid/prebid-server/analytics" + "github.com/prebid/prebid-server/analytics/clients" "github.com/prebid/prebid-server/analytics/filesystem" + "github.com/prebid/prebid-server/analytics/pubstack" "github.com/prebid/prebid-server/config" ) @@ -17,6 +19,21 @@ func NewPBSAnalytics(analytics *config.Analytics) analytics.PBSAnalyticsModule { glog.Fatalf("Could not initialize FileLogger for file %v :%v", analytics.File.Filename, err) } } + if analytics.Pubstack.Enabled { + pubstackModule, err := pubstack.NewPubstackModule( + clients.GetDefaultHttpInstance(), + analytics.Pubstack.ScopeId, + analytics.Pubstack.IntakeUrl, + analytics.Pubstack.ConfRefresh, + analytics.Pubstack.Buffers.EventCount, + analytics.Pubstack.Buffers.BufferSize, + analytics.Pubstack.Buffers.Timeout) + if err == nil { + modules = append(modules, pubstackModule) + } else { + glog.Errorf("Could not initialize PubstackModule: %v", err) + } + } return modules } diff --git a/analytics/config/config_test.go b/analytics/config/config_test.go index 7d97fb5f1be..583d475e786 100644 --- a/analytics/config/config_test.go +++ b/analytics/config/config_test.go @@ -1,6 +1,7 @@ package config import ( + "github.com/stretchr/testify/assert" "net/http" "os" "testing" @@ -73,6 +74,13 @@ func initAnalytics(count *int) analytics.PBSAnalyticsModule { } func TestNewPBSAnalytics(t *testing.T) { + pbsAnalytics := NewPBSAnalytics(&config.Analytics{}) + instance := pbsAnalytics.(enabledAnalytics) + + assert.Equal(t, len(instance), 0) +} + +func TestNewPBSAnalytics_FileLogger(t *testing.T) { if _, err := os.Stat(TEST_DIR); os.IsNotExist(err) { if err = os.MkdirAll(TEST_DIR, 0755); err != nil { t.Fatalf("Could not create test directory for FileLogger") @@ -88,4 +96,37 @@ func TestNewPBSAnalytics(t *testing.T) { default: t.Fatalf("Failed to initialize analytics module") } + + pbsAnalytics := NewPBSAnalytics(&config.Analytics{File: config.FileLogs{Filename: TEST_DIR + "/test"}}) + instance := pbsAnalytics.(enabledAnalytics) + + assert.Equal(t, len(instance), 1) +} + +func TestNewPBSAnalytics_Pubstack(t *testing.T) { + + pbsAnalyticsWithoutError := NewPBSAnalytics(&config.Analytics{ + Pubstack: config.Pubstack{ + Enabled: true, + ScopeId: "scopeId", + IntakeUrl: "https://pubstack.io/intake", + Buffers: config.PubstackBuffer{ + BufferSize: "100KB", + EventCount: 0, + Timeout: "30s", + }, + ConfRefresh: "2h", + }, + }) + instanceWithoutError := pbsAnalyticsWithoutError.(enabledAnalytics) + + assert.Equal(t, len(instanceWithoutError), 1) + + pbsAnalyticsWithError := NewPBSAnalytics(&config.Analytics{ + Pubstack: config.Pubstack{ + Enabled: true, + }, + }) + instanceWithError := pbsAnalyticsWithError.(enabledAnalytics) + assert.Equal(t, len(instanceWithError), 0) } diff --git a/analytics/pubstack/README.md b/analytics/pubstack/README.md new file mode 100644 index 00000000000..51c5fdb6bb3 --- /dev/null +++ b/analytics/pubstack/README.md @@ -0,0 +1,28 @@ +# Pubstack Analytics + +In order to use the pubstack analytics module, it needs to be configured by the host. + +You can configure the server using the following environment variables: + +```bash +export PBS_ANALYTICS_PUBSTACK_ENABLED="true" +export PBS_ANALYTICS_PUBSTACK_ENDPOINT="https://openrtb.preview.pubstack.io/v1/openrtb2" +export PBS_ANALYTICS_PUBSTACK_SCOPEID= # should be an UUIDv4 +``` + +Or using the pbs configuration file and by appending the following block: + +```yaml +analytics: + pubstack: + # Required properties + enabled: true + endpoint: "https://openrtb.preview.pubstack.io/v1/openrtb2" + scopeid: "" # The scopeId provided by the Pubstack Support Team + # Optional properties (advanced configuration) + configuration_refresh_delay: "2h" # Dynamic configuration delay + buffers: # Flush events to Pubstack when (first condition reached) + size: "2MB" # greater than 2MB + count : 100 # greater than 100 events + timeout: "15m" # greater than 15 minutes +``` \ No newline at end of file diff --git a/analytics/pubstack/config.go b/analytics/pubstack/config.go new file mode 100644 index 00000000000..472acf68ead --- /dev/null +++ b/analytics/pubstack/config.go @@ -0,0 +1,51 @@ +package pubstack + +import ( + "encoding/json" + "github.com/docker/go-units" + "net/http" + "net/url" + "time" +) + +func fetchConfig(client *http.Client, endpoint *url.URL) (*Configuration, error) { + + res, err := client.Get(endpoint.String()) + if err != nil { + return nil, err + } + + defer res.Body.Close() + c := Configuration{} + err = json.NewDecoder(res.Body).Decode(&c) + if err != nil { + return nil, err + } + return &c, nil +} + +func newBufferConfig(count int, size, duration string) (*bufferConfig, error) { + pDuration, err := time.ParseDuration(duration) + if err != nil { + return nil, err + } + pSize, err := units.FromHumanSize(size) + if err != nil { + return nil, err + } + return &bufferConfig{ + pDuration, + int64(count), + pSize, + }, nil +} + +func (a *Configuration) isSameAs(b *Configuration) bool { + sameEndpoint := a.Endpoint == b.Endpoint + sameScopeID := a.ScopeID == b.ScopeID + sameFeature := len(a.Features) == len(b.Features) + for key := range a.Features { + sameFeature = sameFeature && a.Features[key] == b.Features[key] + } + return sameFeature && sameEndpoint && sameScopeID +} diff --git a/analytics/pubstack/config_test.go b/analytics/pubstack/config_test.go new file mode 100644 index 00000000000..bb6fd0bddbb --- /dev/null +++ b/analytics/pubstack/config_test.go @@ -0,0 +1,102 @@ +package pubstack + +import ( + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "net/url" + "testing" +) + +func TestFetchConfig(t *testing.T) { + configResponse := `{ + "scopeId": "scopeId", + "endpoint": "https://pubstack.io", + "features": { + "auction": true, + "cookiesync": true, + "amp": true, + "setuid": false, + "video": false + } + }` + + server := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + defer req.Body.Close() + res.Write([]byte(configResponse)) + res.WriteHeader(200) + })) + + defer server.Close() + + endpoint, _ := url.Parse(server.URL) + cfg, _ := fetchConfig(server.Client(), endpoint) + + assert.Equal(t, cfg.ScopeID, "scopeId") + assert.Equal(t, cfg.Endpoint, "https://pubstack.io") + assert.Equal(t, cfg.Features[auction], true) + assert.Equal(t, cfg.Features[cookieSync], true) + assert.Equal(t, cfg.Features[amp], true) + assert.Equal(t, cfg.Features[setUID], false) + assert.Equal(t, cfg.Features[video], false) +} + +func TestFetchConfig_Error(t *testing.T) { + configResponse := `{ + "error": "scopeId", + }` + + server := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + defer req.Body.Close() + res.Write([]byte(configResponse)) + res.WriteHeader(200) + })) + + defer server.Close() + + endpoint, _ := url.Parse(server.URL) + cfg, err := fetchConfig(server.Client(), endpoint) + + assert.Nil(t, cfg) + assert.NotNil(t, err) +} + +func TestIsSameAs(t *testing.T) { + copyConfig := func(conf Configuration) *Configuration { + newConfig := conf + newConfig.Features = make(map[string]bool) + for k := range conf.Features { + newConfig.Features[k] = conf.Features[k] + } + return &newConfig + } + + a := &Configuration{ + ScopeID: "scopeId", + Endpoint: "endpoint", + Features: map[string]bool{ + "auction": true, + "cookiesync": true, + "amp": true, + "setuid": false, + "video": false, + }, + } + + assert.True(t, a.isSameAs(copyConfig(*a))) + + b := copyConfig(*a) + b.ScopeID = "anotherId" + assert.False(t, a.isSameAs(b)) + + b = copyConfig(*a) + b.Endpoint = "anotherEndpoint" + assert.False(t, a.isSameAs(b)) + + b = copyConfig(*a) + b.Features["auction"] = true + assert.True(t, a.isSameAs(b)) + b.Features["auction"] = false + assert.False(t, a.isSameAs(b)) + +} diff --git a/analytics/pubstack/eventchannel/eventchannel.go b/analytics/pubstack/eventchannel/eventchannel.go new file mode 100644 index 00000000000..b8dc4dd8e28 --- /dev/null +++ b/analytics/pubstack/eventchannel/eventchannel.go @@ -0,0 +1,137 @@ +package eventchannel + +import ( + "bytes" + "compress/gzip" + "sync" + "time" + + "github.com/golang/glog" +) + +type Metrics struct { + bufferSize int64 + eventCount int64 +} +type Limit struct { + maxByteSize int64 + maxEventCount int64 + maxTime time.Duration +} +type EventChannel struct { + gz *gzip.Writer + buff *bytes.Buffer + + ch chan []byte + endCh chan int + metrics Metrics + muxGzBuffer sync.RWMutex + send Sender + limit Limit +} + +func NewEventChannel(sender Sender, maxByteSize, maxEventCount int64, maxTime time.Duration) *EventChannel { + b := &bytes.Buffer{} + gzw := gzip.NewWriter(b) + + c := EventChannel{ + gz: gzw, + buff: b, + ch: make(chan []byte), + endCh: make(chan int), + metrics: Metrics{}, + send: sender, + limit: Limit{maxByteSize, maxEventCount, maxTime}, + } + go c.start() + return &c +} + +func (c *EventChannel) Push(event []byte) { + c.ch <- event +} + +func (c *EventChannel) Close() { + c.endCh <- 1 +} + +func (c *EventChannel) buffer(event []byte) { + c.muxGzBuffer.Lock() + defer c.muxGzBuffer.Unlock() + + _, err := c.gz.Write(event) + if err != nil { + glog.Warning("[pubstack] fail to compress, skip the event") + return + } + + c.metrics.eventCount++ + c.metrics.bufferSize += int64(len(event)) +} + +func (c *EventChannel) isBufferFull() bool { + c.muxGzBuffer.RLock() + defer c.muxGzBuffer.RUnlock() + return c.metrics.eventCount >= c.limit.maxEventCount || c.metrics.bufferSize >= c.limit.maxByteSize +} + +func (c *EventChannel) reset() { + // reset buffer + c.gz.Reset(c.buff) + c.buff.Reset() + + // reset metrics + c.metrics.eventCount = 0 + c.metrics.bufferSize = 0 +} + +func (c *EventChannel) flush() { + c.muxGzBuffer.Lock() + defer c.muxGzBuffer.Unlock() + + if c.metrics.eventCount == 0 || c.metrics.bufferSize == 0 { + return + } + + // finish writing gzip header + err := c.gz.Flush() + if err != nil { + glog.Warning("[pubstack] fail to flush gzipped buffer") + return + } + + // copy the current buffer to send the payload in a new thread + payload := make([]byte, c.buff.Len()) + _, err = c.buff.Read(payload) + if err != nil { + glog.Warning("[pubstack] fail to copy the buffer") + return + } + + // reset buffers and writers + c.reset() + + // send events (async) + go c.send(payload) +} + +func (c *EventChannel) start() { + ticker := time.NewTicker(c.limit.maxTime) + + for { + select { + case <-c.endCh: + c.flush() + return + // event is received + case event := <-c.ch: + c.buffer(event) + if c.isBufferFull() { + c.flush() + } + // time between 2 flushes has passed + case <-ticker.C: + c.flush() + } + } +} diff --git a/analytics/pubstack/eventchannel/eventchannel_test.go b/analytics/pubstack/eventchannel/eventchannel_test.go new file mode 100644 index 00000000000..9fdcfe976a6 --- /dev/null +++ b/analytics/pubstack/eventchannel/eventchannel_test.go @@ -0,0 +1,136 @@ +package eventchannel + +import ( + "bytes" + "compress/gzip" + "github.com/stretchr/testify/assert" + "io/ioutil" + "sync" + "testing" + "time" +) + +var maxByteSize = int64(15) +var maxEventCount = int64(3) +var maxTime = 2 * time.Hour + +func readGz(encoded bytes.Buffer) string { + gr, _ := gzip.NewReader(bytes.NewBuffer(encoded.Bytes())) + defer gr.Close() + + decoded, _ := ioutil.ReadAll(gr) + return string(decoded) +} + +func newSender(data *[]byte) Sender { + mux := &sync.Mutex{} + return func(payload []byte) error { + mux.Lock() + defer mux.Unlock() + event := bytes.Buffer{} + event.Write(payload) + *data = append(*data, readGz(event)...) + return nil + } +} + +func TestEventChannel_isBufferFull(t *testing.T) { + + send := func(_ []byte) error { return nil } + + eventChannel := NewEventChannel(send, maxByteSize, maxEventCount, maxTime) + defer eventChannel.Close() + + eventChannel.buffer([]byte("one")) + eventChannel.buffer([]byte("two")) + + assert.Equal(t, eventChannel.isBufferFull(), false) + + eventChannel.buffer([]byte("three")) + + assert.Equal(t, eventChannel.isBufferFull(), true) + + eventChannel.reset() + + assert.Equal(t, eventChannel.isBufferFull(), false) + + eventChannel.buffer([]byte("big-event-abcdefghijklmnopqrstuvwxyz")) + + assert.Equal(t, eventChannel.isBufferFull(), true) + +} + +func TestEventChannel_reset(t *testing.T) { + send := func(_ []byte) error { return nil } + + eventChannel := NewEventChannel(send, maxByteSize, maxEventCount, maxTime) + defer eventChannel.Close() + + assert.Equal(t, eventChannel.metrics.eventCount, int64(0)) + assert.Equal(t, eventChannel.metrics.bufferSize, int64(0)) + + eventChannel.buffer([]byte("one")) + eventChannel.buffer([]byte("two")) + + assert.NotEqual(t, eventChannel.metrics.eventCount, int64(0)) + assert.NotEqual(t, eventChannel.metrics.bufferSize, int64(0)) + + eventChannel.reset() + + assert.Equal(t, eventChannel.buff.Len(), 0) + assert.Equal(t, eventChannel.metrics.eventCount, int64(0)) + assert.Equal(t, eventChannel.metrics.bufferSize, int64(0)) +} + +func TestEventChannel_flush(t *testing.T) { + data := make([]byte, 0) + send := newSender(&data) + + eventChannel := NewEventChannel(send, maxByteSize, maxEventCount, maxTime) + defer eventChannel.Close() + + eventChannel.buffer([]byte("one")) + eventChannel.buffer([]byte("two")) + eventChannel.buffer([]byte("three")) + eventChannel.flush() + time.Sleep(10 * time.Millisecond) + + assert.Equal(t, string(data), "onetwothree") +} + +func TestEventChannel_close(t *testing.T) { + data := make([]byte, 0) + send := newSender(&data) + + eventChannel := NewEventChannel(send, 15000, 15000, 2*time.Hour) + + eventChannel.buffer([]byte("one")) + eventChannel.buffer([]byte("two")) + eventChannel.buffer([]byte("three")) + eventChannel.Close() + + time.Sleep(10 * time.Millisecond) + + assert.Equal(t, string(data), "onetwothree") +} + +func TestEventChannel_Push(t *testing.T) { + data := make([]byte, 0) + send := newSender(&data) + + eventChannel := NewEventChannel(send, 15000, 5, 5*time.Millisecond) + defer eventChannel.Close() + + eventChannel.Push([]byte("one")) + eventChannel.Push([]byte("two")) + eventChannel.Push([]byte("three")) + eventChannel.Push([]byte("four")) + eventChannel.Push([]byte("five")) + eventChannel.Push([]byte("six")) + eventChannel.Push([]byte("seven")) + + time.Sleep(10 * time.Millisecond) + + assert.Equal(t, string(data), "onetwothreefourfivesixseven") + +} diff --git a/analytics/pubstack/eventchannel/sender.go b/analytics/pubstack/eventchannel/sender.go new file mode 100644 index 00000000000..951de4d414e --- /dev/null +++ b/analytics/pubstack/eventchannel/sender.go @@ -0,0 +1,45 @@ +package eventchannel + +import ( + "bytes" + "fmt" + "github.com/golang/glog" + "net/http" + "net/url" + "path" +) + +type Sender = func(payload []byte) error + +func NewHttpSender(client *http.Client, endpoint string) Sender { + return func(payload []byte) error { + req, err := http.NewRequest(http.MethodPost, endpoint, bytes.NewReader(payload)) + if err != nil { + glog.Error(err) + return err + } + + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Encoding", "gzip") + + resp, err := client.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + glog.Errorf("[pubstack] Wrong code received %d instead of %d", resp.StatusCode, http.StatusOK) + return fmt.Errorf("wrong code received %d instead of %d", resp.StatusCode, http.StatusOK) + } + return nil + } +} + +func BuildEndpointSender(client *http.Client, baseUrl string, module string) Sender { + endpoint, err := url.Parse(baseUrl) + if err != nil { + glog.Error(err) + } + endpoint.Path = path.Join(endpoint.Path, "intake", module) + return NewHttpSender(client, endpoint.String()) +} diff --git a/analytics/pubstack/eventchannel/sender_test.go b/analytics/pubstack/eventchannel/sender_test.go new file mode 100644 index 00000000000..1185435e4ab --- /dev/null +++ b/analytics/pubstack/eventchannel/sender_test.go @@ -0,0 +1,40 @@ +package eventchannel + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildEndpointSender(t *testing.T) { + requestBody := make([]byte, 10) + server := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + defer req.Body.Close() + requestBody, _ = ioutil.ReadAll(req.Body) + res.WriteHeader(200) + })) + + defer server.Close() + + sender := BuildEndpointSender(server.Client(), server.URL, "module") + err := sender([]byte("message")) + + assert.Equal(t, requestBody, []byte("message")) + assert.Nil(t, err) +} + +func TestBuildEndpointSender_Error(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + res.WriteHeader(400) + })) + + defer server.Close() + + sender := BuildEndpointSender(server.Client(), server.URL, "module") + err := sender([]byte("message")) + + assert.NotNil(t, err) +} diff --git a/analytics/pubstack/helpers/json.go b/analytics/pubstack/helpers/json.go new file mode 100644 index 00000000000..f02f1120626 --- /dev/null +++ b/analytics/pubstack/helpers/json.go @@ -0,0 +1,88 @@ +package helpers + +import ( + "encoding/json" + "fmt" + + "github.com/prebid/prebid-server/analytics" +) + +func JsonifyAuctionObject(ao *analytics.AuctionObject, scope string) ([]byte, error) { + b, err := json.Marshal(&struct { + Scope string `json:"scope"` + *analytics.AuctionObject + }{ + Scope: scope, + AuctionObject: ao, + }) + + if err == nil { + b = append(b, byte('\n')) + return b, nil + } + return nil, fmt.Errorf("auction object badly formed %v", err) +} + +func JsonifyVideoObject(vo *analytics.VideoObject, scope string) ([]byte, error) { + b, err := json.Marshal(&struct { + Scope string `json:"scope"` + *analytics.VideoObject + }{ + Scope: scope, + VideoObject: vo, + }) + + if err == nil { + b = append(b, byte('\n')) + return b, nil + } + return nil, fmt.Errorf("video object badly formed %v", err) +} + +func JsonifyCookieSync(cso *analytics.CookieSyncObject, scope string) ([]byte, error) { + b, err := json.Marshal(&struct { + Scope string `json:"scope"` + *analytics.CookieSyncObject + }{ + Scope: scope, + CookieSyncObject: cso, + }) + + if err == nil { + b = append(b, byte('\n')) + return b, nil + } + return nil, fmt.Errorf("cookie sync object badly formed %v", err) +} + +func JsonifySetUIDObject(so *analytics.SetUIDObject, scope string) ([]byte, error) { + b, err := json.Marshal(&struct { + Scope string `json:"scope"` + *analytics.SetUIDObject + }{ + Scope: scope, + SetUIDObject: so, + }) + + if err == nil { + b = append(b, byte('\n')) + return b, nil + } + return nil, fmt.Errorf("set UID object badly formed %v", err) +} + +func JsonifyAmpObject(ao *analytics.AmpObject, scope string) ([]byte, error) { + b, err := json.Marshal(&struct { + Scope string `json:"scope"` + *analytics.AmpObject + }{ + Scope: scope, + AmpObject: ao, + }) + + if err == nil { + b = append(b, byte('\n')) + return b, nil + } + return nil, fmt.Errorf("amp object badly formed %v", err) +} diff --git a/analytics/pubstack/helpers/json_test.go b/analytics/pubstack/helpers/json_test.go new file mode 100644 index 00000000000..4e36e8db2be --- /dev/null +++ b/analytics/pubstack/helpers/json_test.go @@ -0,0 +1,61 @@ +package helpers + +import ( + "github.com/mxmCherry/openrtb" + "github.com/prebid/prebid-server/analytics" + "github.com/prebid/prebid-server/usersync" + "net/http" + "testing" +) + +func TestJsonifyAuctionObject(t *testing.T) { + ao := &analytics.AuctionObject{ + Status: http.StatusOK, + } + if _, err := JsonifyAuctionObject(ao, "scopeId"); err != nil { + t.Fail() + } + +} + +func TestJsonifyVideoObject(t *testing.T) { + vo := &analytics.VideoObject{ + Status: http.StatusOK, + } + if _, err := JsonifyVideoObject(vo, "scopeId"); err != nil { + t.Fail() + } +} + +func TestJsonifyCookieSync(t *testing.T) { + cso := &analytics.CookieSyncObject{ + Status: http.StatusOK, + BidderStatus: []*usersync.CookieSyncBidders{}, + } + if _, err := JsonifyCookieSync(cso, "scopeId"); err != nil { + t.Fail() + } +} + +func TestJsonifySetUIDObject(t *testing.T) { + so := &analytics.SetUIDObject{ + Status: http.StatusOK, + Bidder: "any-bidder", + UID: "uid string", + } + if _, err := JsonifySetUIDObject(so, "scopeId"); err != nil { + t.Fail() + } +} + +func TestJsonifyAmpObject(t *testing.T) { + ao := &analytics.AmpObject{ + Status: http.StatusOK, + Errors: make([]error, 0), + AuctionResponse: &openrtb.BidResponse{}, + AmpTargetingValues: map[string]string{}, + } + if _, err := JsonifyAmpObject(ao, "scopeId"); err != nil { + t.Fail() + } +} diff --git a/analytics/pubstack/mocks/mock_openrtb_request.json b/analytics/pubstack/mocks/mock_openrtb_request.json new file mode 100644 index 00000000000..03b9665b247 --- /dev/null +++ b/analytics/pubstack/mocks/mock_openrtb_request.json @@ -0,0 +1,64 @@ +{ + "id": "19c2eeb8-824c-4604-af41-a59b2b7bb895", + "site": { + "page": "https%3A%2F%2Fdebug.mediasquare.fr%2Fdebug%2Fprebid%2Fmsq_desktop.html%3Fpbjs_debug%3Dtrue" + }, + "user": { + "ext": {} + }, + "regs": { + "ext": {} + }, + "test": 1, + "imp": [ + { + "id": "0341252e-b3b0-4dff-a0ef-1ced63369bd5", + "ext": { + "appnexus": { + "placementId": 5724999 + } + }, + "secure": 1, + "banner": { + "format": [ + { + "w": 970, + "h": 250 + } + ] + } + }, + { + "id": "3ac0ffa3-01de-44d2-9baf-1fee79026624", + "ext": { + "msqClassic": { + "placementId": 10471298 + } + }, + "secure": 1, + "banner": { + "format": [ + { + "w": 300, + "h": 250 + }, + { + "w": 300, + "h": 600 + } + ] + } + } + ], + "tmax": 500, + "ext": { + "prebid": { + "bidadjustmentfactors": { + "msqClassic": 0.8, + "msqBrand": 0.8, + "msqMax": 0.8, + "msqMaxView": 0.8 + } + } + } +} \ No newline at end of file diff --git a/analytics/pubstack/mocks/mock_openrtb_response.json b/analytics/pubstack/mocks/mock_openrtb_response.json new file mode 100644 index 00000000000..6f4d1965b8c --- /dev/null +++ b/analytics/pubstack/mocks/mock_openrtb_response.json @@ -0,0 +1,91 @@ +{ + "id": "19c2eeb8-824c-4604-af41-a59b2b7bb895", + "seatbid": [{ + "seat": "958", + "bid": [ + { + "id": "7706636740145184841", + "impid": "0341252e-b3b0-4dff-a0ef-1ced63369bd5", + "price": 0.500000, + "adid": "29681110", + "adm": "some-test-ad", + "adomain": ["appnexus.com"], + "iurl": "http://nym1-ib.adnxs.com/cr?id=29681110", + "cid": "958", + "crid": "29681110", + "h": 970, + "w": 250, + "ext": { + "appnexus": { + "brand_id": 1, + "brand_category_id": 1, + "auction_id": 8189378542222915032, + "bid_ad_type": 0, + "bidder_id": 2, + "ranking_price": 0.000000 + } + } + }, + { + "id": "7706636740145184842", + "impid": "0341252e-b3b0-4dff-a0ef-1ced63369bd5", + "price": 0.0, + "adid": "29681114", + "adm": "some-test-ad2", + "adomain": ["appnexus.com"], + "iurl": "http://nym1-ib.adnxs.com/cr?id=29681114", + "cid": "959", + "crid": "29681114", + "h": 970, + "w": 250, + "ext": { + "appnexus": { + "brand_id": 1, + "brand_category_id": 1, + "auction_id": 8189378542222915032, + "bid_ad_type": 0, + "bidder_id": 2, + "ranking_price": 0.000000 + } + } + },{ + "id": "7706636740145184842", + "impid": "3ac0ffa3-01de-44d2-9baf-1fee79026624", + "price": 0.5234, + "adid": "29681113", + "adm": "some-test-ad2", + "adomain": ["appnexus.com"], + "iurl": "http://nym1-ib.adnxs.com/cr?id=29681113", + "cid": "959", + "crid": "29681113", + "h": 970, + "w": 250, + "ext": { + "appnexus": { + "brand_id": 1, + "brand_category_id": 1, + "auction_id": 8189378542222915032, + "bid_ad_type": 0, + "bidder_id": 2, + "ranking_price": 0.000000 + } + } + }] + }, { + "seat": "improvedigital", + "bid": [{ + "id": "randomid", + "impid": "0341252e-b3b0-4dff-a0ef-1ced63369bd5", + "price": 0.510000, + "adid": "12345678", + "adm": "some-test-ad", + "cid": "987", + "crid": "12345678", + "h": 250, + "w": 300 + }] + } + ], + "bidid": "5778926625248726496", + "cur": "USD" +} diff --git a/analytics/pubstack/pubstack_module.go b/analytics/pubstack/pubstack_module.go new file mode 100644 index 00000000000..9f1a81c7232 --- /dev/null +++ b/analytics/pubstack/pubstack_module.go @@ -0,0 +1,273 @@ +package pubstack + +import ( + "fmt" + "github.com/prebid/prebid-server/analytics/pubstack/eventchannel" + "net/http" + "net/url" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/golang/glog" + "github.com/prebid/prebid-server/analytics/pubstack/helpers" + + "github.com/prebid/prebid-server/analytics" +) + +type Configuration struct { + ScopeID string `json:"scopeId"` + Endpoint string `json:"endpoint"` + Features map[string]bool `json:"features"` +} + +// routes for events +const ( + auction = "auction" + cookieSync = "cookiesync" + amp = "amp" + setUID = "setuid" + video = "video" +) + +type bufferConfig struct { + timeout time.Duration + count int64 + size int64 +} + +type PubstackModule struct { + eventChannels map[string]*eventchannel.EventChannel + httpClient *http.Client + configCh chan *Configuration + sigTermCh chan os.Signal + scope string + cfg *Configuration + buffsCfg *bufferConfig + muxConfig sync.RWMutex +} + +func NewPubstackModule(client *http.Client, scope, endpoint, configRefreshDelay string, maxEventCount int, maxByteSize, maxTime string) (analytics.PBSAnalyticsModule, error) { + glog.Infof("[pubstack] Initializing module scope=%s endpoint=%s\n", scope, endpoint) + + // parse args + + refreshDelay, err := time.ParseDuration(configRefreshDelay) + if err != nil { + return nil, fmt.Errorf("fail to parse the module args, arg=analytics.pubstack.configuration_refresh_delay, :%v", err) + } + + bufferCfg, err := newBufferConfig(maxEventCount, maxByteSize, maxTime) + if err != nil { + return nil, fmt.Errorf("fail to parse the module args, arg=analytics.pubstack.buffers, :%v", err) + } + + defaultFeatures := map[string]bool{ + auction: false, + video: false, + amp: false, + cookieSync: false, + setUID: false, + } + + defaultConfig := &Configuration{ + ScopeID: scope, + Endpoint: endpoint, + Features: defaultFeatures, + } + + pb := PubstackModule{ + scope: scope, + httpClient: client, + cfg: defaultConfig, + buffsCfg: bufferCfg, + sigTermCh: make(chan os.Signal), + configCh: make(chan *Configuration), + eventChannels: make(map[string]*eventchannel.EventChannel), + muxConfig: sync.RWMutex{}, + } + signal.Notify(pb.sigTermCh, os.Interrupt, syscall.SIGTERM) + + configUrl, err := url.Parse(pb.cfg.Endpoint + "/bootstrap?scopeId=" + pb.cfg.ScopeID) + if err != nil { + glog.Error(err) + return nil, err + } + go pb.start(configUrl, refreshDelay) + go func() { + err = pb.reloadConfig(configUrl) + if err != nil { + glog.Errorf("[pubstack] Fail to fetch remote configuration: %v", err) + } + }() + + glog.Info("[pubstack] Pubstack analytics configured and ready") + return &pb, nil +} + +func (p *PubstackModule) LogAuctionObject(ao *analytics.AuctionObject) { + p.muxConfig.RLock() + defer p.muxConfig.RUnlock() + + if !p.isFeatureEnable(auction) { + return + } + + // serialize event + payload, err := helpers.JsonifyAuctionObject(ao, p.scope) + if err != nil { + glog.Warning("[pubstack] Cannot serialize auction") + return + } + + p.eventChannels[auction].Push(payload) +} + +func (p *PubstackModule) LogVideoObject(vo *analytics.VideoObject) { + p.muxConfig.RLock() + defer p.muxConfig.RUnlock() + + if !p.isFeatureEnable(video) { + return + } + + // serialize event + payload, err := helpers.JsonifyVideoObject(vo, p.scope) + if err != nil { + glog.Warning("[pubstack] Cannot serialize video") + return + } + + p.eventChannels[video].Push(payload) +} + +func (p *PubstackModule) LogSetUIDObject(so *analytics.SetUIDObject) { + p.muxConfig.RLock() + defer p.muxConfig.RUnlock() + + if !p.isFeatureEnable(setUID) { + return + } + + // serialize event + payload, err := helpers.JsonifySetUIDObject(so, p.scope) + if err != nil { + glog.Warning("[pubstack] Cannot serialize video") + return + } + + p.eventChannels[setUID].Push(payload) +} + +func (p *PubstackModule) LogCookieSyncObject(cso *analytics.CookieSyncObject) { + p.muxConfig.RLock() + defer p.muxConfig.RUnlock() + + if !p.isFeatureEnable(cookieSync) { + return + } + + // serialize event + payload, err := helpers.JsonifyCookieSync(cso, p.scope) + if err != nil { + glog.Warning("[pubstack] Cannot serialize video") + return + } + + p.eventChannels[cookieSync].Push(payload) + +} + +func (p *PubstackModule) LogAmpObject(ao *analytics.AmpObject) { + p.muxConfig.RLock() + defer p.muxConfig.RUnlock() + + if !p.isFeatureEnable(amp) { + return + } + + // serialize event + payload, err := helpers.JsonifyAmpObject(ao, p.scope) + if err != nil { + glog.Warning("[pubstack] Cannot serialize video") + return + } + + p.eventChannels[amp].Push(payload) + +} + +func (p *PubstackModule) reloadConfig(configUrl *url.URL) error { + config, err := fetchConfig(p.httpClient, configUrl) + if err != nil { + return err + } + p.configCh <- config + return nil +} + +func (p *PubstackModule) start(configUrl *url.URL, refreshDelay time.Duration) { + + tick := time.NewTicker(refreshDelay) + + for { + select { + case <-p.sigTermCh: + p.closeAllEventChannels() + return + case config := <-p.configCh: + p.updateConfig(config) + glog.Infof("[pubstack] Updating config: %v", p.cfg) + case <-tick.C: + go func() { + err := p.reloadConfig(configUrl) + if err != nil { + glog.Errorf("[pubstack] Fail to fetch remote configuration: %v", err) + } + }() + } + } + +} + +func (p *PubstackModule) updateConfig(config *Configuration) { + p.muxConfig.Lock() + defer p.muxConfig.Unlock() + + if p.cfg.isSameAs(config) { + return + } + + p.cfg = config + p.closeAllEventChannels() + + if p.isFeatureEnable(amp) { + p.eventChannels[amp] = eventchannel.NewEventChannel(eventchannel.BuildEndpointSender(p.httpClient, p.cfg.Endpoint, amp), p.buffsCfg.size, p.buffsCfg.count, p.buffsCfg.timeout) + } + if p.isFeatureEnable(auction) { + p.eventChannels[auction] = eventchannel.NewEventChannel(eventchannel.BuildEndpointSender(p.httpClient, p.cfg.Endpoint, auction), p.buffsCfg.size, p.buffsCfg.count, p.buffsCfg.timeout) + } + if p.isFeatureEnable(cookieSync) { + p.eventChannels[cookieSync] = eventchannel.NewEventChannel(eventchannel.BuildEndpointSender(p.httpClient, p.cfg.Endpoint, cookieSync), p.buffsCfg.size, p.buffsCfg.count, p.buffsCfg.timeout) + } + if p.isFeatureEnable(video) { + p.eventChannels[video] = eventchannel.NewEventChannel(eventchannel.BuildEndpointSender(p.httpClient, p.cfg.Endpoint, video), p.buffsCfg.size, p.buffsCfg.count, p.buffsCfg.timeout) + } + if p.isFeatureEnable(setUID) { + p.eventChannels[setUID] = eventchannel.NewEventChannel(eventchannel.BuildEndpointSender(p.httpClient, p.cfg.Endpoint, setUID), p.buffsCfg.size, p.buffsCfg.count, p.buffsCfg.timeout) + } +} + +func (p *PubstackModule) closeAllEventChannels() { + for key, ch := range p.eventChannels { + ch.Close() + delete(p.eventChannels, key) + } +} + +func (p *PubstackModule) isFeatureEnable(feature string) bool { + val, ok := p.cfg.Features[feature] + return ok && val +} diff --git a/analytics/pubstack/pubstack_module_test.go b/analytics/pubstack/pubstack_module_test.go new file mode 100644 index 00000000000..8d4dfdd689f --- /dev/null +++ b/analytics/pubstack/pubstack_module_test.go @@ -0,0 +1,186 @@ +package pubstack + +import ( + "encoding/json" + "github.com/prebid/prebid-server/analytics/pubstack/eventchannel" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "os" + "testing" + "time" + + "github.com/mxmCherry/openrtb" + "github.com/prebid/prebid-server/analytics" + "github.com/stretchr/testify/assert" +) + +func loadJsonFromFile() (*analytics.AuctionObject, error) { + req, err := os.Open("mocks/mock_openrtb_request.json") + if err != nil { + return nil, err + } + defer req.Close() + + reqCtn := openrtb.BidRequest{} + reqPayload, err := ioutil.ReadAll(req) + if err != nil { + return nil, err + } + + err = json.Unmarshal(reqPayload, &reqCtn) + if err != nil { + return nil, err + } + + res, err := os.Open("mocks/mock_openrtb_response.json") + if err != nil { + return nil, err + } + defer res.Close() + + resCtn := openrtb.BidResponse{} + resPayload, err := ioutil.ReadAll(res) + if err != nil { + return nil, err + } + + err = json.Unmarshal(resPayload, &resCtn) + if err != nil { + return nil, err + } + + return &analytics.AuctionObject{ + Request: &reqCtn, + Response: &resCtn, + }, nil +} + +func TestPubstackModule(t *testing.T) { + + remoteConfig := &Configuration{} + server := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + defer req.Body.Close() + data, _ := json.Marshal(remoteConfig) + res.Write(data) + })) + client := server.Client() + + defer server.Close() + + // Loading Issues + _, err := NewPubstackModule(client, "scope", server.URL, "1z", 100, "90MB", "15m") + assert.NotNil(t, err) // should raise an error since we can't parse args // configRefreshDelay + + _, err = NewPubstackModule(client, "scope", server.URL, "1h", 100, "90z", "15m") + assert.NotNil(t, err) // should raise an error since we can't parse args // maxByte + + _, err = NewPubstackModule(client, "scope", server.URL, "1h", 100, "90MB", "15z") + assert.NotNil(t, err) // should raise an error since we can't parse args // maxTime + + // Loading OK + module, err := NewPubstackModule(client, "scope", server.URL, "10ms", 100, "90MB", "15m") + assert.Nil(t, err) + + // Default Configuration + pubstack, ok := module.(*PubstackModule) + assert.Equal(t, ok, true) //PBSAnalyticsModule is also a PubstackModule + assert.Equal(t, len(pubstack.cfg.Features), 5) + assert.Equal(t, pubstack.cfg.Features[auction], false) + assert.Equal(t, pubstack.cfg.Features[video], false) + assert.Equal(t, pubstack.cfg.Features[amp], false) + assert.Equal(t, pubstack.cfg.Features[setUID], false) + assert.Equal(t, pubstack.cfg.Features[cookieSync], false) + + assert.Equal(t, len(pubstack.eventChannels), 0) + + // Process Auction Event + counter := 0 + send := func(_ []byte) error { + counter++ + return nil + } + mockedEvent, err := loadJsonFromFile() + if err != nil { + t.Fail() + } + + pubstack.eventChannels[auction] = eventchannel.NewEventChannel(send, 2000, 1, 10*time.Second) + pubstack.eventChannels[video] = eventchannel.NewEventChannel(send, 2000, 1, 10*time.Second) + pubstack.eventChannels[amp] = eventchannel.NewEventChannel(send, 2000, 1, 10*time.Second) + pubstack.eventChannels[setUID] = eventchannel.NewEventChannel(send, 2000, 1, 10*time.Second) + pubstack.eventChannels[cookieSync] = eventchannel.NewEventChannel(send, 2000, 1, 10*time.Second) + + pubstack.LogAuctionObject(mockedEvent) + pubstack.LogAmpObject(&analytics.AmpObject{ + Status: http.StatusOK, + }) + pubstack.LogCookieSyncObject(&analytics.CookieSyncObject{ + Status: http.StatusOK, + }) + pubstack.LogVideoObject(&analytics.VideoObject{ + Status: http.StatusOK, + }) + pubstack.LogSetUIDObject(&analytics.SetUIDObject{ + Status: http.StatusOK, + }) + + pubstack.closeAllEventChannels() + time.Sleep(10 * time.Millisecond) // process channel + assert.Equal(t, counter, 0) + + // Hot-Reload config + newFeatures := make(map[string]bool) + newFeatures[auction] = true + newFeatures[video] = true + newFeatures[amp] = true + newFeatures[cookieSync] = true + newFeatures[setUID] = true + + remoteConfig = &Configuration{ + ScopeID: "new-scope", + Endpoint: "new-endpoint", + Features: newFeatures, + } + + endpoint, _ := url.Parse(server.URL) + pubstack.reloadConfig(endpoint) + + time.Sleep(2 * time.Millisecond) // process channel + assert.Equal(t, len(pubstack.cfg.Features), 5) + assert.Equal(t, pubstack.cfg.Features[auction], true) + assert.Equal(t, pubstack.cfg.Features[video], true) + assert.Equal(t, pubstack.cfg.Features[amp], true) + assert.Equal(t, pubstack.cfg.Features[setUID], true) + assert.Equal(t, pubstack.cfg.Features[cookieSync], true) + assert.Equal(t, pubstack.cfg.ScopeID, "new-scope") + assert.Equal(t, pubstack.cfg.Endpoint, "new-endpoint") + assert.Equal(t, len(pubstack.eventChannels), 5) + + counter = 0 + pubstack.eventChannels[auction] = eventchannel.NewEventChannel(send, 2000, 1, 10*time.Second) + pubstack.eventChannels[video] = eventchannel.NewEventChannel(send, 2000, 1, 10*time.Second) + pubstack.eventChannels[amp] = eventchannel.NewEventChannel(send, 2000, 1, 10*time.Second) + pubstack.eventChannels[setUID] = eventchannel.NewEventChannel(send, 2000, 1, 10*time.Second) + pubstack.eventChannels[cookieSync] = eventchannel.NewEventChannel(send, 2000, 1, 10*time.Second) + + pubstack.LogAuctionObject(mockedEvent) + pubstack.LogAmpObject(&analytics.AmpObject{ + Status: http.StatusOK, + }) + pubstack.LogCookieSyncObject(&analytics.CookieSyncObject{ + Status: http.StatusOK, + }) + pubstack.LogVideoObject(&analytics.VideoObject{ + Status: http.StatusOK, + }) + pubstack.LogSetUIDObject(&analytics.SetUIDObject{ + Status: http.StatusOK, + }) + pubstack.closeAllEventChannels() + time.Sleep(10 * time.Millisecond) + + assert.Equal(t, counter, 5) + +} diff --git a/config/config.go b/config/config.go index 8545523d238..67689d1ab1a 100755 --- a/config/config.go +++ b/config/config.go @@ -209,7 +209,8 @@ type LMT struct { } type Analytics struct { - File FileLogs `mapstructure:"file"` + File FileLogs `mapstructure:"file"` + Pubstack Pubstack `mapstructure:"pubstack"` } type CurrencyConverter struct { @@ -230,6 +231,20 @@ type FileLogs struct { Filename string `mapstructure:"filename"` } +type Pubstack struct { + Enabled bool `mapstructure:"enabled"` + ScopeId string `mapstructure:"scopeid"` + IntakeUrl string `mapstructure:"endpoint"` + Buffers PubstackBuffer `mapstructure:"buffers"` + ConfRefresh string `mapstructure:"configuration_refresh_delay"` +} + +type PubstackBuffer struct { + BufferSize string `mapstructure:"size"` + EventCount int `mapstructure:"count"` + Timeout string `mapstructure:"timeout"` +} + type HostCookie struct { Domain string `mapstructure:"domain"` Family string `mapstructure:"family"` @@ -855,6 +870,13 @@ func SetupViper(v *viper.Viper, filename string) { v.SetDefault("max_request_size", 1024*256) v.SetDefault("analytics.file.filename", "") + v.SetDefault("analytics.pubstack.endpoint", "https://s2s.pbstck.com/v1") + v.SetDefault("analytics.pubstack.scopeid", "change-me") + v.SetDefault("analytics.pubstack.enabled", false) + v.SetDefault("analytics.pubstack.configuration_refresh_delay", "2h") + v.SetDefault("analytics.pubstack.buffers.size", "2MB") + v.SetDefault("analytics.pubstack.buffers.count", 100) + v.SetDefault("analytics.pubstack.buffers.timeout", "900s") v.SetDefault("amp_timeout_adjustment_ms", 0) v.SetDefault("gdpr.host_vendor_id", 0) v.SetDefault("gdpr.usersync_if_ambiguous", false) diff --git a/go.mod b/go.mod index 00cadd31ce1..a5b5a161cf4 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/cespare/xxhash v1.0.0 // indirect github.com/chasex/glog v0.0.0-20160217080310-c62392af379c github.com/coocood/freecache v1.0.1 + github.com/docker/go-units v0.4.0 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 github.com/evanphx/json-patch v0.0.0-20180720181644-f195058310bd github.com/gofrs/uuid v3.2.0+incompatible diff --git a/go.sum b/go.sum index 5eaf37cad9f..1ddab71332a 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,8 @@ github.com/coocood/freecache v1.0.1/go.mod h1:ePwxCDzOYvARfHdr1pByNct1at3CoKnsip github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/evanphx/json-patch v0.0.0-20180720181644-f195058310bd h1:biTJQdqouE5by89AAffXG8++TY+9Fsdrg5rinbt3tHk=