From aeb7734e2c2f6a54be903c8c480134d994bae957 Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Thu, 30 May 2019 21:30:18 +0000 Subject: [PATCH 1/6] importer transfers cwrs without org --- cmd/mt-whisper-importer-reader/main.go | 25 +- cmd/mt-whisper-importer-writer/main.go | 42 ++- docs/tools.md | 10 +- mdata/cwr.go | 38 ++- mdata/cwr_gen.go | 287 +++++++++++++++--- mdata/cwr_gen_test.go | 143 ++++++++- mdata/cwr_test.go | 21 +- vendor/github.com/raintank/schema/archive.go | 1 + .../github.com/raintank/schema/archive_gen.go | 111 +++++++ .../raintank/schema/archive_gen_test.go | 3 + vendor/github.com/raintank/schema/key.go | 42 --- .../raintank/schema/key_gen_test.go | 236 ++++++++++++++ 12 files changed, 799 insertions(+), 160 deletions(-) create mode 100644 vendor/github.com/raintank/schema/archive_gen.go create mode 100644 vendor/github.com/raintank/schema/archive_gen_test.go create mode 100644 vendor/github.com/raintank/schema/key_gen_test.go diff --git a/cmd/mt-whisper-importer-reader/main.go b/cmd/mt-whisper-importer-reader/main.go index 979ba5224c..dad41302d4 100644 --- a/cmd/mt-whisper-importer-reader/main.go +++ b/cmd/mt-whisper-importer-reader/main.go @@ -30,7 +30,7 @@ import ( var ( httpEndpoint = flag.String( "http-endpoint", - "http://127.0.0.1:8080/chunks", + "http://127.0.0.1:8080/metrics/import", "The http endpoint to send the data to", ) namePrefix = flag.String( @@ -48,11 +48,6 @@ var ( false, "Defines if chunks that have not completed their chunk span should be written", ) - orgId = flag.Int( - "orgid", - 1, - "Organization ID the data belongs to ", - ) insecureSSL = flag.Bool( "insecure-ssl", false, @@ -305,14 +300,9 @@ func getMetric(w *whisper.Whisper, file, name string) (*mdata.ArchiveRequest, er Time: 0, Mtype: "gauge", Tags: []string{}, - OrgId: *orgId, }, } res.MetricData.SetId() - mkey, err := schema.MKeyFromString(res.MetricData.Id) - if err != nil { - panic(err) - } _, selectedSchema := schemas.Match(res.MetricData.Name, int(w.Header.Archives[0].SecondsPerPoint)) conversion := newConversion(w.Header.Archives, points, method) @@ -323,18 +313,15 @@ func getMetric(w *whisper.Whisper, file, name string) (*mdata.ArchiveRequest, er continue } - var amkey schema.AMKey - if retIdx == 0 { - amkey = schema.AMKey{MKey: mkey} - } else { - amkey = schema.GetAMKey(mkey, m, retention.ChunkSpan) + var archive schema.Archive + if retIdx > 0 { + archive = schema.NewArchive(m, retention.ChunkSpan) } encodedChunks := encodedChunksFromPoints(p, uint32(retention.SecondsPerPoint), retention.ChunkSpan) for _, chunk := range encodedChunks { - res.ChunkWriteRequests = append(res.ChunkWriteRequests, mdata.NewChunkWriteRequest( - nil, - amkey, + res.ChunkWriteRequests = append(res.ChunkWriteRequests, mdata.NewChunkWriteRequestWithoutOrg( + archive, uint32(retention.MaxRetention()), chunk.Series.T0, chunk.Encode(retention.ChunkSpan), diff --git a/cmd/mt-whisper-importer-writer/main.go b/cmd/mt-whisper-importer-writer/main.go index 180185e096..e6fd796dc4 100644 --- a/cmd/mt-whisper-importer-writer/main.go +++ b/cmd/mt-whisper-importer-writer/main.go @@ -1,10 +1,12 @@ package main import ( + "errors" "flag" "fmt" "net/http" "os" + "strconv" "strings" "time" @@ -27,7 +29,8 @@ import ( var ( confFile = flag.String("config", "/etc/metrictank/metrictank.ini", "configuration file path") exitOnError = flag.Bool("exit-on-error", false, "Exit with a message when there's an error") - httpEndpoint = flag.String("http-endpoint", "127.0.0.1:8080", "The http endpoint to listen on") + listenAddress = flag.String("listen-address", "127.0.0.1", "The address to listen on") + listenPort = flag.Int("listen-port", 8080, "The port to listen on") ttlsStr = flag.String("ttls", "35d", "list of ttl strings used by MT separated by ','") partitionScheme = flag.String("partition-scheme", "bySeries", "method used for partitioning metrics. This should match the settings of tsdb-gw. (byOrg|bySeries)") uriPath = flag.String("uri-path", "/chunks", "the URI on which we expect chunks to get posted") @@ -137,12 +140,13 @@ func main() { index.Init() + httpEndpoint := fmt.Sprintf("%s:%d", *listenAddress, *listenPort) server := &Server{ partitioner: p, index: index, store: store, HTTPServer: &http.Server{ - Addr: *httpEndpoint, + Addr: httpEndpoint, ReadTimeout: 10 * time.Minute, }, } @@ -150,8 +154,8 @@ func main() { http.HandleFunc(*uriPath, server.chunksHandler) http.HandleFunc("/healthz", server.healthzHandler) - log.Infof("Listening on %q", *httpEndpoint) - err = http.ListenAndServe(*httpEndpoint, nil) + log.Infof("Listening on %q", httpEndpoint) + err = http.ListenAndServe(httpEndpoint, nil) if err != nil { panic(fmt.Sprintf("Error creating listener: %q", err)) } @@ -173,8 +177,15 @@ func (s *Server) healthzHandler(w http.ResponseWriter, req *http.Request) { } func (s *Server) chunksHandler(w http.ResponseWriter, req *http.Request) { + orgId, err := getOrgId(req) + if err != nil { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte(err.Error())) + return + } + data := mdata.ArchiveRequest{} - err := data.UnmarshalCompressed(req.Body) + err = data.UnmarshalCompressed(req.Body) if err != nil { throwError(w, fmt.Sprintf("Error decoding cwr stream: %q", err)) return @@ -189,13 +200,15 @@ func (s *Server) chunksHandler(w http.ResponseWriter, req *http.Request) { "Received %d cwrs for metric %s. The first has Key: %s, T0: %d, TTL: %d. The last has Key: %s, T0: %d, TTL: %d", len(data.ChunkWriteRequests), data.MetricData.Name, - data.ChunkWriteRequests[0].Key.String(), + data.ChunkWriteRequests[0].Archive.String(), data.ChunkWriteRequests[0].T0, data.ChunkWriteRequests[0].TTL, - data.ChunkWriteRequests[len(data.ChunkWriteRequests)-1].Key.String(), + data.ChunkWriteRequests[len(data.ChunkWriteRequests)-1].Archive.String(), data.ChunkWriteRequests[len(data.ChunkWriteRequests)-1].T0, data.ChunkWriteRequests[len(data.ChunkWriteRequests)-1].TTL) + data.MetricData.OrgId = orgId + data.MetricData.SetId() partition, err := s.partitioner.Partition(&data.MetricData, int32(*numPartitions)) if err != nil { throwError(w, fmt.Sprintf("Error partitioning: %q", err)) @@ -210,7 +223,18 @@ func (s *Server) chunksHandler(w http.ResponseWriter, req *http.Request) { s.index.AddOrUpdate(mkey, &data.MetricData, partition) for _, cwr := range data.ChunkWriteRequests { - cwr := cwr // important because we pass by reference and this var will get overwritten in the next loop - s.store.Add(&cwr) + cwrWithOrg := cwr.GetChunkWriteRequest(nil, mkey) + s.store.Add(&cwrWithOrg) + } +} + +func getOrgId(req *http.Request) (int, error) { + if orgIdStr := req.Header.Get("X-Org-Id"); len(orgIdStr) > 0 { + if orgId, err := strconv.Atoi(orgIdStr); err == nil { + return orgId, nil + } else { + return 0, fmt.Errorf("Invalid value in X-Org-Id header (%s): %s", orgIdStr, err) + } } + return 0, errors.New("Missing X-Org-Id header") } diff --git a/docs/tools.md b/docs/tools.md index f3d83fcd42..18791d08ca 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -673,7 +673,7 @@ Usage of ./mt-whisper-importer-reader: -http-auth string The credentials used to authenticate in the format "user:password" -http-endpoint string - The http endpoint to send the data to (default "http://127.0.0.1:8080/chunks") + The http endpoint to send the data to (default "http://127.0.0.1:8080/metrics/import") -import-after uint Only import after the specified timestamp -import-up-to uint @@ -684,8 +684,6 @@ Usage of ./mt-whisper-importer-reader: A regex pattern to be applied to all metric names, only matching ones will be imported -name-prefix string Prefix to prepend before every metric name, should include the '.' if necessary - -orgid int - Organization ID the data belongs to (default 1) -position-file string file to store position and load position from -threads int @@ -707,8 +705,10 @@ Usage of ./mt-whisper-importer-writer: configuration file path (default "/etc/metrictank/metrictank.ini") -exit-on-error Exit with a message when there's an error - -http-endpoint string - The http endpoint to listen on (default "127.0.0.1:8080") + -listen-address string + The address to listen on (default "127.0.0.1") + -listen-port int + The port to listen on (default 8080) -log-level string log level. panic|fatal|error|warning|info|debug (default "info") -num-partitions int diff --git a/mdata/cwr.go b/mdata/cwr.go index 7fd9f151ff..2406ee00be 100644 --- a/mdata/cwr.go +++ b/mdata/cwr.go @@ -13,30 +13,48 @@ import ( "github.com/raintank/schema" ) +//go:generate msgp +//msgp:ignore ChunkWriteRequest + type ChunkSaveCallback func() // ChunkWriteRequest is a request to write a chunk into a store -//go:generate msgp type ChunkWriteRequest struct { - Callback ChunkSaveCallback `msg:"-"` - Key schema.AMKey `msg:"key,extension"` + ChunkWriteRequestPayload + Callback ChunkSaveCallback + Key schema.AMKey +} + +func NewChunkWriteRequest(callback ChunkSaveCallback, key schema.AMKey, ttl, t0 uint32, data []byte, ts time.Time) ChunkWriteRequest { + return ChunkWriteRequest{ChunkWriteRequestPayload{ttl, t0, data, ts}, callback, key} +} + +// ChunkWriteRequestWithoutOrg is used by the importer utility to send cwrs over the network +type ChunkWriteRequestWithoutOrg struct { + ChunkWriteRequestPayload + Archive schema.Archive +} + +func NewChunkWriteRequestWithoutOrg(archive schema.Archive, ttl, t0 uint32, data []byte, ts time.Time) ChunkWriteRequestWithoutOrg { + return ChunkWriteRequestWithoutOrg{ChunkWriteRequestPayload{ttl, t0, data, ts}, archive} +} + +func (c *ChunkWriteRequestWithoutOrg) GetChunkWriteRequest(callback ChunkSaveCallback, key schema.MKey) ChunkWriteRequest { + return ChunkWriteRequest{c.ChunkWriteRequestPayload, callback, schema.AMKey{MKey: key, Archive: c.Archive}} +} + +type ChunkWriteRequestPayload struct { TTL uint32 T0 uint32 Data []byte Timestamp time.Time } -// NewChunkWriteRequest creates a new ChunkWriteRequest -func NewChunkWriteRequest(callback ChunkSaveCallback, key schema.AMKey, ttl, t0 uint32, data []byte, ts time.Time) ChunkWriteRequest { - return ChunkWriteRequest{callback, key, ttl, t0, data, ts} -} - // ArchiveRequest is a complete representation of a Metric together with some // chunk write requests containing data that shall be written into this metric -//go:generate msgp type ArchiveRequest struct { MetricData schema.MetricData - ChunkWriteRequests []ChunkWriteRequest + ChunkWriteRequests []ChunkWriteRequestWithoutOrg } func (a *ArchiveRequest) MarshalCompressed() (*bytes.Buffer, error) { diff --git a/mdata/cwr_gen.go b/mdata/cwr_gen.go index daf38b369e..0d8b8f68b2 100644 --- a/mdata/cwr_gen.go +++ b/mdata/cwr_gen.go @@ -40,14 +40,43 @@ func (z *ArchiveRequest) DecodeMsg(dc *msgp.Reader) (err error) { if cap(z.ChunkWriteRequests) >= int(zb0002) { z.ChunkWriteRequests = (z.ChunkWriteRequests)[:zb0002] } else { - z.ChunkWriteRequests = make([]ChunkWriteRequest, zb0002) + z.ChunkWriteRequests = make([]ChunkWriteRequestWithoutOrg, zb0002) } for za0001 := range z.ChunkWriteRequests { - err = z.ChunkWriteRequests[za0001].DecodeMsg(dc) + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() if err != nil { err = msgp.WrapError(err, "ChunkWriteRequests", za0001) return } + for zb0003 > 0 { + zb0003-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + return + } + switch msgp.UnsafeString(field) { + case "ChunkWriteRequestPayload": + err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") + return + } + case "Archive": + err = z.ChunkWriteRequests[za0001].Archive.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + return + } + } + } } default: err = dc.Skip() @@ -84,9 +113,25 @@ func (z *ArchiveRequest) EncodeMsg(en *msgp.Writer) (err error) { return } for za0001 := range z.ChunkWriteRequests { - err = z.ChunkWriteRequests[za0001].EncodeMsg(en) + // map header, size 2 + // write "ChunkWriteRequestPayload" + err = en.Append(0x82, 0xb8, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64) if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + return + } + err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") + return + } + // write "Archive" + err = en.Append(0xa7, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65) + if err != nil { + return + } + err = z.ChunkWriteRequests[za0001].Archive.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") return } } @@ -108,9 +153,19 @@ func (z *ArchiveRequest) MarshalMsg(b []byte) (o []byte, err error) { o = append(o, 0xb2, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73) o = msgp.AppendArrayHeader(o, uint32(len(z.ChunkWriteRequests))) for za0001 := range z.ChunkWriteRequests { - o, err = z.ChunkWriteRequests[za0001].MarshalMsg(o) + // map header, size 2 + // string "ChunkWriteRequestPayload" + o = append(o, 0x82, 0xb8, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64) + o, err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") + return + } + // string "Archive" + o = append(o, 0xa7, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65) + o, err = z.ChunkWriteRequests[za0001].Archive.MarshalMsg(o) if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") return } } @@ -151,14 +206,43 @@ func (z *ArchiveRequest) UnmarshalMsg(bts []byte) (o []byte, err error) { if cap(z.ChunkWriteRequests) >= int(zb0002) { z.ChunkWriteRequests = (z.ChunkWriteRequests)[:zb0002] } else { - z.ChunkWriteRequests = make([]ChunkWriteRequest, zb0002) + z.ChunkWriteRequests = make([]ChunkWriteRequestWithoutOrg, zb0002) } for za0001 := range z.ChunkWriteRequests { - bts, err = z.ChunkWriteRequests[za0001].UnmarshalMsg(bts) + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { err = msgp.WrapError(err, "ChunkWriteRequests", za0001) return } + for zb0003 > 0 { + zb0003-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + return + } + switch msgp.UnsafeString(field) { + case "ChunkWriteRequestPayload": + bts, err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") + return + } + case "Archive": + bts, err = z.ChunkWriteRequests[za0001].Archive.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + return + } + } + } } default: bts, err = msgp.Skip(bts) @@ -176,13 +260,13 @@ func (z *ArchiveRequest) UnmarshalMsg(bts []byte) (o []byte, err error) { func (z *ArchiveRequest) Msgsize() (s int) { s = 1 + 11 + z.MetricData.Msgsize() + 19 + msgp.ArrayHeaderSize for za0001 := range z.ChunkWriteRequests { - s += z.ChunkWriteRequests[za0001].Msgsize() + s += 1 + 25 + z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.Msgsize() + 8 + z.ChunkWriteRequests[za0001].Archive.Msgsize() } return } // DecodeMsg implements msgp.Decodable -func (z *ChunkWriteRequest) DecodeMsg(dc *msgp.Reader) (err error) { +func (z *ChunkWriteRequestPayload) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte _ = field var zb0001 uint32 @@ -199,12 +283,6 @@ func (z *ChunkWriteRequest) DecodeMsg(dc *msgp.Reader) (err error) { return } switch msgp.UnsafeString(field) { - case "key": - err = dc.ReadExtension(&z.Key) - if err != nil { - err = msgp.WrapError(err, "Key") - return - } case "TTL": z.TTL, err = dc.ReadUint32() if err != nil { @@ -241,20 +319,10 @@ func (z *ChunkWriteRequest) DecodeMsg(dc *msgp.Reader) (err error) { } // EncodeMsg implements msgp.Encodable -func (z *ChunkWriteRequest) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 5 - // write "key" - err = en.Append(0x85, 0xa3, 0x6b, 0x65, 0x79) - if err != nil { - return - } - err = en.WriteExtension(&z.Key) - if err != nil { - err = msgp.WrapError(err, "Key") - return - } +func (z *ChunkWriteRequestPayload) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 // write "TTL" - err = en.Append(0xa3, 0x54, 0x54, 0x4c) + err = en.Append(0x84, 0xa3, 0x54, 0x54, 0x4c) if err != nil { return } @@ -297,18 +365,11 @@ func (z *ChunkWriteRequest) EncodeMsg(en *msgp.Writer) (err error) { } // MarshalMsg implements msgp.Marshaler -func (z *ChunkWriteRequest) MarshalMsg(b []byte) (o []byte, err error) { +func (z *ChunkWriteRequestPayload) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 5 - // string "key" - o = append(o, 0x85, 0xa3, 0x6b, 0x65, 0x79) - o, err = msgp.AppendExtension(o, &z.Key) - if err != nil { - err = msgp.WrapError(err, "Key") - return - } + // map header, size 4 // string "TTL" - o = append(o, 0xa3, 0x54, 0x54, 0x4c) + o = append(o, 0x84, 0xa3, 0x54, 0x54, 0x4c) o = msgp.AppendUint32(o, z.TTL) // string "T0" o = append(o, 0xa2, 0x54, 0x30) @@ -323,7 +384,7 @@ func (z *ChunkWriteRequest) MarshalMsg(b []byte) (o []byte, err error) { } // UnmarshalMsg implements msgp.Unmarshaler -func (z *ChunkWriteRequest) UnmarshalMsg(bts []byte) (o []byte, err error) { +func (z *ChunkWriteRequestPayload) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field var zb0001 uint32 @@ -340,12 +401,6 @@ func (z *ChunkWriteRequest) UnmarshalMsg(bts []byte) (o []byte, err error) { return } switch msgp.UnsafeString(field) { - case "key": - bts, err = msgp.ReadExtensionBytes(bts, &z.Key) - if err != nil { - err = msgp.WrapError(err, "Key") - return - } case "TTL": z.TTL, bts, err = msgp.ReadUint32Bytes(bts) if err != nil { @@ -383,7 +438,143 @@ func (z *ChunkWriteRequest) UnmarshalMsg(bts []byte) (o []byte, err error) { } // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *ChunkWriteRequest) Msgsize() (s int) { - s = 1 + 4 + msgp.ExtensionPrefixSize + z.Key.Len() + 4 + msgp.Uint32Size + 3 + msgp.Uint32Size + 5 + msgp.BytesPrefixSize + len(z.Data) + 10 + msgp.TimeSize +func (z *ChunkWriteRequestPayload) Msgsize() (s int) { + s = 1 + 4 + msgp.Uint32Size + 3 + msgp.Uint32Size + 5 + msgp.BytesPrefixSize + len(z.Data) + 10 + msgp.TimeSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ChunkWriteRequestWithoutOrg) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ChunkWriteRequestPayload": + err = z.ChunkWriteRequestPayload.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequestPayload") + return + } + case "Archive": + err = z.Archive.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Archive") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ChunkWriteRequestWithoutOrg) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "ChunkWriteRequestPayload" + err = en.Append(0x82, 0xb8, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64) + if err != nil { + return + } + err = z.ChunkWriteRequestPayload.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequestPayload") + return + } + // write "Archive" + err = en.Append(0xa7, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65) + if err != nil { + return + } + err = z.Archive.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Archive") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ChunkWriteRequestWithoutOrg) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "ChunkWriteRequestPayload" + o = append(o, 0x82, 0xb8, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64) + o, err = z.ChunkWriteRequestPayload.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequestPayload") + return + } + // string "Archive" + o = append(o, 0xa7, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65) + o, err = z.Archive.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Archive") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ChunkWriteRequestWithoutOrg) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ChunkWriteRequestPayload": + bts, err = z.ChunkWriteRequestPayload.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequestPayload") + return + } + case "Archive": + bts, err = z.Archive.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Archive") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ChunkWriteRequestWithoutOrg) Msgsize() (s int) { + s = 1 + 25 + z.ChunkWriteRequestPayload.Msgsize() + 8 + z.Archive.Msgsize() return } diff --git a/mdata/cwr_gen_test.go b/mdata/cwr_gen_test.go index 1c443f5a13..8e9220d6bf 100644 --- a/mdata/cwr_gen_test.go +++ b/mdata/cwr_gen_test.go @@ -122,8 +122,8 @@ func BenchmarkDecodeArchiveRequest(b *testing.B) { } } -func TestMarshalUnmarshalChunkWriteRequest(t *testing.T) { - v := ChunkWriteRequest{} +func TestMarshalUnmarshalChunkWriteRequestPayload(t *testing.T) { + v := ChunkWriteRequestPayload{} bts, err := v.MarshalMsg(nil) if err != nil { t.Fatal(err) @@ -145,8 +145,8 @@ func TestMarshalUnmarshalChunkWriteRequest(t *testing.T) { } } -func BenchmarkMarshalMsgChunkWriteRequest(b *testing.B) { - v := ChunkWriteRequest{} +func BenchmarkMarshalMsgChunkWriteRequestPayload(b *testing.B) { + v := ChunkWriteRequestPayload{} b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -154,8 +154,8 @@ func BenchmarkMarshalMsgChunkWriteRequest(b *testing.B) { } } -func BenchmarkAppendMsgChunkWriteRequest(b *testing.B) { - v := ChunkWriteRequest{} +func BenchmarkAppendMsgChunkWriteRequestPayload(b *testing.B) { + v := ChunkWriteRequestPayload{} bts := make([]byte, 0, v.Msgsize()) bts, _ = v.MarshalMsg(bts[0:0]) b.SetBytes(int64(len(bts))) @@ -166,8 +166,8 @@ func BenchmarkAppendMsgChunkWriteRequest(b *testing.B) { } } -func BenchmarkUnmarshalChunkWriteRequest(b *testing.B) { - v := ChunkWriteRequest{} +func BenchmarkUnmarshalChunkWriteRequestPayload(b *testing.B) { + v := ChunkWriteRequestPayload{} bts, _ := v.MarshalMsg(nil) b.ReportAllocs() b.SetBytes(int64(len(bts))) @@ -180,8 +180,8 @@ func BenchmarkUnmarshalChunkWriteRequest(b *testing.B) { } } -func TestEncodeDecodeChunkWriteRequest(t *testing.T) { - v := ChunkWriteRequest{} +func TestEncodeDecodeChunkWriteRequestPayload(t *testing.T) { + v := ChunkWriteRequestPayload{} var buf bytes.Buffer msgp.Encode(&buf, &v) @@ -190,7 +190,7 @@ func TestEncodeDecodeChunkWriteRequest(t *testing.T) { t.Logf("WARNING: Msgsize() for %v is inaccurate", v) } - vn := ChunkWriteRequest{} + vn := ChunkWriteRequestPayload{} err := msgp.Decode(&buf, &vn) if err != nil { t.Error(err) @@ -204,8 +204,8 @@ func TestEncodeDecodeChunkWriteRequest(t *testing.T) { } } -func BenchmarkEncodeChunkWriteRequest(b *testing.B) { - v := ChunkWriteRequest{} +func BenchmarkEncodeChunkWriteRequestPayload(b *testing.B) { + v := ChunkWriteRequestPayload{} var buf bytes.Buffer msgp.Encode(&buf, &v) b.SetBytes(int64(buf.Len())) @@ -218,8 +218,121 @@ func BenchmarkEncodeChunkWriteRequest(b *testing.B) { en.Flush() } -func BenchmarkDecodeChunkWriteRequest(b *testing.B) { - v := ChunkWriteRequest{} +func BenchmarkDecodeChunkWriteRequestPayload(b *testing.B) { + v := ChunkWriteRequestPayload{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalChunkWriteRequestWithoutOrg(t *testing.T) { + v := ChunkWriteRequestWithoutOrg{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgChunkWriteRequestWithoutOrg(b *testing.B) { + v := ChunkWriteRequestWithoutOrg{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgChunkWriteRequestWithoutOrg(b *testing.B) { + v := ChunkWriteRequestWithoutOrg{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalChunkWriteRequestWithoutOrg(b *testing.B) { + v := ChunkWriteRequestWithoutOrg{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeChunkWriteRequestWithoutOrg(t *testing.T) { + v := ChunkWriteRequestWithoutOrg{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) + } + + vn := ChunkWriteRequestWithoutOrg{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeChunkWriteRequestWithoutOrg(b *testing.B) { + v := ChunkWriteRequestWithoutOrg{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeChunkWriteRequestWithoutOrg(b *testing.B) { + v := ChunkWriteRequestWithoutOrg{} var buf bytes.Buffer msgp.Encode(&buf, &v) b.SetBytes(int64(buf.Len())) diff --git a/mdata/cwr_test.go b/mdata/cwr_test.go index 3923edcb31..d0562ba8f4 100644 --- a/mdata/cwr_test.go +++ b/mdata/cwr_test.go @@ -54,11 +54,7 @@ func TestArchiveRequestEncodingDecoding(t *testing.T) { testData := getTestData(6000, 60, 100, 0) chunks := chunksFromPoints(testData, 1800) - id := "98.12345678901234567890123456789012" - key, err := schema.AMKeyFromString(id + "_sum_3600") - if err != nil { - t.Fatalf("Expected no error when getting AMKey from string %q", err) - } + archive := schema.NewArchive(schema.Sum, 3600) originalRequest := ArchiveRequest{ MetricData: schema.MetricData{ Id: "98.12345678901234567890123456789012", @@ -74,13 +70,14 @@ func TestArchiveRequestEncodingDecoding(t *testing.T) { } for i := 0; i < len(chunks); i++ { - originalRequest.ChunkWriteRequests = append(originalRequest.ChunkWriteRequests, ChunkWriteRequest{ - Callback: nil, - Key: key, - TTL: 1, - T0: chunks[i].Series.T0, - Data: chunks[i].Encode(1800), - Timestamp: time.Unix(123, 456), + originalRequest.ChunkWriteRequests = append(originalRequest.ChunkWriteRequests, ChunkWriteRequestWithoutOrg{ + ChunkWriteRequestPayload: ChunkWriteRequestPayload{ + TTL: 1, + T0: chunks[i].Series.T0, + Data: chunks[i].Encode(1800), + Timestamp: time.Unix(123, 456), + }, + Archive: archive, }) } diff --git a/vendor/github.com/raintank/schema/archive.go b/vendor/github.com/raintank/schema/archive.go index c42cdb6c69..0b222760e0 100644 --- a/vendor/github.com/raintank/schema/archive.go +++ b/vendor/github.com/raintank/schema/archive.go @@ -14,6 +14,7 @@ import ( // any non-zero value represents a certain // aggregation method (lower 8 bits) and // aggregation span (higher 8 bits) +//go:generate msgp type Archive uint16 // important: caller must make sure to call IsSpanValid first diff --git a/vendor/github.com/raintank/schema/archive_gen.go b/vendor/github.com/raintank/schema/archive_gen.go new file mode 100644 index 0000000000..cbf2f3003f --- /dev/null +++ b/vendor/github.com/raintank/schema/archive_gen.go @@ -0,0 +1,111 @@ +package schema + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *Archive) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint16 + zb0001, err = dc.ReadUint16() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = Archive(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z Archive) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint16(uint16(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z Archive) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint16(o, uint16(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Archive) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint16 + zb0001, bts, err = msgp.ReadUint16Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = Archive(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z Archive) Msgsize() (s int) { + s = msgp.Uint16Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *Method) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint8 + zb0001, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = Method(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z Method) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint8(uint8(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z Method) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Method) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = Method(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z Method) Msgsize() (s int) { + s = msgp.Uint8Size + return +} diff --git a/vendor/github.com/raintank/schema/archive_gen_test.go b/vendor/github.com/raintank/schema/archive_gen_test.go new file mode 100644 index 0000000000..d4387192ff --- /dev/null +++ b/vendor/github.com/raintank/schema/archive_gen_test.go @@ -0,0 +1,3 @@ +package schema + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. diff --git a/vendor/github.com/raintank/schema/key.go b/vendor/github.com/raintank/schema/key.go index e9f50b36e3..a6b7f8011c 100644 --- a/vendor/github.com/raintank/schema/key.go +++ b/vendor/github.com/raintank/schema/key.go @@ -6,23 +6,12 @@ import ( "fmt" "strconv" "strings" - - "github.com/tinylib/msgp/msgp" ) //go:generate msgp //msgp:ignore AMKey // don't ignore Key, MKey because it's used for MetricDefinition -// A random number that does not conflict with any of the msgp internal -// types, and which is also not the first one after the msgp internal types -// to avoid accidental conflicts with other custom types -const msgpExtensionId = 97 - -func init() { - msgp.RegisterExtension(msgpExtensionId, func() msgp.Extension { return new(AMKey) }) -} - var ErrStringTooShort = errors.New("string too short") var ErrInvalidFormat = errors.New("invalid format") @@ -77,37 +66,6 @@ type AMKey struct { Archive Archive } -// ExtensionType is part of the msgp.Extension interface -func (a *AMKey) ExtensionType() int8 { - return msgpExtensionId -} - -// Len is part of the msgp.Extension interface -// It returns the length of the encoded byte slice representing this AMKey -// Length is variable because the AMKey may include the span & method, -// also the org id can have a varying number of digits -func (a *AMKey) Len() int { - return len(a.String()) -} - -// MarshalBinaryTo is part of the msgp.Extension interface -// It takes a buffer which must have the length returned by the Len() function -// and writes the encoded version of this AMKey into it -func (a *AMKey) MarshalBinaryTo(buf []byte) error { - copy(buf, a.String()) - - return nil -} - -// UnmarshalBinary is part of the msgp.Extension interface -// It takes a buffer containing an encoded AMKey and decodes it into this -// AMKey instance -func (a *AMKey) UnmarshalBinary(buf []byte) error { - var err error - *a, err = AMKeyFromString(string(buf)) - return err -} - func (a AMKey) String() string { if a.Archive == 0 { return a.MKey.String() diff --git a/vendor/github.com/raintank/schema/key_gen_test.go b/vendor/github.com/raintank/schema/key_gen_test.go new file mode 100644 index 0000000000..592f6d3fc5 --- /dev/null +++ b/vendor/github.com/raintank/schema/key_gen_test.go @@ -0,0 +1,236 @@ +package schema + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalKey(t *testing.T) { + v := Key{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgKey(b *testing.B) { + v := Key{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgKey(b *testing.B) { + v := Key{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalKey(b *testing.B) { + v := Key{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeKey(t *testing.T) { + v := Key{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) + } + + vn := Key{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeKey(b *testing.B) { + v := Key{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeKey(b *testing.B) { + v := Key{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalMKey(t *testing.T) { + v := MKey{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgMKey(b *testing.B) { + v := MKey{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgMKey(b *testing.B) { + v := MKey{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalMKey(b *testing.B) { + v := MKey{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeMKey(t *testing.T) { + v := MKey{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) + } + + vn := MKey{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeMKey(b *testing.B) { + v := MKey{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeMKey(b *testing.B) { + v := MKey{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} From 47bd3cb72d35462e2044522e31dcba6ffe245946 Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Fri, 7 Jun 2019 22:04:03 +0000 Subject: [PATCH 2/6] change default uri path & listen address syntax these two changes were in reaction to PR comments --- cmd/mt-whisper-importer-writer/main.go | 12 +++++------- docs/tools.md | 16 +++++++--------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/cmd/mt-whisper-importer-writer/main.go b/cmd/mt-whisper-importer-writer/main.go index e6fd796dc4..fe73b98b40 100644 --- a/cmd/mt-whisper-importer-writer/main.go +++ b/cmd/mt-whisper-importer-writer/main.go @@ -29,11 +29,10 @@ import ( var ( confFile = flag.String("config", "/etc/metrictank/metrictank.ini", "configuration file path") exitOnError = flag.Bool("exit-on-error", false, "Exit with a message when there's an error") - listenAddress = flag.String("listen-address", "127.0.0.1", "The address to listen on") - listenPort = flag.Int("listen-port", 8080, "The port to listen on") + httpEndpoint = flag.String("http-endpoint", "0.0.0.0:8080", "The http endpoint to listen on") ttlsStr = flag.String("ttls", "35d", "list of ttl strings used by MT separated by ','") partitionScheme = flag.String("partition-scheme", "bySeries", "method used for partitioning metrics. This should match the settings of tsdb-gw. (byOrg|bySeries)") - uriPath = flag.String("uri-path", "/chunks", "the URI on which we expect chunks to get posted") + uriPath = flag.String("uri-path", "/metrics/import", "the URI on which we expect chunks to get posted") numPartitions = flag.Int("num-partitions", 1, "Number of Partitions") logLevel = flag.String("log-level", "info", "log level. panic|fatal|error|warning|info|debug") @@ -140,13 +139,12 @@ func main() { index.Init() - httpEndpoint := fmt.Sprintf("%s:%d", *listenAddress, *listenPort) server := &Server{ partitioner: p, index: index, store: store, HTTPServer: &http.Server{ - Addr: httpEndpoint, + Addr: *httpEndpoint, ReadTimeout: 10 * time.Minute, }, } @@ -154,8 +152,8 @@ func main() { http.HandleFunc(*uriPath, server.chunksHandler) http.HandleFunc("/healthz", server.healthzHandler) - log.Infof("Listening on %q", httpEndpoint) - err = http.ListenAndServe(httpEndpoint, nil) + log.Infof("Listening on %q", *httpEndpoint) + err = http.ListenAndServe(*httpEndpoint, nil) if err != nil { panic(fmt.Sprintf("Error creating listener: %q", err)) } diff --git a/docs/tools.md b/docs/tools.md index 18791d08ca..925135b9e1 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -674,10 +674,10 @@ Usage of ./mt-whisper-importer-reader: The credentials used to authenticate in the format "user:password" -http-endpoint string The http endpoint to send the data to (default "http://127.0.0.1:8080/metrics/import") - -import-after uint - Only import after the specified timestamp - -import-up-to uint - Only import up to the specified timestamp (default 4294967295) + -import-from uint + Only import starting from the specified timestamp + -import-until uint + Only import up to, but not including, the specified timestamp (default 4294967295) -insecure-ssl Disables ssl certificate verification -name-filter string @@ -705,10 +705,8 @@ Usage of ./mt-whisper-importer-writer: configuration file path (default "/etc/metrictank/metrictank.ini") -exit-on-error Exit with a message when there's an error - -listen-address string - The address to listen on (default "127.0.0.1") - -listen-port int - The port to listen on (default 8080) + -http-endpoint string + The http endpoint to listen on (default "0.0.0.0:8080") -log-level string log level. panic|fatal|error|warning|info|debug (default "info") -num-partitions int @@ -718,6 +716,6 @@ Usage of ./mt-whisper-importer-writer: -ttls string list of ttl strings used by MT separated by ',' (default "35d") -uri-path string - the URI on which we expect chunks to get posted (default "/chunks") + the URI on which we expect chunks to get posted (default "/metrics/import") ``` From 301e767a57870b3e91bf69a8445f41e0df96d0c8 Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Mon, 10 Jun 2019 14:15:46 +0000 Subject: [PATCH 3/6] move cwr into its own "importer" package --- cmd/mt-whisper-importer-reader/main.go | 8 +- cmd/mt-whisper-importer-writer/main.go | 3 +- mdata/cwr.go | 73 ----- mdata/cwr_gen.go | 395 ------------------------ mdata/cwr_gen_test.go | 226 -------------- mdata/cwr_test.go | 136 --------- mdata/importer/cwr.go | 83 +++++ mdata/importer/cwr_gen.go | 402 +++++++++++++++++++++++++ mdata/importer/cwr_gen_test.go | 236 +++++++++++++++ 9 files changed, 727 insertions(+), 835 deletions(-) delete mode 100644 mdata/cwr_test.go create mode 100644 mdata/importer/cwr.go create mode 100644 mdata/importer/cwr_gen.go create mode 100644 mdata/importer/cwr_gen_test.go diff --git a/cmd/mt-whisper-importer-reader/main.go b/cmd/mt-whisper-importer-reader/main.go index dad41302d4..217cf70512 100644 --- a/cmd/mt-whisper-importer-reader/main.go +++ b/cmd/mt-whisper-importer-reader/main.go @@ -20,8 +20,8 @@ import ( "github.com/grafana/metrictank/conf" "github.com/grafana/metrictank/logger" - "github.com/grafana/metrictank/mdata" "github.com/grafana/metrictank/mdata/chunk" + "github.com/grafana/metrictank/mdata/importer" "github.com/kisielk/whisper-go/whisper" "github.com/raintank/schema" log "github.com/sirupsen/logrus" @@ -272,7 +272,7 @@ func convertWhisperMethod(whisperMethod whisper.AggregationMethod) (schema.Metho } } -func getMetric(w *whisper.Whisper, file, name string) (*mdata.ArchiveRequest, error) { +func getMetric(w *whisper.Whisper, file, name string) (*importer.ArchiveRequest, error) { if len(w.Header.Archives) == 0 { return nil, fmt.Errorf("Whisper file contains no archives: %q", file) } @@ -291,7 +291,7 @@ func getMetric(w *whisper.Whisper, file, name string) (*mdata.ArchiveRequest, er points[i] = p } - res := &mdata.ArchiveRequest{ + res := &importer.ArchiveRequest{ MetricData: schema.MetricData{ Name: name, Value: 0, @@ -320,7 +320,7 @@ func getMetric(w *whisper.Whisper, file, name string) (*mdata.ArchiveRequest, er encodedChunks := encodedChunksFromPoints(p, uint32(retention.SecondsPerPoint), retention.ChunkSpan) for _, chunk := range encodedChunks { - res.ChunkWriteRequests = append(res.ChunkWriteRequests, mdata.NewChunkWriteRequestWithoutOrg( + res.ChunkWriteRequests = append(res.ChunkWriteRequests, importer.NewChunkWriteRequest( archive, uint32(retention.MaxRetention()), chunk.Series.T0, diff --git a/cmd/mt-whisper-importer-writer/main.go b/cmd/mt-whisper-importer-writer/main.go index fe73b98b40..fd87ba8bb3 100644 --- a/cmd/mt-whisper-importer-writer/main.go +++ b/cmd/mt-whisper-importer-writer/main.go @@ -22,6 +22,7 @@ import ( "github.com/grafana/metrictank/idx/cassandra" "github.com/grafana/metrictank/logger" "github.com/grafana/metrictank/mdata" + "github.com/grafana/metrictank/mdata/importer" bigTableStore "github.com/grafana/metrictank/store/bigtable" cassandraStore "github.com/grafana/metrictank/store/cassandra" ) @@ -182,7 +183,7 @@ func (s *Server) chunksHandler(w http.ResponseWriter, req *http.Request) { return } - data := mdata.ArchiveRequest{} + data := importer.ArchiveRequest{} err = data.UnmarshalCompressed(req.Body) if err != nil { throwError(w, fmt.Sprintf("Error decoding cwr stream: %q", err)) diff --git a/mdata/cwr.go b/mdata/cwr.go index 2406ee00be..f99181031d 100644 --- a/mdata/cwr.go +++ b/mdata/cwr.go @@ -1,15 +1,8 @@ package mdata import ( - "bufio" - "bytes" - "compress/gzip" - "fmt" - "io" "time" - "github.com/tinylib/msgp/msgp" - "github.com/raintank/schema" ) @@ -29,75 +22,9 @@ func NewChunkWriteRequest(callback ChunkSaveCallback, key schema.AMKey, ttl, t0 return ChunkWriteRequest{ChunkWriteRequestPayload{ttl, t0, data, ts}, callback, key} } -// ChunkWriteRequestWithoutOrg is used by the importer utility to send cwrs over the network -type ChunkWriteRequestWithoutOrg struct { - ChunkWriteRequestPayload - Archive schema.Archive -} - -func NewChunkWriteRequestWithoutOrg(archive schema.Archive, ttl, t0 uint32, data []byte, ts time.Time) ChunkWriteRequestWithoutOrg { - return ChunkWriteRequestWithoutOrg{ChunkWriteRequestPayload{ttl, t0, data, ts}, archive} -} - -func (c *ChunkWriteRequestWithoutOrg) GetChunkWriteRequest(callback ChunkSaveCallback, key schema.MKey) ChunkWriteRequest { - return ChunkWriteRequest{c.ChunkWriteRequestPayload, callback, schema.AMKey{MKey: key, Archive: c.Archive}} -} - type ChunkWriteRequestPayload struct { TTL uint32 T0 uint32 Data []byte Timestamp time.Time } - -// ArchiveRequest is a complete representation of a Metric together with some -// chunk write requests containing data that shall be written into this metric -type ArchiveRequest struct { - MetricData schema.MetricData - ChunkWriteRequests []ChunkWriteRequestWithoutOrg -} - -func (a *ArchiveRequest) MarshalCompressed() (*bytes.Buffer, error) { - var buf bytes.Buffer - - buf.WriteByte(byte(uint8(1))) - - g := gzip.NewWriter(&buf) - err := msgp.Encode(g, a) - if err != nil { - return &buf, fmt.Errorf("ERROR: Encoding MGSP data: %q", err) - } - - err = g.Close() - if err != nil { - return &buf, fmt.Errorf("ERROR: Compressing MSGP data: %q", err) - } - - return &buf, nil -} - -func (a *ArchiveRequest) UnmarshalCompressed(b io.Reader) error { - versionBuf := make([]byte, 1) - readBytes, err := b.Read(versionBuf) - if err != nil || readBytes != 1 { - return fmt.Errorf("ERROR: Failed to read one byte: %s", err) - } - - version := uint8(versionBuf[0]) - if version != 1 { - return fmt.Errorf("ERROR: Only version 1 is supported, received version %d", version) - } - - gzipReader, err := gzip.NewReader(b) - if err != nil { - return fmt.Errorf("ERROR: Creating Gzip reader: %q", err) - } - - err = msgp.Decode(bufio.NewReader(gzipReader), a) - if err != nil { - return fmt.Errorf("ERROR: Unmarshaling Raw: %q", err) - } - gzipReader.Close() - - return nil -} diff --git a/mdata/cwr_gen.go b/mdata/cwr_gen.go index 0d8b8f68b2..3723642258 100644 --- a/mdata/cwr_gen.go +++ b/mdata/cwr_gen.go @@ -6,265 +6,6 @@ import ( "github.com/tinylib/msgp/msgp" ) -// DecodeMsg implements msgp.Decodable -func (z *ArchiveRequest) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zb0001 uint32 - zb0001, err = dc.ReadMapHeader() - if err != nil { - err = msgp.WrapError(err) - return - } - for zb0001 > 0 { - zb0001-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - err = msgp.WrapError(err) - return - } - switch msgp.UnsafeString(field) { - case "MetricData": - err = z.MetricData.DecodeMsg(dc) - if err != nil { - err = msgp.WrapError(err, "MetricData") - return - } - case "ChunkWriteRequests": - var zb0002 uint32 - zb0002, err = dc.ReadArrayHeader() - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests") - return - } - if cap(z.ChunkWriteRequests) >= int(zb0002) { - z.ChunkWriteRequests = (z.ChunkWriteRequests)[:zb0002] - } else { - z.ChunkWriteRequests = make([]ChunkWriteRequestWithoutOrg, zb0002) - } - for za0001 := range z.ChunkWriteRequests { - var zb0003 uint32 - zb0003, err = dc.ReadMapHeader() - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001) - return - } - for zb0003 > 0 { - zb0003-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001) - return - } - switch msgp.UnsafeString(field) { - case "ChunkWriteRequestPayload": - err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.DecodeMsg(dc) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") - return - } - case "Archive": - err = z.ChunkWriteRequests[za0001].Archive.DecodeMsg(dc) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") - return - } - default: - err = dc.Skip() - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001) - return - } - } - } - } - default: - err = dc.Skip() - if err != nil { - err = msgp.WrapError(err) - return - } - } - } - return -} - -// EncodeMsg implements msgp.Encodable -func (z *ArchiveRequest) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 2 - // write "MetricData" - err = en.Append(0x82, 0xaa, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x44, 0x61, 0x74, 0x61) - if err != nil { - return - } - err = z.MetricData.EncodeMsg(en) - if err != nil { - err = msgp.WrapError(err, "MetricData") - return - } - // write "ChunkWriteRequests" - err = en.Append(0xb2, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73) - if err != nil { - return - } - err = en.WriteArrayHeader(uint32(len(z.ChunkWriteRequests))) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests") - return - } - for za0001 := range z.ChunkWriteRequests { - // map header, size 2 - // write "ChunkWriteRequestPayload" - err = en.Append(0x82, 0xb8, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64) - if err != nil { - return - } - err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.EncodeMsg(en) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") - return - } - // write "Archive" - err = en.Append(0xa7, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65) - if err != nil { - return - } - err = z.ChunkWriteRequests[za0001].Archive.EncodeMsg(en) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") - return - } - } - return -} - -// MarshalMsg implements msgp.Marshaler -func (z *ArchiveRequest) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 2 - // string "MetricData" - o = append(o, 0x82, 0xaa, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x44, 0x61, 0x74, 0x61) - o, err = z.MetricData.MarshalMsg(o) - if err != nil { - err = msgp.WrapError(err, "MetricData") - return - } - // string "ChunkWriteRequests" - o = append(o, 0xb2, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73) - o = msgp.AppendArrayHeader(o, uint32(len(z.ChunkWriteRequests))) - for za0001 := range z.ChunkWriteRequests { - // map header, size 2 - // string "ChunkWriteRequestPayload" - o = append(o, 0x82, 0xb8, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64) - o, err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.MarshalMsg(o) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") - return - } - // string "Archive" - o = append(o, 0xa7, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65) - o, err = z.ChunkWriteRequests[za0001].Archive.MarshalMsg(o) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") - return - } - } - return -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *ArchiveRequest) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zb0001 uint32 - zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - for zb0001 > 0 { - zb0001-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - switch msgp.UnsafeString(field) { - case "MetricData": - bts, err = z.MetricData.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "MetricData") - return - } - case "ChunkWriteRequests": - var zb0002 uint32 - zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests") - return - } - if cap(z.ChunkWriteRequests) >= int(zb0002) { - z.ChunkWriteRequests = (z.ChunkWriteRequests)[:zb0002] - } else { - z.ChunkWriteRequests = make([]ChunkWriteRequestWithoutOrg, zb0002) - } - for za0001 := range z.ChunkWriteRequests { - var zb0003 uint32 - zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001) - return - } - for zb0003 > 0 { - zb0003-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001) - return - } - switch msgp.UnsafeString(field) { - case "ChunkWriteRequestPayload": - bts, err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") - return - } - case "Archive": - bts, err = z.ChunkWriteRequests[za0001].Archive.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001) - return - } - } - } - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - } - } - o = bts - return -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *ArchiveRequest) Msgsize() (s int) { - s = 1 + 11 + z.MetricData.Msgsize() + 19 + msgp.ArrayHeaderSize - for za0001 := range z.ChunkWriteRequests { - s += 1 + 25 + z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.Msgsize() + 8 + z.ChunkWriteRequests[za0001].Archive.Msgsize() - } - return -} - // DecodeMsg implements msgp.Decodable func (z *ChunkWriteRequestPayload) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte @@ -442,139 +183,3 @@ func (z *ChunkWriteRequestPayload) Msgsize() (s int) { s = 1 + 4 + msgp.Uint32Size + 3 + msgp.Uint32Size + 5 + msgp.BytesPrefixSize + len(z.Data) + 10 + msgp.TimeSize return } - -// DecodeMsg implements msgp.Decodable -func (z *ChunkWriteRequestWithoutOrg) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zb0001 uint32 - zb0001, err = dc.ReadMapHeader() - if err != nil { - err = msgp.WrapError(err) - return - } - for zb0001 > 0 { - zb0001-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - err = msgp.WrapError(err) - return - } - switch msgp.UnsafeString(field) { - case "ChunkWriteRequestPayload": - err = z.ChunkWriteRequestPayload.DecodeMsg(dc) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequestPayload") - return - } - case "Archive": - err = z.Archive.DecodeMsg(dc) - if err != nil { - err = msgp.WrapError(err, "Archive") - return - } - default: - err = dc.Skip() - if err != nil { - err = msgp.WrapError(err) - return - } - } - } - return -} - -// EncodeMsg implements msgp.Encodable -func (z *ChunkWriteRequestWithoutOrg) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 2 - // write "ChunkWriteRequestPayload" - err = en.Append(0x82, 0xb8, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64) - if err != nil { - return - } - err = z.ChunkWriteRequestPayload.EncodeMsg(en) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequestPayload") - return - } - // write "Archive" - err = en.Append(0xa7, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65) - if err != nil { - return - } - err = z.Archive.EncodeMsg(en) - if err != nil { - err = msgp.WrapError(err, "Archive") - return - } - return -} - -// MarshalMsg implements msgp.Marshaler -func (z *ChunkWriteRequestWithoutOrg) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 2 - // string "ChunkWriteRequestPayload" - o = append(o, 0x82, 0xb8, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64) - o, err = z.ChunkWriteRequestPayload.MarshalMsg(o) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequestPayload") - return - } - // string "Archive" - o = append(o, 0xa7, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65) - o, err = z.Archive.MarshalMsg(o) - if err != nil { - err = msgp.WrapError(err, "Archive") - return - } - return -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *ChunkWriteRequestWithoutOrg) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zb0001 uint32 - zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - for zb0001 > 0 { - zb0001-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - switch msgp.UnsafeString(field) { - case "ChunkWriteRequestPayload": - bts, err = z.ChunkWriteRequestPayload.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequestPayload") - return - } - case "Archive": - bts, err = z.Archive.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Archive") - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - } - } - o = bts - return -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *ChunkWriteRequestWithoutOrg) Msgsize() (s int) { - s = 1 + 25 + z.ChunkWriteRequestPayload.Msgsize() + 8 + z.Archive.Msgsize() - return -} diff --git a/mdata/cwr_gen_test.go b/mdata/cwr_gen_test.go index 8e9220d6bf..d7695beb84 100644 --- a/mdata/cwr_gen_test.go +++ b/mdata/cwr_gen_test.go @@ -9,119 +9,6 @@ import ( "github.com/tinylib/msgp/msgp" ) -func TestMarshalUnmarshalArchiveRequest(t *testing.T) { - v := ArchiveRequest{} - bts, err := v.MarshalMsg(nil) - if err != nil { - t.Fatal(err) - } - left, err := v.UnmarshalMsg(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) - } - - left, err = msgp.Skip(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after Skip(): %q", len(left), left) - } -} - -func BenchmarkMarshalMsgArchiveRequest(b *testing.B) { - v := ArchiveRequest{} - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - v.MarshalMsg(nil) - } -} - -func BenchmarkAppendMsgArchiveRequest(b *testing.B) { - v := ArchiveRequest{} - bts := make([]byte, 0, v.Msgsize()) - bts, _ = v.MarshalMsg(bts[0:0]) - b.SetBytes(int64(len(bts))) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - bts, _ = v.MarshalMsg(bts[0:0]) - } -} - -func BenchmarkUnmarshalArchiveRequest(b *testing.B) { - v := ArchiveRequest{} - bts, _ := v.MarshalMsg(nil) - b.ReportAllocs() - b.SetBytes(int64(len(bts))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := v.UnmarshalMsg(bts) - if err != nil { - b.Fatal(err) - } - } -} - -func TestEncodeDecodeArchiveRequest(t *testing.T) { - v := ArchiveRequest{} - var buf bytes.Buffer - msgp.Encode(&buf, &v) - - m := v.Msgsize() - if buf.Len() > m { - t.Logf("WARNING: Msgsize() for %v is inaccurate", v) - } - - vn := ArchiveRequest{} - err := msgp.Decode(&buf, &vn) - if err != nil { - t.Error(err) - } - - buf.Reset() - msgp.Encode(&buf, &v) - err = msgp.NewReader(&buf).Skip() - if err != nil { - t.Error(err) - } -} - -func BenchmarkEncodeArchiveRequest(b *testing.B) { - v := ArchiveRequest{} - var buf bytes.Buffer - msgp.Encode(&buf, &v) - b.SetBytes(int64(buf.Len())) - en := msgp.NewWriter(msgp.Nowhere) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - v.EncodeMsg(en) - } - en.Flush() -} - -func BenchmarkDecodeArchiveRequest(b *testing.B) { - v := ArchiveRequest{} - var buf bytes.Buffer - msgp.Encode(&buf, &v) - b.SetBytes(int64(buf.Len())) - rd := msgp.NewEndlessReader(buf.Bytes(), b) - dc := msgp.NewReader(rd) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - err := v.DecodeMsg(dc) - if err != nil { - b.Fatal(err) - } - } -} - func TestMarshalUnmarshalChunkWriteRequestPayload(t *testing.T) { v := ChunkWriteRequestPayload{} bts, err := v.MarshalMsg(nil) @@ -234,116 +121,3 @@ func BenchmarkDecodeChunkWriteRequestPayload(b *testing.B) { } } } - -func TestMarshalUnmarshalChunkWriteRequestWithoutOrg(t *testing.T) { - v := ChunkWriteRequestWithoutOrg{} - bts, err := v.MarshalMsg(nil) - if err != nil { - t.Fatal(err) - } - left, err := v.UnmarshalMsg(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) - } - - left, err = msgp.Skip(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after Skip(): %q", len(left), left) - } -} - -func BenchmarkMarshalMsgChunkWriteRequestWithoutOrg(b *testing.B) { - v := ChunkWriteRequestWithoutOrg{} - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - v.MarshalMsg(nil) - } -} - -func BenchmarkAppendMsgChunkWriteRequestWithoutOrg(b *testing.B) { - v := ChunkWriteRequestWithoutOrg{} - bts := make([]byte, 0, v.Msgsize()) - bts, _ = v.MarshalMsg(bts[0:0]) - b.SetBytes(int64(len(bts))) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - bts, _ = v.MarshalMsg(bts[0:0]) - } -} - -func BenchmarkUnmarshalChunkWriteRequestWithoutOrg(b *testing.B) { - v := ChunkWriteRequestWithoutOrg{} - bts, _ := v.MarshalMsg(nil) - b.ReportAllocs() - b.SetBytes(int64(len(bts))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := v.UnmarshalMsg(bts) - if err != nil { - b.Fatal(err) - } - } -} - -func TestEncodeDecodeChunkWriteRequestWithoutOrg(t *testing.T) { - v := ChunkWriteRequestWithoutOrg{} - var buf bytes.Buffer - msgp.Encode(&buf, &v) - - m := v.Msgsize() - if buf.Len() > m { - t.Logf("WARNING: Msgsize() for %v is inaccurate", v) - } - - vn := ChunkWriteRequestWithoutOrg{} - err := msgp.Decode(&buf, &vn) - if err != nil { - t.Error(err) - } - - buf.Reset() - msgp.Encode(&buf, &v) - err = msgp.NewReader(&buf).Skip() - if err != nil { - t.Error(err) - } -} - -func BenchmarkEncodeChunkWriteRequestWithoutOrg(b *testing.B) { - v := ChunkWriteRequestWithoutOrg{} - var buf bytes.Buffer - msgp.Encode(&buf, &v) - b.SetBytes(int64(buf.Len())) - en := msgp.NewWriter(msgp.Nowhere) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - v.EncodeMsg(en) - } - en.Flush() -} - -func BenchmarkDecodeChunkWriteRequestWithoutOrg(b *testing.B) { - v := ChunkWriteRequestWithoutOrg{} - var buf bytes.Buffer - msgp.Encode(&buf, &v) - b.SetBytes(int64(buf.Len())) - rd := msgp.NewEndlessReader(buf.Bytes(), b) - dc := msgp.NewReader(rd) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - err := v.DecodeMsg(dc) - if err != nil { - b.Fatal(err) - } - } -} diff --git a/mdata/cwr_test.go b/mdata/cwr_test.go deleted file mode 100644 index d0562ba8f4..0000000000 --- a/mdata/cwr_test.go +++ /dev/null @@ -1,136 +0,0 @@ -package mdata - -import ( - "bytes" - "reflect" - "testing" - "time" - - "github.com/grafana/metrictank/mdata/chunk" - "github.com/raintank/schema" -) - -func getTestData(ts, interval, length uint32, val float64) []point { - testData := make([]point, 0, length) - - for i := uint32(0); i < length; i++ { - testData = append(testData, point{ts: ts, val: val}) - val++ - ts += interval - } - - return testData -} - -func chunksFromPoints(points []point, chunkSpan uint32) []*chunk.Chunk { - var t0, prevT0 uint32 - var c *chunk.Chunk - var encodedChunks []*chunk.Chunk - - for _, point := range points { - t0 = point.ts - (point.ts % chunkSpan) - - if prevT0 == 0 { - c = chunk.New(t0) - prevT0 = t0 - } else if prevT0 != t0 { - c.Finish() - encodedChunks = append(encodedChunks, c) - - c = chunk.New(t0) - prevT0 = t0 - } - - c.Push(point.ts, point.val) - } - - c.Finish() - encodedChunks = append(encodedChunks, c) - - return encodedChunks -} - -func TestArchiveRequestEncodingDecoding(t *testing.T) { - testData := getTestData(6000, 60, 100, 0) - chunks := chunksFromPoints(testData, 1800) - - archive := schema.NewArchive(schema.Sum, 3600) - originalRequest := ArchiveRequest{ - MetricData: schema.MetricData{ - Id: "98.12345678901234567890123456789012", - OrgId: 1, - Name: "testMetricData", - Interval: 1, - Value: 321, - Unit: "test", - Time: 123456, - Mtype: "test", - Tags: []string{"some=tag"}, - }, - } - - for i := 0; i < len(chunks); i++ { - originalRequest.ChunkWriteRequests = append(originalRequest.ChunkWriteRequests, ChunkWriteRequestWithoutOrg{ - ChunkWriteRequestPayload: ChunkWriteRequestPayload{ - TTL: 1, - T0: chunks[i].Series.T0, - Data: chunks[i].Encode(1800), - Timestamp: time.Unix(123, 456), - }, - Archive: archive, - }) - } - - encoded, err := originalRequest.MarshalCompressed() - if err != nil { - t.Fatalf("Failed when encoding the request: %s", err) - } - - got := ArchiveRequest{} - err = got.UnmarshalCompressed(encoded) - if err != nil { - t.Fatalf("Expected no error when decoding request, got %q", err) - } - - if !reflect.DeepEqual(originalRequest, got) { - t.Fatalf("Decoded request is different than the encoded one.\nexp: %+v\ngot: %+v\n", originalRequest, got) - } - - var decodedData []point - for _, c := range got.ChunkWriteRequests { - itgen, err := chunk.NewIterGen(uint32(c.T0), 0, c.Data) - if err != nil { - t.Fatalf("Expected no error when getting IterGen %q", err) - } - - iter, err := itgen.Get() - if err != nil { - t.Fatalf("Expected no error when getting Iterator %q", err) - } - - for iter.Next() { - ts, val := iter.Values() - decodedData = append(decodedData, point{ts: ts, val: val}) - } - } - - if !reflect.DeepEqual(testData, decodedData) { - t.Fatalf("Decoded request is different than the encoded one.\nexp: %+v\ngot: %+v\n", originalRequest, got) - } -} - -func TestArchiveRequestEncodingWithUnknownVersion(t *testing.T) { - originalRequest := ArchiveRequest{} - encoded, err := originalRequest.MarshalCompressed() - if err != nil { - t.Fatalf("Expected no error when encoding request, got %q", err) - } - - encodedBytes := encoded.Bytes() - encodedBytes[0] = byte(uint8(2)) - decodedRequest := ArchiveRequest{} - err = decodedRequest.UnmarshalCompressed(bytes.NewReader(encodedBytes)) - if err == nil { - t.Fatalf("Expected an error when decoding, but got nil") - } -} diff --git a/mdata/importer/cwr.go b/mdata/importer/cwr.go new file mode 100644 index 0000000000..36f148fd41 --- /dev/null +++ b/mdata/importer/cwr.go @@ -0,0 +1,83 @@ +package importer + +import ( + "bufio" + "bytes" + "compress/gzip" + "fmt" + "io" + "time" + + "github.com/grafana/metrictank/mdata" + "github.com/raintank/schema" + "github.com/tinylib/msgp/msgp" +) + +//go:generate msgp + +// ChunkWriteRequest is used by the importer utility to send cwrs over the network +// It does not contain the org id because this is assumed to be defined by the auth mechanism +type ChunkWriteRequest struct { + mdata.ChunkWriteRequestPayload + Archive schema.Archive +} + +func NewChunkWriteRequest(archive schema.Archive, ttl, t0 uint32, data []byte, ts time.Time) ChunkWriteRequest { + return ChunkWriteRequest{mdata.ChunkWriteRequestPayload{ttl, t0, data, ts}, archive} +} + +func (c *ChunkWriteRequest) GetChunkWriteRequest(callback mdata.ChunkSaveCallback, key schema.MKey) mdata.ChunkWriteRequest { + return mdata.ChunkWriteRequest{c.ChunkWriteRequestPayload, callback, schema.AMKey{MKey: key, Archive: c.Archive}} +} + +// ArchiveRequest is a complete representation of a Metric together with some +// chunk write requests containing data that shall be written into this metric +type ArchiveRequest struct { + MetricData schema.MetricData + ChunkWriteRequests []ChunkWriteRequest +} + +func (a *ArchiveRequest) MarshalCompressed() (*bytes.Buffer, error) { + var buf bytes.Buffer + + buf.WriteByte(byte(uint8(1))) + + g := gzip.NewWriter(&buf) + err := msgp.Encode(g, a) + if err != nil { + return &buf, fmt.Errorf("ERROR: Encoding MGSP data: %q", err) + } + + err = g.Close() + if err != nil { + return &buf, fmt.Errorf("ERROR: Compressing MSGP data: %q", err) + } + + return &buf, nil +} + +func (a *ArchiveRequest) UnmarshalCompressed(b io.Reader) error { + versionBuf := make([]byte, 1) + readBytes, err := b.Read(versionBuf) + if err != nil || readBytes != 1 { + return fmt.Errorf("ERROR: Failed to read one byte: %s", err) + } + + version := uint8(versionBuf[0]) + if version != 1 { + return fmt.Errorf("ERROR: Only version 1 is supported, received version %d", version) + } + + gzipReader, err := gzip.NewReader(b) + if err != nil { + return fmt.Errorf("ERROR: Creating Gzip reader: %q", err) + } + + err = msgp.Decode(bufio.NewReader(gzipReader), a) + if err != nil { + return fmt.Errorf("ERROR: Unmarshaling Raw: %q", err) + } + gzipReader.Close() + + return nil +} diff --git a/mdata/importer/cwr_gen.go b/mdata/importer/cwr_gen.go new file mode 100644 index 0000000000..2406a32d52 --- /dev/null +++ b/mdata/importer/cwr_gen.go @@ -0,0 +1,402 @@ +package importer + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *ArchiveRequest) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "MetricData": + err = z.MetricData.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "MetricData") + return + } + case "ChunkWriteRequests": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests") + return + } + if cap(z.ChunkWriteRequests) >= int(zb0002) { + z.ChunkWriteRequests = (z.ChunkWriteRequests)[:zb0002] + } else { + z.ChunkWriteRequests = make([]ChunkWriteRequest, zb0002) + } + for za0001 := range z.ChunkWriteRequests { + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + return + } + for zb0003 > 0 { + zb0003-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + return + } + switch msgp.UnsafeString(field) { + case "ChunkWriteRequestPayload": + err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") + return + } + case "Archive": + err = z.ChunkWriteRequests[za0001].Archive.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + return + } + } + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ArchiveRequest) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "MetricData" + err = en.Append(0x82, 0xaa, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x44, 0x61, 0x74, 0x61) + if err != nil { + return + } + err = z.MetricData.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "MetricData") + return + } + // write "ChunkWriteRequests" + err = en.Append(0xb2, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.ChunkWriteRequests))) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests") + return + } + for za0001 := range z.ChunkWriteRequests { + // map header, size 2 + // write "ChunkWriteRequestPayload" + err = en.Append(0x82, 0xb8, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64) + if err != nil { + return + } + err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") + return + } + // write "Archive" + err = en.Append(0xa7, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65) + if err != nil { + return + } + err = z.ChunkWriteRequests[za0001].Archive.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ArchiveRequest) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "MetricData" + o = append(o, 0x82, 0xaa, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x44, 0x61, 0x74, 0x61) + o, err = z.MetricData.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "MetricData") + return + } + // string "ChunkWriteRequests" + o = append(o, 0xb2, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.ChunkWriteRequests))) + for za0001 := range z.ChunkWriteRequests { + // map header, size 2 + // string "ChunkWriteRequestPayload" + o = append(o, 0x82, 0xb8, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64) + o, err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") + return + } + // string "Archive" + o = append(o, 0xa7, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65) + o, err = z.ChunkWriteRequests[za0001].Archive.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ArchiveRequest) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "MetricData": + bts, err = z.MetricData.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "MetricData") + return + } + case "ChunkWriteRequests": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests") + return + } + if cap(z.ChunkWriteRequests) >= int(zb0002) { + z.ChunkWriteRequests = (z.ChunkWriteRequests)[:zb0002] + } else { + z.ChunkWriteRequests = make([]ChunkWriteRequest, zb0002) + } + for za0001 := range z.ChunkWriteRequests { + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + return + } + for zb0003 > 0 { + zb0003-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + return + } + switch msgp.UnsafeString(field) { + case "ChunkWriteRequestPayload": + bts, err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") + return + } + case "Archive": + bts, err = z.ChunkWriteRequests[za0001].Archive.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + return + } + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ArchiveRequest) Msgsize() (s int) { + s = 1 + 11 + z.MetricData.Msgsize() + 19 + msgp.ArrayHeaderSize + for za0001 := range z.ChunkWriteRequests { + s += 1 + 25 + z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.Msgsize() + 8 + z.ChunkWriteRequests[za0001].Archive.Msgsize() + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ChunkWriteRequest) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ChunkWriteRequestPayload": + err = z.ChunkWriteRequestPayload.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequestPayload") + return + } + case "Archive": + err = z.Archive.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Archive") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ChunkWriteRequest) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "ChunkWriteRequestPayload" + err = en.Append(0x82, 0xb8, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64) + if err != nil { + return + } + err = z.ChunkWriteRequestPayload.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequestPayload") + return + } + // write "Archive" + err = en.Append(0xa7, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65) + if err != nil { + return + } + err = z.Archive.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Archive") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ChunkWriteRequest) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "ChunkWriteRequestPayload" + o = append(o, 0x82, 0xb8, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64) + o, err = z.ChunkWriteRequestPayload.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequestPayload") + return + } + // string "Archive" + o = append(o, 0xa7, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65) + o, err = z.Archive.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Archive") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ChunkWriteRequest) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ChunkWriteRequestPayload": + bts, err = z.ChunkWriteRequestPayload.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequestPayload") + return + } + case "Archive": + bts, err = z.Archive.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Archive") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ChunkWriteRequest) Msgsize() (s int) { + s = 1 + 25 + z.ChunkWriteRequestPayload.Msgsize() + 8 + z.Archive.Msgsize() + return +} diff --git a/mdata/importer/cwr_gen_test.go b/mdata/importer/cwr_gen_test.go new file mode 100644 index 0000000000..168ed0dc91 --- /dev/null +++ b/mdata/importer/cwr_gen_test.go @@ -0,0 +1,236 @@ +package importer + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalArchiveRequest(t *testing.T) { + v := ArchiveRequest{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgArchiveRequest(b *testing.B) { + v := ArchiveRequest{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgArchiveRequest(b *testing.B) { + v := ArchiveRequest{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalArchiveRequest(b *testing.B) { + v := ArchiveRequest{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeArchiveRequest(t *testing.T) { + v := ArchiveRequest{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) + } + + vn := ArchiveRequest{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeArchiveRequest(b *testing.B) { + v := ArchiveRequest{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeArchiveRequest(b *testing.B) { + v := ArchiveRequest{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalChunkWriteRequest(t *testing.T) { + v := ChunkWriteRequest{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgChunkWriteRequest(b *testing.B) { + v := ChunkWriteRequest{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgChunkWriteRequest(b *testing.B) { + v := ChunkWriteRequest{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalChunkWriteRequest(b *testing.B) { + v := ChunkWriteRequest{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeChunkWriteRequest(t *testing.T) { + v := ChunkWriteRequest{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) + } + + vn := ChunkWriteRequest{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeChunkWriteRequest(b *testing.B) { + v := ChunkWriteRequest{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeChunkWriteRequest(b *testing.B) { + v := ChunkWriteRequest{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} From 24def99da33836082a63fd9ade44657e08c71848 Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Mon, 10 Jun 2019 14:33:36 +0000 Subject: [PATCH 4/6] move conversion and chunk encoding to importer pkg also move all the related tests keep incResolution() and decResolution() as stand-alone functions because that way they're way more testable than if they were an integrated method of the conversion type --- cmd/mt-whisper-importer-reader/main.go | 75 +----- mdata/importer/chunk_encoder.go | 49 ++++ mdata/importer/chunk_encoder_test.go | 93 ++++++++ .../importer}/conversion.go | 93 +++++--- .../importer}/conversion_test.go | 220 ++++++------------ 5 files changed, 272 insertions(+), 258 deletions(-) create mode 100644 mdata/importer/chunk_encoder.go create mode 100644 mdata/importer/chunk_encoder_test.go rename {cmd/mt-whisper-importer-reader => mdata/importer}/conversion.go (85%) rename {cmd/mt-whisper-importer-reader => mdata/importer}/conversion_test.go (84%) diff --git a/cmd/mt-whisper-importer-reader/main.go b/cmd/mt-whisper-importer-reader/main.go index 217cf70512..d8cb5d08f2 100644 --- a/cmd/mt-whisper-importer-reader/main.go +++ b/cmd/mt-whisper-importer-reader/main.go @@ -12,7 +12,6 @@ import ( "os" "path/filepath" "regexp" - "sort" "strings" "sync" "sync/atomic" @@ -20,7 +19,6 @@ import ( "github.com/grafana/metrictank/conf" "github.com/grafana/metrictank/logger" - "github.com/grafana/metrictank/mdata/chunk" "github.com/grafana/metrictank/mdata/importer" "github.com/kisielk/whisper-go/whisper" "github.com/raintank/schema" @@ -73,15 +71,15 @@ var ( "", "A regex pattern to be applied to all metric names, only matching ones will be imported", ) - importUpTo = flag.Uint( - "import-up-to", + importUntil = flag.Uint( + "import-until", math.MaxUint32, - "Only import up to the specified timestamp", + "Only import up to, but not including, the specified timestamp", ) - importAfter = flag.Uint( - "import-after", + importFrom = flag.Uint( + "import-from", 0, - "Only import after the specified timestamp", + "Only import starting from the specified timestamp", ) positionFile = flag.String( "position-file", @@ -241,20 +239,6 @@ func getMetricName(file string) string { return *namePrefix + strings.Replace(strings.TrimSuffix(file, ".wsp"), "/", ".", -1) } -// pointSorter sorts points by timestamp -type pointSorter []whisper.Point - -func (a pointSorter) Len() int { return len(a) } -func (a pointSorter) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp } - -// the whisper archives are organized like a ringbuffer. since we need to -// insert the points into the chunks in order we first need to sort them -func sortPoints(points pointSorter) pointSorter { - sort.Sort(points) - return points -} - func convertWhisperMethod(whisperMethod whisper.AggregationMethod) (schema.Method, error) { switch whisperMethod { case whisper.AggregationAverage: @@ -305,9 +289,9 @@ func getMetric(w *whisper.Whisper, file, name string) (*importer.ArchiveRequest, res.MetricData.SetId() _, selectedSchema := schemas.Match(res.MetricData.Name, int(w.Header.Archives[0].SecondsPerPoint)) - conversion := newConversion(w.Header.Archives, points, method) + conversion := importer.NewConversion(w.Header.Archives, points, method, uint32(*importFrom), uint32(*importUntil)) for retIdx, retention := range selectedSchema.Retentions { - convertedPoints := conversion.getPoints(retIdx, uint32(retention.SecondsPerPoint), uint32(retention.NumberOfPoints)) + convertedPoints := conversion.GetPoints(retIdx, uint32(retention.SecondsPerPoint), uint32(retention.NumberOfPoints)) for m, p := range convertedPoints { if len(p) == 0 { continue @@ -318,7 +302,7 @@ func getMetric(w *whisper.Whisper, file, name string) (*importer.ArchiveRequest, archive = schema.NewArchive(m, retention.ChunkSpan) } - encodedChunks := encodedChunksFromPoints(p, uint32(retention.SecondsPerPoint), retention.ChunkSpan) + encodedChunks := importer.EncodeChunksFromPoints(p, uint32(retention.SecondsPerPoint), retention.ChunkSpan, *writeUnfinishedChunks) for _, chunk := range encodedChunks { res.ChunkWriteRequests = append(res.ChunkWriteRequests, importer.NewChunkWriteRequest( archive, @@ -338,47 +322,6 @@ func getMetric(w *whisper.Whisper, file, name string) (*importer.ArchiveRequest, return res, nil } -func encodedChunksFromPoints(points []whisper.Point, intervalIn, chunkSpan uint32) []*chunk.Chunk { - var point whisper.Point - var t0, prevT0 uint32 - var c *chunk.Chunk - var encodedChunks []*chunk.Chunk - - for _, point = range points { - // this shouldn't happen, but if it would we better catch it here because Metrictank wouldn't handle it well: - // https://github.com/grafana/metrictank/blob/f1868cccfb92fc82cd853914af958f6d187c5f74/mdata/aggmetric.go#L378 - if point.Timestamp == 0 { - continue - } - - t0 = point.Timestamp - (point.Timestamp % chunkSpan) - if prevT0 == 0 { - c = chunk.New(t0) - prevT0 = t0 - } else if prevT0 != t0 { - c.Finish() - encodedChunks = append(encodedChunks, c) - - c = chunk.New(t0) - prevT0 = t0 - } - - err := c.Push(point.Timestamp, point.Value) - if err != nil { - panic(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %q", t0, err)) - } - } - - // if the last written point was also the last one of the current chunk, - // or if writeUnfinishedChunks is on, we close the chunk and push it - if point.Timestamp == t0+chunkSpan-intervalIn || *writeUnfinishedChunks { - c.Finish() - encodedChunks = append(encodedChunks, c) - } - - return encodedChunks -} - // scan a directory and feed the list of whisper files relative to base into the given channel func getFileListIntoChan(pos *posTracker, fileChan chan string) { filepath.Walk( diff --git a/mdata/importer/chunk_encoder.go b/mdata/importer/chunk_encoder.go new file mode 100644 index 0000000000..94f42b9928 --- /dev/null +++ b/mdata/importer/chunk_encoder.go @@ -0,0 +1,49 @@ +package importer + +import ( + "fmt" + + "github.com/grafana/metrictank/mdata/chunk" + "github.com/kisielk/whisper-go/whisper" +) + +func EncodeChunksFromPoints(points []whisper.Point, intervalIn, chunkSpan uint32, writeUnfinishedChunks bool) []*chunk.Chunk { + var point whisper.Point + var t0, prevT0 uint32 + var c *chunk.Chunk + var encodedChunks []*chunk.Chunk + + for _, point = range points { + // this shouldn't happen, but if it would we better catch it here because Metrictank wouldn't handle it well: + // https://github.com/grafana/metrictank/blob/f1868cccfb92fc82cd853914af958f6d187c5f74/mdata/aggmetric.go#L378 + if point.Timestamp == 0 { + continue + } + + t0 = point.Timestamp - (point.Timestamp % chunkSpan) + if prevT0 == 0 { + c = chunk.New(t0) + prevT0 = t0 + } else if prevT0 != t0 { + c.Finish() + encodedChunks = append(encodedChunks, c) + + c = chunk.New(t0) + prevT0 = t0 + } + + err := c.Push(point.Timestamp, point.Value) + if err != nil { + panic(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %q", t0, err)) + } + } + + // if the last written point was also the last one of the current chunk, + // or if writeUnfinishedChunks is on, we close the chunk and push it + if point.Timestamp == t0+chunkSpan-intervalIn || writeUnfinishedChunks { + c.Finish() + encodedChunks = append(encodedChunks, c) + } + + return encodedChunks +} diff --git a/mdata/importer/chunk_encoder_test.go b/mdata/importer/chunk_encoder_test.go new file mode 100644 index 0000000000..b964376c59 --- /dev/null +++ b/mdata/importer/chunk_encoder_test.go @@ -0,0 +1,93 @@ +package importer + +import ( + "testing" + + "github.com/grafana/metrictank/mdata/chunk" + "github.com/kisielk/whisper-go/whisper" +) + +func TestEncodedChunksFromPointsWithUnfinished(t *testing.T) { + points := generatePoints(25200, 10, 10, 0, 8640, func(i float64) float64 { return i + 1 }) + expectedCount := 8640 // count including unfinished chunks + + chunks := EncodeChunksFromPoints(points, 10, 21600, true) + + if len(chunks) != 5 { + t.Fatalf("Expected to get 5 chunks, but got %d", len(chunks)) + } + + i := 0 + for _, c := range chunks { + iterGen, err := chunk.NewIterGen(c.Series.T0, 10, c.Encode(21600)) + if err != nil { + t.Fatalf("Error getting iterator: %s", err) + } + + iter, err := iterGen.Get() + if err != nil { + t.Fatalf("Error getting iterator: %s", err) + } + + for iter.Next() { + ts, val := iter.Values() + if points[i].Timestamp != ts || points[i].Value != val { + t.Fatalf("Unexpected value at index %d:\nExpected: %d:%f\nGot: %d:%f\n", i, ts, val, points[i].Timestamp, points[i].Value) + } + i++ + } + } + if i != expectedCount { + t.Fatalf("Unexpected number of datapoints in chunks:\nExpected: %d\nGot: %d\n", expectedCount, i) + } +} + +func TestEncodedChunksFromPointsWithoutUnfinished(t *testing.T) { + // the actual data in these points doesn't matter, we just want to be sure + // that the chunks resulting from these points include the same data + points := generatePoints(25200, 10, 10, 0, 8640, func(i float64) float64 { return i + 1 }) + expectedCount := 8640 - (2520 % 2160) // count minus what would end up in an unfinished chunk + + chunks := EncodeChunksFromPoints(points, 10, 21600, false) + + if len(chunks) != 4 { + t.Fatalf("Expected to get 4 chunks, but got %d", len(chunks)) + } + + i := 0 + for _, c := range chunks { + iterGen, err := chunk.NewIterGen(c.Series.T0, 10, c.Encode(21600)) + if err != nil { + t.Fatalf("Error getting iterator: %s", err) + } + + iter, err := iterGen.Get() + if err != nil { + t.Fatalf("Error getting iterator: %s", err) + } + + for iter.Next() { + ts, val := iter.Values() + if points[i].Timestamp != ts || points[i].Value != val { + t.Fatalf("Unexpected value at index %d:\nExpected: %d:%f\nGot: %d:%f\n", i, ts, val, points[i].Timestamp, points[i].Value) + } + i++ + } + } + if i != expectedCount { + t.Fatalf("Unexpected number of datapoints in chunks:\nExpected: %d\nGot: %d\n", expectedCount, i) + } +} + +func generatePoints(ts, interval uint32, value float64, offset, count int, inc func(float64) float64) []whisper.Point { + res := make([]whisper.Point, count) + for i := 0; i < count; i++ { + res[(i+offset)%count] = whisper.Point{ + Timestamp: ts, + Value: value, + } + ts += interval + value = inc(value) + } + return res +} diff --git a/cmd/mt-whisper-importer-reader/conversion.go b/mdata/importer/conversion.go similarity index 85% rename from cmd/mt-whisper-importer-reader/conversion.go rename to mdata/importer/conversion.go index 18b2fd0969..a8e035337c 100644 --- a/cmd/mt-whisper-importer-reader/conversion.go +++ b/mdata/importer/conversion.go @@ -1,49 +1,28 @@ -package main +package importer import ( "github.com/grafana/metrictank/mdata" "github.com/kisielk/whisper-go/whisper" "github.com/raintank/schema" + "sort" ) type conversion struct { archives []whisper.ArchiveInfo points map[int][]whisper.Point method schema.Method + from uint32 + until uint32 } const fakeAvg schema.Method = 255 -func newConversion(arch []whisper.ArchiveInfo, points map[int][]whisper.Point, method schema.Method) *conversion { - return &conversion{archives: arch, points: points, method: method} -} - -func (c *conversion) findSmallestLargestArchive(spp, nop uint32) (int, int) { - // find smallest archive that still contains enough data to satisfy requested range - largestArchiveIdx := len(c.archives) - 1 - for i := largestArchiveIdx; i >= 0; i-- { - arch := c.archives[i] - if arch.Points*arch.SecondsPerPoint < nop*spp { - break - } - largestArchiveIdx = i - } - - // find largest archive that still has a higher or equal resolution than requested - smallestArchiveIdx := 0 - for i := 0; i < len(c.archives); i++ { - arch := c.archives[i] - if arch.SecondsPerPoint > spp { - break - } - smallestArchiveIdx = i - } - - return smallestArchiveIdx, largestArchiveIdx +func NewConversion(arch []whisper.ArchiveInfo, points map[int][]whisper.Point, method schema.Method, from, until uint32) *conversion { + return &conversion{archives: arch, points: points, method: method, from: from, until: until} } // generates points according to specified parameters by finding and using the best archives as input -func (c *conversion) getPoints(retIdx int, spp, nop uint32) map[schema.Method][]whisper.Point { +func (c *conversion) GetPoints(retIdx int, spp, nop uint32) map[schema.Method][]whisper.Point { res := make(map[schema.Method][]whisper.Point) if len(c.points) == 0 { @@ -78,7 +57,7 @@ func (c *conversion) getPoints(retIdx int, spp, nop uint32) map[schema.Method][] rawFactor := float64(spp) / float64(rawRes) if retIdx == 0 || c.method != schema.Avg { for _, p := range in { - if p.Timestamp > uint32(*importUpTo) || p.Timestamp < uint32(*importAfter) { + if p.Timestamp > c.until || p.Timestamp < c.from { continue } adjustedPoints[c.method][p.Timestamp] = p.Value @@ -88,7 +67,7 @@ func (c *conversion) getPoints(retIdx int, spp, nop uint32) map[schema.Method][] } } else { for _, p := range in { - if p.Timestamp > uint32(*importUpTo) || p.Timestamp < uint32(*importAfter) { + if p.Timestamp > c.until || p.Timestamp < c.from { continue } adjustedPoints[schema.Sum][p.Timestamp] = p.Value * rawFactor @@ -96,13 +75,13 @@ func (c *conversion) getPoints(retIdx int, spp, nop uint32) map[schema.Method][] } } } else if arch.SecondsPerPoint > spp { - for m, points := range incResolution(in, method, arch.SecondsPerPoint, spp, rawRes) { + for m, points := range incResolution(in, method, arch.SecondsPerPoint, spp, rawRes, c.from, c.until) { for _, p := range points { adjustedPoints[m][p.Timestamp] = p.Value } } } else { - for m, points := range decResolution(in, method, arch.SecondsPerPoint, spp, rawRes) { + for m, points := range decResolution(in, method, arch.SecondsPerPoint, spp, rawRes, c.from, c.until) { for _, p := range points { adjustedPoints[m][p.Timestamp] = p.Value } @@ -113,7 +92,7 @@ func (c *conversion) getPoints(retIdx int, spp, nop uint32) map[schema.Method][] // merge the results that are keyed by timestamp into a slice of points for m, p := range adjustedPoints { for t, v := range p { - if t <= uint32(*importUpTo) && t >= uint32(*importAfter) { + if t <= c.until && t >= c.from { res[m] = append(res[m], whisper.Point{Timestamp: t, Value: v}) } } @@ -130,11 +109,35 @@ func (c *conversion) getPoints(retIdx int, spp, nop uint32) map[schema.Method][] return res } +func (c *conversion) findSmallestLargestArchive(spp, nop uint32) (int, int) { + // find smallest archive that still contains enough data to satisfy requested range + largestArchiveIdx := len(c.archives) - 1 + for i := largestArchiveIdx; i >= 0; i-- { + arch := c.archives[i] + if arch.Points*arch.SecondsPerPoint < nop*spp { + break + } + largestArchiveIdx = i + } + + // find largest archive that still has a higher or equal resolution than requested + smallestArchiveIdx := 0 + for i := 0; i < len(c.archives); i++ { + arch := c.archives[i] + if arch.SecondsPerPoint > spp { + break + } + smallestArchiveIdx = i + } + + return smallestArchiveIdx, largestArchiveIdx +} + // increase resolution of given points according to defined specs by generating // additional datapoints to bridge the gaps between the given points. depending // on what aggregation method is specified, those datapoints may be generated in // slightly different ways. -func incResolution(points []whisper.Point, method schema.Method, inRes, outRes, rawRes uint32) map[schema.Method][]whisper.Point { +func incResolution(points []whisper.Point, method schema.Method, inRes, outRes, rawRes, from, until uint32) map[schema.Method][]whisper.Point { out := make(map[schema.Method][]whisper.Point) resFactor := float64(outRes) / float64(rawRes) for _, inPoint := range points { @@ -151,7 +154,7 @@ func incResolution(points []whisper.Point, method schema.Method, inRes, outRes, // generate datapoints based on inPoint in reverse order var outPoints []whisper.Point for ts := rangeEnd; ts > inPoint.Timestamp-inRes; ts = ts - outRes { - if ts > uint32(*importUpTo) || ts < uint32(*importAfter) { + if ts > until || ts < from { continue } outPoints = append(outPoints, whisper.Point{Timestamp: ts}) @@ -183,7 +186,7 @@ func incResolution(points []whisper.Point, method schema.Method, inRes, outRes, // decreases the resolution of given points by using the aggregation method specified // in the second argument. emulates the way metrictank aggregates data when it generates // rollups of the raw data. -func decResolution(points []whisper.Point, method schema.Method, inRes, outRes, rawRes uint32) map[schema.Method][]whisper.Point { +func decResolution(points []whisper.Point, method schema.Method, inRes, outRes, rawRes, from, until uint32) map[schema.Method][]whisper.Point { out := make(map[schema.Method][]whisper.Point) agg := mdata.NewAggregation() currentBoundary := uint32(0) @@ -241,10 +244,10 @@ func decResolution(points []whisper.Point, method schema.Method, inRes, outRes, continue } boundary := mdata.AggBoundary(inPoint.Timestamp, outRes) - if boundary > uint32(*importUpTo) { + if boundary > until { break } - if boundary < uint32(*importAfter) { + if boundary < from { continue } @@ -262,3 +265,17 @@ func decResolution(points []whisper.Point, method schema.Method, inRes, outRes, return out } + +// pointSorter sorts points by timestamp +type pointSorter []whisper.Point + +func (a pointSorter) Len() int { return len(a) } +func (a pointSorter) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp } + +// the whisper archives are organized like a ringbuffer. since we need to +// insert the points into the chunks in order we first need to sort them +func sortPoints(points pointSorter) pointSorter { + sort.Sort(points) + return points +} diff --git a/cmd/mt-whisper-importer-reader/conversion_test.go b/mdata/importer/conversion_test.go similarity index 84% rename from cmd/mt-whisper-importer-reader/conversion_test.go rename to mdata/importer/conversion_test.go index cae00d72cc..1ffb3d274e 100644 --- a/cmd/mt-whisper-importer-reader/conversion_test.go +++ b/mdata/importer/conversion_test.go @@ -1,18 +1,16 @@ -package main +package importer import ( "math" "testing" - "github.com/grafana/metrictank/mdata/chunk" - "github.com/kisielk/whisper-go/whisper" "github.com/raintank/schema" ) -func testIncResolution(t *testing.T, inData []whisper.Point, expectedResult map[schema.Method][]whisper.Point, method schema.Method, inRes, outRes, rawRes uint32) { +func testIncResolution(t *testing.T, inData []whisper.Point, expectedResult map[schema.Method][]whisper.Point, method schema.Method, inRes, outRes, rawRes, from, until uint32) { t.Helper() - outData := incResolution(inData, method, inRes, outRes, rawRes) + outData := incResolution(inData, method, inRes, outRes, rawRes, from, until) if len(expectedResult) != len(outData) { t.Fatalf("Generated data is not as expected:\nExpected:\n%+v\nGot:\n%+v\n", expectedResult, outData) @@ -24,7 +22,7 @@ func testIncResolution(t *testing.T, inData []whisper.Point, expectedResult map[ if p, ok = outData[m]; !ok { t.Fatalf("testIncResolution.\nExpected:\n%+v\nGot:\n%+v\n", expectedResult, outData) } - if len(p) != len(outData[m]) { + if len(p) != len(ep) { t.Fatalf("testIncResolution.\nExpected:\n%+v\nGot:\n%+v\n", expectedResult, outData) } for i := range p { @@ -49,9 +47,7 @@ func TestIncResolutionUpToTime(t *testing.T) { {5, 5}, }, } - *importUpTo = uint(5) - testIncResolution(t, inData, expectedResult, fakeAvg, 10, 5, 1) - *importUpTo = math.MaxUint32 + testIncResolution(t, inData, expectedResult, fakeAvg, 10, 5, 1, 0, 5) } func TestIncResolutionFakeAvgNonFactorResolutions(t *testing.T) { @@ -102,7 +98,7 @@ func TestIncResolutionFakeAvgNonFactorResolutions(t *testing.T) { }, } - testIncResolution(t, inData, expectedResult, fakeAvg, 10, 3, 1) + testIncResolution(t, inData, expectedResult, fakeAvg, 10, 3, 1, 0, math.MaxUint32) } func TestIncFakeAvgResolutionWithGaps(t *testing.T) { @@ -135,7 +131,7 @@ func TestIncFakeAvgResolutionWithGaps(t *testing.T) { }, } - testIncResolution(t, inData, expectedResult, fakeAvg, 10, 5, 1) + testIncResolution(t, inData, expectedResult, fakeAvg, 10, 5, 1, 0, math.MaxUint32) } func TestIncFakeAvgResolutionOutOfOrder(t *testing.T) { @@ -164,7 +160,7 @@ func TestIncFakeAvgResolutionOutOfOrder(t *testing.T) { }, } - testIncResolution(t, inData, expectedResult, fakeAvg, 10, 5, 1) + testIncResolution(t, inData, expectedResult, fakeAvg, 10, 5, 1, 0, math.MaxUint32) } func TestIncFakeAvgResolutionStrangeRawRes(t *testing.T) { @@ -201,7 +197,7 @@ func TestIncFakeAvgResolutionStrangeRawRes(t *testing.T) { }, } - testIncResolution(t, inData, expectedResult, fakeAvg, 30, 10, 3) + testIncResolution(t, inData, expectedResult, fakeAvg, 30, 10, 3, 0, math.MaxUint32) } func TestIncResolutionSimpleMax(t *testing.T) { @@ -218,7 +214,7 @@ func TestIncResolutionSimpleMax(t *testing.T) { {20, 11}, }, } - testIncResolution(t, inData, expectedResult, schema.Max, 10, 5, 1) + testIncResolution(t, inData, expectedResult, schema.Max, 10, 5, 1, 0, math.MaxUint32) } func TestIncResolutionSimpleLast(t *testing.T) { @@ -235,7 +231,7 @@ func TestIncResolutionSimpleLast(t *testing.T) { {20, 11}, }, } - testIncResolution(t, inData, expectedResult, schema.Lst, 10, 5, 1) + testIncResolution(t, inData, expectedResult, schema.Lst, 10, 5, 1, 0, math.MaxUint32) } func TestIncResolutionSimpleMin(t *testing.T) { @@ -252,7 +248,7 @@ func TestIncResolutionSimpleMin(t *testing.T) { {20, 11}, }, } - testIncResolution(t, inData, expectedResult, schema.Min, 10, 5, 1) + testIncResolution(t, inData, expectedResult, schema.Min, 10, 5, 1, 0, math.MaxUint32) } func TestIncResolutionSimpleAvg(t *testing.T) { @@ -269,7 +265,7 @@ func TestIncResolutionSimpleAvg(t *testing.T) { {20, 11}, }, } - testIncResolution(t, inData, expectedResult, schema.Avg, 10, 5, 1) + testIncResolution(t, inData, expectedResult, schema.Avg, 10, 5, 1, 0, math.MaxUint32) } func TestIncResolutionSimpleFakeAvg(t *testing.T) { @@ -292,7 +288,7 @@ func TestIncResolutionSimpleFakeAvg(t *testing.T) { {20, 5}, }, } - testIncResolution(t, inData, expectedResult, fakeAvg, 10, 5, 1) + testIncResolution(t, inData, expectedResult, fakeAvg, 10, 5, 1, 0, math.MaxUint32) } func TestIncResolutionSimpleSum(t *testing.T) { @@ -315,7 +311,7 @@ func TestIncResolutionSimpleSum(t *testing.T) { {20, 5}, }, } - testIncResolution(t, inData, expectedResult, schema.Sum, 10, 5, 1) + testIncResolution(t, inData, expectedResult, schema.Sum, 10, 5, 1, 0, math.MaxUint32) } func TestIncResolutionNonFactorResolutions(t *testing.T) { @@ -348,7 +344,7 @@ func TestIncResolutionNonFactorResolutions(t *testing.T) { }, } - testIncResolution(t, inData, expectedResult, schema.Max, 10, 3, 1) + testIncResolution(t, inData, expectedResult, schema.Max, 10, 3, 1, 0, math.MaxUint32) } func TestIncResolutionWithGaps(t *testing.T) { @@ -373,7 +369,7 @@ func TestIncResolutionWithGaps(t *testing.T) { }, } - testIncResolution(t, inData, expectedResult, schema.Max, 10, 5, 1) + testIncResolution(t, inData, expectedResult, schema.Max, 10, 5, 1, 0, math.MaxUint32) } func TestIncResolutionOutOfOrder(t *testing.T) { @@ -394,12 +390,12 @@ func TestIncResolutionOutOfOrder(t *testing.T) { }, } - testIncResolution(t, inData, expectedResult, schema.Max, 10, 5, 1) + testIncResolution(t, inData, expectedResult, schema.Max, 10, 5, 1, 0, math.MaxUint32) } -func testDecResolution(t *testing.T, inData []whisper.Point, expectedResult map[schema.Method][]whisper.Point, method schema.Method, inRes, outRes, rawRes uint32) { +func testDecResolution(t *testing.T, inData []whisper.Point, expectedResult map[schema.Method][]whisper.Point, method schema.Method, inRes, outRes, rawRes, from, until uint32) { t.Helper() - outData := decResolution(inData, method, inRes, outRes, rawRes) + outData := decResolution(inData, method, inRes, outRes, rawRes, from, until) if len(expectedResult) != len(outData) { t.Fatalf("Generated data has different length (%d) than expected (%d):\n%+v\n%+v", len(expectedResult), len(outData), outData, expectedResult) @@ -440,7 +436,7 @@ func TestDecResolutionSimpleAvg(t *testing.T) { {60, 14}, }, } - testDecResolution(t, getSimpleInData(), expectedResult, schema.Avg, 10, 30, 1) + testDecResolution(t, getSimpleInData(), expectedResult, schema.Avg, 10, 30, 1, 0, math.MaxUint32) } func TestDecResolutionSimpleFakeAvg(t *testing.T) { @@ -454,7 +450,7 @@ func TestDecResolutionSimpleFakeAvg(t *testing.T) { {60, 30}, }, } - testDecResolution(t, getSimpleInData(), expectedResult, fakeAvg, 10, 30, 1) + testDecResolution(t, getSimpleInData(), expectedResult, fakeAvg, 10, 30, 1, 0, math.MaxUint32) } func TestDecResolutionSimpleSum(t *testing.T) { @@ -468,7 +464,7 @@ func TestDecResolutionSimpleSum(t *testing.T) { {60, 30}, }, } - testDecResolution(t, getSimpleInData(), expectedResult, schema.Sum, 10, 30, 1) + testDecResolution(t, getSimpleInData(), expectedResult, schema.Sum, 10, 30, 1, 0, math.MaxUint32) } func TestDecResolutionSimpleLast(t *testing.T) { @@ -478,7 +474,7 @@ func TestDecResolutionSimpleLast(t *testing.T) { {60, 15}, }, } - testDecResolution(t, getSimpleInData(), expectedResult, schema.Lst, 10, 30, 1) + testDecResolution(t, getSimpleInData(), expectedResult, schema.Lst, 10, 30, 1, 0, math.MaxUint32) } func TestDecResolutionSimpleMax(t *testing.T) { @@ -488,7 +484,7 @@ func TestDecResolutionSimpleMax(t *testing.T) { {60, 15}, }, } - testDecResolution(t, getSimpleInData(), expectedResult, schema.Max, 10, 30, 1) + testDecResolution(t, getSimpleInData(), expectedResult, schema.Max, 10, 30, 1, 0, math.MaxUint32) } func TestDecResolutionSimpleMin(t *testing.T) { @@ -498,7 +494,7 @@ func TestDecResolutionSimpleMin(t *testing.T) { {60, 13}, }, } - testDecResolution(t, getSimpleInData(), expectedResult, schema.Min, 10, 30, 1) + testDecResolution(t, getSimpleInData(), expectedResult, schema.Min, 10, 30, 1, 0, math.MaxUint32) } func TestDecResolutionUpToTime(t *testing.T) { @@ -519,9 +515,7 @@ func TestDecResolutionUpToTime(t *testing.T) { {30, 6}, }, } - *importUpTo = uint(40) - testDecResolution(t, inData, expectedResult, schema.Sum, 10, 30, 5) - *importUpTo = math.MaxUint32 + testDecResolution(t, inData, expectedResult, schema.Sum, 10, 30, 5, 0, 40) } func TestDecResolutionAvg(t *testing.T) { @@ -540,7 +534,7 @@ func TestDecResolutionAvg(t *testing.T) { {60, 14}, }, } - testDecResolution(t, inData, expectedResult, schema.Avg, 10, 30, 1) + testDecResolution(t, inData, expectedResult, schema.Avg, 10, 30, 1, 0, math.MaxUint32) } func TestDecNonFactorResolutions(t *testing.T) { @@ -561,7 +555,7 @@ func TestDecNonFactorResolutions(t *testing.T) { {60, 14.5}, }, } - testDecResolution(t, inData, expectedResult, schema.Avg, 10, 15, 1) + testDecResolution(t, inData, expectedResult, schema.Avg, 10, 15, 1, 0, math.MaxUint32) } func getGapData() []whisper.Point { @@ -586,7 +580,7 @@ func TestDecResolutionWithGapsAvg(t *testing.T) { }, } - testDecResolution(t, getGapData(), expectedResult, schema.Avg, 10, 20, 1) + testDecResolution(t, getGapData(), expectedResult, schema.Avg, 10, 20, 1, 0, math.MaxUint32) } func TestDecResolutionWithGapsFakeAvg(t *testing.T) { @@ -603,7 +597,7 @@ func TestDecResolutionWithGapsFakeAvg(t *testing.T) { }, } - testDecResolution(t, getGapData(), expectedResult, fakeAvg, 10, 20, 1) + testDecResolution(t, getGapData(), expectedResult, fakeAvg, 10, 20, 1, 0, math.MaxUint32) } func TestDecResolutionWithGapsSum(t *testing.T) { @@ -620,7 +614,7 @@ func TestDecResolutionWithGapsSum(t *testing.T) { }, } - testDecResolution(t, getGapData(), expectedResult, schema.Sum, 10, 20, 1) + testDecResolution(t, getGapData(), expectedResult, schema.Sum, 10, 20, 1, 0, math.MaxUint32) } func TestDecResolutionOutOfOrder(t *testing.T) { @@ -639,7 +633,7 @@ func TestDecResolutionOutOfOrder(t *testing.T) { {60, 15}, }, } - testDecResolution(t, inData, expectedResult, schema.Avg, 10, 30, 1) + testDecResolution(t, inData, expectedResult, schema.Avg, 10, 30, 1, 0, math.MaxUint32) } func TestDecFakeAvgNonFactorResolutions(t *testing.T) { @@ -666,7 +660,7 @@ func TestDecFakeAvgNonFactorResolutions(t *testing.T) { {60, 20}, }, } - testDecResolution(t, inData, expectedResult, schema.Sum, 10, 15, 1) + testDecResolution(t, inData, expectedResult, schema.Sum, 10, 15, 1, 0, math.MaxUint32) } func TestDecResolutionFakeAvgOutOfOrder(t *testing.T) { @@ -689,94 +683,7 @@ func TestDecResolutionFakeAvgOutOfOrder(t *testing.T) { {60, 30}, }, } - testDecResolution(t, inData, expectedResult, schema.Sum, 10, 30, 1) -} - -func generatePoints(ts, interval uint32, value float64, offset, count int, inc func(float64) float64) []whisper.Point { - res := make([]whisper.Point, count) - for i := 0; i < count; i++ { - res[(i+offset)%count] = whisper.Point{ - Timestamp: ts, - Value: value, - } - ts += interval - value = inc(value) - } - return res -} - -func TestEncodedChunksFromPointsWithoutUnfinished(t *testing.T) { - // the actual data in these points doesn't matter, we just want to be sure - // that the chunks resulting from these points include the same data - points := generatePoints(25200, 10, 10, 0, 8640, func(i float64) float64 { return i + 1 }) - expectedCount := 8640 - (2520 % 2160) // count minus what would end up in an unfinished chunk - - *writeUnfinishedChunks = false - chunks := encodedChunksFromPoints(points, 10, 21600) - - if len(chunks) != 4 { - t.Fatalf("Expected to get 4 chunks, but got %d", len(chunks)) - } - - i := 0 - for _, c := range chunks { - iterGen, err := chunk.NewIterGen(c.Series.T0, 10, c.Encode(21600)) - if err != nil { - t.Fatalf("Error getting iterator: %s", err) - } - - iter, err := iterGen.Get() - if err != nil { - t.Fatalf("Error getting iterator: %s", err) - } - - for iter.Next() { - ts, val := iter.Values() - if points[i].Timestamp != ts || points[i].Value != val { - t.Fatalf("Unexpected value at index %d:\nExpected: %d:%f\nGot: %d:%f\n", i, ts, val, points[i].Timestamp, points[i].Value) - } - i++ - } - } - if i != expectedCount { - t.Fatalf("Unexpected number of datapoints in chunks:\nExpected: %d\nGot: %d\n", expectedCount, i) - } -} - -func TestEncodedChunksFromPointsWithUnfinished(t *testing.T) { - points := generatePoints(25200, 10, 10, 0, 8640, func(i float64) float64 { return i + 1 }) - expectedCount := 8640 // count including unfinished chunks - - *writeUnfinishedChunks = true - chunks := encodedChunksFromPoints(points, 10, 21600) - - if len(chunks) != 5 { - t.Fatalf("Expected to get 5 chunks, but got %d", len(chunks)) - } - - i := 0 - for _, c := range chunks { - iterGen, err := chunk.NewIterGen(c.Series.T0, 10, c.Encode(21600)) - if err != nil { - t.Fatalf("Error getting iterator: %s", err) - } - - iter, err := iterGen.Get() - if err != nil { - t.Fatalf("Error getting iterator: %s", err) - } - - for iter.Next() { - ts, val := iter.Values() - if points[i].Timestamp != ts || points[i].Value != val { - t.Fatalf("Unexpected value at index %d:\nExpected: %d:%f\nGot: %d:%f\n", i, ts, val, points[i].Timestamp, points[i].Value) - } - i++ - } - } - if i != expectedCount { - t.Fatalf("Unexpected number of datapoints in chunks:\nExpected: %d\nGot: %d\n", expectedCount, i) - } + testDecResolution(t, inData, expectedResult, schema.Sum, 10, 30, 1, 0, math.MaxUint32) } func verifyPointMaps(t *testing.T, points map[schema.Method][]whisper.Point, expected map[schema.Method][]whisper.Point) { @@ -818,6 +725,7 @@ func TestPointsConversionSum1(t *testing.T) { }, }, method: schema.Sum, + until: math.MaxUint32, } expectedPoints1 := map[schema.Method][]whisper.Point{ @@ -847,9 +755,9 @@ func TestPointsConversionSum1(t *testing.T) { }, } - points1 := c.getPoints(0, 1, 8) - points2 := c.getPoints(0, 2, 4) - points3 := c.getPoints(0, 4, 2) + points1 := c.GetPoints(0, 1, 8) + points2 := c.GetPoints(0, 2, 4) + points3 := c.GetPoints(0, 4, 2) verifyPointMaps(t, points1, expectedPoints1) verifyPointMaps(t, points2, expectedPoints2) @@ -878,6 +786,7 @@ func TestPointsConversionLast1(t *testing.T) { }, }, method: schema.Lst, + until: math.MaxUint32, } expectedPoints1 := map[schema.Method][]whisper.Point{ @@ -907,9 +816,9 @@ func TestPointsConversionLast1(t *testing.T) { }, } - points1 := c.getPoints(0, 1, 8) - points2 := c.getPoints(0, 2, 4) - points3 := c.getPoints(0, 4, 2) + points1 := c.GetPoints(0, 1, 8) + points2 := c.GetPoints(0, 2, 4) + points3 := c.GetPoints(0, 4, 2) verifyPointMaps(t, points1, expectedPoints1) verifyPointMaps(t, points2, expectedPoints2) @@ -956,6 +865,7 @@ func TestPointsConversionSum2(t *testing.T) { }, }, method: schema.Sum, + until: math.MaxUint32, } expectedPoints1 := map[schema.Method][]whisper.Point{ @@ -1029,9 +939,9 @@ func TestPointsConversionSum2(t *testing.T) { }, } - points1 := c.getPoints(0, 1, 32) - points2 := c.getPoints(1, 2, 16) - points3 := c.getPoints(2, 4, 8) + points1 := c.GetPoints(0, 1, 32) + points2 := c.GetPoints(1, 2, 16) + points3 := c.GetPoints(2, 4, 8) verifyPointMaps(t, points1, expectedPoints1) verifyPointMaps(t, points2, expectedPoints2) @@ -1060,6 +970,7 @@ func TestPointsConversionAvg1(t *testing.T) { }, }, method: schema.Avg, + until: math.MaxUint32, } expectedPoints1_0 := map[schema.Method][]whisper.Point{ @@ -1177,19 +1088,19 @@ func TestPointsConversionAvg1(t *testing.T) { }, } - points1_0 := c.getPoints(0, 1, 8) - points2_0 := c.getPoints(0, 2, 4) - points3_0 := c.getPoints(0, 4, 2) + points1_0 := c.GetPoints(0, 1, 8) + points2_0 := c.GetPoints(0, 2, 4) + points3_0 := c.GetPoints(0, 4, 2) - points1_1 := c.getPoints(1, 1, 8) - points2_1 := c.getPoints(1, 2, 4) - points3_1 := c.getPoints(1, 4, 100) + points1_1 := c.GetPoints(1, 1, 8) + points2_1 := c.GetPoints(1, 2, 4) + points3_1 := c.GetPoints(1, 4, 100) - *importUpTo = uint(1503407723) - points1_2 := c.getPoints(1, 1, 8) - points2_2 := c.getPoints(1, 2, 100) - points3_2 := c.getPoints(1, 4, 8) - *importUpTo = math.MaxUint32 + c.until = 1503407723 + points1_2 := c.GetPoints(1, 1, 8) + points2_2 := c.GetPoints(1, 2, 100) + points3_2 := c.GetPoints(1, 4, 8) + c.until = math.MaxUint32 verifyPointMaps(t, points1_0, expectedPoints1_0) verifyPointMaps(t, points2_0, expectedPoints2_0) @@ -1229,6 +1140,7 @@ func TestPointsConversionAvg2(t *testing.T) { }, }, method: schema.Avg, + until: math.MaxUint32, } expectedPoints1_0 := map[schema.Method][]whisper.Point{ @@ -1385,13 +1297,13 @@ func TestPointsConversionAvg2(t *testing.T) { }, } - points1_0 := c.getPoints(0, 1, 27) - points2_0 := c.getPoints(0, 3, 100) - points3_0 := c.getPoints(0, 9, 100) + points1_0 := c.GetPoints(0, 1, 27) + points2_0 := c.GetPoints(0, 3, 100) + points3_0 := c.GetPoints(0, 9, 100) - points1_1 := c.getPoints(1, 1, 27) - points2_1 := c.getPoints(1, 3, 9) - points3_1 := c.getPoints(1, 9, 3) + points1_1 := c.GetPoints(1, 1, 27) + points2_1 := c.GetPoints(1, 3, 9) + points3_1 := c.GetPoints(1, 9, 3) verifyPointMaps(t, points1_0, expectedPoints1_0) verifyPointMaps(t, points2_0, expectedPoints2_0) From f0b5305dec22bc3e12afc5c2be25958d3c7f8ad1 Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Tue, 11 Jun 2019 01:14:59 +0000 Subject: [PATCH 5/6] move more converter logic into importer package --- cmd/mt-whisper-importer-reader/main.go | 23 ++------------ .../importer/{conversion.go => converter.go} | 31 +++++++++++++++---- .../{conversion_test.go => converter_test.go} | 0 3 files changed, 28 insertions(+), 26 deletions(-) rename mdata/importer/{conversion.go => converter.go} (89%) rename mdata/importer/{conversion_test.go => converter_test.go} (100%) diff --git a/cmd/mt-whisper-importer-reader/main.go b/cmd/mt-whisper-importer-reader/main.go index d8cb5d08f2..83d3037009 100644 --- a/cmd/mt-whisper-importer-reader/main.go +++ b/cmd/mt-whisper-importer-reader/main.go @@ -239,29 +239,12 @@ func getMetricName(file string) string { return *namePrefix + strings.Replace(strings.TrimSuffix(file, ".wsp"), "/", ".", -1) } -func convertWhisperMethod(whisperMethod whisper.AggregationMethod) (schema.Method, error) { - switch whisperMethod { - case whisper.AggregationAverage: - return schema.Avg, nil - case whisper.AggregationSum: - return schema.Sum, nil - case whisper.AggregationLast: - return schema.Lst, nil - case whisper.AggregationMax: - return schema.Max, nil - case whisper.AggregationMin: - return schema.Min, nil - default: - return 0, fmt.Errorf("Unknown whisper method: %d", whisperMethod) - } -} - func getMetric(w *whisper.Whisper, file, name string) (*importer.ArchiveRequest, error) { if len(w.Header.Archives) == 0 { return nil, fmt.Errorf("Whisper file contains no archives: %q", file) } - method, err := convertWhisperMethod(w.Header.Metadata.AggregationMethod) + method, err := importer.ConvertWhisperMethod(w.Header.Metadata.AggregationMethod) if err != nil { return nil, err } @@ -289,9 +272,9 @@ func getMetric(w *whisper.Whisper, file, name string) (*importer.ArchiveRequest, res.MetricData.SetId() _, selectedSchema := schemas.Match(res.MetricData.Name, int(w.Header.Archives[0].SecondsPerPoint)) - conversion := importer.NewConversion(w.Header.Archives, points, method, uint32(*importFrom), uint32(*importUntil)) + converter := importer.NewConverter(w.Header.Archives, points, method, uint32(*importFrom), uint32(*importUntil)) for retIdx, retention := range selectedSchema.Retentions { - convertedPoints := conversion.GetPoints(retIdx, uint32(retention.SecondsPerPoint), uint32(retention.NumberOfPoints)) + convertedPoints := converter.GetPoints(retIdx, uint32(retention.SecondsPerPoint), uint32(retention.NumberOfPoints)) for m, p := range convertedPoints { if len(p) == 0 { continue diff --git a/mdata/importer/conversion.go b/mdata/importer/converter.go similarity index 89% rename from mdata/importer/conversion.go rename to mdata/importer/converter.go index a8e035337c..ee254a7e2d 100644 --- a/mdata/importer/conversion.go +++ b/mdata/importer/converter.go @@ -1,13 +1,15 @@ package importer import ( + "fmt" + "sort" + "github.com/grafana/metrictank/mdata" "github.com/kisielk/whisper-go/whisper" "github.com/raintank/schema" - "sort" ) -type conversion struct { +type Converter struct { archives []whisper.ArchiveInfo points map[int][]whisper.Point method schema.Method @@ -17,12 +19,12 @@ type conversion struct { const fakeAvg schema.Method = 255 -func NewConversion(arch []whisper.ArchiveInfo, points map[int][]whisper.Point, method schema.Method, from, until uint32) *conversion { - return &conversion{archives: arch, points: points, method: method, from: from, until: until} +func NewConverter(arch []whisper.ArchiveInfo, points map[int][]whisper.Point, method schema.Method, from, until uint32) *Converter { + return &Converter{archives: arch, points: points, method: method, from: from, until: until} } // generates points according to specified parameters by finding and using the best archives as input -func (c *conversion) GetPoints(retIdx int, spp, nop uint32) map[schema.Method][]whisper.Point { +func (c *Converter) GetPoints(retIdx int, spp, nop uint32) map[schema.Method][]whisper.Point { res := make(map[schema.Method][]whisper.Point) if len(c.points) == 0 { @@ -109,7 +111,7 @@ func (c *conversion) GetPoints(retIdx int, spp, nop uint32) map[schema.Method][] return res } -func (c *conversion) findSmallestLargestArchive(spp, nop uint32) (int, int) { +func (c *Converter) findSmallestLargestArchive(spp, nop uint32) (int, int) { // find smallest archive that still contains enough data to satisfy requested range largestArchiveIdx := len(c.archives) - 1 for i := largestArchiveIdx; i >= 0; i-- { @@ -279,3 +281,20 @@ func sortPoints(points pointSorter) pointSorter { sort.Sort(points) return points } + +func ConvertWhisperMethod(whisperMethod whisper.AggregationMethod) (schema.Method, error) { + switch whisperMethod { + case whisper.AggregationAverage: + return schema.Avg, nil + case whisper.AggregationSum: + return schema.Sum, nil + case whisper.AggregationLast: + return schema.Lst, nil + case whisper.AggregationMax: + return schema.Max, nil + case whisper.AggregationMin: + return schema.Min, nil + default: + return 0, fmt.Errorf("Unknown whisper method: %d", whisperMethod) + } +} diff --git a/mdata/importer/conversion_test.go b/mdata/importer/converter_test.go similarity index 100% rename from mdata/importer/conversion_test.go rename to mdata/importer/converter_test.go From d9666457677c31b15887b1d324e496142f281330 Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Tue, 11 Jun 2019 01:42:29 +0000 Subject: [PATCH 6/6] move ArchiveRequest instantiation into importer package the instantiation of the ArchiveRequest struct should be located together with the struct definition, this moves it there --- cmd/mt-whisper-importer-reader/main.go | 69 +----- mdata/importer/archive_request.go | 135 +++++++++++ mdata/importer/archive_request_gen.go | 182 +++++++++++++++ mdata/importer/archive_request_gen_test.go | 123 ++++++++++ mdata/importer/chunk_encoder.go | 2 +- mdata/importer/chunk_encoder_test.go | 4 +- mdata/importer/converter.go | 12 +- mdata/importer/converter_test.go | 58 ++--- mdata/importer/cwr.go | 58 ----- mdata/importer/cwr_gen.go | 259 --------------------- mdata/importer/cwr_gen_test.go | 113 --------- 11 files changed, 479 insertions(+), 536 deletions(-) create mode 100644 mdata/importer/archive_request.go create mode 100644 mdata/importer/archive_request_gen.go create mode 100644 mdata/importer/archive_request_gen_test.go diff --git a/cmd/mt-whisper-importer-reader/main.go b/cmd/mt-whisper-importer-reader/main.go index 83d3037009..ea91022312 100644 --- a/cmd/mt-whisper-importer-reader/main.go +++ b/cmd/mt-whisper-importer-reader/main.go @@ -21,7 +21,6 @@ import ( "github.com/grafana/metrictank/logger" "github.com/grafana/metrictank/mdata/importer" "github.com/kisielk/whisper-go/whisper" - "github.com/raintank/schema" log "github.com/sirupsen/logrus" ) @@ -160,7 +159,7 @@ func processFromChan(pos *posTracker, files chan string, wg *sync.WaitGroup) { name := getMetricName(file) log.Debugf("Processing file %s (%s)", file, name) - data, err := getMetric(w, file, name) + data, err := importer.NewArchiveRequest(w, schemas, file, name, uint32(*importFrom), uint32(*importUntil), *writeUnfinishedChunks) if err != nil { log.Errorf("Failed to get metric: %q", err.Error()) continue @@ -239,72 +238,6 @@ func getMetricName(file string) string { return *namePrefix + strings.Replace(strings.TrimSuffix(file, ".wsp"), "/", ".", -1) } -func getMetric(w *whisper.Whisper, file, name string) (*importer.ArchiveRequest, error) { - if len(w.Header.Archives) == 0 { - return nil, fmt.Errorf("Whisper file contains no archives: %q", file) - } - - method, err := importer.ConvertWhisperMethod(w.Header.Metadata.AggregationMethod) - if err != nil { - return nil, err - } - - points := make(map[int][]whisper.Point) - for i := range w.Header.Archives { - p, err := w.DumpArchive(i) - if err != nil { - return nil, fmt.Errorf("Failed to dump archive %d from whisper file %s", i, file) - } - points[i] = p - } - - res := &importer.ArchiveRequest{ - MetricData: schema.MetricData{ - Name: name, - Value: 0, - Interval: int(w.Header.Archives[0].SecondsPerPoint), - Unit: "unknown", - Time: 0, - Mtype: "gauge", - Tags: []string{}, - }, - } - res.MetricData.SetId() - - _, selectedSchema := schemas.Match(res.MetricData.Name, int(w.Header.Archives[0].SecondsPerPoint)) - converter := importer.NewConverter(w.Header.Archives, points, method, uint32(*importFrom), uint32(*importUntil)) - for retIdx, retention := range selectedSchema.Retentions { - convertedPoints := converter.GetPoints(retIdx, uint32(retention.SecondsPerPoint), uint32(retention.NumberOfPoints)) - for m, p := range convertedPoints { - if len(p) == 0 { - continue - } - - var archive schema.Archive - if retIdx > 0 { - archive = schema.NewArchive(m, retention.ChunkSpan) - } - - encodedChunks := importer.EncodeChunksFromPoints(p, uint32(retention.SecondsPerPoint), retention.ChunkSpan, *writeUnfinishedChunks) - for _, chunk := range encodedChunks { - res.ChunkWriteRequests = append(res.ChunkWriteRequests, importer.NewChunkWriteRequest( - archive, - uint32(retention.MaxRetention()), - chunk.Series.T0, - chunk.Encode(retention.ChunkSpan), - time.Now(), - )) - } - - if res.MetricData.Time < int64(p[len(p)-1].Timestamp) { - res.MetricData.Time = int64(p[len(p)-1].Timestamp) - } - } - } - - return res, nil -} - // scan a directory and feed the list of whisper files relative to base into the given channel func getFileListIntoChan(pos *posTracker, fileChan chan string) { filepath.Walk( diff --git a/mdata/importer/archive_request.go b/mdata/importer/archive_request.go new file mode 100644 index 0000000000..3a0117a5e9 --- /dev/null +++ b/mdata/importer/archive_request.go @@ -0,0 +1,135 @@ +package importer + +import ( + "bufio" + "bytes" + "compress/gzip" + "fmt" + "github.com/tinylib/msgp/msgp" + "io" + "time" + + "github.com/grafana/metrictank/conf" + "github.com/kisielk/whisper-go/whisper" + "github.com/raintank/schema" +) + +//go:generate msgp + +// ArchiveRequest is a complete representation of a Metric together with some +// chunk write requests containing data that shall be written into this metric +type ArchiveRequest struct { + MetricData schema.MetricData + ChunkWriteRequests []ChunkWriteRequest +} + +func NewArchiveRequest(w *whisper.Whisper, schemas conf.Schemas, file, name string, from, until uint32, writeUnfinishedChunks bool) (*ArchiveRequest, error) { + if len(w.Header.Archives) == 0 { + return nil, fmt.Errorf("Whisper file contains no archives: %q", file) + } + + method, err := convertWhisperMethod(w.Header.Metadata.AggregationMethod) + if err != nil { + return nil, err + } + + points := make(map[int][]whisper.Point) + for i := range w.Header.Archives { + p, err := w.DumpArchive(i) + if err != nil { + return nil, fmt.Errorf("Failed to dump archive %d from whisper file %s", i, file) + } + points[i] = p + } + + res := &ArchiveRequest{ + MetricData: schema.MetricData{ + Name: name, + Value: 0, + Interval: int(w.Header.Archives[0].SecondsPerPoint), + Unit: "unknown", + Time: 0, + Mtype: "gauge", + Tags: []string{}, + }, + } + res.MetricData.SetId() + + _, selectedSchema := schemas.Match(res.MetricData.Name, int(w.Header.Archives[0].SecondsPerPoint)) + converter := newConverter(w.Header.Archives, points, method, from, until) + for retIdx, retention := range selectedSchema.Retentions { + convertedPoints := converter.getPoints(retIdx, uint32(retention.SecondsPerPoint), uint32(retention.NumberOfPoints)) + for m, p := range convertedPoints { + if len(p) == 0 { + continue + } + + var archive schema.Archive + if retIdx > 0 { + archive = schema.NewArchive(m, retention.ChunkSpan) + } + + encodedChunks := encodeChunksFromPoints(p, uint32(retention.SecondsPerPoint), retention.ChunkSpan, writeUnfinishedChunks) + for _, chunk := range encodedChunks { + res.ChunkWriteRequests = append(res.ChunkWriteRequests, NewChunkWriteRequest( + archive, + uint32(retention.MaxRetention()), + chunk.Series.T0, + chunk.Encode(retention.ChunkSpan), + time.Now(), + )) + } + + if res.MetricData.Time < int64(p[len(p)-1].Timestamp) { + res.MetricData.Time = int64(p[len(p)-1].Timestamp) + } + } + } + + return res, nil +} + +func (a *ArchiveRequest) MarshalCompressed() (*bytes.Buffer, error) { + var buf bytes.Buffer + + buf.WriteByte(byte(uint8(1))) + + g := gzip.NewWriter(&buf) + err := msgp.Encode(g, a) + if err != nil { + return &buf, fmt.Errorf("ERROR: Encoding MGSP data: %q", err) + } + + err = g.Close() + if err != nil { + return &buf, fmt.Errorf("ERROR: Compressing MSGP data: %q", err) + } + + return &buf, nil +} + +func (a *ArchiveRequest) UnmarshalCompressed(b io.Reader) error { + versionBuf := make([]byte, 1) + readBytes, err := b.Read(versionBuf) + if err != nil || readBytes != 1 { + return fmt.Errorf("ERROR: Failed to read one byte: %s", err) + } + + version := uint8(versionBuf[0]) + if version != 1 { + return fmt.Errorf("ERROR: Only version 1 is supported, received version %d", version) + } + + gzipReader, err := gzip.NewReader(b) + if err != nil { + return fmt.Errorf("ERROR: Creating Gzip reader: %q", err) + } + + err = msgp.Decode(bufio.NewReader(gzipReader), a) + if err != nil { + return fmt.Errorf("ERROR: Unmarshaling Raw: %q", err) + } + gzipReader.Close() + + return nil +} diff --git a/mdata/importer/archive_request_gen.go b/mdata/importer/archive_request_gen.go new file mode 100644 index 0000000000..38e76993e5 --- /dev/null +++ b/mdata/importer/archive_request_gen.go @@ -0,0 +1,182 @@ +package importer + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *ArchiveRequest) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "MetricData": + err = z.MetricData.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "MetricData") + return + } + case "ChunkWriteRequests": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests") + return + } + if cap(z.ChunkWriteRequests) >= int(zb0002) { + z.ChunkWriteRequests = (z.ChunkWriteRequests)[:zb0002] + } else { + z.ChunkWriteRequests = make([]ChunkWriteRequest, zb0002) + } + for za0001 := range z.ChunkWriteRequests { + err = z.ChunkWriteRequests[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ArchiveRequest) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "MetricData" + err = en.Append(0x82, 0xaa, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x44, 0x61, 0x74, 0x61) + if err != nil { + return + } + err = z.MetricData.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "MetricData") + return + } + // write "ChunkWriteRequests" + err = en.Append(0xb2, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.ChunkWriteRequests))) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests") + return + } + for za0001 := range z.ChunkWriteRequests { + err = z.ChunkWriteRequests[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ArchiveRequest) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "MetricData" + o = append(o, 0x82, 0xaa, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x44, 0x61, 0x74, 0x61) + o, err = z.MetricData.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "MetricData") + return + } + // string "ChunkWriteRequests" + o = append(o, 0xb2, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.ChunkWriteRequests))) + for za0001 := range z.ChunkWriteRequests { + o, err = z.ChunkWriteRequests[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ArchiveRequest) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "MetricData": + bts, err = z.MetricData.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "MetricData") + return + } + case "ChunkWriteRequests": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests") + return + } + if cap(z.ChunkWriteRequests) >= int(zb0002) { + z.ChunkWriteRequests = (z.ChunkWriteRequests)[:zb0002] + } else { + z.ChunkWriteRequests = make([]ChunkWriteRequest, zb0002) + } + for za0001 := range z.ChunkWriteRequests { + bts, err = z.ChunkWriteRequests[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ChunkWriteRequests", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ArchiveRequest) Msgsize() (s int) { + s = 1 + 11 + z.MetricData.Msgsize() + 19 + msgp.ArrayHeaderSize + for za0001 := range z.ChunkWriteRequests { + s += z.ChunkWriteRequests[za0001].Msgsize() + } + return +} diff --git a/mdata/importer/archive_request_gen_test.go b/mdata/importer/archive_request_gen_test.go new file mode 100644 index 0000000000..722715068d --- /dev/null +++ b/mdata/importer/archive_request_gen_test.go @@ -0,0 +1,123 @@ +package importer + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalArchiveRequest(t *testing.T) { + v := ArchiveRequest{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgArchiveRequest(b *testing.B) { + v := ArchiveRequest{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgArchiveRequest(b *testing.B) { + v := ArchiveRequest{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalArchiveRequest(b *testing.B) { + v := ArchiveRequest{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeArchiveRequest(t *testing.T) { + v := ArchiveRequest{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) + } + + vn := ArchiveRequest{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeArchiveRequest(b *testing.B) { + v := ArchiveRequest{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeArchiveRequest(b *testing.B) { + v := ArchiveRequest{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/mdata/importer/chunk_encoder.go b/mdata/importer/chunk_encoder.go index 94f42b9928..a04d5f6e93 100644 --- a/mdata/importer/chunk_encoder.go +++ b/mdata/importer/chunk_encoder.go @@ -7,7 +7,7 @@ import ( "github.com/kisielk/whisper-go/whisper" ) -func EncodeChunksFromPoints(points []whisper.Point, intervalIn, chunkSpan uint32, writeUnfinishedChunks bool) []*chunk.Chunk { +func encodeChunksFromPoints(points []whisper.Point, intervalIn, chunkSpan uint32, writeUnfinishedChunks bool) []*chunk.Chunk { var point whisper.Point var t0, prevT0 uint32 var c *chunk.Chunk diff --git a/mdata/importer/chunk_encoder_test.go b/mdata/importer/chunk_encoder_test.go index b964376c59..fba8b57cea 100644 --- a/mdata/importer/chunk_encoder_test.go +++ b/mdata/importer/chunk_encoder_test.go @@ -11,7 +11,7 @@ func TestEncodedChunksFromPointsWithUnfinished(t *testing.T) { points := generatePoints(25200, 10, 10, 0, 8640, func(i float64) float64 { return i + 1 }) expectedCount := 8640 // count including unfinished chunks - chunks := EncodeChunksFromPoints(points, 10, 21600, true) + chunks := encodeChunksFromPoints(points, 10, 21600, true) if len(chunks) != 5 { t.Fatalf("Expected to get 5 chunks, but got %d", len(chunks)) @@ -48,7 +48,7 @@ func TestEncodedChunksFromPointsWithoutUnfinished(t *testing.T) { points := generatePoints(25200, 10, 10, 0, 8640, func(i float64) float64 { return i + 1 }) expectedCount := 8640 - (2520 % 2160) // count minus what would end up in an unfinished chunk - chunks := EncodeChunksFromPoints(points, 10, 21600, false) + chunks := encodeChunksFromPoints(points, 10, 21600, false) if len(chunks) != 4 { t.Fatalf("Expected to get 4 chunks, but got %d", len(chunks)) diff --git a/mdata/importer/converter.go b/mdata/importer/converter.go index ee254a7e2d..34b89a0e2b 100644 --- a/mdata/importer/converter.go +++ b/mdata/importer/converter.go @@ -9,7 +9,7 @@ import ( "github.com/raintank/schema" ) -type Converter struct { +type converter struct { archives []whisper.ArchiveInfo points map[int][]whisper.Point method schema.Method @@ -19,12 +19,12 @@ type Converter struct { const fakeAvg schema.Method = 255 -func NewConverter(arch []whisper.ArchiveInfo, points map[int][]whisper.Point, method schema.Method, from, until uint32) *Converter { - return &Converter{archives: arch, points: points, method: method, from: from, until: until} +func newConverter(arch []whisper.ArchiveInfo, points map[int][]whisper.Point, method schema.Method, from, until uint32) *converter { + return &converter{archives: arch, points: points, method: method, from: from, until: until} } // generates points according to specified parameters by finding and using the best archives as input -func (c *Converter) GetPoints(retIdx int, spp, nop uint32) map[schema.Method][]whisper.Point { +func (c *converter) getPoints(retIdx int, spp, nop uint32) map[schema.Method][]whisper.Point { res := make(map[schema.Method][]whisper.Point) if len(c.points) == 0 { @@ -111,7 +111,7 @@ func (c *Converter) GetPoints(retIdx int, spp, nop uint32) map[schema.Method][]w return res } -func (c *Converter) findSmallestLargestArchive(spp, nop uint32) (int, int) { +func (c *converter) findSmallestLargestArchive(spp, nop uint32) (int, int) { // find smallest archive that still contains enough data to satisfy requested range largestArchiveIdx := len(c.archives) - 1 for i := largestArchiveIdx; i >= 0; i-- { @@ -282,7 +282,7 @@ func sortPoints(points pointSorter) pointSorter { return points } -func ConvertWhisperMethod(whisperMethod whisper.AggregationMethod) (schema.Method, error) { +func convertWhisperMethod(whisperMethod whisper.AggregationMethod) (schema.Method, error) { switch whisperMethod { case whisper.AggregationAverage: return schema.Avg, nil diff --git a/mdata/importer/converter_test.go b/mdata/importer/converter_test.go index 1ffb3d274e..988cc9fd41 100644 --- a/mdata/importer/converter_test.go +++ b/mdata/importer/converter_test.go @@ -704,7 +704,7 @@ func verifyPointMaps(t *testing.T, points map[schema.Method][]whisper.Point, exp } func TestPointsConversionSum1(t *testing.T) { - c := conversion{ + c := converter{ archives: []whisper.ArchiveInfo{ {SecondsPerPoint: 1, Points: 2}, {SecondsPerPoint: 2, Points: 2}, @@ -755,9 +755,9 @@ func TestPointsConversionSum1(t *testing.T) { }, } - points1 := c.GetPoints(0, 1, 8) - points2 := c.GetPoints(0, 2, 4) - points3 := c.GetPoints(0, 4, 2) + points1 := c.getPoints(0, 1, 8) + points2 := c.getPoints(0, 2, 4) + points3 := c.getPoints(0, 4, 2) verifyPointMaps(t, points1, expectedPoints1) verifyPointMaps(t, points2, expectedPoints2) @@ -765,7 +765,7 @@ func TestPointsConversionSum1(t *testing.T) { } func TestPointsConversionLast1(t *testing.T) { - c := conversion{ + c := converter{ archives: []whisper.ArchiveInfo{ {SecondsPerPoint: 1, Points: 2}, {SecondsPerPoint: 2, Points: 2}, @@ -816,9 +816,9 @@ func TestPointsConversionLast1(t *testing.T) { }, } - points1 := c.GetPoints(0, 1, 8) - points2 := c.GetPoints(0, 2, 4) - points3 := c.GetPoints(0, 4, 2) + points1 := c.getPoints(0, 1, 8) + points2 := c.getPoints(0, 2, 4) + points3 := c.getPoints(0, 4, 2) verifyPointMaps(t, points1, expectedPoints1) verifyPointMaps(t, points2, expectedPoints2) @@ -826,7 +826,7 @@ func TestPointsConversionLast1(t *testing.T) { } func TestPointsConversionSum2(t *testing.T) { - c := conversion{ + c := converter{ archives: []whisper.ArchiveInfo{ {SecondsPerPoint: 1, Points: 8}, {SecondsPerPoint: 2, Points: 8}, @@ -939,9 +939,9 @@ func TestPointsConversionSum2(t *testing.T) { }, } - points1 := c.GetPoints(0, 1, 32) - points2 := c.GetPoints(1, 2, 16) - points3 := c.GetPoints(2, 4, 8) + points1 := c.getPoints(0, 1, 32) + points2 := c.getPoints(1, 2, 16) + points3 := c.getPoints(2, 4, 8) verifyPointMaps(t, points1, expectedPoints1) verifyPointMaps(t, points2, expectedPoints2) @@ -949,7 +949,7 @@ func TestPointsConversionSum2(t *testing.T) { } func TestPointsConversionAvg1(t *testing.T) { - c := conversion{ + c := converter{ archives: []whisper.ArchiveInfo{ {SecondsPerPoint: 1, Points: 2}, {SecondsPerPoint: 2, Points: 2}, @@ -1088,18 +1088,18 @@ func TestPointsConversionAvg1(t *testing.T) { }, } - points1_0 := c.GetPoints(0, 1, 8) - points2_0 := c.GetPoints(0, 2, 4) - points3_0 := c.GetPoints(0, 4, 2) + points1_0 := c.getPoints(0, 1, 8) + points2_0 := c.getPoints(0, 2, 4) + points3_0 := c.getPoints(0, 4, 2) - points1_1 := c.GetPoints(1, 1, 8) - points2_1 := c.GetPoints(1, 2, 4) - points3_1 := c.GetPoints(1, 4, 100) + points1_1 := c.getPoints(1, 1, 8) + points2_1 := c.getPoints(1, 2, 4) + points3_1 := c.getPoints(1, 4, 100) c.until = 1503407723 - points1_2 := c.GetPoints(1, 1, 8) - points2_2 := c.GetPoints(1, 2, 100) - points3_2 := c.GetPoints(1, 4, 8) + points1_2 := c.getPoints(1, 1, 8) + points2_2 := c.getPoints(1, 2, 100) + points3_2 := c.getPoints(1, 4, 8) c.until = math.MaxUint32 verifyPointMaps(t, points1_0, expectedPoints1_0) @@ -1116,7 +1116,7 @@ func TestPointsConversionAvg1(t *testing.T) { } func TestPointsConversionAvg2(t *testing.T) { - c := conversion{ + c := converter{ archives: []whisper.ArchiveInfo{ {SecondsPerPoint: 1, Points: 3}, {SecondsPerPoint: 3, Points: 3}, @@ -1297,13 +1297,13 @@ func TestPointsConversionAvg2(t *testing.T) { }, } - points1_0 := c.GetPoints(0, 1, 27) - points2_0 := c.GetPoints(0, 3, 100) - points3_0 := c.GetPoints(0, 9, 100) + points1_0 := c.getPoints(0, 1, 27) + points2_0 := c.getPoints(0, 3, 100) + points3_0 := c.getPoints(0, 9, 100) - points1_1 := c.GetPoints(1, 1, 27) - points2_1 := c.GetPoints(1, 3, 9) - points3_1 := c.GetPoints(1, 9, 3) + points1_1 := c.getPoints(1, 1, 27) + points2_1 := c.getPoints(1, 3, 9) + points3_1 := c.getPoints(1, 9, 3) verifyPointMaps(t, points1_0, expectedPoints1_0) verifyPointMaps(t, points2_0, expectedPoints2_0) diff --git a/mdata/importer/cwr.go b/mdata/importer/cwr.go index 36f148fd41..5ae244da9e 100644 --- a/mdata/importer/cwr.go +++ b/mdata/importer/cwr.go @@ -1,16 +1,10 @@ package importer import ( - "bufio" - "bytes" - "compress/gzip" - "fmt" - "io" "time" "github.com/grafana/metrictank/mdata" "github.com/raintank/schema" - "github.com/tinylib/msgp/msgp" ) //go:generate msgp @@ -29,55 +23,3 @@ func NewChunkWriteRequest(archive schema.Archive, ttl, t0 uint32, data []byte, t func (c *ChunkWriteRequest) GetChunkWriteRequest(callback mdata.ChunkSaveCallback, key schema.MKey) mdata.ChunkWriteRequest { return mdata.ChunkWriteRequest{c.ChunkWriteRequestPayload, callback, schema.AMKey{MKey: key, Archive: c.Archive}} } - -// ArchiveRequest is a complete representation of a Metric together with some -// chunk write requests containing data that shall be written into this metric -type ArchiveRequest struct { - MetricData schema.MetricData - ChunkWriteRequests []ChunkWriteRequest -} - -func (a *ArchiveRequest) MarshalCompressed() (*bytes.Buffer, error) { - var buf bytes.Buffer - - buf.WriteByte(byte(uint8(1))) - - g := gzip.NewWriter(&buf) - err := msgp.Encode(g, a) - if err != nil { - return &buf, fmt.Errorf("ERROR: Encoding MGSP data: %q", err) - } - - err = g.Close() - if err != nil { - return &buf, fmt.Errorf("ERROR: Compressing MSGP data: %q", err) - } - - return &buf, nil -} - -func (a *ArchiveRequest) UnmarshalCompressed(b io.Reader) error { - versionBuf := make([]byte, 1) - readBytes, err := b.Read(versionBuf) - if err != nil || readBytes != 1 { - return fmt.Errorf("ERROR: Failed to read one byte: %s", err) - } - - version := uint8(versionBuf[0]) - if version != 1 { - return fmt.Errorf("ERROR: Only version 1 is supported, received version %d", version) - } - - gzipReader, err := gzip.NewReader(b) - if err != nil { - return fmt.Errorf("ERROR: Creating Gzip reader: %q", err) - } - - err = msgp.Decode(bufio.NewReader(gzipReader), a) - if err != nil { - return fmt.Errorf("ERROR: Unmarshaling Raw: %q", err) - } - gzipReader.Close() - - return nil -} diff --git a/mdata/importer/cwr_gen.go b/mdata/importer/cwr_gen.go index 2406a32d52..9d257cd591 100644 --- a/mdata/importer/cwr_gen.go +++ b/mdata/importer/cwr_gen.go @@ -6,265 +6,6 @@ import ( "github.com/tinylib/msgp/msgp" ) -// DecodeMsg implements msgp.Decodable -func (z *ArchiveRequest) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zb0001 uint32 - zb0001, err = dc.ReadMapHeader() - if err != nil { - err = msgp.WrapError(err) - return - } - for zb0001 > 0 { - zb0001-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - err = msgp.WrapError(err) - return - } - switch msgp.UnsafeString(field) { - case "MetricData": - err = z.MetricData.DecodeMsg(dc) - if err != nil { - err = msgp.WrapError(err, "MetricData") - return - } - case "ChunkWriteRequests": - var zb0002 uint32 - zb0002, err = dc.ReadArrayHeader() - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests") - return - } - if cap(z.ChunkWriteRequests) >= int(zb0002) { - z.ChunkWriteRequests = (z.ChunkWriteRequests)[:zb0002] - } else { - z.ChunkWriteRequests = make([]ChunkWriteRequest, zb0002) - } - for za0001 := range z.ChunkWriteRequests { - var zb0003 uint32 - zb0003, err = dc.ReadMapHeader() - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001) - return - } - for zb0003 > 0 { - zb0003-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001) - return - } - switch msgp.UnsafeString(field) { - case "ChunkWriteRequestPayload": - err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.DecodeMsg(dc) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") - return - } - case "Archive": - err = z.ChunkWriteRequests[za0001].Archive.DecodeMsg(dc) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") - return - } - default: - err = dc.Skip() - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001) - return - } - } - } - } - default: - err = dc.Skip() - if err != nil { - err = msgp.WrapError(err) - return - } - } - } - return -} - -// EncodeMsg implements msgp.Encodable -func (z *ArchiveRequest) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 2 - // write "MetricData" - err = en.Append(0x82, 0xaa, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x44, 0x61, 0x74, 0x61) - if err != nil { - return - } - err = z.MetricData.EncodeMsg(en) - if err != nil { - err = msgp.WrapError(err, "MetricData") - return - } - // write "ChunkWriteRequests" - err = en.Append(0xb2, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73) - if err != nil { - return - } - err = en.WriteArrayHeader(uint32(len(z.ChunkWriteRequests))) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests") - return - } - for za0001 := range z.ChunkWriteRequests { - // map header, size 2 - // write "ChunkWriteRequestPayload" - err = en.Append(0x82, 0xb8, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64) - if err != nil { - return - } - err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.EncodeMsg(en) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") - return - } - // write "Archive" - err = en.Append(0xa7, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65) - if err != nil { - return - } - err = z.ChunkWriteRequests[za0001].Archive.EncodeMsg(en) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") - return - } - } - return -} - -// MarshalMsg implements msgp.Marshaler -func (z *ArchiveRequest) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 2 - // string "MetricData" - o = append(o, 0x82, 0xaa, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x44, 0x61, 0x74, 0x61) - o, err = z.MetricData.MarshalMsg(o) - if err != nil { - err = msgp.WrapError(err, "MetricData") - return - } - // string "ChunkWriteRequests" - o = append(o, 0xb2, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73) - o = msgp.AppendArrayHeader(o, uint32(len(z.ChunkWriteRequests))) - for za0001 := range z.ChunkWriteRequests { - // map header, size 2 - // string "ChunkWriteRequestPayload" - o = append(o, 0x82, 0xb8, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64) - o, err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.MarshalMsg(o) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") - return - } - // string "Archive" - o = append(o, 0xa7, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65) - o, err = z.ChunkWriteRequests[za0001].Archive.MarshalMsg(o) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") - return - } - } - return -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *ArchiveRequest) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zb0001 uint32 - zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - for zb0001 > 0 { - zb0001-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - switch msgp.UnsafeString(field) { - case "MetricData": - bts, err = z.MetricData.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "MetricData") - return - } - case "ChunkWriteRequests": - var zb0002 uint32 - zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests") - return - } - if cap(z.ChunkWriteRequests) >= int(zb0002) { - z.ChunkWriteRequests = (z.ChunkWriteRequests)[:zb0002] - } else { - z.ChunkWriteRequests = make([]ChunkWriteRequest, zb0002) - } - for za0001 := range z.ChunkWriteRequests { - var zb0003 uint32 - zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001) - return - } - for zb0003 > 0 { - zb0003-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001) - return - } - switch msgp.UnsafeString(field) { - case "ChunkWriteRequestPayload": - bts, err = z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "ChunkWriteRequestPayload") - return - } - case "Archive": - bts, err = z.ChunkWriteRequests[za0001].Archive.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001, "Archive") - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - err = msgp.WrapError(err, "ChunkWriteRequests", za0001) - return - } - } - } - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - } - } - o = bts - return -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *ArchiveRequest) Msgsize() (s int) { - s = 1 + 11 + z.MetricData.Msgsize() + 19 + msgp.ArrayHeaderSize - for za0001 := range z.ChunkWriteRequests { - s += 1 + 25 + z.ChunkWriteRequests[za0001].ChunkWriteRequestPayload.Msgsize() + 8 + z.ChunkWriteRequests[za0001].Archive.Msgsize() - } - return -} - // DecodeMsg implements msgp.Decodable func (z *ChunkWriteRequest) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte diff --git a/mdata/importer/cwr_gen_test.go b/mdata/importer/cwr_gen_test.go index 168ed0dc91..36a1846fa2 100644 --- a/mdata/importer/cwr_gen_test.go +++ b/mdata/importer/cwr_gen_test.go @@ -9,119 +9,6 @@ import ( "github.com/tinylib/msgp/msgp" ) -func TestMarshalUnmarshalArchiveRequest(t *testing.T) { - v := ArchiveRequest{} - bts, err := v.MarshalMsg(nil) - if err != nil { - t.Fatal(err) - } - left, err := v.UnmarshalMsg(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) - } - - left, err = msgp.Skip(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after Skip(): %q", len(left), left) - } -} - -func BenchmarkMarshalMsgArchiveRequest(b *testing.B) { - v := ArchiveRequest{} - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - v.MarshalMsg(nil) - } -} - -func BenchmarkAppendMsgArchiveRequest(b *testing.B) { - v := ArchiveRequest{} - bts := make([]byte, 0, v.Msgsize()) - bts, _ = v.MarshalMsg(bts[0:0]) - b.SetBytes(int64(len(bts))) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - bts, _ = v.MarshalMsg(bts[0:0]) - } -} - -func BenchmarkUnmarshalArchiveRequest(b *testing.B) { - v := ArchiveRequest{} - bts, _ := v.MarshalMsg(nil) - b.ReportAllocs() - b.SetBytes(int64(len(bts))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := v.UnmarshalMsg(bts) - if err != nil { - b.Fatal(err) - } - } -} - -func TestEncodeDecodeArchiveRequest(t *testing.T) { - v := ArchiveRequest{} - var buf bytes.Buffer - msgp.Encode(&buf, &v) - - m := v.Msgsize() - if buf.Len() > m { - t.Logf("WARNING: Msgsize() for %v is inaccurate", v) - } - - vn := ArchiveRequest{} - err := msgp.Decode(&buf, &vn) - if err != nil { - t.Error(err) - } - - buf.Reset() - msgp.Encode(&buf, &v) - err = msgp.NewReader(&buf).Skip() - if err != nil { - t.Error(err) - } -} - -func BenchmarkEncodeArchiveRequest(b *testing.B) { - v := ArchiveRequest{} - var buf bytes.Buffer - msgp.Encode(&buf, &v) - b.SetBytes(int64(buf.Len())) - en := msgp.NewWriter(msgp.Nowhere) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - v.EncodeMsg(en) - } - en.Flush() -} - -func BenchmarkDecodeArchiveRequest(b *testing.B) { - v := ArchiveRequest{} - var buf bytes.Buffer - msgp.Encode(&buf, &v) - b.SetBytes(int64(buf.Len())) - rd := msgp.NewEndlessReader(buf.Bytes(), b) - dc := msgp.NewReader(rd) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - err := v.DecodeMsg(dc) - if err != nil { - b.Fatal(err) - } - } -} - func TestMarshalUnmarshalChunkWriteRequest(t *testing.T) { v := ChunkWriteRequest{} bts, err := v.MarshalMsg(nil)