From ac80c69ec175c5347d7c2d4f28fab5e502dd2a79 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Mon, 11 Jan 2021 14:00:46 +0200 Subject: [PATCH 01/22] add benchmark --- .../index/convert/convert_benchmark_test.go | 227 ++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 src/dbnode/storage/index/convert/convert_benchmark_test.go diff --git a/src/dbnode/storage/index/convert/convert_benchmark_test.go b/src/dbnode/storage/index/convert/convert_benchmark_test.go new file mode 100644 index 0000000000..4097bf8216 --- /dev/null +++ b/src/dbnode/storage/index/convert/convert_benchmark_test.go @@ -0,0 +1,227 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package convert + +import ( + "encoding/base64" + "math/rand" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/m3db/m3/src/x/checked" + "github.com/m3db/m3/src/x/ident" + "github.com/m3db/m3/src/x/pool" + "github.com/m3db/m3/src/x/serialize" +) + +type idWithEncodedTags struct { + id ident.ID + encodedTags []byte +} + +type idWithTags struct { + id ident.ID + tags ident.Tags +} + +// Samples of series IDs with corresponding tags. Taken from metrics generated by promremotebench +//nolint:lll +var samples = []struct { + id string + tags string +}{ + { + id: "e19fbmFtZV9fPSJkaXNraW8iLGFyY2g9Ing2NCIsZGF0YWNlbnRlcj0idXMtd2VzdC0yYyIsaG9zdG5hbWU9Imhvc3RfNzgiLG1lYXN1cmVtZW50PSJyZWFkcyIsb3M9IlVidW50dTE1LjEwIixyYWNrPSI4NyIscmVnaW9uPSJ1cy13ZXN0LTIiLHNlcnZpY2U9IjExIixzZXJ2aWNlX2Vudmlyb25tZW50PSJwcm9kdWN0aW9uIixzZXJ2aWNlX3ZlcnNpb249IjEiLHRlYW09IlNGIn0=", + tags: "dScMAAgAX19uYW1lX18GAGRpc2tpbwQAYXJjaAMAeDY0CgBkYXRhY2VudGVyCgB1cy13ZXN0LTJjCABob3N0bmFtZQcAaG9zdF83OAsAbWVhc3VyZW1lbnQFAHJlYWRzAgBvcwsAVWJ1bnR1MTUuMTAEAHJhY2sCADg3BgByZWdpb24JAHVzLXdlc3QtMgcAc2VydmljZQIAMTETAHNlcnZpY2VfZW52aXJvbm1lbnQKAHByb2R1Y3Rpb24PAHNlcnZpY2VfdmVyc2lvbgEAMQQAdGVhbQIAU0Y=", + }, + { + id: "e19fbmFtZV9fPSJuZ2lueCIsYXJjaD0ieDY0IixkYXRhY2VudGVyPSJ1cy13ZXN0LTFhIixob3N0bmFtZT0iaG9zdF8zNyIsbWVhc3VyZW1lbnQ9ImFjdGl2ZSIsb3M9IlVidW50dTE2LjEwIixyYWNrPSI3OCIscmVnaW9uPSJ1cy13ZXN0LTEiLHNlcnZpY2U9IjEwIixzZXJ2aWNlX2Vudmlyb25tZW50PSJ0ZXN0IixzZXJ2aWNlX3ZlcnNpb249IjAiLHRlYW09IkxPTiJ9", + tags: "dScMAAgAX19uYW1lX18FAG5naW54BABhcmNoAwB4NjQKAGRhdGFjZW50ZXIKAHVzLXdlc3QtMWEIAGhvc3RuYW1lBwBob3N0XzM3CwBtZWFzdXJlbWVudAYAYWN0aXZlAgBvcwsAVWJ1bnR1MTYuMTAEAHJhY2sCADc4BgByZWdpb24JAHVzLXdlc3QtMQcAc2VydmljZQIAMTATAHNlcnZpY2VfZW52aXJvbm1lbnQEAHRlc3QPAHNlcnZpY2VfdmVyc2lvbgEAMAQAdGVhbQMATE9O", + }, + { + id: "e19fbmFtZV9fPSJkaXNrIixhcmNoPSJ4NjQiLGRhdGFjZW50ZXI9InNhLWVhc3QtMWIiLGhvc3RuYW1lPSJob3N0XzU0IixtZWFzdXJlbWVudD0iaW5vZGVzX3RvdGFsIixvcz0iVWJ1bnR1MTYuMTAiLHJhY2s9Ijg4IixyZWdpb249InNhLWVhc3QtMSIsc2VydmljZT0iMTUiLHNlcnZpY2VfZW52aXJvbm1lbnQ9InByb2R1Y3Rpb24iLHNlcnZpY2VfdmVyc2lvbj0iMCIsdGVhbT0iQ0hJIn0=", + tags: "dScMAAgAX19uYW1lX18EAGRpc2sEAGFyY2gDAHg2NAoAZGF0YWNlbnRlcgoAc2EtZWFzdC0xYggAaG9zdG5hbWUHAGhvc3RfNTQLAG1lYXN1cmVtZW50DABpbm9kZXNfdG90YWwCAG9zCwBVYnVudHUxNi4xMAQAcmFjawIAODgGAHJlZ2lvbgkAc2EtZWFzdC0xBwBzZXJ2aWNlAgAxNRMAc2VydmljZV9lbnZpcm9ubWVudAoAcHJvZHVjdGlvbg8Ac2VydmljZV92ZXJzaW9uAQAwBAB0ZWFtAwBDSEk=", + }, + { + id: "e19fbmFtZV9fPSJuZXQiLGFyY2g9Ing4NiIsZGF0YWNlbnRlcj0idXMtZWFzdC0xYiIsaG9zdG5hbWU9Imhvc3RfOTMiLG1lYXN1cmVtZW50PSJlcnJfaW4iLG9zPSJVYnVudHUxNS4xMCIscmFjaz0iMzciLHJlZ2lvbj0idXMtZWFzdC0xIixzZXJ2aWNlPSIxMiIsc2VydmljZV9lbnZpcm9ubWVudD0icHJvZHVjdGlvbiIsc2VydmljZV92ZXJzaW9uPSIxIix0ZWFtPSJDSEkifQ==", + tags: "dScMAAgAX19uYW1lX18DAG5ldAQAYXJjaAMAeDg2CgBkYXRhY2VudGVyCgB1cy1lYXN0LTFiCABob3N0bmFtZQcAaG9zdF85MwsAbWVhc3VyZW1lbnQGAGVycl9pbgIAb3MLAFVidW50dTE1LjEwBAByYWNrAgAzNwYAcmVnaW9uCQB1cy1lYXN0LTEHAHNlcnZpY2UCADEyEwBzZXJ2aWNlX2Vudmlyb25tZW50CgBwcm9kdWN0aW9uDwBzZXJ2aWNlX3ZlcnNpb24BADEEAHRlYW0DAENISQ==", + }, + { + id: "e19fbmFtZV9fPSJyZWRpcyIsYXJjaD0ieDg2IixkYXRhY2VudGVyPSJldS1jZW50cmFsLTFhIixob3N0bmFtZT0iaG9zdF83MCIsbWVhc3VyZW1lbnQ9ImtleXNwYWNlX21pc3NlcyIsb3M9IlVidW50dTE2LjA0TFRTIixyYWNrPSI0NyIscmVnaW9uPSJldS1jZW50cmFsLTEiLHNlcnZpY2U9IjEyIixzZXJ2aWNlX2Vudmlyb25tZW50PSJzdGFnaW5nIixzZXJ2aWNlX3ZlcnNpb249IjEiLHRlYW09IkxPTiJ9", + tags: "dScMAAgAX19uYW1lX18FAHJlZGlzBABhcmNoAwB4ODYKAGRhdGFjZW50ZXINAGV1LWNlbnRyYWwtMWEIAGhvc3RuYW1lBwBob3N0XzcwCwBtZWFzdXJlbWVudA8Aa2V5c3BhY2VfbWlzc2VzAgBvcw4AVWJ1bnR1MTYuMDRMVFMEAHJhY2sCADQ3BgByZWdpb24MAGV1LWNlbnRyYWwtMQcAc2VydmljZQIAMTITAHNlcnZpY2VfZW52aXJvbm1lbnQHAHN0YWdpbmcPAHNlcnZpY2VfdmVyc2lvbgEAMQQAdGVhbQMATE9O", + }, + { + id: "e19fbmFtZV9fPSJuZ2lueCIsYXJjaD0ieDg2IixkYXRhY2VudGVyPSJ1cy1lYXN0LTFiIixob3N0bmFtZT0iaG9zdF84NCIsbWVhc3VyZW1lbnQ9InJlcXVlc3RzIixvcz0iVWJ1bnR1MTYuMDRMVFMiLHJhY2s9IjkwIixyZWdpb249InVzLWVhc3QtMSIsc2VydmljZT0iMTMiLHNlcnZpY2VfZW52aXJvbm1lbnQ9InRlc3QiLHNlcnZpY2VfdmVyc2lvbj0iMCIsdGVhbT0iTllDIn0=", + tags: "dScMAAgAX19uYW1lX18FAG5naW54BABhcmNoAwB4ODYKAGRhdGFjZW50ZXIKAHVzLWVhc3QtMWIIAGhvc3RuYW1lBwBob3N0Xzg0CwBtZWFzdXJlbWVudAgAcmVxdWVzdHMCAG9zDgBVYnVudHUxNi4wNExUUwQAcmFjawIAOTAGAHJlZ2lvbgkAdXMtZWFzdC0xBwBzZXJ2aWNlAgAxMxMAc2VydmljZV9lbnZpcm9ubWVudAQAdGVzdA8Ac2VydmljZV92ZXJzaW9uAQAwBAB0ZWFtAwBOWUM=", + }, + { + id: "e19fbmFtZV9fPSJtZW0iLGFyY2g9Ing2NCIsZGF0YWNlbnRlcj0iZXUtY2VudHJhbC0xYiIsaG9zdG5hbWU9Imhvc3RfMjciLG1lYXN1cmVtZW50PSJidWZmZXJlZCIsb3M9IlVidW50dTE2LjA0TFRTIixyYWNrPSI1OCIscmVnaW9uPSJldS1jZW50cmFsLTEiLHNlcnZpY2U9IjAiLHNlcnZpY2VfZW52aXJvbm1lbnQ9InRlc3QiLHNlcnZpY2VfdmVyc2lvbj0iMCIsdGVhbT0iTllDIn0=", + tags: "dScMAAgAX19uYW1lX18DAG1lbQQAYXJjaAMAeDY0CgBkYXRhY2VudGVyDQBldS1jZW50cmFsLTFiCABob3N0bmFtZQcAaG9zdF8yNwsAbWVhc3VyZW1lbnQIAGJ1ZmZlcmVkAgBvcw4AVWJ1bnR1MTYuMDRMVFMEAHJhY2sCADU4BgByZWdpb24MAGV1LWNlbnRyYWwtMQcAc2VydmljZQEAMBMAc2VydmljZV9lbnZpcm9ubWVudAQAdGVzdA8Ac2VydmljZV92ZXJzaW9uAQAwBAB0ZWFtAwBOWUM=", + }, + { + id: "e19fbmFtZV9fPSJrZXJuZWwiLGFyY2g9Ing4NiIsZGF0YWNlbnRlcj0idXMtd2VzdC0yYSIsaG9zdG5hbWU9Imhvc3RfODAiLG1lYXN1cmVtZW50PSJkaXNrX3BhZ2VzX2luIixvcz0iVWJ1bnR1MTYuMTAiLHJhY2s9IjQyIixyZWdpb249InVzLXdlc3QtMiIsc2VydmljZT0iMTMiLHNlcnZpY2VfZW52aXJvbm1lbnQ9InRlc3QiLHNlcnZpY2VfdmVyc2lvbj0iMSIsdGVhbT0iU0YifQ==", + tags: "dScMAAgAX19uYW1lX18GAGtlcm5lbAQAYXJjaAMAeDg2CgBkYXRhY2VudGVyCgB1cy13ZXN0LTJhCABob3N0bmFtZQcAaG9zdF84MAsAbWVhc3VyZW1lbnQNAGRpc2tfcGFnZXNfaW4CAG9zCwBVYnVudHUxNi4xMAQAcmFjawIANDIGAHJlZ2lvbgkAdXMtd2VzdC0yBwBzZXJ2aWNlAgAxMxMAc2VydmljZV9lbnZpcm9ubWVudAQAdGVzdA8Ac2VydmljZV92ZXJzaW9uAQAxBAB0ZWFtAgBTRg==", + }, + { + id: "e19fbmFtZV9fPSJkaXNrIixhcmNoPSJ4NjQiLGRhdGFjZW50ZXI9ImFwLW5vcnRoZWFzdC0xYyIsaG9zdG5hbWU9Imhvc3RfNzciLG1lYXN1cmVtZW50PSJpbm9kZXNfdXNlZCIsb3M9IlVidW50dTE2LjA0TFRTIixyYWNrPSI4NCIscmVnaW9uPSJhcC1ub3J0aGVhc3QtMSIsc2VydmljZT0iNSIsc2VydmljZV9lbnZpcm9ubWVudD0icHJvZHVjdGlvbiIsc2VydmljZV92ZXJzaW9uPSIwIix0ZWFtPSJMT04ifQ==", + tags: "dScMAAgAX19uYW1lX18EAGRpc2sEAGFyY2gDAHg2NAoAZGF0YWNlbnRlcg8AYXAtbm9ydGhlYXN0LTFjCABob3N0bmFtZQcAaG9zdF83NwsAbWVhc3VyZW1lbnQLAGlub2Rlc191c2VkAgBvcw4AVWJ1bnR1MTYuMDRMVFMEAHJhY2sCADg0BgByZWdpb24OAGFwLW5vcnRoZWFzdC0xBwBzZXJ2aWNlAQA1EwBzZXJ2aWNlX2Vudmlyb25tZW50CgBwcm9kdWN0aW9uDwBzZXJ2aWNlX3ZlcnNpb24BADAEAHRlYW0DAExPTg==", + }, + { + id: "e19fbmFtZV9fPSJwb3N0Z3Jlc2wiLGFyY2g9Ing2NCIsZGF0YWNlbnRlcj0iZXUtY2VudHJhbC0xYiIsaG9zdG5hbWU9Imhvc3RfMjciLG1lYXN1cmVtZW50PSJ4YWN0X3JvbGxiYWNrIixvcz0iVWJ1bnR1MTYuMDRMVFMiLHJhY2s9IjU4IixyZWdpb249ImV1LWNlbnRyYWwtMSIsc2VydmljZT0iMCIsc2VydmljZV9lbnZpcm9ubWVudD0idGVzdCIsc2VydmljZV92ZXJzaW9uPSIwIix0ZWFtPSJOWUMifQ==", + tags: "dScMAAgAX19uYW1lX18JAHBvc3RncmVzbAQAYXJjaAMAeDY0CgBkYXRhY2VudGVyDQBldS1jZW50cmFsLTFiCABob3N0bmFtZQcAaG9zdF8yNwsAbWVhc3VyZW1lbnQNAHhhY3Rfcm9sbGJhY2sCAG9zDgBVYnVudHUxNi4wNExUUwQAcmFjawIANTgGAHJlZ2lvbgwAZXUtY2VudHJhbC0xBwBzZXJ2aWNlAQAwEwBzZXJ2aWNlX2Vudmlyb25tZW50BAB0ZXN0DwBzZXJ2aWNlX3ZlcnNpb24BADAEAHRlYW0DAE5ZQw==", + }, + { + id: "e19fbmFtZV9fPSJjcHUiLGFyY2g9Ing2NCIsZGF0YWNlbnRlcj0ic2EtZWFzdC0xYiIsaG9zdG5hbWU9Imhvc3RfNDMiLG1lYXN1cmVtZW50PSJ1c2FnZV9uaWNlIixvcz0iVWJ1bnR1MTYuMTAiLHJhY2s9Ijk1IixyZWdpb249InNhLWVhc3QtMSIsc2VydmljZT0iNCIsc2VydmljZV9lbnZpcm9ubWVudD0idGVzdCIsc2VydmljZV92ZXJzaW9uPSIwIix0ZWFtPSJTRiJ9", + tags: "dScMAAgAX19uYW1lX18DAGNwdQQAYXJjaAMAeDY0CgBkYXRhY2VudGVyCgBzYS1lYXN0LTFiCABob3N0bmFtZQcAaG9zdF80MwsAbWVhc3VyZW1lbnQKAHVzYWdlX25pY2UCAG9zCwBVYnVudHUxNi4xMAQAcmFjawIAOTUGAHJlZ2lvbgkAc2EtZWFzdC0xBwBzZXJ2aWNlAQA0EwBzZXJ2aWNlX2Vudmlyb25tZW50BAB0ZXN0DwBzZXJ2aWNlX3ZlcnNpb24BADAEAHRlYW0CAFNG", + }, + { + id: "e19fbmFtZV9fPSJkaXNrIixhcmNoPSJ4NjQiLGRhdGFjZW50ZXI9ImFwLW5vcnRoZWFzdC0xYyIsaG9zdG5hbWU9Imhvc3RfMTciLG1lYXN1cmVtZW50PSJpbm9kZXNfdG90YWwiLG9zPSJVYnVudHUxNi4xMCIscmFjaz0iOTQiLHJlZ2lvbj0iYXAtbm9ydGhlYXN0LTEiLHNlcnZpY2U9IjkiLHNlcnZpY2VfZW52aXJvbm1lbnQ9InN0YWdpbmciLHNlcnZpY2VfdmVyc2lvbj0iMCIsdGVhbT0iU0YifQ==", + tags: "dScMAAgAX19uYW1lX18EAGRpc2sEAGFyY2gDAHg2NAoAZGF0YWNlbnRlcg8AYXAtbm9ydGhlYXN0LTFjCABob3N0bmFtZQcAaG9zdF8xNwsAbWVhc3VyZW1lbnQMAGlub2Rlc190b3RhbAIAb3MLAFVidW50dTE2LjEwBAByYWNrAgA5NAYAcmVnaW9uDgBhcC1ub3J0aGVhc3QtMQcAc2VydmljZQEAORMAc2VydmljZV9lbnZpcm9ubWVudAcAc3RhZ2luZw8Ac2VydmljZV92ZXJzaW9uAQAwBAB0ZWFtAgBTRg==", + }, + { + id: "e19fbmFtZV9fPSJyZWRpcyIsYXJjaD0ieDg2IixkYXRhY2VudGVyPSJ1cy13ZXN0LTJhIixob3N0bmFtZT0iaG9zdF84MCIsbWVhc3VyZW1lbnQ9InN5bmNfcGFydGlhbF9lcnIiLG9zPSJVYnVudHUxNi4xMCIscmFjaz0iNDIiLHJlZ2lvbj0idXMtd2VzdC0yIixzZXJ2aWNlPSIxMyIsc2VydmljZV9lbnZpcm9ubWVudD0idGVzdCIsc2VydmljZV92ZXJzaW9uPSIxIix0ZWFtPSJTRiJ9", + tags: "dScMAAgAX19uYW1lX18FAHJlZGlzBABhcmNoAwB4ODYKAGRhdGFjZW50ZXIKAHVzLXdlc3QtMmEIAGhvc3RuYW1lBwBob3N0XzgwCwBtZWFzdXJlbWVudBAAc3luY19wYXJ0aWFsX2VycgIAb3MLAFVidW50dTE2LjEwBAByYWNrAgA0MgYAcmVnaW9uCQB1cy13ZXN0LTIHAHNlcnZpY2UCADEzEwBzZXJ2aWNlX2Vudmlyb25tZW50BAB0ZXN0DwBzZXJ2aWNlX3ZlcnNpb24BADEEAHRlYW0CAFNG", + }, + { + id: "e19fbmFtZV9fPSJuZXQiLGFyY2g9Ing4NiIsZGF0YWNlbnRlcj0idXMtZWFzdC0xYSIsaG9zdG5hbWU9Imhvc3RfNzkiLG1lYXN1cmVtZW50PSJkcm9wX291dCIsb3M9IlVidW50dTE2LjA0TFRTIixyYWNrPSIxNyIscmVnaW9uPSJ1cy1lYXN0LTEiLHNlcnZpY2U9IjE3IixzZXJ2aWNlX2Vudmlyb25tZW50PSJzdGFnaW5nIixzZXJ2aWNlX3ZlcnNpb249IjEiLHRlYW09IlNGIn0=", + tags: "dScMAAgAX19uYW1lX18DAG5ldAQAYXJjaAMAeDg2CgBkYXRhY2VudGVyCgB1cy1lYXN0LTFhCABob3N0bmFtZQcAaG9zdF83OQsAbWVhc3VyZW1lbnQIAGRyb3Bfb3V0AgBvcw4AVWJ1bnR1MTYuMDRMVFMEAHJhY2sCADE3BgByZWdpb24JAHVzLWVhc3QtMQcAc2VydmljZQIAMTcTAHNlcnZpY2VfZW52aXJvbm1lbnQHAHN0YWdpbmcPAHNlcnZpY2VfdmVyc2lvbgEAMQQAdGVhbQIAU0Y=", + }, + { + id: "e19fbmFtZV9fPSJyZWRpcyIsYXJjaD0ieDg2IixkYXRhY2VudGVyPSJhcC1zb3V0aGVhc3QtMmIiLGhvc3RuYW1lPSJob3N0XzEwMCIsbWVhc3VyZW1lbnQ9InVzZWRfY3B1X3VzZXJfY2hpbGRyZW4iLG9zPSJVYnVudHUxNi4wNExUUyIscmFjaz0iNDAiLHJlZ2lvbj0iYXAtc291dGhlYXN0LTIiLHNlcnZpY2U9IjE0IixzZXJ2aWNlX2Vudmlyb25tZW50PSJzdGFnaW5nIixzZXJ2aWNlX3ZlcnNpb249IjEiLHRlYW09Ik5ZQyJ9", + tags: "dScMAAgAX19uYW1lX18FAHJlZGlzBABhcmNoAwB4ODYKAGRhdGFjZW50ZXIPAGFwLXNvdXRoZWFzdC0yYggAaG9zdG5hbWUIAGhvc3RfMTAwCwBtZWFzdXJlbWVudBYAdXNlZF9jcHVfdXNlcl9jaGlsZHJlbgIAb3MOAFVidW50dTE2LjA0TFRTBAByYWNrAgA0MAYAcmVnaW9uDgBhcC1zb3V0aGVhc3QtMgcAc2VydmljZQIAMTQTAHNlcnZpY2VfZW52aXJvbm1lbnQHAHN0YWdpbmcPAHNlcnZpY2VfdmVyc2lvbgEAMQQAdGVhbQMATllD", + }, + { + id: "e19fbmFtZV9fPSJkaXNrIixhcmNoPSJ4NjQiLGRhdGFjZW50ZXI9ImFwLXNvdXRoZWFzdC0xYSIsaG9zdG5hbWU9Imhvc3RfODciLG1lYXN1cmVtZW50PSJpbm9kZXNfdG90YWwiLG9zPSJVYnVudHUxNS4xMCIscmFjaz0iMCIscmVnaW9uPSJhcC1zb3V0aGVhc3QtMSIsc2VydmljZT0iMTEiLHNlcnZpY2VfZW52aXJvbm1lbnQ9InN0YWdpbmciLHNlcnZpY2VfdmVyc2lvbj0iMCIsdGVhbT0iTE9OIn0=", + tags: "dScMAAgAX19uYW1lX18EAGRpc2sEAGFyY2gDAHg2NAoAZGF0YWNlbnRlcg8AYXAtc291dGhlYXN0LTFhCABob3N0bmFtZQcAaG9zdF84NwsAbWVhc3VyZW1lbnQMAGlub2Rlc190b3RhbAIAb3MLAFVidW50dTE1LjEwBAByYWNrAQAwBgByZWdpb24OAGFwLXNvdXRoZWFzdC0xBwBzZXJ2aWNlAgAxMRMAc2VydmljZV9lbnZpcm9ubWVudAcAc3RhZ2luZw8Ac2VydmljZV92ZXJzaW9uAQAwBAB0ZWFtAwBMT04=", + }, + { + id: "e19fbmFtZV9fPSJjcHUiLGFyY2g9Ing2NCIsZGF0YWNlbnRlcj0idXMtd2VzdC0yYSIsaG9zdG5hbWU9Imhvc3RfNiIsbWVhc3VyZW1lbnQ9InVzYWdlX2lkbGUiLG9zPSJVYnVudHUxNi4xMCIscmFjaz0iMTAiLHJlZ2lvbj0idXMtd2VzdC0yIixzZXJ2aWNlPSI2IixzZXJ2aWNlX2Vudmlyb25tZW50PSJ0ZXN0IixzZXJ2aWNlX3ZlcnNpb249IjAiLHRlYW09IkNISSJ9", + tags: "dScMAAgAX19uYW1lX18DAGNwdQQAYXJjaAMAeDY0CgBkYXRhY2VudGVyCgB1cy13ZXN0LTJhCABob3N0bmFtZQYAaG9zdF82CwBtZWFzdXJlbWVudAoAdXNhZ2VfaWRsZQIAb3MLAFVidW50dTE2LjEwBAByYWNrAgAxMAYAcmVnaW9uCQB1cy13ZXN0LTIHAHNlcnZpY2UBADYTAHNlcnZpY2VfZW52aXJvbm1lbnQEAHRlc3QPAHNlcnZpY2VfdmVyc2lvbgEAMAQAdGVhbQMAQ0hJ", + }, + { + id: "e19fbmFtZV9fPSJuZ2lueCIsYXJjaD0ieDg2IixkYXRhY2VudGVyPSJ1cy1lYXN0LTFhIixob3N0bmFtZT0iaG9zdF80NCIsbWVhc3VyZW1lbnQ9ImhhbmRsZWQiLG9zPSJVYnVudHUxNi4wNExUUyIscmFjaz0iNjEiLHJlZ2lvbj0idXMtZWFzdC0xIixzZXJ2aWNlPSIyIixzZXJ2aWNlX2Vudmlyb25tZW50PSJzdGFnaW5nIixzZXJ2aWNlX3ZlcnNpb249IjEiLHRlYW09Ik5ZQyJ9", + tags: "dScMAAgAX19uYW1lX18FAG5naW54BABhcmNoAwB4ODYKAGRhdGFjZW50ZXIKAHVzLWVhc3QtMWEIAGhvc3RuYW1lBwBob3N0XzQ0CwBtZWFzdXJlbWVudAcAaGFuZGxlZAIAb3MOAFVidW50dTE2LjA0TFRTBAByYWNrAgA2MQYAcmVnaW9uCQB1cy1lYXN0LTEHAHNlcnZpY2UBADITAHNlcnZpY2VfZW52aXJvbm1lbnQHAHN0YWdpbmcPAHNlcnZpY2VfdmVyc2lvbgEAMQQAdGVhbQMATllD", + }, + { + id: "e19fbmFtZV9fPSJuZ2lueCIsYXJjaD0ieDg2IixkYXRhY2VudGVyPSJ1cy13ZXN0LTFhIixob3N0bmFtZT0iaG9zdF8yOSIsbWVhc3VyZW1lbnQ9IndhaXRpbmciLG9zPSJVYnVudHUxNS4xMCIscmFjaz0iMTUiLHJlZ2lvbj0idXMtd2VzdC0xIixzZXJ2aWNlPSI0IixzZXJ2aWNlX2Vudmlyb25tZW50PSJ0ZXN0IixzZXJ2aWNlX3ZlcnNpb249IjEiLHRlYW09Ik5ZQyJ9", + tags: "dScMAAgAX19uYW1lX18FAG5naW54BABhcmNoAwB4ODYKAGRhdGFjZW50ZXIKAHVzLXdlc3QtMWEIAGhvc3RuYW1lBwBob3N0XzI5CwBtZWFzdXJlbWVudAcAd2FpdGluZwIAb3MLAFVidW50dTE1LjEwBAByYWNrAgAxNQYAcmVnaW9uCQB1cy13ZXN0LTEHAHNlcnZpY2UBADQTAHNlcnZpY2VfZW52aXJvbm1lbnQEAHRlc3QPAHNlcnZpY2VfdmVyc2lvbgEAMQQAdGVhbQMATllD", + }, + { + id: "e19fbmFtZV9fPSJkaXNraW8iLGFyY2g9Ing2NCIsZGF0YWNlbnRlcj0iYXAtbm9ydGhlYXN0LTFjIixob3N0bmFtZT0iaG9zdF8zOCIsbWVhc3VyZW1lbnQ9IndyaXRlX3RpbWUiLG9zPSJVYnVudHUxNS4xMCIscmFjaz0iMjAiLHJlZ2lvbj0iYXAtbm9ydGhlYXN0LTEiLHNlcnZpY2U9IjAiLHNlcnZpY2VfZW52aXJvbm1lbnQ9InN0YWdpbmciLHNlcnZpY2VfdmVyc2lvbj0iMCIsdGVhbT0iU0YifQ==", + tags: "dScMAAgAX19uYW1lX18GAGRpc2tpbwQAYXJjaAMAeDY0CgBkYXRhY2VudGVyDwBhcC1ub3J0aGVhc3QtMWMIAGhvc3RuYW1lBwBob3N0XzM4CwBtZWFzdXJlbWVudAoAd3JpdGVfdGltZQIAb3MLAFVidW50dTE1LjEwBAByYWNrAgAyMAYAcmVnaW9uDgBhcC1ub3J0aGVhc3QtMQcAc2VydmljZQEAMBMAc2VydmljZV9lbnZpcm9ubWVudAcAc3RhZ2luZw8Ac2VydmljZV92ZXJzaW9uAQAwBAB0ZWFtAgBTRg==", + }, +} + +// BenchmarkFromSeriesIDAndTagIter-12 189643 6096 ns/op +func BenchmarkFromSeriesIDAndTagIter(b *testing.B) { + testData, err := prepareIDAndEncodedTags(b) + require.NoError(b, err) + + decoderPool := serialize.NewTagDecoderPool( + serialize.NewTagDecoderOptions(serialize.TagDecoderOptionsConfig{}), + pool.NewObjectPoolOptions(), + ) + decoderPool.Init() + tagDecoder := decoderPool.Get() + defer tagDecoder.Close() + + b.ResetTimer() + for i := range testData { + tagDecoder.Reset(checked.NewBytes(testData[i].encodedTags, nil)) + + _, err := FromSeriesIDAndTagIter(testData[i].id, tagDecoder) + require.NoError(b, err) + } +} + +// BenchmarkFromSeriesIDAndTags-12 586689 2584 ns/op +func BenchmarkFromSeriesIDAndTags(b *testing.B) { + testData, err := prepareIDAndTags(b) + require.NoError(b, err) + + b.ResetTimer() + for i := range testData { + _, err := FromSeriesIDAndTags(testData[i].id, testData[i].tags) + require.NoError(b, err) + } +} + +func prepareIDAndEncodedTags(b *testing.B) ([]idWithEncodedTags, error) { + var ( + rnd = rand.New(rand.NewSource(42)) //nolint:gosec + b64 = base64.StdEncoding + result = make([]idWithEncodedTags, 0, b.N) + ) + + for i := 0; i < b.N; i++ { + k := rnd.Intn(len(samples)) + id, err := b64.DecodeString(samples[k].id) + if err != nil { + return nil, err + } + tags, err := b64.DecodeString(samples[k].tags) + if err != nil { + return nil, err + } + + result = append(result, idWithEncodedTags{ + id: ident.BytesID(id), + encodedTags: tags, + }) + } + + return result, nil +} + +func prepareIDAndTags(b *testing.B) ([]idWithTags, error) { + testData, err := prepareIDAndEncodedTags(b) + if err != nil { + return nil, err + } + + decoderPool := serialize.NewTagDecoderPool( + serialize.NewTagDecoderOptions(serialize.TagDecoderOptionsConfig{}), + pool.NewObjectPoolOptions(), + ) + decoderPool.Init() + + bytesPool := pool.NewCheckedBytesPool(nil, nil, func(s []pool.Bucket) pool.BytesPool { + return pool.NewBytesPool(s, nil) + }) + bytesPool.Init() + + identPool := ident.NewPool(bytesPool, ident.PoolOptions{}) + + tagDecoder := decoderPool.Get() + defer tagDecoder.Close() + + result := make([]idWithTags, 0, len(testData)) + for i := range testData { + tagDecoder.Reset(checked.NewBytes(testData[i].encodedTags, nil)) + tags, err := TagsFromTagsIter(testData[i].id, tagDecoder, identPool) + if err != nil { + return nil, err + } + result = append(result, idWithTags{id: testData[i].id, tags: tags}) + } + return result, nil +} From f0370d82c68b5699c6b5b5960182135a8cf4cf64 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Mon, 11 Jan 2021 14:38:59 +0200 Subject: [PATCH 02/22] speed up the search of tag bytes in series ID --- src/dbnode/storage/index/convert/convert.go | 57 +++++++++++-------- .../index/convert/convert_benchmark_test.go | 4 +- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/src/dbnode/storage/index/convert/convert.go b/src/dbnode/storage/index/convert/convert.go index e6013362ee..a59a4887db 100644 --- a/src/dbnode/storage/index/convert/convert.go +++ b/src/dbnode/storage/index/convert/convert.go @@ -32,6 +32,17 @@ import ( "github.com/m3db/m3/src/x/pool" ) +const ( + // NB: this assumes that series ID has a format: + // {tag1="value1",tag2="value2",...} + // + // Thus seriesIDFirstTagBytesIdx points to the 't' immediately after curly brace '{', and + // seriesIDDistanceBetweenTagBytes corresponds to either '="' or '",' that separate + // tag from it's value or value from the next tag. + seriesIDFirstTagBytesIdx int = 1 + seriesIDDistanceBetweenTagBytes int = 2 +) + var ( // ReservedFieldNameID is the field name used to index the ID in the // m3ninx subsytem. @@ -108,22 +119,15 @@ func ValidateSeriesTag(tag ident.Tag) error { // FromSeriesIDAndTags converts the provided series id+tags into a document. func FromSeriesIDAndTags(id ident.ID, tags ident.Tags) (doc.Metadata, error) { - clonedID := clone(id) + clonedID := clone(id.Bytes()) fields := make([]doc.Field, 0, len(tags.Values())) + expectedIdx := seriesIDFirstTagBytesIdx for _, tag := range tags.Values() { nameBytes, valueBytes := tag.Name.Bytes(), tag.Value.Bytes() var clonedName, clonedValue []byte - if idx := bytes.Index(clonedID, nameBytes); idx != -1 { - clonedName = clonedID[idx : idx+len(nameBytes)] - } else { - clonedName = append([]byte(nil), nameBytes...) - } - if idx := bytes.Index(clonedID, valueBytes); idx != -1 { - clonedValue = clonedID[idx : idx+len(valueBytes)] - } else { - clonedValue = append([]byte(nil), valueBytes...) - } + clonedName, expectedIdx = findSliceOrClone(clonedID, nameBytes, expectedIdx) + clonedValue, expectedIdx = findSliceOrClone(clonedID, valueBytes, expectedIdx) fields = append(fields, doc.Field{ Name: clonedName, @@ -143,23 +147,17 @@ func FromSeriesIDAndTags(id ident.ID, tags ident.Tags) (doc.Metadata, error) { // FromSeriesIDAndTagIter converts the provided series id+tags into a document. func FromSeriesIDAndTagIter(id ident.ID, tags ident.TagIterator) (doc.Metadata, error) { - clonedID := clone(id) + clonedID := clone(id.Bytes()) fields := make([]doc.Field, 0, tags.Remaining()) + + expectedIdx := 1 for tags.Next() { tag := tags.Current() nameBytes, valueBytes := tag.Name.Bytes(), tag.Value.Bytes() var clonedName, clonedValue []byte - if idx := bytes.Index(clonedID, nameBytes); idx != -1 { - clonedName = clonedID[idx : idx+len(nameBytes)] - } else { - clonedName = append([]byte(nil), nameBytes...) - } - if idx := bytes.Index(clonedID, valueBytes); idx != -1 { - clonedValue = clonedID[idx : idx+len(valueBytes)] - } else { - clonedValue = append([]byte(nil), valueBytes...) - } + clonedName, expectedIdx = findSliceOrClone(clonedID, nameBytes, expectedIdx) + clonedValue, expectedIdx = findSliceOrClone(clonedID, valueBytes, expectedIdx) fields = append(fields, doc.Field{ Name: clonedName, @@ -180,6 +178,18 @@ func FromSeriesIDAndTagIter(id ident.ID, tags ident.TagIterator) (doc.Metadata, return d, nil } +func findSliceOrClone(id, tag []byte, expectedIdx int) ([]byte, int) { + n := len(tag) + expectedEnd := expectedIdx + n + if expectedIdx != -1 && expectedEnd <= len(id) && bytes.Equal(id[expectedIdx:expectedEnd], tag) { + return id[expectedIdx:expectedEnd], expectedEnd + seriesIDDistanceBetweenTagBytes + } else if idx := bytes.Index(id, tag); idx != -1 { + return id[idx : idx+n], expectedEnd + seriesIDDistanceBetweenTagBytes + } else { + return clone(tag), -1 + } +} + // TagsFromTagsIter returns an ident.Tags from a TagIterator. It also tries // to re-use bytes from the seriesID if they're also present in the tags // instead of re-allocating them. This requires that the ident.Tags that is @@ -252,8 +262,7 @@ func TagsFromTagsIter( // NB(prateek): we take an independent copy of the bytes underlying // any ids provided, as we need to maintain the lifecycle of the indexed // bytes separately from the rest of the storage subsystem. -func clone(id ident.ID) []byte { - original := id.Bytes() +func clone(original []byte) []byte { clone := make([]byte, len(original)) copy(clone, original) return clone diff --git a/src/dbnode/storage/index/convert/convert_benchmark_test.go b/src/dbnode/storage/index/convert/convert_benchmark_test.go index 4097bf8216..246fab0d5f 100644 --- a/src/dbnode/storage/index/convert/convert_benchmark_test.go +++ b/src/dbnode/storage/index/convert/convert_benchmark_test.go @@ -131,7 +131,7 @@ var samples = []struct { }, } -// BenchmarkFromSeriesIDAndTagIter-12 189643 6096 ns/op +// BenchmarkFromSeriesIDAndTagIter-12 254090 4689 ns/op func BenchmarkFromSeriesIDAndTagIter(b *testing.B) { testData, err := prepareIDAndEncodedTags(b) require.NoError(b, err) @@ -153,7 +153,7 @@ func BenchmarkFromSeriesIDAndTagIter(b *testing.B) { } } -// BenchmarkFromSeriesIDAndTags-12 586689 2584 ns/op +// BenchmarkFromSeriesIDAndTags-12 1000000 1311 ns/op func BenchmarkFromSeriesIDAndTags(b *testing.B) { testData, err := prepareIDAndTags(b) require.NoError(b, err) From 5e77e4a107df4d58410fb4974199d409e3659a0b Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Mon, 11 Jan 2021 15:15:04 +0200 Subject: [PATCH 03/22] update tests --- .../storage/index/convert/convert_test.go | 80 +++++++++++++++---- 1 file changed, 64 insertions(+), 16 deletions(-) diff --git a/src/dbnode/storage/index/convert/convert_test.go b/src/dbnode/storage/index/convert/convert_test.go index 07c5d37873..7166bff2f6 100644 --- a/src/dbnode/storage/index/convert/convert_test.go +++ b/src/dbnode/storage/index/convert/convert_test.go @@ -68,37 +68,85 @@ func TestFromSeriesIDAndTagIteratorInvalid(t *testing.T) { } func TestFromSeriesIDAndTagsValid(t *testing.T) { - id := ident.StringID("foo") + tests := []struct { + name string + id string + }{ + { + name: "no tags in ID", + id: "foo", + }, + { + name: "tags in ID", + id: "bar=baz", + }, + { + name: "tags in ID with specific format", + id: `{bar="baz"}`, + }, + { + name: "inexact tag occurrence in ID", + id: "bazillion_barometers", + }, + } tags := ident.NewTags( ident.StringTag("bar", "baz"), ) - d, err := convert.FromSeriesIDAndTags(id, tags) - assert.NoError(t, err) - assert.Equal(t, "foo", string(d.ID)) - assert.Len(t, d.Fields, 1) - assert.Equal(t, "bar", string(d.Fields[0].Name)) - assert.Equal(t, "baz", string(d.Fields[0].Value)) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d, err := convert.FromSeriesIDAndTags(ident.StringID(tt.id), tags) + assert.NoError(t, err) + assert.Equal(t, tt.id, string(d.ID)) + assert.Len(t, d.Fields, 1) + assert.Equal(t, "bar", string(d.Fields[0].Name)) + assert.Equal(t, "baz", string(d.Fields[0].Value)) + }) + } } func TestFromSeriesIDAndTagIterValid(t *testing.T) { - id := ident.StringID("foo") + tests := []struct { + name string + id string + }{ + { + name: "no tags in ID", + id: "foo", + }, + { + name: "tags in ID", + id: "bar=baz", + }, + { + name: "tags in ID with specific format", + id: `{bar="baz"}`, + }, + { + name: "inexact tag occurrence in ID", + id: "bazillion_barometers", + }, + } tags := ident.NewTags( ident.StringTag("bar", "baz"), ) - d, err := convert.FromSeriesIDAndTagIter(id, ident.NewTagsIterator(tags)) - assert.NoError(t, err) - assert.Equal(t, "foo", string(d.ID)) - assert.Len(t, d.Fields, 1) - assert.Equal(t, "bar", string(d.Fields[0].Name)) - assert.Equal(t, "baz", string(d.Fields[0].Value)) + + for _, tt := range tests { + d, err := convert.FromSeriesIDAndTagIter(ident.StringID(tt.id), ident.NewTagsIterator(tags)) + assert.NoError(t, err) + assert.Equal(t, tt.id, string(d.ID)) + assert.Len(t, d.Fields, 1) + assert.Equal(t, "bar", string(d.Fields[0].Name)) + assert.Equal(t, "baz", string(d.Fields[0].Value)) + } } func TestToSeriesValid(t *testing.T) { d := doc.Metadata{ ID: []byte("foo"), Fields: []doc.Field{ - doc.Field{Name: []byte("bar"), Value: []byte("baz")}, - doc.Field{Name: []byte("some"), Value: []byte("others")}, + {Name: []byte("bar"), Value: []byte("baz")}, + {Name: []byte("some"), Value: []byte("others")}, }, } id, tags, err := convert.ToSeries(d, testOpts) From fe8ee23c07194f60a63e8ebb8aa7c79fb7fdc614 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Tue, 12 Jan 2021 10:08:46 +0200 Subject: [PATCH 04/22] PR comments --- src/dbnode/storage/index/convert/convert.go | 53 +++++++++++-------- .../index/convert/convert_benchmark_test.go | 2 +- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/src/dbnode/storage/index/convert/convert.go b/src/dbnode/storage/index/convert/convert.go index a59a4887db..023589a51e 100644 --- a/src/dbnode/storage/index/convert/convert.go +++ b/src/dbnode/storage/index/convert/convert.go @@ -36,11 +36,14 @@ const ( // NB: this assumes that series ID has a format: // {tag1="value1",tag2="value2",...} // - // Thus seriesIDFirstTagBytesIdx points to the 't' immediately after curly brace '{', and - // seriesIDDistanceBetweenTagBytes corresponds to either '="' or '",' that separate - // tag from it's value or value from the next tag. - seriesIDFirstTagBytesIdx int = 1 - seriesIDDistanceBetweenTagBytes int = 2 + // Thus firstTagBytesPosition points to the 't' immediately after curly brace '{' + firstTagBytesPosition int = 1 + // distanceBetweenTagNameAndValue corresponds to '="' in series ID that separates tag name from + // it's value + distanceBetweenTagNameAndValue int = 2 + // distanceBetweenTagValueAndNextName corresponds to '",' in series ID that separates + // tag's value from the following tag name + distanceBetweenTagValueAndNextName int = 2 ) var ( @@ -119,15 +122,19 @@ func ValidateSeriesTag(tag ident.Tag) error { // FromSeriesIDAndTags converts the provided series id+tags into a document. func FromSeriesIDAndTags(id ident.ID, tags ident.Tags) (doc.Metadata, error) { - clonedID := clone(id.Bytes()) - fields := make([]doc.Field, 0, len(tags.Values())) - expectedIdx := seriesIDFirstTagBytesIdx + var ( + clonedID = clone(id.Bytes()) + fields = make([]doc.Field, 0, len(tags.Values())) + expectedStart = firstTagBytesPosition + ) for _, tag := range tags.Values() { nameBytes, valueBytes := tag.Name.Bytes(), tag.Value.Bytes() var clonedName, clonedValue []byte - clonedName, expectedIdx = findSliceOrClone(clonedID, nameBytes, expectedIdx) - clonedValue, expectedIdx = findSliceOrClone(clonedID, valueBytes, expectedIdx) + clonedName, expectedStart = findSliceOrClone(clonedID, nameBytes, expectedStart, + distanceBetweenTagNameAndValue) + clonedValue, expectedStart = findSliceOrClone(clonedID, valueBytes, expectedStart, + distanceBetweenTagValueAndNextName) fields = append(fields, doc.Field{ Name: clonedName, @@ -147,17 +154,20 @@ func FromSeriesIDAndTags(id ident.ID, tags ident.Tags) (doc.Metadata, error) { // FromSeriesIDAndTagIter converts the provided series id+tags into a document. func FromSeriesIDAndTagIter(id ident.ID, tags ident.TagIterator) (doc.Metadata, error) { - clonedID := clone(id.Bytes()) - fields := make([]doc.Field, 0, tags.Remaining()) - - expectedIdx := 1 + var ( + clonedID = clone(id.Bytes()) + fields = make([]doc.Field, 0, tags.Remaining()) + expectedStart = firstTagBytesPosition + ) for tags.Next() { tag := tags.Current() nameBytes, valueBytes := tag.Name.Bytes(), tag.Value.Bytes() var clonedName, clonedValue []byte - clonedName, expectedIdx = findSliceOrClone(clonedID, nameBytes, expectedIdx) - clonedValue, expectedIdx = findSliceOrClone(clonedID, valueBytes, expectedIdx) + clonedName, expectedStart = findSliceOrClone(clonedID, nameBytes, expectedStart, + distanceBetweenTagNameAndValue) + clonedValue, expectedStart = findSliceOrClone(clonedID, valueBytes, expectedStart, + distanceBetweenTagValueAndNextName) fields = append(fields, doc.Field{ Name: clonedName, @@ -178,13 +188,14 @@ func FromSeriesIDAndTagIter(id ident.ID, tags ident.TagIterator) (doc.Metadata, return d, nil } -func findSliceOrClone(id, tag []byte, expectedIdx int) ([]byte, int) { +func findSliceOrClone(id, tag []byte, expectedStart, nextPositionDistance int) ([]byte, int) { //nolint:unparam n := len(tag) - expectedEnd := expectedIdx + n - if expectedIdx != -1 && expectedEnd <= len(id) && bytes.Equal(id[expectedIdx:expectedEnd], tag) { - return id[expectedIdx:expectedEnd], expectedEnd + seriesIDDistanceBetweenTagBytes + expectedEnd := expectedStart + n + if expectedStart != -1 && expectedEnd <= len(id) && + bytes.Equal(id[expectedStart:expectedEnd], tag) { + return id[expectedStart:expectedEnd], expectedEnd + nextPositionDistance } else if idx := bytes.Index(id, tag); idx != -1 { - return id[idx : idx+n], expectedEnd + seriesIDDistanceBetweenTagBytes + return id[idx : idx+n], expectedEnd + nextPositionDistance } else { return clone(tag), -1 } diff --git a/src/dbnode/storage/index/convert/convert_benchmark_test.go b/src/dbnode/storage/index/convert/convert_benchmark_test.go index 246fab0d5f..5124a0a2b1 100644 --- a/src/dbnode/storage/index/convert/convert_benchmark_test.go +++ b/src/dbnode/storage/index/convert/convert_benchmark_test.go @@ -43,7 +43,7 @@ type idWithTags struct { tags ident.Tags } -// Samples of series IDs with corresponding tags. Taken from metrics generated by promremotebench +// Samples of series IDs with corresponding tags. Taken from metrics generated by promremotebench. //nolint:lll var samples = []struct { id string From 4221b848a55e2edd8c4bba770d7a9631c1c7d0db Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Tue, 12 Jan 2021 12:06:26 +0200 Subject: [PATCH 05/22] add tests that check if bytes tag slices reuse data from series ID --- .../storage/index/convert/convert_test.go | 80 +++++++++++++------ src/x/test/util.go | 14 ++++ src/x/test/util_test.go | 61 ++++++++++++++ 3 files changed, 130 insertions(+), 25 deletions(-) create mode 100644 src/x/test/util_test.go diff --git a/src/dbnode/storage/index/convert/convert_test.go b/src/dbnode/storage/index/convert/convert_test.go index 7166bff2f6..a95359d07d 100644 --- a/src/dbnode/storage/index/convert/convert_test.go +++ b/src/dbnode/storage/index/convert/convert_test.go @@ -29,6 +29,7 @@ import ( "github.com/m3db/m3/src/x/checked" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/pool" + "github.com/m3db/m3/src/x/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -68,76 +69,96 @@ func TestFromSeriesIDAndTagIteratorInvalid(t *testing.T) { } func TestFromSeriesIDAndTagsValid(t *testing.T) { + id := ident.StringID("foo") + tags := ident.NewTags( + ident.StringTag("bar", "baz"), + ) + d, err := convert.FromSeriesIDAndTags(id, tags) + assert.NoError(t, err) + assertContentsMatch(t, id, tags.Values(), d) +} + +func TestFromSeriesIDAndTagsReuseBytesFromSeriesId(t *testing.T) { tests := []struct { name string id string }{ - { - name: "no tags in ID", - id: "foo", - }, { name: "tags in ID", - id: "bar=baz", + id: "bar=baz,quip=quix", }, { name: "tags in ID with specific format", - id: `{bar="baz"}`, + id: `{bar="baz",quip="quix"}`, }, { name: "inexact tag occurrence in ID", - id: "bazillion_barometers", + id: "quixquip_bazillion_barometers", }, } tags := ident.NewTags( ident.StringTag("bar", "baz"), + ident.StringTag("quip", "quix"), ) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d, err := convert.FromSeriesIDAndTags(ident.StringID(tt.id), tags) + seriesID := ident.StringID(tt.id) + d, err := convert.FromSeriesIDAndTags(seriesID, tags) assert.NoError(t, err) - assert.Equal(t, tt.id, string(d.ID)) - assert.Len(t, d.Fields, 1) - assert.Equal(t, "bar", string(d.Fields[0].Name)) - assert.Equal(t, "baz", string(d.Fields[0].Value)) + assertContentsMatch(t, seriesID, tags.Values(), d) + for i := range d.Fields { + assert.True(t, test.ByteSliceDataContained(d.ID, d.Fields[i].Name)) + assert.True(t, test.ByteSliceDataContained(d.ID, d.Fields[i].Value)) + } }) } } func TestFromSeriesIDAndTagIterValid(t *testing.T) { + id := ident.StringID("foo") + tags := ident.NewTags( + ident.StringTag("bar", "baz"), + ) + d, err := convert.FromSeriesIDAndTagIter(id, ident.NewTagsIterator(tags)) + assert.NoError(t, err) + assertContentsMatch(t, id, tags.Values(), d) +} + +func TestFromSeriesIDAndTagIterReuseBytesFromSeriesId(t *testing.T) { tests := []struct { name string id string }{ - { - name: "no tags in ID", - id: "foo", - }, { name: "tags in ID", - id: "bar=baz", + id: "bar=baz,quip=quix", }, { name: "tags in ID with specific format", - id: `{bar="baz"}`, + id: `{bar="baz",quip="quix"}`, }, { name: "inexact tag occurrence in ID", - id: "bazillion_barometers", + id: "quixquip_bazillion_barometers", }, } tags := ident.NewTags( ident.StringTag("bar", "baz"), + ident.StringTag("quip", "quix"), ) for _, tt := range tests { - d, err := convert.FromSeriesIDAndTagIter(ident.StringID(tt.id), ident.NewTagsIterator(tags)) - assert.NoError(t, err) - assert.Equal(t, tt.id, string(d.ID)) - assert.Len(t, d.Fields, 1) - assert.Equal(t, "bar", string(d.Fields[0].Name)) - assert.Equal(t, "baz", string(d.Fields[0].Value)) + t.Run(tt.name, func(t *testing.T) { + seriesID := ident.StringID(tt.id) + d, err := convert.FromSeriesIDAndTagIter(seriesID, ident.NewTagsIterator(tags)) + assert.NoError(t, err) + assertContentsMatch(t, seriesID, tags.Values(), d) + for i := range d.Fields { + assert.True(t, test.ByteSliceDataContained(d.ID, d.Fields[i].Name)) + assert.True(t, test.ByteSliceDataContained(d.ID, d.Fields[i].Value)) + } + }) } } @@ -263,3 +284,12 @@ func TestValidateSeries(t *testing.T) { } // TODO(prateek): add a test to ensure we're interacting with the Pools as expected + +func assertContentsMatch(t *testing.T, seriesID ident.ID, tags []ident.Tag, doc doc.Metadata) { + assert.Equal(t, seriesID.String(), string(doc.ID)) + assert.Len(t, doc.Fields, len(tags)) + for i, f := range doc.Fields { //nolint:gocritic + assert.Equal(t, tags[i].Name.String(), string(f.Name)) + assert.Equal(t, tags[i].Value.String(), string(f.Value)) + } +} diff --git a/src/x/test/util.go b/src/x/test/util.go index f10f1cb5d6..e083cdd6c1 100644 --- a/src/x/test/util.go +++ b/src/x/test/util.go @@ -32,3 +32,17 @@ func ByteSlicesBackedBySameData(a, b []byte) bool { bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b)) return aHeader.Data == bHeader.Data } + +// ByteSliceDataContained checks data of 'inner' slice is fully contained inside data of +// 'outter' slice. +func ByteSliceDataContained(outter, inner []byte) bool { + var ( + outterHeader = (*reflect.SliceHeader)(unsafe.Pointer(&outter)) //nolint:gosec + innerHeader = (*reflect.SliceHeader)(unsafe.Pointer(&inner)) //nolint:gosec + outterStart = uint(outterHeader.Data) + outterEnd = outterStart + uint(outterHeader.Len) + innerStart = uint(innerHeader.Data) + innerEnd = innerStart + uint(innerHeader.Len) + ) + return outterStart <= innerStart && innerEnd <= outterEnd +} diff --git a/src/x/test/util_test.go b/src/x/test/util_test.go new file mode 100644 index 0000000000..842980c80e --- /dev/null +++ b/src/x/test/util_test.go @@ -0,0 +1,61 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package test + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestByteSliceDataContained(t *testing.T) { + outter := []byte("abcde") + for i := 0; i < len(outter); i++ { + for j := i; j <= len(outter); j++ { + inner := outter[i:j] + require.True(t, ByteSliceDataContained(outter, inner)) + } + } +} + +func TestByteSliceDataContainedNot(t *testing.T) { + data := []byte("__abcde__") + outterStart := 2 + outterEnd := 7 + outter := data[outterStart:outterEnd] + + tests := [][]byte{ + data, + data[outterStart-1 : outterEnd+1], + + data[outterStart : outterEnd+1], + data[outterStart : outterEnd+2], + + data[outterStart-1 : outterEnd], + data[outterStart-2 : outterEnd], + } + + for _, tt := range tests { + t.Run(string(tt), func(t *testing.T) { + require.False(t, ByteSliceDataContained(outter, tt)) + }) + } +} From 4927f6f0f5ed19ed898f3067123310e92f1a2e57 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Tue, 12 Jan 2021 12:11:18 +0200 Subject: [PATCH 06/22] update benchmark --- .../index/convert/convert_benchmark_test.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/dbnode/storage/index/convert/convert_benchmark_test.go b/src/dbnode/storage/index/convert/convert_benchmark_test.go index 5124a0a2b1..04bae01672 100644 --- a/src/dbnode/storage/index/convert/convert_benchmark_test.go +++ b/src/dbnode/storage/index/convert/convert_benchmark_test.go @@ -131,24 +131,14 @@ var samples = []struct { }, } -// BenchmarkFromSeriesIDAndTagIter-12 254090 4689 ns/op +// BenchmarkFromSeriesIDAndTagIter-12 772224 1649 ns/op func BenchmarkFromSeriesIDAndTagIter(b *testing.B) { - testData, err := prepareIDAndEncodedTags(b) + testData, err := prepareIDAndTags(b) require.NoError(b, err) - decoderPool := serialize.NewTagDecoderPool( - serialize.NewTagDecoderOptions(serialize.TagDecoderOptionsConfig{}), - pool.NewObjectPoolOptions(), - ) - decoderPool.Init() - tagDecoder := decoderPool.Get() - defer tagDecoder.Close() - b.ResetTimer() for i := range testData { - tagDecoder.Reset(checked.NewBytes(testData[i].encodedTags, nil)) - - _, err := FromSeriesIDAndTagIter(testData[i].id, tagDecoder) + _, err := FromSeriesIDAndTagIter(testData[i].id, ident.NewTagsIterator(testData[i].tags)) require.NoError(b, err) } } From 6bb6d9b40510ae5404955846d12c5319a9dc0a9e Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Wed, 13 Jan 2021 11:41:33 +0200 Subject: [PATCH 07/22] PR comments --- .../storage/index/convert/convert_test.go | 26 ++++++-- src/x/test/util.go | 14 ----- src/x/test/util_test.go | 61 ------------------- 3 files changed, 22 insertions(+), 79 deletions(-) delete mode 100644 src/x/test/util_test.go diff --git a/src/dbnode/storage/index/convert/convert_test.go b/src/dbnode/storage/index/convert/convert_test.go index a95359d07d..aa72449797 100644 --- a/src/dbnode/storage/index/convert/convert_test.go +++ b/src/dbnode/storage/index/convert/convert_test.go @@ -20,6 +20,7 @@ package convert_test import ( + "bytes" "encoding/hex" "testing" "unicode/utf8" @@ -91,6 +92,10 @@ func TestFromSeriesIDAndTagsReuseBytesFromSeriesId(t *testing.T) { name: "tags in ID with specific format", id: `{bar="baz",quip="quix"}`, }, + { + name: "tags in ID with specific format reverse order", + id: `{quip="quix",bar="baz"}`, + }, { name: "inexact tag occurrence in ID", id: "quixquip_bazillion_barometers", @@ -108,8 +113,8 @@ func TestFromSeriesIDAndTagsReuseBytesFromSeriesId(t *testing.T) { assert.NoError(t, err) assertContentsMatch(t, seriesID, tags.Values(), d) for i := range d.Fields { - assert.True(t, test.ByteSliceDataContained(d.ID, d.Fields[i].Name)) - assert.True(t, test.ByteSliceDataContained(d.ID, d.Fields[i].Value)) + assertBackedBySameData(t, d.ID, d.Fields[i].Name) + assertBackedBySameData(t, d.ID, d.Fields[i].Value) } }) } @@ -138,6 +143,10 @@ func TestFromSeriesIDAndTagIterReuseBytesFromSeriesId(t *testing.T) { name: "tags in ID with specific format", id: `{bar="baz",quip="quix"}`, }, + { + name: "tags in ID with specific format reverse order", + id: `{quip="quix",bar="baz"}`, + }, { name: "inexact tag occurrence in ID", id: "quixquip_bazillion_barometers", @@ -155,8 +164,8 @@ func TestFromSeriesIDAndTagIterReuseBytesFromSeriesId(t *testing.T) { assert.NoError(t, err) assertContentsMatch(t, seriesID, tags.Values(), d) for i := range d.Fields { - assert.True(t, test.ByteSliceDataContained(d.ID, d.Fields[i].Name)) - assert.True(t, test.ByteSliceDataContained(d.ID, d.Fields[i].Value)) + assertBackedBySameData(t, d.ID, d.Fields[i].Name) + assertBackedBySameData(t, d.ID, d.Fields[i].Value) } }) } @@ -293,3 +302,12 @@ func assertContentsMatch(t *testing.T, seriesID ident.ID, tags []ident.Tag, doc assert.Equal(t, tags[i].Value.String(), string(f.Value)) } } + +func assertBackedBySameData(t *testing.T, outer, inner []byte) { + if idx := bytes.Index(outer, inner); idx != -1 { + subslice := outer[idx : idx+len(inner)] + assert.True(t, test.ByteSlicesBackedBySameData(subslice, inner)) + } else { + assert.Fail(t, "inner byte sequence wasn't found") + } +} diff --git a/src/x/test/util.go b/src/x/test/util.go index e083cdd6c1..f10f1cb5d6 100644 --- a/src/x/test/util.go +++ b/src/x/test/util.go @@ -32,17 +32,3 @@ func ByteSlicesBackedBySameData(a, b []byte) bool { bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b)) return aHeader.Data == bHeader.Data } - -// ByteSliceDataContained checks data of 'inner' slice is fully contained inside data of -// 'outter' slice. -func ByteSliceDataContained(outter, inner []byte) bool { - var ( - outterHeader = (*reflect.SliceHeader)(unsafe.Pointer(&outter)) //nolint:gosec - innerHeader = (*reflect.SliceHeader)(unsafe.Pointer(&inner)) //nolint:gosec - outterStart = uint(outterHeader.Data) - outterEnd = outterStart + uint(outterHeader.Len) - innerStart = uint(innerHeader.Data) - innerEnd = innerStart + uint(innerHeader.Len) - ) - return outterStart <= innerStart && innerEnd <= outterEnd -} diff --git a/src/x/test/util_test.go b/src/x/test/util_test.go deleted file mode 100644 index 842980c80e..0000000000 --- a/src/x/test/util_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2021 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package test - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestByteSliceDataContained(t *testing.T) { - outter := []byte("abcde") - for i := 0; i < len(outter); i++ { - for j := i; j <= len(outter); j++ { - inner := outter[i:j] - require.True(t, ByteSliceDataContained(outter, inner)) - } - } -} - -func TestByteSliceDataContainedNot(t *testing.T) { - data := []byte("__abcde__") - outterStart := 2 - outterEnd := 7 - outter := data[outterStart:outterEnd] - - tests := [][]byte{ - data, - data[outterStart-1 : outterEnd+1], - - data[outterStart : outterEnd+1], - data[outterStart : outterEnd+2], - - data[outterStart-1 : outterEnd], - data[outterStart-2 : outterEnd], - } - - for _, tt := range tests { - t.Run(string(tt), func(t *testing.T) { - require.False(t, ByteSliceDataContained(outter, tt)) - }) - } -} From 6d007e3cc4491a6abaaabba44286b4813dc0c946 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Wed, 13 Jan 2021 11:55:40 +0200 Subject: [PATCH 08/22] un-base64 series ID samples --- .../index/convert/convert_benchmark_test.go | 45 +++++++++---------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/src/dbnode/storage/index/convert/convert_benchmark_test.go b/src/dbnode/storage/index/convert/convert_benchmark_test.go index 04bae01672..0847e557e3 100644 --- a/src/dbnode/storage/index/convert/convert_benchmark_test.go +++ b/src/dbnode/storage/index/convert/convert_benchmark_test.go @@ -50,83 +50,83 @@ var samples = []struct { tags string }{ { - id: "e19fbmFtZV9fPSJkaXNraW8iLGFyY2g9Ing2NCIsZGF0YWNlbnRlcj0idXMtd2VzdC0yYyIsaG9zdG5hbWU9Imhvc3RfNzgiLG1lYXN1cmVtZW50PSJyZWFkcyIsb3M9IlVidW50dTE1LjEwIixyYWNrPSI4NyIscmVnaW9uPSJ1cy13ZXN0LTIiLHNlcnZpY2U9IjExIixzZXJ2aWNlX2Vudmlyb25tZW50PSJwcm9kdWN0aW9uIixzZXJ2aWNlX3ZlcnNpb249IjEiLHRlYW09IlNGIn0=", + id: `{__name__="diskio",arch="x64",datacenter="us-west-2c",hostname="host_78",measurement="reads",os="Ubuntu15.10",rack="87",region="us-west-2",service="11",service_environment="production",service_version="1",team="SF"}`, tags: "dScMAAgAX19uYW1lX18GAGRpc2tpbwQAYXJjaAMAeDY0CgBkYXRhY2VudGVyCgB1cy13ZXN0LTJjCABob3N0bmFtZQcAaG9zdF83OAsAbWVhc3VyZW1lbnQFAHJlYWRzAgBvcwsAVWJ1bnR1MTUuMTAEAHJhY2sCADg3BgByZWdpb24JAHVzLXdlc3QtMgcAc2VydmljZQIAMTETAHNlcnZpY2VfZW52aXJvbm1lbnQKAHByb2R1Y3Rpb24PAHNlcnZpY2VfdmVyc2lvbgEAMQQAdGVhbQIAU0Y=", }, { - id: "e19fbmFtZV9fPSJuZ2lueCIsYXJjaD0ieDY0IixkYXRhY2VudGVyPSJ1cy13ZXN0LTFhIixob3N0bmFtZT0iaG9zdF8zNyIsbWVhc3VyZW1lbnQ9ImFjdGl2ZSIsb3M9IlVidW50dTE2LjEwIixyYWNrPSI3OCIscmVnaW9uPSJ1cy13ZXN0LTEiLHNlcnZpY2U9IjEwIixzZXJ2aWNlX2Vudmlyb25tZW50PSJ0ZXN0IixzZXJ2aWNlX3ZlcnNpb249IjAiLHRlYW09IkxPTiJ9", + id: `{__name__="nginx",arch="x64",datacenter="us-west-1a",hostname="host_37",measurement="active",os="Ubuntu16.10",rack="78",region="us-west-1",service="10",service_environment="test",service_version="0",team="LON"}`, tags: "dScMAAgAX19uYW1lX18FAG5naW54BABhcmNoAwB4NjQKAGRhdGFjZW50ZXIKAHVzLXdlc3QtMWEIAGhvc3RuYW1lBwBob3N0XzM3CwBtZWFzdXJlbWVudAYAYWN0aXZlAgBvcwsAVWJ1bnR1MTYuMTAEAHJhY2sCADc4BgByZWdpb24JAHVzLXdlc3QtMQcAc2VydmljZQIAMTATAHNlcnZpY2VfZW52aXJvbm1lbnQEAHRlc3QPAHNlcnZpY2VfdmVyc2lvbgEAMAQAdGVhbQMATE9O", }, { - id: "e19fbmFtZV9fPSJkaXNrIixhcmNoPSJ4NjQiLGRhdGFjZW50ZXI9InNhLWVhc3QtMWIiLGhvc3RuYW1lPSJob3N0XzU0IixtZWFzdXJlbWVudD0iaW5vZGVzX3RvdGFsIixvcz0iVWJ1bnR1MTYuMTAiLHJhY2s9Ijg4IixyZWdpb249InNhLWVhc3QtMSIsc2VydmljZT0iMTUiLHNlcnZpY2VfZW52aXJvbm1lbnQ9InByb2R1Y3Rpb24iLHNlcnZpY2VfdmVyc2lvbj0iMCIsdGVhbT0iQ0hJIn0=", + id: `{__name__="disk",arch="x64",datacenter="sa-east-1b",hostname="host_54",measurement="inodes_total",os="Ubuntu16.10",rack="88",region="sa-east-1",service="15",service_environment="production",service_version="0",team="CHI"}`, tags: "dScMAAgAX19uYW1lX18EAGRpc2sEAGFyY2gDAHg2NAoAZGF0YWNlbnRlcgoAc2EtZWFzdC0xYggAaG9zdG5hbWUHAGhvc3RfNTQLAG1lYXN1cmVtZW50DABpbm9kZXNfdG90YWwCAG9zCwBVYnVudHUxNi4xMAQAcmFjawIAODgGAHJlZ2lvbgkAc2EtZWFzdC0xBwBzZXJ2aWNlAgAxNRMAc2VydmljZV9lbnZpcm9ubWVudAoAcHJvZHVjdGlvbg8Ac2VydmljZV92ZXJzaW9uAQAwBAB0ZWFtAwBDSEk=", }, { - id: "e19fbmFtZV9fPSJuZXQiLGFyY2g9Ing4NiIsZGF0YWNlbnRlcj0idXMtZWFzdC0xYiIsaG9zdG5hbWU9Imhvc3RfOTMiLG1lYXN1cmVtZW50PSJlcnJfaW4iLG9zPSJVYnVudHUxNS4xMCIscmFjaz0iMzciLHJlZ2lvbj0idXMtZWFzdC0xIixzZXJ2aWNlPSIxMiIsc2VydmljZV9lbnZpcm9ubWVudD0icHJvZHVjdGlvbiIsc2VydmljZV92ZXJzaW9uPSIxIix0ZWFtPSJDSEkifQ==", + id: `{__name__="net",arch="x86",datacenter="us-east-1b",hostname="host_93",measurement="err_in",os="Ubuntu15.10",rack="37",region="us-east-1",service="12",service_environment="production",service_version="1",team="CHI"}`, tags: "dScMAAgAX19uYW1lX18DAG5ldAQAYXJjaAMAeDg2CgBkYXRhY2VudGVyCgB1cy1lYXN0LTFiCABob3N0bmFtZQcAaG9zdF85MwsAbWVhc3VyZW1lbnQGAGVycl9pbgIAb3MLAFVidW50dTE1LjEwBAByYWNrAgAzNwYAcmVnaW9uCQB1cy1lYXN0LTEHAHNlcnZpY2UCADEyEwBzZXJ2aWNlX2Vudmlyb25tZW50CgBwcm9kdWN0aW9uDwBzZXJ2aWNlX3ZlcnNpb24BADEEAHRlYW0DAENISQ==", }, { - id: "e19fbmFtZV9fPSJyZWRpcyIsYXJjaD0ieDg2IixkYXRhY2VudGVyPSJldS1jZW50cmFsLTFhIixob3N0bmFtZT0iaG9zdF83MCIsbWVhc3VyZW1lbnQ9ImtleXNwYWNlX21pc3NlcyIsb3M9IlVidW50dTE2LjA0TFRTIixyYWNrPSI0NyIscmVnaW9uPSJldS1jZW50cmFsLTEiLHNlcnZpY2U9IjEyIixzZXJ2aWNlX2Vudmlyb25tZW50PSJzdGFnaW5nIixzZXJ2aWNlX3ZlcnNpb249IjEiLHRlYW09IkxPTiJ9", + id: `{__name__="redis",arch="x86",datacenter="eu-central-1a",hostname="host_70",measurement="keyspace_misses",os="Ubuntu16.04LTS",rack="47",region="eu-central-1",service="12",service_environment="staging",service_version="1",team="LON"}`, tags: "dScMAAgAX19uYW1lX18FAHJlZGlzBABhcmNoAwB4ODYKAGRhdGFjZW50ZXINAGV1LWNlbnRyYWwtMWEIAGhvc3RuYW1lBwBob3N0XzcwCwBtZWFzdXJlbWVudA8Aa2V5c3BhY2VfbWlzc2VzAgBvcw4AVWJ1bnR1MTYuMDRMVFMEAHJhY2sCADQ3BgByZWdpb24MAGV1LWNlbnRyYWwtMQcAc2VydmljZQIAMTITAHNlcnZpY2VfZW52aXJvbm1lbnQHAHN0YWdpbmcPAHNlcnZpY2VfdmVyc2lvbgEAMQQAdGVhbQMATE9O", }, { - id: "e19fbmFtZV9fPSJuZ2lueCIsYXJjaD0ieDg2IixkYXRhY2VudGVyPSJ1cy1lYXN0LTFiIixob3N0bmFtZT0iaG9zdF84NCIsbWVhc3VyZW1lbnQ9InJlcXVlc3RzIixvcz0iVWJ1bnR1MTYuMDRMVFMiLHJhY2s9IjkwIixyZWdpb249InVzLWVhc3QtMSIsc2VydmljZT0iMTMiLHNlcnZpY2VfZW52aXJvbm1lbnQ9InRlc3QiLHNlcnZpY2VfdmVyc2lvbj0iMCIsdGVhbT0iTllDIn0=", + id: `{__name__="nginx",arch="x86",datacenter="us-east-1b",hostname="host_84",measurement="requests",os="Ubuntu16.04LTS",rack="90",region="us-east-1",service="13",service_environment="test",service_version="0",team="NYC"}`, tags: "dScMAAgAX19uYW1lX18FAG5naW54BABhcmNoAwB4ODYKAGRhdGFjZW50ZXIKAHVzLWVhc3QtMWIIAGhvc3RuYW1lBwBob3N0Xzg0CwBtZWFzdXJlbWVudAgAcmVxdWVzdHMCAG9zDgBVYnVudHUxNi4wNExUUwQAcmFjawIAOTAGAHJlZ2lvbgkAdXMtZWFzdC0xBwBzZXJ2aWNlAgAxMxMAc2VydmljZV9lbnZpcm9ubWVudAQAdGVzdA8Ac2VydmljZV92ZXJzaW9uAQAwBAB0ZWFtAwBOWUM=", }, { - id: "e19fbmFtZV9fPSJtZW0iLGFyY2g9Ing2NCIsZGF0YWNlbnRlcj0iZXUtY2VudHJhbC0xYiIsaG9zdG5hbWU9Imhvc3RfMjciLG1lYXN1cmVtZW50PSJidWZmZXJlZCIsb3M9IlVidW50dTE2LjA0TFRTIixyYWNrPSI1OCIscmVnaW9uPSJldS1jZW50cmFsLTEiLHNlcnZpY2U9IjAiLHNlcnZpY2VfZW52aXJvbm1lbnQ9InRlc3QiLHNlcnZpY2VfdmVyc2lvbj0iMCIsdGVhbT0iTllDIn0=", + id: `{__name__="mem",arch="x64",datacenter="eu-central-1b",hostname="host_27",measurement="buffered",os="Ubuntu16.04LTS",rack="58",region="eu-central-1",service="0",service_environment="test",service_version="0",team="NYC"}`, tags: "dScMAAgAX19uYW1lX18DAG1lbQQAYXJjaAMAeDY0CgBkYXRhY2VudGVyDQBldS1jZW50cmFsLTFiCABob3N0bmFtZQcAaG9zdF8yNwsAbWVhc3VyZW1lbnQIAGJ1ZmZlcmVkAgBvcw4AVWJ1bnR1MTYuMDRMVFMEAHJhY2sCADU4BgByZWdpb24MAGV1LWNlbnRyYWwtMQcAc2VydmljZQEAMBMAc2VydmljZV9lbnZpcm9ubWVudAQAdGVzdA8Ac2VydmljZV92ZXJzaW9uAQAwBAB0ZWFtAwBOWUM=", }, { - id: "e19fbmFtZV9fPSJrZXJuZWwiLGFyY2g9Ing4NiIsZGF0YWNlbnRlcj0idXMtd2VzdC0yYSIsaG9zdG5hbWU9Imhvc3RfODAiLG1lYXN1cmVtZW50PSJkaXNrX3BhZ2VzX2luIixvcz0iVWJ1bnR1MTYuMTAiLHJhY2s9IjQyIixyZWdpb249InVzLXdlc3QtMiIsc2VydmljZT0iMTMiLHNlcnZpY2VfZW52aXJvbm1lbnQ9InRlc3QiLHNlcnZpY2VfdmVyc2lvbj0iMSIsdGVhbT0iU0YifQ==", + id: `{__name__="kernel",arch="x86",datacenter="us-west-2a",hostname="host_80",measurement="disk_pages_in",os="Ubuntu16.10",rack="42",region="us-west-2",service="13",service_environment="test",service_version="1",team="SF"}`, tags: "dScMAAgAX19uYW1lX18GAGtlcm5lbAQAYXJjaAMAeDg2CgBkYXRhY2VudGVyCgB1cy13ZXN0LTJhCABob3N0bmFtZQcAaG9zdF84MAsAbWVhc3VyZW1lbnQNAGRpc2tfcGFnZXNfaW4CAG9zCwBVYnVudHUxNi4xMAQAcmFjawIANDIGAHJlZ2lvbgkAdXMtd2VzdC0yBwBzZXJ2aWNlAgAxMxMAc2VydmljZV9lbnZpcm9ubWVudAQAdGVzdA8Ac2VydmljZV92ZXJzaW9uAQAxBAB0ZWFtAgBTRg==", }, { - id: "e19fbmFtZV9fPSJkaXNrIixhcmNoPSJ4NjQiLGRhdGFjZW50ZXI9ImFwLW5vcnRoZWFzdC0xYyIsaG9zdG5hbWU9Imhvc3RfNzciLG1lYXN1cmVtZW50PSJpbm9kZXNfdXNlZCIsb3M9IlVidW50dTE2LjA0TFRTIixyYWNrPSI4NCIscmVnaW9uPSJhcC1ub3J0aGVhc3QtMSIsc2VydmljZT0iNSIsc2VydmljZV9lbnZpcm9ubWVudD0icHJvZHVjdGlvbiIsc2VydmljZV92ZXJzaW9uPSIwIix0ZWFtPSJMT04ifQ==", + id: `{__name__="disk",arch="x64",datacenter="ap-northeast-1c",hostname="host_77",measurement="inodes_used",os="Ubuntu16.04LTS",rack="84",region="ap-northeast-1",service="5",service_environment="production",service_version="0",team="LON"}`, tags: "dScMAAgAX19uYW1lX18EAGRpc2sEAGFyY2gDAHg2NAoAZGF0YWNlbnRlcg8AYXAtbm9ydGhlYXN0LTFjCABob3N0bmFtZQcAaG9zdF83NwsAbWVhc3VyZW1lbnQLAGlub2Rlc191c2VkAgBvcw4AVWJ1bnR1MTYuMDRMVFMEAHJhY2sCADg0BgByZWdpb24OAGFwLW5vcnRoZWFzdC0xBwBzZXJ2aWNlAQA1EwBzZXJ2aWNlX2Vudmlyb25tZW50CgBwcm9kdWN0aW9uDwBzZXJ2aWNlX3ZlcnNpb24BADAEAHRlYW0DAExPTg==", }, { - id: "e19fbmFtZV9fPSJwb3N0Z3Jlc2wiLGFyY2g9Ing2NCIsZGF0YWNlbnRlcj0iZXUtY2VudHJhbC0xYiIsaG9zdG5hbWU9Imhvc3RfMjciLG1lYXN1cmVtZW50PSJ4YWN0X3JvbGxiYWNrIixvcz0iVWJ1bnR1MTYuMDRMVFMiLHJhY2s9IjU4IixyZWdpb249ImV1LWNlbnRyYWwtMSIsc2VydmljZT0iMCIsc2VydmljZV9lbnZpcm9ubWVudD0idGVzdCIsc2VydmljZV92ZXJzaW9uPSIwIix0ZWFtPSJOWUMifQ==", + id: `{__name__="postgresl",arch="x64",datacenter="eu-central-1b",hostname="host_27",measurement="xact_rollback",os="Ubuntu16.04LTS",rack="58",region="eu-central-1",service="0",service_environment="test",service_version="0",team="NYC"}`, tags: "dScMAAgAX19uYW1lX18JAHBvc3RncmVzbAQAYXJjaAMAeDY0CgBkYXRhY2VudGVyDQBldS1jZW50cmFsLTFiCABob3N0bmFtZQcAaG9zdF8yNwsAbWVhc3VyZW1lbnQNAHhhY3Rfcm9sbGJhY2sCAG9zDgBVYnVudHUxNi4wNExUUwQAcmFjawIANTgGAHJlZ2lvbgwAZXUtY2VudHJhbC0xBwBzZXJ2aWNlAQAwEwBzZXJ2aWNlX2Vudmlyb25tZW50BAB0ZXN0DwBzZXJ2aWNlX3ZlcnNpb24BADAEAHRlYW0DAE5ZQw==", }, { - id: "e19fbmFtZV9fPSJjcHUiLGFyY2g9Ing2NCIsZGF0YWNlbnRlcj0ic2EtZWFzdC0xYiIsaG9zdG5hbWU9Imhvc3RfNDMiLG1lYXN1cmVtZW50PSJ1c2FnZV9uaWNlIixvcz0iVWJ1bnR1MTYuMTAiLHJhY2s9Ijk1IixyZWdpb249InNhLWVhc3QtMSIsc2VydmljZT0iNCIsc2VydmljZV9lbnZpcm9ubWVudD0idGVzdCIsc2VydmljZV92ZXJzaW9uPSIwIix0ZWFtPSJTRiJ9", + id: `{__name__="cpu",arch="x64",datacenter="sa-east-1b",hostname="host_43",measurement="usage_nice",os="Ubuntu16.10",rack="95",region="sa-east-1",service="4",service_environment="test",service_version="0",team="SF"}`, tags: "dScMAAgAX19uYW1lX18DAGNwdQQAYXJjaAMAeDY0CgBkYXRhY2VudGVyCgBzYS1lYXN0LTFiCABob3N0bmFtZQcAaG9zdF80MwsAbWVhc3VyZW1lbnQKAHVzYWdlX25pY2UCAG9zCwBVYnVudHUxNi4xMAQAcmFjawIAOTUGAHJlZ2lvbgkAc2EtZWFzdC0xBwBzZXJ2aWNlAQA0EwBzZXJ2aWNlX2Vudmlyb25tZW50BAB0ZXN0DwBzZXJ2aWNlX3ZlcnNpb24BADAEAHRlYW0CAFNG", }, { - id: "e19fbmFtZV9fPSJkaXNrIixhcmNoPSJ4NjQiLGRhdGFjZW50ZXI9ImFwLW5vcnRoZWFzdC0xYyIsaG9zdG5hbWU9Imhvc3RfMTciLG1lYXN1cmVtZW50PSJpbm9kZXNfdG90YWwiLG9zPSJVYnVudHUxNi4xMCIscmFjaz0iOTQiLHJlZ2lvbj0iYXAtbm9ydGhlYXN0LTEiLHNlcnZpY2U9IjkiLHNlcnZpY2VfZW52aXJvbm1lbnQ9InN0YWdpbmciLHNlcnZpY2VfdmVyc2lvbj0iMCIsdGVhbT0iU0YifQ==", + id: `{__name__="disk",arch="x64",datacenter="ap-northeast-1c",hostname="host_17",measurement="inodes_total",os="Ubuntu16.10",rack="94",region="ap-northeast-1",service="9",service_environment="staging",service_version="0",team="SF"}`, tags: "dScMAAgAX19uYW1lX18EAGRpc2sEAGFyY2gDAHg2NAoAZGF0YWNlbnRlcg8AYXAtbm9ydGhlYXN0LTFjCABob3N0bmFtZQcAaG9zdF8xNwsAbWVhc3VyZW1lbnQMAGlub2Rlc190b3RhbAIAb3MLAFVidW50dTE2LjEwBAByYWNrAgA5NAYAcmVnaW9uDgBhcC1ub3J0aGVhc3QtMQcAc2VydmljZQEAORMAc2VydmljZV9lbnZpcm9ubWVudAcAc3RhZ2luZw8Ac2VydmljZV92ZXJzaW9uAQAwBAB0ZWFtAgBTRg==", }, { - id: "e19fbmFtZV9fPSJyZWRpcyIsYXJjaD0ieDg2IixkYXRhY2VudGVyPSJ1cy13ZXN0LTJhIixob3N0bmFtZT0iaG9zdF84MCIsbWVhc3VyZW1lbnQ9InN5bmNfcGFydGlhbF9lcnIiLG9zPSJVYnVudHUxNi4xMCIscmFjaz0iNDIiLHJlZ2lvbj0idXMtd2VzdC0yIixzZXJ2aWNlPSIxMyIsc2VydmljZV9lbnZpcm9ubWVudD0idGVzdCIsc2VydmljZV92ZXJzaW9uPSIxIix0ZWFtPSJTRiJ9", + id: `{__name__="redis",arch="x86",datacenter="us-west-2a",hostname="host_80",measurement="sync_partial_err",os="Ubuntu16.10",rack="42",region="us-west-2",service="13",service_environment="test",service_version="1",team="SF"}`, tags: "dScMAAgAX19uYW1lX18FAHJlZGlzBABhcmNoAwB4ODYKAGRhdGFjZW50ZXIKAHVzLXdlc3QtMmEIAGhvc3RuYW1lBwBob3N0XzgwCwBtZWFzdXJlbWVudBAAc3luY19wYXJ0aWFsX2VycgIAb3MLAFVidW50dTE2LjEwBAByYWNrAgA0MgYAcmVnaW9uCQB1cy13ZXN0LTIHAHNlcnZpY2UCADEzEwBzZXJ2aWNlX2Vudmlyb25tZW50BAB0ZXN0DwBzZXJ2aWNlX3ZlcnNpb24BADEEAHRlYW0CAFNG", }, { - id: "e19fbmFtZV9fPSJuZXQiLGFyY2g9Ing4NiIsZGF0YWNlbnRlcj0idXMtZWFzdC0xYSIsaG9zdG5hbWU9Imhvc3RfNzkiLG1lYXN1cmVtZW50PSJkcm9wX291dCIsb3M9IlVidW50dTE2LjA0TFRTIixyYWNrPSIxNyIscmVnaW9uPSJ1cy1lYXN0LTEiLHNlcnZpY2U9IjE3IixzZXJ2aWNlX2Vudmlyb25tZW50PSJzdGFnaW5nIixzZXJ2aWNlX3ZlcnNpb249IjEiLHRlYW09IlNGIn0=", + id: `{__name__="net",arch="x86",datacenter="us-east-1a",hostname="host_79",measurement="drop_out",os="Ubuntu16.04LTS",rack="17",region="us-east-1",service="17",service_environment="staging",service_version="1",team="SF"}`, tags: "dScMAAgAX19uYW1lX18DAG5ldAQAYXJjaAMAeDg2CgBkYXRhY2VudGVyCgB1cy1lYXN0LTFhCABob3N0bmFtZQcAaG9zdF83OQsAbWVhc3VyZW1lbnQIAGRyb3Bfb3V0AgBvcw4AVWJ1bnR1MTYuMDRMVFMEAHJhY2sCADE3BgByZWdpb24JAHVzLWVhc3QtMQcAc2VydmljZQIAMTcTAHNlcnZpY2VfZW52aXJvbm1lbnQHAHN0YWdpbmcPAHNlcnZpY2VfdmVyc2lvbgEAMQQAdGVhbQIAU0Y=", }, { - id: "e19fbmFtZV9fPSJyZWRpcyIsYXJjaD0ieDg2IixkYXRhY2VudGVyPSJhcC1zb3V0aGVhc3QtMmIiLGhvc3RuYW1lPSJob3N0XzEwMCIsbWVhc3VyZW1lbnQ9InVzZWRfY3B1X3VzZXJfY2hpbGRyZW4iLG9zPSJVYnVudHUxNi4wNExUUyIscmFjaz0iNDAiLHJlZ2lvbj0iYXAtc291dGhlYXN0LTIiLHNlcnZpY2U9IjE0IixzZXJ2aWNlX2Vudmlyb25tZW50PSJzdGFnaW5nIixzZXJ2aWNlX3ZlcnNpb249IjEiLHRlYW09Ik5ZQyJ9", + id: `{__name__="redis",arch="x86",datacenter="ap-southeast-2b",hostname="host_100",measurement="used_cpu_user_children",os="Ubuntu16.04LTS",rack="40",region="ap-southeast-2",service="14",service_environment="staging",service_version="1",team="NYC"}`, tags: "dScMAAgAX19uYW1lX18FAHJlZGlzBABhcmNoAwB4ODYKAGRhdGFjZW50ZXIPAGFwLXNvdXRoZWFzdC0yYggAaG9zdG5hbWUIAGhvc3RfMTAwCwBtZWFzdXJlbWVudBYAdXNlZF9jcHVfdXNlcl9jaGlsZHJlbgIAb3MOAFVidW50dTE2LjA0TFRTBAByYWNrAgA0MAYAcmVnaW9uDgBhcC1zb3V0aGVhc3QtMgcAc2VydmljZQIAMTQTAHNlcnZpY2VfZW52aXJvbm1lbnQHAHN0YWdpbmcPAHNlcnZpY2VfdmVyc2lvbgEAMQQAdGVhbQMATllD", }, { - id: "e19fbmFtZV9fPSJkaXNrIixhcmNoPSJ4NjQiLGRhdGFjZW50ZXI9ImFwLXNvdXRoZWFzdC0xYSIsaG9zdG5hbWU9Imhvc3RfODciLG1lYXN1cmVtZW50PSJpbm9kZXNfdG90YWwiLG9zPSJVYnVudHUxNS4xMCIscmFjaz0iMCIscmVnaW9uPSJhcC1zb3V0aGVhc3QtMSIsc2VydmljZT0iMTEiLHNlcnZpY2VfZW52aXJvbm1lbnQ9InN0YWdpbmciLHNlcnZpY2VfdmVyc2lvbj0iMCIsdGVhbT0iTE9OIn0=", + id: `{__name__="disk",arch="x64",datacenter="ap-southeast-1a",hostname="host_87",measurement="inodes_total",os="Ubuntu15.10",rack="0",region="ap-southeast-1",service="11",service_environment="staging",service_version="0",team="LON"}`, tags: "dScMAAgAX19uYW1lX18EAGRpc2sEAGFyY2gDAHg2NAoAZGF0YWNlbnRlcg8AYXAtc291dGhlYXN0LTFhCABob3N0bmFtZQcAaG9zdF84NwsAbWVhc3VyZW1lbnQMAGlub2Rlc190b3RhbAIAb3MLAFVidW50dTE1LjEwBAByYWNrAQAwBgByZWdpb24OAGFwLXNvdXRoZWFzdC0xBwBzZXJ2aWNlAgAxMRMAc2VydmljZV9lbnZpcm9ubWVudAcAc3RhZ2luZw8Ac2VydmljZV92ZXJzaW9uAQAwBAB0ZWFtAwBMT04=", }, { - id: "e19fbmFtZV9fPSJjcHUiLGFyY2g9Ing2NCIsZGF0YWNlbnRlcj0idXMtd2VzdC0yYSIsaG9zdG5hbWU9Imhvc3RfNiIsbWVhc3VyZW1lbnQ9InVzYWdlX2lkbGUiLG9zPSJVYnVudHUxNi4xMCIscmFjaz0iMTAiLHJlZ2lvbj0idXMtd2VzdC0yIixzZXJ2aWNlPSI2IixzZXJ2aWNlX2Vudmlyb25tZW50PSJ0ZXN0IixzZXJ2aWNlX3ZlcnNpb249IjAiLHRlYW09IkNISSJ9", + id: `{__name__="cpu",arch="x64",datacenter="us-west-2a",hostname="host_6",measurement="usage_idle",os="Ubuntu16.10",rack="10",region="us-west-2",service="6",service_environment="test",service_version="0",team="CHI"}`, tags: "dScMAAgAX19uYW1lX18DAGNwdQQAYXJjaAMAeDY0CgBkYXRhY2VudGVyCgB1cy13ZXN0LTJhCABob3N0bmFtZQYAaG9zdF82CwBtZWFzdXJlbWVudAoAdXNhZ2VfaWRsZQIAb3MLAFVidW50dTE2LjEwBAByYWNrAgAxMAYAcmVnaW9uCQB1cy13ZXN0LTIHAHNlcnZpY2UBADYTAHNlcnZpY2VfZW52aXJvbm1lbnQEAHRlc3QPAHNlcnZpY2VfdmVyc2lvbgEAMAQAdGVhbQMAQ0hJ", }, { - id: "e19fbmFtZV9fPSJuZ2lueCIsYXJjaD0ieDg2IixkYXRhY2VudGVyPSJ1cy1lYXN0LTFhIixob3N0bmFtZT0iaG9zdF80NCIsbWVhc3VyZW1lbnQ9ImhhbmRsZWQiLG9zPSJVYnVudHUxNi4wNExUUyIscmFjaz0iNjEiLHJlZ2lvbj0idXMtZWFzdC0xIixzZXJ2aWNlPSIyIixzZXJ2aWNlX2Vudmlyb25tZW50PSJzdGFnaW5nIixzZXJ2aWNlX3ZlcnNpb249IjEiLHRlYW09Ik5ZQyJ9", + id: `{__name__="nginx",arch="x86",datacenter="us-east-1a",hostname="host_44",measurement="handled",os="Ubuntu16.04LTS",rack="61",region="us-east-1",service="2",service_environment="staging",service_version="1",team="NYC"}`, tags: "dScMAAgAX19uYW1lX18FAG5naW54BABhcmNoAwB4ODYKAGRhdGFjZW50ZXIKAHVzLWVhc3QtMWEIAGhvc3RuYW1lBwBob3N0XzQ0CwBtZWFzdXJlbWVudAcAaGFuZGxlZAIAb3MOAFVidW50dTE2LjA0TFRTBAByYWNrAgA2MQYAcmVnaW9uCQB1cy1lYXN0LTEHAHNlcnZpY2UBADITAHNlcnZpY2VfZW52aXJvbm1lbnQHAHN0YWdpbmcPAHNlcnZpY2VfdmVyc2lvbgEAMQQAdGVhbQMATllD", }, { - id: "e19fbmFtZV9fPSJuZ2lueCIsYXJjaD0ieDg2IixkYXRhY2VudGVyPSJ1cy13ZXN0LTFhIixob3N0bmFtZT0iaG9zdF8yOSIsbWVhc3VyZW1lbnQ9IndhaXRpbmciLG9zPSJVYnVudHUxNS4xMCIscmFjaz0iMTUiLHJlZ2lvbj0idXMtd2VzdC0xIixzZXJ2aWNlPSI0IixzZXJ2aWNlX2Vudmlyb25tZW50PSJ0ZXN0IixzZXJ2aWNlX3ZlcnNpb249IjEiLHRlYW09Ik5ZQyJ9", + id: `{__name__="nginx",arch="x86",datacenter="us-west-1a",hostname="host_29",measurement="waiting",os="Ubuntu15.10",rack="15",region="us-west-1",service="4",service_environment="test",service_version="1",team="NYC"}`, tags: "dScMAAgAX19uYW1lX18FAG5naW54BABhcmNoAwB4ODYKAGRhdGFjZW50ZXIKAHVzLXdlc3QtMWEIAGhvc3RuYW1lBwBob3N0XzI5CwBtZWFzdXJlbWVudAcAd2FpdGluZwIAb3MLAFVidW50dTE1LjEwBAByYWNrAgAxNQYAcmVnaW9uCQB1cy13ZXN0LTEHAHNlcnZpY2UBADQTAHNlcnZpY2VfZW52aXJvbm1lbnQEAHRlc3QPAHNlcnZpY2VfdmVyc2lvbgEAMQQAdGVhbQMATllD", }, { - id: "e19fbmFtZV9fPSJkaXNraW8iLGFyY2g9Ing2NCIsZGF0YWNlbnRlcj0iYXAtbm9ydGhlYXN0LTFjIixob3N0bmFtZT0iaG9zdF8zOCIsbWVhc3VyZW1lbnQ9IndyaXRlX3RpbWUiLG9zPSJVYnVudHUxNS4xMCIscmFjaz0iMjAiLHJlZ2lvbj0iYXAtbm9ydGhlYXN0LTEiLHNlcnZpY2U9IjAiLHNlcnZpY2VfZW52aXJvbm1lbnQ9InN0YWdpbmciLHNlcnZpY2VfdmVyc2lvbj0iMCIsdGVhbT0iU0YifQ==", + id: `{__name__="diskio",arch="x64",datacenter="ap-northeast-1c",hostname="host_38",measurement="write_time",os="Ubuntu15.10",rack="20",region="ap-northeast-1",service="0",service_environment="staging",service_version="0",team="SF"}`, tags: "dScMAAgAX19uYW1lX18GAGRpc2tpbwQAYXJjaAMAeDY0CgBkYXRhY2VudGVyDwBhcC1ub3J0aGVhc3QtMWMIAGhvc3RuYW1lBwBob3N0XzM4CwBtZWFzdXJlbWVudAoAd3JpdGVfdGltZQIAb3MLAFVidW50dTE1LjEwBAByYWNrAgAyMAYAcmVnaW9uDgBhcC1ub3J0aGVhc3QtMQcAc2VydmljZQEAMBMAc2VydmljZV9lbnZpcm9ubWVudAcAc3RhZ2luZw8Ac2VydmljZV92ZXJzaW9uAQAwBAB0ZWFtAgBTRg==", }, } @@ -164,10 +164,7 @@ func prepareIDAndEncodedTags(b *testing.B) ([]idWithEncodedTags, error) { for i := 0; i < b.N; i++ { k := rnd.Intn(len(samples)) - id, err := b64.DecodeString(samples[k].id) - if err != nil { - return nil, err - } + id := clone([]byte(samples[k].id)) tags, err := b64.DecodeString(samples[k].tags) if err != nil { return nil, err From a879a2acff0e9acafd2818001360f8817ee9e8b8 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Thu, 14 Jan 2021 11:02:49 +0200 Subject: [PATCH 09/22] [dbnode] Direct conversion of encoded tags to doc.Metadata --- src/dbnode/storage/index/convert/convert.go | 119 ++++++++++++++++++ .../index/convert/convert_benchmark_test.go | 43 +++++++ src/x/serialize/decoder_fast.go | 52 ++++++-- 3 files changed, 206 insertions(+), 8 deletions(-) diff --git a/src/dbnode/storage/index/convert/convert.go b/src/dbnode/storage/index/convert/convert.go index 023589a51e..dace2c75b9 100644 --- a/src/dbnode/storage/index/convert/convert.go +++ b/src/dbnode/storage/index/convert/convert.go @@ -22,14 +22,17 @@ package convert import ( "bytes" + "encoding/binary" "errors" "fmt" "unicode/utf8" + "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/m3ninx/doc" "github.com/m3db/m3/src/query/graphite/graphite" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/pool" + "github.com/m3db/m3/src/x/serialize" ) const ( @@ -188,6 +191,122 @@ func FromSeriesIDAndTagIter(id ident.ID, tags ident.TagIterator) (doc.Metadata, return d, nil } +func FromSeriesIDAndEncodedTags(id ident.ID, encodedTags ts.EncodedTags) (doc.Metadata, error) { + total := len(encodedTags) + if total < 4 { + return doc.Metadata{}, fmt.Errorf( + "encoded tags too short: size=%d, need=%d", total, 4) + } + + var ( + byteOrder = binary.LittleEndian + headerMagicNumber uint16 = 10101 + ) + + header := byteOrder.Uint16(encodedTags[:2]) + encodedTags = encodedTags[2:] + if header != headerMagicNumber { + return doc.Metadata{}, errors.New("") + } + + length := int(byteOrder.Uint16(encodedTags[:2])) + encodedTags = encodedTags[2:] + + var ( + clonedID = clone(id.Bytes()) + fields = make([]doc.Field, 0, length) + expectedStart = firstTagBytesPosition + ) + + for i := 0; i < length; i++ { + if len(encodedTags) < 2 { + return doc.Metadata{}, fmt.Errorf("missing size for tag name: index=%d", i) + } + numBytesName := int(byteOrder.Uint16(encodedTags[:2])) + if numBytesName <= 0 { + return doc.Metadata{}, errors.New("") + } + encodedTags = encodedTags[2:] + + bytesName := encodedTags[:numBytesName] + encodedTags = encodedTags[numBytesName:] + + if len(encodedTags) < 2 { + return doc.Metadata{}, fmt.Errorf("missing size for tag value: index=%d", i) + } + + numBytesValue := int(byteOrder.Uint16(encodedTags[:2])) + encodedTags = encodedTags[2:] + + bytesValue := encodedTags[:numBytesValue] + encodedTags = encodedTags[numBytesValue:] + + var clonedName, clonedValue []byte + clonedName, expectedStart = findSliceOrClone(clonedID, bytesName, expectedStart, + distanceBetweenTagNameAndValue) + clonedValue, expectedStart = findSliceOrClone(clonedID, bytesValue, expectedStart, + distanceBetweenTagValueAndNextName) + + fields = append(fields, doc.Field{ + Name: clonedName, + Value: clonedValue, + }) + } + + d := doc.Metadata{ + ID: clonedID, + Fields: fields, + } + if err := Validate(d); err != nil { + return doc.Metadata{}, err + } + return d, nil +} + +type mapTagsVisitor struct { + clonedID []byte + fields []doc.Field + expectedStart int +} + +func (v *mapTagsVisitor) Length(n int) { + v.fields = make([]doc.Field, 0, n) +} + +func (v *mapTagsVisitor) Visit(tag, value []byte) bool { + var clonedName, clonedValue []byte + clonedName, v.expectedStart = findSliceOrClone(v.clonedID, tag, v.expectedStart, + distanceBetweenTagNameAndValue) + clonedValue, v.expectedStart = findSliceOrClone(v.clonedID, value, v.expectedStart, + distanceBetweenTagValueAndNextName) + v.fields = append(v.fields, doc.Field{ + Name: clonedName, + Value: clonedValue, + }) + return true +} + +func FromSeriesIDAndEncodedTags2(id ident.ID, encodedTags ts.EncodedTags) (doc.Metadata, error) { + visitor := mapTagsVisitor{ + clonedID: clone(id.Bytes()), + fields: nil, + expectedStart: firstTagBytesPosition, + } + + err := serialize.VisitTags(encodedTags, &visitor) + if err != nil { + return doc.Metadata{}, err + } + d := doc.Metadata{ + ID: visitor.clonedID, + Fields: visitor.fields, + } + if err := Validate(d); err != nil { + return doc.Metadata{}, err + } + return d, nil +} + func findSliceOrClone(id, tag []byte, expectedStart, nextPositionDistance int) ([]byte, int) { //nolint:unparam n := len(tag) expectedEnd := expectedStart + n diff --git a/src/dbnode/storage/index/convert/convert_benchmark_test.go b/src/dbnode/storage/index/convert/convert_benchmark_test.go index 0847e557e3..cfe32cb60f 100644 --- a/src/dbnode/storage/index/convert/convert_benchmark_test.go +++ b/src/dbnode/storage/index/convert/convert_benchmark_test.go @@ -155,6 +155,49 @@ func BenchmarkFromSeriesIDAndTags(b *testing.B) { } } +func BenchmarkFromSeriesIDAndEncodedTags(b *testing.B) { + testData, err := prepareIDAndEncodedTags(b) + require.NoError(b, err) + + b.ResetTimer() + for i := range testData { + _, err := FromSeriesIDAndEncodedTags(testData[i].id, testData[i].encodedTags) + require.NoError(b, err) + } +} + +func BenchmarkFromSeriesIDAndEncodedTags2(b *testing.B) { + testData, err := prepareIDAndEncodedTags(b) + require.NoError(b, err) + + b.ResetTimer() + for i := range testData { + _, err := FromSeriesIDAndEncodedTags2(testData[i].id, testData[i].encodedTags) + require.NoError(b, err) + } +} + +func BenchmarkFromSeriesIDAndTagIter_TagDecoder(b *testing.B) { + testData, err := prepareIDAndEncodedTags(b) + require.NoError(b, err) + + decoderPool := serialize.NewTagDecoderPool( + serialize.NewTagDecoderOptions(serialize.TagDecoderOptionsConfig{}), + pool.NewObjectPoolOptions(), + ) + decoderPool.Init() + + decoder := decoderPool.Get() + defer decoder.Close() + + b.ResetTimer() + for i := range testData { + decoder.Reset(checked.NewBytes(testData[i].encodedTags, nil)) + _, err := FromSeriesIDAndTagIter(testData[i].id, decoder) + require.NoError(b, err) + } +} + func prepareIDAndEncodedTags(b *testing.B) ([]idWithEncodedTags, error) { var ( rnd = rand.New(rand.NewSource(42)) //nolint:gosec diff --git a/src/x/serialize/decoder_fast.go b/src/x/serialize/decoder_fast.go index ef5ac8d406..0b7dc3391b 100644 --- a/src/x/serialize/decoder_fast.go +++ b/src/x/serialize/decoder_fast.go @@ -27,32 +27,68 @@ import ( // TagValueFromEncodedTagsFast returns a tag from a set of encoded tags without // any pooling required. + +type TagVisitor interface { + Length(n int) + Visit(tag, value []byte) bool +} + func TagValueFromEncodedTagsFast( encodedTags []byte, tagName []byte, ) ([]byte, bool, error) { + visitor := findTagValueVisitor{ + tagName: tagName, + tagValue: nil, + } + err := VisitTags(encodedTags, &visitor) + return visitor.tagValue, visitor.tagValue != nil, err +} + +type findTagValueVisitor struct { + tagName []byte + tagValue []byte +} + +func (v *findTagValueVisitor) Length(_ int) { +} + +func (v *findTagValueVisitor) Visit(tag, value []byte) bool { + if bytes.Equal(tag, v.tagName) { + v.tagValue = value + return false + } + return true +} + +func VisitTags( + encodedTags []byte, + visitor TagVisitor, +) error { total := len(encodedTags) if total < 4 { - return nil, false, fmt.Errorf( + return fmt.Errorf( "encoded tags too short: size=%d, need=%d", total, 4) } header := byteOrder.Uint16(encodedTags[:2]) encodedTags = encodedTags[2:] if header != headerMagicNumber { - return nil, false, errIncorrectHeader + return errIncorrectHeader } length := int(byteOrder.Uint16(encodedTags[:2])) encodedTags = encodedTags[2:] + visitor.Length(length) + for i := 0; i < length; i++ { if len(encodedTags) < 2 { - return nil, false, fmt.Errorf("missing size for tag name: index=%d", i) + return fmt.Errorf("missing size for tag name: index=%d", i) } numBytesName := int(byteOrder.Uint16(encodedTags[:2])) if numBytesName <= 0 { - return nil, false, errEmptyTagNameLiteral + return errEmptyTagNameLiteral } encodedTags = encodedTags[2:] @@ -60,7 +96,7 @@ func TagValueFromEncodedTagsFast( encodedTags = encodedTags[numBytesName:] if len(encodedTags) < 2 { - return nil, false, fmt.Errorf("missing size for tag value: index=%d", i) + return fmt.Errorf("missing size for tag value: index=%d", i) } numBytesValue := int(byteOrder.Uint16(encodedTags[:2])) @@ -69,10 +105,10 @@ func TagValueFromEncodedTagsFast( bytesValue := encodedTags[:numBytesValue] encodedTags = encodedTags[numBytesValue:] - if bytes.Compare(bytesName, tagName) == 0 { - return bytesValue, true, nil + if !visitor.Visit(bytesName, bytesValue) { + return nil } } - return nil, false, nil + return nil } From db2023e192a01f3f040cd7e81175468bd5229736 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Thu, 14 Jan 2021 15:20:54 +0200 Subject: [PATCH 10/22] extract decoding steps into smaller functions --- src/dbnode/storage/index/convert/convert.go | 64 ++++++------ src/x/serialize/decoder_fast.go | 104 +++++++++----------- 2 files changed, 83 insertions(+), 85 deletions(-) diff --git a/src/dbnode/storage/index/convert/convert.go b/src/dbnode/storage/index/convert/convert.go index dace2c75b9..f90baaa55e 100644 --- a/src/dbnode/storage/index/convert/convert.go +++ b/src/dbnode/storage/index/convert/convert.go @@ -263,43 +263,51 @@ func FromSeriesIDAndEncodedTags(id ident.ID, encodedTags ts.EncodedTags) (doc.Me return d, nil } -type mapTagsVisitor struct { - clonedID []byte - fields []doc.Field - expectedStart int -} +func FromSeriesIDAndEncodedTags2(id ident.ID, encodedTags ts.EncodedTags) (doc.Metadata, error) { + var ( + length int + err error + ) + encodedTags, length, err = serialize.DecodeHeader(encodedTags) + if err != nil { + return doc.Metadata{}, err + } -func (v *mapTagsVisitor) Length(n int) { - v.fields = make([]doc.Field, 0, n) -} + var ( + clonedID = clone(id.Bytes()) + fields = make([]doc.Field, 0, length) + expectedStart = firstTagBytesPosition + ) -func (v *mapTagsVisitor) Visit(tag, value []byte) bool { - var clonedName, clonedValue []byte - clonedName, v.expectedStart = findSliceOrClone(v.clonedID, tag, v.expectedStart, - distanceBetweenTagNameAndValue) - clonedValue, v.expectedStart = findSliceOrClone(v.clonedID, value, v.expectedStart, - distanceBetweenTagValueAndNextName) - v.fields = append(v.fields, doc.Field{ - Name: clonedName, - Value: clonedValue, - }) - return true -} + for i := 0; i < length; i++ { + var nameBytes, valueBytes []byte + encodedTags, nameBytes, err = serialize.DecodeTagName(encodedTags) + if err != nil { + return doc.Metadata{}, err + } + encodedTags, valueBytes, err = serialize.DecodeTagValue(encodedTags) + if err != nil { + return doc.Metadata{}, err + } -func FromSeriesIDAndEncodedTags2(id ident.ID, encodedTags ts.EncodedTags) (doc.Metadata, error) { - visitor := mapTagsVisitor{ - clonedID: clone(id.Bytes()), - fields: nil, - expectedStart: firstTagBytesPosition, + var clonedName, clonedValue []byte + clonedName, expectedStart = findSliceOrClone(clonedID, nameBytes, expectedStart, + distanceBetweenTagNameAndValue) + clonedValue, expectedStart = findSliceOrClone(clonedID, valueBytes, expectedStart, + distanceBetweenTagValueAndNextName) + + fields = append(fields, doc.Field{ + Name: clonedName, + Value: clonedValue, + }) } - err := serialize.VisitTags(encodedTags, &visitor) if err != nil { return doc.Metadata{}, err } d := doc.Metadata{ - ID: visitor.clonedID, - Fields: visitor.fields, + ID: clonedID, + Fields: fields, } if err := Validate(d); err != nil { return doc.Metadata{}, err diff --git a/src/x/serialize/decoder_fast.go b/src/x/serialize/decoder_fast.go index 0b7dc3391b..23c949f0a0 100644 --- a/src/x/serialize/decoder_fast.go +++ b/src/x/serialize/decoder_fast.go @@ -27,88 +27,78 @@ import ( // TagValueFromEncodedTagsFast returns a tag from a set of encoded tags without // any pooling required. - -type TagVisitor interface { - Length(n int) - Visit(tag, value []byte) bool -} - func TagValueFromEncodedTagsFast( encodedTags []byte, tagName []byte, ) ([]byte, bool, error) { - visitor := findTagValueVisitor{ - tagName: tagName, - tagValue: nil, + var ( + length int + err error + ) + encodedTags, length, err = DecodeHeader(encodedTags) + if err != nil { + return nil, false, err } - err := VisitTags(encodedTags, &visitor) - return visitor.tagValue, visitor.tagValue != nil, err -} -type findTagValueVisitor struct { - tagName []byte - tagValue []byte -} + for i := 0; i < length; i++ { + var bytesName, bytesValue []byte -func (v *findTagValueVisitor) Length(_ int) { -} + encodedTags, bytesName, err = DecodeTagName(encodedTags) + if err != nil { + return nil, false, err + } -func (v *findTagValueVisitor) Visit(tag, value []byte) bool { - if bytes.Equal(tag, v.tagName) { - v.tagValue = value - return false + encodedTags, bytesValue, err = DecodeTagValue(encodedTags) + if err != nil { + return nil, false, err + } + + if bytes.Equal(bytesName, tagName) { + return bytesValue, true, nil + } } - return true + + return nil, false, nil } -func VisitTags( - encodedTags []byte, - visitor TagVisitor, -) error { +func DecodeHeader(encodedTags []byte) ([]byte, int, error) { total := len(encodedTags) if total < 4 { - return fmt.Errorf( + return nil, 0, fmt.Errorf( "encoded tags too short: size=%d, need=%d", total, 4) } header := byteOrder.Uint16(encodedTags[:2]) encodedTags = encodedTags[2:] if header != headerMagicNumber { - return errIncorrectHeader + return nil, 0, errIncorrectHeader } length := int(byteOrder.Uint16(encodedTags[:2])) encodedTags = encodedTags[2:] + return encodedTags, length, nil +} - visitor.Length(length) - - for i := 0; i < length; i++ { - if len(encodedTags) < 2 { - return fmt.Errorf("missing size for tag name: index=%d", i) - } - numBytesName := int(byteOrder.Uint16(encodedTags[:2])) - if numBytesName <= 0 { - return errEmptyTagNameLiteral - } - encodedTags = encodedTags[2:] - - bytesName := encodedTags[:numBytesName] - encodedTags = encodedTags[numBytesName:] - - if len(encodedTags) < 2 { - return fmt.Errorf("missing size for tag value: index=%d", i) - } - - numBytesValue := int(byteOrder.Uint16(encodedTags[:2])) - encodedTags = encodedTags[2:] - - bytesValue := encodedTags[:numBytesValue] - encodedTags = encodedTags[numBytesValue:] +func DecodeTagName(encodedTags []byte) ([]byte, []byte, error) { + encodedTags, stringBytes, err := DecodeTagValue(encodedTags) + if err != nil { + return encodedTags, stringBytes, err + } + if len(stringBytes) == 0 { + return nil, nil, errEmptyTagNameLiteral + } + return encodedTags, stringBytes, nil +} - if !visitor.Visit(bytesName, bytesValue) { - return nil - } +func DecodeTagValue(encodedTags []byte) ([]byte, []byte, error) { + if len(encodedTags) < 2 { + return nil, nil, fmt.Errorf("missing or incomplete size bytes") } + numBytes := int(byteOrder.Uint16(encodedTags[:2])) + encodedTags = encodedTags[2:] + + stringBytes := encodedTags[:numBytes] + encodedTags = encodedTags[numBytes:] - return nil + return encodedTags, stringBytes, nil } From f7c6bcdcb6e96cd42a361397a2cc1326f16ee8b4 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Fri, 15 Jan 2021 13:55:55 +0200 Subject: [PATCH 11/22] thinner encoded tag iterator and benchmarks --- src/dbnode/storage/index/convert/convert.go | 37 +++++++- .../index/convert/convert_benchmark_test.go | 13 +++ src/metrics/metric/id/tag.go | 2 + src/x/serialize/decoder_fast_iter.go | 85 +++++++++++++++++++ src/x/serialize/types.go | 1 - 5 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 src/x/serialize/decoder_fast_iter.go diff --git a/src/dbnode/storage/index/convert/convert.go b/src/dbnode/storage/index/convert/convert.go index f90baaa55e..421786f901 100644 --- a/src/dbnode/storage/index/convert/convert.go +++ b/src/dbnode/storage/index/convert/convert.go @@ -29,6 +29,7 @@ import ( "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/m3ninx/doc" + "github.com/m3db/m3/src/metrics/metric/id" "github.com/m3db/m3/src/query/graphite/graphite" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/pool" @@ -223,7 +224,7 @@ func FromSeriesIDAndEncodedTags(id ident.ID, encodedTags ts.EncodedTags) (doc.Me return doc.Metadata{}, fmt.Errorf("missing size for tag name: index=%d", i) } numBytesName := int(byteOrder.Uint16(encodedTags[:2])) - if numBytesName <= 0 { + if numBytesName == 0 { return doc.Metadata{}, errors.New("") } encodedTags = encodedTags[2:] @@ -315,6 +316,40 @@ func FromSeriesIDAndEncodedTags2(id ident.ID, encodedTags ts.EncodedTags) (doc.M return d, nil } +func FromSeriesIDAndFastTagIter(id ident.ID, tags id.SortedTagIterator) (doc.Metadata, error) { + var ( + clonedID = clone(id.Bytes()) + fields = make([]doc.Field, 0, tags.NumTags()) + expectedStart = firstTagBytesPosition + ) + for tags.Next() { + nameBytes, valueBytes := tags.Current() + + var clonedName, clonedValue []byte + clonedName, expectedStart = findSliceOrClone(clonedID, nameBytes, expectedStart, + distanceBetweenTagNameAndValue) + clonedValue, expectedStart = findSliceOrClone(clonedID, valueBytes, expectedStart, + distanceBetweenTagValueAndNextName) + + fields = append(fields, doc.Field{ + Name: clonedName, + Value: clonedValue, + }) + } + if err := tags.Err(); err != nil { + return doc.Metadata{}, err + } + + d := doc.Metadata{ + ID: clonedID, + Fields: fields, + } + if err := Validate(d); err != nil { + return doc.Metadata{}, err + } + return d, nil +} + func findSliceOrClone(id, tag []byte, expectedStart, nextPositionDistance int) ([]byte, int) { //nolint:unparam n := len(tag) expectedEnd := expectedStart + n diff --git a/src/dbnode/storage/index/convert/convert_benchmark_test.go b/src/dbnode/storage/index/convert/convert_benchmark_test.go index cfe32cb60f..7fcb6d7813 100644 --- a/src/dbnode/storage/index/convert/convert_benchmark_test.go +++ b/src/dbnode/storage/index/convert/convert_benchmark_test.go @@ -177,6 +177,19 @@ func BenchmarkFromSeriesIDAndEncodedTags2(b *testing.B) { } } +func BenchmarkFromSeriesIDAndFastTagIter(b *testing.B) { + testData, err := prepareIDAndEncodedTags(b) + require.NoError(b, err) + decoder := serialize.NewFastTagDecoder() + + b.ResetTimer() + for i := range testData { + decoder.Reset(testData[i].encodedTags) + _, err := FromSeriesIDAndFastTagIter(testData[i].id, decoder) + require.NoError(b, err) + } +} + func BenchmarkFromSeriesIDAndTagIter_TagDecoder(b *testing.B) { testData, err := prepareIDAndEncodedTags(b) require.NoError(b, err) diff --git a/src/metrics/metric/id/tag.go b/src/metrics/metric/id/tag.go index d5561f3f3b..5bb7e368f5 100644 --- a/src/metrics/metric/id/tag.go +++ b/src/metrics/metric/id/tag.go @@ -61,6 +61,8 @@ type SortedTagIterator interface { // Close closes the iterator. Close() + + NumTags() int } // SortedTagIteratorAlloc allocates a new sorted tag iterator. diff --git a/src/x/serialize/decoder_fast_iter.go b/src/x/serialize/decoder_fast_iter.go new file mode 100644 index 0000000000..646da4e74f --- /dev/null +++ b/src/x/serialize/decoder_fast_iter.go @@ -0,0 +1,85 @@ +package serialize + +import ( + "errors" + "fmt" + + "github.com/m3db/m3/src/metrics/metric/id" +) + +type decoderFast struct { + length int + remaining int + + encodedTags []byte + + currentName, currentValue []byte + + err error +} + +func NewFastTagDecoder() id.SortedTagIterator { + return &decoderFast{} +} + +func (d *decoderFast) Reset(encodedTags []byte) { + header := byteOrder.Uint16(encodedTags[:2]) + encodedTags = encodedTags[2:] + if header != headerMagicNumber { + d.err = errors.New("") + return + } + + d.length = int(byteOrder.Uint16(encodedTags[:2])) + d.encodedTags = encodedTags[2:] + d.remaining = d.length +} + +func (d *decoderFast) Next() bool { + if d.remaining == 0 { + return false + } + + d.remaining-- + + if len(d.encodedTags) < 2 { + d.err = fmt.Errorf("missing size for tag name: index=%d", d.length-d.remaining) + return false + } + numBytesName := int(byteOrder.Uint16(d.encodedTags[:2])) + if numBytesName == 0 { + d.err = errors.New("") + return false + } + d.encodedTags = d.encodedTags[2:] + + d.currentName = d.encodedTags[:numBytesName] + d.encodedTags = d.encodedTags[numBytesName:] + + if len(d.encodedTags) < 2 { + d.err = fmt.Errorf("missing size for tag value: index=%d", d.length-d.remaining) + return false + } + + numBytesValue := int(byteOrder.Uint16(d.encodedTags[:2])) + d.encodedTags = d.encodedTags[2:] + + d.currentValue = d.encodedTags[:numBytesValue] + d.encodedTags = d.encodedTags[numBytesValue:] + return true +} + +func (d *decoderFast) Current() ([]byte, []byte) { + return d.currentName, d.currentValue +} + +func (d *decoderFast) Err() error { + return d.err +} + +func (d *decoderFast) Close() { +} + +func (d *decoderFast) NumTags() int { + return d.length +} diff --git a/src/x/serialize/types.go b/src/x/serialize/types.go index a694ae7dc7..600113d81b 100644 --- a/src/x/serialize/types.go +++ b/src/x/serialize/types.go @@ -138,7 +138,6 @@ type TagSerializationLimits interface { type MetricTagsIterator interface { id.ID id.SortedTagIterator - NumTags() int } // MetricTagsIteratorPool pools MetricTagsIterator. From f769ce04cda1dc5b34ad770664b7a9600f17e0cd Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Fri, 15 Jan 2021 15:48:25 +0200 Subject: [PATCH 12/22] merge DecodeTagName() and DecodeTagValue() methods --- src/dbnode/storage/index/convert/convert.go | 6 +--- src/x/serialize/decoder_fast.go | 37 ++++++++++----------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/src/dbnode/storage/index/convert/convert.go b/src/dbnode/storage/index/convert/convert.go index 421786f901..dc62123369 100644 --- a/src/dbnode/storage/index/convert/convert.go +++ b/src/dbnode/storage/index/convert/convert.go @@ -282,11 +282,7 @@ func FromSeriesIDAndEncodedTags2(id ident.ID, encodedTags ts.EncodedTags) (doc.M for i := 0; i < length; i++ { var nameBytes, valueBytes []byte - encodedTags, nameBytes, err = serialize.DecodeTagName(encodedTags) - if err != nil { - return doc.Metadata{}, err - } - encodedTags, valueBytes, err = serialize.DecodeTagValue(encodedTags) + encodedTags, nameBytes, valueBytes, err = serialize.DecodeTag(encodedTags) if err != nil { return doc.Metadata{}, err } diff --git a/src/x/serialize/decoder_fast.go b/src/x/serialize/decoder_fast.go index 23c949f0a0..f73cf9c131 100644 --- a/src/x/serialize/decoder_fast.go +++ b/src/x/serialize/decoder_fast.go @@ -43,12 +43,7 @@ func TagValueFromEncodedTagsFast( for i := 0; i < length; i++ { var bytesName, bytesValue []byte - encodedTags, bytesName, err = DecodeTagName(encodedTags) - if err != nil { - return nil, false, err - } - - encodedTags, bytesValue, err = DecodeTagValue(encodedTags) + encodedTags, bytesName, bytesValue, err = DecodeTag(encodedTags) if err != nil { return nil, false, err } @@ -79,26 +74,28 @@ func DecodeHeader(encodedTags []byte) ([]byte, int, error) { return encodedTags, length, nil } -func DecodeTagName(encodedTags []byte) ([]byte, []byte, error) { - encodedTags, stringBytes, err := DecodeTagValue(encodedTags) - if err != nil { - return encodedTags, stringBytes, err +func DecodeTag(encodedTags []byte) ([]byte, []byte, []byte, error) { + if len(encodedTags) < 2 { + return nil, nil, nil, fmt.Errorf("missing size for tag name") } - if len(stringBytes) == 0 { - return nil, nil, errEmptyTagNameLiteral + numBytesName := int(byteOrder.Uint16(encodedTags[:2])) + if numBytesName == 0 { + return nil, nil, nil, errEmptyTagNameLiteral } - return encodedTags, stringBytes, nil -} + encodedTags = encodedTags[2:] + + bytesName := encodedTags[:numBytesName] + encodedTags = encodedTags[numBytesName:] -func DecodeTagValue(encodedTags []byte) ([]byte, []byte, error) { if len(encodedTags) < 2 { - return nil, nil, fmt.Errorf("missing or incomplete size bytes") + return nil, nil, nil, fmt.Errorf("missing size for tag value") } - numBytes := int(byteOrder.Uint16(encodedTags[:2])) + + numBytesValue := int(byteOrder.Uint16(encodedTags[:2])) encodedTags = encodedTags[2:] - stringBytes := encodedTags[:numBytes] - encodedTags = encodedTags[numBytes:] + bytesValue := encodedTags[:numBytesValue] + encodedTags = encodedTags[numBytesValue:] - return encodedTags, stringBytes, nil + return encodedTags, bytesName, bytesValue, nil } From d3a8c387174329621e5d9904d23589778962dbdb Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Mon, 18 Jan 2021 14:36:11 +0200 Subject: [PATCH 13/22] function that uses indexing instead of sliding slice --- src/dbnode/storage/index/convert/convert.go | 74 +++++++++++++++++++ .../index/convert/convert_benchmark_test.go | 11 +++ 2 files changed, 85 insertions(+) diff --git a/src/dbnode/storage/index/convert/convert.go b/src/dbnode/storage/index/convert/convert.go index dc62123369..8ac23cbbd4 100644 --- a/src/dbnode/storage/index/convert/convert.go +++ b/src/dbnode/storage/index/convert/convert.go @@ -264,6 +264,80 @@ func FromSeriesIDAndEncodedTags(id ident.ID, encodedTags ts.EncodedTags) (doc.Me return d, nil } +func FromSeriesIDAndEncodedTagsIndex(id ident.ID, encodedTags ts.EncodedTags) (doc.Metadata, error) { + total := len(encodedTags) + if total < 4 { + return doc.Metadata{}, fmt.Errorf( + "encoded tags too short: size=%d, need=%d", total, 4) + } + + var ( + byteOrder = binary.LittleEndian + headerMagicNumber uint16 = 10101 + + pos = 0 + ) + + header := byteOrder.Uint16(encodedTags[pos : pos+2]) + if header != headerMagicNumber { + return doc.Metadata{}, errors.New("") + } + pos += 2 + + length := int(byteOrder.Uint16(encodedTags[pos : pos+2])) + pos += 2 + + var ( + clonedID = clone(id.Bytes()) + fields = make([]doc.Field, 0, length) + expectedStart = firstTagBytesPosition + ) + + for i := 0; i < length; i++ { + if pos+2 > total { + return doc.Metadata{}, fmt.Errorf("missing size for tag name: index=%d", i) + } + numBytesName := int(byteOrder.Uint16(encodedTags[pos : pos+2])) + if numBytesName == 0 { + return doc.Metadata{}, errors.New("") + } + pos += 2 + + bytesName := encodedTags[pos : pos+numBytesName] + pos += numBytesName + + if pos+2 > total { + return doc.Metadata{}, fmt.Errorf("missing size for tag value: index=%d", i) + } + + numBytesValue := int(byteOrder.Uint16(encodedTags[pos : pos+2])) + pos += 2 + + bytesValue := encodedTags[pos : pos+numBytesValue] + pos += numBytesValue + + var clonedName, clonedValue []byte + clonedName, expectedStart = findSliceOrClone(clonedID, bytesName, expectedStart, + distanceBetweenTagNameAndValue) + clonedValue, expectedStart = findSliceOrClone(clonedID, bytesValue, expectedStart, + distanceBetweenTagValueAndNextName) + + fields = append(fields, doc.Field{ + Name: clonedName, + Value: clonedValue, + }) + } + + d := doc.Metadata{ + ID: clonedID, + Fields: fields, + } + if err := Validate(d); err != nil { + return doc.Metadata{}, err + } + return d, nil +} + func FromSeriesIDAndEncodedTags2(id ident.ID, encodedTags ts.EncodedTags) (doc.Metadata, error) { var ( length int diff --git a/src/dbnode/storage/index/convert/convert_benchmark_test.go b/src/dbnode/storage/index/convert/convert_benchmark_test.go index 7fcb6d7813..6b18f8ff87 100644 --- a/src/dbnode/storage/index/convert/convert_benchmark_test.go +++ b/src/dbnode/storage/index/convert/convert_benchmark_test.go @@ -166,6 +166,17 @@ func BenchmarkFromSeriesIDAndEncodedTags(b *testing.B) { } } +func BenchmarkFromSeriesIDAndEncodedTagsIndex(b *testing.B) { + testData, err := prepareIDAndEncodedTags(b) + require.NoError(b, err) + + b.ResetTimer() + for i := range testData { + _, err := FromSeriesIDAndEncodedTagsIndex(testData[i].id, testData[i].encodedTags) + require.NoError(b, err) + } +} + func BenchmarkFromSeriesIDAndEncodedTags2(b *testing.B) { testData, err := prepareIDAndEncodedTags(b) require.NoError(b, err) From c4296b65fd467cac4df36021059c53865ef1ca05 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Mon, 18 Jan 2021 15:35:04 +0200 Subject: [PATCH 14/22] benchmark TagValueFromEncodedTagsFast --- .../convert/decoder_fast_benchmark_test.go | 74 +++++++++++++++++++ src/x/serialize/decoder_fast.go | 50 +++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 src/dbnode/storage/index/convert/decoder_fast_benchmark_test.go diff --git a/src/dbnode/storage/index/convert/decoder_fast_benchmark_test.go b/src/dbnode/storage/index/convert/decoder_fast_benchmark_test.go new file mode 100644 index 0000000000..88efae2d7d --- /dev/null +++ b/src/dbnode/storage/index/convert/decoder_fast_benchmark_test.go @@ -0,0 +1,74 @@ +package convert + +import ( + "github.com/m3db/m3/src/x/checked" + "github.com/m3db/m3/src/x/pool" + "github.com/m3db/m3/src/x/serialize" + "github.com/stretchr/testify/require" + "math/rand" + "testing" +) + +type encodedTagsWithTagName struct { + encodedTags, tagName []byte +} + +func BenchmarkTagValueFromEncodedTagsFast(b *testing.B) { + testData, err := prepareData(b) + require.NoError(b, err) + + b.ResetTimer() + for i := range testData { + _, _, err := serialize.TagValueFromEncodedTagsFast(testData[i].encodedTags, + testData[i].tagName) + require.NoError(b, err) + } +} + +func BenchmarkTagValueFromEncodedTagsFast2(b *testing.B) { + testData, err := prepareData(b) + require.NoError(b, err) + + b.ResetTimer() + for i := range testData { + _, _, err := serialize.TagValueFromEncodedTagsFast2(testData[i].encodedTags, + testData[i].tagName) + require.NoError(b, err) + } +} + +func prepareData(b *testing.B) ([]encodedTagsWithTagName, error) { + data, err := prepareIDAndEncodedTags(b) + if err != nil { + return nil, err + } + + decoderPool := serialize.NewTagDecoderPool( + serialize.NewTagDecoderOptions(serialize.TagDecoderOptionsConfig{}), + pool.NewObjectPoolOptions(), + ) + decoderPool.Init() + decoder := decoderPool.Get() + defer decoder.Close() + + var tagNames [][]byte + decoder.Reset(checked.NewBytes(data[0].encodedTags, nil)) + for decoder.Next() { + tagNames = append(tagNames, clone(decoder.Current().Name.Bytes())) + } + tagNames = append(tagNames, []byte("not_exist")) + + var ( + result = make([]encodedTagsWithTagName, 0, b.N) + rnd = rand.New(rand.NewSource(42)) + ) + + for i := 0; i < b.N; i++ { + result = append(result, encodedTagsWithTagName{ + encodedTags: data[i].encodedTags, + tagName: tagNames[rnd.Intn(len(tagNames))], + }) + } + + return result, nil +} diff --git a/src/x/serialize/decoder_fast.go b/src/x/serialize/decoder_fast.go index f73cf9c131..60d8bb053d 100644 --- a/src/x/serialize/decoder_fast.go +++ b/src/x/serialize/decoder_fast.go @@ -30,6 +30,56 @@ import ( func TagValueFromEncodedTagsFast( encodedTags []byte, tagName []byte, +) ([]byte, bool, error) { + total := len(encodedTags) + if total < 4 { + return nil, false, fmt.Errorf( + "encoded tags too short: size=%d, need=%d", total, 4) + } + + header := byteOrder.Uint16(encodedTags[:2]) + encodedTags = encodedTags[2:] + if header != headerMagicNumber { + return nil, false, errIncorrectHeader + } + + length := int(byteOrder.Uint16(encodedTags[:2])) + encodedTags = encodedTags[2:] + + for i := 0; i < length; i++ { + if len(encodedTags) < 2 { + return nil, false, fmt.Errorf("missing size for tag name: index=%d", i) + } + numBytesName := int(byteOrder.Uint16(encodedTags[:2])) + if numBytesName == 0 { + return nil, false, errEmptyTagNameLiteral + } + encodedTags = encodedTags[2:] + + bytesName := encodedTags[:numBytesName] + encodedTags = encodedTags[numBytesName:] + + if len(encodedTags) < 2 { + return nil, false, fmt.Errorf("missing size for tag value: index=%d", i) + } + + numBytesValue := int(byteOrder.Uint16(encodedTags[:2])) + encodedTags = encodedTags[2:] + + bytesValue := encodedTags[:numBytesValue] + encodedTags = encodedTags[numBytesValue:] + + if bytes.Equal(bytesName, tagName) { + return bytesValue, true, nil + } + } + + return nil, false, nil +} + +func TagValueFromEncodedTagsFast2( + encodedTags []byte, + tagName []byte, ) ([]byte, bool, error) { var ( length int From 9b1ad47656a0466fa8c27e6b180dfbd4c9c01ef6 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Tue, 19 Jan 2021 10:12:17 +0200 Subject: [PATCH 15/22] revert decoder_fast.go changes --- src/dbnode/storage/index/convert/convert.go | 49 ------------ .../index/convert/convert_benchmark_test.go | 11 --- .../convert/decoder_fast_benchmark_test.go | 74 ------------------ src/x/serialize/decoder_fast.go | 77 +------------------ 4 files changed, 2 insertions(+), 209 deletions(-) delete mode 100644 src/dbnode/storage/index/convert/decoder_fast_benchmark_test.go diff --git a/src/dbnode/storage/index/convert/convert.go b/src/dbnode/storage/index/convert/convert.go index 8ac23cbbd4..5a373a42ab 100644 --- a/src/dbnode/storage/index/convert/convert.go +++ b/src/dbnode/storage/index/convert/convert.go @@ -33,7 +33,6 @@ import ( "github.com/m3db/m3/src/query/graphite/graphite" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/pool" - "github.com/m3db/m3/src/x/serialize" ) const ( @@ -338,54 +337,6 @@ func FromSeriesIDAndEncodedTagsIndex(id ident.ID, encodedTags ts.EncodedTags) (d return d, nil } -func FromSeriesIDAndEncodedTags2(id ident.ID, encodedTags ts.EncodedTags) (doc.Metadata, error) { - var ( - length int - err error - ) - encodedTags, length, err = serialize.DecodeHeader(encodedTags) - if err != nil { - return doc.Metadata{}, err - } - - var ( - clonedID = clone(id.Bytes()) - fields = make([]doc.Field, 0, length) - expectedStart = firstTagBytesPosition - ) - - for i := 0; i < length; i++ { - var nameBytes, valueBytes []byte - encodedTags, nameBytes, valueBytes, err = serialize.DecodeTag(encodedTags) - if err != nil { - return doc.Metadata{}, err - } - - var clonedName, clonedValue []byte - clonedName, expectedStart = findSliceOrClone(clonedID, nameBytes, expectedStart, - distanceBetweenTagNameAndValue) - clonedValue, expectedStart = findSliceOrClone(clonedID, valueBytes, expectedStart, - distanceBetweenTagValueAndNextName) - - fields = append(fields, doc.Field{ - Name: clonedName, - Value: clonedValue, - }) - } - - if err != nil { - return doc.Metadata{}, err - } - d := doc.Metadata{ - ID: clonedID, - Fields: fields, - } - if err := Validate(d); err != nil { - return doc.Metadata{}, err - } - return d, nil -} - func FromSeriesIDAndFastTagIter(id ident.ID, tags id.SortedTagIterator) (doc.Metadata, error) { var ( clonedID = clone(id.Bytes()) diff --git a/src/dbnode/storage/index/convert/convert_benchmark_test.go b/src/dbnode/storage/index/convert/convert_benchmark_test.go index 6b18f8ff87..956b88effa 100644 --- a/src/dbnode/storage/index/convert/convert_benchmark_test.go +++ b/src/dbnode/storage/index/convert/convert_benchmark_test.go @@ -177,17 +177,6 @@ func BenchmarkFromSeriesIDAndEncodedTagsIndex(b *testing.B) { } } -func BenchmarkFromSeriesIDAndEncodedTags2(b *testing.B) { - testData, err := prepareIDAndEncodedTags(b) - require.NoError(b, err) - - b.ResetTimer() - for i := range testData { - _, err := FromSeriesIDAndEncodedTags2(testData[i].id, testData[i].encodedTags) - require.NoError(b, err) - } -} - func BenchmarkFromSeriesIDAndFastTagIter(b *testing.B) { testData, err := prepareIDAndEncodedTags(b) require.NoError(b, err) diff --git a/src/dbnode/storage/index/convert/decoder_fast_benchmark_test.go b/src/dbnode/storage/index/convert/decoder_fast_benchmark_test.go deleted file mode 100644 index 88efae2d7d..0000000000 --- a/src/dbnode/storage/index/convert/decoder_fast_benchmark_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package convert - -import ( - "github.com/m3db/m3/src/x/checked" - "github.com/m3db/m3/src/x/pool" - "github.com/m3db/m3/src/x/serialize" - "github.com/stretchr/testify/require" - "math/rand" - "testing" -) - -type encodedTagsWithTagName struct { - encodedTags, tagName []byte -} - -func BenchmarkTagValueFromEncodedTagsFast(b *testing.B) { - testData, err := prepareData(b) - require.NoError(b, err) - - b.ResetTimer() - for i := range testData { - _, _, err := serialize.TagValueFromEncodedTagsFast(testData[i].encodedTags, - testData[i].tagName) - require.NoError(b, err) - } -} - -func BenchmarkTagValueFromEncodedTagsFast2(b *testing.B) { - testData, err := prepareData(b) - require.NoError(b, err) - - b.ResetTimer() - for i := range testData { - _, _, err := serialize.TagValueFromEncodedTagsFast2(testData[i].encodedTags, - testData[i].tagName) - require.NoError(b, err) - } -} - -func prepareData(b *testing.B) ([]encodedTagsWithTagName, error) { - data, err := prepareIDAndEncodedTags(b) - if err != nil { - return nil, err - } - - decoderPool := serialize.NewTagDecoderPool( - serialize.NewTagDecoderOptions(serialize.TagDecoderOptionsConfig{}), - pool.NewObjectPoolOptions(), - ) - decoderPool.Init() - decoder := decoderPool.Get() - defer decoder.Close() - - var tagNames [][]byte - decoder.Reset(checked.NewBytes(data[0].encodedTags, nil)) - for decoder.Next() { - tagNames = append(tagNames, clone(decoder.Current().Name.Bytes())) - } - tagNames = append(tagNames, []byte("not_exist")) - - var ( - result = make([]encodedTagsWithTagName, 0, b.N) - rnd = rand.New(rand.NewSource(42)) - ) - - for i := 0; i < b.N; i++ { - result = append(result, encodedTagsWithTagName{ - encodedTags: data[i].encodedTags, - tagName: tagNames[rnd.Intn(len(tagNames))], - }) - } - - return result, nil -} diff --git a/src/x/serialize/decoder_fast.go b/src/x/serialize/decoder_fast.go index 60d8bb053d..ef5ac8d406 100644 --- a/src/x/serialize/decoder_fast.go +++ b/src/x/serialize/decoder_fast.go @@ -51,7 +51,7 @@ func TagValueFromEncodedTagsFast( return nil, false, fmt.Errorf("missing size for tag name: index=%d", i) } numBytesName := int(byteOrder.Uint16(encodedTags[:2])) - if numBytesName == 0 { + if numBytesName <= 0 { return nil, false, errEmptyTagNameLiteral } encodedTags = encodedTags[2:] @@ -69,83 +69,10 @@ func TagValueFromEncodedTagsFast( bytesValue := encodedTags[:numBytesValue] encodedTags = encodedTags[numBytesValue:] - if bytes.Equal(bytesName, tagName) { + if bytes.Compare(bytesName, tagName) == 0 { return bytesValue, true, nil } } return nil, false, nil } - -func TagValueFromEncodedTagsFast2( - encodedTags []byte, - tagName []byte, -) ([]byte, bool, error) { - var ( - length int - err error - ) - encodedTags, length, err = DecodeHeader(encodedTags) - if err != nil { - return nil, false, err - } - - for i := 0; i < length; i++ { - var bytesName, bytesValue []byte - - encodedTags, bytesName, bytesValue, err = DecodeTag(encodedTags) - if err != nil { - return nil, false, err - } - - if bytes.Equal(bytesName, tagName) { - return bytesValue, true, nil - } - } - - return nil, false, nil -} - -func DecodeHeader(encodedTags []byte) ([]byte, int, error) { - total := len(encodedTags) - if total < 4 { - return nil, 0, fmt.Errorf( - "encoded tags too short: size=%d, need=%d", total, 4) - } - - header := byteOrder.Uint16(encodedTags[:2]) - encodedTags = encodedTags[2:] - if header != headerMagicNumber { - return nil, 0, errIncorrectHeader - } - - length := int(byteOrder.Uint16(encodedTags[:2])) - encodedTags = encodedTags[2:] - return encodedTags, length, nil -} - -func DecodeTag(encodedTags []byte) ([]byte, []byte, []byte, error) { - if len(encodedTags) < 2 { - return nil, nil, nil, fmt.Errorf("missing size for tag name") - } - numBytesName := int(byteOrder.Uint16(encodedTags[:2])) - if numBytesName == 0 { - return nil, nil, nil, errEmptyTagNameLiteral - } - encodedTags = encodedTags[2:] - - bytesName := encodedTags[:numBytesName] - encodedTags = encodedTags[numBytesName:] - - if len(encodedTags) < 2 { - return nil, nil, nil, fmt.Errorf("missing size for tag value") - } - - numBytesValue := int(byteOrder.Uint16(encodedTags[:2])) - encodedTags = encodedTags[2:] - - bytesValue := encodedTags[:numBytesValue] - encodedTags = encodedTags[numBytesValue:] - - return encodedTags, bytesName, bytesValue, nil -} From ba7721717efa327f543ad8ce302cfbf354ffe82a Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Tue, 19 Jan 2021 10:19:18 +0200 Subject: [PATCH 16/22] revert decoder_fast_iter.go changes --- src/dbnode/storage/index/convert/convert.go | 35 -------- .../index/convert/convert_benchmark_test.go | 13 --- src/metrics/metric/id/tag.go | 2 - src/x/serialize/decoder_fast_iter.go | 85 ------------------- src/x/serialize/types.go | 1 + 5 files changed, 1 insertion(+), 135 deletions(-) delete mode 100644 src/x/serialize/decoder_fast_iter.go diff --git a/src/dbnode/storage/index/convert/convert.go b/src/dbnode/storage/index/convert/convert.go index 5a373a42ab..aadd0d9556 100644 --- a/src/dbnode/storage/index/convert/convert.go +++ b/src/dbnode/storage/index/convert/convert.go @@ -29,7 +29,6 @@ import ( "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/m3ninx/doc" - "github.com/m3db/m3/src/metrics/metric/id" "github.com/m3db/m3/src/query/graphite/graphite" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/pool" @@ -337,40 +336,6 @@ func FromSeriesIDAndEncodedTagsIndex(id ident.ID, encodedTags ts.EncodedTags) (d return d, nil } -func FromSeriesIDAndFastTagIter(id ident.ID, tags id.SortedTagIterator) (doc.Metadata, error) { - var ( - clonedID = clone(id.Bytes()) - fields = make([]doc.Field, 0, tags.NumTags()) - expectedStart = firstTagBytesPosition - ) - for tags.Next() { - nameBytes, valueBytes := tags.Current() - - var clonedName, clonedValue []byte - clonedName, expectedStart = findSliceOrClone(clonedID, nameBytes, expectedStart, - distanceBetweenTagNameAndValue) - clonedValue, expectedStart = findSliceOrClone(clonedID, valueBytes, expectedStart, - distanceBetweenTagValueAndNextName) - - fields = append(fields, doc.Field{ - Name: clonedName, - Value: clonedValue, - }) - } - if err := tags.Err(); err != nil { - return doc.Metadata{}, err - } - - d := doc.Metadata{ - ID: clonedID, - Fields: fields, - } - if err := Validate(d); err != nil { - return doc.Metadata{}, err - } - return d, nil -} - func findSliceOrClone(id, tag []byte, expectedStart, nextPositionDistance int) ([]byte, int) { //nolint:unparam n := len(tag) expectedEnd := expectedStart + n diff --git a/src/dbnode/storage/index/convert/convert_benchmark_test.go b/src/dbnode/storage/index/convert/convert_benchmark_test.go index 956b88effa..e74c8d03b9 100644 --- a/src/dbnode/storage/index/convert/convert_benchmark_test.go +++ b/src/dbnode/storage/index/convert/convert_benchmark_test.go @@ -177,19 +177,6 @@ func BenchmarkFromSeriesIDAndEncodedTagsIndex(b *testing.B) { } } -func BenchmarkFromSeriesIDAndFastTagIter(b *testing.B) { - testData, err := prepareIDAndEncodedTags(b) - require.NoError(b, err) - decoder := serialize.NewFastTagDecoder() - - b.ResetTimer() - for i := range testData { - decoder.Reset(testData[i].encodedTags) - _, err := FromSeriesIDAndFastTagIter(testData[i].id, decoder) - require.NoError(b, err) - } -} - func BenchmarkFromSeriesIDAndTagIter_TagDecoder(b *testing.B) { testData, err := prepareIDAndEncodedTags(b) require.NoError(b, err) diff --git a/src/metrics/metric/id/tag.go b/src/metrics/metric/id/tag.go index 5bb7e368f5..d5561f3f3b 100644 --- a/src/metrics/metric/id/tag.go +++ b/src/metrics/metric/id/tag.go @@ -61,8 +61,6 @@ type SortedTagIterator interface { // Close closes the iterator. Close() - - NumTags() int } // SortedTagIteratorAlloc allocates a new sorted tag iterator. diff --git a/src/x/serialize/decoder_fast_iter.go b/src/x/serialize/decoder_fast_iter.go deleted file mode 100644 index 646da4e74f..0000000000 --- a/src/x/serialize/decoder_fast_iter.go +++ /dev/null @@ -1,85 +0,0 @@ -package serialize - -import ( - "errors" - "fmt" - - "github.com/m3db/m3/src/metrics/metric/id" -) - -type decoderFast struct { - length int - remaining int - - encodedTags []byte - - currentName, currentValue []byte - - err error -} - -func NewFastTagDecoder() id.SortedTagIterator { - return &decoderFast{} -} - -func (d *decoderFast) Reset(encodedTags []byte) { - header := byteOrder.Uint16(encodedTags[:2]) - encodedTags = encodedTags[2:] - if header != headerMagicNumber { - d.err = errors.New("") - return - } - - d.length = int(byteOrder.Uint16(encodedTags[:2])) - d.encodedTags = encodedTags[2:] - d.remaining = d.length -} - -func (d *decoderFast) Next() bool { - if d.remaining == 0 { - return false - } - - d.remaining-- - - if len(d.encodedTags) < 2 { - d.err = fmt.Errorf("missing size for tag name: index=%d", d.length-d.remaining) - return false - } - numBytesName := int(byteOrder.Uint16(d.encodedTags[:2])) - if numBytesName == 0 { - d.err = errors.New("") - return false - } - d.encodedTags = d.encodedTags[2:] - - d.currentName = d.encodedTags[:numBytesName] - d.encodedTags = d.encodedTags[numBytesName:] - - if len(d.encodedTags) < 2 { - d.err = fmt.Errorf("missing size for tag value: index=%d", d.length-d.remaining) - return false - } - - numBytesValue := int(byteOrder.Uint16(d.encodedTags[:2])) - d.encodedTags = d.encodedTags[2:] - - d.currentValue = d.encodedTags[:numBytesValue] - d.encodedTags = d.encodedTags[numBytesValue:] - return true -} - -func (d *decoderFast) Current() ([]byte, []byte) { - return d.currentName, d.currentValue -} - -func (d *decoderFast) Err() error { - return d.err -} - -func (d *decoderFast) Close() { -} - -func (d *decoderFast) NumTags() int { - return d.length -} diff --git a/src/x/serialize/types.go b/src/x/serialize/types.go index 600113d81b..a694ae7dc7 100644 --- a/src/x/serialize/types.go +++ b/src/x/serialize/types.go @@ -138,6 +138,7 @@ type TagSerializationLimits interface { type MetricTagsIterator interface { id.ID id.SortedTagIterator + NumTags() int } // MetricTagsIteratorPool pools MetricTagsIterator. From 3e949affcc5eb495591a7774b977b582efea1711 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Tue, 19 Jan 2021 10:20:55 +0200 Subject: [PATCH 17/22] remove convert.FromSeriesIDAndEncodedTagsIndex() --- src/dbnode/storage/index/convert/convert.go | 74 ------------------- .../index/convert/convert_benchmark_test.go | 11 --- 2 files changed, 85 deletions(-) diff --git a/src/dbnode/storage/index/convert/convert.go b/src/dbnode/storage/index/convert/convert.go index aadd0d9556..d89b4b3bed 100644 --- a/src/dbnode/storage/index/convert/convert.go +++ b/src/dbnode/storage/index/convert/convert.go @@ -262,80 +262,6 @@ func FromSeriesIDAndEncodedTags(id ident.ID, encodedTags ts.EncodedTags) (doc.Me return d, nil } -func FromSeriesIDAndEncodedTagsIndex(id ident.ID, encodedTags ts.EncodedTags) (doc.Metadata, error) { - total := len(encodedTags) - if total < 4 { - return doc.Metadata{}, fmt.Errorf( - "encoded tags too short: size=%d, need=%d", total, 4) - } - - var ( - byteOrder = binary.LittleEndian - headerMagicNumber uint16 = 10101 - - pos = 0 - ) - - header := byteOrder.Uint16(encodedTags[pos : pos+2]) - if header != headerMagicNumber { - return doc.Metadata{}, errors.New("") - } - pos += 2 - - length := int(byteOrder.Uint16(encodedTags[pos : pos+2])) - pos += 2 - - var ( - clonedID = clone(id.Bytes()) - fields = make([]doc.Field, 0, length) - expectedStart = firstTagBytesPosition - ) - - for i := 0; i < length; i++ { - if pos+2 > total { - return doc.Metadata{}, fmt.Errorf("missing size for tag name: index=%d", i) - } - numBytesName := int(byteOrder.Uint16(encodedTags[pos : pos+2])) - if numBytesName == 0 { - return doc.Metadata{}, errors.New("") - } - pos += 2 - - bytesName := encodedTags[pos : pos+numBytesName] - pos += numBytesName - - if pos+2 > total { - return doc.Metadata{}, fmt.Errorf("missing size for tag value: index=%d", i) - } - - numBytesValue := int(byteOrder.Uint16(encodedTags[pos : pos+2])) - pos += 2 - - bytesValue := encodedTags[pos : pos+numBytesValue] - pos += numBytesValue - - var clonedName, clonedValue []byte - clonedName, expectedStart = findSliceOrClone(clonedID, bytesName, expectedStart, - distanceBetweenTagNameAndValue) - clonedValue, expectedStart = findSliceOrClone(clonedID, bytesValue, expectedStart, - distanceBetweenTagValueAndNextName) - - fields = append(fields, doc.Field{ - Name: clonedName, - Value: clonedValue, - }) - } - - d := doc.Metadata{ - ID: clonedID, - Fields: fields, - } - if err := Validate(d); err != nil { - return doc.Metadata{}, err - } - return d, nil -} - func findSliceOrClone(id, tag []byte, expectedStart, nextPositionDistance int) ([]byte, int) { //nolint:unparam n := len(tag) expectedEnd := expectedStart + n diff --git a/src/dbnode/storage/index/convert/convert_benchmark_test.go b/src/dbnode/storage/index/convert/convert_benchmark_test.go index e74c8d03b9..2f93341b89 100644 --- a/src/dbnode/storage/index/convert/convert_benchmark_test.go +++ b/src/dbnode/storage/index/convert/convert_benchmark_test.go @@ -166,17 +166,6 @@ func BenchmarkFromSeriesIDAndEncodedTags(b *testing.B) { } } -func BenchmarkFromSeriesIDAndEncodedTagsIndex(b *testing.B) { - testData, err := prepareIDAndEncodedTags(b) - require.NoError(b, err) - - b.ResetTimer() - for i := range testData { - _, err := FromSeriesIDAndEncodedTagsIndex(testData[i].id, testData[i].encodedTags) - require.NoError(b, err) - } -} - func BenchmarkFromSeriesIDAndTagIter_TagDecoder(b *testing.B) { testData, err := prepareIDAndEncodedTags(b) require.NoError(b, err) From e0cad90c44232458bf6f6892bbe4304a5a5e2cf0 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Tue, 19 Jan 2021 11:17:23 +0200 Subject: [PATCH 18/22] export and use variables from serialize package --- src/dbnode/storage/index/convert/convert.go | 22 ++++++++++----------- src/x/serialize/decoder.go | 10 ++++++---- src/x/serialize/decoder_fast.go | 14 ++++++------- src/x/serialize/encoder.go | 18 +++++++++-------- src/x/serialize/types.go | 6 +++--- 5 files changed, 36 insertions(+), 34 deletions(-) diff --git a/src/dbnode/storage/index/convert/convert.go b/src/dbnode/storage/index/convert/convert.go index d89b4b3bed..e44c0d1f15 100644 --- a/src/dbnode/storage/index/convert/convert.go +++ b/src/dbnode/storage/index/convert/convert.go @@ -22,7 +22,6 @@ package convert import ( "bytes" - "encoding/binary" "errors" "fmt" "unicode/utf8" @@ -32,6 +31,7 @@ import ( "github.com/m3db/m3/src/query/graphite/graphite" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/pool" + "github.com/m3db/m3/src/x/serialize" ) const ( @@ -190,22 +190,20 @@ func FromSeriesIDAndTagIter(id ident.ID, tags ident.TagIterator) (doc.Metadata, return d, nil } +// FromSeriesIDAndEncodedTags converts the provided series id and encoded tags into a document. func FromSeriesIDAndEncodedTags(id ident.ID, encodedTags ts.EncodedTags) (doc.Metadata, error) { - total := len(encodedTags) - if total < 4 { - return doc.Metadata{}, fmt.Errorf( - "encoded tags too short: size=%d, need=%d", total, 4) - } - var ( - byteOrder = binary.LittleEndian - headerMagicNumber uint16 = 10101 + byteOrder = serialize.ByteOrder + total = len(encodedTags) ) + if total < 4 { + return doc.Metadata{}, fmt.Errorf("encoded tags too short: size=%d, need=%d", total, 4) + } header := byteOrder.Uint16(encodedTags[:2]) encodedTags = encodedTags[2:] - if header != headerMagicNumber { - return doc.Metadata{}, errors.New("") + if header != serialize.HeaderMagicNumber { + return doc.Metadata{}, serialize.ErrIncorrectHeader } length := int(byteOrder.Uint16(encodedTags[:2])) @@ -223,7 +221,7 @@ func FromSeriesIDAndEncodedTags(id ident.ID, encodedTags ts.EncodedTags) (doc.Me } numBytesName := int(byteOrder.Uint16(encodedTags[:2])) if numBytesName == 0 { - return doc.Metadata{}, errors.New("") + return doc.Metadata{}, serialize.ErrEmptyTagNameLiteral } encodedTags = encodedTags[2:] diff --git a/src/x/serialize/decoder.go b/src/x/serialize/decoder.go index 9d1b5d9dce..574b4c1109 100644 --- a/src/x/serialize/decoder.go +++ b/src/x/serialize/decoder.go @@ -29,7 +29,9 @@ import ( ) var ( - errIncorrectHeader = errors.New("header magic number does not match expected value") + // ErrIncorrectHeader is an error when encoded tag byte sequence doesn't start with + // an expected magic number. + ErrIncorrectHeader = errors.New("header magic number does not match expected value") errInvalidByteStreamIDDecoding = errors.New("internal error, invalid byte stream while decoding ID") errInvalidByteStreamUintDecoding = errors.New("internal error, invalid byte stream while decoding uint") ) @@ -78,8 +80,8 @@ func (d *decoder) Reset(b checked.Bytes) { return } - if header != headerMagicNumber { - d.err = errIncorrectHeader + if header != HeaderMagicNumber { + d.err = ErrIncorrectHeader return } @@ -129,7 +131,7 @@ func (d *decoder) decodeTag() error { // safe to call Bytes() as d.current.Name has inc'd a ref if len(d.currentTagName.Bytes()) == 0 { d.releaseCurrent() - return errEmptyTagNameLiteral + return ErrEmptyTagNameLiteral } if err := d.decodeIDInto(d.currentTagValue); err != nil { diff --git a/src/x/serialize/decoder_fast.go b/src/x/serialize/decoder_fast.go index ef5ac8d406..10e44488ce 100644 --- a/src/x/serialize/decoder_fast.go +++ b/src/x/serialize/decoder_fast.go @@ -37,22 +37,22 @@ func TagValueFromEncodedTagsFast( "encoded tags too short: size=%d, need=%d", total, 4) } - header := byteOrder.Uint16(encodedTags[:2]) + header := ByteOrder.Uint16(encodedTags[:2]) encodedTags = encodedTags[2:] - if header != headerMagicNumber { - return nil, false, errIncorrectHeader + if header != HeaderMagicNumber { + return nil, false, ErrIncorrectHeader } - length := int(byteOrder.Uint16(encodedTags[:2])) + length := int(ByteOrder.Uint16(encodedTags[:2])) encodedTags = encodedTags[2:] for i := 0; i < length; i++ { if len(encodedTags) < 2 { return nil, false, fmt.Errorf("missing size for tag name: index=%d", i) } - numBytesName := int(byteOrder.Uint16(encodedTags[:2])) + numBytesName := int(ByteOrder.Uint16(encodedTags[:2])) if numBytesName <= 0 { - return nil, false, errEmptyTagNameLiteral + return nil, false, ErrEmptyTagNameLiteral } encodedTags = encodedTags[2:] @@ -63,7 +63,7 @@ func TagValueFromEncodedTagsFast( return nil, false, fmt.Errorf("missing size for tag value: index=%d", i) } - numBytesValue := int(byteOrder.Uint16(encodedTags[:2])) + numBytesValue := int(ByteOrder.Uint16(encodedTags[:2])) encodedTags = encodedTags[2:] bytesValue := encodedTags[:numBytesValue] diff --git a/src/x/serialize/encoder.go b/src/x/serialize/encoder.go index a682a639ee..5a67c98152 100644 --- a/src/x/serialize/encoder.go +++ b/src/x/serialize/encoder.go @@ -51,18 +51,20 @@ import ( */ var ( - byteOrder binary.ByteOrder = binary.LittleEndian + // ByteOrder is the byte order used for encoding tags into a byte sequence. + ByteOrder binary.ByteOrder = binary.LittleEndian headerMagicBytes = make([]byte, 2) ) func init() { - encodeUInt16(headerMagicNumber, headerMagicBytes) + encodeUInt16(HeaderMagicNumber, headerMagicBytes) } var ( - errTagEncoderInUse = errors.New("encoder already in use") - errTagLiteralTooLong = errors.New("literal is too long") - errEmptyTagNameLiteral = xerrors.NewInvalidParamsError(errors.New("tag name cannot be empty")) + errTagEncoderInUse = errors.New("encoder already in use") + errTagLiteralTooLong = errors.New("literal is too long") + // ErrEmptyTagNameLiteral is an error when encoded tag name is empty + ErrEmptyTagNameLiteral = xerrors.NewInvalidParamsError(errors.New("tag name cannot be empty")) ) type newCheckedBytesFn func([]byte, checked.BytesOptions) checked.Bytes @@ -166,7 +168,7 @@ func (e *encoder) Finalize() { func (e *encoder) encodeTag(t ident.Tag) error { if len(t.Name.Bytes()) == 0 { - return errEmptyTagNameLiteral + return ErrEmptyTagNameLiteral } if err := e.encodeID(t.Name); err != nil { @@ -204,10 +206,10 @@ func (e *encoder) encodeUInt16(v uint16) []byte { } func encodeUInt16(v uint16, dest []byte) []byte { - byteOrder.PutUint16(dest, v) + ByteOrder.PutUint16(dest, v) return dest } func decodeUInt16(b []byte) uint16 { - return byteOrder.Uint16(b) + return ByteOrder.Uint16(b) } diff --git a/src/x/serialize/types.go b/src/x/serialize/types.go index a694ae7dc7..e3b31e6e7f 100644 --- a/src/x/serialize/types.go +++ b/src/x/serialize/types.go @@ -27,10 +27,10 @@ import ( "github.com/m3db/m3/src/x/ident" ) -var ( - // headerMagicNumber is an internal header used to denote the beginning of +const ( + // HeaderMagicNumber is an internal header used to denote the beginning of // an encoded stream. - headerMagicNumber uint16 = 10101 + HeaderMagicNumber uint16 = 10101 ) // TagEncoder encodes provided Tag iterators. From f77ccefad3d475c27f6d7c6cb7d3ac81ea022016 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Tue, 19 Jan 2021 11:51:57 +0200 Subject: [PATCH 19/22] add tests --- .../storage/index/convert/convert_test.go | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/src/dbnode/storage/index/convert/convert_test.go b/src/dbnode/storage/index/convert/convert_test.go index aa72449797..5537df2f45 100644 --- a/src/dbnode/storage/index/convert/convert_test.go +++ b/src/dbnode/storage/index/convert/convert_test.go @@ -30,6 +30,7 @@ import ( "github.com/m3db/m3/src/x/checked" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/pool" + "github.com/m3db/m3/src/x/serialize" "github.com/m3db/m3/src/x/test" "github.com/stretchr/testify/assert" @@ -171,6 +172,113 @@ func TestFromSeriesIDAndTagIterReuseBytesFromSeriesId(t *testing.T) { } } +func TestFromSeriesIDAndEncodedTagsInvalid(t *testing.T) { + var ( + id = ident.StringID("foo") + tags = ident.NewTags( + ident.StringTag(string(convert.ReservedFieldNameID), "value"), + ) + encodedTags = toEncodedTags(t, tags) + ) + _, err := convert.FromSeriesIDAndEncodedTags(id, encodedTags) + assert.Error(t, err) +} + +func TestFromSeriesIDAndEncodedTagsValid(t *testing.T) { + var ( + id = ident.StringID("foo") + tags = ident.NewTags( + ident.StringTag("bar", "baz"), + ) + encodedTags = toEncodedTags(t, tags) + ) + d, err := convert.FromSeriesIDAndEncodedTags(id, encodedTags) + assert.NoError(t, err) + assertContentsMatch(t, id, tags.Values(), d) +} + +func TestFromSeriesIDAndEncodedTagsReuseBytesFromSeriesId(t *testing.T) { + tests := []struct { + name string + id string + }{ + { + name: "tags in ID", + id: "bar=baz,quip=quix", + }, + { + name: "tags in ID with specific format", + id: `{bar="baz",quip="quix"}`, + }, + { + name: "tags in ID with specific format reverse order", + id: `{quip="quix",bar="baz"}`, + }, + { + name: "inexact tag occurrence in ID", + id: "quixquip_bazillion_barometers", + }, + } + var ( + tags = ident.NewTags( + ident.StringTag("bar", "baz"), + ident.StringTag("quip", "quix"), + ) + encodedTags = toEncodedTags(t, tags) + ) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + seriesID := ident.StringID(tt.id) + d, err := convert.FromSeriesIDAndEncodedTags(seriesID, encodedTags) + assert.NoError(t, err) + assertContentsMatch(t, seriesID, tags.Values(), d) + for i := range d.Fields { + assertBackedBySameData(t, d.ID, d.Fields[i].Name) + assertBackedBySameData(t, d.ID, d.Fields[i].Value) + } + }) + } +} + +func TestFromSeriesIDAndEncodedTagsInvalidEncodedTags(t *testing.T) { + validEncodedTags := []byte{117, 39, 1, 0, 3, 0, 98, 97, 114, 3, 0, 98, 97, 122} + + tests := []struct { + name string + encodedTags []byte + }{ + { + name: "incomplete header", + encodedTags: validEncodedTags[:3], + }, + { + name: "incomplete tag name length", + encodedTags: validEncodedTags[:5], + }, + { + name: "incomplete tag value length", + encodedTags: validEncodedTags[:10], + }, + { + name: "invalid magic number", + encodedTags: []byte{42, 42, 0, 0}, + }, + { + name: "empty tag name", + encodedTags: []byte{117, 39, 1, 0, 0, 0, 3, 0, 98, 97, 122}, + }, + } + seriesID := ident.StringID("foo") + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := convert.FromSeriesIDAndEncodedTags(seriesID, tt.encodedTags) + assert.Error(t, err) + }) + } +} + func TestToSeriesValid(t *testing.T) { d := doc.Metadata{ ID: []byte("foo"), @@ -311,3 +419,15 @@ func assertBackedBySameData(t *testing.T, outer, inner []byte) { assert.Fail(t, "inner byte sequence wasn't found") } } + +func toEncodedTags(t *testing.T, tags ident.Tags) []byte { + pool := serialize.NewTagEncoderPool(serialize.NewTagEncoderOptions(), nil) + pool.Init() + encoder := pool.Get() + defer encoder.Finalize() + + require.NoError(t, encoder.Encode(ident.NewTagsIterator(tags))) + data, ok := encoder.Data() + require.True(t, ok) + return append([]byte(nil), data.Bytes()...) +} From 45378ef731ac2e59567cb6dab72b3d059cc6185a Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Tue, 19 Jan 2021 22:23:00 +1100 Subject: [PATCH 20/22] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Linas Medžiūnas --- src/dbnode/storage/index/convert/convert.go | 2 +- src/x/serialize/encoder.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dbnode/storage/index/convert/convert.go b/src/dbnode/storage/index/convert/convert.go index e44c0d1f15..a79fc9c2dd 100644 --- a/src/dbnode/storage/index/convert/convert.go +++ b/src/dbnode/storage/index/convert/convert.go @@ -190,7 +190,7 @@ func FromSeriesIDAndTagIter(id ident.ID, tags ident.TagIterator) (doc.Metadata, return d, nil } -// FromSeriesIDAndEncodedTags converts the provided series id and encoded tags into a document. +// FromSeriesIDAndEncodedTags converts the provided series id and encoded tags into a doc.Metadata. func FromSeriesIDAndEncodedTags(id ident.ID, encodedTags ts.EncodedTags) (doc.Metadata, error) { var ( byteOrder = serialize.ByteOrder diff --git a/src/x/serialize/encoder.go b/src/x/serialize/encoder.go index 5a67c98152..19bd884a25 100644 --- a/src/x/serialize/encoder.go +++ b/src/x/serialize/encoder.go @@ -63,7 +63,7 @@ func init() { var ( errTagEncoderInUse = errors.New("encoder already in use") errTagLiteralTooLong = errors.New("literal is too long") - // ErrEmptyTagNameLiteral is an error when encoded tag name is empty + // ErrEmptyTagNameLiteral is an error when encoded tag name is empty. ErrEmptyTagNameLiteral = xerrors.NewInvalidParamsError(errors.New("tag name cannot be empty")) ) From a5436e8235c627486c9b0aec6ae438a82e5c30c7 Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Tue, 19 Jan 2021 13:35:30 +0200 Subject: [PATCH 21/22] PR comments --- .../storage/index/convert/convert_test.go | 46 +++++++------------ 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/src/dbnode/storage/index/convert/convert_test.go b/src/dbnode/storage/index/convert/convert_test.go index 5537df2f45..005d932110 100644 --- a/src/dbnode/storage/index/convert/convert_test.go +++ b/src/dbnode/storage/index/convert/convert_test.go @@ -172,36 +172,15 @@ func TestFromSeriesIDAndTagIterReuseBytesFromSeriesId(t *testing.T) { } } -func TestFromSeriesIDAndEncodedTagsInvalid(t *testing.T) { - var ( - id = ident.StringID("foo") - tags = ident.NewTags( - ident.StringTag(string(convert.ReservedFieldNameID), "value"), - ) - encodedTags = toEncodedTags(t, tags) - ) - _, err := convert.FromSeriesIDAndEncodedTags(id, encodedTags) - assert.Error(t, err) -} - -func TestFromSeriesIDAndEncodedTagsValid(t *testing.T) { - var ( - id = ident.StringID("foo") - tags = ident.NewTags( - ident.StringTag("bar", "baz"), - ) - encodedTags = toEncodedTags(t, tags) - ) - d, err := convert.FromSeriesIDAndEncodedTags(id, encodedTags) - assert.NoError(t, err) - assertContentsMatch(t, id, tags.Values(), d) -} - -func TestFromSeriesIDAndEncodedTagsReuseBytesFromSeriesId(t *testing.T) { +func TestFromSeriesIDAndEncodedTags(t *testing.T) { tests := []struct { name string id string }{ + { + name: "no tags in ID", + id: "foo", + }, { name: "tags in ID", id: "bar=baz,quip=quix", @@ -241,13 +220,22 @@ func TestFromSeriesIDAndEncodedTagsReuseBytesFromSeriesId(t *testing.T) { } } -func TestFromSeriesIDAndEncodedTagsInvalidEncodedTags(t *testing.T) { - validEncodedTags := []byte{117, 39, 1, 0, 3, 0, 98, 97, 114, 3, 0, 98, 97, 122} +func TestFromSeriesIDAndEncodedTagsInvalid(t *testing.T) { + var ( + validEncodedTags = []byte{117, 39, 1, 0, 3, 0, 98, 97, 114, 3, 0, 98, 97, 122} + tagsWithReservedName = toEncodedTags(t, ident.NewTags( + ident.StringTag(string(convert.ReservedFieldNameID), "some_value"), + )) + ) tests := []struct { name string encodedTags []byte }{ + { + name: "reserved tag name", + encodedTags: tagsWithReservedName, + }, { name: "incomplete header", encodedTags: validEncodedTags[:3], @@ -415,8 +403,6 @@ func assertBackedBySameData(t *testing.T, outer, inner []byte) { if idx := bytes.Index(outer, inner); idx != -1 { subslice := outer[idx : idx+len(inner)] assert.True(t, test.ByteSlicesBackedBySameData(subslice, inner)) - } else { - assert.Fail(t, "inner byte sequence wasn't found") } } From 8b725f006584dce7951b0a8d4a83b64e05c644bd Mon Sep 17 00:00:00 2001 From: Vilius Pranckaitis Date: Tue, 19 Jan 2021 14:10:04 +0200 Subject: [PATCH 22/22] use ident.BytesID instead of ident.ID --- src/dbnode/storage/index/convert/convert.go | 2 +- src/dbnode/storage/index/convert/convert_benchmark_test.go | 4 ++-- src/dbnode/storage/index/convert/convert_test.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/dbnode/storage/index/convert/convert.go b/src/dbnode/storage/index/convert/convert.go index a79fc9c2dd..a6178b3ac9 100644 --- a/src/dbnode/storage/index/convert/convert.go +++ b/src/dbnode/storage/index/convert/convert.go @@ -191,7 +191,7 @@ func FromSeriesIDAndTagIter(id ident.ID, tags ident.TagIterator) (doc.Metadata, } // FromSeriesIDAndEncodedTags converts the provided series id and encoded tags into a doc.Metadata. -func FromSeriesIDAndEncodedTags(id ident.ID, encodedTags ts.EncodedTags) (doc.Metadata, error) { +func FromSeriesIDAndEncodedTags(id ident.BytesID, encodedTags ts.EncodedTags) (doc.Metadata, error) { var ( byteOrder = serialize.ByteOrder total = len(encodedTags) diff --git a/src/dbnode/storage/index/convert/convert_benchmark_test.go b/src/dbnode/storage/index/convert/convert_benchmark_test.go index 2f93341b89..90781ed7ac 100644 --- a/src/dbnode/storage/index/convert/convert_benchmark_test.go +++ b/src/dbnode/storage/index/convert/convert_benchmark_test.go @@ -34,12 +34,12 @@ import ( ) type idWithEncodedTags struct { - id ident.ID + id ident.BytesID encodedTags []byte } type idWithTags struct { - id ident.ID + id ident.BytesID tags ident.Tags } diff --git a/src/dbnode/storage/index/convert/convert_test.go b/src/dbnode/storage/index/convert/convert_test.go index 005d932110..8aa6c727d8 100644 --- a/src/dbnode/storage/index/convert/convert_test.go +++ b/src/dbnode/storage/index/convert/convert_test.go @@ -208,7 +208,7 @@ func TestFromSeriesIDAndEncodedTags(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - seriesID := ident.StringID(tt.id) + seriesID := ident.BytesID(tt.id) d, err := convert.FromSeriesIDAndEncodedTags(seriesID, encodedTags) assert.NoError(t, err) assertContentsMatch(t, seriesID, tags.Values(), d) @@ -257,7 +257,7 @@ func TestFromSeriesIDAndEncodedTagsInvalid(t *testing.T) { encodedTags: []byte{117, 39, 1, 0, 0, 0, 3, 0, 98, 97, 122}, }, } - seriesID := ident.StringID("foo") + seriesID := ident.BytesID("foo") for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {