Skip to content

Commit

Permalink
Merge branch 'master' into linasn/dbnode-metrics
Browse files Browse the repository at this point in the history
* master: (26 commits)
  [dbnode][m3ninx] Rename doc.Document -> doc.Metadata (#3062)
  Don't attempt to lint genny gen'd code (#3063)
  [coordinator] Add ability to turn off rule matching cache (#3059)
  [r2ctl] Bump urijs from 1.19.1 to 1.19.5 in /src/ctl/ui (#3055)
  [aggregator/client] Metric for dropped metrics (#3054)
  [query] Add Graphite support for ** with metric path selectors (#3020)
  [tools] Fix failing linter (#3052)
  [query] implemented Graphite's `powSeries` function (#3038)
  [coordinator] Add Graphite Carbon rewrite cleanup integration test (#3049)
  [coordinator] Add Graphite rewrite cleanup directive for cleansing incoming metrics (#3047)
  [coordinator] Add Graphite Carbon ingest contains rule matcher for fast match (#3046)
  [query] Update Graphite quoted strings to just escape quotes and a flag for previous escape behavior (#3022)
  [query] Graphite match regexp all values for __g0__ tag matcher instead of match field (#3021)
  [coordinator] Add Carbon ingest latency metrics (#3045)
  [aggregator] Fix panic on invalid unaggregated payloads (#3041)
  [coordinator] Ignore rollup rule storage policies when deciding to utilize auto-mapping rules (#3044)
  [dbnode] Support checkpointing in cold flusher and preparing index writer per block (#3040)
  [dbnode] Client borrow connection API (#3019)
  [DOCS] Add back FAQ and troubleshooting sections (#3029)
  [DOCS] SEO and config changes (#3024)
  ...
  • Loading branch information
soundvibe committed Jan 5, 2021
2 parents a548480 + a5e31f3 commit 2e98407
Show file tree
Hide file tree
Showing 230 changed files with 3,909 additions and 1,572 deletions.
5 changes: 3 additions & 2 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ run:
# won't be reported. Default value is empty list, but there is
# no need to include all autogenerated files, we confidently recognize
# autogenerated files. If it's not please let us know.
skip-files: []
skip-files:
- ".*_gen.go$"

# by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules":
# If invoked with -mod=readonly, the go command is disallowed from the implicit
Expand Down Expand Up @@ -113,7 +114,7 @@ linters-settings:
lll:
# max line length, lines longer will be reported. Default is 120.
# '\t' is counted as 1 character by default, and can be changed with the tab-width option
line-length: 100
line-length: 120
# tab width in spaces. Default to 1.
tab-width: 1
unused:
Expand Down
10 changes: 1 addition & 9 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ require (
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d // indirect
github.com/DataDog/datadog-go v3.7.1+incompatible // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/MichaelTJones/pcg v0.0.0-20180122055547-df440c6ed7ed
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
Expand All @@ -23,7 +22,6 @@ require (
github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect
github.com/davecgh/go-spew v1.1.1
github.com/docker/go-connections v0.4.0 // indirect
github.com/fatih/color v1.10.0 // indirect
github.com/fortytw2/leaktest v1.2.1-0.20180901000122-b433bbd6d743
github.com/fossas/fossa-cli v1.0.30
github.com/garethr/kubeval v0.0.0-20180821130434-c44f5193dc94
Expand All @@ -36,6 +34,7 @@ require (
github.com/golang/mock v1.4.4
github.com/golang/protobuf v1.4.2
github.com/golang/snappy v0.0.1
github.com/golangci/golangci-lint v1.33.0
github.com/google/go-cmp v0.5.2
github.com/google/go-jsonnet v0.16.0
github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f // indirect
Expand All @@ -47,7 +46,6 @@ require (
github.com/influxdata/influxdb v1.7.7
github.com/jhump/protoreflect v1.6.1
github.com/json-iterator/go v1.1.9
github.com/kr/text v0.2.0 // indirect
github.com/leanovate/gopter v0.2.8
github.com/lib/pq v1.6.0 // indirect
github.com/lightstep/lightstep-tracer-go v0.18.1
Expand All @@ -66,9 +64,6 @@ require (
github.com/m3dbx/vellum v0.0.0-20201119082309-5b47f7a70f69
github.com/mauricelam/genny v0.0.0-20180903214747-eb2c5232c885
github.com/mjibson/esc v0.1.0
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/onsi/ginkgo v1.14.1 // indirect
github.com/onsi/gomega v1.10.2 // indirect
github.com/opencontainers/image-spec v1.0.1 // indirect
github.com/opencontainers/runc v0.1.1 // indirect
github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9
Expand All @@ -92,7 +87,6 @@ require (
github.com/rveen/ogdl v0.0.0-20200522080342-eeeda1a978e7 // indirect
github.com/sergi/go-diff v1.1.0
github.com/shirou/gopsutil v2.20.5+incompatible // indirect
github.com/sirupsen/logrus v1.7.0 // indirect
github.com/spf13/cast v1.3.1-0.20190531151931-f31dc0aaab5a // indirect
github.com/spf13/cobra v1.1.1
github.com/spf13/jwalterweatherman v1.1.0 // indirect
Expand Down Expand Up @@ -123,7 +117,6 @@ require (
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634
golang.org/x/tools v0.0.0-20201013201025-64a9e34f3752
google.golang.org/grpc v1.29.1
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
gopkg.in/go-ini/ini.v1 v1.57.0 // indirect
gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
gopkg.in/go-playground/validator.v9 v9.7.0
Expand All @@ -133,7 +126,6 @@ require (
gopkg.in/vmihailenco/msgpack.v2 v2.8.3
gopkg.in/yaml.v2 v2.3.0
gotest.tools v2.2.0+incompatible
honnef.co/go/tools v0.0.1-2020.1.6 // indirect
)

// branch 0.9.3-pool-read-binary-3
Expand Down
192 changes: 192 additions & 0 deletions go.sum

Large diffs are not rendered by default.

10 changes: 5 additions & 5 deletions netlify.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,17 @@

[[redirects]]
from = "/docs/how_to/single_node"
to = "docs/quickstart"
to = "/docs/quickstart"

[[redirects]]
from="docs/how_to/kubernetes"
to="/docs/getting_started/kubernetes_cluster"
from="/docs/how_to/kubernetes"
to="/docs/cluster/kubernetes_cluster"

[[redirects]]
from="docs/quickstart/kubernetes"
from="/docs/quickstart/kubernetes"
to="/docs/cluster/kubernetes_cluster"

# TODO: Fix this with new content type
[[redirects]]
from = "/talks"
to = "docs/overview/media"
to = "/docs/overview/media"
9 changes: 9 additions & 0 deletions scripts/docker-integration-tests/carbon/expected/dbaz.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
[
{
"id": "d.bar.baz",
"text": "baz",
"leaf": 1,
"expandable": 0,
"allowChildren": 0
}
]
9 changes: 9 additions & 0 deletions scripts/docker-integration-tests/carbon/expected/ebaz.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
[
{
"id": "e.bar.baz",
"text": "baz",
"leaf": 1,
"expandable": 0,
"allowChildren": 0
}
]
9 changes: 9 additions & 0 deletions scripts/docker-integration-tests/carbon/expected/fbaz.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
[
{
"id": "f.bar.baz",
"text": "baz",
"leaf": 1,
"expandable": 0,
"allowChildren": 0
}
]
9 changes: 9 additions & 0 deletions scripts/docker-integration-tests/carbon/expected/gbaz.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
[
{
"id": "g.bar.baz",
"text": "baz",
"leaf": 1,
"expandable": 0,
"allowChildren": 0
}
]
9 changes: 9 additions & 0 deletions scripts/docker-integration-tests/carbon/expected/hbarbaz.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
[
{
"id": "h.bar_baz",
"text": "bar_baz",
"leaf": 1,
"expandable": 0,
"allowChildren": 0
}
]
9 changes: 9 additions & 0 deletions scripts/docker-integration-tests/carbon/expected/ibarbaz.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
[
{
"id": "i.bar_baz",
"text": "bar_baz",
"leaf": 1,
"expandable": 0,
"allowChildren": 0
}
]
2 changes: 2 additions & 0 deletions scripts/docker-integration-tests/carbon/m3coordinator.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ clusters:
carbon:
ingester:
listenAddress: "0.0.0.0:7204"
rewrite:
cleanup: true
rules:
- pattern: .*min.aggregate.*
aggregation:
Expand Down
65 changes: 50 additions & 15 deletions scripts/docker-integration-tests/carbon/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ t=$(date +%s)
echo "foo.min.aggregate.baz 41 $t" | nc 0.0.0.0 7204
echo "foo.min.aggregate.baz 42 $t" | nc 0.0.0.0 7204
echo "Attempting to read min aggregated carbon metric"
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff read_carbon foo.min.aggregate.baz 41
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.min.aggregate.baz' 41"

echo "Writing out a carbon metric that should not be aggregated"
t=$(date +%s)
Expand All @@ -64,41 +64,76 @@ t=$(date +%s)
echo "foo.min.already-aggregated.baz 42 $t" | nc 0.0.0.0 7204
echo "foo.min.already-aggregated.baz 43 $t" | nc 0.0.0.0 7204
echo "Attempting to read unaggregated carbon metric"
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff read_carbon foo.min.already-aggregated.baz 43
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.min.already-aggregated.baz' 43"

echo "Writing out a carbon metric that should should use the default mean aggregation"
t=$(date +%s)
# Mean of 10 and 20 is 15. Same comment as the min aggregation test above.
echo "foo.min.catch-all.baz 10 $t" | nc 0.0.0.0 7204
echo "foo.min.catch-all.baz 20 $t" | nc 0.0.0.0 7204
echo "Attempting to read mean aggregated carbon metric"
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff read_carbon foo.min.catch-all.baz 15
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.min.catch-all.baz' 15"

# Test writing and reading IDs with colons in them.
t=$(date +%s)
echo "foo.bar:baz.qux 42 $t" | nc 0.0.0.0 7204
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff read_carbon 'foo.bar:*.*' 42
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.bar:*.*' 42"

# Test writing and reading IDs with a single element.
t=$(date +%s)
echo "quail 42 $t" | nc 0.0.0.0 7204
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff read_carbon 'quail' 42
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'quail' 42"

# Test using "**" in queries
t=$(date +%s)
echo "qux.pos1-a.pos2-0 1 $t" | nc 0.0.0.0 7204
echo "qux.pos1-a.pos2-1 1 $t" | nc 0.0.0.0 7204
echo "qux.pos1-b.pos2-0 1 $t" | nc 0.0.0.0 7204
echo "qux.pos1-b.pos2-1 1 $t" | nc 0.0.0.0 7204
echo "qux.pos1-c.pos2-0 1 $t" | nc 0.0.0.0 7204
echo "qux.pos1-c.pos2-1 1 $t" | nc 0.0.0.0 7204
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(qux**)' 6"
ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(qux.pos1-a**)' 2"
ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(**pos1-a**)' 2"
ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(**pos2-1**)' 3"
ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(**pos2-1)' 3"

t=$(date +%s)

# Test basic cases
echo "a 0 $t" | nc 0.0.0.0 7204
echo "a.bar 0 $t" | nc 0.0.0.0 7204
echo "a.biz 0 $t" | nc 0.0.0.0 7204
echo "a.biz.cake 0 $t" | nc 0.0.0.0 7204
echo "a.bar.caw.daz 0 $t" | nc 0.0.0.0 7204
echo "a.bag 0 $t" | nc 0.0.0.0 7204
echo "c:bar.c:baz 0 $t" | nc 0.0.0.0 7204
ATTEMPTS=10 TIMEOUT=1 retry_with_backoff find_carbon a* a.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon a.b* a.b.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon a.ba[rg] a.ba.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon a.b*.c* a.b.c.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon a.b*.caw.* a.b.c.d.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon x none.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon a.d none.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon *.*.*.*.* none.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon c:* cbar.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon c:bar.* cbaz.json

# Test rewrite multiple dots
echo "d..bar.baz 0 $t" | nc 0.0.0.0 7204
echo "e.bar...baz 0 $t" | nc 0.0.0.0 7204

# Test rewrite leading or trailing dots
echo "..f.bar.baz 0 $t" | nc 0.0.0.0 7204
echo "g.bar.baz.. 0 $t" | nc 0.0.0.0 7204

# Test rewrite bad chars
echo "h.bar@@baz 0 $t" | nc 0.0.0.0 7204
echo "i.bar!!baz 0 $t" | nc 0.0.0.0 7204

ATTEMPTS=10 TIMEOUT=1 retry_with_backoff "find_carbon 'a*' a.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.b*' ab.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.ba[rg]' aba.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.b*.c*' abc.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.b*.caw.*' abcd.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'x' none.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.d' none.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon '*.*.*.*.*' none.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'c:*' cbar.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'c:bar.*' cbaz.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'd.bar.*' dbaz.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'e.bar.*' ebaz.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'f.bar.*' fbaz.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'g.bar.*' gbaz.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'h.bar*' hbarbaz.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'i.bar*' ibarbaz.json"
2 changes: 1 addition & 1 deletion scripts/docker-integration-tests/coordinator_noop/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ if ! curl -vvvsSf localhost:7201/api/v1/services/m3coordinator/placement; then
exit 1
fi

QUERY_EXP='{"error":"operation not valid for noop client"}'
QUERY_EXP='{"status":"error","error":"operation not valid for noop client"}'
RES=$(curl "localhost:7201/m3query/api/v1/query_range?start=$(date '+%s')&end=$(date '+%s')&step=10&query=foo")
if [[ "$RES" != "$QUERY_EXP" ]]; then
echo "Expected resp '$QUERY_EXP', GOT '$RES'"
Expand Down
12 changes: 6 additions & 6 deletions site/config/_default/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
target = "archetypes"

ignoreFiles = [ "\\.ttf$", "\\.woff$", "\\.woff2$", "\\.eot$" ]
enableRobotsTXT = true
metaDataFormat = "yaml"
# theme = "docs-theme"
# baseURL = "/"
Expand All @@ -44,8 +43,8 @@ defaultContentLanguageInSubdir = true

[languages]
[languages.en]
title = "Kubernetes"
description = "Production-Grade Container Orchestration"
title = "M3 Documentation"
description = "M3 is a Prometheus compatible, easy to adopt metrics engine that provides visibility for some of the world’s largest brands."
languageName ="English"
# Weight used for sorting.
weight = 1
Expand Down Expand Up @@ -110,9 +109,10 @@ offlineSearch = false
# Prefix URL to edit current page. Will display an "Edit this page" button on top right hand corner of every page.
# Useful to give opportunity to people to create merge request for your doc.
# See the config.toml file from this documentation site to have an example.
# TODO: pattern to branch?
# TODO: bring back
# editURL = "https://github.com/m3db/m3/tree/master/site/content/"

helpBlock = true
editURL = "https://github.com/m3db/m3/tree/master/site/content/"

# Author of the site, will be used in meta information
author = "The M3 team and community"
# Description of the site, will be used in meta information
Expand Down
9 changes: 3 additions & 6 deletions site/config/production/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ ignoreFiles = [ "\\.ttf$", "\\.woff$", "\\.woff2$", "\\.eot$" ]
languageCode = "en-US"
defaultContentLanguage = "en"
# staticDir = ["static"]
enableRobotsTXT = true

metaDataFormat = "yaml"
defaultContentLanguageInSubdir= true
Expand Down Expand Up @@ -101,11 +102,7 @@ offlineSearch = false
# Prefix URL to edit current page. Will display an "Edit this page" button on top right hand corner of every page.
# Useful to give opportunity to people to create merge request for your doc.
# See the config.toml file from this documentation site to have an example.
# TODO: pattern to branch?
# TODO: bring back
# editURL = "https://github.com/m3db/m3/tree/master/site/content/"
# Shows a checkmark for visited pages on the menu
showVisitedLinks = false

# Disable search function. It will hide search bar
disableSearch = false
# Javascript and CSS cache are automatically busted when new version of site is generated.
Expand All @@ -124,7 +121,7 @@ offlineSearch = false
# Order sections in menu by "weight" or "title". Default to "weight"
ordersectionsby = "weight"
# Change default color scheme with a variant one. Can be "red", "blue", "green".
themeVariant = "blue"
# themeVariant = "blue"
twitter = "m3db_io"
disableHomeIcon = true

Expand Down
3 changes: 1 addition & 2 deletions site/content/cluster/binaries_cluster.md
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
---
linktitle: "Binaries"
title: Creating an M3 Cluster with Binaries
weight: 2
---

# Creating an M3 Cluster with Binaries

This guide shows you the steps involved in creating an M3 cluster using M3 binaries, typically you would automate this with infrastructure as code tools such as Terraform or [Kubernetes](/docs/operator).

{{% notice note %}}
Expand Down
3 changes: 1 addition & 2 deletions site/content/cluster/kubernetes_cluster.md
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
---
linktitle: "Kubernetes"
title: Creating an M3 Cluster with Kubernetes
weight: 1
---

# Creating an M3 Cluster with Kubernetes

This guide shows you how to create an M3 cluster of 3 nodes, designed to run locally on the same machine. It is designed to show you how M3 and Kubernetes can work together, but not as a production example.

{{% notice note %}}
Expand Down
Loading

0 comments on commit 2e98407

Please sign in to comment.