Change coordinator to query and add m3query service (#817)

.codecov.yml

@@ -6,8 +6,8 @@ coverage:
   status:
     project:
       default: on
-      coordinator:
-        flags: coordinator
+      query:
+        flags: query
       dbnode:
         flags: dbnode
       m3ninx:
@@ -18,10 +18,10 @@ coverage:
       default: off
 
   flags:
-    coordinator:
+    query:
       paths:
-        - src/coordinator/
-        - src/cmd/services/m3coordinator/
+        - src/query/
+        - src/cmd/services/m3query/
     dbnode:
       paths:
         - src/dbnode/

Makefile

@@ -41,11 +41,12 @@ SERVICES :=     \
 	m3coordinator \
 	m3nsch_server \
 	m3nsch_client \
+	m3query       \
 
 SUBDIRS :=    \
 	cmd         \
 	dbnode      \
-	coordinator \
+	query       \
 	m3nsch      \
 	m3ninx      \
 

docker/m3coordinator/Dockerfile

@@ -21,7 +21,7 @@ LABEL maintainer="The M3DB Authors <[email protected]>"
 EXPOSE 7201/tcp 7203/tcp
 
 COPY --from=builder /go/src/github.com/m3db/m3db/bin/m3coordinator /bin/
-COPY --from=builder /go/src/github.com/m3db/m3db/src/coordinator/config/m3coordinator-local-etcd.yml /etc/m3coordinator/m3coordinator.yml
+COPY --from=builder /go/src/github.com/m3db/m3db/src/query/config/m3coordinator-local-etcd.yml /etc/m3coordinator/m3coordinator.yml
 
 ENTRYPOINT [ "/bin/m3coordinator" ]
 CMD [ "-f", "/etc/m3coordinator/m3coordinator.yml" ]

docker/m3query/Dockerfile

@@ -0,0 +1,27 @@
+# stage 1: build
+FROM golang:1.10-alpine AS builder
+LABEL maintainer="The M3DB Authors <[email protected]>"
+
+# Install Glide
+RUN apk add --update glide git make bash
+
+# Add source code
+RUN mkdir -p /go/src/github.com/m3db/m3db
+ADD . /go/src/github.com/m3db/m3db
+
+# Build m3coordinator binary
+RUN cd /go/src/github.com/m3db/m3db/ && \
+    git submodule update --init      && \
+    make m3query-linux-amd64
+
+# stage 2: lightweight "release"
+FROM alpine:latest
+LABEL maintainer="The M3DB Authors <[email protected]>"
+
+EXPOSE 7201/tcp 7203/tcp
+
+COPY --from=builder /go/src/github.com/m3db/m3db/bin/m3query /bin/
+COPY --from=builder /go/src/github.com/m3db/m3db/src/query/config/m3query-local-etcd.yml /etc/m3query/m3query.yml
+
+ENTRYPOINT [ "/bin/m3query" ]
+CMD [ "-f", "/etc/m3query/m3query.yml" ]

scripts/integration-tests/docker-integration-test.sh

@@ -2,15 +2,17 @@
 
 set -xe
 
-echo "Build docker image" 
+echo "Build docker images"
 
 docker build -t "m3dbnode:$(git rev-parse HEAD)" -f ./docker/m3dbnode/Dockerfile .
+docker build -t "m3coordinator:$(git rev-parse HEAD)" -f ./docker/m3coordinator/Dockerfile .
+docker build -t "m3query:$(git rev-parse HEAD)" -f ./docker/m3query/Dockerfile .
 
-echo "Run docker container" 
+echo "Run m3dbnode docker container"
 
 docker run --name "m3dbnode-version-$(git rev-parse HEAD)" -d --rm -p 9000:9000 -p 9001:9001 -p 9002:9002 -p 9003:9003 -p 9004:9004 -p 7201:7201 "m3dbnode:$(git rev-parse HEAD)"
 
-echo "Sleeping for a bit to ensure db"
+echo "Sleeping for a bit to ensure db up"
 
 sleep 10 # TODO Replace sleeps with logic to determine when to proceed
 
@@ -40,13 +42,13 @@ curl -vvvsSf -X POST localhost:7201/api/v1/namespace -d '{
   }
 }'
 
-echo "Sleep while namespace is init'd" 
+echo "Sleep while namespace is init'd"
 
 sleep 10 # TODO Replace sleeps with logic to determine when to proceed
 
 [ "$(curl -sSf localhost:7201/api/v1/namespace | jq .registry.namespaces.default.indexOptions.enabled)" == true ]
 
-echo "Initialization placement" 
+echo "Initialization placement"
 
 curl -vvvsSf -X POST localhost:7201/api/v1/placement/init -d '{
     "num_shards": 64,
@@ -66,11 +68,11 @@ curl -vvvsSf -X POST localhost:7201/api/v1/placement/init -d '{
 
 [ "$(curl -sSf localhost:7201/api/v1/placement | jq .placement.instances.m3db_local.id)" == '"m3db_local"' ]
 
-echo "Wait for placement to fully initialize" 
+echo "Wait for placement to fully initialize"
 
 sleep 60 # TODO Replace sleeps with logic to determine when to proceed
 
-echo "Write data" 
+echo "Write data"
 
 curl -vvvsSf -X POST localhost:9003/writetagged -d '{
   "namespace": "default",
@@ -103,24 +105,24 @@ queryResult=$(curl -sSf -X POST localhost:9003/query -d '{
   },
   "rangeStart": 0,
   "rangeEnd":'"$(date +"%s")"'
-}' | jq '.results | length') 
+}' | jq '.results | length')
 
-if [ "$queryResult" -lt 1 ]; then 
-  echo "Result not found" 
+if [ "$queryResult" -lt 1 ]; then
+  echo "Result not found"
   exit 1
-else 
+else
   echo "Result found"
 fi
 
-echo "Deleting placement" 
+echo "Deleting placement"
 
 curl -vvvsSf -X DELETE localhost:7201/api/v1/placement
 
 echo "Deleting namespace"
 
 curl -vvvsSf -X DELETE localhost:7201/api/v1/namespace/default
 
-echo "Stop docker container" 
+echo "Stop docker container"
 
 docker stop "m3dbnode-version-$(git rev-parse HEAD)"
 

scripts/process-cover.sh

@@ -8,11 +8,11 @@ fi
 COVERFILE=$1
 SUBMIT_COVER="$(dirname $0)/../.ci/codecov.sh"
 
-TARGETS=("dbnode" "coordinator" "m3ninx")
+TARGETS=("dbnode" "query" "m3ninx")
 target_patterns() {
     case $1 in
         'dbnode') echo "^mode|github.com/m3db/m3db/src/dbnode|github.com/m3db/m3db/src/cmd/services/m3dbnode";;
-        'coordinator') echo "^mode|github.com/m3db/m3db/src/coordinator|github.com/m3db/m3db/src/cmd/services/m3coordinator";;
+        'query') echo "^mode|github.com/m3db/m3db/src/query|github.com/m3db/m3db/src/cmd/services/m3query";;
         'm3ninx') echo "^mode|github.com/m3db/m3db/src/m3ninx";;
         *)   echo "unknown key: $1"; exit 1;;
     esac

scripts/site-build.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-ASSET_DIR=${1:-src/coordinator/generated/assets/openapi}
+ASSET_DIR=${1:-src/query/generated/assets/openapi}
 
 # Copy over OpenAPI doc.
 mkdir -p m3metrics.io/openapi

src/cmd/services/m3coordinator/main/main.go

@@ -25,7 +25,7 @@ import (
 	_ "net/http/pprof" // pprof: for debug listen server if configured
 	"os"
 
-	"github.com/m3db/m3db/src/coordinator/services/m3coordinator/server"
+	"github.com/m3db/m3db/src/query/server"
 )
 
 var (

src/cmd/services/m3dbnode/config/config.go

@@ -29,7 +29,7 @@ import (
 	"strings"
 	"time"
 
-	coordinatorcfg "github.com/m3db/m3db/src/cmd/services/m3coordinator/config"
+	coordinatorcfg "github.com/m3db/m3db/src/cmd/services/m3query/config"
 	"github.com/m3db/m3db/src/dbnode/client"
 	"github.com/m3db/m3db/src/dbnode/environment"
 	"github.com/m3db/m3x/config/hostid"

src/cmd/services/m3dbnode/main/main.go

@@ -28,9 +28,9 @@ import (
 
 	clusterclient "github.com/m3db/m3cluster/client"
 	"github.com/m3db/m3db/src/cmd/services/m3dbnode/config"
-	dbserver "github.com/m3db/m3db/src/cmd/services/m3dbnode/server"
-	coordinatorserver "github.com/m3db/m3db/src/coordinator/services/m3coordinator/server"
 	"github.com/m3db/m3db/src/dbnode/client"
+	dbserver "github.com/m3db/m3db/src/dbnode/server"
+	coordinatorserver "github.com/m3db/m3db/src/query/server"
 	xconfig "github.com/m3db/m3x/config"
 )
 

src/cmd/services/m3dbnode/main/main_index_test.go

@@ -34,9 +34,9 @@ import (
 	"github.com/m3db/m3cluster/placement"
 	"github.com/m3db/m3cluster/services"
 	"github.com/m3db/m3db/src/cmd/services/m3dbnode/config"
-	"github.com/m3db/m3db/src/cmd/services/m3dbnode/server"
 	"github.com/m3db/m3db/src/dbnode/client"
 	"github.com/m3db/m3db/src/dbnode/kvconfig"
+	"github.com/m3db/m3db/src/dbnode/server"
 	"github.com/m3db/m3db/src/dbnode/storage/index"
 	m3ninxidx "github.com/m3db/m3db/src/m3ninx/idx"
 	xconfig "github.com/m3db/m3x/config"

src/cmd/services/m3dbnode/main/main_test.go

@@ -35,9 +35,9 @@ import (
 	"github.com/m3db/m3cluster/services"
 	"github.com/m3db/m3cluster/shard"
 	"github.com/m3db/m3db/src/cmd/services/m3dbnode/config"
-	"github.com/m3db/m3db/src/cmd/services/m3dbnode/server"
 	"github.com/m3db/m3db/src/dbnode/client"
 	"github.com/m3db/m3db/src/dbnode/kvconfig"
+	"github.com/m3db/m3db/src/dbnode/server"
 	xconfig "github.com/m3db/m3x/config"
 	"github.com/m3db/m3x/ident"
 	"github.com/m3db/m3x/instrument"

src/cmd/services/m3coordinator/config/config.go → src/cmd/services/m3query/config/config.go

@@ -24,11 +24,11 @@ import (
 	"time"
 
 	etcdclient "github.com/m3db/m3cluster/client/etcd"
-	"github.com/m3db/m3db/src/coordinator/storage/local"
+	"github.com/m3db/m3db/src/query/storage/local"
 	"github.com/m3db/m3x/instrument"
 )
 
-// Configuration is the configuration for the coordinator.
+// Configuration is the configuration for the query service.
 type Configuration struct {
 	// Metrics configuration.
 	Metrics instrument.MetricsConfiguration `yaml:"metrics"`

src/cmd/services/m3query/main/main.go

@@ -0,0 +1,46 @@
+// Copyright (c) 2018 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package main
+
+import (
+	"flag"
+	_ "net/http/pprof" // pprof: for debug listen server if configured
+	"os"
+
+	"github.com/m3db/m3db/src/query/server"
+)
+
+var (
+	configFile = flag.String("f", "", "configuration file")
+)
+
+func main() {
+	flag.Parse()
+
+	if len(*configFile) == 0 {
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	server.Run(server.RunOptions{
+		ConfigFile: *configFile,
+	})
+}

src/coordinator/README.md → src/query/README.md

@@ -50,13 +50,13 @@ Run m3coordinator binary:
 
 Run Prometheus Docker image:
 
-    $ docker run -p 9090:9090 -v $GOPATH/src/github.com/m3db/m3db/src/coordinator/docker/prometheus-mac.yml:/etc/prometheus/prometheus.yml quay.io/prometheus/prometheus
+    $ docker run -p 9090:9090 -v $GOPATH/src/github.com/m3db/m3db/src/query/docker/prometheus-mac.yml:/etc/prometheus/prometheus.yml quay.io/prometheus/prometheus
 
 ### Running on GCP
 
 Setup GCP for [single m3db node](https://github.com/m3db/m3db/pull/452/files?short_path=20bfc3f#diff-20bfc3ff6a860483887b93bf9cf0d135)
 
-> For a multi-node cluster, [see here](https://github.com/m3db/m3db/src/coordinator/tree/master/benchmark)
+> For a multi-node cluster, [see here](https://github.com/m3db/m3db/src/query/tree/master/benchmark)
 
 Setup GCP for m3coordinator:
 
@@ -80,7 +80,7 @@ Setup and run Prometheus:
                 - url: http://10.142.0.8:7201/api/v1/prom/remote/write
             ```
     3. Run Prometheus
-        $ sudo docker run -p 9090:9090 -v $GOPATH/src/github.com/m3db/m3db/src/coordinator/docker/prometheus.yml:/etc/prometheus/prometheus.yml quay.io/prometheus/prometheus
+        $ sudo docker run -p 9090:9090 -v $GOPATH/src/github.com/m3db/m3db/src/query/docker/prometheus.yml:/etc/prometheus/prometheus.yml quay.io/prometheus/prometheus
 
-[doc-img]: https://godoc.org/github.com/m3db/m3db/src/coordinator?status.svg
-[doc]: https://godoc.org/github.com/m3db/m3db/src/coordinator
+[doc-img]: https://godoc.org/github.com/m3db/m3db/src/query?status.svg
+[doc]: https://godoc.org/github.com/m3db/m3db/src/query

src/coordinator/api/v1/handler/common.go → src/query/api/v1/handler/common.go

@@ -29,7 +29,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/m3db/m3db/src/coordinator/util/logging"
+	"github.com/m3db/m3db/src/query/util/logging"
 
 	"github.com/golang/protobuf/jsonpb"
 	"github.com/golang/protobuf/proto"

src/coordinator/api/v1/handler/database/common.go → src/query/api/v1/handler/database/common.go

@@ -22,9 +22,9 @@ package database
 
 import (
 	clusterclient "github.com/m3db/m3cluster/client"
-	"github.com/m3db/m3db/src/cmd/services/m3coordinator/config"
 	dbconfig "github.com/m3db/m3db/src/cmd/services/m3dbnode/config"
-	"github.com/m3db/m3db/src/coordinator/util/logging"
+	"github.com/m3db/m3db/src/cmd/services/m3query/config"
+	"github.com/m3db/m3db/src/query/util/logging"
 
 	"github.com/gorilla/mux"
 )

src/coordinator/api/v1/handler/database/create.go → src/query/api/v1/handler/database/create.go

@@ -32,15 +32,15 @@ import (
 
 	clusterclient "github.com/m3db/m3cluster/client"
 	"github.com/m3db/m3cluster/generated/proto/placementpb"
-	"github.com/m3db/m3db/src/cmd/services/m3coordinator/config"
 	dbconfig "github.com/m3db/m3db/src/cmd/services/m3dbnode/config"
-	"github.com/m3db/m3db/src/coordinator/api/v1/handler"
-	"github.com/m3db/m3db/src/coordinator/api/v1/handler/namespace"
-	"github.com/m3db/m3db/src/coordinator/api/v1/handler/placement"
-	"github.com/m3db/m3db/src/coordinator/generated/proto/admin"
-	"github.com/m3db/m3db/src/coordinator/util"
-	"github.com/m3db/m3db/src/coordinator/util/logging"
+	"github.com/m3db/m3db/src/cmd/services/m3query/config"
 	dbnamespace "github.com/m3db/m3db/src/dbnode/storage/namespace"
+	"github.com/m3db/m3db/src/query/api/v1/handler"
+	"github.com/m3db/m3db/src/query/api/v1/handler/namespace"
+	"github.com/m3db/m3db/src/query/api/v1/handler/placement"
+	"github.com/m3db/m3db/src/query/generated/proto/admin"
+	"github.com/m3db/m3db/src/query/util"
+	"github.com/m3db/m3db/src/query/util/logging"
 
 	"github.com/golang/protobuf/jsonpb"
 	"go.uber.org/zap"

src/coordinator/api/v1/handler/database/create_test.go → src/query/api/v1/handler/database/create_test.go

@@ -36,11 +36,11 @@ import (
 	"github.com/m3db/m3cluster/kv"
 	"github.com/m3db/m3cluster/placement"
 	"github.com/m3db/m3cluster/services"
-	"github.com/m3db/m3db/src/cmd/services/m3coordinator/config"
 	dbconfig "github.com/m3db/m3db/src/cmd/services/m3dbnode/config"
-	"github.com/m3db/m3db/src/coordinator/api/v1/handler/namespace"
-	"github.com/m3db/m3db/src/coordinator/util/logging"
+	"github.com/m3db/m3db/src/cmd/services/m3query/config"
 	xtest "github.com/m3db/m3db/src/dbnode/x/test"
+	"github.com/m3db/m3db/src/query/api/v1/handler/namespace"
+	"github.com/m3db/m3db/src/query/util/logging"
 
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"

src/coordinator/api/v1/handler/namespace/add.go → src/query/api/v1/handler/namespace/add.go

@@ -26,11 +26,11 @@ import (
 	"net/http"
 
 	clusterclient "github.com/m3db/m3cluster/client"
-	"github.com/m3db/m3db/src/coordinator/api/v1/handler"
-	"github.com/m3db/m3db/src/coordinator/generated/proto/admin"
-	"github.com/m3db/m3db/src/coordinator/util/logging"
 	nsproto "github.com/m3db/m3db/src/dbnode/generated/proto/namespace"
 	"github.com/m3db/m3db/src/dbnode/storage/namespace"
+	"github.com/m3db/m3db/src/query/api/v1/handler"
+	"github.com/m3db/m3db/src/query/generated/proto/admin"
+	"github.com/m3db/m3db/src/query/util/logging"
 
 	"github.com/gogo/protobuf/jsonpb"
 	"go.uber.org/zap"