From dd96f668d545d979bc0a6380280df0092c53e476 Mon Sep 17 00:00:00 2001 From: chris Date: Fri, 11 Sep 2020 09:32:19 -0400 Subject: [PATCH] [WIP] v1.0.0 --- .drone.yml | 4 +- .goreleaser.yml | 8 +- Makefile | 59 +- README.md | 169 +- {deploy => build}/criticalstack.repo | 0 {deploy => build}/e2d.service | 3 +- cmd/e2d/app/certs/certs.go | 20 + cmd/e2d/app/certs/generate/generate.go | 73 + cmd/e2d/app/certs/init/init.go | 34 + cmd/e2d/app/completion.go | 30 - cmd/e2d/app/e2d.go | 61 + cmd/e2d/app/pki.go | 173 -- cmd/e2d/app/root.go | 26 - cmd/e2d/app/run.go | 253 --- cmd/e2d/app/run/run.go | 78 + cmd/e2d/app/{ => version}/version.go | 19 +- cmd/e2d/main.go | 4 +- e2e/context.go | 113 ++ e2e/manager_test.go | 353 ++++ e2e/node.go | 169 ++ e2e/testcluster.go | 223 +++ e2e/utils.go | 21 + go.mod | 16 +- go.sum | 164 ++ hack/boilerplate.go.txt | 15 + hack/tools/go.mod | 2 +- hack/tools/tools.go | 2 + hack/tools/update-codegen.sh | 23 + {pkg => internal}/buildinfo/version.go | 0 {pkg => internal}/provider/aws/client.go | 3 +- {pkg => internal}/provider/aws/config.go | 3 +- .../provider/digitalocean/client.go | 7 +- pkg/client/config.go | 38 +- pkg/config/v1alpha1/defaults.go | 62 + pkg/config/v1alpha1/doc.go | 5 + pkg/config/v1alpha1/register.go | 38 + pkg/config/v1alpha1/types.go | 289 ++++ pkg/config/v1alpha1/zz_generated.deepcopy.go | 145 ++ pkg/config/v1alpha1/zz_generated.default.go | 37 + pkg/discovery/discovery_aws.go | 3 +- pkg/discovery/discovery_do.go | 12 +- pkg/e2db/config.go | 2 +- pkg/e2db/db_test.go | 20 +- pkg/etcdserver/members.go | 47 + pkg/etcdserver/metadata.go | 107 ++ pkg/etcdserver/peers.go | 57 + pkg/etcdserver/server.go | 242 +++ pkg/etcdserver/snapshot.go | 68 + pkg/gossip/gossip.go | 187 +++ pkg/{manager => gossip}/gossip_test.go | 8 +- pkg/gossip/logger.go | 31 + pkg/gossip/member.go | 82 + pkg/gossip/messages.go | 101 ++ pkg/log/log.go | 22 +- pkg/manager/certs.go | 168 ++ pkg/manager/client.go | 31 +- pkg/manager/config.go | 284 ---- pkg/manager/config_test.go | 36 - pkg/manager/gossip.go | 349 ---- pkg/manager/manager.go | 701 ++++---- pkg/manager/manager_test.go | 1482 ----------------- pkg/manager/membership.go | 161 +- pkg/manager/server.go | 473 ------ pkg/manager/service.go | 21 +- pkg/manager/snapshotter.go | 98 ++ pkg/manager/utils.go | 64 + pkg/pki/pki.go | 211 --- pkg/pki/pki_test.go | 38 - pkg/snapshot/snapshot_aws.go | 3 +- pkg/{cmdutil => util/env}/env.go | 48 +- pkg/{cmdutil => util/env}/env_test.go | 2 +- pkg/{netutil/netutil.go => util/net/net.go} | 5 +- .../netutil_test.go => util/net/net_test.go} | 2 +- scripts/install.sh | 53 + 74 files changed, 4019 insertions(+), 3942 deletions(-) rename {deploy => build}/criticalstack.repo (100%) rename {deploy => build}/e2d.service (56%) create mode 100644 cmd/e2d/app/certs/certs.go create mode 100644 cmd/e2d/app/certs/generate/generate.go create mode 100644 cmd/e2d/app/certs/init/init.go delete mode 100644 cmd/e2d/app/completion.go create mode 100644 cmd/e2d/app/e2d.go delete mode 100644 cmd/e2d/app/pki.go delete mode 100644 cmd/e2d/app/root.go delete mode 100644 cmd/e2d/app/run.go create mode 100644 cmd/e2d/app/run/run.go rename cmd/e2d/app/{ => version}/version.go (62%) create mode 100644 e2e/context.go create mode 100644 e2e/manager_test.go create mode 100644 e2e/node.go create mode 100644 e2e/testcluster.go create mode 100644 e2e/utils.go create mode 100644 hack/boilerplate.go.txt create mode 100755 hack/tools/update-codegen.sh rename {pkg => internal}/buildinfo/version.go (100%) rename {pkg => internal}/provider/aws/client.go (98%) rename {pkg => internal}/provider/aws/config.go (99%) rename {pkg => internal}/provider/digitalocean/client.go (90%) create mode 100644 pkg/config/v1alpha1/defaults.go create mode 100644 pkg/config/v1alpha1/doc.go create mode 100644 pkg/config/v1alpha1/register.go create mode 100644 pkg/config/v1alpha1/types.go create mode 100644 pkg/config/v1alpha1/zz_generated.deepcopy.go create mode 100644 pkg/config/v1alpha1/zz_generated.default.go create mode 100644 pkg/etcdserver/members.go create mode 100644 pkg/etcdserver/metadata.go create mode 100644 pkg/etcdserver/peers.go create mode 100644 pkg/etcdserver/server.go create mode 100644 pkg/etcdserver/snapshot.go create mode 100644 pkg/gossip/gossip.go rename pkg/{manager => gossip}/gossip_test.go (93%) create mode 100644 pkg/gossip/logger.go create mode 100644 pkg/gossip/member.go create mode 100644 pkg/gossip/messages.go create mode 100644 pkg/manager/certs.go delete mode 100644 pkg/manager/config.go delete mode 100644 pkg/manager/config_test.go delete mode 100644 pkg/manager/gossip.go delete mode 100644 pkg/manager/manager_test.go delete mode 100644 pkg/manager/server.go create mode 100644 pkg/manager/snapshotter.go create mode 100644 pkg/manager/utils.go delete mode 100644 pkg/pki/pki.go delete mode 100644 pkg/pki/pki_test.go rename pkg/{cmdutil => util/env}/env.go (99%) rename pkg/{cmdutil => util/env}/env_test.go (98%) rename pkg/{netutil/netutil.go => util/net/net.go} (97%) rename pkg/{netutil/netutil_test.go => util/net/net_test.go} (96%) create mode 100755 scripts/install.sh diff --git a/.drone.yml b/.drone.yml index cc0a191..34982f4 100644 --- a/.drone.yml +++ b/.drone.yml @@ -22,13 +22,13 @@ steps: event: - push -- name: manager-testing +- name: e2e image: golang:1.14 volumes: - name: gocache path: /go commands: - - go test -v ./pkg/manager -test.long + - make test-e2e when: branch: - master diff --git a/.goreleaser.yml b/.goreleaser.yml index ad1f3ee..ddc0679 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -26,9 +26,9 @@ builds: - all=-trimpath={{.Env.GOPATH}} ldflags: - -s -w - - -X "github.com/criticalstack/e2d/pkg/buildinfo.Date={{.Date}}" - - -X "github.com/criticalstack/e2d/pkg/buildinfo.GitSHA={{.ShortCommit}}" - - -X "github.com/criticalstack/e2d/pkg/buildinfo.Version={{.Tag}}" + - -X "github.com/criticalstack/e2d/internal/buildinfo.Date={{.Date}}" + - -X "github.com/criticalstack/e2d/internal/buildinfo.GitSHA={{.ShortCommit}}" + - -X "github.com/criticalstack/e2d/internal/buildinfo.Version={{.Tag}}" archives: - replacements: darwin: Darwin @@ -55,7 +55,7 @@ nfpms: - /etc/systemd/system/e2d.service.d - /var/lib/etcd files: - deploy/e2d.service: /etc/systemd/system/e2d.service + build/e2d.service: /etc/systemd/system/e2d.service checksum: name_template: 'checksums.txt' snapshot: diff --git a/Makefile b/Makefile index 54d55c7..b8cd9a8 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,10 @@ -# If you update this file, please follow -# https://suva.sh/posts/well-documented-makefiles - .DEFAULT_GOAL:=help -ifeq ($(GOPROXY),) -export GOPROXY = direct -endif - -# Directories. -TOOLS_DIR := hack/tools +BIN_DIR ?= bin +TOOLS_DIR := hack/tools TOOLS_BIN_DIR := $(TOOLS_DIR)/bin -BIN_DIR := bin - -# Binaries. GOLANGCI_LINT := $(TOOLS_BIN_DIR)/golangci-lint -# Golang build env -LDFLAGS := -s -w - GIT_BRANCH = $(shell git rev-parse --abbrev-ref HEAD | sed 's/\///g') GIT_COMMIT = $(shell git rev-parse HEAD) GIT_SHA = $(shell git rev-parse --short HEAD) @@ -28,29 +15,36 @@ ifneq ($(GIT_TAG),) VERSION = $(GIT_TAG) endif -LDFLAGS += -X "github.com/criticalstack/e2d/pkg/buildinfo.Date=$(shell date -u +'%Y-%m-%dT%TZ')" -LDFLAGS += -X "github.com/criticalstack/e2d/pkg/buildinfo.GitSHA=$(GIT_SHA)" -LDFLAGS += -X "github.com/criticalstack/e2d/pkg/buildinfo.Version=$(VERSION)" +LDFLAGS := -s -w +LDFLAGS += -X "github.com/criticalstack/e2d/internal/buildinfo.Date=$(shell date -u +'%Y-%m-%dT%TZ')" +LDFLAGS += -X "github.com/criticalstack/e2d/internal/buildinfo.GitSHA=$(GIT_SHA)" +LDFLAGS += -X "github.com/criticalstack/e2d/internal/buildinfo.Version=$(VERSION)" GOFLAGS = -gcflags "all=-trimpath=$(PWD)" -asmflags "all=-trimpath=$(PWD)" GO_BUILD_ENV_VARS := GO111MODULE=on CGO_ENABLED=0 -.PHONY: build test test-manager clean +##@ Building -build: clean ## Build the e2d golang binary +.PHONY: e2d + +e2d: ## Build the e2d golang binary $(GO_BUILD_ENV_VARS) go build -o bin/e2d $(GOFLAGS) -ldflags '$(LDFLAGS)' ./cmd/e2d -test: ## Run all tests - go test ./... +.PHONY: update-codegen +update-codegen: ## Update generated code (slow) + @echo "Updating generated code files ..." + @echo " *** This can be slow and does not need to run every build ***" + @hack/tools/update-codegen.sh -test-manager: ## Test the manager package - go test ./pkg/manager -test.long +##@ Testing -clean: ## Cleanup the project folders - @rm -rf ./bin/* - @rm -rf hack/tools/bin +.PHONY: test test-e2e lint lint-full + +test: ## Run all tests + @go test $(shell go list ./... | grep -v e2e) -.PHONY: lint +test-e2e: ## Run e2e tests + @go test ./e2e -parallel=16 -count=1 lint: $(GOLANGCI_LINT) ## Lint codebase $(GOLANGCI_LINT) run -v @@ -60,15 +54,20 @@ lint-full: $(GOLANGCI_LINT) ## Run slower linters to detect possible issues ##@ Helpers -.PHONY: help +.PHONY: help clean $(GOLANGCI_LINT): $(TOOLS_DIR)/go.mod # Build golangci-lint from tools folder. cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/golangci-lint github.com/golangci/golangci-lint/cmd/golangci-lint +clean: ## Cleanup the project folders + @rm -rf ./bin/* + @rm -rf hack/tools/bin + help: ## Display this help - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) +# TODO: move to hack/tools generate: protoc -I pkg/manager/e2dpb \ -I vendor/ \ diff --git a/README.md b/README.md index 3b6d416..edd16f2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # e2d -[![GoDoc](https://godoc.org/github.com/criticalstack/e2d?status.svg)](https://godoc.org/github.com/criticalstack/e2d) +[![PkgGoDev](https://pkg.go.dev/badge/github.com/criticalstack/e2d)](https://pkg.go.dev/github.com/criticalstack/e2d) [![Build Status](https://cloud.drone.io/api/badges/criticalstack/e2d/status.svg)](https://cloud.drone.io/criticalstack/e2d) e2d is a command-line tool for deploying and managing etcd clusters, both in the cloud or on bare-metal. It also includes [e2db](https://github.com/criticalstack/e2d/tree/master/pkg/e2db), an ORM-like abstraction for working with etcd. @@ -9,8 +9,8 @@ e2d is a command-line tool for deploying and managing etcd clusters, both in the - [What is e2d](#what-is-e2d) - [Features](#features) - - [Design](#design) -- [Getting started](#getting-started) + - [Installation](#installation) + - [Getting started](#getting-started) - [Required ports](#required-ports) - [Configuration](#configuration) - [Peer discovery](#peer-discovery) @@ -21,7 +21,6 @@ e2d is a command-line tool for deploying and managing etcd clusters, both in the - [Usage](#usage) - [Generating certificates](#generating-certificates) - [Running with systemd](#running-with-systemd) - - [Running with Kubernetes](#running-with-kubernetes) - [FAQ](#faq) ## What is e2d @@ -43,30 +42,80 @@ e2d is designed to manage highly available etcd clusters in the cloud. It can be In dynamic cloud environments, etcd membership is seeded from cloud provider APIs and maintained via a gossip network. This ensures etcd stays healthy and available when nodes might be automatically created or destroyed. -### Design +### Installation -A key design philosophy of e2d is ease-of-use, so having minimal and/or automatic configuration is an important part of the user experience. Since e2d uses a gossip network for peer discovery, cloud metadata services can be leveraged to dynamically create the initial etcd bootstrap configuration. The gossip network is also used for determining node liveness, ensuring that healthy members can safely (and automatically) remove and replace a failing minority of members. An automatic snapshot feature creates periodic backups, which can be restored in the case of a majority failure of etcd members. This is all handled for the user by simply setting a shared file location for the snapshot, and e2d handles the rest. This ends up being incredibly helpful for those using Kubernetes in their dev environments, where cost-savings policies might stop instances over nights/weekends. +The easiest way to install: -While e2d makes use of cloud provider specific features, it never depends on them. The abstraction for peer discovery and snapshot storage are generalized so they can be ported to many different platforms trivially. Another neat aspect of e2d is that it embeds etcd directly into its own binary, effectively using it like a library. This is what enables some of the more complex automation, and with only one binary it reduces the complexity of deploying into production. +```sh +curl -sSfL https://raw.githubusercontent.com/criticalstack/e2d/master/scripts/install.sh | sh +``` + +Pre-built binaries are also available in [Releases](https://github.com/criticalstack/e2d/releases/latest). e2d is written in Go so it is also pretty simple to install via go: + +```sh +go get github.com/criticalstack/e2d/cmd/e2d +``` + +Packages can also be installed from [packagecloud.io](https://packagecloud.io/criticalstack/public) (includes systemd service file). + +Debian/Ubuntu: + +```sh +curl -sL https://packagecloud.io/criticalstack/public/gpgkey | apt-key add - +apt-add-repository https://packagecloud.io/criticalstack/public/ubuntu +apt-get install -y e2d +``` + +Fedora: -## Getting started +```sh +dnf config-manager --add-repo https://packagecloud.io/criticalstack/public/fedora +dnf install -y e2d +``` + +### Getting started Running a single-node cluster: -```bash -$ e2d run +```sh +❯ e2d run ``` -Multi-node clusters require that the `--required-cluster-size/-n` flag be set with the desired size of the cluster. Each node must also either provide a seed of peers (via the `--bootstrap-addrs` flag): +Configuration is made through a single yaml file passed to `e2d run` via the `--config/-c` flag: -```bash -$ e2d run -n 3 --bootstrap-addrs 10.0.2.15,10.0.2.17 +```sh +❯ e2d run -c config.yaml +``` + +Multi-node clusters require the `requiredClusterSize` value be set with the desired size of the cluster. Each node must also either provide a seed of peers (via `initialPeers`): + +```yaml +requiredClusterSize: 3 +discovery: + initialPeers: + - 10.0.2.15:7980 + - 10.0.2.17:7980 ``` or specify a method for [peer discovery](#peer-discovery): -```bash -$ e2d run -n 3 --peer-discovery aws-autoscaling-group +```yaml +requiredClusterSize: 3 +discovery: + type: aws/autoscaling-group + + # optionally provide the name of the ASG, otherwise will default to detecting + # the ASG for the EC2 instance + matches: + name: my-asg +``` + +The e2d configuration file uses Kubernetes-like API versioning, ensuring that compatbility is maintained through implicit conversions. If `apiVersion` and `kind` are not provided, the version for the e2d binary is presumed, otherwise an explicit version can be provided as needed: + +```yaml +apiVersion: e2d.crit.sh/v1alpha1 +kind: Configuration +... ``` ### Required ports @@ -88,16 +137,23 @@ The same ports required by etcd are necessary, along with a couple new ones: Peers can be automatically discovered based upon several different built-in methods: -| Method | Usage | +| Method | Name | | --- | --- | -| AWS Autoscaling Group | `aws-autoscaling-group` | -| AWS EC2 tags | `ec2-tags[:=,=]` | -| Digital Ocean tags | `do-tags[:,]` | +| AWS Autoscaling Group | `aws/autoscaling-group` | +| AWS EC2 tags | `aws/tags` | +| Digital Ocean tags | `digitalocean/tags` | For example, running a 3-node cluster in AWS where initial peers are found via ec2 tags: -```bash -$ e2d run -n 3 --peer-discovery ec2-tags:Name=my-cluster,Email=admin@example.com +```sh +❯ e2d run -c - <`. +Snapshot storage options like S3 use TLS and offer encryption-at-rest, however, it is possible that encryption of the snapshot file itself might be needed. This is especially true for other storage options that do not offer these features. Enabling snapshot encryption is simply a flag in the e2d configuration file, however, must be combined with `caCert`/`caKey` since the encryption key itself is derived from the CA private key: + +```yaml +caCert: /etc/kubernetes/pki/etcd/ca.crt +caKey: /etc/kubernetes/pki/etcd/ca.key +snapshot: + encryption: true +``` The encryption being used is AES-256 in [CTR mode](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_(CTR)), with message authentication provided by HMAC-512_256. This mode was used because the Go implementation of AES-GCM would require the entire snapshot to be in-memory, and CTR mode allows for memory efficient streaming. -It is possible to use compression alongside of encryption, however, it is important to note that because of the possibility of opening up side-channel attacks, compression is not performed before encryption. The nature of how strong encryption works causes the encrypted snapshot to not gain benefits from compression. So enabling snapshot compression with encryption will cause the gzip level to be set to `gzip.NoCompression`, meaning it still creates a valid gzip file, but doesn't waste nearly as many compute resources while doing so. +It is possible to use compression alongside of encryption, however, it is important to note that because of the possibility of opening up side-channel attacks, compression is not performed before encryption. The nature of how strong encryption works causes the encrypted snapshot to not gain benefits from compression. So enabling snapshot compression with encryption will cause the gzip level to be set to `gzip.NoCompression`, meaning it still creates a valid gzip file, but doesn't waste nearly as many compute resources while doing so (it is used because the gzip header is useful for file-type detection and checksum validation). + +```yaml +caCert: /etc/kubernetes/pki/etcd/ca.crt +caKey: /etc/kubernetes/pki/etcd/ca.key +snapshot: + compression: true + encryption: true + interval: 5m + file: s3://bucket/snapshot.tar.gz +``` #### Storage options -The `--snapshot-backup-url` has several schemes it implicitly understands: +The `snapshot.file` value has several schemes it implicitly understands: | Storage Type | Usage | | --- | --- | @@ -141,39 +226,23 @@ Note: these examples assume e2d is being deployed in AWS. Mutual TLS authentication is highly recommended and e2d embeds the necessary functionality to generate the required key pairs. To get started with a new e2d cluster, first initialize a new key/cert pair: ```bash -$ e2d pki init +❯ e2d certs init ``` -Then create the remaining certificates for the client, peer, and server communication: - -```bash -$ e2d pki gencerts -``` - -This will create the remaining key pairs needed to run e2d based on the initial cluster key pair. +The remaining certificates for the client, peer, and server communication are created automatically and placed in the same directory as the CA key/cert. ### Running with systemd -An example unit file for running via systemd in an AWS ASG: +This systemd service file is provided with the Debian/Ubuntu packages: ``` [Unit] Description=e2d [Service] -ExecStart=/usr/local/bin/e2d run \ - --data-dir=/var/lib/etcd \ - --ca-cert=/etc/kubernetes/pki/etcd/ca.crt \ - --ca-key=/etc/kubernetes/pki/etcd/ca.crt \ - --client-cert=/etc/kubernetes/pki/etcd/client.crt \ - --client-key=/etc/kubernetes/pki/etcd/client.key \ - --peer-cert=/etc/kubernetes/pki/etcd/peer.crt \ - --peer-key=/etc/kubernetes/pki/etcd/peer.key \ - --server-cert=/etc/kubernetes/pki/etcd/server.crt \ - --server-key=/etc/kubernetes/pki/etcd/server.key \ - --peer-discovery=aws-autoscaling-group \ - --required-cluster-size=3 \ - --snapshot-backup-url=s3://e2d_snapshot_bucket +Environment="E2D_CONFIG_ARGS=--config=/etc/e2d.yaml" +ExecStart=/usr/local/bin/e2d run $E2D_CONFIG_ARGS +EnvironmentFile=-/etc/e2d.conf Restart=on-failure RestartSec=30 @@ -181,9 +250,7 @@ RestartSec=30 WantedBy=multi-user.target ``` -### Running in Kubernetes - -e2d currently doesn't have the integration necessary to run correctly within Kubernetes, however, it should be relatively easy to add the necessary discovery features to make that work and is planned for future releases of e2d. +It relies on providing the e2d configuration in a fixed location: `/etc/e2d.yaml`. Environment variables can be set for the service by providing them in the `/etc/e2d.conf` file. ## FAQ diff --git a/deploy/criticalstack.repo b/build/criticalstack.repo similarity index 100% rename from deploy/criticalstack.repo rename to build/criticalstack.repo diff --git a/deploy/e2d.service b/build/e2d.service similarity index 56% rename from deploy/e2d.service rename to build/e2d.service index 550564e..fd8a0db 100644 --- a/deploy/e2d.service +++ b/build/e2d.service @@ -2,7 +2,8 @@ Description=e2d [Service] -ExecStart=/usr/local/bin/e2d run +Environment="E2D_CONFIG_ARGS=--config=/etc/e2d.yaml" +ExecStart=/usr/local/bin/e2d run $E2D_CONFIG_ARGS EnvironmentFile=-/etc/e2d.conf Restart=on-failure RestartSec=30 diff --git a/cmd/e2d/app/certs/certs.go b/cmd/e2d/app/certs/certs.go new file mode 100644 index 0000000..521f279 --- /dev/null +++ b/cmd/e2d/app/certs/certs.go @@ -0,0 +1,20 @@ +package certs + +import ( + "github.com/spf13/cobra" + + certsgenerate "github.com/criticalstack/e2d/cmd/e2d/app/certs/generate" + certsinit "github.com/criticalstack/e2d/cmd/e2d/app/certs/init" +) + +func NewCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "certs", + Short: "manage e2d certs", + } + cmd.AddCommand( + certsinit.NewCommand(), + certsgenerate.NewCommand(), + ) + return cmd +} diff --git a/cmd/e2d/app/certs/generate/generate.go b/cmd/e2d/app/certs/generate/generate.go new file mode 100644 index 0000000..631addd --- /dev/null +++ b/cmd/e2d/app/certs/generate/generate.go @@ -0,0 +1,73 @@ +package generate + +import ( + "path/filepath" + "strings" + + "github.com/spf13/cobra" + + "github.com/criticalstack/e2d/pkg/log" + "github.com/criticalstack/e2d/pkg/manager" +) + +var opts struct { + CertDir string + AltNames string +} + +func NewCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "generate [flags] [arg]\n\nValidArgs:\n all, server, peer, client", + Short: "generate certificates/private keys", + Aliases: []string{"gen"}, + Args: cobra.ExactValidArgs(1), + ValidArgs: []string{ + "all", + "server", + "peer", + "client", + }, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + var names []string + if opts.AltNames != "" { + names = strings.Split(opts.AltNames, ",") + } + ca, err := manager.LoadCertificateAuthority(filepath.Join(opts.CertDir, "ca.crt"), filepath.Join(opts.CertDir, "ca.key"), names...) + if err != nil { + return err + } + switch args[0] { + case "all": + if err := ca.WriteAll(); err != nil { + return err + } + log.Info("generated all certificates successfully.") + return nil + case "server": + if err := ca.WriteServerCertAndKey(); err != nil { + return err + } + log.Info("generated server certificates successfully.") + return nil + case "peer": + if err := ca.WritePeerCertAndKey(); err != nil { + return err + } + log.Info("generated peer certificates successfully.") + return nil + case "client": + if err := ca.WriteClientCertAndKey(); err != nil { + return err + } + log.Info("generated client certificates successfully.") + return nil + } + return nil + }, + } + + cmd.Flags().StringVar(&opts.CertDir, "cert-dir", "", "") + cmd.Flags().StringVar(&opts.AltNames, "alt-names", "", "") + return cmd +} diff --git a/cmd/e2d/app/certs/init/init.go b/cmd/e2d/app/certs/init/init.go new file mode 100644 index 0000000..ddb132f --- /dev/null +++ b/cmd/e2d/app/certs/init/init.go @@ -0,0 +1,34 @@ +package init + +import ( + "os" + + "github.com/spf13/cobra" + + "github.com/criticalstack/e2d/pkg/manager" +) + +var opts struct { + CertDir string +} + +func NewCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "init", + Short: "initialize a new CA", + Args: cobra.NoArgs, + SilenceErrors: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + if opts.CertDir != "" { + if err := os.MkdirAll(opts.CertDir, 0755); err != nil && !os.IsExist(err) { + return err + } + } + return manager.WriteNewCA(opts.CertDir) + }, + } + + cmd.Flags().StringVar(&opts.CertDir, "cert-dir", "", "") + return cmd +} diff --git a/cmd/e2d/app/completion.go b/cmd/e2d/app/completion.go deleted file mode 100644 index b8b9f53..0000000 --- a/cmd/e2d/app/completion.go +++ /dev/null @@ -1,30 +0,0 @@ -package app - -import ( - "os" - - "github.com/criticalstack/e2d/pkg/log" - "github.com/spf13/cobra" -) - -func newCompletionCmd(rootCmd *cobra.Command) *cobra.Command { - cmd := &cobra.Command{ - Use: "completion", - Short: "Generates bash completion scripts", - Run: func(cmd *cobra.Command, args []string) { - w := os.Stdout - if len(args) > 0 { - var err error - w, err = os.OpenFile(args[0], os.O_RDWR|os.O_CREATE, 0644) - if err != nil { - log.Fatal(err) - } - defer w.Close() - } - if err := rootCmd.GenBashCompletion(w); err != nil { - log.Fatal(err) - } - }, - } - return cmd -} diff --git a/cmd/e2d/app/e2d.go b/cmd/e2d/app/e2d.go new file mode 100644 index 0000000..ef42688 --- /dev/null +++ b/cmd/e2d/app/e2d.go @@ -0,0 +1,61 @@ +package app + +import ( + "os" + + "github.com/spf13/cobra" + "go.uber.org/zap/zapcore" + + "github.com/criticalstack/e2d/cmd/e2d/app/certs" + "github.com/criticalstack/e2d/cmd/e2d/app/run" + "github.com/criticalstack/e2d/cmd/e2d/app/version" + "github.com/criticalstack/e2d/pkg/log" +) + +var opts struct { + Verbose bool +} + +func NewCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "e2d", + Short: "etcd manager", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + if opts.Verbose { + log.SetLevel(zapcore.DebugLevel) + } + }, + } + + cmd.AddCommand( + newCompletionCmd(cmd), + certs.NewCommand(), + run.NewCommand(), + version.NewCommand(), + ) + + cmd.PersistentFlags().BoolVarP(&opts.Verbose, "verbose", "v", false, "verbose log output (debug)") + return cmd +} + +func newCompletionCmd(rootCmd *cobra.Command) *cobra.Command { + cmd := &cobra.Command{ + Use: "completion", + Short: "Generates bash completion scripts", + Run: func(cmd *cobra.Command, args []string) { + w := os.Stdout + if len(args) > 0 { + var err error + w, err = os.OpenFile(args[0], os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + log.Fatal(err) + } + defer w.Close() + } + if err := rootCmd.GenBashCompletion(w); err != nil { + log.Fatal(err) + } + }, + } + return cmd +} diff --git a/cmd/e2d/app/pki.go b/cmd/e2d/app/pki.go deleted file mode 100644 index d86c321..0000000 --- a/cmd/e2d/app/pki.go +++ /dev/null @@ -1,173 +0,0 @@ -package app - -import ( - "io/ioutil" - "os" - "path/filepath" - "strings" - - "github.com/cloudflare/cfssl/csr" - "github.com/criticalstack/e2d/pkg/log" - "github.com/criticalstack/e2d/pkg/netutil" - "github.com/criticalstack/e2d/pkg/pki" - "github.com/spf13/cobra" -) - -type pkiOptions struct { - CACert string - CAKey string -} - -func newPKICmd() *cobra.Command { - o := &pkiOptions{} - - cmd := &cobra.Command{ - Use: "pki", - Short: "manage e2d pki", - } - - cmd.PersistentFlags().StringVar(&o.CACert, "ca-cert", "", "") - cmd.PersistentFlags().StringVar(&o.CAKey, "ca-key", "", "") - - cmd.AddCommand( - newPKIInitCmd(o), - newPKIGenCertsCmd(o), - ) - return cmd -} - -func newPKIInitCmd(pkiOpts *pkiOptions) *cobra.Command { - cmd := &cobra.Command{ - Use: "init", - Short: "initialize a new CA", - Run: func(cmd *cobra.Command, args []string) { - path := filepath.Dir(pkiOpts.CACert) - if path != "" { - if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) { - log.Fatal(err) - } - } - r, err := pki.NewDefaultRootCA() - if err != nil { - log.Fatal(err) - } - if err := writeFile(pkiOpts.CACert, r.CA.CertPEM, 0644); err != nil { - log.Fatal(err) - } - if err := writeFile(pkiOpts.CAKey, r.CA.KeyPEM, 0600); err != nil { - log.Fatal(err) - } - }, - } - return cmd -} - -type pkiGenCertsOptions struct { - Hosts string - OutputDir string -} - -func newPKIGenCertsCmd(pkiOpts *pkiOptions) *cobra.Command { - o := pkiGenCertsOptions{} - - cmd := &cobra.Command{ - Use: "gencerts", - Short: "generate certificates/private keys", - Run: func(cmd *cobra.Command, args []string) { - var hosts []string - if o.Hosts != "" { - hosts = strings.Split(o.Hosts, ",") - } - r, err := pki.NewRootCAFromFile(pkiOpts.CACert, pkiOpts.CAKey) - if err != nil { - log.Fatal(err) - } - hostIP, err := netutil.DetectHostIPv4() - if err != nil { - log.Fatal(err) - } - if o.OutputDir != "" { - if err := os.MkdirAll(o.OutputDir, 0755); err != nil && !os.IsExist(err) { - log.Fatal(err) - } - } - hosts = appendHosts(hosts, "127.0.0.1", hostIP) - certs, err := r.GenerateCertificates(pki.ServerSigningProfile, newCertificateRequest("etcd server", hosts...)) - if err != nil { - log.Fatal(err) - } - if err := writeFile(filepath.Join(o.OutputDir, "server.crt"), certs.CertPEM, 0644); err != nil { - log.Fatal(err) - } - if err := writeFile(filepath.Join(o.OutputDir, "server.key"), certs.KeyPEM, 0600); err != nil { - log.Fatal(err) - } - certs, err = r.GenerateCertificates(pki.PeerSigningProfile, newCertificateRequest("etcd peer", hosts...)) - if err != nil { - log.Fatal(err) - } - if err := writeFile(filepath.Join(o.OutputDir, "peer.crt"), certs.CertPEM, 0644); err != nil { - log.Fatal(err) - } - if err := writeFile(filepath.Join(o.OutputDir, "peer.key"), certs.KeyPEM, 0600); err != nil { - log.Fatal(err) - } - certs, err = r.GenerateCertificates(pki.ClientSigningProfile, newCertificateRequest("etcd client")) - if err != nil { - log.Fatal(err) - } - if err := writeFile(filepath.Join(o.OutputDir, "client.crt"), certs.CertPEM, 0644); err != nil { - log.Fatal(err) - } - if err := writeFile(filepath.Join(o.OutputDir, "client.key"), certs.KeyPEM, 0600); err != nil { - log.Fatal(err) - } - log.Info("generated certificates successfully.") - }, - } - - cmd.Flags().StringVar(&o.Hosts, "hosts", "", "") - cmd.Flags().StringVar(&o.OutputDir, "output-dir", "", "") - - return cmd -} - -func appendHosts(hosts []string, newHosts ...string) []string { - for _, newHost := range newHosts { - if newHost == "" { - continue - } - for _, host := range hosts { - if newHost == host { - continue - } - } - hosts = append(hosts, newHost) - } - return hosts -} - -func newCertificateRequest(commonName string, hosts ...string) *csr.CertificateRequest { - return &csr.CertificateRequest{ - Names: []csr.Name{ - { - C: "US", - ST: "Boston", - L: "MA", - }, - }, - KeyRequest: &csr.KeyRequest{ - A: "rsa", - S: 2048, - }, - Hosts: hosts, - CN: commonName, - } -} - -func writeFile(filename string, data []byte, perm os.FileMode) error { - if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil { - return err - } - return ioutil.WriteFile(filename, data, perm) -} diff --git a/cmd/e2d/app/root.go b/cmd/e2d/app/root.go deleted file mode 100644 index 510a1ea..0000000 --- a/cmd/e2d/app/root.go +++ /dev/null @@ -1,26 +0,0 @@ -package app - -import ( - "github.com/spf13/cobra" -) - -var globalOptions struct { - verbose bool -} - -func NewRootCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "e2d", - Short: "etcd manager", - } - cmd.PersistentFlags().BoolVarP(&globalOptions.verbose, "verbose", "v", false, "verbose log output (debug)") - - cmd.AddCommand( - newCompletionCmd(cmd), - newRunCmd(), - newPKICmd(), - newVersionCmd(), - ) - - return cmd -} diff --git a/cmd/e2d/app/run.go b/cmd/e2d/app/run.go deleted file mode 100644 index b81e96e..0000000 --- a/cmd/e2d/app/run.go +++ /dev/null @@ -1,253 +0,0 @@ -package app - -import ( - "context" - "fmt" - "strings" - "time" - - "github.com/criticalstack/e2d/pkg/client" - "github.com/criticalstack/e2d/pkg/cmdutil" - "github.com/criticalstack/e2d/pkg/discovery" - "github.com/criticalstack/e2d/pkg/log" - "github.com/criticalstack/e2d/pkg/manager" - "github.com/criticalstack/e2d/pkg/snapshot" - "github.com/pkg/errors" - "github.com/spf13/cobra" - "go.uber.org/zap" -) - -type runOptions struct { - Name string `env:"E2D_NAME"` - DataDir string `env:"E2D_DATA_DIR"` - Host string `env:"E2D_HOST"` - ClientAddr string `env:"E2D_CLIENT_ADDR"` - PeerAddr string `env:"E2D_PEER_ADDR"` - GossipAddr string `env:"E2D_GOSSIP_ADDR"` - - CACert string `env:"E2D_CA_CERT"` - CAKey string `env:"E2D_CA_KEY"` - PeerCert string `env:"E2D_PEER_CERT"` - PeerKey string `env:"E2D_PEER_KEY"` - ServerCert string `env:"E2D_SERVER_CERT"` - ServerKey string `env:"E2D_SERVER_KEY"` - - BootstrapAddrs string `env:"E2D_BOOTSTRAP_ADDRS"` - RequiredClusterSize int `env:"E2D_REQUIRED_CLUSTER_SIZE"` - - HealthCheckInterval time.Duration `env:"E2D_HEALTH_CHECK_INTERVAL"` - HealthCheckTimeout time.Duration `env:"E2D_HEALTH_CHECK_TIMEOUT"` - - PeerDiscovery string `env:"E2D_PEER_DISCOVERY"` - - SnapshotBackupURL string `env:"E2D_SNAPSHOT_BACKUP_URL"` - SnapshotCompression bool `env:"E2D_SNAPSHOT_COMPRESSION"` - SnapshotEncryption bool `env:"E2D_SNAPSHOT_ENCRYPTION"` - SnapshotInterval time.Duration `env:"E2D_SNAPSHOT_INTERVAL"` - - AWSAccessKey string `env:"E2D_AWS_ACCESS_KEY"` - AWSSecretKey string `env:"E2D_AWS_SECRET_KEY"` - AWSRoleSessionName string `env:"E2D_AWS_ROLE_SESSION_NAME"` - - DOAccessToken string `env:"E2D_DO_ACCESS_TOKEN"` - DOSpacesKey string `env:"E2D_DO_SPACES_KEY"` - DOSpacesSecret string `env:"E2D_DO_SPACES_SECRET"` -} - -func newRunCmd() *cobra.Command { - o := &runOptions{} - - cmd := &cobra.Command{ - Use: "run", - Short: "start a managed etcd instance", - Run: func(cmd *cobra.Command, args []string) { - peerGetter, err := getPeerGetter(o) - if err != nil { - log.Fatalf("%+v", err) - } - - baddrs, err := getInitialBootstrapAddrs(o, peerGetter) - if err != nil { - log.Fatalf("%+v", err) - } - - snapshotter, err := getSnapshotProvider(o) - if err != nil { - log.Fatalf("%+v", err) - } - - m, err := manager.New(&manager.Config{ - Name: o.Name, - Dir: o.DataDir, - Host: o.Host, - ClientAddr: o.ClientAddr, - PeerAddr: o.PeerAddr, - GossipAddr: o.GossipAddr, - BootstrapAddrs: baddrs, - RequiredClusterSize: o.RequiredClusterSize, - SnapshotInterval: o.SnapshotInterval, - SnapshotCompression: o.SnapshotCompression, - SnapshotEncryption: o.SnapshotEncryption, - HealthCheckInterval: o.HealthCheckInterval, - HealthCheckTimeout: o.HealthCheckTimeout, - ClientSecurity: client.SecurityConfig{ - CertFile: o.ServerCert, - KeyFile: o.ServerKey, - TrustedCAFile: o.CACert, - }, - PeerSecurity: client.SecurityConfig{ - CertFile: o.PeerCert, - KeyFile: o.PeerKey, - TrustedCAFile: o.CACert, - }, - CACertFile: o.CACert, - CAKeyFile: o.CAKey, - PeerGetter: peerGetter, - Snapshotter: snapshotter, - Debug: globalOptions.verbose, - }) - if err != nil { - log.Fatalf("%+v", err) - } - if err := m.Run(); err != nil { - log.Fatalf("%+v", err) - } - }, - } - - cmd.Flags().StringVar(&o.Name, "name", "", "specify a name for the node") - cmd.Flags().StringVar(&o.DataDir, "data-dir", "", "etcd data-dir") - cmd.Flags().StringVar(&o.Host, "host", "", "host IPv4 (defaults to 127.0.0.1 if unset)") - cmd.Flags().StringVar(&o.ClientAddr, "client-addr", "0.0.0.0:2379", "etcd client addrress") - cmd.Flags().StringVar(&o.PeerAddr, "peer-addr", "0.0.0.0:2380", "etcd peer addrress") - cmd.Flags().StringVar(&o.GossipAddr, "gossip-addr", "0.0.0.0:7980", "gossip address") - - cmd.Flags().StringVar(&o.CACert, "ca-cert", "", "etcd trusted ca certificate") - cmd.Flags().StringVar(&o.CAKey, "ca-key", "", "etcd ca key") - cmd.Flags().StringVar(&o.PeerCert, "peer-cert", "", "etcd peer certificate") - cmd.Flags().StringVar(&o.PeerKey, "peer-key", "", "etcd peer private key") - cmd.Flags().StringVar(&o.ServerCert, "server-cert", "", "etcd server certificate") - cmd.Flags().StringVar(&o.ServerKey, "server-key", "", "etcd server private key") - - cmd.Flags().StringVar(&o.BootstrapAddrs, "bootstrap-addrs", "", "initial addresses used for node discovery") - cmd.Flags().IntVarP(&o.RequiredClusterSize, "required-cluster-size", "n", 1, "size of the etcd cluster should be {1,3,5}") - - cmd.Flags().DurationVar(&o.HealthCheckInterval, "health-check-interval", 1*time.Minute, "") - cmd.Flags().DurationVar(&o.HealthCheckTimeout, "health-check-timeout", 5*time.Minute, "") - - cmd.Flags().StringVar(&o.PeerDiscovery, "peer-discovery", "", "which method {aws-autoscaling-group,ec2-tags,do-tags} to use to discover peers") - - cmd.Flags().DurationVar(&o.SnapshotInterval, "snapshot-interval", 1*time.Minute, "frequency of etcd snapshots") - cmd.Flags().StringVar(&o.SnapshotBackupURL, "snapshot-backup-url", "", "an absolute path to shared filesystem storage (like file:///etcd-backups) or cloud storage bucket (like s3://etcd-backups) for snapshot backups") - cmd.Flags().BoolVar(&o.SnapshotCompression, "snapshot-compression", false, "compression snapshots with gzip") - cmd.Flags().BoolVar(&o.SnapshotEncryption, "snapshot-encryption", false, "encrypt snapshots with aes-256") - - cmd.Flags().StringVar(&o.AWSAccessKey, "aws-access-key", "", "") - cmd.Flags().StringVar(&o.AWSSecretKey, "aws-secret-key", "", "") - cmd.Flags().StringVar(&o.AWSRoleSessionName, "aws-role-session-name", "", "") - - cmd.Flags().StringVar(&o.DOAccessToken, "do-access-token", "", "DigitalOcean personal access token") - cmd.Flags().StringVar(&o.DOSpacesKey, "do-spaces-key", "", "DigitalOcean spaces access key") - cmd.Flags().StringVar(&o.DOSpacesSecret, "do-spaces-secret", "", "DigitalOcean spaces secret") - if err := cmdutil.SetEnvs(o); err != nil { - log.Debug("cannot set environment variables", zap.Error(err)) - } - - return cmd -} - -func parsePeerDiscovery(s string) (string, []discovery.KeyValue) { - kvs := make([]discovery.KeyValue, 0) - parts := strings.SplitN(s, ":", 2) - if len(parts) != 2 { - return s, kvs - } - pairs := strings.Split(parts[1], ",") - for _, pair := range pairs { - parts := strings.SplitN(pair, "=", 2) - switch len(parts) { - case 1: - kvs = append(kvs, discovery.KeyValue{Key: parts[0], Value: ""}) - case 2: - kvs = append(kvs, discovery.KeyValue{Key: parts[0], Value: parts[1]}) - } - } - return parts[0], kvs -} - -func getPeerGetter(o *runOptions) (discovery.PeerGetter, error) { - method, kvs := parsePeerDiscovery(o.PeerDiscovery) - log.Info("peer-discovery", zap.String("method", method), zap.String("kvs", fmt.Sprintf("%v", kvs))) - switch strings.ToLower(method) { - case "aws-autoscaling-group": - // TODO(chris): needs to take access key/secret - return discovery.NewAmazonAutoScalingPeerGetter() - case "ec2-tags": - return discovery.NewAmazonInstanceTagPeerGetter(kvs) - case "do-tags": - if len(kvs) == 0 { - return nil, errors.New("must provide at least 1 tag") - } - return discovery.NewDigitalOceanPeerGetter(&discovery.DigitalOceanConfig{ - AccessToken: o.DOAccessToken, - TagValue: kvs[0].Key, - }) - case "k8s-labels": - return nil, errors.New("peer getter not yet implemented") - } - return &discovery.NoopGetter{}, nil -} - -func getInitialBootstrapAddrs(o *runOptions, peerGetter discovery.PeerGetter) ([]string, error) { - baddrs := make([]string, 0) - - // user-provided bootstrap addresses take precedence - if o.BootstrapAddrs != "" { - baddrs = strings.Split(o.BootstrapAddrs, ",") - } - - if o.RequiredClusterSize > 1 && len(baddrs) == 0 { - addrs, err := peerGetter.GetAddrs(context.Background()) - if err != nil { - return nil, err - } - log.Debugf("cloud provided addresses: %v", addrs) - for _, addr := range addrs { - baddrs = append(baddrs, fmt.Sprintf("%s:%d", addr, manager.DefaultGossipPort)) - } - log.Debugf("bootstrap addrs: %v", baddrs) - if len(baddrs) == 0 { - return nil, errors.Errorf("bootstrap addresses must be provided") - } - } - return baddrs, nil -} - -func getSnapshotProvider(o *runOptions) (snapshot.Snapshotter, error) { - if o.SnapshotBackupURL == "" { - return nil, nil - } - u, err := snapshot.ParseSnapshotBackupURL(o.SnapshotBackupURL) - if err != nil { - return nil, err - } - - switch u.Type { - case snapshot.FileType: - return snapshot.NewFileSnapshotter(u.Path) - case snapshot.S3Type: - return snapshot.NewAmazonSnapshotter(&snapshot.AmazonConfig{ - RoleSessionName: o.AWSRoleSessionName, - Bucket: u.Bucket, - Key: u.Path, - }) - case snapshot.SpacesType: - return snapshot.NewDigitalOceanSnapshotter(&snapshot.DigitalOceanConfig{ - SpacesURL: o.SnapshotBackupURL, - SpacesAccessKey: o.DOSpacesKey, - SpacesSecretKey: o.DOSpacesSecret, - }) - default: - return nil, errors.Errorf("unsupported snapshot url format: %#v", o.SnapshotBackupURL) - } -} diff --git a/cmd/e2d/app/run/run.go b/cmd/e2d/app/run/run.go new file mode 100644 index 0000000..a99d3d9 --- /dev/null +++ b/cmd/e2d/app/run/run.go @@ -0,0 +1,78 @@ +package run + +import ( + "encoding/json" + "os" + + configutil "github.com/criticalstack/crit/pkg/config/util" + "github.com/criticalstack/crit/pkg/kubernetes/yaml" + "github.com/pkg/errors" + "github.com/spf13/cobra" + + configv1alpha1 "github.com/criticalstack/e2d/pkg/config/v1alpha1" + "github.com/criticalstack/e2d/pkg/manager" +) + +var opts struct { + ConfigFile string +} + +func NewCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "run", + Short: "start a managed etcd instance", + Args: cobra.NoArgs, + SilenceErrors: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + if _, err := os.Stat(opts.ConfigFile); os.IsNotExist(err) { + m, err := manager.New(&configv1alpha1.Configuration{}) + if err != nil { + return err + } + return m.Run() + } + data, err := configutil.ReadFile(opts.ConfigFile) + if err != nil { + return err + } + data, err = injectVersion(data) + if err != nil { + return err + } + obj, err := yaml.UnmarshalFromYaml(data, configv1alpha1.SchemeGroupVersion) + if err != nil { + return err + } + cfg, ok := obj.(*configv1alpha1.Configuration) + if !ok { + return errors.Errorf("expected %q, received %T", configv1alpha1.SchemeGroupVersion, obj) + } + m, err := manager.New(cfg) + if err != nil { + return err + } + return m.Run() + }, + } + cmd.Flags().StringVarP(&opts.ConfigFile, "config", "c", "config.yaml", "config file") + return cmd +} + +func injectVersion(data []byte) ([]byte, error) { + resources, err := yaml.UnmarshalFromYamlUnstructured(data) + if err != nil { + return nil, err + } + if len(resources) == 0 { + return nil, errors.Errorf("cannot find resources in configuration file: %q", opts.ConfigFile) + } + u := resources[0] + if u.GetAPIVersion() == "" { + u.SetAPIVersion(configv1alpha1.SchemeGroupVersion.String()) + } + if u.GetKind() == "" { + u.SetKind("Configuration") + } + return json.Marshal(u) +} diff --git a/cmd/e2d/app/version.go b/cmd/e2d/app/version/version.go similarity index 62% rename from cmd/e2d/app/version.go rename to cmd/e2d/app/version/version.go index bfa803a..e7405f7 100644 --- a/cmd/e2d/app/version.go +++ b/cmd/e2d/app/version/version.go @@ -1,20 +1,22 @@ -package app +package version import ( "encoding/json" "fmt" - "github.com/criticalstack/e2d/pkg/buildinfo" - "github.com/criticalstack/e2d/pkg/log" "github.com/spf13/cobra" "go.etcd.io/etcd/version" + + "github.com/criticalstack/e2d/internal/buildinfo" ) -func newVersionCmd() *cobra.Command { +func NewCommand() *cobra.Command { cmd := &cobra.Command{ - Use: "version", - Short: "etcd version", - Run: func(cmd *cobra.Command, args []string) { + Use: "version", + Short: "etcd version", + SilenceErrors: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { data, err := json.Marshal(map[string]map[string]string{ "etcd": { "Version": version.Version, @@ -29,9 +31,10 @@ func newVersionCmd() *cobra.Command { }, }) if err != nil { - log.Fatal(err) + return err } fmt.Printf("%s\n", data) + return nil }, } return cmd diff --git a/cmd/e2d/main.go b/cmd/e2d/main.go index 8cd63b6..bb7d4cd 100644 --- a/cmd/e2d/main.go +++ b/cmd/e2d/main.go @@ -6,7 +6,7 @@ import ( ) func main() { - if err := app.NewRootCmd().Execute(); err != nil { - log.Fatal(err) + if err := app.NewCommand().Execute(); err != nil { + log.Fatalf("%+v", err) } } diff --git a/e2e/context.go b/e2e/context.go new file mode 100644 index 0000000..9140103 --- /dev/null +++ b/e2e/context.go @@ -0,0 +1,113 @@ +package e2e + +import ( + "sync" + "time" + + "github.com/criticalstack/e2d/pkg/log" +) + +type Context struct { + c *TestCluster + nodes []*Node +} + +func (c *Context) Start() *Context { + for _, node := range c.nodes { + node.Start() + } + return c +} + +func (c *Context) Remove() *Context { + for _, n := range c.nodes { + n.Remove() + } + return c +} + +func (c *Context) Restart() *Context { + var wg sync.WaitGroup + for _, n := range c.nodes { + wg.Add(1) + + go func(n *Node) { + defer wg.Done() + + n.Restart() + }(n) + } + wg.Wait() + return c +} + +func (c *Context) Stop() *Context { + for _, n := range c.nodes { + n.Stop() + } + return c +} + +func (c *Context) Wait() *Context { + waitChan := make(chan struct{}) + + go func() { + var wg sync.WaitGroup + for _, node := range c.nodes { + wg.Add(1) + + go func(node *Node) { + defer wg.Done() + + node.Wait() + }(node) + } + wg.Wait() + waitChan <- struct{}{} + }() + + select { + case <-waitChan: + break + case <-time.After(1 * time.Minute): + c.c.t.Fatal("timed out waiting for node state") + } + log.Info("healthy!") + return c +} + +func (c *Context) SaveSnapshot() *Context { + for _, n := range c.nodes { + n.SaveSnapshot() + } + return c +} + +func (c *Context) TestClientSet(name string) *Context { + cl := c.c.Node(name).Client() + if err := cl.Set(testKey1, testValue1); err != nil { + c.c.t.Fatal(err) + } + v, err := cl.Get(testKey1) + if err != nil { + c.c.t.Fatal(err) + } + cl.Close() + if string(v) != testValue1 { + c.c.t.Fatalf("expected %#v, received %#v", testValue1, string(v)) + } + return c +} + +func (c *Context) TestClientGet(name string) *Context { + cl := c.c.Node(name).Client() + v, err := cl.Get(testKey1) + if err != nil { + c.c.t.Fatal(err) + } + cl.Close() + if string(v) != testValue1 { + c.c.t.Fatalf("expected %#v, received %#v", testValue1, string(v)) + } + return c +} diff --git a/e2e/manager_test.go b/e2e/manager_test.go new file mode 100644 index 0000000..2a54810 --- /dev/null +++ b/e2e/manager_test.go @@ -0,0 +1,353 @@ +//nolint:goconst +package e2e + +import ( + "context" + "fmt" + "net/http" + "path/filepath" + "testing" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + configv1alpha1 "github.com/criticalstack/e2d/pkg/config/v1alpha1" + "github.com/criticalstack/e2d/pkg/etcdserver" + "github.com/criticalstack/e2d/pkg/log" +) + +func init() { + log.SetLevel(zapcore.DebugLevel) +} + +func TestManagerSingleFaultRecoveryFollower(t *testing.T) { + c := NewTestCluster(t, 3).Setup() + defer c.Cleanup() + + c.AddNodes("node1", "node2", "node3"). + Start().Wait(). + TestClientSet("node1") + c.Follower(). + Stop().Remove().Wait() + c.AddNodes("node4"). + Start().Wait(). + TestClientGet("node4") + + log.Info("test completed successfully", zap.String("test", t.Name())) +} + +func TestManagerSingleFaultRecoveryLeader(t *testing.T) { + c := NewTestCluster(t, 3).Setup() + defer c.Cleanup() + + c.AddNodes("node1", "node2", "node3"). + Start().Wait(). + TestClientSet("node1") + c.Leader(). + Stop().Remove().Wait() + c.AddNodes("node4"). + Start().Wait(). + TestClientGet("node4") + + log.Info("test completed successfully", zap.String("test", t.Name())) +} + +func TestManagerRestoreClusterFromSnapshotNoCompression(t *testing.T) { + c := NewTestCluster(t, 3).Setup() + c.WithConfig(func(cfg *configv1alpha1.Configuration) { + cfg.SnapshotConfiguration.File = "file://" + filepath.Join(c.dir, "snapshots") + }) + defer c.Cleanup() + + c.AddNodes("node1", "node2", "node3"). + Start().Wait(). + TestClientSet("node2") + c.Leader(). + SaveSnapshot() + c.Nodes("node1", "node2", "node3"). + Stop().Remove() + + // need to wait a bit to ensure the port is free to bind + time.Sleep(1 * time.Second) + + // SnapshotInterval is 0 so creating snapshots is disabled, however, + // SnapshotDir is being replaced with default SnapshotDir from node1 so + // that this new node can restore that snapshot + c.AddNodes("node4", "node5", "node6"). + Start().Wait(). + TestClientGet("node4") + + log.Info("test completed successfully", zap.String("test", t.Name())) +} + +func TestManagerRestoreClusterFromSnapshotCompression(t *testing.T) { + c := NewTestCluster(t, 3).Setup() + c.WithConfig(func(cfg *configv1alpha1.Configuration) { + cfg.SnapshotConfiguration.Compression = true + cfg.SnapshotConfiguration.File = "file://" + filepath.Join(c.dir, "snapshots") + }) + defer c.Cleanup() + + c.AddNodes("node1", "node2", "node3"). + Start().Wait(). + TestClientSet("node2") + c.Leader(). + SaveSnapshot() + c.Nodes("node1", "node2", "node3"). + Stop().Remove() + + // need to wait a bit to ensure the port is free to bind + time.Sleep(1 * time.Second) + + // SnapshotInterval is 0 so creating snapshots is disabled, however, + // SnapshotDir is being replaced with default SnapshotDir from node1 so + // that this new node can restore that snapshot + c.WithConfig(func(cfg *configv1alpha1.Configuration) { + cfg.SnapshotConfiguration.Compression = false + }).AddNodes("node4", "node5", "node6"). + Start().Wait(). + TestClientGet("node4") + + log.Info("test completed successfully", zap.String("test", t.Name())) +} + +func TestManagerRestoreClusterFromSnapshotEncryption(t *testing.T) { + c := NewTestCluster(t, 3).Setup().WriteCerts() + c.WithConfig(func(cfg *configv1alpha1.Configuration) { + cfg.SnapshotConfiguration.Compression = true + cfg.SnapshotConfiguration.Encryption = true + cfg.SnapshotConfiguration.File = "file://" + filepath.Join(c.dir, "snapshots") + cfg.CACert = filepath.Join(c.dir, "ca.crt") + cfg.CAKey = filepath.Join(c.dir, "ca.key") + }) + defer c.Cleanup() + + c.AddNodes("node1", "node2", "node3"). + Start().Wait(). + TestClientSet("node2") + c.Leader(). + SaveSnapshot() + c.Nodes("node1", "node2", "node3"). + Stop().Remove() + + // need to wait a bit to ensure the port is free to bind + time.Sleep(1 * time.Second) + + // SnapshotInterval is 0 so creating snapshots is disabled, however, + // SnapshotDir is being replaced with default SnapshotDir from node1 so + // that this new node can restore that snapshot + c.WithConfig(func(cfg *configv1alpha1.Configuration) { + cfg.SnapshotConfiguration.Encryption = false + }).AddNodes("node4", "node5", "node6"). + Start().Wait(). + TestClientGet("node4") + + log.Info("test completed successfully", zap.String("test", t.Name())) +} + +func TestManagerSingleNodeRestart(t *testing.T) { + c := NewTestCluster(t, 3).Setup() + defer c.Cleanup() + + c.AddNodes("node1", "node2", "node3"). + Start().Wait(). + TestClientSet("node1") + + // The important part for this test to work is that the cluster cannot + // remove node1, which is why we are not waiting for the node to be + // removed. The existing node is started again after being stopped so it + // should use the same data-dir. + c.Nodes("node1"). + Stop().Start().Wait(). + TestClientGet("node1") + + log.Info("test completed successfully", zap.String("test", t.Name())) +} + +func TestManagerNodeReplacementUsedPeerAddr(t *testing.T) { + c := NewTestCluster(t, 3).Setup() + defer c.Cleanup() + + c.AddNodes("node1", "node2", "node3"). + Start().Wait(). + TestClientSet("node1") + + // Impersonate node1 and replace it with node4 which happens to have the + // same inet. It's expected that node1 will be removed from the cluster + // during node4 join because it's not allowed to join a new node that uses + // an existing peerAddr (and having two nodes with the same peerAddr is + // impossible since the peerAddr must be a routable IP:port). + c.Nodes("node1"). + Stop().Remove().Wait() + c.WithConfig(func(cfg *configv1alpha1.Configuration) { + cfg.ClientAddr = c.Node("node1").Config().ClientAddr + cfg.PeerAddr = c.Node("node1").Config().PeerAddr + cfg.GossipAddr = c.Node("node1").Config().GossipAddr + }).AddNodes("node4"). + Start().Wait(). + TestClientGet("node4") +} + +func TestManagerSecurityConfig(t *testing.T) { + c := NewTestCluster(t, 3).Setup().WriteCerts() + c.WithConfig(func(cfg *configv1alpha1.Configuration) { + cfg.CACert = filepath.Join(c.dir, "ca.crt") + cfg.CAKey = filepath.Join(c.dir, "ca.key") + }) + defer c.Cleanup() + + c.AddNodes("node1", "node2", "node3"). + Start().Wait(). + TestClientSet("node1"). + TestClientGet("node2") + + log.Info("test completed successfully", zap.String("test", t.Name())) +} + +func TestManagerReadExistingName(t *testing.T) { + c := NewTestCluster(t, 1).Setup() + defer c.Cleanup() + + name := "node1" + + c.AddNodes(name). + Start().Wait(). + TestClientSet("node1"). + Stop() + c.AddNodes(name). + Start().Wait() + + if c.Node(name).Name() != name { + t.Fatalf("expected %#v, received %#v", name, c.Node(name).Etcd().Name) + } + + log.Info("test completed successfully", zap.String("test", t.Name())) +} + +func TestManagerDeleteVolatile(t *testing.T) { + c := NewTestCluster(t, 1).Setup() + c.WithConfig(func(cfg *configv1alpha1.Configuration) { + cfg.SnapshotConfiguration.File = "file://" + filepath.Join(c.dir, "snapshots") + }) + defer c.Cleanup() + + c.AddNodes("node1"). + Start().Wait() + cl := c.Node("node1").Client() + nkeys := 10 + for i := 0; i < nkeys; i++ { + if err := cl.Set(fmt.Sprintf("%s/%d", etcdserver.VolatilePrefix, i), "testvalue1"); err != nil { + t.Fatal(err) + } + } + n, err := cl.Count(string(etcdserver.VolatilePrefix)) + if err != nil { + t.Fatal(err) + } + cl.Close() + + // cluster-info is added by e2db which contains a key with the cluster-info + // itself, and another key for the e2db table schema + if n != int64(nkeys+2) { + t.Fatalf("expected %d keys, received %d", nkeys+2, n) + } + c.Nodes("node1"). + SaveSnapshot() + c.Nodes("node1").Stop().Remove() + + // need to wait a bit to ensure the port is free to bind + time.Sleep(1 * time.Second) + + // SnapshotInterval is 0 so creating snapshots is disabled, however, + // SnapshotDir is being replaced with default SnapshotDir from node1 so + // that this new node can restore that snapshot + c.WithConfig(func(cfg *configv1alpha1.Configuration) { + cfg.ClientAddr = c.Node("node1").Config().ClientAddr + cfg.PeerAddr = c.Node("node1").Config().PeerAddr + cfg.GossipAddr = c.Node("node1").Config().GossipAddr + }).AddNodes("node2"). + Start().Wait() + cl = c.Node("node2").Client() + + // There is a short race after etcdserver is ready and the manager places + // the snapshot marker. This will be fixed in the future, for now just make + // sure the marker gets placed before stopping the node. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if _, err := cl.MustGet(ctx, string(etcdserver.SnapshotMarkerKey)); err != nil { + t.Fatal(err) + } + + n, err = cl.Count(string(etcdserver.VolatilePrefix)) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Fatalf("after snapshot recover, only 1 key/value should remain, received %d", n) + } + cl.Close() + + log.Info("test completed successfully", zap.String("test", t.Name())) +} + +func TestManagerServerRestartCertRenewal(t *testing.T) { + c := NewTestCluster(t, 3).Setup().WriteCerts() + c.WithConfig(func(cfg *configv1alpha1.Configuration) { + cfg.CACert = filepath.Join(c.dir, "ca.crt") + cfg.CAKey = filepath.Join(c.dir, "ca.key") + }) + defer c.Cleanup() + + c.AddNodes("node1", "node2", "node3"). + Start().Wait(). + TestClientSet("node1") + + // replace certs on disk + c.WriteCerts() + c.All().Restart().Wait(). + TestClientGet("node2") + + log.Info("test completed successfully", zap.String("test", t.Name())) +} + +func TestManagerRollingUpdate(t *testing.T) { + c := NewTestCluster(t, 3).Setup() + defer c.Cleanup() + + c.AddNodes("node1", "node2", "node3"). + Start().Wait(). + TestClientSet("node1") + c.AddNodes("node4"). + Start().Wait() + c.Nodes("node1"). + Stop().Remove().Wait(). + TestClientGet("node4") + + log.Info("test completed successfully", zap.String("test", t.Name())) +} + +func TestManagerMetricsDisableAuth(t *testing.T) { + c := NewTestCluster(t, 1).Setup() + c.WithConfig(func(cfg *configv1alpha1.Configuration) { + cfg.MetricsConfiguration.Addr = ParseAddr(":4001") + cfg.MetricsConfiguration.DisableAuth = true + }) + defer c.Cleanup() + + c.AddNodes("node1"). + Start().Wait() + + resp, err := http.Get("http://127.0.0.1:4001/metrics") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + t.Fatalf("expected status code 200, received %d", resp.StatusCode) + } + + log.Info("test completed successfully", zap.String("test", t.Name())) +} diff --git a/e2e/node.go b/e2e/node.go new file mode 100644 index 0000000..ace03d1 --- /dev/null +++ b/e2e/node.go @@ -0,0 +1,169 @@ +//nolint +package e2e + +import ( + "context" + "net/url" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/criticalstack/e2d/pkg/client" + "github.com/criticalstack/e2d/pkg/log" + "github.com/criticalstack/e2d/pkg/manager" + snapshotutil "github.com/criticalstack/e2d/pkg/snapshot/util" +) + +type Node struct { + *manager.Manager + c *TestCluster + + removed bool + started bool +} + +func (n *Node) Client() *manager.Client { + scheme := "https" + if n.Config().CAKey == "" { + scheme = "http" + } + clientURL := url.URL{Scheme: scheme, Host: n.Config().ClientAddr.String()} + cfg := &client.Config{ + ClientURLs: []string{clientURL.String()}, + Timeout: 5 * time.Second, + } + if n.Config().CACert != "" { + dir := filepath.Dir(n.Config().CACert) + cfg.SecurityConfig = client.SecurityConfig{ + CertFile: filepath.Join(dir, "client.crt"), + KeyFile: filepath.Join(dir, "client.key"), + CertAuth: true, + TrustedCAFile: n.Config().CACert, + } + } + cc, err := client.New(cfg) + if err != nil { + panic(err) + } + return &manager.Client{ + Client: cc, + Timeout: 5 * time.Second, + } +} + +func (n *Node) Name() string { + return n.Etcd().Name +} + +func (n *Node) Remove() *Node { + if n.started { + n.c.t.Fatalf("cannot remove node %q, must be stopped first", n.Name()) + } + n.removed = true + return n +} + +func (n *Node) Start() *Node { + if n.removed || n.started { + return nil + } + log.Infof("starting node: %q\n", n.Name()) + n.started = true + + go func() { + if err := n.Run(); err != nil { + n.c.t.Fatalf("cannot start node %q: %v", n.Name(), err) + } + }() + return n +} + +func (n *Node) Stop() *Node { + if !n.started { + return n + } + log.Infof("stopping node: %q\n", n.Name()) + n.HardStop() + n.started = false + return n +} + +func (n *Node) Restart() *Node { + if !n.started { + return n + } + if err := n.Manager.Restart(); err != nil { + n.c.t.Fatalf("cannot restart node %q: %v", n.Name(), err) + } + return n +} + +func (n *Node) Wait() *Node { + switch { + case n.removed: + nodes := n.c.getStarted() + log.Infof("waiting for the node %#v to be removed from the following nodes: %v\n", n.Name(), nodes) + waiting := nodes + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + + var wg sync.WaitGroup + for _, name := range nodes { + wg.Add(1) + + go func(name string) { + defer wg.Done() + + for { + select { + case removedNode := <-n.c.Node(name).RemoveCh(): + if removedNode == n.Name() { + for i, name := range waiting { + if name == removedNode { + waiting = append(waiting[:i], waiting[i+1:]...) + } + } + return + } + case <-ctx.Done(): + n.c.t.Fatalf("timed out waiting for nodes to be removed: %q", strings.Join(waiting, ",")) + return + } + } + }(name) + } + wg.Wait() + case n.started: + log.Infof("waiting for the node %q to be started\n", n.Name()) + for { + if n.Etcd().IsRunning() { + log.Infof("node %q started successfully!\n", n.Name()) + return n + } + time.Sleep(100 * time.Millisecond) + } + } + return n +} + +func (n *Node) SaveSnapshot() *Node { + data, size, _, err := n.Etcd().CreateSnapshot(0) + if err != nil { + n.c.t.Fatal(err) + } + if n.Config().SnapshotConfiguration.Encryption { + key, err := manager.ReadEncryptionKey(n.Config().CAKey) + if err != nil { + n.c.t.Fatal(err) + } + data = snapshotutil.NewEncrypterReadCloser(data, &key, size) + } + if n.Config().SnapshotConfiguration.Compression { + data = snapshotutil.NewGzipReadCloser(data) + } + if err := n.Snapshotter().Save(data); err != nil { + n.c.t.Fatal(err) + } + return n +} diff --git a/e2e/testcluster.go b/e2e/testcluster.go new file mode 100644 index 0000000..cd51817 --- /dev/null +++ b/e2e/testcluster.go @@ -0,0 +1,223 @@ +//nolint +package e2e + +import ( + "context" + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/pkg/errors" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configv1alpha1 "github.com/criticalstack/e2d/pkg/config/v1alpha1" + "github.com/criticalstack/e2d/pkg/log" + "github.com/criticalstack/e2d/pkg/manager" + netutil "github.com/criticalstack/e2d/pkg/util/net" +) + +var ( + nextPort int32 = 2000 + + testKey1 = "testkey1" + testValue1 = "testvalue1" +) + +type TestCluster struct { + Name string + t *testing.T + nodes map[string]*Node + requiredClusterSize int + dir string + cfg *configv1alpha1.Configuration +} + +func NewTestCluster(t *testing.T, n int) *TestCluster { + t.Parallel() + h := sha256.Sum256([]byte(t.Name())) + cfg := &configv1alpha1.Configuration{ + RequiredClusterSize: n, + HealthCheckInterval: metav1.Duration{Duration: 1 * time.Second}, + // This setting is not set based upon a realistic time we expect + // failure. It is set to be the lowest timeout it can be here to + // test scenarios involving health check timeout while accounting + // for the worst case scenario in performance of the test runner. + // In other words, we don't want this to fail when we aren't + // testing something like the removal of dead nodes just because + // the system resources cause it to take too long to run the test. + HealthCheckTimeout: metav1.Duration{Duration: 15 * time.Second}, + EtcdLogLevel: "info", + MemberlistLogLevel: "debug", + } + return &TestCluster{ + Name: fmt.Sprintf("%x", h[:5]), + t: t, + nodes: make(map[string]*Node), + requiredClusterSize: n, + dir: filepath.Join("testdata", fmt.Sprintf("%x", h[:5])), + cfg: cfg, + } +} + +func (c *TestCluster) Setup() *TestCluster { + if err := os.RemoveAll(c.dir); err != nil { + c.t.Fatal(err) + } + return c +} + +func (c *TestCluster) WriteCerts() *TestCluster { + if err := manager.WriteNewCA(c.dir); err != nil { + c.t.Fatal(err) + } + hostIP, err := netutil.DetectHostIPv4() + if err != nil { + c.t.Fatal(err) + } + ca, err := manager.LoadCertificateAuthority(filepath.Join(c.dir, "ca.crt"), filepath.Join(c.dir, "ca.key"), hostIP) + if err != nil { + c.t.Fatal(err) + } + if err := ca.WriteAll(); err != nil { + c.t.Fatal(err) + } + return c +} + +func (c *TestCluster) Cleanup() { + for _, node := range c.nodes { + node.HardStop() + } +} + +func (c *TestCluster) Node(name string) *Node { + node, ok := c.nodes[name] + if !ok { + c.t.Fatalf("node not found: %#v", name) + } + return node +} + +func (c *TestCluster) Nodes(names ...string) *Context { + nodes := make([]*Node, 0) + for _, name := range names { + nodes = append(nodes, c.Node(name)) + } + return &Context{c: c, nodes: nodes} +} + +func (c *TestCluster) All() *Context { + nodes := make([]*Node, 0) + for _, node := range c.nodes { + nodes = append(nodes, node) + } + return &Context{c: c, nodes: nodes} +} + +func (c *TestCluster) Leader() *Node { + for _, node := range c.nodes { + if node.Etcd().IsLeader() { + return node + } + } + return nil +} + +func (c *TestCluster) Follower() *Node { + for _, node := range c.nodes { + if !node.Etcd().IsLeader() { + return node + } + } + return nil +} + +func (c *TestCluster) IsHealthy() error { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + for _, n := range c.nodes { + cl := n.Client() + if err := cl.IsHealthy(ctx); err != nil { + log.Debug("node failed health check", zap.String("name", n.Name()), zap.Error(err)) + continue + } + } + return nil + case <-ctx.Done(): + return errors.Errorf("cluster %q unhealthy", c.Name) + } + } +} + +func (c *TestCluster) getStarted() []string { + nodes := make([]string, 0) + for _, n := range c.nodes { + if n.started { + nodes = append(nodes, n.Etcd().Name) + } + } + return nodes +} + +func (c *TestCluster) WithConfig(fn func(*configv1alpha1.Configuration)) *TestCluster { + fn(c.cfg) + return c +} + +func (c *TestCluster) AddNodes(names ...string) *Context { + selected := make([]*Node, 0) + for i := 0; i < len(names); i++ { + cfg := c.cfg.DeepCopy() + cfg.OverrideName = names[i] + cfg.DataDir = filepath.Join(c.dir, cfg.OverrideName) + if cfg.ClientAddr.IsZero() { + cfg.ClientAddr.Port = atomic.AddInt32(&nextPort, 1) + } + if cfg.PeerAddr.IsZero() { + cfg.PeerAddr.Port = atomic.AddInt32(&nextPort, 1) + } + if cfg.GossipAddr.IsZero() { + cfg.GossipAddr.Host = "127.0.0.1" + cfg.GossipAddr.Port = atomic.AddInt32(&nextPort, 1) + } + // This is a placeholder and will be replaced below after all nodes + // have been added. It has to be here to ensure that the config + // validation doesn't error when constructing a new manager. + cfg.DiscoveryConfiguration.InitialPeers = []string{fmt.Sprintf(":%d", cfg.GossipAddr.Port)} + m, err := manager.New(cfg) + if err != nil { + c.t.Fatal(err) + } + n := &Node{Manager: m, c: c} + c.nodes[cfg.OverrideName] = n + selected = append(selected, n) + } + + // Here we ensure the initial peers are optimal when adding new nodes. This + // helps avoid a situation where the node joins the gossip network with + // only itself, and does not perform an initial push/pull, exchanging node + // status information. When this happens it takes a full push/pull interval + // (usually 30s) to exchange this information, slowing the test down. + nodes := make([]*Node, 0) + for _, n := range c.nodes { + if !n.removed { + nodes = append(nodes, n) + } + } + for i := range nodes { + nodes[i].Config().DiscoveryConfiguration.InitialPeers = []string{nodes[(i+1)%len(nodes)].Config().GossipAddr.String()} + } + return &Context{c: c, nodes: selected} +} diff --git a/e2e/utils.go b/e2e/utils.go new file mode 100644 index 0000000..c585f6b --- /dev/null +++ b/e2e/utils.go @@ -0,0 +1,21 @@ +//nolint +package e2e + +import ( + "encoding/json" + "strconv" + + configv1alpha1 "github.com/criticalstack/e2d/pkg/config/v1alpha1" +) + +func ParseAddr(s string) configv1alpha1.APIEndpoint { + s = strconv.Quote(s) + var v configv1alpha1.APIEndpoint + if err := json.Unmarshal([]byte(s), &v); err != nil { + panic(err) + } + return configv1alpha1.APIEndpoint{ + Host: v.Host, + Port: v.Port, + } +} diff --git a/go.mod b/go.mod index ff6ab60..9f6bb84 100644 --- a/go.mod +++ b/go.mod @@ -3,21 +3,21 @@ module github.com/criticalstack/e2d go 1.14 require ( - github.com/aws/aws-sdk-go v1.30.7 - github.com/cloudflare/cfssl v1.4.1 - github.com/coreos/go-semver v0.3.0 // indirect + github.com/aws/aws-sdk-go v1.34.23 + github.com/criticalstack/crit v1.0.1 github.com/digitalocean/go-metadata v0.0.0-20180111002115-15bd36e5f6f7 - github.com/digitalocean/godo v1.34.0 + github.com/digitalocean/godo v1.44.0 github.com/fatih/color v1.7.0 github.com/gogo/protobuf v1.3.1 github.com/google/go-cmp v0.5.0 - github.com/hashicorp/memberlist v0.2.0 + github.com/hashicorp/memberlist v0.2.2 github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.0.0 go.etcd.io/bbolt v1.3.5 - go.etcd.io/etcd v0.5.0-alpha.5.0.20200707173218-d3a702a09d92 - go.uber.org/zap v1.15.0 + go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b + go.uber.org/zap v1.16.0 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d - golang.org/x/text v0.3.3 // indirect google.golang.org/grpc v1.29.1 + k8s.io/apimachinery v0.18.5 + k8s.io/client-go v0.18.5 ) diff --git a/go.sum b/go.sum index 76a7129..028653e 100644 --- a/go.sum +++ b/go.sum @@ -1,24 +1,39 @@ bitbucket.org/liamstask/goose v0.0.0-20150115234039-8488cc47d90c/go.mod h1:hSVuE3qU7grINVSwrmzHfpg9k87ALBk+XaualNyUzI4= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alessio/shellescape v1.2.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/aws/aws-sdk-go v1.30.7 h1:IaXfqtioP6p9SFAnNfsqdNczbR5UNbYqvcZUSsCAdTY= github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.34.23 h1:ZUqMEJRjQUpZNA/OOhFmjWtWxD3n8XbOrC5rC1djl5s= +github.com/aws/aws-sdk-go v1.34.23/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -42,12 +57,16 @@ github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5t github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf h1:CAKfRE2YtTUIjjh1bkBtyYFaUT/WmOqsJjgtihT0vMI= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/criticalstack/crit v1.0.1 h1:nq9GiRVm7vOceni+qOyFvV3h+u937frVufp8tT9yh+s= +github.com/criticalstack/crit v1.0.1/go.mod h1:ULOKHjqXNtCGg4sH46kYxTBje0P9mgkpV8forMVxhGk= +github.com/criticalstack/e2d v0.4.14/go.mod h1:Bxbt5zWKhtA81n/YibGi8dlOdTVjNuBzy2zkbjJBf98= github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -59,25 +78,39 @@ github.com/digitalocean/go-metadata v0.0.0-20180111002115-15bd36e5f6f7 h1:ESwakc github.com/digitalocean/go-metadata v0.0.0-20180111002115-15bd36e5f6f7/go.mod h1:lNrzMwI4fx6xfzieyLEpYIJPLWjT/Sak4G/hIzGTEL4= github.com/digitalocean/godo v1.34.0 h1:OXJhLLJS2VTB5SziTyCq8valKVZ0uBHCFQsDW3/HF78= github.com/digitalocean/godo v1.34.0/go.mod h1:gfLm3JSupWD9V/ibQygXWW3IVz7hranzckH5UimhZsI= +github.com/digitalocean/godo v1.44.0 h1:IMElzMUpO1dVR8qjSg53+5vDkOLzMbhJt4yTAq7NGCQ= +github.com/digitalocean/godo v1.44.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.0.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/getsentry/raven-go v0.0.0-20180121060056-563b81fc02b7/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -90,12 +123,16 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -103,18 +140,30 @@ github.com/google/certificate-transparency-go v1.0.21 h1:Yf1aXowfZ2nuboBsg7iYGLm github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c h1:Lh2aW+HnU2Nbe1gqD9SOJLJxW1jBMmQOktN2acDyJk8= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -137,9 +186,15 @@ github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCS github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/memberlist v0.2.0 h1:WeeNspppWi5s1OFefTviPQueC/Bq8dONfvNjPhiEQKE= github.com/hashicorp/memberlist v0.2.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/memberlist v0.2.2 h1:5+RffWKwqJ71YPu9mWsF7ZOscZmwfasdA8kbdC7AO2g= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -153,6 +208,9 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -170,13 +228,24 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/go-gypsy v0.0.0-20160905020020-08cad365cd28/go.mod h1:T/T7jsxVqf9k/zYOqbgNAsANsjxTd1Yq3htjDhQ1H0c= +github.com/kyokomi/emoji v2.2.1+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA= +github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/lib/pq v0.0.0-20180201184707-88edab080323 h1:Ou506ViB5uo2GloKFWIYi5hwRJn4AAOXuLVv8RMY9+4= github.com/lib/pq v0.0.0-20180201184707-88edab080323/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -193,14 +262,24 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pires/go-proxyproto v0.1.3/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -217,6 +296,8 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= @@ -242,16 +323,20 @@ github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -270,6 +355,7 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/weppos/publicsuffix-go v0.4.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.5.0 h1:rutRtjBJViU/YjcI5d80t4JAVvDltS6bciJg2K1HrLU= github.com/weppos/publicsuffix-go v0.5.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= @@ -290,6 +376,9 @@ go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.5.0-alpha.5.0.20200707173218-d3a702a09d92 h1:9puP5UohLxJHfYVvSTIPzMeCdHGfE7qdED7zlj7poEc= go.etcd.io/etcd v0.5.0-alpha.5.0.20200707173218-d3a702a09d92/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b h1:3kC4J3eQF6p1UEfQTkC67eEeb3rTk+shQqdX6tFyq9Q= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -305,22 +394,31 @@ go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392 h1:ACG4HJsFiNMf47Y4PeRoebLNy/2lXT9EtprMuTFWt1M= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= +golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -328,54 +426,79 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -385,11 +508,17 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -406,14 +535,49 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= +k8s.io/api v0.18.5/go.mod h1:tN+e/2nbdGKOAH55NMV8oGrMG+3uRlA9GaRfvnCCSNk= +k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= +k8s.io/apimachinery v0.18.5 h1:Lh6tgsM9FMkC12K5T5QjRm7rDs6aQN5JHkA0JomULDM= +k8s.io/apimachinery v0.18.5/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= +k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU= +k8s.io/client-go v0.18.5 h1:cLhGZdOmyPhwtt20Lrb7uAqxxB1uvY+NTmNJvno1oKA= +k8s.io/client-go v0.18.5/go.mod h1:EsiD+7Fx+bRckKWZXnAXRKKetm1WuzPagH4iOSC8x58= +k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= +k8s.io/component-base v0.18.5/go.mod h1:RSbcboNk4B+S8Acs2JaBOVW3XNz1+A637s2jL+QQrlU= +k8s.io/cri-api v0.18.2/go.mod h1:OJtpjDvfsKoLGhvcc0qfygved0S0dGX56IJzPbqTG1s= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= +k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= +k8s.io/kube-proxy v0.18.2/go.mod h1:VTgyDMdylYGgHVqLQo/Nt4yDWkh/LRsSnxRiG8GVgDo= +k8s.io/kubelet v0.18.2/go.mod h1:7x/nzlIWJLg7vOfmbQ4lgsYazEB0gOhjiYiHK1Gii4M= +k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +sigs.k8s.io/kind v0.8.1/go.mod h1:oNKTxUVPYkV9lWzY6CVMNluVq8cBsyq+UgPJdvA3uu4= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt new file mode 100644 index 0000000..a1b404d --- /dev/null +++ b/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2020 Critical Stack, LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/hack/tools/go.mod b/hack/tools/go.mod index cf1b87b..566e43d 100644 --- a/hack/tools/go.mod +++ b/hack/tools/go.mod @@ -1,4 +1,4 @@ -module github.com/critical-stack/e2d/hack/tools +module github.com/criticalstack/e2d/hack/tools go 1.13 diff --git a/hack/tools/tools.go b/hack/tools/tools.go index 140f4be..2b98f22 100644 --- a/hack/tools/tools.go +++ b/hack/tools/tools.go @@ -6,6 +6,8 @@ package tools import ( _ "github.com/golangci/golangci-lint/cmd/golangi-lint" _ "k8s.io/code-generator/cmd/conversion-gen" + _ "k8s.io/code-generator/cmd/deepcopy-gen" + _ "k8s.io/code-generator/cmd/defaulter-gen" _ "sigs.k8s.io/controller-tools/cmd/controller-gen" _ "sigs.k8s.io/testing_frameworks/integration" ) diff --git a/hack/tools/update-codegen.sh b/hack/tools/update-codegen.sh new file mode 100755 index 0000000..959b592 --- /dev/null +++ b/hack/tools/update-codegen.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -o nounset +set -o errexit +set -o pipefail + +REPO_ROOT="${REPO_ROOT:-$(git rev-parse --show-toplevel)}" +TOOLS_DIR=${REPO_ROOT}/hack/tools +TOOLS_BIN=${TOOLS_DIR}/bin + +# build tools +cd "${TOOLS_DIR}" +go build -o "bin/defaulter-gen" k8s.io/code-generator/cmd/defaulter-gen +go build -o "bin/deepcopy-gen" k8s.io/code-generator/cmd/deepcopy-gen +go build -o "bin/conversion-gen" k8s.io/code-generator/cmd/conversion-gen +cd "${REPO_ROOT}" + +# run generators +"${TOOLS_BIN}/deepcopy-gen" -i ./pkg/config/v1alpha1 -o . -O zz_generated.deepcopy --go-header-file hack/boilerplate.go.txt +"${TOOLS_BIN}/defaulter-gen" -i ./pkg/config/v1alpha1 -o . -O zz_generated.default --go-header-file hack/boilerplate.go.txt +#"${TOOLS_BIN}/conversion-gen" -i ./pkg/config/v1alpha1 -o . -O zz_generated.conversion --go-header-file hack/boilerplate.go.txt + +# gofmt the tree +find . -name "*.go" -type f -print0 | xargs -0 gofmt -s -w diff --git a/pkg/buildinfo/version.go b/internal/buildinfo/version.go similarity index 100% rename from pkg/buildinfo/version.go rename to internal/buildinfo/version.go diff --git a/pkg/provider/aws/client.go b/internal/provider/aws/client.go similarity index 98% rename from pkg/provider/aws/client.go rename to internal/provider/aws/client.go index ec136ef..13870fc 100644 --- a/pkg/provider/aws/client.go +++ b/internal/provider/aws/client.go @@ -9,8 +9,9 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/criticalstack/e2d/pkg/netutil" "github.com/pkg/errors" + + netutil "github.com/criticalstack/e2d/pkg/util/net" ) type Client struct { diff --git a/pkg/provider/aws/config.go b/internal/provider/aws/config.go similarity index 99% rename from pkg/provider/aws/config.go rename to internal/provider/aws/config.go index 64876b6..c521015 100644 --- a/pkg/provider/aws/config.go +++ b/internal/provider/aws/config.go @@ -14,8 +14,9 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/sts" - "github.com/criticalstack/e2d/pkg/log" "github.com/pkg/errors" + + "github.com/criticalstack/e2d/pkg/log" ) func NewConfig() (*aws.Config, error) { diff --git a/pkg/provider/digitalocean/client.go b/internal/provider/digitalocean/client.go similarity index 90% rename from pkg/provider/digitalocean/client.go rename to internal/provider/digitalocean/client.go index 9f27476..262163e 100644 --- a/pkg/provider/digitalocean/client.go +++ b/internal/provider/digitalocean/client.go @@ -3,10 +3,11 @@ package digitalocean import ( "context" - "github.com/criticalstack/e2d/pkg/netutil" meta "github.com/digitalocean/go-metadata" "github.com/digitalocean/godo" "golang.org/x/oauth2" + + netutil "github.com/criticalstack/e2d/pkg/util/net" ) type Config struct { @@ -16,9 +17,7 @@ type Config struct { } func (cfg *Config) Token() (*oauth2.Token, error) { - return &oauth2.Token{ - AccessToken: cfg.AccessToken, - }, nil + return &oauth2.Token{AccessToken: cfg.AccessToken}, nil } type Client struct { diff --git a/pkg/client/config.go b/pkg/client/config.go index 7a99bb6..e4618d2 100644 --- a/pkg/client/config.go +++ b/pkg/client/config.go @@ -7,6 +7,25 @@ import ( "go.etcd.io/etcd/pkg/transport" ) +type Config struct { + ClientURLs []string + SecurityConfig SecurityConfig + Timeout time.Duration + + // NOTE: AutoSync sets client endpoints based upon the current members. + // This can cause the endpoints to become unreachable if the members are + // not directly accessible (e.g. a terminating load balancer). This is + // disabled by default and can be enabled by passed a non-zero duration. + AutoSyncInterval time.Duration +} + +func (c *Config) validate() error { + if c.Timeout == 0 { + c.Timeout = 2 * time.Second + } + return nil +} + type SecurityConfig struct { CertFile string KeyFile string @@ -34,22 +53,3 @@ func (sc SecurityConfig) TLSInfo() transport.TLSInfo { TrustedCAFile: sc.TrustedCAFile, } } - -type Config struct { - ClientURLs []string - SecurityConfig SecurityConfig - Timeout time.Duration - - // NOTE: AutoSync sets client endpoints based upon the current members. - // This can cause the endpoints to become unreachable if the members are - // not directly accessible (e.g. a terminating load balancer). This is - // disabled by default and can be enabled by passed a non-zero duration. - AutoSyncInterval time.Duration -} - -func (c *Config) validate() error { - if c.Timeout == 0 { - c.Timeout = 2 * time.Second - } - return nil -} diff --git a/pkg/config/v1alpha1/defaults.go b/pkg/config/v1alpha1/defaults.go new file mode 100644 index 0000000..a17354b --- /dev/null +++ b/pkg/config/v1alpha1/defaults.go @@ -0,0 +1,62 @@ +package v1alpha1 + +import ( + "time" + + "k8s.io/apimachinery/pkg/runtime" +) + +func addDefaultingFuncs(scheme *runtime.Scheme) error { + return RegisterDefaults(scheme) +} + +func SetDefaults_Configuration(obj *Configuration) { + if obj.DataDir == "" { + obj.DataDir = "data" + } + if obj.RequiredClusterSize == 0 { + obj.RequiredClusterSize = 1 + } + if obj.ClientAddr.Host == "" { + obj.ClientAddr.Host = "0.0.0.0" + } + if obj.ClientAddr.Port == 0 { + obj.ClientAddr.Port = DefaultClientPort + } + if obj.PeerAddr.Host == "" { + obj.PeerAddr.Host = "0.0.0.0" + } + if obj.PeerAddr.Port == 0 { + obj.PeerAddr.Port = DefaultPeerPort + } + if obj.GossipAddr.Host == "" { + obj.GossipAddr.Host = "0.0.0.0" + } + if obj.GossipAddr.Port == 0 { + obj.GossipAddr.Port = DefaultGossipPort + } + if obj.HealthCheckInterval.Duration == 0 { + obj.HealthCheckInterval.Duration = 1 * time.Minute + } + if obj.HealthCheckTimeout.Duration == 0 { + obj.HealthCheckTimeout.Duration = 5 * time.Minute + } + if obj.DiscoveryConfiguration.BootstrapTimeout.Duration == 0 { + obj.DiscoveryConfiguration.BootstrapTimeout.Duration = 30 * time.Minute + } + if obj.SnapshotConfiguration.Interval.Duration == 0 { + obj.SnapshotConfiguration.Interval.Duration = 1 * time.Minute + } + if obj.DiscoveryConfiguration.ExtraArgs == nil { + obj.DiscoveryConfiguration.ExtraArgs = make(map[string]string) + } + if obj.EtcdLogLevel == "" { + obj.EtcdLogLevel = "error" + } + if obj.MemberlistLogLevel == "" { + obj.MemberlistLogLevel = "error" + } + if obj.MetricsConfiguration.Type == "" { + obj.MetricsConfiguration.Type = MetricsBasic + } +} diff --git a/pkg/config/v1alpha1/doc.go b/pkg/config/v1alpha1/doc.go new file mode 100644 index 0000000..729fec8 --- /dev/null +++ b/pkg/config/v1alpha1/doc.go @@ -0,0 +1,5 @@ +// +k8s:deepcopy-gen=package +// +k8s:defaulter-gen=TypeMeta +// +groupName=e2d.crit.sh + +package v1alpha1 diff --git a/pkg/config/v1alpha1/register.go b/pkg/config/v1alpha1/register.go new file mode 100644 index 0000000..e81cf6c --- /dev/null +++ b/pkg/config/v1alpha1/register.go @@ -0,0 +1,38 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// GroupName is the group name used in this package. +const GroupName = "e2d.crit.sh" + +var ( + // SchemeGroupVersion is group version used to register these objects. + SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"} + // SchemeBuilder is a type to collect functions that add data to an API + // object through a scheme. + SchemeBuilder runtime.SchemeBuilder + localSchemeBuilder = &SchemeBuilder + // AddToScheme applies all the stored functions in the localSchemeBuilder + // to the scheme. + AddToScheme = localSchemeBuilder.AddToScheme +) + +func init() { + // We only register manually written functions here. The registration of the + // generated functions takes place in the generated files. The separation + // makes the code compile even when the generated files are missing. + localSchemeBuilder.Register(addKnownTypes, addDefaultingFuncs) +} + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &Configuration{}, + ) + + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/pkg/config/v1alpha1/types.go b/pkg/config/v1alpha1/types.go new file mode 100644 index 0000000..0aecbef --- /dev/null +++ b/pkg/config/v1alpha1/types.go @@ -0,0 +1,289 @@ +package v1alpha1 + +import ( + "encoding/json" + "fmt" + "net" + + fmtutil "github.com/criticalstack/crit/pkg/util/fmt" + "github.com/pkg/errors" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientsetscheme "k8s.io/client-go/kubernetes/scheme" + + "github.com/criticalstack/e2d/pkg/discovery" + "github.com/criticalstack/e2d/pkg/log" + netutil "github.com/criticalstack/e2d/pkg/util/net" +) + +const ( + DefaultClientPort = 2379 + DefaultPeerPort = 2380 + DefaultGossipPort = 7980 + DefaultMetricsPort = 4001 +) + +func init() { + _ = AddToScheme(clientsetscheme.Scheme) +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type Configuration struct { + metav1.TypeMeta `json:",inline"` + + // directory used for etcd data-dir, wal and snapshot dirs derived from + // this by etcd + DataDir string `json:"dataDir"` + + // the required number of nodes that must be present to start a cluster + RequiredClusterSize int `json:"requiredClusterSize"` + + // client endpoint for accessing etcd + ClientAddr APIEndpoint `json:"clientAddr"` + + // address used for traffic within the cluster + PeerAddr APIEndpoint `json:"peerAddr"` + + // address used for gossip network + GossipAddr APIEndpoint `json:"gossipAddr"` + + // CA certificate file location + // +optional + CACert string `json:"caCert"` + + // CA private key file location + // +optional + CAKey string `json:"caKey"` + + // how often to perform a health check + HealthCheckInterval metav1.Duration `json:"healthCheckInterval"` + + // time until an unreachable member is considered unhealthy + HealthCheckTimeout metav1.Duration `json:"healthCheckTimeout"` + + // DiscoveryConfiguration provides configuration for discovery of e2d + // peers. + DiscoveryConfiguration DiscoveryConfiguration `json:"discovery"` + // SnapshotConfiguration provides configuration for periodic snapshot + // backups of etcd. + SnapshotConfiguration SnapshotConfiguration `json:"snapshot"` + // MetricsConfiguration provides configuration for serving prometheus + // metrics. + MetricsConfiguration MetricsConfiguration `json:"metrics"` + + // name used for etcd.Embed instance, should generally be left alone so + // that a random name is generated + // +optional + OverrideName string `json:"overrideName"` + + // Default: error + // +optional + EtcdLogLevel string + + // Default: error + // +optional + MemberlistLogLevel string + + // +optional + CompactionInterval metav1.Duration `json:"compactionInterval"` + // UnsafeNoFsync disables all uses of fsync. Setting this is unsafe and + // will cause data loss. + // +optional + UnsafeNoFsync bool `json:"unsafeNoFsync"` + + // +optional + DisablePreVote bool `json:"DisablePreVote"` +} + +func (cfg *Configuration) Validate() error { + SetDefaults_Configuration(cfg) + + switch cfg.RequiredClusterSize { + case 1, 3, 5: + default: + return errors.New("value of RequiredClusterSize must be 1, 3, or 5") + } + for i, p := range cfg.DiscoveryConfiguration.InitialPeers { + addr, err := netutil.FixUnspecifiedHostAddr(p) + if err != nil { + return errors.Wrapf(err, "cannot determine ipv4 address from host string: %#v", p) + } + cfg.DiscoveryConfiguration.InitialPeers[i] = addr + } + if len(cfg.DiscoveryConfiguration.InitialPeers) == 0 && cfg.RequiredClusterSize > 1 { + return errors.New("must provide at least 1 initial peer when not a single-host cluster") + } + + if cfg.SnapshotConfiguration.Encryption && cfg.CAKey == "" { + return errors.New("must provide ca key for snapshot encryption") + } + host, err := netutil.DetectHostIPv4() + if err != nil { + return err + } + + if cfg.ClientAddr.IsUnspecified() { + cfg.ClientAddr.Host = host + } + if cfg.ClientAddr.Port == 0 { + cfg.ClientAddr.Port = DefaultClientPort + } + if cfg.PeerAddr.IsUnspecified() { + cfg.PeerAddr.Host = host + } + if cfg.PeerAddr.Port == 0 { + cfg.PeerAddr.Port = DefaultPeerPort + } + if cfg.GossipAddr.IsUnspecified() { + cfg.GossipAddr.Host = host + } + if cfg.GossipAddr.Port == 0 { + cfg.GossipAddr.Port = DefaultGossipPort + } + if !cfg.MetricsConfiguration.Addr.IsZero() { + if cfg.MetricsConfiguration.Addr.Host == "" { + cfg.MetricsConfiguration.Addr.Host = "0.0.0.0" + } + if cfg.MetricsConfiguration.Addr.IsUnspecified() { + cfg.MetricsConfiguration.Addr.Host = host + } + if cfg.MetricsConfiguration.Addr.Port == 0 { + cfg.MetricsConfiguration.Addr.Port = DefaultMetricsPort + } + } + + return nil +} + +type DiscoveryType string + +const ( + AmazonAutoscalingGroup DiscoveryType = "aws/autoscaling-group" + AmazonTags DiscoveryType = "aws/tags" + DigitalOceanTags DiscoveryType = "digitalocean/tags" +) + +type DiscoveryConfiguration struct { + // initial set of addresses used to bootstrap the gossip network + // +optional + InitialPeers []string `json:"initialPeers"` + + // amount of time to attempt bootstrapping before failing + BootstrapTimeout metav1.Duration `json:"bootstrapTimeout"` + + // +optional + Type DiscoveryType `json:"type"` + + // +optional + ExtraArgs map[string]string `json:"extraArgs"` +} + +func (d *DiscoveryConfiguration) Setup() (discovery.PeerGetter, error) { + kvs := make([]discovery.KeyValue, 0) + for k, v := range d.ExtraArgs { + kvs = append(kvs, discovery.KeyValue{Key: k, Value: v}) + } + switch d.Type { + case AmazonAutoscalingGroup: + return discovery.NewAmazonAutoScalingPeerGetter() + case AmazonTags: + return discovery.NewAmazonInstanceTagPeerGetter(kvs) + case DigitalOceanTags: + if len(kvs) == 0 { + return nil, errors.New("must provide at least 1 tag") + } + docfg := &discovery.DigitalOceanConfig{ + TagValue: kvs[0].Key, + } + return discovery.NewDigitalOceanPeerGetter(docfg) + //case "k8s-labels": + //return nil, errors.New("peer getter not yet implemented") + } + return &discovery.NoopGetter{}, nil +} + +type SnapshotConfiguration struct { + // use gzip compression for snapshot backup + // +optional + Compression bool `json:"compression"` + + // use aes-256 encryption for snapshot backup + // +optional + Encryption bool `json:"encryption"` + + // interval for creating etcd snapshots + Interval metav1.Duration `json:"interval"` + + // +optional + File string `json:"file"` + + // +optional + ExtraArgs map[string]string `json:"extraArgs"` +} + +// APIEndpoint represents a reachable endpoint using scheme-less host:port +// (port is optional). +type APIEndpoint struct { + // The IP or DNS name portion of an address. + Host string `json:"host"` + + // The port part of an address (optional). + Port int32 `json:"port"` +} + +// String returns a formatted version HOST:PORT of this APIEndpoint. +func (v APIEndpoint) String() string { + return fmt.Sprintf("%s:%d", v.Host, v.Port) +} + +// IsZero returns true if host and the port are zero values. +func (v APIEndpoint) IsZero() bool { + return v.Host == "" && v.Port == 0 +} + +func (v APIEndpoint) IsUnspecified() bool { + return net.ParseIP(v.Host).IsUnspecified() +} + +func (v *APIEndpoint) UnmarshalJSON(data []byte) error { + type alias APIEndpoint + if reterr := json.Unmarshal(data, (*alias)(v)); reterr != nil { + if err := v.tryUnmarshalText(fmtutil.Unquote(string(data))); err != nil { + log.Debug("tryUnmarshalText", zap.Error(err)) + return reterr + } + } + return nil +} + +func (v *APIEndpoint) tryUnmarshalText(s string) (err error) { + host, port, err := netutil.SplitHostPort(s) + v.Host = host + v.Port = int32(port) + return err +} + +type MetricsType string + +const ( + MetricsBasic MetricsType = "basic" + MetricsExtensive MetricsType = "extensive" +) + +type MetricsConfiguration struct { + // Addr is the addressed used to serve prometheus metrics. + // +optional + Addr APIEndpoint `json:"addr"` + // Type is used to specify the type of metrics to be served. It can + // currently be only "basic" or "extensive". + // Default: "basic + // +optional + Type MetricsType `json:"type"` + // DisableAuth disables the usage of TLS authentication for the metrics + // endpoint. By default, the usage of TLS is determined if CACert and CAKey + // were provided, and if so the client derived security is used. + // Default: false + // +optional + DisableAuth bool `json:"disableAuth"` +} diff --git a/pkg/config/v1alpha1/zz_generated.deepcopy.go b/pkg/config/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..ad4728a --- /dev/null +++ b/pkg/config/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,145 @@ +// +build !ignore_autogenerated + +/* +Copyright 2020 Critical Stack, LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *APIEndpoint) DeepCopyInto(out *APIEndpoint) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIEndpoint. +func (in *APIEndpoint) DeepCopy() *APIEndpoint { + if in == nil { + return nil + } + out := new(APIEndpoint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Configuration) DeepCopyInto(out *Configuration) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ClientAddr = in.ClientAddr + out.PeerAddr = in.PeerAddr + out.GossipAddr = in.GossipAddr + out.HealthCheckInterval = in.HealthCheckInterval + out.HealthCheckTimeout = in.HealthCheckTimeout + in.DiscoveryConfiguration.DeepCopyInto(&out.DiscoveryConfiguration) + in.SnapshotConfiguration.DeepCopyInto(&out.SnapshotConfiguration) + out.MetricsConfiguration = in.MetricsConfiguration + out.CompactionInterval = in.CompactionInterval + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Configuration. +func (in *Configuration) DeepCopy() *Configuration { + if in == nil { + return nil + } + out := new(Configuration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Configuration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DiscoveryConfiguration) DeepCopyInto(out *DiscoveryConfiguration) { + *out = *in + if in.InitialPeers != nil { + in, out := &in.InitialPeers, &out.InitialPeers + *out = make([]string, len(*in)) + copy(*out, *in) + } + out.BootstrapTimeout = in.BootstrapTimeout + if in.ExtraArgs != nil { + in, out := &in.ExtraArgs, &out.ExtraArgs + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DiscoveryConfiguration. +func (in *DiscoveryConfiguration) DeepCopy() *DiscoveryConfiguration { + if in == nil { + return nil + } + out := new(DiscoveryConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricsConfiguration) DeepCopyInto(out *MetricsConfiguration) { + *out = *in + out.Addr = in.Addr + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsConfiguration. +func (in *MetricsConfiguration) DeepCopy() *MetricsConfiguration { + if in == nil { + return nil + } + out := new(MetricsConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SnapshotConfiguration) DeepCopyInto(out *SnapshotConfiguration) { + *out = *in + out.Interval = in.Interval + if in.ExtraArgs != nil { + in, out := &in.ExtraArgs, &out.ExtraArgs + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SnapshotConfiguration. +func (in *SnapshotConfiguration) DeepCopy() *SnapshotConfiguration { + if in == nil { + return nil + } + out := new(SnapshotConfiguration) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/config/v1alpha1/zz_generated.default.go b/pkg/config/v1alpha1/zz_generated.default.go new file mode 100644 index 0000000..2020420 --- /dev/null +++ b/pkg/config/v1alpha1/zz_generated.default.go @@ -0,0 +1,37 @@ +// +build !ignore_autogenerated + +/* +Copyright 2020 Critical Stack, LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by defaulter-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + scheme.AddTypeDefaultingFunc(&Configuration{}, func(obj interface{}) { SetObjectDefaults_Configuration(obj.(*Configuration)) }) + return nil +} + +func SetObjectDefaults_Configuration(in *Configuration) { + SetDefaults_Configuration(in) +} diff --git a/pkg/discovery/discovery_aws.go b/pkg/discovery/discovery_aws.go index 40e2d25..606d8d7 100644 --- a/pkg/discovery/discovery_aws.go +++ b/pkg/discovery/discovery_aws.go @@ -3,8 +3,9 @@ package discovery import ( "context" - e2daws "github.com/criticalstack/e2d/pkg/provider/aws" "github.com/pkg/errors" + + e2daws "github.com/criticalstack/e2d/internal/provider/aws" ) type AmazonAutoScalingPeerGetter struct { diff --git a/pkg/discovery/discovery_do.go b/pkg/discovery/discovery_do.go index 960416e..bcad5ec 100644 --- a/pkg/discovery/discovery_do.go +++ b/pkg/discovery/discovery_do.go @@ -2,13 +2,13 @@ package discovery import ( "context" + "os" - "github.com/criticalstack/e2d/pkg/provider/digitalocean" + "github.com/criticalstack/e2d/internal/provider/digitalocean" ) type DigitalOceanConfig struct { - AccessToken string - TagValue string + TagValue string } type DigitalOceanPeerGetter struct { @@ -18,12 +18,14 @@ type DigitalOceanPeerGetter struct { func NewDigitalOceanPeerGetter(cfg *DigitalOceanConfig) (*DigitalOceanPeerGetter, error) { client, err := digitalocean.NewClient(&digitalocean.Config{ - AccessToken: cfg.AccessToken, + AccessToken: os.Getenv("ACCESS_TOKEN"), + SpacesAccessKey: os.Getenv("SPACES_ACCESS_KEY"), + SpacesSecretKey: os.Getenv("SPACES_SECRET_KEY"), }) if err != nil { return nil, err } - return &DigitalOceanPeerGetter{client, cfg}, nil + return &DigitalOceanPeerGetter{Client: client, cfg: cfg}, nil } func (p *DigitalOceanPeerGetter) GetAddrs(ctx context.Context) ([]string, error) { diff --git a/pkg/e2db/config.go b/pkg/e2db/config.go index dabe7c0..f65ce60 100644 --- a/pkg/e2db/config.go +++ b/pkg/e2db/config.go @@ -11,7 +11,7 @@ import ( "github.com/pkg/errors" "github.com/criticalstack/e2d/pkg/client" - "github.com/criticalstack/e2d/pkg/netutil" + netutil "github.com/criticalstack/e2d/pkg/util/net" ) type Config struct { diff --git a/pkg/e2db/db_test.go b/pkg/e2db/db_test.go index 31687c6..382a491 100644 --- a/pkg/e2db/db_test.go +++ b/pkg/e2db/db_test.go @@ -11,7 +11,10 @@ import ( "github.com/google/go-cmp/cmp" "github.com/pkg/errors" "go.uber.org/zap/zapcore" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/criticalstack/e2d/e2e" + configv1alpha1 "github.com/criticalstack/e2d/pkg/config/v1alpha1" "github.com/criticalstack/e2d/pkg/e2db" "github.com/criticalstack/e2d/pkg/log" "github.com/criticalstack/e2d/pkg/manager" @@ -26,16 +29,15 @@ func init() { log.Fatal(err) } - m, err := manager.New(&manager.Config{ - Name: "node1", - ClientAddr: ":2479", - PeerAddr: ":2480", - GossipAddr: ":7980", - Dir: filepath.Join("testdata", "node1"), + m, err := manager.New(&configv1alpha1.Configuration{ + OverrideName: "node1", + ClientAddr: e2e.ParseAddr(":2479"), + PeerAddr: e2e.ParseAddr(":2480"), + GossipAddr: e2e.ParseAddr(":7980"), + DataDir: filepath.Join("testdata", "node1"), RequiredClusterSize: 1, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 5 * time.Second, - EtcdLogLevel: zapcore.WarnLevel, + HealthCheckInterval: metav1.Duration{Duration: 1 * time.Second}, + HealthCheckTimeout: metav1.Duration{Duration: 5 * time.Second}, }) if err != nil { log.Fatal(err) diff --git a/pkg/etcdserver/members.go b/pkg/etcdserver/members.go new file mode 100644 index 0000000..af90107 --- /dev/null +++ b/pkg/etcdserver/members.go @@ -0,0 +1,47 @@ +package etcdserver + +import ( + "context" + "time" + + "github.com/pkg/errors" + "go.etcd.io/etcd/etcdserver/api/membership" +) + +var ( + ErrCannotFindMember = errors.New("cannot find member") +) + +func (s *Server) LookupMemberNameByPeerAddr(addr string) (string, error) { + for _, member := range s.Etcd.Server.Cluster().Members() { + for _, url := range member.PeerURLs { + if url == addr { + return member.Name, nil + } + } + } + return "", errors.Wrap(ErrCannotFindMember, addr) +} + +func (s *Server) RemoveMember(ctx context.Context, name string) error { + id, err := s.lookupMember(name) + if err != nil { + return err + } + cctx, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + _, err = s.Server.RemoveMember(cctx, id) + if err != nil && err != membership.ErrIDRemoved { + return errors.Errorf("cannot remove member %#v: %v", name, err) + } + return nil +} + +func (s *Server) lookupMember(name string) (uint64, error) { + for _, member := range s.Etcd.Server.Cluster().Members() { + if member.Name == name { + return uint64(member.ID), nil + } + } + return 0, errors.Wrap(ErrCannotFindMember, name) +} diff --git a/pkg/etcdserver/metadata.go b/pkg/etcdserver/metadata.go new file mode 100644 index 0000000..a753784 --- /dev/null +++ b/pkg/etcdserver/metadata.go @@ -0,0 +1,107 @@ +package etcdserver + +import ( + "bytes" + "context" + "time" + + "github.com/pkg/errors" + "go.etcd.io/etcd/lease" + "go.etcd.io/etcd/mvcc" + + "github.com/criticalstack/e2d/pkg/e2db" +) + +// move to manager? +type Cluster struct { + ID int `e2db:"id"` + Created time.Time + RequiredClusterSize int +} + +// writeClusterInfo attempts to write basic cluster info whenever a server +// starts or joins a new cluster. The e2db namespace matches the volatile +// prefix so that this information will not survive being restored from +// snapshot. This is because the cluster requirements could change for the +// restored cluster (e.g. going from RequiredClusterSize 1 -> 3). +func (s *Server) writeClusterInfo(ctx context.Context) error { + // NOTE(chrism): As the naming can be confusing it is worth pointing out + // that the ClientSecurity field is specifying the server certs and NOT the + // client certs. Since the server certs do not have client auth key usage, + // we need to use the peer certs here (they have client auth key usage). + db, err := e2db.New(ctx, &e2db.Config{ + ClientAddr: s.ClientURL.String(), + CAFile: s.PeerSecurity.TrustedCAFile, + CertFile: s.PeerSecurity.CertFile, + KeyFile: s.PeerSecurity.KeyFile, + Namespace: string(VolatilePrefix), + }) + if err != nil { + return err + } + defer db.Close() + + return db.Table(new(Cluster)).Tx(func(tx *e2db.Tx) error { + var cluster *Cluster + if err := tx.Find("ID", 1, &cluster); err != nil && errors.Cause(err) != e2db.ErrNoRows { + return err + } + + if cluster != nil { + // check RequiredClusterSize for discrepancies + if cluster.RequiredClusterSize != s.RequiredClusterSize { + return errors.Errorf("server %s attempted to join cluster with incorrect RequiredClusterSize, cluster expects %d, this server is configured with %d", s.Name, cluster.RequiredClusterSize, s.RequiredClusterSize) + } + return nil + } + + return tx.Insert(&Cluster{ + ID: 1, + Created: time.Now(), + RequiredClusterSize: s.RequiredClusterSize, + }) + }) +} + +var ( + // VolatilePrefix is the key prefix used for keys that will NOT be + // preserved after a cluster is recovered from snapshot + VolatilePrefix = []byte("/_e2d") + + // SnapshotMarkerKey is the key used to indicate when a cluster recovered + // from snapshot + SnapshotMarkerKey = []byte("/_e2d/snapshot") +) + +var ErrServerStopped = errors.New("server stopped") + +func (s *Server) ClearVolatilePrefix() (rev, deleted int64, err error) { + s.mu.Lock() + defer s.mu.Unlock() + + if !s.IsRunning() { + return 0, 0, ErrServerStopped + } + res, err := s.Server.KV().Range(VolatilePrefix, []byte{}, mvcc.RangeOptions{}) + if err != nil { + return 0, 0, err + } + for _, kv := range res.KVs { + if bytes.HasPrefix(kv.Key, VolatilePrefix) { + n, _ := s.Server.KV().DeleteRange(kv.Key, nil) + deleted += n + } + } + return res.Rev, deleted, nil +} + +func (s *Server) PlaceSnapshotMarker(v []byte) (int64, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if !s.IsRunning() { + return 0, ErrServerStopped + } + rev := s.Server.KV().Put(SnapshotMarkerKey, v, lease.NoLease) + return rev, nil +} diff --git a/pkg/etcdserver/peers.go b/pkg/etcdserver/peers.go new file mode 100644 index 0000000..10b0703 --- /dev/null +++ b/pkg/etcdserver/peers.go @@ -0,0 +1,57 @@ +package etcdserver + +import ( + "fmt" + "sort" + "strings" + + "github.com/pkg/errors" +) + +type Peer struct { + Name string + URL string +} + +func (p *Peer) String() string { + return fmt.Sprintf("%s=%s", p.Name, p.URL) +} + +func initialClusterStringFromPeers(peers []*Peer) string { + initialCluster := make([]string, 0) + for _, p := range peers { + initialCluster = append(initialCluster, fmt.Sprintf("%s=%s", p.Name, p.URL)) + } + if len(initialCluster) == 0 { + return "" + } + sort.Strings(initialCluster) + return strings.Join(initialCluster, ",") +} + +// validatePeers ensures that a group of peers are capable of starting, joining +// or recovering a cluster. It must be used whenever the initial cluster string +// will be built. +func validatePeers(peers []*Peer, requiredClusterSize int) error { + // When the name part of the initial cluster string is blank, etcd behaves + // abnormally. The same raft id is generated when providing the same + // connection information, so in cases where a member was removed from the + // cluster and replaced by a new member with the same address, having a + // blank name caused it to not be removed from the removed member tracking + // done by rafthttp. The member is "accepted" into the cluster, but cannot + // participate since the transport layer won't allow it. + for _, p := range peers { + if p.Name == "" || p.URL == "" { + return errors.Errorf("peer name/url cannot be blank: %+v", p) + } + } + + // The number of peers used to start etcd should always be the same as the + // cluster size. Otherwise, the etcd cluster can (very likely) fail to + // become healthy, therefore we go ahead and return early rather than deal + // with an invalid state. + if len(peers) < requiredClusterSize { + return errors.Errorf("expected %d members, but received %d: %v", requiredClusterSize, len(peers), peers) + } + return nil +} diff --git a/pkg/etcdserver/server.go b/pkg/etcdserver/server.go new file mode 100644 index 0000000..18079f1 --- /dev/null +++ b/pkg/etcdserver/server.go @@ -0,0 +1,242 @@ +package etcdserver + +import ( + "context" + "fmt" + "io/ioutil" + "net/url" + "os" + "sync" + "sync/atomic" + "time" + + "github.com/pkg/errors" + "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/embed" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "google.golang.org/grpc" + "google.golang.org/grpc/grpclog" + + "github.com/criticalstack/e2d/pkg/client" + "github.com/criticalstack/e2d/pkg/log" + netutil "github.com/criticalstack/e2d/pkg/util/net" +) + +type Server struct { + *embed.Etcd + + // name used for etcd.Embed instance, should generally be left alone so + // that a random name is generated + Name string + + // directory used for etcd data-dir, wal and snapshot dirs derived from + // this by etcd + DataDir string + + // client endpoint for accessing etcd + ClientURL url.URL + + // address used for traffic within the cluster + PeerURL url.URL + + // address serving prometheus metrics + MetricsURL *url.URL + + // the required number of nodes that must be present to start a cluster + RequiredClusterSize int + + // configures authentication/transport security for clients + ClientSecurity client.SecurityConfig + + // configures authentication/transport security within the etcd cluster + PeerSecurity client.SecurityConfig + + // add a local client listener (i.e. 127.0.0.1) + EnableLocalListener bool + + CompactionInterval time.Duration + + PreVote bool + + UnsafeNoFsync bool + + // configures the level of the logger used by etcd + LogLevel zapcore.Level + + ServiceRegister func(*grpc.Server) + + // used to determine if the instance of Etcd has already been started + started uint64 + // set when server is being restarted + restarting uint64 + + // mu is used to coordinate potentially unsafe access to etcd + mu sync.Mutex +} + +func (s *Server) IsRestarting() bool { + return atomic.LoadUint64(&s.restarting) == 1 +} + +func (s *Server) IsRunning() bool { + return atomic.LoadUint64(&s.started) == 1 +} + +func (s *Server) IsLeader() bool { + if !s.IsRunning() { + return false + } + return s.Etcd.Server.Leader() == s.Etcd.Server.ID() +} + +func (s *Server) Restart(ctx context.Context, peers []*Peer) error { + atomic.StoreUint64(&s.restarting, 1) + defer atomic.StoreUint64(&s.restarting, 0) + + s.HardStop() + return s.startEtcd(ctx, embed.ClusterStateFlagNew, peers) +} + +func (s *Server) HardStop() { + s.mu.Lock() + defer s.mu.Unlock() + + if s.Etcd != nil { + // This shuts down the etcdserver.Server instance without coordination + // with other members of the cluster. This ensures that a transfer of + // leadership is not attempted, which can cause an issue where a member + // can no longer join after a snapshot restore, should it fail during + // the attempted transfer of leadership. + s.Server.HardStop() + + // This must be called after HardStop since Close will start a graceful + // shutdown. + s.Etcd.Close() + } + atomic.StoreUint64(&s.started, 0) +} + +func (s *Server) GracefulStop() { + s.mu.Lock() + defer s.mu.Unlock() + + if s.Etcd != nil { + // There is no need to call Stop on the underlying etcdserver.Server + // since it is called in Close. + s.Etcd.Close() + } + atomic.StoreUint64(&s.started, 0) +} + +func (s *Server) StartNew(ctx context.Context, peers []*Peer) error { + return s.startEtcd(ctx, embed.ClusterStateFlagNew, peers) +} + +func (s *Server) JoinExisting(ctx context.Context, peers []*Peer) error { + return s.startEtcd(ctx, embed.ClusterStateFlagExisting, peers) +} + +func (s *Server) startEtcd(ctx context.Context, state string, peers []*Peer) error { + if err := validatePeers(peers, s.RequiredClusterSize); err != nil { + return err + } + + cfg := embed.NewConfig() + cfg.Name = s.Name + cfg.Dir = s.DataDir + if s.DataDir != "" { + cfg.Dir = s.DataDir + } + if err := os.MkdirAll(cfg.Dir, 0700); err != nil && !os.IsExist(err) { + return errors.Wrapf(err, "cannot create etcd data dir: %#v", cfg.Dir) + } + + // NOTE(chrism): etcd 3.4.9 introduced a check on the data directory + // permissions that require 0700. Since this causes the server to not come + // up we will attempt to change the perms. + log.Info("chmod data dir", zap.String("dir", s.DataDir)) + if err := os.Chmod(cfg.Dir, 0700); err != nil { + log.Error("chmod failed", zap.String("dir", s.DataDir), zap.Error(err)) + } + cfg.Logger = "zap" + if s.LogLevel == zapcore.DebugLevel { + cfg.Debug = true + } + cfg.ZapLoggerBuilder = func(c *embed.Config) error { + l := log.NewLoggerWithLevel("etcd", s.LogLevel) + return embed.NewZapCoreLoggerBuilder(l, l.Core(), zapcore.AddSync(os.Stderr))(c) + } + cfg.PreVote = s.PreVote + cfg.UnsafeNoFsync = s.UnsafeNoFsync + cfg.AutoCompactionMode = embed.CompactorModePeriodic + if s.CompactionInterval != 0 { + cfg.AutoCompactionRetention = s.CompactionInterval.String() + } + cfg.LPUrls = []url.URL{s.PeerURL} + cfg.APUrls = []url.URL{s.PeerURL} + cfg.LCUrls = []url.URL{s.ClientURL} + if s.EnableLocalListener { + _, port, _ := netutil.SplitHostPort(s.ClientURL.Host) + cfg.LCUrls = append(cfg.LCUrls, url.URL{Scheme: s.ClientSecurity.Scheme(), Host: fmt.Sprintf("127.0.0.1:%d", port)}) + } + cfg.ACUrls = []url.URL{s.ClientURL} + cfg.ClientAutoTLS = s.ClientSecurity.AutoTLS + cfg.PeerAutoTLS = s.PeerSecurity.AutoTLS + if s.ClientSecurity.Enabled() { + cfg.ClientTLSInfo = s.ClientSecurity.TLSInfo() + } + if s.PeerSecurity.Enabled() { + cfg.PeerTLSInfo = s.PeerSecurity.TLSInfo() + } + if s.MetricsURL != nil { + cfg.ListenMetricsUrls = []url.URL{*s.MetricsURL} + if s.EnableLocalListener { + _, port, _ := netutil.SplitHostPort(s.MetricsURL.Host) + cfg.ListenMetricsUrls = append(cfg.ListenMetricsUrls, url.URL{Scheme: s.MetricsURL.Scheme, Host: fmt.Sprintf("127.0.0.1:%d", port)}) + } + } + cfg.EnableV2 = false + cfg.ClusterState = state + cfg.InitialCluster = initialClusterStringFromPeers(peers) + cfg.StrictReconfigCheck = true + cfg.ServiceRegister = s.ServiceRegister + + // XXX(chris): not sure about this + clientv3.SetLogger(grpclog.NewLoggerV2(ioutil.Discard, ioutil.Discard, ioutil.Discard)) + + log.Info("starting etcd", + zap.String("name", cfg.Name), + zap.String("dir", s.DataDir), + zap.String("cluster-state", cfg.ClusterState), + zap.String("initial-cluster", cfg.InitialCluster), + zap.Int("required-cluster-size", s.RequiredClusterSize), + zap.Bool("debug", cfg.Debug), + ) + var err error + s.Etcd, err = embed.StartEtcd(cfg) + if err != nil { + return err + } + select { + case <-s.Server.ReadyNotify(): + if err := s.writeClusterInfo(ctx); err != nil { + return errors.Wrap(err, "cannot write cluster-info") + } + log.Debug("write cluster-info successful!") + atomic.StoreUint64(&s.started, 1) + log.Info("Server is ready!") + + go func() { + <-s.Server.StopNotify() + atomic.StoreUint64(&s.started, 0) + }() + return nil + case err := <-s.Err(): + return errors.Wrap(err, "etcd.Server.Start") + case <-ctx.Done(): + s.Server.Stop() + log.Info("Server was unable to start") + return ctx.Err() + } +} diff --git a/pkg/etcdserver/snapshot.go b/pkg/etcdserver/snapshot.go new file mode 100644 index 0000000..342687a --- /dev/null +++ b/pkg/etcdserver/snapshot.go @@ -0,0 +1,68 @@ +package etcdserver + +import ( + "io" + + "github.com/pkg/errors" + "go.etcd.io/etcd/clientv3/snapshot" + "go.etcd.io/etcd/embed" + "go.etcd.io/etcd/mvcc/backend" + + "github.com/criticalstack/e2d/pkg/log" +) + +func (s *Server) CreateSnapshot(minRevision int64) (io.ReadCloser, int64, int64, error) { + // Get the current revision and compare with the minimum requested revision. + revision := s.Etcd.Server.KV().Rev() + if revision <= minRevision { + return nil, 0, revision, errors.Errorf("member revision too old, wanted %d, received: %d", minRevision, revision) + } + sp := s.Etcd.Server.Backend().Snapshot() + if sp == nil { + return nil, 0, revision, errors.New("no snappy") + } + return newSnapshotReadCloser(sp), sp.Size(), revision, nil +} + +func newSnapshotReadCloser(snapshot backend.Snapshot) io.ReadCloser { + pr, pw := io.Pipe() + go func() { + n, err := snapshot.WriteTo(pw) + if err == nil { + log.Infof("wrote database snapshot out [total bytes: %d]", n) + } + _ = pw.CloseWithError(err) + snapshot.Close() + }() + return pr +} + +func (s *Server) RestoreSnapshot(snapshotFilename string, peers []*Peer) error { + if err := validatePeers(peers, s.RequiredClusterSize); err != nil { + return err + } + snapshotMgr := snapshot.NewV3(log.NewLoggerWithLevel("etcd", s.LogLevel)) + return snapshotMgr.Restore(snapshot.RestoreConfig{ + // SnapshotPath is the path of snapshot file to restore from. + SnapshotPath: snapshotFilename, + + // Name is the human-readable name of this member. + Name: s.Name, + + // OutputDataDir is the target data directory to save restored data. + // OutputDataDir should not conflict with existing etcd data directory. + // If OutputDataDir already exists, it will return an error to prevent + // unintended data directory overwrites. + // If empty, defaults to "[Name].etcd" if not given. + OutputDataDir: s.DataDir, + + // PeerURLs is a list of member's peer URLs to advertise to the rest of the cluster. + PeerURLs: []string{s.PeerURL.String()}, + + // InitialCluster is the initial cluster configuration for restore bootstrap. + InitialCluster: initialClusterStringFromPeers(peers), + + InitialClusterToken: embed.NewConfig().InitialClusterToken, + SkipHashCheck: true, + }) +} diff --git a/pkg/gossip/gossip.go b/pkg/gossip/gossip.go new file mode 100644 index 0000000..fc44b04 --- /dev/null +++ b/pkg/gossip/gossip.go @@ -0,0 +1,187 @@ +package gossip + +import ( + "context" + "fmt" + stdlog "log" + "strings" + "sync" + "time" + + "github.com/hashicorp/memberlist" + "github.com/pkg/errors" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + configv1alpha1 "github.com/criticalstack/e2d/pkg/config/v1alpha1" + "github.com/criticalstack/e2d/pkg/log" + netutil "github.com/criticalstack/e2d/pkg/util/net" +) + +type Config struct { + Name string + ClientURL string + PeerURL string + GossipHost string + GossipPort int + SecretKey []byte + LogLevel zapcore.Level +} + +type Gossip struct { + m memberlister + + config *memberlist.Config + events chan memberlist.NodeEvent + + broadcasts *memberlist.TransmitLimitedQueue + mu sync.RWMutex + nodes map[string]NodeStatus + self *Member +} + +func New(cfg *Config) *Gossip { + c := memberlist.DefaultLANConfig() + c.Name = cfg.Name + c.BindAddr = cfg.GossipHost + c.BindPort = cfg.GossipPort + c.Logger = stdlog.New(&logger{log.NewLoggerWithLevel("memberlist", cfg.LogLevel, zap.AddCallerSkip(2))}, "", 0) + c.SecretKey = cfg.SecretKey + + g := &Gossip{ + m: &noopMemberlist{}, + config: c, + events: make(chan memberlist.NodeEvent, 100), + nodes: make(map[string]NodeStatus), + self: &Member{ + Name: cfg.Name, + ClientURL: cfg.ClientURL, + PeerURL: cfg.PeerURL, + GossipAddr: fmt.Sprintf("%s:%d", cfg.GossipHost, cfg.GossipPort), + }, + } + g.broadcasts = &memberlist.TransmitLimitedQueue{ + NumNodes: func() int { + return g.m.NumMembers() + }, + RetransmitMult: 4, + } + c.Delegate = g + c.Events = &memberlist.ChannelEventDelegate{Ch: g.events} + return g +} + +func (g *Gossip) Shutdown() error { + if err := g.m.Shutdown(); err != nil { + return err + } + if g.config.Events != nil { + g.config.Events = nil + } + if g.events != nil { + close(g.events) + g.events = nil + } + return nil +} + +// Start attempts to join a gossip network using the given bootstrap addresses. +func (g *Gossip) Start(ctx context.Context, baddrs []string) error { + m, err := memberlist.Create(g.config) + if err != nil { + return err + } + g.m = m + + if err := g.Update(Unknown); err != nil { + return err + } + + peers := make([]string, 0) + for _, addr := range baddrs { + host, port, err := netutil.SplitHostPort(addr) + if err != nil { + return errors.Wrapf(err, "cannot split bootstrap address: %#v", addr) + } + if host == "" { + host = "127.0.0.1" + } + if port == 0 { + port = configv1alpha1.DefaultGossipPort + } + peers = append(peers, fmt.Sprintf("%s:%d", host, port)) + } + + log.Debug("attempting to join gossip network ...", + zap.String("bootstrap-addrs", strings.Join(peers, ",")), + ) + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + _, err := g.m.Join(peers) + if err != nil { + log.Errorf("cannot join gossip network: %v", err) + continue + } + log.Debug("joined gossip network successfully") + return nil + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// Events returns a read-only channel of memberlist events. +func (g *Gossip) Events() <-chan memberlist.NodeEvent { return g.events } + +// Members returns all members currently participating in the gossip network. +func (g *Gossip) Members() []*Member { + g.mu.RLock() + defer g.mu.RUnlock() + + members := make([]*Member, 0) + for _, m := range g.m.Members() { + // A member may be in the memberlist but Meta may be nil if the local + // metadata has yet to be propagated to this node. In this case, we + // ignore that member considering it to not ready. + if m.Meta == nil { + continue + } + meta := &Member{} + if err := meta.Unmarshal(m.Meta); err != nil { + log.Debugf("cannot unmarshal member: %v", err) + continue + } + + // status information shared via delegate is presumed to be more + // accurate + if status, ok := g.nodes[meta.Name]; ok { + meta.Status = status + } + members = append(members, meta) + } + return members +} + +func (g *Gossip) PendingMembers() []*Member { + members := make([]*Member, 0) + for _, member := range g.Members() { + if member.Status == Pending { + members = append(members, member) + } + } + return members +} + +func (g *Gossip) RunningMembers() []*Member { + members := make([]*Member, 0) + for _, member := range g.Members() { + if member.Status == Running { + members = append(members, member) + } + } + return members +} diff --git a/pkg/manager/gossip_test.go b/pkg/gossip/gossip_test.go similarity index 93% rename from pkg/manager/gossip_test.go rename to pkg/gossip/gossip_test.go index 4ad658b..6b762e4 100644 --- a/pkg/manager/gossip_test.go +++ b/pkg/gossip/gossip_test.go @@ -1,5 +1,5 @@ //nolint:errcheck -package manager +package gossip import ( "context" @@ -37,7 +37,7 @@ func TestMemberEncodeDecode(t *testing.T) { func TestGossipDelegate(t *testing.T) { t.Skip() - g1 := newGossip(&gossipConfig{ + g1 := New(&Config{ Name: "node1", GossipPort: 7980, }) @@ -47,7 +47,7 @@ func TestGossipDelegate(t *testing.T) { t.Fatal(err) } }() - g2 := newGossip(&gossipConfig{ + g2 := New(&Config{ Name: "node2", GossipPort: 7981, }) @@ -57,7 +57,7 @@ func TestGossipDelegate(t *testing.T) { t.Fatal(err) } }() - g3 := newGossip(&gossipConfig{ + g3 := New(&Config{ Name: "node3", GossipPort: 7982, }) diff --git a/pkg/gossip/logger.go b/pkg/gossip/logger.go new file mode 100644 index 0000000..73df11e --- /dev/null +++ b/pkg/gossip/logger.go @@ -0,0 +1,31 @@ +package gossip + +import ( + "strings" + + "go.uber.org/zap" +) + +type logger struct { + l *zap.Logger +} + +func (l *logger) Write(p []byte) (n int, err error) { + msg := string(p) + parts := strings.SplitN(msg, " ", 2) + lvl := "[DEBUG]" + if len(parts) > 1 { + lvl = parts[0] + msg = strings.TrimPrefix(parts[1], "memberlist: ") + } + + switch lvl { + case "[DEBUG]": + l.l.Debug(msg) + case "[WARN]": + l.l.Warn(msg) + case "[INFO]": + l.l.Info(msg) + } + return len(p), nil +} diff --git a/pkg/gossip/member.go b/pkg/gossip/member.go new file mode 100644 index 0000000..52c1003 --- /dev/null +++ b/pkg/gossip/member.go @@ -0,0 +1,82 @@ +package gossip + +import ( + "bytes" + "encoding/gob" + + "github.com/hashicorp/memberlist" +) + +type NodeStatus int + +const ( + Unknown NodeStatus = iota + Pending + Running +) + +func (s NodeStatus) String() string { + switch s { + case Unknown: + return "Unknown" + case Pending: + return "Pending" + case Running: + return "Running" + } + return "" +} + +type Member struct { + ID uint64 + Name string + ClientAddr string + ClientURL string + PeerAddr string + PeerURL string + GossipAddr string + BootstrapAddrs []string + Status NodeStatus +} + +func (m *Member) Marshal() ([]byte, error) { + var b bytes.Buffer + if err := gob.NewEncoder(&b).Encode(*m); err != nil { + return nil, err + } + return b.Bytes(), nil +} + +func (m *Member) Unmarshal(data []byte) error { + return gob.NewDecoder(bytes.NewReader(data)).Decode(m) +} + +type memberlister interface { + Join([]string) (int, error) + LocalNode() *memberlist.Node + Members() []*memberlist.Node + NumMembers() int + Shutdown() error +} + +type noopMemberlist struct{} + +func (noopMemberlist) Join([]string) (int, error) { + return 0, nil +} + +func (noopMemberlist) LocalNode() *memberlist.Node { + return &memberlist.Node{} +} + +func (noopMemberlist) Members() []*memberlist.Node { + return nil +} + +func (noopMemberlist) NumMembers() int { + return 0 +} + +func (noopMemberlist) Shutdown() error { + return nil +} diff --git a/pkg/gossip/messages.go b/pkg/gossip/messages.go new file mode 100644 index 0000000..3f0fa8e --- /dev/null +++ b/pkg/gossip/messages.go @@ -0,0 +1,101 @@ +package gossip + +import ( + "bytes" + "encoding/gob" + + "github.com/hashicorp/memberlist" + "go.uber.org/zap" + + "github.com/criticalstack/e2d/pkg/log" +) + +// Update uses the provided NodeStatus to updates the node metadata and +// broadcast the updated NodeStatus to all currently known members. +func (g *Gossip) Update(status NodeStatus) error { + log.Debug("attempting to update node status", + zap.String("name", g.self.Name), + zap.Stringer("status", g.nodes[g.self.Name]), + zap.Stringer("update-status", status), + ) + g.mu.Lock() + g.nodes[g.self.Name] = status + g.self.Status = status + g.mu.Unlock() + data, err := g.self.Marshal() + if err != nil { + return err + } + g.m.LocalNode().Meta = data + var b bytes.Buffer + if err := gob.NewEncoder(&b).Encode(statusMsg{Name: g.self.Name, Status: status}); err != nil { + return err + } + g.broadcasts.QueueBroadcast(&msg{b.Bytes()}) + return nil +} + +// msg implements the memberlist.Broadcast interface and is required to send +// messages over the gossip network +type msg struct { + data []byte +} + +func (m *msg) Invalidates(other memberlist.Broadcast) bool { return false } +func (m *msg) Message() []byte { return m.data } +func (m *msg) Finished() {} + +type statusMsg struct { + Name string + Status NodeStatus +} + +func (g *Gossip) NodeMeta(limit int) []byte { return g.m.LocalNode().Meta } + +func (g *Gossip) NotifyMsg(data []byte) { + if len(data) == 0 { + return + } + var n statusMsg + if err := gob.NewDecoder(bytes.NewReader(data)).Decode(&n); err != nil { + log.Debugf("cannot unmarshal: %v", err) + return + } + log.Debug("received status update", + zap.String("name", g.self.Name), + zap.String("peer", n.Name), + zap.Stringer("peer-status", n.Status), + ) + g.mu.Lock() + g.nodes[n.Name] = n.Status + g.mu.Unlock() +} + +func (g *Gossip) GetBroadcasts(overhead, limit int) [][]byte { + return g.broadcasts.GetBroadcasts(overhead, limit) +} + +func (g *Gossip) LocalState(join bool) []byte { + var b bytes.Buffer + if err := gob.NewEncoder(&b).Encode(statusMsg{Name: g.self.Name, Status: g.self.Status}); err != nil { + log.Error("cannot send gossip local state", zap.Error(err)) + return nil + } + return b.Bytes() +} + +func (g *Gossip) MergeRemoteState(buf []byte, join bool) { + if len(buf) == 0 { + return + } + var n statusMsg + if err := gob.NewDecoder(bytes.NewReader(buf)).Decode(&n); err != nil { + log.Error("cannot merge gossip remote state", zap.Error(err)) + return + } + if g.nodes[n.Name] != n.Status { + g.mu.Lock() + g.nodes[n.Name] = n.Status + g.mu.Unlock() + } +} diff --git a/pkg/log/log.go b/pkg/log/log.go index 4772180..d2587bd 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -19,31 +19,37 @@ var ( ) // NewLogger creates a new child logger with the provided namespace. -func NewLogger(ns string) *zap.Logger { +func NewLogger(ns string, opts ...zap.Option) *zap.Logger { encoder := NewEncoder(NewDefaultEncoderConfig()) encoder.OpenNamespace(ns) - return log.WithOptions(zap.WrapCore(func(c zapcore.Core) zapcore.Core { + opts = append(opts, zap.WrapCore(func(c zapcore.Core) zapcore.Core { return zapcore.NewCore( encoder, zapcore.AddSync(os.Stderr), level, ) - }), zap.AddCaller()) + })) + return log.WithOptions(opts...) } -// NewLogger creates a new child logger with the provided namespace and level. -// Since this specifies a level, it overrides the global package level for this -// child logger only. -func NewLoggerWithLevel(ns string, lvl zapcore.Level) *zap.Logger { +// NewLoggerWithLevel creates a new child logger with the provided namespace +// and level. Since this specifies a level, it overrides the global package +// level for this child logger only. +func NewLoggerWithLevel(ns string, lvl zapcore.Level, opts ...zap.Option) *zap.Logger { encoder := NewEncoder(NewDefaultEncoderConfig()) encoder.OpenNamespace(ns) - return log.WithOptions(zap.WrapCore(func(c zapcore.Core) zapcore.Core { + opts = append(opts, zap.WrapCore(func(c zapcore.Core) zapcore.Core { return zapcore.NewCore( encoder, zapcore.AddSync(os.Stderr), lvl, ) })) + return log.WithOptions(opts...) +} + +func Level() zapcore.Level { + return level.Level() } func SetLevel(lvl zapcore.Level) { diff --git a/pkg/manager/certs.go b/pkg/manager/certs.go new file mode 100644 index 0000000..f33a44b --- /dev/null +++ b/pkg/manager/certs.go @@ -0,0 +1,168 @@ +package manager + +import ( + "bytes" + "crypto/sha512" + "crypto/x509" + "encoding/pem" + "io" + "io/ioutil" + "net" + "path/filepath" + + "github.com/criticalstack/crit/pkg/kubernetes/pki" + "github.com/pkg/errors" + + netutil "github.com/criticalstack/e2d/pkg/util/net" +) + +func WriteNewCA(dir string) error { + ca, err := pki.NewCertificateAuthority("ca", &pki.Config{ + CommonName: "etcd", + }) + if err != nil { + return err + } + return ca.WriteFiles(dir) +} + +type CertificateAuthority struct { + *pki.CertificateAuthority + + dir string + ips []net.IP + dnsnames []string +} + +func LoadCertificateAuthority(cert, key string, names ...string) (*CertificateAuthority, error) { + caCert, err := pki.ReadCertFromFile(cert) + if err != nil { + return nil, err + } + caKey, err := pki.ReadKeyFromFile(key) + if err != nil { + return nil, err + } + ca := &CertificateAuthority{ + CertificateAuthority: &pki.CertificateAuthority{ + KeyPair: &pki.KeyPair{ + Name: "ca", + Cert: caCert, + Key: caKey, + }, + }, + dir: filepath.Dir(cert), + } + if !contains(names, "127.0.0.1") { + names = append(names, "127.0.0.1") + } + hostIP, err := netutil.DetectHostIPv4() + if err != nil { + return nil, err + } + if !contains(names, hostIP) { + names = append(names, hostIP) + } + for _, name := range names { + if ip := net.ParseIP(name); ip != nil { + ca.ips = append(ca.ips, ip) + continue + } + ca.dnsnames = append(ca.dnsnames, name) + } + return ca, nil +} + +func (ca *CertificateAuthority) WriteAll() error { + fns := []func() error{ + ca.WriteServerCertAndKey, + ca.WritePeerCertAndKey, + ca.WriteClientCertAndKey, + } + for _, fn := range fns { + if err := fn(); err != nil { + return err + } + } + return nil +} + +func (ca *CertificateAuthority) WriteCA() error { + return ca.WriteFiles(ca.dir) +} + +func (ca *CertificateAuthority) WriteServerCertAndKey() error { + server, err := ca.NewSignedKeyPair("server", &pki.Config{ + CommonName: "etcd-server", + Usages: []x509.ExtKeyUsage{ + x509.ExtKeyUsageServerAuth, + }, + AltNames: pki.AltNames{ + DNSNames: ca.dnsnames, + IPs: ca.ips, + }, + }) + if err != nil { + return err + } + return server.WriteFiles(ca.dir) +} + +func (ca *CertificateAuthority) WritePeerCertAndKey() error { + peer, err := ca.NewSignedKeyPair("peer", &pki.Config{ + CommonName: "etcd-peer", + Usages: []x509.ExtKeyUsage{ + x509.ExtKeyUsageClientAuth, + x509.ExtKeyUsageServerAuth, + }, + AltNames: pki.AltNames{ + DNSNames: ca.dnsnames, + IPs: ca.ips, + }, + }) + if err != nil { + return err + } + return peer.WriteFiles(ca.dir) +} + +func (ca *CertificateAuthority) WriteClientCertAndKey() error { + client, err := ca.NewSignedKeyPair("client", &pki.Config{ + CommonName: "etcd-client", + Usages: []x509.ExtKeyUsage{ + x509.ExtKeyUsageClientAuth, + }, + }) + if err != nil { + return err + } + return client.WriteFiles(ca.dir) +} + +func contains(ss []string, match string) bool { + for _, s := range ss { + if s == match { + return true + } + } + return false +} + +func ReadEncryptionKey(caKey string) (key [32]byte, err error) { + data, err := ioutil.ReadFile(caKey) + if err != nil { + return key, err + } + block, _ := pem.Decode(data) + if _, err := x509.ParsePKCS1PrivateKey(block.Bytes); err != nil { + return key, errors.Wrapf(err, "cannot parse ca key file: %#v", caKey) + } + h := sha512.New512_256() + if _, err := h.Write(block.Bytes); err != nil { + return key, err + } + if _, err := io.ReadFull(bytes.NewReader(h.Sum(nil)), key[:]); err != nil { + return key, err + } + return key, nil +} diff --git a/pkg/manager/client.go b/pkg/manager/client.go index 21de35b..0de4209 100644 --- a/pkg/manager/client.go +++ b/pkg/manager/client.go @@ -8,24 +8,17 @@ import ( "go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes" "github.com/criticalstack/e2d/pkg/client" + "github.com/criticalstack/e2d/pkg/gossip" ) type Client struct { *client.Client - cfg *client.Config + Timeout time.Duration } -func newClient(cfg *client.Config) (*Client, error) { - c, err := client.New(cfg) - if err != nil { - return nil, err - } - return &Client{c, cfg}, nil -} - -func (c *Client) members(ctx context.Context) (map[string]*Member, error) { - ctx, cancel := context.WithTimeout(ctx, c.cfg.Timeout) +func (c *Client) members(ctx context.Context) (map[string]*gossip.Member, error) { + ctx, cancel := context.WithTimeout(ctx, c.Timeout) defer cancel() resp, err := c.MemberList(ctx) @@ -33,9 +26,9 @@ func (c *Client) members(ctx context.Context) (map[string]*Member, error) { return nil, err } - members := make(map[string]*Member) + members := make(map[string]*gossip.Member) for _, member := range resp.Members { - m := &Member{ + m := &gossip.Member{ ID: member.ID, Name: member.Name, } @@ -50,15 +43,15 @@ func (c *Client) members(ctx context.Context) (map[string]*Member, error) { return members, nil } -func (c *Client) addMember(ctx context.Context, peerURL string) (*Member, error) { - ctx, cancel := context.WithTimeout(ctx, c.cfg.Timeout) +func (c *Client) addMember(ctx context.Context, peerURL string) (*gossip.Member, error) { + ctx, cancel := context.WithTimeout(ctx, c.Timeout) defer cancel() resp, err := c.MemberAdd(ctx, []string{peerURL}) if err != nil { return nil, err } - m := &Member{ + m := &gossip.Member{ ID: resp.Member.ID, Name: resp.Member.Name, } @@ -72,7 +65,7 @@ func (c *Client) addMember(ctx context.Context, peerURL string) (*Member, error) } func (c *Client) removeMember(ctx context.Context, id uint64) error { - ctx, cancel := context.WithTimeout(ctx, c.cfg.Timeout) + ctx, cancel := context.WithTimeout(ctx, c.Timeout) defer cancel() if _, err := c.MemberRemove(ctx, id); err != nil && err != rpctypes.ErrMemberNotFound { @@ -81,14 +74,14 @@ func (c *Client) removeMember(ctx context.Context, id uint64) error { return nil } -func (c *Client) removeMemberLocked(ctx context.Context, member *Member) error { +func (c *Client) removeMemberLocked(ctx context.Context, member *gossip.Member) error { unlock, err := c.Lock(member.Name, 10*time.Second) if err != nil { return err } defer unlock() - ctx, cancel := context.WithTimeout(ctx, c.cfg.Timeout) + ctx, cancel := context.WithTimeout(ctx, c.Timeout) defer cancel() return c.removeMember(ctx, member.ID) diff --git a/pkg/manager/config.go b/pkg/manager/config.go deleted file mode 100644 index b432adf..0000000 --- a/pkg/manager/config.go +++ /dev/null @@ -1,284 +0,0 @@ -//nolint:maligned -package manager - -import ( - "bytes" - "crypto/sha512" - "crypto/x509" - "encoding/json" - "encoding/pem" - "fmt" - "io" - "io/ioutil" - "math/rand" - "net/url" - "path/filepath" - "strings" - "time" - - "github.com/criticalstack/e2d/pkg/client" - "github.com/criticalstack/e2d/pkg/discovery" - "github.com/criticalstack/e2d/pkg/log" - "github.com/criticalstack/e2d/pkg/netutil" - "github.com/criticalstack/e2d/pkg/snapshot" - "github.com/pkg/errors" - bolt "go.etcd.io/bbolt" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -func init() { - rand.Seed(time.Now().UnixNano()) -} - -type Config struct { - // name used for etcd.Embed instance, should generally be left alone so - // that a random name is generated - Name string - - // directory used for etcd data-dir, wal and snapshot dirs derived from - // this by etcd - Dir string - - // the required number of nodes that must be present to start a cluster - RequiredClusterSize int - - // allows for explicit setting of the host ip - Host string - - // client endpoint for accessing etcd - ClientAddr string - - // client url created based upon the client address and use of TLS - ClientURL url.URL - - // address used for traffic within the cluster - PeerAddr string - - // peer url created based upon the peer address and use of TLS - PeerURL url.URL - - // address used for gossip network - GossipAddr string - - // host used for gossip network, derived from GossipAddr - GossipHost string - - // port used for gossip network, derived from GossipAddr - GossipPort int - - // addresses used to bootstrap the gossip network - BootstrapAddrs []string - - // amount of time to attempt bootstrapping before failing - BootstrapTimeout time.Duration - - // interval for creating etcd snapshots - SnapshotInterval time.Duration - - // use gzip compression for snapshot backup - SnapshotCompression bool - - // use aes-256 encryption for snapshot backup - SnapshotEncryption bool - - // how often to perform a health check - HealthCheckInterval time.Duration - - // time until an unreachable member is considered unhealthy - HealthCheckTimeout time.Duration - - // configures authentication/transport security for clients - ClientSecurity client.SecurityConfig - - // configures authentication/transport security within the etcd cluster - PeerSecurity client.SecurityConfig - - CACertFile string - CAKeyFile string - - // configures the level of the logger used by etcd - EtcdLogLevel zapcore.Level - - discovery.PeerGetter - snapshot.Snapshotter - - gossipSecretKey []byte - snapshotEncryptionKey *[32]byte - - Debug bool -} - -//nolint:gocyclo -func (c *Config) validate() error { - if c.Dir == "" { - c.Dir = "data" - } - if c.SnapshotInterval == 0 { - c.SnapshotInterval = 1 * time.Minute - } - if c.HealthCheckInterval == 0 { - c.HealthCheckInterval = 1 * time.Minute - } - if c.HealthCheckTimeout == 0 { - c.HealthCheckTimeout = 5 * time.Minute - } - if c.BootstrapTimeout == 0 { - c.BootstrapTimeout = 30 * time.Minute - } - for i, baddr := range c.BootstrapAddrs { - addr, err := netutil.FixUnspecifiedHostAddr(baddr) - if err != nil { - return errors.Wrapf(err, "cannot determine ipv4 address from host string: %#v", baddr) - } - c.BootstrapAddrs[i] = addr - } - - // If the host is not set the IPv4 of the first non-loopback network - // adapter is used. This value is only used when the host is unspecified in - // an address. - if c.Host == "" { - var err error - c.Host, err = netutil.DetectHostIPv4() - if err != nil { - return err - } - } - - // parse etcd client address - caddr, err := netutil.ParseAddr(c.ClientAddr) - if err != nil { - return err - } - if caddr.IsUnspecified() { - caddr.Host = c.Host - } - if caddr.Port == 0 { - caddr.Port = 2379 - } - c.ClientAddr = caddr.String() - c.ClientURL = url.URL{Scheme: c.ClientSecurity.Scheme(), Host: c.ClientAddr} - - // parse etcd peer address - paddr, err := netutil.ParseAddr(c.PeerAddr) - if err != nil { - return err - } - if paddr.IsUnspecified() { - paddr.Host = c.Host - } - if paddr.Port == 0 { - paddr.Port = 2380 - } - c.PeerAddr = paddr.String() - c.PeerURL = url.URL{Scheme: c.PeerSecurity.Scheme(), Host: c.PeerAddr} - - // parse gossip address - gaddr, err := netutil.ParseAddr(c.GossipAddr) - if err != nil { - return err - } - if gaddr.IsUnspecified() { - gaddr.Host = c.Host - } - if gaddr.Port == 0 { - gaddr.Port = DefaultGossipPort - } - c.GossipAddr = gaddr.String() - c.GossipHost, c.GossipPort, err = netutil.SplitHostPort(c.GossipAddr) - if err != nil { - return errors.Wrapf(err, "cannot split GossipAddr: %#v", c.GossipAddr) - } - - // both memberlist security and snapshot encryption are implicitly based - // upon the CA key - if c.CAKeyFile != "" { - data, err := ioutil.ReadFile(c.CAKeyFile) - if err != nil { - return err - } - block, _ := pem.Decode(data) - if _, err := x509.ParsePKCS1PrivateKey(block.Bytes); err != nil { - return errors.Wrapf(err, "cannot parse ca key file: %#v", c.CAKeyFile) - } - h := sha512.New512_256() - if _, err := h.Write(block.Bytes); err != nil { - return err - } - key := [32]byte{} - if _, err := io.ReadFull(bytes.NewReader(h.Sum(nil)), key[:]); err != nil { - return err - } - c.gossipSecretKey = key[:] - c.snapshotEncryptionKey = &key - } - - if c.SnapshotEncryption && c.CAKeyFile == "" { - return errors.New("must provide ca key for snapshot encryption") - } - - if len(c.BootstrapAddrs) == 0 && c.RequiredClusterSize > 1 { - return errors.New("must provide at least 1 BootstrapAddrs when not a single-host cluster") - } - switch c.RequiredClusterSize { - case 0: - c.RequiredClusterSize = 1 - case 1, 3, 5: - default: - return errors.New("value of RequiredClusterSize must be 1, 3, or 5") - } - if c.Name == "" { - if name, err := getExistingNameFromDataDir(filepath.Join(c.Dir, "member/snap/db"), c.PeerURL); err == nil { - log.Debugf("reusing name from existing data-dir: %v", name) - c.Name = name - } else { - log.Debug("cannot read existing data-dir", zap.Error(err)) - c.Name = fmt.Sprintf("%X", rand.Uint64()) - } - } - return nil -} - -// shortName returns a shorter, lowercase version of the node name. The intent -// is to make log reading easier. -func shortName(name string) string { - if len(name) > 5 { - name = name[:5] - } - return strings.ToLower(name) -} - -func getExistingNameFromDataDir(path string, peerURL url.URL) (string, error) { - db, err := bolt.Open(path, 0600, &bolt.Options{Timeout: 1 * time.Second}) - if err != nil { - return "", err - } - defer db.Close() - var name string - err = db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("members")) - if b == nil { - return errors.New("existing name not found") - } - c := b.Cursor() - for k, v := c.First(); k != nil; k, v = c.Next() { - var m struct { - ID uint64 `json:"id"` - Name string `json:"name"` - PeerURLs []string `json:"peerURLs"` - } - if err := json.Unmarshal(v, &m); err != nil { - log.Error("cannot unmarshal etcd member", zap.Error(err)) - continue - } - for _, u := range m.PeerURLs { - if u == peerURL.String() { - name = m.Name - return nil - } - } - } - return errors.New("existing name not found") - }) - return name, err -} diff --git a/pkg/manager/config_test.go b/pkg/manager/config_test.go deleted file mode 100644 index 253f289..0000000 --- a/pkg/manager/config_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package manager - -import ( - "fmt" - "testing" - - "github.com/criticalstack/e2d/pkg/netutil" -) - -func TestConfigUnspecifiedAddr(t *testing.T) { - host, err := netutil.DetectHostIPv4() - if err != nil { - t.Fatal(err) - } - cfg := &Config{ - ClientAddr: "0.0.0.0:2379", - PeerAddr: "0.0.0.0:2380", - GossipAddr: "0.0.0.0:7980", - BootstrapAddrs: []string{"0.0.0.0:7981"}, - } - if err := cfg.validate(); err != nil { - t.Fatal(err) - } - if cfg.ClientAddr != fmt.Sprintf("%s:%d", host, 2379) { - t.Fatalf("ClientAddr unspecified address not fixed: %v", cfg.ClientAddr) - } - if cfg.PeerAddr != fmt.Sprintf("%s:%d", host, 2380) { - t.Fatalf("PeerAddr unspecified address not fixed: %v", cfg.PeerAddr) - } - if cfg.GossipAddr != fmt.Sprintf("%s:%d", host, 7980) { - t.Fatalf("GossipAddr unspecified address not fixed: %v", cfg.GossipAddr) - } - if cfg.BootstrapAddrs[0] != fmt.Sprintf("%s:%d", host, 7981) { - t.Fatalf("BootstrapAddr unspecified address not fixed: %v", cfg.BootstrapAddrs[0]) - } -} diff --git a/pkg/manager/gossip.go b/pkg/manager/gossip.go deleted file mode 100644 index 787f67f..0000000 --- a/pkg/manager/gossip.go +++ /dev/null @@ -1,349 +0,0 @@ -package manager - -import ( - "bytes" - "context" - "encoding/gob" - "fmt" - stdlog "log" - "strings" - "sync" - "time" - - "github.com/criticalstack/e2d/pkg/log" - "github.com/criticalstack/e2d/pkg/netutil" - "github.com/hashicorp/memberlist" - "github.com/pkg/errors" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -const ( - DefaultGossipPort = 7980 -) - -type NodeStatus int - -const ( - Unknown NodeStatus = iota - Pending - Running -) - -func (s NodeStatus) String() string { - switch s { - case Unknown: - return "Unknown" - case Pending: - return "Pending" - case Running: - return "Running" - } - return "" -} - -type Member struct { - ID uint64 - Name string - ClientAddr string - ClientURL string - PeerAddr string - PeerURL string - GossipAddr string - BootstrapAddrs []string - Status NodeStatus -} - -func (m *Member) Marshal() ([]byte, error) { - var b bytes.Buffer - if err := gob.NewEncoder(&b).Encode(*m); err != nil { - return nil, err - } - return b.Bytes(), nil -} - -func (m *Member) Unmarshal(data []byte) error { - return gob.NewDecoder(bytes.NewReader(data)).Decode(m) -} - -type memberlister interface { - Join([]string) (int, error) - LocalNode() *memberlist.Node - Members() []*memberlist.Node - NumMembers() int - Shutdown() error -} - -type noopMemberlist struct{} - -func (noopMemberlist) Join([]string) (int, error) { - return 0, nil -} - -func (noopMemberlist) LocalNode() *memberlist.Node { - return &memberlist.Node{} -} - -func (noopMemberlist) Members() []*memberlist.Node { - return nil -} - -func (noopMemberlist) NumMembers() int { - return 0 -} - -func (noopMemberlist) Shutdown() error { - return nil -} - -type logger struct { - l *zap.Logger -} - -func (l *logger) Write(p []byte) (n int, err error) { - msg := string(p) - parts := strings.SplitN(msg, " ", 2) - lvl := "[DEBUG]" - if len(parts) > 1 { - lvl = parts[0] - msg = strings.TrimPrefix(parts[1], "memberlist: ") - } - - switch lvl { - case "[DEBUG]": - l.l.Debug(msg) - case "[WARN]": - l.l.Warn(msg) - case "[INFO]": - l.l.Info(msg) - } - return len(p), nil -} - -type gossipConfig struct { - Name string - ClientURL string - PeerURL string - GossipHost string - GossipPort int - SecretKey []byte - Debug bool -} - -type gossip struct { - m memberlister - - config *memberlist.Config - events chan memberlist.NodeEvent - - broadcasts *memberlist.TransmitLimitedQueue - mu sync.RWMutex - nodes map[string]NodeStatus - self *Member -} - -func newGossip(cfg *gossipConfig) *gossip { - c := memberlist.DefaultLANConfig() - c.Name = cfg.Name - c.BindAddr = cfg.GossipHost - c.BindPort = cfg.GossipPort - c.Logger = stdlog.New(&logger{log.NewLoggerWithLevel("memberlist", zapcore.InfoLevel)}, "", 0) - c.SecretKey = cfg.SecretKey - - g := &gossip{ - m: &noopMemberlist{}, - config: c, - events: make(chan memberlist.NodeEvent, 100), - nodes: make(map[string]NodeStatus), - self: &Member{ - Name: cfg.Name, - ClientURL: cfg.ClientURL, - PeerURL: cfg.PeerURL, - GossipAddr: fmt.Sprintf("%s:%d", cfg.GossipHost, cfg.GossipPort), - }, - } - g.broadcasts = &memberlist.TransmitLimitedQueue{ - NumNodes: func() int { - return g.m.NumMembers() - }, - RetransmitMult: 3, - } - c.Delegate = g - c.Events = &memberlist.ChannelEventDelegate{Ch: g.events} - return g -} - -func (g *gossip) Shutdown() error { - if err := g.m.Shutdown(); err != nil { - return err - } - if g.config.Events != nil { - g.config.Events = nil - } - if g.events != nil { - close(g.events) - g.events = nil - } - return nil -} - -// Start attempts to join a gossip network using the given bootstrap addresses. -func (g *gossip) Start(ctx context.Context, baddrs []string) error { - m, err := memberlist.Create(g.config) - if err != nil { - return err - } - g.m = m - if err := g.Update(Unknown); err != nil { - return err - } - peers := make([]string, 0) - for _, addr := range baddrs { - host, port, err := netutil.SplitHostPort(addr) - if err != nil { - return errors.Wrapf(err, "cannot split bootstrap address: %#v", addr) - } - if host == "" { - host = "127.0.0.1" - } - if port == 0 { - port = DefaultGossipPort - } - peers = append(peers, fmt.Sprintf("%s:%d", host, port)) - } - - log.Debug("attempting to join gossip network ...", - zap.String("bootstrap-addrs", strings.Join(peers, ",")), - ) - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - _, err := g.m.Join(peers) - if err != nil { - log.Errorf("cannot join gossip network: %v", err) - continue - } - log.Debug("joined gossip network successfully") - return nil - case <-ctx.Done(): - return ctx.Err() - } - } -} - -// msg implements the memberlist.Broadcast interface and is required to send -// messages over the gossip network -type msg struct { - data []byte -} - -func (m *msg) Invalidates(other memberlist.Broadcast) bool { return false } -func (m *msg) Message() []byte { return m.data } -func (m *msg) Finished() {} - -type statusMsg struct { - Name string - Status NodeStatus -} - -// Update uses the provided NodeStatus to updates the node metadata and -// broadcast the updated NodeStatus to all currently known members. -func (g *gossip) Update(status NodeStatus) error { - g.mu.Lock() - g.nodes[g.self.Name] = status - g.self.Status = status - g.mu.Unlock() - data, err := g.self.Marshal() - if err != nil { - return err - } - g.m.LocalNode().Meta = data - var b bytes.Buffer - if err := gob.NewEncoder(&b).Encode(statusMsg{Name: g.self.Name, Status: status}); err != nil { - return err - } - g.broadcasts.QueueBroadcast(&msg{b.Bytes()}) - return nil -} - -// Events returns a read-only channel of memberlist events. -func (g *gossip) Events() <-chan memberlist.NodeEvent { return g.events } - -// Members returns all members currently participating in the gossip network. -func (g *gossip) Members() []*Member { - g.mu.RLock() - defer g.mu.RUnlock() - - members := make([]*Member, 0) - for _, m := range g.m.Members() { - // A member may be in the memberlist but Meta may be nil if the local - // metadata has yet to be propagated to this node. In this case, we - // ignore that member considering it to not ready. - if m.Meta == nil { - continue - } - meta := &Member{} - if err := meta.Unmarshal(m.Meta); err != nil { - log.Debugf("cannot unmarshal member: %v", err) - continue - } - - // status information shared via delegate is presumed to be more - // accurate - if status, ok := g.nodes[meta.Name]; ok { - meta.Status = status - } - members = append(members, meta) - } - return members -} - -func (g *gossip) pendingMembers() []*Member { - members := make([]*Member, 0) - for _, member := range g.Members() { - if member.Status == Pending { - members = append(members, member) - } - } - return members -} - -func (g *gossip) runningMembers() []*Member { - members := make([]*Member, 0) - for _, member := range g.Members() { - if member.Status == Running { - members = append(members, member) - } - } - return members -} - -func (g *gossip) NodeMeta(limit int) []byte { return g.m.LocalNode().Meta } - -func (g *gossip) NotifyMsg(data []byte) { - if len(data) == 0 { - return - } - var n statusMsg - if err := gob.NewDecoder(bytes.NewReader(data)).Decode(&n); err != nil { - log.Debugf("cannot unmarshal: %v", err) - return - } - g.mu.Lock() - g.nodes[n.Name] = n.Status - g.mu.Unlock() -} - -func (g *gossip) GetBroadcasts(overhead, limit int) [][]byte { - return g.broadcasts.GetBroadcasts(overhead, limit) -} - -func (g *gossip) LocalState(join bool) []byte { - return nil -} - -func (g *gossip) MergeRemoteState(buf []byte, join bool) { -} diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 715d24f..1db1f21 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -2,17 +2,25 @@ package manager import ( "context" + "fmt" "io" "io/ioutil" + "math/rand" + "net/url" "os" + "path/filepath" "time" - "github.com/hashicorp/memberlist" "github.com/pkg/errors" "go.uber.org/zap" + "go.uber.org/zap/zapcore" "google.golang.org/grpc" "github.com/criticalstack/e2d/pkg/client" + configv1alpha1 "github.com/criticalstack/e2d/pkg/config/v1alpha1" + "github.com/criticalstack/e2d/pkg/discovery" + "github.com/criticalstack/e2d/pkg/etcdserver" + "github.com/criticalstack/e2d/pkg/gossip" "github.com/criticalstack/e2d/pkg/log" "github.com/criticalstack/e2d/pkg/manager/e2dpb" "github.com/criticalstack/e2d/pkg/snapshot" @@ -24,75 +32,180 @@ type Manager struct { ctx context.Context cancel context.CancelFunc - cfg *Config - gossip *gossip - etcd *server - cluster *clusterMembership - snapshotter snapshot.Snapshotter + cfg *configv1alpha1.Configuration + etcd *etcdserver.Server + gossip *gossip.Gossip + + peerGetter discovery.PeerGetter + snapshotter snapshot.Snapshotter + snapshotEncryptionKey *[32]byte removeCh chan string } // New creates a new instance of Manager. -func New(cfg *Config) (*Manager, error) { - if err := cfg.validate(); err != nil { +func New(cfg *configv1alpha1.Configuration) (*Manager, error) { + if err := cfg.Validate(); err != nil { + return nil, errors.Wrap(err, "e2d config is invalid") + } + + var clientSecurity, peerSecurity client.SecurityConfig + if cfg.CACert != "" && cfg.CAKey != "" { + ca, err := LoadCertificateAuthority(cfg.CACert, cfg.CAKey) + if err != nil { + return nil, err + } + if err := ca.WriteAll(); err != nil { + return nil, err + } + dir := filepath.Dir(cfg.CACert) + clientSecurity = client.SecurityConfig{ + CertFile: filepath.Join(dir, "server.crt"), + KeyFile: filepath.Join(dir, "server.key"), + TrustedCAFile: cfg.CACert, + } + peerSecurity = client.SecurityConfig{ + CertFile: filepath.Join(dir, "peer.crt"), + KeyFile: filepath.Join(dir, "peer.key"), + TrustedCAFile: cfg.CACert, + } + } + clientURL := url.URL{Scheme: clientSecurity.Scheme(), Host: cfg.ClientAddr.String()} + peerURL := url.URL{Scheme: peerSecurity.Scheme(), Host: cfg.PeerAddr.String()} + + name, err := getExistingNameFromDataDir(filepath.Join(cfg.DataDir, "member/snap/db"), peerURL) + if err != nil { + log.Debug("cannot read existing data-dir", zap.Error(err)) + name = fmt.Sprintf("%X", rand.Uint64()) + } + if cfg.OverrideName != "" { + name = cfg.OverrideName + } + + peerGetter, err := cfg.DiscoveryConfiguration.Setup() + if err != nil { + return nil, err + } + + // the initial peers will only be seeded when the user does not provide any + // initial peers + if cfg.RequiredClusterSize > 1 && len(cfg.DiscoveryConfiguration.InitialPeers) == 0 { + addrs, err := peerGetter.GetAddrs(context.Background()) + if err != nil { + return nil, err + } + log.Debugf("cloud provided addresses: %v", addrs) + for _, addr := range addrs { + cfg.DiscoveryConfiguration.InitialPeers = append(cfg.DiscoveryConfiguration.InitialPeers, fmt.Sprintf("%s:%d", addr, configv1alpha1.DefaultGossipPort)) + } + log.Debugf("bootstrap addrs: %v", cfg.DiscoveryConfiguration.InitialPeers) + if len(cfg.DiscoveryConfiguration.InitialPeers) == 0 { + return nil, errors.Errorf("bootstrap addresses must be provided") + } + } + + snapshotter, err := getSnapshotProvider(cfg.SnapshotConfiguration) + if err != nil { + return nil, err + } + + // both memberlist security and snapshot encryption are implicitly based + // upon the CA key + var key [32]byte + if cfg.CAKey != "" { + key, err = ReadEncryptionKey(cfg.CAKey) + if err != nil { + return nil, err + } + } + + var etcdLogLevel, memberlistLogLevel zapcore.Level + if err := etcdLogLevel.Set(cfg.EtcdLogLevel); err != nil { + return nil, err + } + if err := memberlistLogLevel.Set(cfg.MemberlistLogLevel); err != nil { return nil, err } m := &Manager{ cfg: cfg, - etcd: newServer(&serverConfig{ - Name: cfg.Name, - Dir: cfg.Dir, - ClientURL: cfg.ClientURL, - PeerURL: cfg.PeerURL, + etcd: &etcdserver.Server{ + Name: name, + DataDir: cfg.DataDir, + ClientURL: clientURL, + PeerURL: peerURL, RequiredClusterSize: cfg.RequiredClusterSize, - ClientSecurity: cfg.ClientSecurity, - PeerSecurity: cfg.PeerSecurity, - EtcdLogLevel: cfg.EtcdLogLevel, - Debug: cfg.Debug, + ClientSecurity: clientSecurity, + PeerSecurity: peerSecurity, + LogLevel: etcdLogLevel, EnableLocalListener: true, + PreVote: !cfg.DisablePreVote, + UnsafeNoFsync: cfg.UnsafeNoFsync, + }, + gossip: gossip.New(&gossip.Config{ + Name: name, + ClientURL: clientURL.String(), + PeerURL: peerURL.String(), + GossipHost: cfg.GossipAddr.Host, + GossipPort: int(cfg.GossipAddr.Port), + SecretKey: key[:], + LogLevel: memberlistLogLevel, }), - gossip: newGossip(&gossipConfig{ - Name: cfg.Name, - ClientURL: cfg.ClientURL.String(), - PeerURL: cfg.PeerURL.String(), - GossipHost: cfg.GossipHost, - GossipPort: cfg.GossipPort, - SecretKey: cfg.gossipSecretKey, - }), - removeCh: make(chan string, 10), - snapshotter: cfg.Snapshotter, + snapshotEncryptionKey: &key, + removeCh: make(chan string, 10), + peerGetter: peerGetter, + snapshotter: snapshotter, + } + if cfg.CompactionInterval.Duration != 0 { + m.etcd.CompactionInterval = cfg.CompactionInterval.Duration + } + if !cfg.MetricsConfiguration.Addr.IsZero() { + metricsURL := &url.URL{Scheme: clientSecurity.Scheme(), Host: cfg.MetricsConfiguration.Addr.String()} + if cfg.MetricsConfiguration.DisableAuth { + metricsURL.Scheme = "http" + } + m.etcd.MetricsURL = metricsURL } m.ctx, m.cancel = context.WithCancel(context.Background()) - m.cluster = newClusterMembership(m.ctx, m.cfg.HealthCheckTimeout, func(name string) error { - log.Debug("removing member ...", - zap.String("name", shortName(m.cfg.Name)), - zap.String("removed", shortName(name)), - ) - if err := m.etcd.removeMember(m.ctx, name); err != nil && errors.Cause(err) != errCannotFindMember { - return err - } - log.Debug("member removed", - zap.String("name", shortName(m.cfg.Name)), - zap.String("removed", shortName(name)), - ) - - // TODO(chris): this is mostly used for testing atm and - // should evolve in the future to be part of a more - // complete event broadcast system - select { - case m.removeCh <- name: - default: - } - return nil - }) - m.etcd.cfg.ServiceRegister = func(s *grpc.Server) { + m.etcd.ServiceRegister = func(s *grpc.Server) { e2dpb.RegisterManagerServer(s, &ManagerService{m}) } return m, nil } +func (m *Manager) Name() string { + if m.etcd == nil { + return "" + } + return m.etcd.Name +} + +func (m *Manager) Etcd() *etcdserver.Server { + if m.etcd == nil { + return nil + } + return m.etcd +} + +func (m *Manager) Gossip() *gossip.Gossip { + if m.gossip == nil { + return nil + } + return m.gossip +} + +func (m *Manager) Config() *configv1alpha1.Configuration { + return m.cfg +} + +func (m *Manager) RemoveCh() <-chan string { + return m.removeCh +} + +func (m *Manager) Snapshotter() snapshot.Snapshotter { + return m.snapshotter +} + // HardStop stops all services and cleans up the Manager state. Unlike // GracefulStop, it does not attempt to gracefully shutdown etcd. func (m *Manager) HardStop() { @@ -102,10 +215,12 @@ func (m *Manager) HardStop() { } m.cancel() m.ctx, m.cancel = context.WithCancel(context.Background()) - log.Debug("attempting hard stop of etcd server ...") - m.etcd.hardStop() - <-m.etcd.Server.StopNotify() - log.Debug("etcd server stopped") + if m.etcd != nil { + log.Debug("attempting hard stop of etcd server ...") + m.etcd.HardStop() + <-m.etcd.Server.StopNotify() + log.Debug("etcd server stopped") + } if err := m.gossip.Shutdown(); err != nil { log.Debug("gossip shutdown failed", zap.Error(err)) } @@ -121,7 +236,7 @@ func (m *Manager) GracefulStop() { m.cancel() m.ctx, m.cancel = context.WithCancel(context.Background()) log.Debug("attempting graceful stop of etcd server ...") - m.etcd.gracefulStop() + m.etcd.GracefulStop() <-m.etcd.Server.StopNotify() log.Debug("etcd server stopped") if err := m.gossip.Shutdown(); err != nil { @@ -130,54 +245,132 @@ func (m *Manager) GracefulStop() { } func (m *Manager) Restart() error { - peers := make([]*Peer, 0) + peers := make([]*etcdserver.Peer, 0) for _, member := range m.etcd.Etcd.Server.Cluster().Members() { if len(member.PeerURLs) == 0 { continue } - peers = append(peers, &Peer{member.Name, member.PeerURLs[0]}) + peers = append(peers, &etcdserver.Peer{Name: member.Name, URL: member.PeerURLs[0]}) } ctx, cancel := context.WithTimeout(m.ctx, 30*time.Second) defer cancel() - return m.etcd.restart(ctx, peers) + return m.etcd.Restart(ctx, peers) } -func (m *Manager) restoreFromSnapshot(peers []*Peer) (bool, error) { - if m.snapshotter == nil { - return false, nil +// Run starts and manages an etcd node based upon the provided configuration. +// In the case of a fault, or if the manager is otherwise stopped, this method +// exits. +func (m *Manager) Run() error { + if m.etcd.IsRunning() { + return errors.New("etcd is already running") } - r, err := m.snapshotter.Load() - if err != nil { - return false, err - } - defer r.Close() + switch m.cfg.RequiredClusterSize { + case 1: + // a single-node etcd cluster does not require gossip or need to wait for + // other members and therefore can start immediately + if err := m.startEtcdCluster([]*etcdserver.Peer{{Name: m.etcd.Name, URL: m.etcd.PeerURL.String()}}); err != nil { + return err + } + case 3, 5: + // all multi-node clusters require the gossip network to be started + if err := m.gossip.Start(m.ctx, m.cfg.DiscoveryConfiguration.InitialPeers); err != nil { + return err + } - log.Debugf("[%v]: attempting snapshot restore with members: %s", shortName(m.cfg.Name), peers) - tmpFile, err := ioutil.TempFile("", "snapshot.load") - if err != nil { - return false, err - } - defer tmpFile.Close() + // a multi-node etcd cluster will either be created or an existing one will + // be joined + if err := m.startOrJoinEtcdCluster(); err != nil { + return err + } - r = snapshotutil.NewGunzipReadCloser(r) - r = snapshotutil.NewDecrypterReadCloser(r, m.cfg.snapshotEncryptionKey) - if _, err := io.Copy(tmpFile, r); err != nil { - return false, err + if err := m.gossip.Update(gossip.Running); err != nil { + log.Debugf("[%v]: cannot update member metadata: %v", m.etcd.Name, err) + } } - // if the process is restarted, this will fail if the data-dir already - // exists, so it must be deleted here - if err := os.RemoveAll(m.cfg.Dir); err != nil { - log.Errorf("cannot remove data-dir: %v", err) + // cluster is ready so start maintenance loops + go m.runMembershipCleanup() + go m.runSnapshotter() + + for { + select { + case <-m.etcd.Server.StopNotify(): + log.Info("etcd server stopping ...", + zap.Stringer("id", m.etcd.Server.ID()), + zap.String("name", m.etcd.Name), + ) + if m.etcd.IsRestarting() { + time.Sleep(1 * time.Second) + continue + } + if m.cfg.RequiredClusterSize == 1 { + return nil + } + if err := m.gossip.Update(gossip.Unknown); err != nil { + log.Debugf("[%v]: cannot update member metadata: %v", m.etcd.Name, err) + } + return nil + case err := <-m.etcd.Err(): + return err + case <-m.ctx.Done(): + return nil + } } - log.Infof("loading snapshot from: %#v", tmpFile.Name()) - if err := m.etcd.restoreSnapshot(tmpFile.Name(), peers); err != nil { - return false, err +} + +func (m *Manager) startOrJoinEtcdCluster() error { + ctx, cancel := context.WithTimeout(m.ctx, m.cfg.DiscoveryConfiguration.BootstrapTimeout.Duration) + defer cancel() + + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + // first use peers to attempt joining an existing cluster + for _, member := range m.gossip.Members() { + if member.Name == m.etcd.Name { + continue + } + log.Debugf("[%v]: gossip peer: %+v", shortName(m.etcd.Name), member) + if member.Status != gossip.Running { + log.Debugf("[%v]: cannot join peer %#v in current status: %s", shortName(m.etcd.Name), shortName(member.Name), member.Status) + continue + } + if err := m.joinEtcdCluster(member.ClientURL); err != nil { + log.Debugf("[%v]: cannot join node %#v: %v", shortName(m.etcd.Name), member.ClientURL, err) + continue + } + log.Debug("joined an existing etcd cluster successfully") + return nil + } + log.Debugf("[%v]: cluster currently has %d members", shortName(m.etcd.Name), len(m.gossip.Members())) + if len(m.gossip.Members()) < m.cfg.RequiredClusterSize { + continue + } + if err := m.gossip.Update(gossip.Pending); err != nil { + log.Debugf("[%v]: cannot update member metadata: %v", shortName(m.etcd.Name), err) + } + + // when enough members are reporting in as pending, it means that a + // majority of members were unable to connect to an existing + // cluster + if len(m.gossip.PendingMembers()) < m.cfg.RequiredClusterSize { + log.Debugf("[%v]: members pending: %d", shortName(m.etcd.Name), len(m.gossip.PendingMembers())) + continue + } + peers := make([]*etcdserver.Peer, 0) + for _, m := range m.gossip.Members() { + peers = append(peers, &etcdserver.Peer{Name: m.Name, URL: m.PeerURL}) + } + return m.startEtcdCluster(peers) + case <-ctx.Done(): + return ctx.Err() + } } - log.Infof("successfully loaded snapshot from: %#v", tmpFile.Name()) - return true, nil } // startEtcdCluster starts a new etcd cluster with the provided peers. The list @@ -188,7 +381,7 @@ func (m *Manager) restoreFromSnapshot(peers []*Peer) (bool, error) { // marker is created. This enables clients using e2d to coordinate their // cluster, by conveying information about whether this is a brand new cluster // or an existing cluster that recovered from total cluster failure. -func (m *Manager) startEtcdCluster(peers []*Peer) error { +func (m *Manager) startEtcdCluster(peers []*etcdserver.Peer) error { snapshot, err := m.restoreFromSnapshot(peers) if err != nil { log.Error("cannot restore snapshot", zap.Error(err)) @@ -196,7 +389,7 @@ func (m *Manager) startEtcdCluster(peers []*Peer) error { ctx, cancel := context.WithTimeout(m.ctx, 5*time.Minute) defer cancel() - if err := m.etcd.startNew(ctx, peers); err != nil { + if err := m.etcd.StartNew(ctx, peers); err != nil { return err } if !snapshot { @@ -207,9 +400,9 @@ func (m *Manager) startEtcdCluster(peers []*Peer) error { // therefore do NOT get committed through the raft log. This is OK // since all servers that recover from a snapshot will perform the same // operations and the outcome is deterministic. - rev, deleted, err := m.etcd.clearVolatilePrefix() + rev, deleted, err := m.etcd.ClearVolatilePrefix() if err != nil { - if errors.Cause(err) != errServerStopped { + if errors.Cause(err) != etcdserver.ErrServerStopped { return err } log.Debug("cannot clear volatile prefix", zap.Error(err)) @@ -220,33 +413,74 @@ func (m *Manager) startEtcdCluster(peers []*Peer) error { zap.Int64("revision", rev), ) v := []byte(time.Now().Format(time.RFC3339)) - rev, err = m.etcd.placeSnapshotMarker(v) + rev, err = m.etcd.PlaceSnapshotMarker(v) if err != nil { - if errors.Cause(err) != errServerStopped { + if errors.Cause(err) != etcdserver.ErrServerStopped { return err } log.Debug("cannot place snapshot marker", zap.Error(err)) return nil } log.Debug("placed snapshot marker", - zap.String("key", string(snapshotMarkerKey)), + zap.String("key", string(etcdserver.SnapshotMarkerKey)), zap.String("value", string(v)), zap.Int64("rev", rev), ) return nil } +func (m *Manager) restoreFromSnapshot(peers []*etcdserver.Peer) (bool, error) { + if m.snapshotter == nil { + return false, nil + } + + r, err := m.snapshotter.Load() + if err != nil { + return false, err + } + defer r.Close() + + log.Debugf("[%v]: attempting snapshot restore with members: %s", shortName(m.etcd.Name), peers) + tmpFile, err := ioutil.TempFile("", "snapshot.load") + if err != nil { + return false, err + } + defer tmpFile.Close() + + r = snapshotutil.NewGunzipReadCloser(r) + r = snapshotutil.NewDecrypterReadCloser(r, m.snapshotEncryptionKey) + if _, err := io.Copy(tmpFile, r); err != nil { + return false, err + } + + // if the process is restarted, this will fail if the data-dir already + // exists, so it must be deleted here + if err := os.RemoveAll(m.cfg.DataDir); err != nil { + log.Errorf("cannot remove data-dir: %v", err) + } + log.Infof("loading snapshot from: %#v", tmpFile.Name()) + if err := m.etcd.RestoreSnapshot(tmpFile.Name(), peers); err != nil { + return false, err + } + log.Infof("successfully loaded snapshot from: %#v", tmpFile.Name()) + return true, nil +} + // joinEtcdCluster attempts to join an etcd cluster by establishing a client // connection with the provided peer URL. func (m *Manager) joinEtcdCluster(peerURL string) error { ctx, cancel := context.WithTimeout(m.ctx, 5*time.Minute) defer cancel() - c, err := newClient(&client.Config{ + cc, err := client.New(&client.Config{ ClientURLs: []string{peerURL}, - SecurityConfig: m.cfg.PeerSecurity, + SecurityConfig: m.etcd.PeerSecurity, Timeout: 1 * time.Second, }) + c := &Client{ + Client: cc, + Timeout: 1 * time.Second, + } if err != nil { return err } @@ -262,32 +496,32 @@ func (m *Manager) joinEtcdCluster(peerURL string) error { // happens when restarting a node and specifying the previous node name. // The previous node name MUST be specified since otherwise a new Name is // generated. - if members[m.cfg.Name] != nil { - peers := make([]*Peer, 0) + if members[m.etcd.Name] != nil { + peers := make([]*etcdserver.Peer, 0) for _, m := range members { - peers = append(peers, &Peer{m.Name, m.PeerURL}) + peers = append(peers, &etcdserver.Peer{Name: m.Name, URL: m.PeerURL}) } - log.Infof("%s is already considered a member, attempting to start ...", m.cfg.Name) - if err := m.etcd.joinExisting(ctx, peers); err == nil { + log.Infof("%s is already considered a member, attempting to start ...", m.etcd.Name) + if err := m.etcd.JoinExisting(ctx, peers); err == nil { return nil } - log.Infof("%s is already considered a member, but failed to start, attempting to remove ...", m.cfg.Name) - if err := c.removeMemberLocked(ctx, members[m.cfg.Name]); err != nil { + log.Infof("%s is already considered a member, but failed to start, attempting to remove ...", m.etcd.Name) + if err := c.removeMemberLocked(ctx, members[m.etcd.Name]); err != nil { return err } } - log.Infof("%s is NOT a member, attempting to add member and start ...", m.cfg.Name) - if err := os.RemoveAll(m.cfg.Dir); err != nil { - log.Errorf("failed to remove data dir %s, %v", m.cfg.Dir, err) + log.Infof("%s is NOT a member, attempting to add member and start ...", m.etcd.Name) + if err := os.RemoveAll(m.cfg.DataDir); err != nil { + log.Errorf("failed to remove data dir %s, %v", m.cfg.DataDir, err) } - unlock, err := c.Lock(m.cfg.Name, 10*time.Second) + unlock, err := c.Lock(m.etcd.Name, 10*time.Second) if err != nil { return err } defer unlock() - member, err := c.addMember(ctx, m.cfg.PeerURL.String()) + member, err := c.addMember(ctx, m.etcd.PeerURL.String()) if err != nil { return err } @@ -295,11 +529,11 @@ func (m *Manager) joinEtcdCluster(peerURL string) error { // The name will not be available immediately after adding a new member. // Since the member missing is this member, we can safely use the local // member name. - peers := []*Peer{{m.cfg.Name, m.cfg.PeerURL.String()}} + peers := []*etcdserver.Peer{{Name: m.etcd.Name, URL: m.etcd.PeerURL.String()}} for _, m := range members { - peers = append(peers, &Peer{m.Name, m.PeerURL}) + peers = append(peers, &etcdserver.Peer{Name: m.Name, URL: m.PeerURL}) } - if err := m.etcd.joinExisting(ctx, peers); err != nil { + if err := m.etcd.JoinExisting(ctx, peers); err != nil { if err := c.removeMember(m.ctx, member.ID); err != nil { log.Debug("unable to remove member", zap.Error(err)) } @@ -307,252 +541,3 @@ func (m *Manager) joinEtcdCluster(peerURL string) error { } return nil } - -func (m *Manager) startOrJoinEtcdCluster() error { - ctx, cancel := context.WithTimeout(m.ctx, m.cfg.BootstrapTimeout) - defer cancel() - - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - // first use peers to attempt joining an existing cluster - for _, member := range m.gossip.Members() { - if member.Name == m.cfg.Name { - continue - } - log.Debugf("[%v]: gossip peer: %+v", shortName(m.cfg.Name), member) - if member.Status != Running { - log.Debugf("[%v]: cannot join peer %#v in current status: %s", shortName(m.cfg.Name), shortName(member.Name), member.Status) - continue - } - if err := m.joinEtcdCluster(member.ClientURL); err != nil { - log.Debugf("[%v]: cannot join node %#v: %v", shortName(m.cfg.Name), member.ClientURL, err) - continue - } - log.Debug("joined an existing etcd cluster successfully") - return nil - } - log.Debugf("[%v]: cluster currently has %d members", shortName(m.cfg.Name), len(m.gossip.Members())) - if len(m.gossip.Members()) < m.cfg.RequiredClusterSize { - continue - } - if err := m.gossip.Update(Pending); err != nil { - log.Debugf("[%v]: cannot update member metadata: %v", shortName(m.cfg.Name), err) - } - - // when enough members are reporting in as pending, it means that a - // majority of members were unable to connect to an existing - // cluster - if len(m.gossip.pendingMembers()) < m.cfg.RequiredClusterSize { - log.Debugf("[%v]: members pending: %d", shortName(m.cfg.Name), len(m.gossip.pendingMembers())) - continue - } - peers := make([]*Peer, 0) - for _, m := range m.gossip.Members() { - peers = append(peers, &Peer{m.Name, m.PeerURL}) - } - return m.startEtcdCluster(peers) - case <-ctx.Done(): - return ctx.Err() - } - } -} - -func (m *Manager) runMembershipCleanup() { - if m.cfg.RequiredClusterSize == 1 { - return - } - for { - select { - case ev := <-m.gossip.Events(): - log.Debugf("[%v]: received membership event: %v", shortName(m.cfg.Name), ev) - - // It is possible to receive an event from memberlist where the - // Node is nil. This most likely happens when starting and stopping - // the server quickly, so is mostly observed during testing. - if ev.Node == nil { - continue - } - - // When this member's gossip network does not have enough members - // to be considered a majority, it is no longer eligible to affect - // cluster membership. This helps ensure that when a network - // partition takes place that minority partition(s) will not - // attempt to change cluster membership. Only members in Running - // status are considered. - if !m.cluster.ensureQuorum(len(m.gossip.runningMembers()) > m.cfg.RequiredClusterSize/2) { - log.Info("not enough members are healthy to remove other members", - zap.String("name", shortName(m.cfg.Name)), - zap.Int("gossip-members", len(m.gossip.runningMembers())), - zap.Int("required-cluster-size", m.cfg.RequiredClusterSize), - ) - } - - member := &Member{} - if err := member.Unmarshal(ev.Node.Meta); err != nil { - log.Debugf("[%v]: cannot unmarshal node meta: %v", shortName(m.cfg.Name), err) - continue - } - - // This member must not acknowledge membership changes related to - // itself. Gossip events are used to determine when a member needs - // to be evicted, and this responsibility falls to peers only (i.e. - // a member should never evict itself). The PeerURL is used rather - // than the name or gossip address as it better represents a - // distinct member of the cluster as only one PeerURL will ever be - // present on a network. - if member.PeerURL == m.cfg.PeerURL.String() { - continue - } - switch ev.Event { - case memberlist.NodeJoin: - log.Debugf("[%v]: member joined: %#v", shortName(m.cfg.Name), member.Name) - - // The name of the new member is compared with any members with - // a matching PeerURL that are currently part of the etcd - // cluster membership. In the case that a member is still part - // of the etcd cluster membership, but has a different name - // than the joining member, the assertion can be made that the - // existing member is now defunct and can be removed - // immediately to allow the new member to join. Since members - // do not handle gossip events for their own PeerURL, this - // check will only ever be performed by peers of the member - // joining the gossip network. - if oldName, err := m.etcd.lookupMemberNameByPeerAddr(member.PeerURL); err == nil { - log.Debugf("[%v]: member %v peerAddr in use by member %v", shortName(m.cfg.Name), member.Name, oldName) - if oldName != member.Name { - log.Debugf("[%v]: members name mismatched, evicting %v", shortName(m.cfg.Name), oldName) - if err := m.cluster.removeMember(oldName); err != nil { - log.Debug("unable to remove member", zap.Error(err)) - } - } - } - - m.cluster.removeSuspect(member.Name) - case memberlist.NodeLeave: - m.cluster.addSuspect(member.Name) - case memberlist.NodeUpdate: - } - case <-m.ctx.Done(): - return - } - } -} - -func (m *Manager) runSnapshotter() { - if m.snapshotter == nil { - log.Info("snapshotting disabled: no snapshot backup set") - return - } - log.Debug("starting snapshotter") - ticker := time.NewTicker(m.cfg.SnapshotInterval) - defer ticker.Stop() - - var latestRev int64 - - for { - select { - case <-ticker.C: - if m.etcd.isRestarting() { - log.Debug("server is restarting, skipping snapshot backup") - continue - } - if !m.etcd.isLeader() { - log.Debug("not leader, skipping snapshot backup") - continue - } - log.Debug("starting snapshot backup") - snapshotData, snapshotSize, rev, err := m.etcd.createSnapshot(latestRev) - if err != nil { - log.Debug("cannot create snapshot", - zap.String("name", shortName(m.cfg.Name)), - zap.Error(err), - ) - continue - } - if m.cfg.SnapshotEncryption { - snapshotData = snapshotutil.NewEncrypterReadCloser(snapshotData, m.cfg.snapshotEncryptionKey, snapshotSize) - } - if m.cfg.SnapshotCompression { - snapshotData = snapshotutil.NewGzipReadCloser(snapshotData) - } - if err := m.snapshotter.Save(snapshotData); err != nil { - log.Debug("cannot save snapshot", - zap.String("name", shortName(m.cfg.Name)), - zap.Error(err), - ) - continue - } - latestRev = rev - log.Infof("wrote snapshot (rev %d) to backup", latestRev) - case <-m.ctx.Done(): - log.Debug("stopping snapshotter") - return - } - } -} - -// Run starts and manages an etcd node based upon the provided configuration. -// In the case of a fault, or if the manager is otherwise stopped, this method -// exits. -func (m *Manager) Run() error { - if m.etcd.isRunning() { - return errors.New("etcd is already running") - } - - switch m.cfg.RequiredClusterSize { - case 1: - // a single-node etcd cluster does not require gossip or need to wait for - // other members and therefore can start immediately - if err := m.startEtcdCluster([]*Peer{{m.cfg.Name, m.cfg.PeerURL.String()}}); err != nil { - return err - } - case 3, 5: - // all multi-node clusters require the gossip network to be started - if err := m.gossip.Start(m.ctx, m.cfg.BootstrapAddrs); err != nil { - return err - } - - // a multi-node etcd cluster will either be created or an existing one will - // be joined - if err := m.startOrJoinEtcdCluster(); err != nil { - return err - } - - if err := m.gossip.Update(Running); err != nil { - log.Debugf("[%v]: cannot update member metadata: %v", m.cfg.Name, err) - } - } - - // cluster is ready so start maintenance loops - go m.runMembershipCleanup() - go m.runSnapshotter() - - for { - select { - case <-m.etcd.Server.StopNotify(): - log.Info("etcd server stopping ...", - zap.Stringer("id", m.etcd.Server.ID()), - zap.String("name", m.cfg.Name), - ) - if m.etcd.isRestarting() { - time.Sleep(1 * time.Second) - continue - } - if m.cfg.RequiredClusterSize == 1 { - return nil - } - if err := m.gossip.Update(Unknown); err != nil { - log.Debugf("[%v]: cannot update member metadata: %v", m.cfg.Name, err) - } - return nil - case err := <-m.etcd.Err(): - return err - case <-m.ctx.Done(): - return nil - } - } -} diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go deleted file mode 100644 index 6420f49..0000000 --- a/pkg/manager/manager_test.go +++ /dev/null @@ -1,1482 +0,0 @@ -//nolint:goconst -package manager - -import ( - "flag" - "fmt" - "io/ioutil" - "net/url" - "os" - "path/filepath" - "sync" - "testing" - "time" - - "github.com/cloudflare/cfssl/csr" - "go.uber.org/zap/zapcore" - - "github.com/criticalstack/e2d/pkg/client" - "github.com/criticalstack/e2d/pkg/log" - "github.com/criticalstack/e2d/pkg/netutil" - "github.com/criticalstack/e2d/pkg/pki" - "github.com/criticalstack/e2d/pkg/snapshot" - snapshotutil "github.com/criticalstack/e2d/pkg/snapshot/util" -) - -func writeFile(filename string, data []byte, perm os.FileMode) error { - if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil { - return err - } - return ioutil.WriteFile(filename, data, perm) -} - -type testCluster struct { - t *testing.T - nodes map[string]*Manager -} - -func newTestCluster(t *testing.T) *testCluster { - return &testCluster{t: t, nodes: make(map[string]*Manager)} -} - -func (n *testCluster) addNode(name string, cfg *Config) { - cfg.Name = name - cfg.Dir = filepath.Join("testdata", name) - m, err := New(cfg) - if err != nil { - n.t.Fatal(err) - } - n.nodes[name] = m -} - -func (n *testCluster) lookupNode(name string) *Manager { - node, ok := n.nodes[name] - if !ok { - n.t.Fatalf("node not found: %#v", name) - } - return node -} - -func (n *testCluster) start(names ...string) { - log.Infof("starting the following nodes: %v\n", names) - for _, name := range names { - go func(name string) { - if err := n.lookupNode(name).Run(); err != nil { - n.t.Fatal(err) - } - }(name) - } -} - -func (n *testCluster) restart(names ...string) { - var wg sync.WaitGroup - for _, name := range names { - wg.Add(1) - go func(name string) { - defer wg.Done() - - if err := n.lookupNode(name).Restart(); err != nil { - n.t.Fatal(err) - } - }(name) - } - wg.Wait() -} - -func (n *testCluster) startAll() { - for k := range n.nodes { - n.start(k) - } -} - -func (n *testCluster) saveSnapshot(name string) { - node := n.lookupNode(name) - data, size, _, err := node.etcd.createSnapshot(0) - if err != nil { - n.t.Fatal(err) - } - if node.cfg.SnapshotEncryption { - data = snapshotutil.NewEncrypterReadCloser(data, node.cfg.snapshotEncryptionKey, size) - } - if node.cfg.SnapshotCompression { - data = snapshotutil.NewGzipReadCloser(data) - } - if err := node.snapshotter.Save(data); err != nil { - n.t.Fatal(err) - } -} - -func (n *testCluster) stop(name string) { - log.Infof("stopping node: %#v\n", name) - n.lookupNode(name).HardStop() -} - -func (n *testCluster) wait(names ...string) { - log.Infof("waiting for the following nodes to be running: %v\n", names) - var wg sync.WaitGroup - for _, name := range names { - wg.Add(1) - go func(name string) { - defer wg.Done() - for { - if n.lookupNode(name).etcd.isRunning() { - return - } - time.Sleep(100 * time.Millisecond) - } - }(name) - } - wg.Wait() -} - -func (n *testCluster) waitRemoved(removed string, nodes ...string) { - log.Infof("waiting for the node %#v to be removed from the following nodes: %v\n", removed, nodes) - var wg sync.WaitGroup - for _, name := range nodes { - wg.Add(1) - go func(name string) { - defer wg.Done() - for removedNode := range n.lookupNode(name).removeCh { - if removedNode == removed { - return - } - } - }(name) - } - wg.Wait() -} - -func (n *testCluster) leader() *Manager { - for _, node := range n.nodes { - if node.etcd.isLeader() { - return node - } - } - return nil -} - -func (n *testCluster) cleanup() { - for _, node := range n.nodes { - node.HardStop() - } -} - -func newTestClient(addr string) *Client { - caddr, _ := netutil.ParseAddr(addr) - if caddr.Port == 0 { - caddr.Port = 2379 - } - clientURL := url.URL{Scheme: "http", Host: caddr.String()} - c, err := newClient(&client.Config{ - ClientURLs: []string{clientURL.String()}, - Timeout: 5 * time.Second, - }) - if err != nil { - panic(err) - } - return c -} - -func newSecureTestClient(addr, certFile, clientCertFile, clientKeyFile string) *Client { - caddr, _ := netutil.ParseAddr(addr) - if caddr.Port == 0 { - caddr.Port = 2379 - } - clientURL := url.URL{Scheme: "https", Host: caddr.String()} - c, err := newClient(&client.Config{ - ClientURLs: []string{clientURL.String()}, - SecurityConfig: client.SecurityConfig{ - CertFile: clientCertFile, - KeyFile: clientKeyFile, - CertAuth: true, - TrustedCAFile: certFile, - }, - Timeout: 5 * time.Second, - }) - if err != nil { - panic(err) - } - return c -} - -func newFileSnapshotter(path string) *snapshot.FileSnapshotter { - s, _ := snapshot.NewFileSnapshotter(path) - return s -} - -var testLong = flag.Bool("test.long", false, "enable running larger tests") - -func init() { - for _, arg := range os.Args[1:] { - if arg == "-test.long" { - *testLong = true - } - } - log.SetLevel(zapcore.DebugLevel) -} - -// TODO(chris): a lot of cases here create a healthy 3 node cluster, so create -// a function to do that to make the test code more succinct - -func TestManagerSingleFaultRecovery(t *testing.T) { - // TODO(chris): break this test into two version, one where the leader dies - // and one where a follower dies - if !*testLong { - t.Skip() - } - - if err := os.RemoveAll("testdata"); err != nil { - t.Fatal(err) - } - - c := newTestCluster(t) - defer c.cleanup() - - c.addNode("node1", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 5 * time.Second, - }) - c.addNode("node2", &Config{ - ClientAddr: ":2479", - PeerAddr: ":2480", - GossipAddr: ":7981", - BootstrapAddrs: []string{":7980"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 5 * time.Second, - }) - c.addNode("node3", &Config{ - ClientAddr: ":2579", - PeerAddr: ":2580", - GossipAddr: ":7982", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 5 * time.Second, - }) - - c.startAll() - c.wait("node1", "node2", "node3") - fmt.Println("ready") - cl := newTestClient(":2379") - testKey1 := "testkey1" - testValue1 := "testvalue1" - if err := cl.Set(testKey1, testValue1); err != nil { - t.Fatal(err) - } - v, err := cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } - - c.stop("node1") - c.waitRemoved("node1", "node2", "node3") - c.addNode("node4", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - }) - c.start("node4") - c.wait("node2", "node3", "node4") - fmt.Println("healthy!") - cl = newTestClient(":2379") - v, err = cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } -} - -func TestManagerRestoreClusterFromSnapshotNoCompression(t *testing.T) { - if !*testLong { - t.Skip() - } - if err := os.RemoveAll("testdata"); err != nil { - t.Fatal(err) - } - - c := newTestCluster(t) - defer c.cleanup() - - c.addNode("node1", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - }) - c.addNode("node2", &Config{ - ClientAddr: ":2479", - PeerAddr: ":2480", - GossipAddr: ":7981", - BootstrapAddrs: []string{":7980"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - }) - c.addNode("node3", &Config{ - ClientAddr: ":2579", - PeerAddr: ":2580", - GossipAddr: ":7982", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - }) - - c.startAll() - c.wait("node1", "node2", "node3") - fmt.Println("ready") - cl := newTestClient(":2479") - testKey1 := "testkey1" - testValue1 := "testvalue1" - if err := cl.Set(testKey1, testValue1); err != nil { - t.Fatal(err) - } - v, err := cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } - leader := c.leader().cfg.Name - fmt.Printf("leader = %+v\n", leader) - c.saveSnapshot(leader) - c.stop("node1") - c.stop("node2") - c.stop("node3") - - // need to wait a bit to ensure the port is free to bind - time.Sleep(1 * time.Second) - - // SnapshotInterval is 0 so creating snapshots is disabled, however, - // SnapshotDir is being replaced with default SnapshotDir from node1 so - // that this new node can restore that snapshot - c.addNode("node4", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - }) - c.addNode("node5", &Config{ - ClientAddr: ":2479", - PeerAddr: ":2480", - GossipAddr: ":7981", - BootstrapAddrs: []string{":7980"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - }) - c.addNode("node6", &Config{ - ClientAddr: ":2579", - PeerAddr: ":2580", - GossipAddr: ":7982", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - }) - c.start("node4", "node5", "node6") - c.wait("node4", "node5", "node6") - cl = newTestClient(":2379") - v, err = cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } -} - -func TestManagerRestoreClusterFromSnapshotCompression(t *testing.T) { - if !*testLong { - t.Skip() - } - - if err := os.RemoveAll("testdata"); err != nil { - t.Fatal(err) - } - - c := newTestCluster(t) - defer c.cleanup() - - c.addNode("node1", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - SnapshotCompression: true, - }) - c.addNode("node2", &Config{ - ClientAddr: ":2479", - PeerAddr: ":2480", - GossipAddr: ":7981", - BootstrapAddrs: []string{":7980"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - SnapshotCompression: true, - }) - c.addNode("node3", &Config{ - ClientAddr: ":2579", - PeerAddr: ":2580", - GossipAddr: ":7982", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - SnapshotCompression: true, - }) - - c.startAll() - c.wait("node1", "node2", "node3") - fmt.Println("ready") - cl := newTestClient(":2479") - testKey1 := "testkey1" - testValue1 := "testvalue1" - if err := cl.Set(testKey1, testValue1); err != nil { - t.Fatal(err) - } - v, err := cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } - leader := c.leader().cfg.Name - fmt.Printf("leader = %+v\n", leader) - c.saveSnapshot(leader) - c.stop("node1") - c.stop("node2") - c.stop("node3") - - // need to wait a bit to ensure the port is free to bind - time.Sleep(1 * time.Second) - - // SnapshotInterval is 0 so creating snapshots is disabled, however, - // SnapshotDir is being replaced with default SnapshotDir from node1 so - // that this new node can restore that snapshot - c.addNode("node4", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - SnapshotCompression: true, - }) - c.addNode("node5", &Config{ - ClientAddr: ":2479", - PeerAddr: ":2480", - GossipAddr: ":7981", - BootstrapAddrs: []string{":7980"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - SnapshotCompression: true, - }) - c.addNode("node6", &Config{ - ClientAddr: ":2579", - PeerAddr: ":2580", - GossipAddr: ":7982", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - SnapshotCompression: true, - }) - c.start("node4", "node5", "node6") - c.wait("node4", "node5", "node6") - cl = newTestClient(":2379") - v, err = cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } -} - -func TestManagerRestoreClusterFromSnapshotEncryption(t *testing.T) { - if !*testLong { - t.Skip() - } - if err := os.RemoveAll("testdata"); err != nil { - t.Fatal(err) - } - - if err := writeTestingCerts(); err != nil { - t.Fatal(err) - } - - caCertFile := "testdata/ca.crt" - caKeyFile := "testdata/ca.key" - serverCertFile := "testdata/server.crt" - serverKeyFile := "testdata/server.key" - peerCertFile := "testdata/peer.crt" - peerKeyFile := "testdata/peer.key" - clientCertFile := "testdata/client.crt" - clientKeyFile := "testdata/client.key" - - c := newTestCluster(t) - defer c.cleanup() - - c.addNode("node1", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - SnapshotCompression: true, - SnapshotEncryption: true, - ClientSecurity: client.SecurityConfig{ - CertFile: serverCertFile, - KeyFile: serverKeyFile, - TrustedCAFile: caCertFile, - }, - PeerSecurity: client.SecurityConfig{ - CertFile: peerCertFile, - KeyFile: peerKeyFile, - TrustedCAFile: caCertFile, - }, - CACertFile: caCertFile, - CAKeyFile: caKeyFile, - }) - c.addNode("node2", &Config{ - ClientAddr: ":2479", - PeerAddr: ":2480", - GossipAddr: ":7981", - BootstrapAddrs: []string{":7980"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - SnapshotCompression: true, - SnapshotEncryption: true, - ClientSecurity: client.SecurityConfig{ - CertFile: serverCertFile, - KeyFile: serverKeyFile, - TrustedCAFile: caCertFile, - }, - PeerSecurity: client.SecurityConfig{ - CertFile: peerCertFile, - KeyFile: peerKeyFile, - TrustedCAFile: caCertFile, - }, - CACertFile: caCertFile, - CAKeyFile: caKeyFile, - }) - c.addNode("node3", &Config{ - ClientAddr: ":2579", - PeerAddr: ":2580", - GossipAddr: ":7982", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - SnapshotCompression: true, - SnapshotEncryption: true, - ClientSecurity: client.SecurityConfig{ - CertFile: serverCertFile, - KeyFile: serverKeyFile, - TrustedCAFile: caCertFile, - }, - PeerSecurity: client.SecurityConfig{ - CertFile: peerCertFile, - KeyFile: peerKeyFile, - TrustedCAFile: caCertFile, - }, - CACertFile: caCertFile, - CAKeyFile: caKeyFile, - }) - - c.startAll() - c.wait("node1", "node2", "node3") - fmt.Println("ready") - cl := newSecureTestClient(":2479", caCertFile, clientCertFile, clientKeyFile) - testKey1 := "testkey1" - testValue1 := "testvalue1" - if err := cl.Set(testKey1, testValue1); err != nil { - t.Fatal(err) - } - v, err := cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } - leader := c.leader().cfg.Name - fmt.Printf("leader = %+v\n", leader) - c.saveSnapshot(leader) - c.stop("node1") - c.stop("node2") - c.stop("node3") - - // need to wait a bit to ensure the port is free to bind - time.Sleep(1 * time.Second) - - // SnapshotInterval is 0 so creating snapshots is disabled, however, - // SnapshotDir is being replaced with default SnapshotDir from node1 so - // that this new node can restore that snapshot - c.addNode("node4", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - SnapshotCompression: true, - ClientSecurity: client.SecurityConfig{ - CertFile: serverCertFile, - KeyFile: serverKeyFile, - TrustedCAFile: caCertFile, - }, - PeerSecurity: client.SecurityConfig{ - CertFile: peerCertFile, - KeyFile: peerKeyFile, - TrustedCAFile: caCertFile, - }, - CACertFile: caCertFile, - CAKeyFile: caKeyFile, - }) - c.addNode("node5", &Config{ - ClientAddr: ":2479", - PeerAddr: ":2480", - GossipAddr: ":7981", - BootstrapAddrs: []string{":7980"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - SnapshotCompression: true, - ClientSecurity: client.SecurityConfig{ - CertFile: serverCertFile, - KeyFile: serverKeyFile, - TrustedCAFile: caCertFile, - }, - PeerSecurity: client.SecurityConfig{ - CertFile: peerCertFile, - KeyFile: peerKeyFile, - TrustedCAFile: caCertFile, - }, - CACertFile: caCertFile, - CAKeyFile: caKeyFile, - }) - c.addNode("node6", &Config{ - ClientAddr: ":2579", - PeerAddr: ":2580", - GossipAddr: ":7982", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - SnapshotCompression: true, - ClientSecurity: client.SecurityConfig{ - CertFile: serverCertFile, - KeyFile: serverKeyFile, - TrustedCAFile: caCertFile, - }, - PeerSecurity: client.SecurityConfig{ - CertFile: peerCertFile, - KeyFile: peerKeyFile, - TrustedCAFile: caCertFile, - }, - CACertFile: caCertFile, - CAKeyFile: caKeyFile, - }) - c.start("node4", "node5", "node6") - c.wait("node4", "node5", "node6") - cl = newSecureTestClient(":2379", caCertFile, clientCertFile, clientKeyFile) - v, err = cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } -} - -func TestManagerSingleNodeRestart(t *testing.T) { - if !*testLong { - t.Skip() - } - if err := os.RemoveAll("testdata"); err != nil { - t.Fatal(err) - } - - c := newTestCluster(t) - defer c.cleanup() - - c.addNode("node1", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 15 * time.Second, - }) - c.addNode("node2", &Config{ - ClientAddr: ":2479", - PeerAddr: ":2480", - GossipAddr: ":7981", - BootstrapAddrs: []string{":7980"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 15 * time.Second, - }) - c.addNode("node3", &Config{ - ClientAddr: ":2579", - PeerAddr: ":2580", - GossipAddr: ":7982", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 15 * time.Second, - }) - - c.startAll() - c.wait("node1", "node2", "node3") - fmt.Println("ready") - cl := newTestClient(":2379") - testKey1 := "testkey1" - testValue1 := "testvalue1" - if err := cl.Set(testKey1, testValue1); err != nil { - t.Fatal(err) - } - v, err := cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } - - c.stop("node1") - - // The important part for this test to work is that the cluster cannot - // remove node1, which is why the HealthCheckInterval has been increased - // and we are not waiting for the node to be removed. The existing node is - // started again after being stopped so it should use the same data-dir. - c.start("node1") - c.wait("node1", "node2", "node3") - fmt.Println("healthy!") - - // It is possible that the client might fail here because the new member - // takes longer than usual to respond. The timeout has been increased in - // the test client in response to this, but this may need to be - // reevaluated. - cl = newTestClient(":2379") - v, err = cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } -} - -func TestManagerNodeReplacementUsedPeerAddr(t *testing.T) { - if !*testLong { - t.Skip() - } - if err := os.RemoveAll("testdata"); err != nil { - t.Fatal(err) - } - - c := newTestCluster(t) - defer c.cleanup() - - c.addNode("node1", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 15 * time.Second, - }) - c.addNode("node2", &Config{ - ClientAddr: ":2479", - PeerAddr: ":2480", - GossipAddr: ":7981", - BootstrapAddrs: []string{":7980"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 15 * time.Second, - }) - c.addNode("node3", &Config{ - ClientAddr: ":2579", - PeerAddr: ":2580", - GossipAddr: ":7982", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 15 * time.Second, - }) - - c.startAll() - c.wait("node1", "node2", "node3") - fmt.Println("ready") - cl := newTestClient(":2379") - testKey1 := "testkey1" - testValue1 := "testvalue1" - if err := cl.Set(testKey1, testValue1); err != nil { - t.Fatal(err) - } - v, err := cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } - - // Impersonate node1 and replace it with node4 which happens to have the - // same inet. It's expected that node1 will be removed from the cluster - // during node4 join because it's not allowed to join a new node that uses - // an existing peerAddr (and having two nodes with the same peerAddr is - // impossible since the peerAddr must be a routable IP:port). - c.addNode("node4", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7983", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 15 * time.Second, - }) - - c.stop("node1") - c.start("node4") - - // node1 should no longer be present in the etcd membership - c.waitRemoved("node1", "node2", "node3") - - // cluster should be up with node4 - waitChan := make(chan struct{}) - - go func() { - c.wait("node2", "node3", "node4") - waitChan <- struct{}{} - }() - - select { - case <-waitChan: - break - case <-time.After(30 * time.Second): - t.Fatal("timed out waiting for node4 to become healthy") - } - - fmt.Println("healthy!") - - // It is possible that the client might fail here because the new member - // takes longer than usual to respond. The timeout has been increased in - // the test client in response to this, but this may need to be - // reevaluated. - cl = newTestClient(":2379") - v, err = cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } -} - -func writeTestingCerts() error { - r, err := pki.NewDefaultRootCA() - if err != nil { - return err - } - if err := writeFile("testdata/ca.crt", r.CA.CertPEM, 0644); err != nil { - return err - } - if err := writeFile("testdata/ca.key", r.CA.KeyPEM, 0600); err != nil { - return err - } - certs, err := r.GenerateCertificates(pki.ServerSigningProfile, &csr.CertificateRequest{ - Names: []csr.Name{ - { - C: "US", - ST: "Boston", - L: "MA", - }, - }, - KeyRequest: &csr.KeyRequest{ - A: "rsa", - S: 2048, - }, - Hosts: []string{"127.0.0.1"}, - CN: "etcd server", - }) - if err != nil { - return err - } - - if err := writeFile("testdata/server.crt", certs.CertPEM, 0644); err != nil { - return err - } - if err := writeFile("testdata/server.key", certs.KeyPEM, 0600); err != nil { - return err - } - certs, err = r.GenerateCertificates(pki.PeerSigningProfile, &csr.CertificateRequest{ - Names: []csr.Name{ - { - C: "US", - ST: "Boston", - L: "MA", - }, - }, - KeyRequest: &csr.KeyRequest{ - A: "rsa", - S: 2048, - }, - Hosts: []string{"127.0.0.1"}, - CN: "etcd peer", - }) - if err != nil { - return err - } - - if err := writeFile("testdata/peer.crt", certs.CertPEM, 0644); err != nil { - return err - } - if err := writeFile("testdata/peer.key", certs.KeyPEM, 0600); err != nil { - return err - } - certs, err = r.GenerateCertificates(pki.ClientSigningProfile, &csr.CertificateRequest{ - Names: []csr.Name{ - { - C: "US", - ST: "Boston", - L: "MA", - }, - }, - KeyRequest: &csr.KeyRequest{ - A: "rsa", - S: 2048, - }, - Hosts: []string{""}, - CN: "etcd client", - }) - if err != nil { - return err - } - - if err := writeFile("testdata/client.crt", certs.CertPEM, 0644); err != nil { - return err - } - if err := writeFile("testdata/client.key", certs.KeyPEM, 0600); err != nil { - return err - } - return nil -} - -func TestManagerSecurityConfig(t *testing.T) { - if !*testLong { - t.Skip() - } - if err := os.RemoveAll("testdata"); err != nil { - t.Fatal(err) - } - - if err := writeTestingCerts(); err != nil { - t.Fatal(err) - } - - caCertFile := "testdata/ca.crt" - caKeyFile := "testdata/ca.key" - serverCertFile := "testdata/server.crt" - serverKeyFile := "testdata/server.key" - peerCertFile := "testdata/peer.crt" - peerKeyFile := "testdata/peer.key" - clientCertFile := "testdata/client.crt" - clientKeyFile := "testdata/client.key" - - c := newTestCluster(t) - defer c.cleanup() - - c.addNode("node1", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 5 * time.Second, - ClientSecurity: client.SecurityConfig{ - CertFile: serverCertFile, - KeyFile: serverKeyFile, - TrustedCAFile: caCertFile, - }, - PeerSecurity: client.SecurityConfig{ - CertFile: peerCertFile, - KeyFile: peerKeyFile, - TrustedCAFile: caCertFile, - }, - CACertFile: caCertFile, - CAKeyFile: caKeyFile, - }) - c.addNode("node2", &Config{ - ClientAddr: ":2479", - PeerAddr: ":2480", - GossipAddr: ":7981", - BootstrapAddrs: []string{":7980"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 5 * time.Second, - ClientSecurity: client.SecurityConfig{ - CertFile: serverCertFile, - KeyFile: serverKeyFile, - TrustedCAFile: caCertFile, - }, - PeerSecurity: client.SecurityConfig{ - CertFile: peerCertFile, - KeyFile: peerKeyFile, - TrustedCAFile: caCertFile, - }, - CACertFile: caCertFile, - CAKeyFile: caKeyFile, - }) - c.addNode("node3", &Config{ - ClientAddr: ":2579", - PeerAddr: ":2580", - GossipAddr: ":7982", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 5 * time.Second, - ClientSecurity: client.SecurityConfig{ - CertFile: serverCertFile, - KeyFile: serverKeyFile, - TrustedCAFile: caCertFile, - }, - PeerSecurity: client.SecurityConfig{ - CertFile: peerCertFile, - KeyFile: peerKeyFile, - TrustedCAFile: caCertFile, - }, - CACertFile: caCertFile, - CAKeyFile: caKeyFile, - }) - - c.start("node1", "node2", "node3") - c.wait("node1", "node2", "node3") - fmt.Println("ready") - cl := newSecureTestClient(":2379", caCertFile, clientCertFile, clientKeyFile) - testKey1 := "testkey1" - testValue1 := "testvalue1" - if err := cl.Set(testKey1, testValue1); err != nil { - t.Fatal(err) - } - cl.Close() - cl = newSecureTestClient(":2479", caCertFile, clientCertFile, clientKeyFile) - v, err := cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } -} - -func TestManagerReadExistingName(t *testing.T) { - if !*testLong { - t.Skip() - } - - if err := os.RemoveAll("testdata"); err != nil { - t.Fatal(err) - } - - c := newTestCluster(t) - defer c.cleanup() - name := "node1" - - var err error - c.nodes[name], err = New(&Config{ - Name: name, - Dir: filepath.Join("testdata", name), - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 1, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 5 * time.Second, - }) - if err != nil { - t.Fatal(err) - } - c.startAll() - c.wait("node1") - - fmt.Println("ready") - cl := newTestClient(":2379") - testKey1 := "testkey1" - testValue1 := "testvalue1" - if err := cl.Set(testKey1, testValue1); err != nil { - t.Fatal(err) - } - v, err := cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } - - c.stop(name) - newNode, err := New(&Config{ - Dir: filepath.Join("testdata", name), - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 1, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 5 * time.Second, - }) - if err != nil { - t.Fatal(err) - } - if newNode.cfg.Name != name { - t.Fatalf("expected %#v, received %#v", name, newNode.cfg.Name) - } -} - -func TestManagerDeleteVolatile(t *testing.T) { - if !*testLong { - t.Skip() - } - if err := os.RemoveAll("testdata"); err != nil { - t.Fatal(err) - } - - c := newTestCluster(t) - defer c.cleanup() - - c.addNode("node1", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 1, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - }) - - c.startAll() - c.wait("node1") - fmt.Println("ready") - cl := newTestClient(":2379") - volatilePrefix := "/_e2d" - nkeys := 10 - for i := 0; i < nkeys; i++ { - if err := cl.Set(fmt.Sprintf("%s/%d", volatilePrefix, i), "testvalue1"); err != nil { - t.Fatal(err) - } - } - n, err := cl.Count(volatilePrefix) - if err != nil { - t.Fatal(err) - } - cl.Close() - - // cluster-info is added by e2db which contains a key with the cluster-info - // itself, and another key for the e2db table schema - if n != int64(nkeys+2) { - t.Fatalf("expected %d keys, received %d", nkeys+2, n) - } - c.saveSnapshot("node1") - c.stop("node1") - - // need to wait a bit to ensure the port is free to bind - time.Sleep(1 * time.Second) - - // SnapshotInterval is 0 so creating snapshots is disabled, however, - // SnapshotDir is being replaced with default SnapshotDir from node1 so - // that this new node can restore that snapshot - c.addNode("node2", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 1, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - Snapshotter: newFileSnapshotter("testdata/snapshots"), - }) - c.start("node2") - c.wait("node2") - cl = newTestClient(":2379") - _, err = cl.Get("/_e2d/snapshot") - if err != nil { - t.Fatal(err) - } - n, err = cl.Count("/_e2d") - if err != nil { - t.Fatal(err) - } - if n != 1 { - t.Fatalf("after snapshot recover, only 1 key/value should remain, received %d", n) - } - cl.Close() -} - -func TestManagerServerRestartCertRenewal(t *testing.T) { - if !*testLong { - t.Skip() - } - if err := os.RemoveAll("testdata"); err != nil { - t.Fatal(err) - } - - if err := writeTestingCerts(); err != nil { - t.Fatal(err) - } - - caCertFile := "testdata/ca.crt" - caKeyFile := "testdata/ca.key" - serverCertFile := "testdata/server.crt" - serverKeyFile := "testdata/server.key" - peerCertFile := "testdata/peer.crt" - peerKeyFile := "testdata/peer.key" - clientCertFile := "testdata/client.crt" - clientKeyFile := "testdata/client.key" - - c := newTestCluster(t) - defer c.cleanup() - - c.addNode("node1", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 5 * time.Second, - ClientSecurity: client.SecurityConfig{ - CertFile: serverCertFile, - KeyFile: serverKeyFile, - TrustedCAFile: caCertFile, - }, - PeerSecurity: client.SecurityConfig{ - CertFile: peerCertFile, - KeyFile: peerKeyFile, - TrustedCAFile: caCertFile, - }, - CACertFile: caCertFile, - CAKeyFile: caKeyFile, - }) - c.addNode("node2", &Config{ - ClientAddr: ":2479", - PeerAddr: ":2480", - GossipAddr: ":7981", - BootstrapAddrs: []string{":7980"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 5 * time.Second, - ClientSecurity: client.SecurityConfig{ - CertFile: serverCertFile, - KeyFile: serverKeyFile, - TrustedCAFile: caCertFile, - }, - PeerSecurity: client.SecurityConfig{ - CertFile: peerCertFile, - KeyFile: peerKeyFile, - TrustedCAFile: caCertFile, - }, - CACertFile: caCertFile, - CAKeyFile: caKeyFile, - }) - c.addNode("node3", &Config{ - ClientAddr: ":2579", - PeerAddr: ":2580", - GossipAddr: ":7982", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 5 * time.Second, - ClientSecurity: client.SecurityConfig{ - CertFile: serverCertFile, - KeyFile: serverKeyFile, - TrustedCAFile: caCertFile, - }, - PeerSecurity: client.SecurityConfig{ - CertFile: peerCertFile, - KeyFile: peerKeyFile, - TrustedCAFile: caCertFile, - }, - CACertFile: caCertFile, - CAKeyFile: caKeyFile, - }) - - c.start("node1", "node2", "node3") - c.wait("node1", "node2", "node3") - fmt.Println("ready") - cl := newSecureTestClient(":2379", caCertFile, clientCertFile, clientKeyFile) - testKey1 := "testkey1" - testValue1 := "testvalue1" - if err := cl.Set(testKey1, testValue1); err != nil { - t.Fatal(err) - } - cl.Close() - if err := writeTestingCerts(); err != nil { - t.Fatal(err) - } - c.restart("node1", "node2", "node3") - cl = newSecureTestClient(":2479", caCertFile, clientCertFile, clientKeyFile) - v, err := cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } -} - -func TestManagerRollingUpdate(t *testing.T) { - if !*testLong { - t.Skip() - } - - if err := os.RemoveAll("testdata"); err != nil { - t.Fatal(err) - } - - c := newTestCluster(t) - defer c.cleanup() - - c.addNode("node1", &Config{ - ClientAddr: ":2379", - PeerAddr: ":2380", - GossipAddr: ":7980", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 5 * time.Second, - }) - c.addNode("node2", &Config{ - ClientAddr: ":2479", - PeerAddr: ":2480", - GossipAddr: ":7981", - BootstrapAddrs: []string{":7980"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 5 * time.Second, - }) - c.addNode("node3", &Config{ - ClientAddr: ":2579", - PeerAddr: ":2580", - GossipAddr: ":7982", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 5 * time.Second, - }) - - c.startAll() - c.wait("node1", "node2", "node3") - fmt.Println("ready") - cl := newTestClient(":2379") - testKey1 := "testkey1" - testValue1 := "testvalue1" - if err := cl.Set(testKey1, testValue1); err != nil { - t.Fatal(err) - } - v, err := cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } - - c.addNode("node4", &Config{ - ClientAddr: ":2679", - PeerAddr: ":2680", - GossipAddr: ":7983", - BootstrapAddrs: []string{":7981"}, - RequiredClusterSize: 3, - HealthCheckInterval: 1 * time.Second, - HealthCheckTimeout: 10 * time.Second, - }) - c.start("node4") - c.wait("node2", "node3", "node4") - fmt.Println("healthy!") - c.stop("node1") - c.waitRemoved("node1", "node2", "node3", "node4") - cl = newTestClient(":2679") - v, err = cl.Get(testKey1) - if err != nil { - t.Fatal(err) - } - cl.Close() - if string(v) != testValue1 { - t.Fatalf("expected %#v, received %#v", testValue1, string(v)) - } -} diff --git a/pkg/manager/membership.go b/pkg/manager/membership.go index 0d9d561..4dc4d84 100644 --- a/pkg/manager/membership.go +++ b/pkg/manager/membership.go @@ -2,16 +2,150 @@ package manager import ( "context" + "fmt" + "strings" "sync" "time" - "github.com/criticalstack/e2d/pkg/log" + "github.com/hashicorp/memberlist" + "github.com/pkg/errors" "go.uber.org/zap" + + "github.com/criticalstack/e2d/pkg/etcdserver" + "github.com/criticalstack/e2d/pkg/gossip" + "github.com/criticalstack/e2d/pkg/log" ) +func (m *Manager) runMembershipCleanup() { + if m.cfg.RequiredClusterSize == 1 { + return + } + + membership := newMembership(m.ctx, m.cfg.HealthCheckTimeout.Duration, func(name string) error { + log.Debug("removing member ...", + zap.String("name", shortName(m.etcd.Name)), + zap.String("removed", shortName(name)), + ) + if err := m.etcd.RemoveMember(m.ctx, name); err != nil && errors.Cause(err) != etcdserver.ErrCannotFindMember { + return err + } + log.Debug("member removed", + zap.String("name", shortName(m.etcd.Name)), + zap.String("removed", shortName(name)), + ) + + // TODO(chris): this is mostly used for testing atm and + // should evolve in the future to be part of a more + // complete event broadcast system + select { + case m.removeCh <- name: + default: + } + return nil + }) + + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + // When this member's gossip network does not have enough members + // to be considered a majority, it is no longer eligible to affect + // cluster membership. This helps ensure that when a network + // partition takes place that minority partition(s) will not + // attempt to change cluster membership. Only members in Running + // status are considered. + if membership.ensureQuorum(len(m.gossip.RunningMembers()) > m.cfg.RequiredClusterSize/2) { + continue + } + members := make([]string, 0) + for _, m := range m.gossip.Members() { + members = append(members, fmt.Sprintf("%s=%s", m.Name, m.Status)) + } + suspects := make([]string, 0) + for k, v := range membership.suspects { + suspects = append(suspects, fmt.Sprintf("%s=%s", k, v.Format(time.RFC3339))) + } + log.Debug("not enough members are healthy to remove other members", + zap.String("name", shortName(m.etcd.Name)), + zap.Int("gossip-members-running", len(m.gossip.RunningMembers())), + zap.String("gossip-members", strings.Join(members, ",")), + zap.Int("required-cluster-size", m.cfg.RequiredClusterSize), + zap.Bool("hasQuorum", membership.hasQuorum), + zap.String("suspects", strings.Join(suspects, ",")), + ) + case ev := <-m.gossip.Events(): + // It is possible to receive an event from memberlist where the + // Node is nil. This most likely happens when starting and stopping + // the server quickly, so is mostly observed during testing. + if ev.Node == nil { + log.Debug("discarded null event") + continue + } + member := &gossip.Member{} + if err := member.Unmarshal(ev.Node.Meta); err != nil { + log.Debugf("[%v]: cannot unmarshal node meta: %v", shortName(m.etcd.Name), err) + continue + } + log.Debug("received membership event", + zap.String("name", shortName(m.etcd.Name)), + zap.Int("event-type", int(ev.Event)), + zap.Uint64("member-id", member.ID), + zap.String("member-name", member.Name), + zap.String("member-client-addr", member.ClientAddr), + zap.String("member-peer-addr", member.PeerAddr), + zap.String("member-gossip-addr", member.GossipAddr), + ) + + // This member must not acknowledge membership changes related to + // itself. Gossip events are used to determine when a member needs + // to be evicted, and this responsibility falls to peers only (i.e. + // a member should never evict itself). The PeerURL is used rather + // than the name or gossip address as it better represents a + // distinct member of the cluster as only one PeerURL will ever be + // present on a network. + if member.PeerURL == m.etcd.PeerURL.String() { + continue + } + switch ev.Event { + case memberlist.NodeJoin: + log.Debugf("[%v]: member joined: %#v", shortName(m.etcd.Name), member.Name) + + // The name of the new member is compared with any members with + // a matching PeerURL that are currently part of the etcd + // cluster membership. In the case that a member is still part + // of the etcd cluster membership, but has a different name + // than the joining member, the assertion can be made that the + // existing member is now defunct and can be removed + // immediately to allow the new member to join. Since members + // do not handle gossip events for their own PeerURL, this + // check will only ever be performed by peers of the member + // joining the gossip network. + if oldName, err := m.etcd.LookupMemberNameByPeerAddr(member.PeerURL); err == nil { + log.Debugf("[%v]: member %v peerAddr %q in use by member %v", shortName(m.etcd.Name), member.Name, member.PeerURL, oldName) + if oldName != member.Name { + log.Debugf("[%v]: members name mismatched, evicting %v", shortName(m.etcd.Name), oldName) + if err := membership.removeMember(oldName); err != nil { + log.Debug("unable to remove member", zap.Error(err)) + } + } + } + + membership.removeSuspect(member.Name) + case memberlist.NodeLeave: + membership.addSuspect(member.Name) + case memberlist.NodeUpdate: + } + case <-m.ctx.Done(): + return + } + } +} + type removerFunc func(string) error -type clusterMembership struct { +type membership struct { timeout time.Duration fn removerFunc @@ -20,12 +154,13 @@ type clusterMembership struct { hasQuorum bool } -func newClusterMembership(ctx context.Context, d time.Duration, fn removerFunc) *clusterMembership { - c := &clusterMembership{ +func newMembership(ctx context.Context, d time.Duration, fn removerFunc) *membership { + c := &membership{ timeout: d, fn: fn, suspects: make(map[string]time.Time), } + go func() { ticker := time.NewTicker(time.Second) defer ticker.Stop() @@ -51,23 +186,26 @@ func newClusterMembership(ctx context.Context, d time.Duration, fn removerFunc) return c } -func (c *clusterMembership) addSuspect(name string) { +func (c *membership) addSuspect(name string) { c.mu.Lock() c.suspects[name] = time.Now() c.mu.Unlock() } -func (c *clusterMembership) removeSuspect(name string) { +func (c *membership) removeSuspect(name string) { c.mu.Lock() delete(c.suspects, name) c.mu.Unlock() } -func (c *clusterMembership) removeMember(name string) error { +func (c *membership) removeMember(name string) error { c.mu.Lock() defer c.mu.Unlock() if !c.hasQuorum { + log.Debug("gossip network lost quorum, cannot remove member", + zap.String("name", name), + ) return nil } if err := c.fn(name); err != nil { @@ -77,7 +215,7 @@ func (c *clusterMembership) removeMember(name string) error { return nil } -func (c *clusterMembership) ensureQuorum(q bool) bool { +func (c *membership) ensureQuorum(q bool) bool { c.mu.Lock() defer c.mu.Unlock() @@ -86,6 +224,11 @@ func (c *clusterMembership) ensureQuorum(q bool) bool { return c.hasQuorum } c.hasQuorum = q - c.suspects = make(map[string]time.Time) + if c.suspects == nil { + c.suspects = make(map[string]time.Time) + } + for name := range c.suspects { + c.suspects[name] = time.Now() + } return c.hasQuorum } diff --git a/pkg/manager/server.go b/pkg/manager/server.go deleted file mode 100644 index bbefc3e..0000000 --- a/pkg/manager/server.go +++ /dev/null @@ -1,473 +0,0 @@ -package manager - -import ( - "bytes" - "context" - "fmt" - "io" - "io/ioutil" - "net/url" - "os" - "sort" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/pkg/errors" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/clientv3/snapshot" - "go.etcd.io/etcd/embed" - "go.etcd.io/etcd/etcdserver/api/membership" - "go.etcd.io/etcd/lease" - "go.etcd.io/etcd/mvcc" - "go.etcd.io/etcd/mvcc/backend" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - "google.golang.org/grpc" - "google.golang.org/grpc/grpclog" - - "github.com/criticalstack/e2d/pkg/client" - "github.com/criticalstack/e2d/pkg/e2db" - "github.com/criticalstack/e2d/pkg/log" - "github.com/criticalstack/e2d/pkg/netutil" -) - -type serverConfig struct { - // name used for etcd.Embed instance, should generally be left alone so - // that a random name is generated - Name string - - // directory used for etcd data-dir, wal and snapshot dirs derived from - // this by etcd - Dir string - - // client endpoint for accessing etcd - ClientURL url.URL - - // address used for traffic within the cluster - PeerURL url.URL - - // the required number of nodes that must be present to start a cluster - RequiredClusterSize int - - // configures authentication/transport security for clients - ClientSecurity client.SecurityConfig - - // configures authentication/transport security within the etcd cluster - PeerSecurity client.SecurityConfig - - // add a local client listener (i.e. 127.0.0.1) - EnableLocalListener bool - - // configures the level of the logger used by etcd - EtcdLogLevel zapcore.Level - - ServiceRegister func(*grpc.Server) - - Debug bool -} - -type server struct { - *embed.Etcd - cfg *serverConfig - - // used to determine if the instance of Etcd has already been started - started uint64 - // set when server is being restarted - restarting uint64 - - // mu is used to coordinate potentially unsafe access to etcd - mu sync.Mutex -} - -func newServer(cfg *serverConfig) *server { - return &server{cfg: cfg} -} - -func (s *server) isRestarting() bool { - return atomic.LoadUint64(&s.restarting) == 1 -} - -func (s *server) isRunning() bool { - return atomic.LoadUint64(&s.started) == 1 -} - -func (s *server) isLeader() bool { - if !s.isRunning() { - return false - } - return s.Etcd.Server.Leader() == s.Etcd.Server.ID() -} - -func (s *server) restart(ctx context.Context, peers []*Peer) error { - atomic.StoreUint64(&s.restarting, 1) - defer atomic.StoreUint64(&s.restarting, 0) - - s.hardStop() - return s.startEtcd(ctx, embed.ClusterStateFlagNew, peers) -} - -func (s *server) hardStop() { - s.mu.Lock() - defer s.mu.Unlock() - - if s.Etcd != nil { - // This shuts down the etcdserver.Server instance without coordination - // with other members of the cluster. This ensures that a transfer of - // leadership is not attempted, which can cause an issue where a member - // can no longer join after a snapshot restore, should it fail during - // the attempted transfer of leadership. - s.Server.HardStop() - - // This must be called after HardStop since Close will start a graceful - // shutdown. - s.Etcd.Close() - } - atomic.StoreUint64(&s.started, 0) -} - -func (s *server) gracefulStop() { - s.mu.Lock() - defer s.mu.Unlock() - - if s.Etcd != nil { - // There is no need to call Stop on the underlying etcdserver.Server - // since it is called in Close. - s.Etcd.Close() - } - atomic.StoreUint64(&s.started, 0) -} - -type Peer struct { - Name string - URL string -} - -func (p *Peer) String() string { - return fmt.Sprintf("%s=%s", p.Name, p.URL) -} - -func initialClusterStringFromPeers(peers []*Peer) string { - initialCluster := make([]string, 0) - for _, p := range peers { - initialCluster = append(initialCluster, fmt.Sprintf("%s=%s", p.Name, p.URL)) - } - if len(initialCluster) == 0 { - return "" - } - sort.Strings(initialCluster) - return strings.Join(initialCluster, ",") -} - -// validatePeers ensures that a group of peers are capable of starting, joining -// or recovering a cluster. It must be used whenever the initial cluster string -// will be built. -func validatePeers(peers []*Peer, requiredClusterSize int) error { - // When the name part of the initial cluster string is blank, etcd behaves - // abnormally. The same raft id is generated when providing the same - // connection information, so in cases where a member was removed from the - // cluster and replaced by a new member with the same address, having a - // blank name caused it to not be removed from the removed member tracking - // done by rafthttp. The member is "accepted" into the cluster, but cannot - // participate since the transport layer won't allow it. - for _, p := range peers { - if p.Name == "" || p.URL == "" { - return errors.Errorf("peer name/url cannot be blank: %+v", p) - } - } - - // The number of peers used to start etcd should always be the same as the - // cluster size. Otherwise, the etcd cluster can (very likely) fail to - // become healthy, therefore we go ahead and return early rather than deal - // with an invalid state. - if len(peers) < requiredClusterSize { - return errors.Errorf("expected %d members, but received %d: %v", requiredClusterSize, len(peers), peers) - } - return nil -} - -func (s *server) startEtcd(ctx context.Context, state string, peers []*Peer) error { - if err := validatePeers(peers, s.cfg.RequiredClusterSize); err != nil { - return err - } - - cfg := embed.NewConfig() - cfg.Name = s.cfg.Name - cfg.Dir = s.cfg.Dir - if s.cfg.Dir != "" { - cfg.Dir = s.cfg.Dir - } - if err := os.MkdirAll(cfg.Dir, 0700); err != nil && !os.IsExist(err) { - return errors.Wrapf(err, "cannot create etcd data dir: %#v", cfg.Dir) - } - - // NOTE(chrism): etcd 3.4.9 introduced a check on the data directory - // permissions that require 0700. Since this causes the server to not come - // up we will attempt to change the perms. - log.Info("chmod data dir", zap.String("dir", s.cfg.Dir)) - if err := os.Chmod(cfg.Dir, 0700); err != nil { - log.Error("chmod failed", zap.String("dir", s.cfg.Dir), zap.Error(err)) - } - cfg.Logger = "zap" - cfg.Debug = s.cfg.Debug - cfg.ZapLoggerBuilder = func(c *embed.Config) error { - l := log.NewLoggerWithLevel("etcd", s.cfg.EtcdLogLevel) - return embed.NewZapCoreLoggerBuilder(l, l.Core(), zapcore.AddSync(os.Stderr))(c) - } - cfg.AutoCompactionMode = embed.CompactorModePeriodic - cfg.LPUrls = []url.URL{s.cfg.PeerURL} - cfg.APUrls = []url.URL{s.cfg.PeerURL} - cfg.LCUrls = []url.URL{s.cfg.ClientURL} - if s.cfg.EnableLocalListener { - _, port, _ := netutil.SplitHostPort(s.cfg.ClientURL.Host) - cfg.LCUrls = append(cfg.LCUrls, url.URL{Scheme: s.cfg.ClientSecurity.Scheme(), Host: fmt.Sprintf("127.0.0.1:%d", port)}) - } - cfg.ACUrls = []url.URL{s.cfg.ClientURL} - cfg.ClientAutoTLS = s.cfg.ClientSecurity.AutoTLS - cfg.PeerAutoTLS = s.cfg.PeerSecurity.AutoTLS - if s.cfg.ClientSecurity.Enabled() { - cfg.ClientTLSInfo = s.cfg.ClientSecurity.TLSInfo() - } - if s.cfg.PeerSecurity.Enabled() { - cfg.PeerTLSInfo = s.cfg.PeerSecurity.TLSInfo() - } - cfg.EnableV2 = false - cfg.ClusterState = state - cfg.InitialCluster = initialClusterStringFromPeers(peers) - cfg.StrictReconfigCheck = true - cfg.ServiceRegister = s.cfg.ServiceRegister - - // XXX(chris): not sure about this - clientv3.SetLogger(grpclog.NewLoggerV2(ioutil.Discard, ioutil.Discard, ioutil.Discard)) - - log.Info("starting etcd", - zap.String("name", cfg.Name), - zap.String("dir", s.cfg.Dir), - zap.String("cluster-state", cfg.ClusterState), - zap.String("initial-cluster", cfg.InitialCluster), - zap.Int("required-cluster-size", s.cfg.RequiredClusterSize), - zap.Bool("debug", s.cfg.Debug), - ) - var err error - s.Etcd, err = embed.StartEtcd(cfg) - if err != nil { - return err - } - select { - case <-s.Server.ReadyNotify(): - if err := s.writeClusterInfo(ctx); err != nil { - return errors.Wrap(err, "cannot write cluster-info") - } - log.Debug("write cluster-info successful!") - atomic.StoreUint64(&s.started, 1) - log.Info("Server is ready!") - - go func() { - <-s.Server.StopNotify() - atomic.StoreUint64(&s.started, 0) - }() - return nil - case err := <-s.Err(): - return errors.Wrap(err, "etcd.Server.Start") - case <-ctx.Done(): - s.Server.Stop() - log.Info("Server was unable to start") - return ctx.Err() - } -} - -func (s *server) startNew(ctx context.Context, peers []*Peer) error { - return s.startEtcd(ctx, embed.ClusterStateFlagNew, peers) -} - -func (s *server) joinExisting(ctx context.Context, peers []*Peer) error { - return s.startEtcd(ctx, embed.ClusterStateFlagExisting, peers) -} - -func newSnapshotReadCloser(snapshot backend.Snapshot) io.ReadCloser { - pr, pw := io.Pipe() - go func() { - n, err := snapshot.WriteTo(pw) - if err == nil { - log.Infof("wrote database snapshot out [total bytes: %d]", n) - } - _ = pw.CloseWithError(err) - snapshot.Close() - }() - return pr -} - -func (s *server) createSnapshot(minRevision int64) (io.ReadCloser, int64, int64, error) { - // Get the current revision and compare with the minimum requested revision. - revision := s.Etcd.Server.KV().Rev() - if revision <= minRevision { - return nil, 0, revision, errors.Errorf("member revision too old, wanted %d, received: %d", minRevision, revision) - } - sp := s.Etcd.Server.Backend().Snapshot() - if sp == nil { - return nil, 0, revision, errors.New("no snappy") - } - return newSnapshotReadCloser(sp), sp.Size(), revision, nil -} - -func (s *server) restoreSnapshot(snapshotFilename string, peers []*Peer) error { - if err := validatePeers(peers, s.cfg.RequiredClusterSize); err != nil { - return err - } - snapshotMgr := snapshot.NewV3(nil) - return snapshotMgr.Restore(snapshot.RestoreConfig{ - // SnapshotPath is the path of snapshot file to restore from. - SnapshotPath: snapshotFilename, - - // Name is the human-readable name of this member. - Name: s.cfg.Name, - - // OutputDataDir is the target data directory to save restored data. - // OutputDataDir should not conflict with existing etcd data directory. - // If OutputDataDir already exists, it will return an error to prevent - // unintended data directory overwrites. - // If empty, defaults to "[Name].etcd" if not given. - OutputDataDir: s.cfg.Dir, - - // PeerURLs is a list of member's peer URLs to advertise to the rest of the cluster. - PeerURLs: []string{s.cfg.PeerURL.String()}, - - // InitialCluster is the initial cluster configuration for restore bootstrap. - InitialCluster: initialClusterStringFromPeers(peers), - - InitialClusterToken: embed.NewConfig().InitialClusterToken, - SkipHashCheck: true, - }) -} - -var ( - errCannotFindMember = errors.New("cannot find member") -) - -func (s *server) lookupMember(name string) (uint64, error) { - for _, member := range s.Etcd.Server.Cluster().Members() { - if member.Name == name { - return uint64(member.ID), nil - } - } - return 0, errors.Wrap(errCannotFindMember, name) -} - -func (s *server) lookupMemberNameByPeerAddr(addr string) (string, error) { - for _, member := range s.Etcd.Server.Cluster().Members() { - for _, url := range member.PeerURLs { - if url == addr { - return member.Name, nil - } - } - } - return "", errors.Wrap(errCannotFindMember, addr) -} - -func (s *server) removeMember(ctx context.Context, name string) error { - id, err := s.lookupMember(name) - if err != nil { - return err - } - cctx, cancel := context.WithTimeout(ctx, 1*time.Second) - defer cancel() - _, err = s.Server.RemoveMember(cctx, id) - if err != nil && err != membership.ErrIDRemoved { - return errors.Errorf("cannot remove member %#v: %v", name, err) - } - return nil -} - -type Cluster struct { - ID int `e2db:"id"` - Created time.Time - RequiredClusterSize int -} - -// writeClusterInfo attempts to write basic cluster info whenever a server -// starts or joins a new cluster. The e2db namespace matches the volatile -// prefix so that this information will not survive being restored from -// snapshot. This is because the cluster requirements could change for the -// restored cluster (e.g. going from RequiredClusterSize 1 -> 3). -func (s *server) writeClusterInfo(ctx context.Context) error { - // NOTE(chrism): As the naming can be confusing it is worth pointing out - // that the ClientSecurity field is specifying the server certs and NOT the - // client certs. Since the server certs do not have client auth key usage, - // we need to use the peer certs here (they have client auth key usage). - db, err := e2db.New(ctx, &e2db.Config{ - ClientAddr: s.cfg.ClientURL.String(), - CAFile: s.cfg.PeerSecurity.TrustedCAFile, - CertFile: s.cfg.PeerSecurity.CertFile, - KeyFile: s.cfg.PeerSecurity.KeyFile, - Namespace: string(volatilePrefix), - }) - if err != nil { - return err - } - defer db.Close() - - return db.Table(new(Cluster)).Tx(func(tx *e2db.Tx) error { - var cluster *Cluster - if err := tx.Find("ID", 1, &cluster); err != nil && errors.Cause(err) != e2db.ErrNoRows { - return err - } - - if cluster != nil { - // check RequiredClusterSize for discrepancies - if cluster.RequiredClusterSize != s.cfg.RequiredClusterSize { - return errors.Errorf("server %s attempted to join cluster with incorrect RequiredClusterSize, cluster expects %d, this server is configured with %d", s.cfg.Name, cluster.RequiredClusterSize, s.cfg.RequiredClusterSize) - } - return nil - } - - return tx.Insert(&Cluster{ - ID: 1, - Created: time.Now(), - RequiredClusterSize: s.cfg.RequiredClusterSize, - }) - }) -} - -var ( - // volatilePrefix is the key prefix used for keys that will NOT be - // preserved after a cluster is recovered from snapshot - volatilePrefix = []byte("/_e2d") - - // snapshotMarkerKey is the key used to indicate when a cluster recovered - // from snapshot - snapshotMarkerKey = []byte("/_e2d/snapshot") -) - -var errServerStopped = errors.New("server stopped") - -func (s *server) clearVolatilePrefix() (rev, deleted int64, err error) { - s.mu.Lock() - defer s.mu.Unlock() - - if !s.isRunning() { - return 0, 0, errServerStopped - } - res, err := s.Server.KV().Range(volatilePrefix, []byte{}, mvcc.RangeOptions{}) - if err != nil { - return 0, 0, err - } - for _, kv := range res.KVs { - if bytes.HasPrefix(kv.Key, volatilePrefix) { - n, _ := s.Server.KV().DeleteRange(kv.Key, nil) - deleted += n - } - } - return res.Rev, deleted, nil -} - -func (s *server) placeSnapshotMarker(v []byte) (int64, error) { - s.mu.Lock() - defer s.mu.Unlock() - - if !s.isRunning() { - return 0, errServerStopped - } - rev := s.Server.KV().Put(snapshotMarkerKey, v, lease.NoLease) - return rev, nil -} diff --git a/pkg/manager/service.go b/pkg/manager/service.go index d4ebc20..7685565 100644 --- a/pkg/manager/service.go +++ b/pkg/manager/service.go @@ -8,6 +8,7 @@ import ( "github.com/criticalstack/e2d/pkg/client" "github.com/criticalstack/e2d/pkg/e2db" + "github.com/criticalstack/e2d/pkg/etcdserver" "github.com/criticalstack/e2d/pkg/log" "github.com/criticalstack/e2d/pkg/manager/e2dpb" ) @@ -21,24 +22,24 @@ func (s *ManagerService) Health(ctx context.Context, _ *types.Empty) (*e2dpb.Hea Status: "not great, bob", } db, err := e2db.New(ctx, &e2db.Config{ - ClientAddr: s.m.cfg.ClientURL.String(), - CAFile: s.m.cfg.PeerSecurity.TrustedCAFile, - CertFile: s.m.cfg.PeerSecurity.CertFile, - KeyFile: s.m.cfg.PeerSecurity.KeyFile, - Namespace: string(volatilePrefix), + ClientAddr: s.m.etcd.ClientURL.String(), + CAFile: s.m.etcd.PeerSecurity.TrustedCAFile, + CertFile: s.m.etcd.PeerSecurity.CertFile, + KeyFile: s.m.etcd.PeerSecurity.KeyFile, + Namespace: string(etcdserver.VolatilePrefix), }) if err != nil { return resp, err } defer db.Close() - var cluster *Cluster - if err := db.Table(new(Cluster)).Find("ID", 1, &cluster); err != nil { + var cluster *etcdserver.Cluster + if err := db.Table(new(etcdserver.Cluster)).Find("ID", 1, &cluster); err != nil { return resp, err } c, err := client.New(&client.Config{ - ClientURLs: []string{s.m.cfg.ClientURL.String()}, - SecurityConfig: s.m.cfg.PeerSecurity, + ClientURLs: []string{s.m.etcd.ClientURL.String()}, + SecurityConfig: s.m.etcd.PeerSecurity, }) if err != nil { return resp, err @@ -60,7 +61,7 @@ func (s *ManagerService) Restart(ctx context.Context, _ *types.Empty) (*e2dpb.Re resp := &e2dpb.RestartResponse{ Msg: "attempting restarting ...", } - if s.m.etcd.isRestarting() { + if s.m.etcd.IsRestarting() { resp.Msg = "a restart is already in progress" return resp, nil } diff --git a/pkg/manager/snapshotter.go b/pkg/manager/snapshotter.go new file mode 100644 index 0000000..c88b27b --- /dev/null +++ b/pkg/manager/snapshotter.go @@ -0,0 +1,98 @@ +package manager + +import ( + "time" + + "github.com/pkg/errors" + "go.uber.org/zap" + + configv1alpha1 "github.com/criticalstack/e2d/pkg/config/v1alpha1" + "github.com/criticalstack/e2d/pkg/log" + "github.com/criticalstack/e2d/pkg/snapshot" + snapshotutil "github.com/criticalstack/e2d/pkg/snapshot/util" +) + +func (m *Manager) runSnapshotter() { + if m.snapshotter == nil { + log.Info("snapshotting disabled: no snapshot backup set") + return + } + log.Debug("starting snapshotter") + ticker := time.NewTicker(m.cfg.SnapshotConfiguration.Interval.Duration) + defer ticker.Stop() + + var latestRev int64 + + for { + select { + case <-ticker.C: + if m.etcd.IsRestarting() { + log.Debug("server is restarting, skipping snapshot backup") + continue + } + if !m.etcd.IsLeader() { + log.Debug("not leader, skipping snapshot backup") + continue + } + log.Debug("starting snapshot backup") + snapshotData, snapshotSize, rev, err := m.etcd.CreateSnapshot(latestRev) + if err != nil { + log.Debug("cannot create snapshot", + zap.String("name", shortName(m.etcd.Name)), + zap.Error(err), + ) + continue + } + if m.cfg.SnapshotConfiguration.Encryption { + snapshotData = snapshotutil.NewEncrypterReadCloser(snapshotData, m.snapshotEncryptionKey, snapshotSize) + } + if m.cfg.SnapshotConfiguration.Compression { + snapshotData = snapshotutil.NewGzipReadCloser(snapshotData) + } + if err := m.snapshotter.Save(snapshotData); err != nil { + log.Debug("cannot save snapshot", + zap.String("name", shortName(m.etcd.Name)), + zap.Error(err), + ) + continue + } + latestRev = rev + log.Infof("wrote snapshot (rev %d) to backup", latestRev) + case <-m.ctx.Done(): + log.Debug("stopping snapshotter") + return + } + } +} + +func getSnapshotProvider(cfg configv1alpha1.SnapshotConfiguration) (snapshot.Snapshotter, error) { + if cfg.File == "" { + return nil, nil + } + u, err := snapshot.ParseSnapshotBackupURL(cfg.File) + if err != nil { + return nil, err + } + + switch u.Type { + case snapshot.FileType: + return snapshot.NewFileSnapshotter(u.Path) + case snapshot.S3Type: + awscfg := &snapshot.AmazonConfig{ + Bucket: u.Bucket, + Key: u.Path, + } + if v, ok := cfg.ExtraArgs["RoleSessionName"]; ok { + awscfg.RoleSessionName = v + } + return snapshot.NewAmazonSnapshotter(awscfg) + case snapshot.SpacesType: + return snapshot.NewDigitalOceanSnapshotter(&snapshot.DigitalOceanConfig{ + SpacesURL: cfg.File, + //SpacesAccessKey: opts.DOSpacesKey, + //SpacesSecretKey: opts.DOSpacesSecret, + }) + default: + return nil, errors.Errorf("unsupported snapshot url format: %#v", cfg.File) + } +} diff --git a/pkg/manager/utils.go b/pkg/manager/utils.go new file mode 100644 index 0000000..fd386f2 --- /dev/null +++ b/pkg/manager/utils.go @@ -0,0 +1,64 @@ +package manager + +import ( + "encoding/json" + "math/rand" + "net/url" + "strings" + "time" + + "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" + "go.uber.org/zap" + + "github.com/criticalstack/e2d/pkg/log" +) + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +// shortName returns a shorter, lowercase version of the node name. The intent +// is to make log reading easier. +func shortName(name string) string { + if len(name) > 5 { + name = name[:5] + } + return strings.ToLower(name) +} + +func getExistingNameFromDataDir(path string, peerURL url.URL) (string, error) { + db, err := bolt.Open(path, 0600, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + return "", err + } + defer db.Close() + + var name string + err = db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("members")) + if b == nil { + return errors.New("existing name not found") + } + c := b.Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + var m struct { + ID uint64 `json:"id"` + Name string `json:"name"` + PeerURLs []string `json:"peerURLs"` + } + if err := json.Unmarshal(v, &m); err != nil { + log.Error("cannot unmarshal etcd member", zap.Error(err)) + continue + } + for _, u := range m.PeerURLs { + if u == peerURL.String() { + name = m.Name + return nil + } + } + } + return errors.New("existing name not found") + }) + return name, err +} diff --git a/pkg/pki/pki.go b/pkg/pki/pki.go deleted file mode 100644 index 631f824..0000000 --- a/pkg/pki/pki.go +++ /dev/null @@ -1,211 +0,0 @@ -package pki - -import ( - "crypto" - "crypto/sha256" - "crypto/x509" - "encoding/pem" - "io/ioutil" - "time" - - "github.com/cloudflare/cfssl/cli/genkey" - "github.com/cloudflare/cfssl/config" - "github.com/cloudflare/cfssl/csr" - "github.com/cloudflare/cfssl/helpers" - "github.com/cloudflare/cfssl/initca" - clog "github.com/cloudflare/cfssl/log" - "github.com/cloudflare/cfssl/signer" - "github.com/cloudflare/cfssl/signer/local" - "github.com/criticalstack/e2d/pkg/log" - "github.com/pkg/errors" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -const ( - ClientSigningProfile = "client" - PeerSigningProfile = "peer" - ServerSigningProfile = "server" -) - -var ( - SigningProfiles = &config.Signing{ - Default: &config.SigningProfile{ - Expiry: 5 * 365 * 24 * time.Hour, - }, - Profiles: map[string]*config.SigningProfile{ - ClientSigningProfile: { - Expiry: 5 * 365 * 24 * time.Hour, - Usage: []string{ - "signing", - "key encipherment", - "client auth", - }, - }, - PeerSigningProfile: { - Expiry: 5 * 365 * 24 * time.Hour, - Usage: []string{ - "signing", - "key encipherment", - "server auth", - "client auth", - }, - }, - ServerSigningProfile: { - Expiry: 5 * 365 * 24 * time.Hour, - Usage: []string{ - "signing", - "key encipherment", - "server auth", - }, - }, - }, - } -) - -type nopLogger struct { -} - -func (nopLogger) Debug(msg string) {} -func (nopLogger) Info(msg string) {} -func (nopLogger) Warning(msg string) {} -func (nopLogger) Err(msg string) {} -func (nopLogger) Crit(msg string) {} -func (nopLogger) Emerg(msg string) {} - -type logger struct { - l *zap.Logger -} - -func (l *logger) Debug(msg string) { l.l.Debug(msg) } -func (l *logger) Info(msg string) { l.l.Info(msg) } -func (l *logger) Warning(msg string) { l.l.Warn(msg) } -func (l *logger) Err(msg string) { l.l.Error(msg) } -func (l *logger) Crit(msg string) { l.l.Error(msg) } -func (l *logger) Emerg(msg string) { l.l.Fatal(msg) } - -func init() { - clog.SetLogger(&logger{log.NewLoggerWithLevel("cfssl", zapcore.ErrorLevel)}) -} - -type KeyPair struct { - Cert *x509.Certificate - CertPEM []byte - Key crypto.Signer - KeyPEM []byte -} - -func NewKeyPairFromPEM(certPEM, keyPEM []byte) (*KeyPair, error) { - cert, err := helpers.ParseCertificatePEM(certPEM) - if err != nil { - return nil, err - } - key, err := helpers.ParsePrivateKeyPEM(keyPEM) - if err != nil { - return nil, err - } - return &KeyPair{ - Cert: cert, - CertPEM: certPEM, - Key: key, - KeyPEM: keyPEM, - }, nil -} - -type RootCA struct { - CA *KeyPair - g *csr.Generator - sp *config.Signing -} - -func NewRootCA(cr *csr.CertificateRequest) (*RootCA, error) { - certPEM, _, keyPEM, err := initca.New(cr) - if err != nil { - return nil, err - } - ca, err := NewKeyPairFromPEM(certPEM, keyPEM) - if err != nil { - return nil, err - } - r := &RootCA{ - CA: ca, - g: &csr.Generator{Validator: genkey.Validator}, - sp: SigningProfiles, - } - return r, nil -} - -func NewRootCAFromFile(certpath, keypath string) (*RootCA, error) { - certPEM, err := ioutil.ReadFile(certpath) - if err != nil { - return nil, err - } - keyPEM, err := ioutil.ReadFile(keypath) - if err != nil { - return nil, err - } - ca, err := NewKeyPairFromPEM(certPEM, keyPEM) - if err != nil { - return nil, err - } - r := &RootCA{ - CA: ca, - g: &csr.Generator{Validator: genkey.Validator}, - sp: SigningProfiles, - } - return r, nil -} - -func NewDefaultRootCA() (*RootCA, error) { - return NewRootCA(&csr.CertificateRequest{ - Names: []csr.Name{ - { - C: "US", - ST: "Boston", - L: "MA", - O: "Critical Stack", - }, - }, - KeyRequest: &csr.KeyRequest{ - A: "rsa", - S: 2048, - }, - CN: "e2d-ca", - }) -} - -func (r *RootCA) GenerateCertificates(profile string, cr *csr.CertificateRequest) (*KeyPair, error) { - csrBytes, keyPEM, err := r.g.ProcessRequest(cr) - if err != nil { - return nil, err - } - s, err := local.NewSigner(r.CA.Key, r.CA.Cert, signer.DefaultSigAlgo(r.CA.Key), r.sp) - if err != nil { - return nil, err - } - certPEM, err := s.Sign(signer.SignRequest{ - Request: string(csrBytes), - Profile: profile, - }) - if err != nil { - return nil, err - } - return NewKeyPairFromPEM(certPEM, keyPEM) -} - -func GenerateCertHash(caCertPath string) ([]byte, error) { - data, err := ioutil.ReadFile(caCertPath) - if err != nil { - return nil, err - } - block, _ := pem.Decode(data) - if block == nil { - return nil, errors.New("cannot parse PEM formatted block") - } - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return nil, err - } - h := sha256.Sum256(cert.RawSubjectPublicKeyInfo) - return h[:], nil -} diff --git a/pkg/pki/pki_test.go b/pkg/pki/pki_test.go deleted file mode 100644 index 80f4070..0000000 --- a/pkg/pki/pki_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package pki - -import ( - "fmt" - "testing" - - "github.com/cloudflare/cfssl/csr" -) - -func TestGenerateCertificates(t *testing.T) { - hosts := []string{"10.10.0.1", "10.10.0.2"} - - r, err := NewDefaultRootCA() - if err != nil { - t.Fatal(err) - } - - kp, err := r.GenerateCertificates(PeerSigningProfile, &csr.CertificateRequest{ - Names: []csr.Name{ - { - C: "US", - ST: "Boston", - L: "MA", - }, - }, - KeyRequest: &csr.KeyRequest{ - A: "rsa", - S: 2048, - }, - Hosts: hosts, - CN: "etcd peer", - }) - if err != nil { - t.Fatal(err) - } - fmt.Printf("kp.certPEM = %s\n", kp.CertPEM) - fmt.Printf("kp.keyPEM = %s\n", kp.KeyPEM) -} diff --git a/pkg/snapshot/snapshot_aws.go b/pkg/snapshot/snapshot_aws.go index 228316e..7b492cc 100644 --- a/pkg/snapshot/snapshot_aws.go +++ b/pkg/snapshot/snapshot_aws.go @@ -12,8 +12,9 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - e2daws "github.com/criticalstack/e2d/pkg/provider/aws" "github.com/pkg/errors" + + e2daws "github.com/criticalstack/e2d/internal/provider/aws" ) func newAWSConfig(name string) (*aws.Config, error) { diff --git a/pkg/cmdutil/env.go b/pkg/util/env/env.go similarity index 99% rename from pkg/cmdutil/env.go rename to pkg/util/env/env.go index 5f6488f..9920855 100644 --- a/pkg/cmdutil/env.go +++ b/pkg/util/env/env.go @@ -1,4 +1,4 @@ -package cmdutil +package env import ( "os" @@ -10,8 +10,27 @@ import ( "github.com/pkg/errors" ) -func isTimeDuration(v reflect.Value) bool { - return v.Kind() == reflect.Int64 && v.Type().PkgPath() == "time" && v.Type().Name() == "Duration" +// SetEnvs sets field values for the provided struct passed in based on +// environment variables. Fields are mapped to environment variables using the +// `env` struct tag. Non-tagged fields are skipped. +func SetEnvs(iface interface{}) error { + v := reflect.Indirect(reflect.ValueOf(iface)) + if v.Kind() != reflect.Struct { + return errors.Errorf("expected struct, received %v", v.Type()) + } + for i := 0; i < v.Type().NumField(); i++ { + fv := v.Field(i) + tv, ok := v.Type().Field(i).Tag.Lookup("env") + if !ok { + continue + } + if v, ok := os.LookupEnv(strings.ToUpper(tv)); ok { + if err := setValue(fv, v); err != nil { + return err + } + } + } + return nil } func setValue(v reflect.Value, s string) error { @@ -51,25 +70,6 @@ func setValue(v reflect.Value, s string) error { return nil } -// SetEnvs sets field values for the provided struct passed in based on -// environment variables. Fields are mapped to environment variables using the -// `env` struct tag. Non-tagged fields are skipped. -func SetEnvs(iface interface{}) error { - v := reflect.Indirect(reflect.ValueOf(iface)) - if v.Kind() != reflect.Struct { - return errors.Errorf("expected struct, received %v", v.Type()) - } - for i := 0; i < v.Type().NumField(); i++ { - fv := v.Field(i) - tv, ok := v.Type().Field(i).Tag.Lookup("env") - if !ok { - continue - } - if v, ok := os.LookupEnv(strings.ToUpper(tv)); ok { - if err := setValue(fv, v); err != nil { - return err - } - } - } - return nil +func isTimeDuration(v reflect.Value) bool { + return v.Kind() == reflect.Int64 && v.Type().PkgPath() == "time" && v.Type().Name() == "Duration" } diff --git a/pkg/cmdutil/env_test.go b/pkg/util/env/env_test.go similarity index 98% rename from pkg/cmdutil/env_test.go rename to pkg/util/env/env_test.go index 2b2f421..16d126e 100644 --- a/pkg/cmdutil/env_test.go +++ b/pkg/util/env/env_test.go @@ -1,4 +1,4 @@ -package cmdutil +package env import ( "os" diff --git a/pkg/netutil/netutil.go b/pkg/util/net/net.go similarity index 97% rename from pkg/netutil/netutil.go rename to pkg/util/net/net.go index 6c07ee0..1ac8085 100644 --- a/pkg/netutil/netutil.go +++ b/pkg/util/net/net.go @@ -1,4 +1,4 @@ -package netutil +package net import ( "fmt" @@ -42,9 +42,6 @@ func SplitHostPort(addr string) (string, int, error) { if err != nil { return "", 0, err } - if host == "" { - host = "127.0.0.1" - } p, _ := strconv.Atoi(port) return host, p, nil } diff --git a/pkg/netutil/netutil_test.go b/pkg/util/net/net_test.go similarity index 96% rename from pkg/netutil/netutil_test.go rename to pkg/util/net/net_test.go index b7e11b2..02b642b 100644 --- a/pkg/netutil/netutil_test.go +++ b/pkg/util/net/net_test.go @@ -1,4 +1,4 @@ -package netutil +package net import "testing" diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000..439d4be --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,53 @@ +#!/bin/sh + +# Copyright 2020 Critical Stack, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +USER="criticalstack" +REPO="e2d" +VERSION=$(curl -s "https://api.github.com/repos/${USER}/${REPO}/releases/latest" | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/') +OS=$(uname) +ARCH=$(uname -m) +FILENAME="${REPO}_${VERSION}_${OS}_${ARCH}.tar.gz" +TMP_DIR="${REPO}_tmp" +INSTALL_DIR=/usr/local/bin + +# Download archive from GitHub Releases +curl -sLO -w '' "https://github.com/${USER}/${REPO}/releases/download/v${VERSION}/${FILENAME}" + +if [ $? -ne 0 ]; then + echo "Failed to download ${USER}/${REPO}" + exit 1 +fi + +# Unpack into tmp directory +mkdir -p ${TMP_DIR} +tar xzf ${FILENAME} -C ${TMP_DIR} + +if [ $? -ne 0 ]; then + echo "Failed to unpack ${FILENAME}" + exit 1 +fi + +# Install any executables +for f in ${TMP_DIR}/*; do + if [ -x $f ]; then + sudo mv $f $INSTALL_DIR + fi +done + +# Cleanup +rm -f ${TMP_DIR}/* +rmdir ${TMP_DIR} +echo "${REPO} has been installed!"