Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master' into horizon-planning
Browse files Browse the repository at this point in the history
Signed-off-by: Andres Taylor <[email protected]>
  • Loading branch information
systay committed Jan 11, 2021
2 parents 26b008b + bb3c7dd commit 35e41cd
Show file tree
Hide file tree
Showing 63 changed files with 8,043 additions and 5,658 deletions.
1 change: 1 addition & 0 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,4 @@ Components that this PR will affect:
- [ ] VReplication
- [ ] Cluster Management
- [ ] Build
- [ ] VTAdmin
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ endif
$(PROTO_GO_OUTS): install_protoc-gen-go proto/*.proto
for name in $(PROTO_SRC_NAMES); do \
$(VTROOT)/bin/protoc --go_out=plugins=grpc:. -Iproto proto/$${name}.proto && \
goimports -w vitess.io/vitess/go/vt/proto/$${name}/$${name}.pb.go; \
goimports -local vitess.io/vitess -w vitess.io/vitess/go/vt/proto/$${name}/$${name}.pb.go; \
done
cp -Rf vitess.io/vitess/go/vt/proto/* go/vt/proto
rm -rf vitess.io/vitess/go/vt/proto/
Expand Down
41 changes: 28 additions & 13 deletions docker/lite/install_dependencies.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,19 @@ add_apt_key() {
done
}

# Set number of times to retry a download
MAX_RETRY=20

do_fetch() {
wget \
--tries=$MAX_RETRY\
--read-timeout=30\
--timeout=30\
--retry-connrefused\
--waitretry=1\
--no-dns-cache \
$1 -O $2
}

# Install base packages that are common to all flavors.
BASE_PACKAGES=(
Expand Down Expand Up @@ -63,11 +76,11 @@ mysql56)
;;
mysql57)
mysql57_version=5.7.31
wget https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/libmysqlclient20_${mysql57_version}-1debian10_amd64.deb -O /tmp/libmysqlclient20_${mysql57_version}-1debian10_amd64.deb
wget https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-community-client_${mysql57_version}-1debian10_amd64.deb -O /tmp/mysql-community-client_${mysql57_version}-1debian10_amd64.deb
wget https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-client_${mysql57_version}-1debian10_amd64.deb -O /tmp/mysql-client_${mysql57_version}-1debian10_amd64.deb
wget https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-community-server_${mysql57_version}-1debian10_amd64.deb -O /tmp/mysql-community-server_${mysql57_version}-1debian10_amd64.deb
wget https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-server_${mysql57_version}-1debian10_amd64.deb -O /tmp/mysql-server_${mysql57_version}-1debian10_amd64.deb
do_fetch https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/libmysqlclient20_${mysql57_version}-1debian10_amd64.deb /tmp/libmysqlclient20_${mysql57_version}-1debian10_amd64.deb
do_fetch https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-community-client_${mysql57_version}-1debian10_amd64.deb /tmp/mysql-community-client_${mysql57_version}-1debian10_amd64.deb
do_fetch https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-client_${mysql57_version}-1debian10_amd64.deb /tmp/mysql-client_${mysql57_version}-1debian10_amd64.deb
do_fetch https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-community-server_${mysql57_version}-1debian10_amd64.deb /tmp/mysql-community-server_${mysql57_version}-1debian10_amd64.deb
do_fetch https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-server_${mysql57_version}-1debian10_amd64.deb /tmp/mysql-server_${mysql57_version}-1debian10_amd64.deb
PACKAGES=(
/tmp/libmysqlclient20_${mysql57_version}-1debian10_amd64.deb
/tmp/mysql-community-client_${mysql57_version}-1debian10_amd64.deb
Expand All @@ -79,13 +92,13 @@ mysql57)
;;
mysql80)
mysql8_version=8.0.21
wget https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/libmysqlclient21_${mysql8_version}-1debian10_amd64.deb -O /tmp/libmysqlclient21_${mysql8_version}-1debian10_amd64.deb
wget https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-client-core_${mysql8_version}-1debian10_amd64.deb -O /tmp/mysql-community-client-core_${mysql8_version}-1debian10_amd64.deb
wget https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-client_${mysql8_version}-1debian10_amd64.deb -O /tmp/mysql-community-client_${mysql8_version}-1debian10_amd64.deb
wget https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-client_${mysql8_version}-1debian10_amd64.deb -O /tmp/mysql-client_${mysql8_version}-1debian10_amd64.deb
wget https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-server-core_${mysql8_version}-1debian10_amd64.deb -O /tmp/mysql-community-server-core_${mysql8_version}-1debian10_amd64.deb
wget https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-server_${mysql8_version}-1debian10_amd64.deb -O /tmp/mysql-community-server_${mysql8_version}-1debian10_amd64.deb
wget https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-server_${mysql8_version}-1debian10_amd64.deb -O /tmp/mysql-server_${mysql8_version}-1debian10_amd64.deb
do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/libmysqlclient21_${mysql8_version}-1debian10_amd64.deb /tmp/libmysqlclient21_${mysql8_version}-1debian10_amd64.deb
do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-client-core_${mysql8_version}-1debian10_amd64.deb /tmp/mysql-community-client-core_${mysql8_version}-1debian10_amd64.deb
do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-client_${mysql8_version}-1debian10_amd64.deb /tmp/mysql-community-client_${mysql8_version}-1debian10_amd64.deb
do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-client_${mysql8_version}-1debian10_amd64.deb /tmp/mysql-client_${mysql8_version}-1debian10_amd64.deb
do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-server-core_${mysql8_version}-1debian10_amd64.deb /tmp/mysql-community-server-core_${mysql8_version}-1debian10_amd64.deb
do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-server_${mysql8_version}-1debian10_amd64.deb /tmp/mysql-community-server_${mysql8_version}-1debian10_amd64.deb
do_fetch https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-server_${mysql8_version}-1debian10_amd64.deb /tmp/mysql-server_${mysql8_version}-1debian10_amd64.deb
PACKAGES=(
/tmp/libmysqlclient21_${mysql8_version}-1debian10_amd64.deb
/tmp/mysql-community-client-core_${mysql8_version}-1debian10_amd64.deb
Expand Down Expand Up @@ -200,6 +213,8 @@ percona57)
debconf debconf/frontend select Noninteractive
percona-server-server-5.7 percona-server-server/root_password password 'unused'
percona-server-server-5.7 percona-server-server/root_password_again password 'unused'
percona-server-server-5.7 percona-server-server-5.7/root-pass password 'unused'
percona-server-server-5.7 percona-server-server-5.7/re-root-pass password 'unused'
EOF
;;
percona80)
Expand All @@ -213,7 +228,7 @@ esac

# Install flavor-specific packages
apt-get update
apt-get install -y --no-install-recommends "${PACKAGES[@]}"
for i in $(seq 1 $MAX_RETRY); do apt-get install -y --no-install-recommends "${PACKAGES[@]}" && break; done

# Clean up files we won't need in the final image.
rm -rf /var/lib/apt/lists/*
Expand Down
58 changes: 58 additions & 0 deletions docker/vttestserver/Dockerfile.mysql57
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# Copyright 2021 The Vitess Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# NOTE: We have to build the Vitess binaries from scratch instead of sharing
# a base image because Docker Hub dropped the feature we relied upon to
# ensure images contain the right binaries.

# Use a temporary layer for the build stage.
ARG bootstrap_version=1
ARG image="vitess/bootstrap:${bootstrap_version}-mysql57"

FROM "${image}" AS builder

# Allows docker builds to set the BUILD_NUMBER
ARG BUILD_NUMBER

# Re-copy sources from working tree.
COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess

# Build and install Vitess in a temporary output directory.
USER vitess
RUN make install-testing PREFIX=/vt/install

# Start over and build the final image.
FROM debian:buster-slim

# Install dependencies
COPY docker/lite/install_dependencies.sh /vt/dist/install_dependencies.sh
RUN /vt/dist/install_dependencies.sh mysql57

# Set up Vitess user and directory tree.
RUN groupadd -r vitess && useradd -r -g vitess vitess
RUN mkdir -p /vt/vtdataroot && chown -R vitess:vitess /vt

# Set up Vitess environment (just enough to run pre-built Go binaries)
ENV VTROOT /vt
ENV VTDATAROOT /vt/vtdataroot
ENV PATH $VTROOT/bin:$PATH

# Copy artifacts from builder layer.
COPY --from=builder --chown=vitess:vitess /vt/install /vt

# Create mount point for actual data (e.g. MySQL data dir)
VOLUME /vt/vtdataroot
USER vitess

CMD /vt/bin/vttestserver -port $PORT -keyspaces $KEYSPACES -num_shards $NUM_SHARDS
58 changes: 58 additions & 0 deletions docker/vttestserver/Dockerfile.mysql80
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# Copyright 2021 The Vitess Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# NOTE: We have to build the Vitess binaries from scratch instead of sharing
# a base image because Docker Hub dropped the feature we relied upon to
# ensure images contain the right binaries.

# Use a temporary layer for the build stage.
ARG bootstrap_version=1
ARG image="vitess/bootstrap:${bootstrap_version}-mysql80"

FROM "${image}" AS builder

# Allows docker builds to set the BUILD_NUMBER
ARG BUILD_NUMBER

# Re-copy sources from working tree.
COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess

# Build and install Vitess in a temporary output directory.
USER vitess
RUN make install-testing PREFIX=/vt/install

# Start over and build the final image.
FROM debian:buster-slim

# Install dependencies
COPY docker/lite/install_dependencies.sh /vt/dist/install_dependencies.sh
RUN /vt/dist/install_dependencies.sh mysql80

# Set up Vitess user and directory tree.
RUN groupadd -r vitess && useradd -r -g vitess vitess
RUN mkdir -p /vt/vtdataroot && chown -R vitess:vitess /vt

# Set up Vitess environment (just enough to run pre-built Go binaries)
ENV VTROOT /vt
ENV VTDATAROOT /vt/vtdataroot
ENV PATH $VTROOT/bin:$PATH

# Copy artifacts from builder layer.
COPY --from=builder --chown=vitess:vitess /vt/install /vt

# Create mount point for actual data (e.g. MySQL data dir)
VOLUME /vt/vtdataroot
USER vitess

CMD /vt/bin/vttestserver -port $PORT -keyspaces $KEYSPACES -num_shards $NUM_SHARDS
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/DataDog/datadog-go v2.2.0+incompatible
github.com/GeertJohan/go.rice v1.0.0
github.com/PuerkitoBio/goquery v1.5.1
github.com/aquarapid/vaultlib v0.5.1
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878
github.com/aws/aws-sdk-go v1.28.8
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/aquarapid/vaultlib v0.5.1 h1:vuLWR6bZzLHybjJBSUYPgZlIp6KZ+SXeHLRRYTuk6d4=
github.com/aquarapid/vaultlib v0.5.1/go.mod h1:yT7AlEXtuabkxylOc/+Ulyp18tff1+QjgNLTnFWTlOs=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 h1:G1bPvciwNyF7IUmKXNt9Ak3m6u9DE1rF+RmtIkBpVdA=
Expand Down Expand Up @@ -472,6 +474,8 @@ github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/
github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mch1307/vaultlib v0.5.0 h1:+tI8YCG033aVI+kAKwo0fwrUylFs+wO6DB7DM5qXJzU=
github.com/mch1307/vaultlib v0.5.0/go.mod h1:phFbO1oIDL1xTqUrNXbrAG0VdcYEKP8TNa9FJd7hFic=
github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU=
Expand Down
28 changes: 28 additions & 0 deletions go/cmd/vtgate/plugin_auth_vault.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/*
Copyright 2020 The Vitess Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

// This plugin imports InitAuthServerVault to register the HashiCorp Vault implementation of AuthServer.

import (
"vitess.io/vitess/go/mysql"
"vitess.io/vitess/go/vt/vtgate"
)

func init() {
vtgate.RegisterPluginInitializer(func() { mysql.InitAuthServerVault() })
}
4 changes: 2 additions & 2 deletions go/cmd/vttablet/vttablet.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func main() {
log.Exitf("failed to parse -tablet-path: %v", err)
}

// config and mycnf intializations are intertwined.
// config and mycnf initializations are intertwined.
config, mycnf := initConfig(tabletAlias)

ts := topo.Open()
Expand Down Expand Up @@ -106,7 +106,7 @@ func main() {
VREngine: vreplication.NewEngine(config, ts, tabletAlias.Cell, mysqld),
}
if err := tm.Start(tablet, config.Healthcheck.IntervalSeconds.Get()); err != nil {
log.Exitf("failed to parse -tablet-path: %v", err)
log.Exitf("failed to parse -tablet-path or initialize DB credentials: %v", err)
}
servenv.OnClose(func() {
// Close the tm so that our topo entry gets pruned properly and any
Expand Down
Loading

0 comments on commit 35e41cd

Please sign in to comment.