-
Notifications
You must be signed in to change notification settings - Fork 3.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
cmd: replacing release-20.1 version of roachprod, roachtest, workload…
…(ccl) This change does the following: - Deletes release-20.1 roachprod, roachtest, workload and workloadccl packages. - Adds the above packages from master. - Make all the above packages compatible with 20.1. I have highlighted some of the major changes in each package and will inline links to why they were changed on master. I took some decisions on what required a backport and what looked too invasive, but am open to suggestions here. roachtest: - Deleted roachtests with min version 20.2.0 namely: - backup/KMS - join-init/mixed - disk_full - multitenant acceptance - tpchvec/bench - tpchvec/perf_no_stats - Backported `tpcdsvec` roachtest workload: - Backported `schemachange` workload - deleted geospatial.go - switched all `rowenc` pkg references back to `sqlbase` - type representation in 20.2 differed from what was supported in 20.1. All workloads had to be switched back to using `colTypes` workloadccl: - More type representation related changes - Set --deprecated-fk-indexes to be always true for tpcc workload Release note: None
- Loading branch information
1 parent
4ea9b3b
commit 6e90d0b
Showing
226 changed files
with
12,549 additions
and
6,060 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| #!/usr/bin/env bash | ||
| set -euo pipefail | ||
|
|
||
| bin/roachtest run \ | ||
| --cloud="${CLOUD}" \ | ||
| --artifacts="${ARTIFACTS}" \ | ||
| --parallelism="${PARALLELISM}" \ | ||
| --cpu-quota="${CPUQUOTA}" \ | ||
| --zones="${ZONES}" \ | ||
| --count="${COUNT-1}" \ | ||
| --debug="${DEBUG-false}" \ | ||
| --build-tag="${BUILD_TAG}" \ | ||
| --cockroach="${COCKROACH_BINARY}" \ | ||
| --roachprod="${PWD}/bin/roachprod" \ | ||
| --workload="${PWD}/bin/workload" \ | ||
| --teamcity=true \ | ||
| --slack-token="${SLACK_TOKEN}" \ | ||
| --cluster-id="${TC_BUILD_ID}" \ | ||
| "${TESTS}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| #!/usr/bin/env bash | ||
| set -euxo pipefail | ||
|
|
||
| # Entry point for the nightly roachtests. These are run from CI and require | ||
| # appropriate secrets for the ${CLOUD} parameter (along with other things, | ||
| # apologies, you're going to have to dig around for them below or even better | ||
| # yet, look at the job). | ||
|
|
||
| # Note that when this script is called, the cockroach binary to be tested | ||
| # already exists in the current directory. | ||
| COCKROACH_BINARY="${PWD}/cockroach.linux-2.6.32-gnu-amd64" | ||
| chmod +x "${COCKROACH_BINARY}" | ||
|
|
||
| if [[ ! -f ~/.ssh/id_rsa.pub ]]; then | ||
| ssh-keygen -q -C "roachtest-nightly $(date)" -N "" -f ~/.ssh/id_rsa | ||
| fi | ||
|
|
||
| # The artifacts dir should match up with that supplied by TC. | ||
| artifacts=$PWD/artifacts | ||
| mkdir -p "${artifacts}" | ||
| chmod o+rwx "${artifacts}" | ||
|
|
||
| # Disable global -json flag. | ||
| export PATH=$PATH:$(GOFLAGS=; go env GOPATH)/bin | ||
|
|
||
| make bin/workload bin/roachtest bin/roachprod > "${artifacts}/build.txt" 2>&1 || cat "${artifacts}/build.txt" | ||
|
|
||
| # Set up Google credentials. Note that we need this for all clouds since we upload | ||
| # perf artifacts to Google Storage at the end. | ||
| if [[ "$GOOGLE_EPHEMERAL_CREDENTIALS" ]]; then | ||
| echo "$GOOGLE_EPHEMERAL_CREDENTIALS" > creds.json | ||
| gcloud auth activate-service-account --key-file=creds.json | ||
| export ROACHPROD_USER=teamcity | ||
| else | ||
| echo 'warning: GOOGLE_EPHEMERAL_CREDENTIALS not set' >&2 | ||
| echo "Assuming that you've run \`gcloud auth login\` from inside the builder." >&2 | ||
| fi | ||
|
|
||
| # Early bind the stats dir. Roachtest invocations can take ages, and we want the | ||
| # date at the time of the start of the run (which identifies the version of the | ||
| # code run best). | ||
| stats_dir="$(date +"%Y%m%d")-${TC_BUILD_ID}" | ||
|
|
||
| # Set up a function we'll invoke at the end. | ||
| function upload_stats { | ||
| if [[ "${TC_BUILD_BRANCH}" == "master" ]]; then | ||
| bucket="cockroach-nightly-${CLOUD}" | ||
| if [[ "${CLOUD}" == "gce" ]]; then | ||
| # GCE, having been there first, gets an exemption. | ||
| bucket="cockroach-nightly" | ||
| fi | ||
| # The stats.json files need some path translation: | ||
| # ${artifacts}/path/to/test/stats.json | ||
| # to | ||
| # gs://${bucket}/artifacts/${stats_dir}/path/to/test/stats.json | ||
| # | ||
| # `find` below will expand "{}" as ./path/to/test/stats.json. We need | ||
| # to bend over backwards to remove the `./` prefix or gsutil will have | ||
| # a `.` folder in ${stats_dir}, which we don't want. | ||
| (cd "${artifacts}" && \ | ||
| while IFS= read -r f; do | ||
| if [[ -n "${f}" ]]; then | ||
| gsutil cp "${f}" "gs://${bucket}/artifacts/${stats_dir}/${f}" | ||
| fi | ||
| done <<< "$(find . -name stats.json | sed 's/^\.\///')") | ||
| fi | ||
| } | ||
|
|
||
| # Upload any stats.json we can find, no matter what happens. | ||
| trap upload_stats EXIT | ||
|
|
||
| # Set up the parameters for the roachtest invocation. | ||
|
|
||
| ARTIFACTS="${artifacts}" | ||
| PARALLELISM=16 | ||
| CPUQUOTA=1024 | ||
| ZONES="" | ||
| TESTS="" | ||
| case "${CLOUD}" in | ||
| gce) | ||
| # We specify --zones below so that nodes are created in us-central1-b by | ||
| # default. This reserves us-east1-b (the roachprod default zone) for use by | ||
| # manually created clusters. | ||
| ZONES="us-central1-b,us-west1-b,europe-west2-b" | ||
| ;; | ||
| aws) | ||
| PARALLELISM=3 | ||
| CPUQUOTA=384 | ||
| if [ -z "${TESTS}" ]; then | ||
| TESTS="kv(0|95)|ycsb|tpcc/(headroom/n4cpu16)|tpccbench/(nodes=3/cpu=16)|scbench/randomload/(nodes=3/ops=2000/conc=1)|backup/(KMS/n3cpu4)" | ||
| fi | ||
| ;; | ||
| *) | ||
| echo "unknown cloud ${CLOUD}" | ||
| exit 1 | ||
| ;; | ||
| esac | ||
|
|
||
| export \ | ||
| CLOUD="${CLOUD}" \ | ||
| ARTIFACTS="${ARTIFACTS}" \ | ||
| PARALLELISM="${PARALLELISM}" \ | ||
| CPUQUOTA="${CPUQUOTA}" \ | ||
| ZONES="${ZONES}" \ | ||
| COUNT="${COUNT-1}" \ | ||
| DEBUG="${DEBUG-false}" \ | ||
| BUILD_TAG="${BUILD_TAG}" \ | ||
| COCKROACH_BINARY="${COCKROACH_BINARY}" \ | ||
| SLACK_TOKEN="${SLACK_TOKEN}" \ | ||
| TC_BUILD_ID="${TC_BUILD_ID}" \ | ||
| TESTS="${TESTS}" | ||
|
|
||
| # Teamcity has a 1300 minute timeout that, when reached, kills the process | ||
| # without a stack trace (probably SIGKILL). We'd love to see a stack trace | ||
| # though, so after 1200 minutes, kill with SIGINT which will allow roachtest to | ||
| # fail tests and cleanup. | ||
| timeout -s INT $((1200*60)) "build/teamcity-nightly-roachtest-invoke.sh" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.