ci: do not publish to S3

Previously, we published CI and release artifacts to both S3 and GCS. After switching the CDN from CloudFront to Google CDN, we can stop publishing to both locations. This PR removes S3 related code and references from various CI tools. Fixes: RE-342 Epic: none Release note: None
cockroachdb · Dec 29, 2022 · c00dc57 · c00dc57
1 parent b04fee2
commit c00dc57
Show file tree

Hide file tree

Showing 13 changed files with 148 additions and 419 deletions.
diff --git a/build/teamcity/cockroach/post-merge/publish-bleeding-edge.sh b/build/teamcity/cockroach/post-merge/publish-bleeding-edge.sh
@@ -9,18 +9,16 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
 source "$dir/teamcity-support.sh"
 source "$dir/teamcity-bazel-support.sh"
 
-# s3 pushes to the "cockroach" bucket. There is no test/dev bucket fir this build type.
-bucket="cockroach"
 gcs_bucket="cockroach-edge-artifacts-prod"
 # export the variable to avoid shell escaping
 export gcs_credentials="$GCS_CREDENTIALS_PROD"
 
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e gcs_credentials -e bucket=$bucket -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
 bazel build --config ci //pkg/cmd/publish-artifacts
 BAZEL_BIN=$(bazel info bazel-bin --config ci)
 export google_credentials="$gcs_credentials"
 source "build/teamcity-support.sh"  # For log_into_gcloud
 log_into_gcloud
 export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
-$BAZEL_BIN/pkg/cmd/publish-artifacts/publish-artifacts_/publish-artifacts --gcs-bucket="$gcs_bucket" --bucket="$bucket"
+$BAZEL_BIN/pkg/cmd/publish-artifacts/publish-artifacts_/publish-artifacts --gcs-bucket="$gcs_bucket"
 EOF
diff --git a/build/teamcity/internal/cockroach/release/publish/sign_macos_release.sh b/build/teamcity/internal/cockroach/release/publish/sign_macos_release.sh
@@ -14,14 +14,12 @@ trap remove_files_on_exit EXIT
 
 # By default, set dry-run variables
 google_credentials="$GCS_CREDENTIALS_DEV"
-s3_bucket="cockroach-builds-test"
 gcs_bucket="cockroach-release-artifacts-dryrun"
 
 # override dev defaults with production values
 if [[ -z "${DRY_RUN}" ]] ; then
   echo "Setting production variable values"
   google_credentials="$GCS_CREDENTIALS_PROD"
-  s3_bucket="binaries.cockroachdb.com"
   gcs_bucket="cockroach-release-artifacts-prod"
 fi
 
@@ -76,10 +74,6 @@ for product in cockroach cockroach-sql; do
     rm -rf "$base" "$unsigned_file" "$unsigned_file.sha256sum" crl.zip
 
     shasum --algorithm 256 "$target" > "$target.sha256sum"
-    "$BAZEL_BIN/pkg/cmd/cloudupload/cloudupload_/cloudupload" \
-      "$target" "s3://$s3_bucket/$target"
-    "$BAZEL_BIN/pkg/cmd/cloudupload/cloudupload_/cloudupload" \
-      "$target.sha256sum" "s3://$s3_bucket/$target.sha256sum"
     "$BAZEL_BIN/pkg/cmd/cloudupload/cloudupload_/cloudupload" \
       "$target" "gs://$gcs_bucket/$target"
     "$BAZEL_BIN/pkg/cmd/cloudupload/cloudupload_/cloudupload" \

diff --git a/build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh b/build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh
@@ -15,7 +15,6 @@ release_branch="$(echo "$build_name" | grep -Eo "^v[0-9]+\.[0-9]+" || echo"")"
 is_custom_build="$(echo "$TC_BUILD_BRANCH" | grep -Eo "^custombuild-" || echo "")"
 
 if [[ -z "${DRY_RUN}" ]] ; then
-  bucket="cockroach-builds"
   gcs_bucket="cockroach-builds-artifacts-prod"
   google_credentials=$GOOGLE_COCKROACH_CLOUD_IMAGES_COCKROACHDB_CREDENTIALS
   gcr_repository="us-docker.pkg.dev/cockroach-cloud-images/cockroachdb/cockroach"
@@ -24,7 +23,6 @@ if [[ -z "${DRY_RUN}" ]] ; then
   # export the variable to avoid shell escaping
   export gcs_credentials="$GCS_CREDENTIALS_PROD"
 else
-  bucket="cockroach-builds-test"
   gcs_bucket="cockroach-builds-artifacts-dryrun"
   google_credentials="$GOOGLE_COCKROACH_RELEASE_CREDENTIALS"
   gcr_repository="us.gcr.io/cockroach-release/cockroach-test"
@@ -40,7 +38,6 @@ cat << EOF
   build_name:      $build_name
   release_branch:  $release_branch
   is_custom_build: $is_custom_build
-  bucket:          $bucket
   gcs_bucket:      $gcs_bucket
   gcr_repository:  $gcr_repository
 
@@ -54,17 +51,17 @@ tc_start_block "Tag the release"
 git tag "${build_name}"
 tc_end_block "Tag the release"
 
-tc_start_block "Compile and publish S3 artifacts"
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
+tc_start_block "Compile and publish artifacts"
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
 bazel build --config ci //pkg/cmd/publish-provisional-artifacts
 BAZEL_BIN=$(bazel info bazel-bin --config ci)
 export google_credentials="$gcs_credentials"
 source "build/teamcity-support.sh"  # For log_into_gcloud
 log_into_gcloud
 export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
-$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket" --gcs-bucket="$gcs_bucket"
+$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release --gcs-bucket="$gcs_bucket"
 EOF
-tc_end_block "Compile and publish S3 artifacts"
+tc_end_block "Compile and publish artifacts"
 
 tc_start_block "Make and push multiarch docker images"
 configure_docker_creds

diff --git a/build/teamcity/internal/release/process/publish-cockroach-release.sh b/build/teamcity/internal/release/process/publish-cockroach-release.sh
@@ -24,7 +24,6 @@ fi
 release_branch=$(echo ${build_name} | grep -E -o '^v[0-9]+\.[0-9]+')
 
 if [[ -z "${DRY_RUN}" ]] ; then
-  bucket="binaries.cockroachdb.com"
   gcs_bucket="cockroach-release-artifacts-prod"
   google_credentials="$GOOGLE_COCKROACH_CLOUD_IMAGES_COCKROACHDB_CREDENTIALS"
   # export the variable to avoid shell escaping
@@ -39,7 +38,6 @@ if [[ -z "${DRY_RUN}" ]] ; then
   gcr_hostname="us-docker.pkg.dev"
   git_repo_for_tag="cockroachdb/cockroach"
 else
-  bucket="cockroach-builds-test"
   gcs_bucket="cockroach-release-artifacts-dryrun"
   google_credentials="$GOOGLE_COCKROACH_RELEASE_CREDENTIALS"
   # export the variable to avoid shell escaping
@@ -79,19 +77,19 @@ git tag "${build_name}"
 tc_end_block "Tag the release"
 
 
-tc_start_block "Make and publish release S3 artifacts"
+tc_start_block "Make and publish release artifacts"
 # Using publish-provisional-artifacts here is funky. We're directly publishing
 # the official binaries, not provisional ones. Legacy naming. To clean up...
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
 bazel build --config ci //pkg/cmd/publish-provisional-artifacts
 BAZEL_BIN=$(bazel info bazel-bin --config ci)
 export google_credentials="$gcs_credentials"
 source "build/teamcity-support.sh"  # For log_into_gcloud
 log_into_gcloud
 export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
-$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket" --gcs-bucket="$gcs_bucket"
+$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release --gcs-bucket="$gcs_bucket"
 EOF
-tc_end_block "Make and publish release S3 artifacts"
+tc_end_block "Make and publish release artifacts"
 
 
 tc_start_block "Make and push multiarch docker images"
@@ -167,35 +165,35 @@ git_wrapped push "ssh://[email protected]/${git_repo_for_tag}.git" "$build_name"
 tc_end_block "Push release tag to GitHub"
 
 
-tc_start_block "Publish S3 binaries and archive as latest-RELEASE_BRANCH"
+tc_start_block "Publish binaries and archive as latest-RELEASE_BRANCH"
 # example: v20.1-latest
 if [[ -z "$PRE_RELEASE" ]]; then
   #TODO: implement me!
-  echo "Pushing latest-RELEASE_BRANCH S3 binaries and archive is not implemented."
+  echo "Pushing latest-RELEASE_BRANCH binaries and archive is not implemented."
 else
-  echo "Pushing latest-RELEASE_BRANCH S3 binaries and archive is not implemented."
+  echo "Pushing latest-RELEASE_BRANCH binaries and archive is not implemented."
 fi
-tc_end_block "Publish S3 binaries and archive as latest-RELEASE_BRANCH"
+tc_end_block "Publish binaries and archive as latest-RELEASE_BRANCH"
 
 
-tc_start_block "Publish S3 binaries and archive as latest"
+tc_start_block "Publish binaries and archive as latest"
 # Only push the "latest" for our most recent release branch.
 # https://github.com/cockroachdb/cockroach/issues/41067
 if [[ -n "${PUBLISH_LATEST}" && -z "${PRE_RELEASE}" ]]; then
-    BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
+    BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
 bazel build --config ci //pkg/cmd/publish-provisional-artifacts
 BAZEL_BIN=$(bazel info bazel-bin --config ci)
 export google_credentials="$gcs_credentials"
 source "build/teamcity-support.sh"  # For log_into_gcloud
 log_into_gcloud
 export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
-$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -bless -release -bucket "$bucket" --gcs-bucket="$gcs_bucket"
+$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -bless -release --gcs-bucket="$gcs_bucket"
 EOF
 
 else
-  echo "The latest S3 binaries and archive were _not_ updated."
+  echo "The latest binaries and archive were _not_ updated."
 fi
-tc_end_block "Publish S3 binaries and archive as latest"
+tc_end_block "Publish binaries and archive as latest"
 
 
 tc_start_block "Tag docker image as latest-RELEASE_BRANCH"

diff --git a/pkg/cmd/cloudupload/main.go b/pkg/cmd/cloudupload/main.go
@@ -39,10 +39,6 @@ func main() {
 		Body: handler,
 	}
 	switch parsedDst.provider {
-	case "s3":
-		if err := s3Upload(parsedDst.bucket, dstObj); err != nil {
-			log.Fatalf("failed to upload %s to %s: %s", src, dst, err)
-		}
 	case "gs":
 		if err := gcsUpload(parsedDst.bucket, dstObj); err != nil {
 			log.Fatalf("failed to upload %s to %s: %s", src, dst, err)
@@ -70,25 +66,6 @@ func parseURL(dst string) (target, error) {
 	}, nil
 }
 
-func s3Upload(bucket string, dstObj release.PutObjectInput) error {
-	if _, ok := os.LookupEnv("AWS_ACCESS_KEY_ID"); !ok {
-		return fmt.Errorf("AWS_ACCESS_KEY_ID environment variable is not set")
-	}
-	if _, ok := os.LookupEnv("AWS_SECRET_ACCESS_KEY"); !ok {
-		return fmt.Errorf("AWS_SECRET_ACCESS_KEY environment variable is not set")
-	}
-	s3, err := release.NewS3("us-east-1", bucket)
-	if err != nil {
-		return fmt.Errorf("creating AWS S3 session: %w", err)
-	}
-	// Make sure the object doesn't exist. Potentially can race.
-	obj := release.GetObjectInput{Key: dstObj.Key}
-	if _, err := s3.GetObject(&obj); err == nil {
-		return fmt.Errorf("cannot overwrite %s in bucket %s", *dstObj.Key, bucket)
-	}
-	return s3.PutObject(&dstObj)
-}
-
 func gcsUpload(bucket string, dstObj release.PutObjectInput) error {
 	if _, ok := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS"); !ok {
 		return fmt.Errorf("GOOGLE_APPLICATION_CREDENTIALS environment variable is not set")

diff --git a/pkg/cmd/publish-artifacts/main.go b/pkg/cmd/publish-artifacts/main.go
@@ -23,15 +23,11 @@ import (
 )
 
 const (
-	awsAccessKeyIDKey      = "AWS_ACCESS_KEY_ID"
-	awsSecretAccessKeyKey  = "AWS_SECRET_ACCESS_KEY"
 	teamcityBuildBranchKey = "TC_BUILD_BRANCH"
 )
 
 func main() {
-	var s3Bucket string
 	var gcsBucket string
-	flag.StringVar(&s3Bucket, "bucket", "", "S3 bucket")
 	flag.StringVar(&gcsBucket, "gcs-bucket", "", "GCS bucket")
 	flag.Parse()
 
@@ -45,25 +41,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("Creating GCS session: %s", err)
 	}
-	var providers []release.ObjectPutGetter
-	providers = append(providers, gcs)
-
-	if s3Bucket != "" {
-		log.Printf("Using S3 bucket: %s", s3Bucket)
-		if _, ok := os.LookupEnv(awsAccessKeyIDKey); !ok {
-			log.Fatalf("AWS access key ID environment variable %s is not set", awsAccessKeyIDKey)
-		}
-		if _, ok := os.LookupEnv(awsSecretAccessKeyKey); !ok {
-			log.Fatalf("AWS secret access key environment variable %s is not set", awsSecretAccessKeyKey)
-		}
-		s3, err := release.NewS3("us-east-1", s3Bucket)
-		if err != nil {
-			log.Fatalf("Creating AWS S3 session: %s", err)
-		}
-		providers = append(providers, s3)
-	} else {
-		log.Println("Not using S3 bucket")
-	}
+	providers := []release.ObjectPutGetter{gcs}
 
 	branch, ok := os.LookupEnv(teamcityBuildBranchKey)
 	if !ok {