Merge pull request #94673 from cockroachdb/blathers/backport-release-…

…22.2-94624

release-22.2: ci: do not publish to S3

build/teamcity/cockroach/post-merge/publish-bleeding-edge.sh

@@ -9,18 +9,16 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
 source "$dir/teamcity-support.sh"
 source "$dir/teamcity-bazel-support.sh"
 
-# s3 pushes to the "cockroach" bucket. There is no test/dev bucket fir this build type.
-bucket="cockroach"
 gcs_bucket="cockroach-edge-artifacts-prod"
 # export the variable to avoid shell escaping
 export gcs_credentials="$GCS_CREDENTIALS_PROD"
 
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e gcs_credentials -e bucket=$bucket -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
 bazel build --config ci //pkg/cmd/publish-artifacts
 BAZEL_BIN=$(bazel info bazel-bin --config ci)
 export google_credentials="$gcs_credentials"
 source "build/teamcity-support.sh"  # For log_into_gcloud
 log_into_gcloud
 export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
-$BAZEL_BIN/pkg/cmd/publish-artifacts/publish-artifacts_/publish-artifacts --gcs-bucket="$gcs_bucket" --bucket="$bucket"
+$BAZEL_BIN/pkg/cmd/publish-artifacts/publish-artifacts_/publish-artifacts --gcs-bucket="$gcs_bucket"
 EOF

build/teamcity/internal/cockroach/release/publish/sign_macos_release.sh

@@ -14,14 +14,12 @@ trap remove_files_on_exit EXIT
 
 # By default, set dry-run variables
 google_credentials="$GCS_CREDENTIALS_DEV"
-s3_bucket="cockroach-builds-test"
 gcs_bucket="cockroach-release-artifacts-dryrun"
 
 # override dev defaults with production values
 if [[ -z "${DRY_RUN}" ]] ; then
   echo "Setting production variable values"
   google_credentials="$GCS_CREDENTIALS_PROD"
-  s3_bucket="binaries.cockroachdb.com"
   gcs_bucket="cockroach-release-artifacts-prod"
 fi
 
@@ -76,10 +74,6 @@ for product in cockroach cockroach-sql; do
     rm -rf "$base" "$unsigned_file" "$unsigned_file.sha256sum" crl.zip
 
     shasum --algorithm 256 "$target" > "$target.sha256sum"
-    "$BAZEL_BIN/pkg/cmd/cloudupload/cloudupload_/cloudupload" \
-      "$target" "s3://$s3_bucket/$target"
-    "$BAZEL_BIN/pkg/cmd/cloudupload/cloudupload_/cloudupload" \
-      "$target.sha256sum" "s3://$s3_bucket/$target.sha256sum"
     "$BAZEL_BIN/pkg/cmd/cloudupload/cloudupload_/cloudupload" \
       "$target" "gs://$gcs_bucket/$target"
     "$BAZEL_BIN/pkg/cmd/cloudupload/cloudupload_/cloudupload" \

build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh

@@ -15,7 +15,6 @@ release_branch="$(echo "$build_name" | grep -Eo "^v[0-9]+\.[0-9]+" || echo"")"
 is_custom_build="$(echo "$TC_BUILD_BRANCH" | grep -Eo "^custombuild-" || echo "")"
 
 if [[ -z "${DRY_RUN}" ]] ; then
-  bucket="cockroach-builds"
   gcs_bucket="cockroach-builds-artifacts-prod"
   google_credentials=$GOOGLE_COCKROACH_CLOUD_IMAGES_COCKROACHDB_CREDENTIALS
   gcr_repository="us-docker.pkg.dev/cockroach-cloud-images/cockroachdb/cockroach"
@@ -24,7 +23,6 @@ if [[ -z "${DRY_RUN}" ]] ; then
   # export the variable to avoid shell escaping
   export gcs_credentials="$GCS_CREDENTIALS_PROD"
 else
-  bucket="cockroach-builds-test"
   gcs_bucket="cockroach-builds-artifacts-dryrun"
   google_credentials="$GOOGLE_COCKROACH_RELEASE_CREDENTIALS"
   gcr_repository="us.gcr.io/cockroach-release/cockroach-test"
@@ -40,7 +38,6 @@ cat << EOF
   build_name:      $build_name
   release_branch:  $release_branch
   is_custom_build: $is_custom_build
-  bucket:          $bucket
   gcs_bucket:      $gcs_bucket
   gcr_repository:  $gcr_repository
 
@@ -54,17 +51,17 @@ tc_start_block "Tag the release"
 git tag "${build_name}"
 tc_end_block "Tag the release"
 
-tc_start_block "Compile and publish S3 artifacts"
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
+tc_start_block "Compile and publish artifacts"
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
 bazel build --config ci //pkg/cmd/publish-provisional-artifacts
 BAZEL_BIN=$(bazel info bazel-bin --config ci)
 export google_credentials="$gcs_credentials"
 source "build/teamcity-support.sh"  # For log_into_gcloud
 log_into_gcloud
 export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
-$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket" --gcs-bucket="$gcs_bucket"
+$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release --gcs-bucket="$gcs_bucket"
 EOF
-tc_end_block "Compile and publish S3 artifacts"
+tc_end_block "Compile and publish artifacts"
 
 tc_start_block "Make and push multiarch docker images"
 configure_docker_creds

build/teamcity/internal/release/process/publish-cockroach-release.sh

@@ -24,7 +24,6 @@ fi
 release_branch=$(echo ${build_name} | grep -E -o '^v[0-9]+\.[0-9]+')
 
 if [[ -z "${DRY_RUN}" ]] ; then
-  bucket="binaries.cockroachdb.com"
   gcs_bucket="cockroach-release-artifacts-prod"
   google_credentials="$GOOGLE_COCKROACH_CLOUD_IMAGES_COCKROACHDB_CREDENTIALS"
   # export the variable to avoid shell escaping
@@ -39,7 +38,6 @@ if [[ -z "${DRY_RUN}" ]] ; then
   gcr_hostname="us-docker.pkg.dev"
   git_repo_for_tag="cockroachdb/cockroach"
 else
-  bucket="cockroach-builds-test"
   gcs_bucket="cockroach-release-artifacts-dryrun"
   google_credentials="$GOOGLE_COCKROACH_RELEASE_CREDENTIALS"
   # export the variable to avoid shell escaping
@@ -79,19 +77,19 @@ git tag "${build_name}"
 tc_end_block "Tag the release"
 
 
-tc_start_block "Make and publish release S3 artifacts"
+tc_start_block "Make and publish release artifacts"
 # Using publish-provisional-artifacts here is funky. We're directly publishing
 # the official binaries, not provisional ones. Legacy naming. To clean up...
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
 bazel build --config ci //pkg/cmd/publish-provisional-artifacts
 BAZEL_BIN=$(bazel info bazel-bin --config ci)
 export google_credentials="$gcs_credentials"
 source "build/teamcity-support.sh"  # For log_into_gcloud
 log_into_gcloud
 export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
-$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release -bucket "$bucket" --gcs-bucket="$gcs_bucket"
+$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release --gcs-bucket="$gcs_bucket"
 EOF
-tc_end_block "Make and publish release S3 artifacts"
+tc_end_block "Make and publish release artifacts"
 
 
 tc_start_block "Make and push multiarch docker images"
@@ -167,35 +165,35 @@ git_wrapped push "ssh://[email protected]/${git_repo_for_tag}.git" "$build_name"
 tc_end_block "Push release tag to GitHub"
 
 
-tc_start_block "Publish S3 binaries and archive as latest-RELEASE_BRANCH"
+tc_start_block "Publish binaries and archive as latest-RELEASE_BRANCH"
 # example: v20.1-latest
 if [[ -z "$PRE_RELEASE" ]]; then
   #TODO: implement me!
-  echo "Pushing latest-RELEASE_BRANCH S3 binaries and archive is not implemented."
+  echo "Pushing latest-RELEASE_BRANCH binaries and archive is not implemented."
 else
-  echo "Pushing latest-RELEASE_BRANCH S3 binaries and archive is not implemented."
+  echo "Pushing latest-RELEASE_BRANCH binaries and archive is not implemented."
 fi
-tc_end_block "Publish S3 binaries and archive as latest-RELEASE_BRANCH"
+tc_end_block "Publish binaries and archive as latest-RELEASE_BRANCH"
 
 
-tc_start_block "Publish S3 binaries and archive as latest"
+tc_start_block "Publish binaries and archive as latest"
 # Only push the "latest" for our most recent release branch.
 # https://github.com/cockroachdb/cockroach/issues/41067
 if [[ -n "${PUBLISH_LATEST}" && -z "${PRE_RELEASE}" ]]; then
-    BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e bucket=$bucket -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
+    BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
 bazel build --config ci //pkg/cmd/publish-provisional-artifacts
 BAZEL_BIN=$(bazel info bazel-bin --config ci)
 export google_credentials="$gcs_credentials"
 source "build/teamcity-support.sh"  # For log_into_gcloud
 log_into_gcloud
 export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
-$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -bless -release -bucket "$bucket" --gcs-bucket="$gcs_bucket"
+$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -bless -release --gcs-bucket="$gcs_bucket"
 EOF
 
 else
-  echo "The latest S3 binaries and archive were _not_ updated."
+  echo "The latest binaries and archive were _not_ updated."
 fi
-tc_end_block "Publish S3 binaries and archive as latest"
+tc_end_block "Publish binaries and archive as latest"
 
 
 tc_start_block "Tag docker image as latest-RELEASE_BRANCH"

pkg/cmd/cloudupload/main.go

@@ -39,10 +39,6 @@ func main() {
 		Body: handler,
 	}
 	switch parsedDst.provider {
-	case "s3":
-		if err := s3Upload(parsedDst.bucket, dstObj); err != nil {
-			log.Fatalf("failed to upload %s to %s: %s", src, dst, err)
-		}
 	case "gs":
 		if err := gcsUpload(parsedDst.bucket, dstObj); err != nil {
 			log.Fatalf("failed to upload %s to %s: %s", src, dst, err)
@@ -70,25 +66,6 @@ func parseURL(dst string) (target, error) {
 	}, nil
 }
 
-func s3Upload(bucket string, dstObj release.PutObjectInput) error {
-	if _, ok := os.LookupEnv("AWS_ACCESS_KEY_ID"); !ok {
-		return fmt.Errorf("AWS_ACCESS_KEY_ID environment variable is not set")
-	}
-	if _, ok := os.LookupEnv("AWS_SECRET_ACCESS_KEY"); !ok {
-		return fmt.Errorf("AWS_SECRET_ACCESS_KEY environment variable is not set")
-	}
-	s3, err := release.NewS3("us-east-1", bucket)
-	if err != nil {
-		return fmt.Errorf("creating AWS S3 session: %w", err)
-	}
-	// Make sure the object doesn't exist. Potentially can race.
-	obj := release.GetObjectInput{Key: dstObj.Key}
-	if _, err := s3.GetObject(&obj); err == nil {
-		return fmt.Errorf("cannot overwrite %s in bucket %s", *dstObj.Key, bucket)
-	}
-	return s3.PutObject(&dstObj)
-}
-
 func gcsUpload(bucket string, dstObj release.PutObjectInput) error {
 	if _, ok := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS"); !ok {
 		return fmt.Errorf("GOOGLE_APPLICATION_CREDENTIALS environment variable is not set")

pkg/cmd/publish-artifacts/main.go

@@ -23,15 +23,11 @@ import (
 )
 
 const (
-	awsAccessKeyIDKey      = "AWS_ACCESS_KEY_ID"
-	awsSecretAccessKeyKey  = "AWS_SECRET_ACCESS_KEY"
 	teamcityBuildBranchKey = "TC_BUILD_BRANCH"
 )
 
 func main() {
-	var s3Bucket string
 	var gcsBucket string
-	flag.StringVar(&s3Bucket, "bucket", "", "S3 bucket")
 	flag.StringVar(&gcsBucket, "gcs-bucket", "", "GCS bucket")
 	flag.Parse()
 
@@ -45,25 +41,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("Creating GCS session: %s", err)
 	}
-	var providers []release.ObjectPutGetter
-	providers = append(providers, gcs)
-
-	if s3Bucket != "" {
-		log.Printf("Using S3 bucket: %s", s3Bucket)
-		if _, ok := os.LookupEnv(awsAccessKeyIDKey); !ok {
-			log.Fatalf("AWS access key ID environment variable %s is not set", awsAccessKeyIDKey)
-		}
-		if _, ok := os.LookupEnv(awsSecretAccessKeyKey); !ok {
-			log.Fatalf("AWS secret access key environment variable %s is not set", awsSecretAccessKeyKey)
-		}
-		s3, err := release.NewS3("us-east-1", s3Bucket)
-		if err != nil {
-			log.Fatalf("Creating AWS S3 session: %s", err)
-		}
-		providers = append(providers, s3)
-	} else {
-		log.Println("Not using S3 bucket")
-	}
+	providers := []release.ObjectPutGetter{gcs}
 
 	branch, ok := os.LookupEnv(teamcityBuildBranchKey)
 	if !ok {