diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml index 5c35aa744198..eac2655492ce 100644 --- a/.github/workflows/build_release_candidate.yml +++ b/.github/workflows/build_release_candidate.yml @@ -122,18 +122,18 @@ jobs: mkdir -p beam/${{ github.event.inputs.RELEASE }} cd beam/${{ github.event.inputs.RELEASE }} RC_DIR="beam-${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" - RC_ZIP="${RC_DIR}.tar.gz" + RC_ZIP="${RC_DIR}.zip" RELEASE_DIR="beam-${{ github.event.inputs.RELEASE }}" RC_TAG="v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" - SOURCE_RELEASE_ZIP="apache-beam-${{ github.event.inputs.RELEASE }}-source-release.tar.gz" + SOURCE_RELEASE_ZIP="apache-beam-${{ github.event.inputs.RELEASE }}-source-release.zip" # Check whether there is an existing dist dir if (svn ls "${SOURCE_RELEASE_ZIP}"); then echo "Removing existing ${SOURCE_RELEASE_ZIP}." svn delete "${SOURCE_RELEASE_ZIP}" fi - echo "Downloading: https://github.com/apache/beam/archive/${RC_TAG}.tar.gz" - wget https://github.com/apache/beam/archive/${RC_TAG}.tar.gz -O "${RC_ZIP}" + echo "Downloading: https://github.com/apache/beam/archive/${RC_TAG}.zip" + wget https://github.com/apache/beam/archive/${RC_TAG}.zip -O "${RC_ZIP}" unzip "$RC_ZIP" rm "$RC_ZIP" @@ -155,6 +155,8 @@ jobs: if: ${{github.event.inputs.STAGE_PYTHON_ARTIFACTS == 'yes'}} runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v4 - name: Validate and mask apache id/password run: | echo "::add-mask::${{ github.event.inputs.APACHE_PASSWORD }}" @@ -175,28 +177,37 @@ jobs: uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + - name: Install dependencies + run: | + pip install python-dateutil + pip install requests - name: stage python artifacts env: RC_TAG: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" GIT_REPO_BASE_URL: https://github.com/apache/beam RELEASE_DIR: "beam/${{ github.event.inputs.RELEASE }}" + RELEASE: "${{ github.event.inputs.RELEASE }}" SCRIPT_DIR: release/src/main/scripts GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SVN_ARTIFACTS_DIR: "beam/${{ github.event.inputs.RELEASE }}/python" run: | svn co https://dist.apache.org/repos/dist/dev/beam mkdir -p "${SVN_ARTIFACTS_DIR}" - + + git fetch --all --tags --prune RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}") python "${SCRIPT_DIR}/download_github_actions_artifacts.py" \ --github-token-var GITHUB_TOKEN \ - --repo-url "${GIT_REPO_BASE_URL}" \ + --repo-url "apache/beam" \ --rc-tag "${RC_TAG}" \ --release-commit "${RELEASE_COMMIT}" \ - --artifacts_dir "${RELEASE_DIR}/python" + --artifacts_dir "${RELEASE_DIR}/python" \ + --yes True cd "${RELEASE_DIR}"/python - + ls + echo "------Checking Hash Value for apache-beam-${RELEASE}.tar.gz-----" sha512sum -c "apache-beam-${RELEASE}.tar.gz.sha512" @@ -216,7 +227,7 @@ jobs: cd .. svn add --force python svn status - svn commit --no-auth-cache -m "Staging Python artifacts for Apache Beam ${RELEASE} RC${RC_NUM}" + svn commit -m "Staging Python artifacts for Apache Beam ${RELEASE} RC${RC_NUM}" --non-interactive --username ${{ github.event.inputs.APACHE_ID }} --password ${{ github.event.inputs.APACHE_PASSWORD }} stage_docker: diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py index 5fbeb51a10cd..99526f1ac7d1 100644 --- a/release/src/main/scripts/download_github_actions_artifacts.py +++ b/release/src/main/scripts/download_github_actions_artifacts.py @@ -50,6 +50,7 @@ def parse_arguments(): parser.add_argument("--release-commit", required=True) parser.add_argument("--artifacts_dir", required=True) parser.add_argument("--rc_number", required=False, default="") + parser.add_argument("--yes", required=False, default=False) args = parser.parse_args() github_token = get_github_token(args.github_token_var) @@ -57,7 +58,7 @@ def parse_arguments(): print("You passed following arguments:") pprint.pprint({**vars(args), **{"github_token": github_token}}) - if not get_yes_or_no_answer("Do you want to continue?"): + if not args.yes and not get_yes_or_no_answer("Do you want to continue?"): print("You said NO. Quitting ...") sys.exit(1) @@ -67,8 +68,9 @@ def parse_arguments(): artifacts_dir = args.artifacts_dir if os.path.isabs(args.artifacts_dir) \ else os.path.abspath(args.artifacts_dir) rc_number = args.rc_number + skip_prompts = args.yes - return github_token, repo_url, rc_tag, release_commit, artifacts_dir, rc_number + return github_token, repo_url, rc_tag, release_commit, artifacts_dir, rc_number, skip_prompts def get_github_token(github_token_var): @@ -241,7 +243,7 @@ def wait_for_workflow_run_to_finish( ) -def prepare_directory(artifacts_dir): +def prepare_directory(artifacts_dir, skip_prompts): """Creates given directory and asks for confirmation if directory exists before clearing it.""" print(f"Preparing Artifacts directory: {artifacts_dir}") if os.path.isdir(artifacts_dir): @@ -249,7 +251,7 @@ def prepare_directory(artifacts_dir): f"Found that directory already exists.\n" f"Any existing content in it will be erased. Proceed?\n" f"Your answer") - if get_yes_or_no_answer(question): + if skip_prompts or get_yes_or_no_answer(question): print(f"Clearing directory: {artifacts_dir}") shutil.rmtree(artifacts_dir, ignore_errors=True) else: @@ -328,6 +330,7 @@ def extract_single_artifact(file_path, output_dir): release_commit, artifacts_dir, rc_number, + skip_prompts, ) = parse_arguments() try: @@ -335,7 +338,7 @@ def extract_single_artifact(file_path, output_dir): run_id = get_last_run_id( workflow_id, repo_url, rc_tag, release_commit, github_token) validate_run(run_id, repo_url, github_token) - prepare_directory(artifacts_dir) + prepare_directory(artifacts_dir, skip_prompts) fetch_github_artifacts(run_id, repo_url, artifacts_dir, github_token, rc_number) print("Script finished successfully!") print(f"Artifacts available in directory: {artifacts_dir}")