Merge remote-tracking branch 'origin/feature/add-error-handling-for-b…

…qio' into feature/add-error-handling-for-bqio

.asf.yaml

@@ -49,6 +49,7 @@ github:
 
   protected_branches:
     master: {}
+    release-2.54.0: {}
     release-2.53.0: {}
     release-2.52.0: {}
     release-2.51.0: {}

.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json

@@ -0,0 +1 @@
+

.github/workflows/README.md

@@ -79,6 +79,7 @@ jobs:
       github.event_name == 'push' ||
       github.event_name == 'pull_request_target' ||
       (github.event_name == 'schedule' && github.repository == 'apache/beam') ||
+      github.event_name == 'workflow_dispatch' ||
       startsWith(github.event.comment.body, 'Run Job With Matrix')
     steps:
       - uses: actions/checkout@v3

.github/workflows/beam_CancelStaleDataflowJobs.yml

@@ -71,6 +71,8 @@ jobs:
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
+        with:
+          disable-cache: true
       - name: Authenticate on GCP
         id: auth
         uses: google-github-actions/auth@v1

.github/workflows/beam_CleanUpDataprocResources.yml

@@ -17,7 +17,7 @@ name: Cleanup Dataproc Resources
 
 on:
   schedule:
-    - cron: '0 */6 * * *'
+    - cron: '0 0 * * *'
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event

.github/workflows/beam_CleanUpGCPResources.yml

@@ -71,6 +71,8 @@ jobs:
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
+        with:
+          disable-cache: true
       - name: Authenticate on GCP
         id: auth
         uses: google-github-actions/setup-gcloud@v0

.github/workflows/beam_CleanUpPrebuiltSDKImages.yml

@@ -52,7 +52,7 @@ jobs:
   beam_CleanUpPrebuiltSDKImages:
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
     runs-on: [self-hosted, ubuntu-20.04, main]
-    timeout-minutes: 100
+    timeout-minutes: 180
     strategy:
       matrix: 
         job_name: [beam_CleanUpPrebuiltSDKImages]
@@ -71,12 +71,18 @@ jobs:
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
+        with:
+          disable-cache: true
       - name: Authenticate on GCP
         id: auth
         uses: google-github-actions/auth@v1
         with:
           credentials_json: ${{ secrets.GCP_SA_KEY }}
           project_id: ${{ secrets.GCP_PROJECT_ID }}
+      - name: GCloud Docker credential helper
+        run: |
+          gcloud auth configure-docker gcr.io && \
+          gcloud auth configure-docker us.gcr.io
       - name: run remove stale sdk container images
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:

.github/workflows/beam_PostCommit_Java_DataflowV2.yml

@@ -19,7 +19,7 @@ name: PostCommit Java Dataflow V2
 
 on:
   schedule:
-    - cron: '30 3/6 * * *'
+    - cron: '30 3/8 * * *'
   pull_request_target:
     paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_DataflowV2.json']
   workflow_dispatch:
@@ -54,7 +54,7 @@ jobs:
   beam_PostCommit_Java_DataflowV2:
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
     runs-on: [self-hosted, ubuntu-20.04, main]
-    timeout-minutes: 240
+    timeout-minutes: 360
     strategy:
       matrix:
         job_name: [beam_PostCommit_Java_DataflowV2]

.github/workflows/beam_PostCommit_TransformService_Direct.yml

@@ -91,7 +91,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python Test Results
+          name: Python ${{ matrix.python_version }} Test Results
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2

.github/workflows/beam_PreCommit_Python_Coverage.yml

@@ -53,6 +53,8 @@ env:
   GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
   GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
   GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+  HF_INFERENCE_TOKEN: ${{ secrets.HF_INFERENCE_TOKEN }}
+
 
 jobs:
   beam_PreCommit_Python_Coverage:

.github/workflows/playground_frontend_test.yml

@@ -48,7 +48,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: 'Cache Flutter Dependencies'
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /opt/hostedtoolcache/flutter
           key: ${{ runner.OS }}-flutter-install-cache-${{ env.FLUTTER_VERSION }}

.github/workflows/tour_of_beam_frontend_test.yml

@@ -50,7 +50,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: 'Cache Flutter Dependencies'
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /opt/hostedtoolcache/flutter
           key: ${{ runner.OS }}-flutter-install-cache-${{ env.FLUTTER_VERSION }}

.github/workflows/update_python_dependencies.yml

@@ -24,7 +24,14 @@ on:
   schedule:
   - cron: "0 0 * * 0"
   workflow_dispatch:
-permissions: read-all
+permissions:
+  contents: write
+  pull-requests: write
+  checks: read
+  issues: read
+  statuses: read
+
+
 
 env:
   GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
@@ -55,6 +62,8 @@ jobs:
           python-version: |
             3.8
             3.9
+            3.10
+            3.11
           java-version: 8
           go-version: 1.21
       - name: Update Python Dependencies
@@ -68,6 +77,8 @@ jobs:
           git config user.name $GITHUB_ACTOR
           git config user.email actions@"$RUNNER_NAME".local
       - name: Commit Changes and create new PR
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           branchName=weekly_update_python_dependencies_$(date +%s)
           git checkout -b $branchName
@@ -81,4 +92,4 @@ jobs:
             echo "Link of the new PR  $GITHUB_PR_URL"
           else
             echo "No changes on the files"
-          fi
+          fi

.test-infra/dataproc/cleanup.sh

@@ -44,8 +44,8 @@ function filterClusters(){
 
 		elapsedHours=$((($currentDate - $clusterStartTime)/3600))
 
-		#Most of the jobs commonly finish in less than one hour
-		if [[ $elapsedHours -ge 2 ]]; then 
+		# teardown clusters >= 6 hr old
+		if [[ $elapsedHours -ge 6 ]]; then
 			for name in ${generatedResources[@]}; do
 				# Only resources generated by the groovy jobs set are queued for deletion
 				if [[ "$cluster" == *${name}* && ! ("$cluster" =~ nokill) ]]; then