Remove a note about cloudpickle support being experimental

.asf.yaml

@@ -35,12 +35,14 @@ github:
     projects: true
 
   # Give some users issue triage permissions
+  # Cannot have >10 at any given time per https://github.com/apache/infrastructure-asfyaml?tab=readme-ov-file#triage
   collaborators:
-    - pcoet
-    - olehborysevych
-    - rshamunov
-    - andreydevyatkin
-    - liferoad
+    - Amar3tto
+    - mrshakirov
+    - akashorabek
+    - derrickaw
+    - claudevdm
+    - shunping
 
   enabled_merge_buttons:
     squash: true
@@ -49,6 +51,19 @@ github:
 
   protected_branches:
     master: {}
+    release-2.63.0-postrelease: {}
+    release-2.63: {}
+    release-2.62.0: {}
+    release-2.61.0: {}
+    release-2.60.0: {}
+    release-2.59.0: {}
+    release-2.58.1: {}
+    release-2.58.0: {}
+    release-2.57.0: {}
+    release-2.56.0: {}
+    release-2.55.1: {}
+    release-2.55.0: {}
+    release-2.54.0: {}
     release-2.53.0: {}
     release-2.52.0: {}
     release-2.51.0: {}

.github/ACTIONS.md

@@ -86,15 +86,20 @@ jobs:
 * **DO NOT** Approve and Run changes in the workflows in the PR Conversation tab, under "Workflow(s) awaiting approval".
 * For approving the updates in the workflows, you should go to the Repository Actions and filter All Workflows by `action_required`. The search will display the workflows that need to be reviewed before running. **Please make sure reviewing the file that is referenced by the workflow.**
 * Seed job will be emulated using the `Approve and Run` built-in feature of GitHub Actions, since the workflows will use the `pull_request_target` directive; no modifications would be allowed either for new or existent jobs unless a committer explicitly approves the job from GitHub Actions UI.
-### Issue Management
 
+#### Issue Management
 Phrases self-assign, close, or manage labels on an issue:
 | Phrase | Effect |
 |--------|--------|
 | `.take-issue` | Self-assign the issue |
+| `.free-issue` | Self-unassign the issue |
 | `.close-issue` | Close the issue as completed |
 | `.close-issue not_planned` | Close the issue as not-planned |
 | `.reopen-issue` | Reopen a closed issue |
 | `.add-labels` | Add comma separated labels to the issue (e.g. `add-labels l1, 'l2 with spaces'`) |
 | `.remove-labels` | Remove comma separated labels to the issue (e.g. `remove-labels l1, 'l2 with spaces'`) |
 | `.set-labels` | Sets comma separated labels to the issue and removes any other labels (e.g. `set-labels l1, 'l2 with spaces'`) |
+
+## Security Model
+
+For information on the Beam CI security model, see https://cwiki.apache.org/confluence/display/BEAM/CI+Security+Model

.github/ISSUE_TEMPLATE/bug.yml

@@ -25,7 +25,7 @@ body:
   - type: markdown
     attributes:
       value: |
-        Thanks for taking the time to fill out this bug report! Once you've created an issue, you can self-assign by commenting `.take-issue` and you can close by commenting `.close-issue`.
+        Thanks for taking the time to fill out this bug report! Once you've created an issue, you can self-assign by commenting `.take-issue`, self-unassign by commenting `.free-issue`, and close by commenting `.close-issue`.
         Anyone can reopen a closed issue by commenting `.reopen-issue`.
         You can also add/remove labels by commenting `.add-labels label1,label2,'label 3 with spaces'` or `.remove-labels label1,label2,'label 3 with spaces'`,
         or with `.set-labels label1,label2,'label 3 with spaces'` (which removes any labels not in that set).
@@ -50,6 +50,7 @@ body:
         - "Priority: 2 (default / most bugs should be filed as P2)"
         - "Priority: 1 (data loss / total loss of function)"
         - "Priority: 0 (outage / urgent vulnerability)"
+      default: 1
     validations:
       required: true
   - type: checkboxes
@@ -68,6 +69,7 @@ body:
        - label: "Component: Beam playground"
        - label: "Component: Beam katas"
        - label: "Component: Website"
+       - label: "Component: Infrastructure"
        - label: "Component: Spark Runner"
        - label: "Component: Flink Runner"
        - label: "Component: Samza Runner"

.github/ISSUE_TEMPLATE/failing_test.yml

@@ -25,7 +25,7 @@ body:
   - type: markdown
     attributes:
       value: |
-        Thanks for taking the time to fill out this feature request! Once you've created an issue, you can self-assign by commenting `.take-issue` and you can close by commenting `.close-issue`.
+        Thanks for taking the time to fill out this failing test report! Once you've created an issue, you can self-assign by commenting `.take-issue`, self-unassign by commenting `.free-issue`, and close by commenting `.close-issue`.
         Anyone can reopen a closed issue by commenting `.reopen-issue`.
         You can also add/remove labels by commenting `.add-labels label1,label2,'label 3 with spaces'` or `.remove-labels label1,label2,'label 3 with spaces'`,
         or with `.set-labels label1,label2,'label 3 with spaces'` (which removes any labels not in that set).
@@ -56,6 +56,7 @@ body:
         - "Priority: 2 (backlog / disabled test but we think the product is healthy)"
         - "Priority: 1 (unhealthy code / failing or flaky postcommit so we cannot be sure the product is healthy)"
         - "Priority: 0 (outage / failing precommit test impacting development)"
+      default: 1
     validations:
       required: true
   - type: checkboxes
@@ -74,6 +75,7 @@ body:
        - label: "Component: Beam playground"
        - label: "Component: Beam katas"
        - label: "Component: Website"
+       - label: "Component: Infrastructure"
        - label: "Component: Spark Runner"
        - label: "Component: Flink Runner"
        - label: "Component: Samza Runner"

.github/ISSUE_TEMPLATE/feature.yml

@@ -25,7 +25,7 @@ body:
   - type: markdown
     attributes:
       value: |
-        Thanks for taking the time to fill out this feature request! Once you've created an issue, you can self-assign by commenting `.take-issue` and you can close by commenting `.close-issue`.
+        Thanks for taking the time to fill out this feature request! Once you've created an issue, you can self-assign by commenting `.take-issue`, self-unassign by commenting `.free-issue`, and close by commenting `.close-issue`.
         Anyone can reopen a closed issue by commenting `.reopen-issue`.
         You can also add/remove labels by commenting `.add-labels label1,label2,'label 3 with spaces'` or `.remove-labels label1,label2,'label 3 with spaces'`,
         or with `.set-labels label1,label2,'label 3 with spaces'` (which removes any labels not in that set).
@@ -44,6 +44,7 @@ body:
       options:
         - "Priority: 3 (nice-to-have improvement)"
         - "Priority: 2 (default / most feature requests should be filed as P2)"
+      default: 1
     validations:
       required: true
   - type: checkboxes
@@ -62,6 +63,7 @@ body:
        - label: "Component: Beam playground"
        - label: "Component: Beam katas"
        - label: "Component: Website"
+       - label: "Component: Infrastructure"
        - label: "Component: Spark Runner"
        - label: "Component: Flink Runner"
        - label: "Component: Samza Runner"

.github/ISSUE_TEMPLATE/task.yml

@@ -25,7 +25,7 @@ body:
   - type: markdown
     attributes:
       value: |
-        Thanks for taking the time to fill out this task! Once you've created an issue, you can self-assign by commenting `.take-issue` and you can close by commenting `.close-issue`.
+        Thanks for taking the time to fill out this task! Once you've created an issue, you can self-assign by commenting `.take-issue`, self-unassign by commenting `.free-issue`, and close by commenting `.close-issue`.
         Anyone can reopen a closed issue by commenting `.reopen-issue`.
         You can also add/remove labels by commenting `.add-labels label1,label2,'label 3 with spaces'` or `.remove-labels label1,label2,'label 3 with spaces'`,
         or with `.set-labels label1,label2,'label 3 with spaces'` (which removes any labels not in that set).
@@ -45,6 +45,7 @@ body:
         - "Priority: 3 (nice-to-have improvement)"
         - "Priority: 2 (default / most normal work should be filed as P2)"
         - "Priority: 1 (urgent / mostly reserved for critical bugs)"
+      default: 1
     validations:
       required: true
   - type: checkboxes
@@ -63,6 +64,7 @@ body:
        - label: "Component: Beam playground"
        - label: "Component: Beam katas"
        - label: "Component: Website"
+       - label: "Component: Infrastructure"
        - label: "Component: Spark Runner"
        - label: "Component: Flink Runner"
        - label: "Component: Samza Runner"

.github/PULL_REQUEST_TEMPLATE.md

@@ -14,9 +14,9 @@ To check the build health, please visit [https://github.com/apache/beam/blob/mas
 
 GitHub Actions Tests Status (on master branch)
 ------------------------------------------------------------------------------------------------
-[![Build python source distribution and wheels](https://github.com/apache/beam/workflows/Build%20python%20source%20distribution%20and%20wheels/badge.svg?branch=master&event=schedule)](https://github.com/apache/beam/actions?query=workflow%3A%22Build+python+source+distribution+and+wheels%22+branch%3Amaster+event%3Aschedule)
-[![Python tests](https://github.com/apache/beam/workflows/Python%20tests/badge.svg?branch=master&event=schedule)](https://github.com/apache/beam/actions?query=workflow%3A%22Python+Tests%22+branch%3Amaster+event%3Aschedule)
-[![Java tests](https://github.com/apache/beam/workflows/Java%20Tests/badge.svg?branch=master&event=schedule)](https://github.com/apache/beam/actions?query=workflow%3A%22Java+Tests%22+branch%3Amaster+event%3Aschedule)
-[![Go tests](https://github.com/apache/beam/workflows/Go%20tests/badge.svg?branch=master&event=schedule)](https://github.com/apache/beam/actions?query=workflow%3A%22Go+tests%22+branch%3Amaster+event%3Aschedule)
+[![Build python source distribution and wheels](https://github.com/apache/beam/actions/workflows/build_wheels.yml/badge.svg?event=schedule&&?branch=master)](https://github.com/apache/beam/actions?query=workflow%3A%22Build+python+source+distribution+and+wheels%22+branch%3Amaster+event%3Aschedule)
+[![Python tests](https://github.com/apache/beam/actions/workflows/python_tests.yml/badge.svg?event=schedule&&?branch=master)](https://github.com/apache/beam/actions?query=workflow%3A%22Python+Tests%22+branch%3Amaster+event%3Aschedule)
+[![Java tests](https://github.com/apache/beam/actions/workflows/java_tests.yml/badge.svg?event=schedule&&?branch=master)](https://github.com/apache/beam/actions?query=workflow%3A%22Java+Tests%22+branch%3Amaster+event%3Aschedule)
+[![Go tests](https://github.com/apache/beam/actions/workflows/go_tests.yml/badge.svg?event=schedule&&?branch=master)](https://github.com/apache/beam/actions?query=workflow%3A%22Go+tests%22+branch%3Amaster+event%3Aschedule)
 
 See [CI.md](https://github.com/apache/beam/blob/master/CI.md) for more information about GitHub Actions CI or the [workflows README](https://github.com/apache/beam/blob/master/.github/workflows/README.md) to see a list of phrases to trigger workflows.

.github/REVIEWERS.yml

@@ -22,24 +22,21 @@ labels:
     reviewers:
       - lostluck
       - jrmccluskey
-      - riteshghorse
     exclusionList:
       - youngoli
   - name: Python
     reviewers:
-      - AnandInguva
       - damccorm
       - jrmccluskey
       - tvalentyn
-      - riteshghorse
       - liferoad
+      - shunping
     exclusionList: []
   - name: Java
     reviewers:
       - Abacn
       - kennknowles
       - robertwb
-      - bvolpato
       - m-trieu
       - damondouglas
     exclusionList: []
@@ -49,8 +46,8 @@ labels:
       - johnjcasey
       - Abacn
       - ahmedabu98
-      - bvolpato
       - damondouglas
+      - shunping
     exclusionList: []
   - name: spanner
     reviewers:
@@ -59,7 +56,20 @@ labels:
     reviewers:
       - igorbernstein2
       - mutianf
+      - djyau
+      - andre-sampaio
+      - meeral-k
     exclusionList: []
+  - name: healthcare
+    reviewers:
+      - svetakvsundhar
+    exclusionList: []
+  - name: kafka
+    reviewers:
+      - johnjcasey
+      - fozzie15
+      - Dippatel98
+      - sjvanrossum
   - name: Build
     reviewers:
       - damccorm
@@ -75,14 +85,13 @@ labels:
     exclusionList: []
 fallbackReviewers:
   - Abacn
-  - AnandInguva
   - chamikaramj
   - damccorm
   - damondouglas
   - johnjcasey
   - jrmccluskey
   - kennknowles
   - lostluck
-  - riteshghorse
   - robertwb
+  - shunping
   - tvalentyn

.github/actions/setup-default-test-properties/test-properties.json

@@ -1,23 +1,23 @@
 {
   "PythonTestProperties": {
-    "ALL_SUPPORTED_VERSIONS": ["3.8", "3.9", "3.10", "3.11"],
-    "LOWEST_SUPPORTED": ["3.8"],
-    "HIGHEST_SUPPORTED": ["3.11"],
-    "ESSENTIAL_VERSIONS": ["3.8", "3.11"],
-    "CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.8", "3.11"],
+    "ALL_SUPPORTED_VERSIONS": ["3.9", "3.10", "3.11", "3.12"],
+    "LOWEST_SUPPORTED": ["3.9"],
+    "HIGHEST_SUPPORTED": ["3.12"],
+    "ESSENTIAL_VERSIONS": ["3.9", "3.12"],
+    "CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS": ["3.9", "3.12"],
     "CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS": ["3.11"],
-    "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.8", "3.9", "3.10", "3.11" ]
-    "LOAD_TEST_PYTHON_VERSION": "3.8",
-    "CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.8",
-    "DEFAULT_INTERPRETER": "python3.8",
+    "VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS": ["3.9", "3.10", "3.11", "3.12"],
+    "LOAD_TEST_PYTHON_VERSION": "3.9",
+    "CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION": "3.9",
+    "DEFAULT_INTERPRETER": "python3.9",
     "TOX_ENV": ["Cloud", "Cython"]
   },
   "JavaTestProperties": {
-    "SUPPORTED_VERSIONS": ["8", "11", "17"],
-    "FLINK_VERSIONS": ["1.13", "1.14", "1.15"],
+    "SUPPORTED_VERSIONS": ["8", "11", "17", "21"],
+    "FLINK_VERSIONS": ["1.17", "1.18", "1.19"],
     "SPARK_VERSIONS": ["2", "3"]
   },
   "GoTestProperties": {
-    "SUPPORTED_VERSIONS": ["1.20"]
+    "SUPPORTED_VERSIONS": ["1.23"]
   }
 }

.github/actions/setup-environment-action/action.yml

@@ -34,27 +34,54 @@ inputs:
     required: false
     description: 'Whether to disable the gradle cache'
     default: false
+  python-cache:
+    required: false
+    description: 'Whether to enable Python pip caching'
+    default: true
+  tox-cache:
+    required: false
+    description: 'Whether to enable tox environment caching'
+    default: true
 
 runs:
   using: "composite"
   steps:
     - name: Install Python
       if: ${{ inputs.python-version != '' }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        python-version: ${{ inputs.python-version }}
+        python-version: ${{ inputs.python-version == 'default' && '3.9' || inputs.python-version }}
+        cache: ${{ inputs.python-cache && 'pip' || 'none' }}
+        cache-dependency-path: |
+          sdks/python/setup.py
+          sdks/python/tox.ini
+
+    - name: Cache tox environments
+      if: ${{ inputs.python-version != '' && inputs.tox-cache == 'true' }}
+      uses: actions/cache@v3
+      with:
+        path: |
+          sdks/python/target/.tox
+          !sdks/python/target/.tox/**/log
+          !sdks/python/target/.tox/.package_cache
+        key: tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '39' || inputs.python-version }}-${{ hashFiles('sdks/python/tox.ini') }}-${{ hashFiles('sdks/python/setup.py') }}
+        restore-keys: |
+          tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '39' || inputs.python-version }}-${{ hashFiles('sdks/python/tox.ini') }}-
+          tox-${{ runner.os }}-py${{ inputs.python-version == 'default' && '39' || inputs.python-version }}-
+
     - name: Install Java
       if: ${{ inputs.java-version != '' }}
       uses: actions/setup-java@v3
       with:
         distribution: 'temurin'
-        java-version: ${{ inputs.java-version }}
+        java-version: ${{ inputs.java-version == 'default' && '11' || inputs.java-version }}
     - name: Setup Gradle
       uses: gradle/gradle-build-action@v2
       with:
-        cache-read-only: ${{ inputs.disable-cache }}
+        cache-disabled: ${{ inputs.disable-cache }}
     - name: Install Go
       if: ${{ inputs.go-version != '' }}
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v5
       with:
-        go-version: ${{ inputs.go-version }} # never set patch, to get latest patch releases.
+        go-version: ${{ inputs.go-version == 'default' && '1.24' || inputs.go-version }} # never set patch, to get latest patch releases.
+        cache-dependency-path: $${{ inputs.disable-cache && '' || 'sdks/go.sum' }}

.github/actions/setup-k8s-access/action.yml

@@ -34,7 +34,10 @@ inputs:
     description: "Zone of the cluster to be created"
     required: true
     default: "us-central1-a"
-
+  remove_finalizer:
+    description: "Remove finalizers from the cluster"
+    required: false
+    default: ""
 
 runs:
   using: composite
@@ -70,4 +73,7 @@ runs:
       main: echo "Post Cleanup"
       post: |
         echo "Post Cleanup"
+        if [ -n "${{ inputs.remove_finalizer }}" ]; then
+          kubectl patch ${{ inputs.remove_finalizer }} -p '[{"op": "remove", "path": "/metadata/finalizers"}]' --type=json
+        fi
         kubectl delete namespace ${{ steps.replace_namespace.outputs.TEST_NAMESPACE }}