Merge branch 'apache:master' into master

.github/actions/job_init/action.yml

@@ -0,0 +1,96 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: "Initializes the job"
+description: "Does all the necessary steps to set up the job"
+inputs:
+  jdk_version:
+    description: "The JDK version that's supposed to be used."
+    required: true
+  maven_repo_folder:
+    description: "The location of the local Maven repository (not setting this parameter will omit the caching of Maven artifacts)."
+    required: false
+    default: ""
+  source_directory:
+    description: "Specifies the directory from which the code should be moved from (needed for containerized runs; not setting this parameter will omit moving the checkout)."
+    required: false
+    default: ""
+  target_directory:
+    description: "Specifies the directory to which the code should be moved to (needed for containerized runs; not setting this parameter will omit moving the checkout)."
+    required: false
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: "Initializes GHA_PIPELINE_START_TIME with the job's start time"
+      shell: bash
+      run: |
+        job_start_time="$(date --rfc-3339=seconds)"
+        echo "GHA_PIPELINE_START_TIME=${job_start_time}" >> "${GITHUB_ENV}"
+        echo "The job's start time is set to ${job_start_time}."
+
+    - name: "Pre-cleanup Disk Info"
+      shell: bash
+      run: df -h
+
+    - name: "Delete unused binaries"
+      shell: bash
+      run: |
+        # inspired by https://github.com/easimon/maximize-build-space
+        for label_with_path in \
+              "Android SDK:/usr/local/lib/android" \
+              "CodeQL:/opt/hostedtoolcache/CodeQL" \
+              ".NET:/usr/share/dotnet"; do
+          dependency_name="$(echo "${label_with_path}" | cut -d: -f1)"
+          dependency_path="$(echo "${label_with_path}" | cut -d: -f2)"
+        
+          if [ -d "${dependency_path}" ]; then
+            echo "[INFO] Deleting binaries of ${dependency_name} in ${dependency_path}."
+            sudo rm -rf "${dependency_path}"
+            df -h
+          else
+            echo "[INFO] The directory '${dependency_path}' doesn't exist. ${dependency_name} won't be removed."
+          fi
+        done
+
+    - name: "Set JDK version to ${{ inputs.jdk_version }}"
+      shell: bash
+      run: |
+        echo "JAVA_HOME=$JAVA_HOME_${{ inputs.jdk_version }}_X64" >> "${GITHUB_ENV}"
+        echo "PATH=$JAVA_HOME_${{ inputs.jdk_version }}_X64/bin:$PATH" >> "${GITHUB_ENV}"
+
+    - name: "Setup Maven package cache"
+      if: ${{ inputs.maven_repo_folder != '' }}
+      uses: actions/cache@v4
+      with:
+        path: ${{ inputs.maven_repo_folder }}
+        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: ${{ runner.os }}-maven
+
+    - name: "Moves checkout content from ${{ inputs.source_directory }} to ${{ inputs.target_directory }}."
+      if: ${{ inputs.source_directory != '' && inputs.target_directory != '' }}
+      shell: bash
+      run: |
+        mkdir -p ${{ inputs.target_directory }}
+
+        # .scalafmt.conf is needed for Scala format checks
+        # .mvn is needed to make the Maven wrapper accessible in test runs
+        mv ${{ inputs.source_directory }}/* \
+          ${{ inputs.source_directory }}/.scalafmt.conf \
+          ${{ inputs.source_directory }}/.mvn \
+          ${{ inputs.target_directory }}

.github/actions/run_mvn/action.yml

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: "Runs Maven Command"
+description: "Executes Flink's Maven wrapper with the passed Maven parameters."
+inputs:
+  working_directory:
+    description: "The directory under which the Maven command should be executed."
+    default: "${{ github.workspace }}"
+  maven-parameters:
+    description: "Any parameters of the Maven command."
+    default: ""
+  env:
+    description: "Any environment-specifics that are meant to be available in the context of the call."
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: "Runs Maven Command"
+      working-directory: "${{ inputs.working_directory }}"
+      shell: bash
+      run: |
+        # errexit needs to be disabled explicitly here because maven-utils.sh handles the error if a mirror is not available
+        set +o errexit
+        
+        ${{ inputs.env }} source "./tools/ci/maven-utils.sh"
+        run_mvn ${{ inputs.maven-parameters }}

.github/workflows/ci.yml

@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This workflow is meant for as a basic CI run covering the most-important features
+# and default Java version. It is meant to run before a PullRequest is merged.
+
+name: "Flink CI (beta)"
+
+on:
+  push:
+  workflow_dispatch:
+
+permissions: read-all
+
+jobs:
+  pre-compile-checks:
+    name: "Pre-compile Checks"
+    uses: ./.github/workflows/template.pre-compile-checks.yml
+  ci:
+    name: "Default (Java 8)"
+    uses: ./.github/workflows/template.flink-ci.yml
+    with:
+      environment: 'PROFILE="-Dinclude_hadoop_aws"'
+      jdk_version: 8
+    secrets:
+      s3_bucket: ${{ secrets.IT_CASE_S3_BUCKET }}
+      s3_access_key: ${{ secrets.IT_CASE_S3_ACCESS_KEY }}
+      s3_secret_key: ${{ secrets.IT_CASE_S3_SECRET_KEY }}

.github/workflows/docs.sh

@@ -22,19 +22,20 @@ mvn --version
 java -version
 javadoc -J-version
 
-# setup hugo
-HUGO_REPO=https://github.com/gohugoio/hugo/releases/download/v0.110.0/hugo_extended_0.110.0_Linux-64bit.tar.gz
-HUGO_ARTIFACT=hugo_extended_0.110.0_Linux-64bit.tar.gz
-if ! curl --fail -OL $HUGO_REPO ; then
-	echo "Failed to download Hugo binary"
-	exit 1
-fi
-tar -zxvf $HUGO_ARTIFACT -C /usr/local/bin
+# workaround for a git security patch
+git config --global --add safe.directory /root/flink
 git submodule update --init --recursive
-# Setup the external documentation modules
+
 cd docs
+
+# setup hugo
+source setup_hugo.sh
+
+# Setup the external documentation modules
 source setup_docs.sh
+
 cd ..
+
 # Build the docs
 hugo --source docs
 

.github/workflows/docs.yml

@@ -28,9 +28,10 @@ jobs:
       matrix:
         branch:
           - master
+          - release-1.19
+          - release-1.18
           - release-1.17
           - release-1.16
-          - release-1.15
     steps:
       - uses: actions/checkout@v3
         with:
@@ -42,13 +43,13 @@ jobs:
           echo "flink_branch=${currentBranch}" >> ${GITHUB_ENV}
 
           if [ "${currentBranch}" = "master" ]; then
-            echo "flink_alias=release-1.18" >> ${GITHUB_ENV}
-          elif [ "${currentBranch}" = "release-1.17" ]; then
+            echo "flink_alias=release-1.19" >> ${GITHUB_ENV}
+          elif [ "${currentBranch}" = "release-1.18" ]; then
             echo "flink_alias=stable" >> ${GITHUB_ENV}
           fi
       - name: Build documentation
         run: |
-          docker run  --rm --volume "$PWD:/root/flink" chesnay/flink-ci:java_8_11 bash -c "cd /root/flink && ./.github/workflows/docs.sh"
+          docker run --rm --volume "$PWD:/root/flink" mapohl/flink-ci:FLINK-34194 bash -c "cd /root/flink && ./.github/workflows/docs.sh"
       - name: Upload documentation
         uses: burnett01/[email protected]
         with:

.github/workflows/nightly-trigger.yml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This workflow is meant as an extended CI run that includes certain features that shall be tested
+# and JDK versions that are supported but not considered default.
+
+name: "Nightly trigger (beta)"
+
+on:
+  schedule:
+    - cron: '0 2 * * *'
+
+jobs:
+  Trigger:
+    if: github.repository == 'apache/flink'
+    permissions:
+      actions: write
+    strategy:
+      matrix:
+        branch:
+          - master
+          - release-1.19
+          - release-1.18
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger Workflow
+        uses: actions/github-script@v7
+        with:
+          script: |
+            github.rest.actions.createWorkflowDispatch({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: 'nightly.yml',
+              ref: '${{ matrix.branch }}'
+            })

.github/workflows/nightly.yml

@@ -0,0 +1,96 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This workflow is meant as an extended CI run that includes certain features that shall be tested
+# and JDK versions that are supported but not considered default.
+
+name: "Nightly (beta)"
+
+on:
+  workflow_dispatch:
+
+permissions: read-all
+
+jobs:
+  pre-compile-checks:
+    name: "Pre-compile Checks"
+    uses: ./.github/workflows/template.pre-compile-checks.yml
+
+  java8:
+    name: "Java 8"
+    uses: ./.github/workflows/template.flink-ci.yml
+    with:
+      workflow-caller-id: java8
+      environment: 'PROFILE="-Dinclude_hadoop_aws"'
+      jdk_version: 8
+    secrets:
+      s3_bucket: ${{ secrets.IT_CASE_S3_BUCKET }}
+      s3_access_key: ${{ secrets.IT_CASE_S3_ACCESS_KEY }}
+      s3_secret_key: ${{ secrets.IT_CASE_S3_SECRET_KEY }}
+  java11:
+    name: "Java 11"
+    uses: ./.github/workflows/template.flink-ci.yml
+    with:
+      workflow-caller-id: java11
+      environment: 'PROFILE="-Dinclude_hadoop_aws -Djdk11 -Pjava11-target"'
+      jdk_version: 11
+    secrets:
+      s3_bucket: ${{ secrets.IT_CASE_S3_BUCKET }}
+      s3_access_key: ${{ secrets.IT_CASE_S3_ACCESS_KEY }}
+      s3_secret_key: ${{ secrets.IT_CASE_S3_SECRET_KEY }}
+  java17:
+    name: "Java 17"
+    uses: ./.github/workflows/template.flink-ci.yml
+    with:
+      workflow-caller-id: java17
+      environment: 'PROFILE="-Dinclude_hadoop_aws -Djdk11 -Djdk17 -Pjava17-target"'
+      jdk_version: 17
+    secrets:
+      s3_bucket: ${{ secrets.IT_CASE_S3_BUCKET }}
+      s3_access_key: ${{ secrets.IT_CASE_S3_ACCESS_KEY }}
+      s3_secret_key: ${{ secrets.IT_CASE_S3_SECRET_KEY }}
+  java21:
+    name: "Java 21"
+    uses: ./.github/workflows/template.flink-ci.yml
+    with:
+      workflow-caller-id: java21
+      environment: 'PROFILE="-Dinclude_hadoop_aws -Djdk11 -Djdk17 -Djdk21 -Pjava21-target"'
+      jdk_version: 21
+    secrets:
+      s3_bucket: ${{ secrets.IT_CASE_S3_BUCKET }}
+      s3_access_key: ${{ secrets.IT_CASE_S3_ACCESS_KEY }}
+      s3_secret_key: ${{ secrets.IT_CASE_S3_SECRET_KEY }}
+  hadoop313:
+    name: "Hadoop 3.1.3"
+    uses: ./.github/workflows/template.flink-ci.yml
+    with:
+      workflow-caller-id: hadoop313
+      environment: 'PROFILE="-Dflink.hadoop.version=3.2.3 -Phadoop3-tests,hive3"'
+      jdk_version: 8
+    secrets:
+      s3_bucket: ${{ secrets.IT_CASE_S3_BUCKET }}
+      s3_access_key: ${{ secrets.IT_CASE_S3_ACCESS_KEY }}
+      s3_secret_key: ${{ secrets.IT_CASE_S3_SECRET_KEY }}
+  adaptive-scheduler:
+    name: "AdaptiveScheduler"
+    uses: ./.github/workflows/template.flink-ci.yml
+    with:
+      workflow-caller-id: adaptive-scheduler
+      environment: 'PROFILE="-Penable-adaptive-scheduler"'
+      jdk_version: 8
+    secrets:
+      s3_bucket: ${{ secrets.IT_CASE_S3_BUCKET }}
+      s3_access_key: ${{ secrets.IT_CASE_S3_ACCESS_KEY }}
+      s3_secret_key: ${{ secrets.IT_CASE_S3_SECRET_KEY }}