Implement provider versioning tools (#13767)

This change implements per-provider versioning tools. Version of the providers is retrieved from provider.yaml file (top-level verion). Documentation is generated in the documentation folder rather than in sources and embedded in provider's index. Backport providers remain as they were until we delete all the backport references in April 2021 nd then the code can be simplified and the backport functionality can be removed then. When generating multiple providers, only those that have version that has no corresponding `providers-<PROVIDER>/<VERSION>` are generated. Other providers are skipped with warnings. Old documentation is removed and new CHANGELOG.rst have been prepared for all providers to accomodate to the new process (which is comming as a follow-up commit) Fixes: #13272, #13271, #13274, #13276, #13277, #13275, #13273 (cherry picked from commit ac2f72c)
apache · Mar 3, 2021 · 07d924d · 07d924d
1 parent 76c249e
commit 07d924d
Show file tree

Hide file tree

Showing 257 changed files with 5,539 additions and 15,458 deletions.
diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml
@@ -22,7 +22,7 @@ on:  # yamllint disable-line rule:truthy
     workflows: ["CI Build"]
     types: ['requested']
 env:
-  MOUNT_LOCAL_SOURCES: "false"
+  MOUNT_SELECTED_LOCAL_SOURCES: "false"
   FORCE_ANSWER_TO_QUESTIONS: "yes"
   FORCE_PULL_IMAGES: "true"
   CHECK_IMAGE_FOR_REBUILD: "true"
@@ -405,6 +405,8 @@ jobs:
       GITHUB_REGISTRY_PULL_IMAGE_TAG: ${{ github.event.workflow_run.id }}
       UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
       DOCKER_CACHE: ${{ needs.cancel-workflow-runs.outputs.cacheDirective }}
+      VERSION_SUFFIX_FOR_PYPI: "dev"
+      VERSION_SUFFIX_FOR_SVN: "dev"
     steps:
       - name: >
           Checkout [${{ needs.cancel-workflow-runs.outputs.sourceEvent }}]

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -27,7 +27,7 @@ on:  # yamllint disable-line rule:truthy
 
 env:
 
-  MOUNT_LOCAL_SOURCES: "false"
+  MOUNT_SELECTED_LOCAL_SOURCES: "false"
   FORCE_ANSWER_TO_QUESTIONS: "yes"
   FORCE_PULL_IMAGES: "true"
   CHECK_IMAGE_FOR_REBUILD: "true"
@@ -275,7 +275,7 @@ jobs:
     needs: [build-info, ci-images]
     env:
       SKIP: "pylint,identity"
-      MOUNT_LOCAL_SOURCES: "true"
+      MOUNT_SELECTED_LOCAL_SOURCES: "true"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
       GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     if: needs.build-info.outputs.basic-checks-only == 'false'
@@ -327,7 +327,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
     needs: [build-info]
     env:
       SKIP: "build,mypy,flake8,pylint,bats-in-container-tests,identity"
-      MOUNT_LOCAL_SOURCES: "true"
+      MOUNT_SELECTED_LOCAL_SOURCES: "true"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
     if: needs.build-info.outputs.basic-checks-only == 'true'
     steps:
@@ -380,7 +380,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
     env:
       # We want to make sure we have latest sources as only in_container scripts are added
       # to the image but we want to static-check all of them
-      MOUNT_LOCAL_SOURCES: "true"
+      MOUNT_SELECTED_LOCAL_SOURCES: "true"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
       GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     steps:
@@ -470,6 +470,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
       BACKPORT_PACKAGES: "true"
       VERSION_SUFFIX_FOR_PYPI: "dev"
+      VERSION_SUFFIX_FOR_SVN: "dev"
       PACKAGE_FORMAT: ${{ matrix.package-format }}
       GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     if: needs.build-info.outputs.image-build == 'true'
@@ -487,8 +488,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
         if: "!contains(needs.build-info.outputs.runsOn, 'self-hosted')"
       - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
         run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
-      - name: "Prepare provider readmes"
-        run: ./scripts/ci/provider_packages/ci_prepare_provider_readmes.sh
+      - name: "Prepare provider documentation"
+        run: ./scripts/ci/provider_packages/ci_prepare_provider_documentation.sh
       - name: "Prepare provider packages: ${{ matrix.package-format }}"
         run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh
       - name: "Install and test provider packages and airflow via ${{ matrix.package-format }} files"
@@ -502,7 +503,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           name: airflow-backport-packages
           path: "./dist/apache*"
           retention-days: 7
-      - name: "Upload readme artifacts"
+      - name: "Upload documentation artifacts"
         uses: actions/upload-artifact@v2
         if: always() && matrix.package-format == 'wheel'
         with:
@@ -520,6 +521,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       AIRFLOW_EXTRAS: "all"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
       VERSION_SUFFIX_FOR_PYPI: "dev"
+      VERSION_SUFFIX_FOR_SVN: "dev"
       PACKAGE_FORMAT: ${{ matrix.package-format }}
       GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     strategy:
@@ -540,8 +542,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
         if: "!contains(needs.build-info.outputs.runsOn, 'self-hosted')"
       - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
         run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
-      - name: "Prepare provider readmes"
-        run: ./scripts/ci/provider_packages/ci_prepare_provider_readmes.sh
+      - name: "Prepare provider documentation"
+        run: ./scripts/ci/provider_packages/ci_prepare_provider_documentation.sh
       - name: "Prepare provider packages: ${{ matrix.package-format }}"
         run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh
       - name: "Prepare airflow packages: ${{ matrix.package-format }}"
@@ -555,13 +557,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           name: airflow-provider-packages
           path: "./dist/apache-*"
           retention-days: 7
-      - name: "Upload readme artifacts"
-        uses: actions/upload-artifact@v2
-        if: always() && matrix.package-format == 'wheel'
-        with:
-          name: airflow-provider-readmes
-          path: "./files/airflow-readme-*"
-          retention-days: 7
 
   test-provider-packages-released-airflow:
     timeout-minutes: 30
@@ -573,6 +568,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       AIRFLOW_EXTRAS: "all"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
       VERSION_SUFFIX_FOR_PYPI: "dev"
+      VERSION_SUFFIX_FOR_SVN: "dev"
       PACKAGE_FORMAT: ${{ matrix.package-format }}
       GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     strategy:
@@ -593,8 +589,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
         if: "!contains(needs.build-info.outputs.runsOn, 'self-hosted')"
       - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
         run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
-      - name: "Prepare provider readmes"
-        run: ./scripts/ci/provider_packages/ci_prepare_provider_readmes.sh
+      - name: "Prepare provider documentation"
+        run: ./scripts/ci/provider_packages/ci_prepare_provider_documentation.sh
       - name: "Prepare provider packages: ${{ matrix.package-format }}"
         run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh
       - name: "Install and test provider packages and airflow via ${{ matrix.package-format }} files"
@@ -606,7 +602,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
     runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
     needs: [build-info, ci-images]
     env:
-      MOUNT_LOCAL_SOURCES: "true"
+      MOUNT_SELECTED_LOCAL_SOURCES: "true"
       RUN_TESTS: true
       TEST_TYPES: "Helm"
       BACKEND: "sqlite"

diff --git a/.github/workflows/scheduled_quarantined.yml b/.github/workflows/scheduled_quarantined.yml
@@ -23,7 +23,7 @@ on:  # yamllint disable-line rule:truthy
     - cron: '12 */6 * * *'
 
 env:
-  MOUNT_LOCAL_SOURCES: "false"
+  MOUNT_SELECTED_LOCAL_SOURCES: "false"
   FORCE_ANSWER_TO_QUESTIONS: "yes"
   FORCE_PULL_IMAGES: "true"
   CHECK_IMAGE_FOR_REBUILD: "true"

diff --git a/BREEZE.rst b/BREEZE.rst
@@ -558,20 +558,19 @@ The below example builds provider packages in the wheel format.
      ./breeze prepare-provider-packages
 
 If you run this command without packages, you will prepare all packages, you can however specify
-providers that you would like to build. By default only ``wheel`` packages are prepared,
-but you can change it providing optional --package-format flag.
-
+providers that you would like to build. By default ``both`` types of packages are prepared (
+``wheel`` and ``sdist``, but you can change it providing optional --package-format flag.
 
 .. code-block:: bash
 
-     ./breeze prepare-provider-packages --package-format=both google amazon
+     ./breeze prepare-provider-packages google amazon
 
 You can also prepare backport provider packages, if you specify ``--backport`` flag. You can read more
 about backport packages in `dev <dev/README.md>`_
 
 .. code-block:: bash
 
-     ./breeze prepare-provider-packages --backports --package-format=both google amazon
+     ./breeze prepare-provider-packages --backports google amazon
 
 You can see all providers available by running this command:
 
@@ -588,11 +587,12 @@ You can also prepare airflow packages using breeze:
 
 This prepares airflow .whl package in the dist folder.
 
-Again, you can specify optional ``--package-format`` flag to build airflow packages.
+Again, you can specify optional ``--package-format`` flag to build selected formats of airflow packages,
+default is to build ``both`` type of packages ``sdist`` and ``wheel``.
 
 .. code-block:: bash
 
-     ./breeze prepare-airflow-packages --package-format=bot
+     ./breeze prepare-airflow-packages --package-format=wheel
 
 
 Building Production images
@@ -1167,12 +1167,12 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
   Commands with arguments:
 
-    docker-compose                <ARG>      Executes specified docker-compose command
-    kind-cluster                  <ARG>      Manages KinD cluster on the host
-    prepare-provider-readme       <ARG>      Prepares provider packages readme files
-    prepare-provider-packages     <ARG>      Prepares provider packages
-    static-check                  <ARG>      Performs selected static check for changed files
-    tests                         <ARG>      Runs selected tests in the container
+    docker-compose                     <ARG>      Executes specified docker-compose command
+    kind-cluster                       <ARG>      Manages KinD cluster on the host
+    prepare-provider-documentation     <ARG>      Prepares provider packages documentation
+    prepare-provider-packages          <ARG>      Prepares provider packages
+    static-check                       <ARG>      Performs selected static check for changed files
+    tests                              <ARG>      Runs selected tests in the container
 
   Help commands:
 
@@ -1467,7 +1467,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
           and you need to be committer to push to Apache Airflow' GitHub registry.
 
   --github-registry GITHUB_REGISTRY
-          Github registry used. GitHub has legacy Packages registry and Public Beta Container
+          GitHub registry used. GitHub has legacy Packages registry and Public Beta Container
           registry.
 
           Default: docker.pkg.github.com.
@@ -1661,7 +1661,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
           and you need to be committer to push to Apache Airflow' GitHub registry.
 
   --github-registry GITHUB_REGISTRY
-          Github registry used. GitHub has legacy Packages registry and Public Beta Container
+          GitHub registry used. GitHub has legacy Packages registry and Public Beta Container
           registry.
 
           Default: docker.pkg.github.com.
@@ -1753,9 +1753,13 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
           One of:
 
-                 wheel,sdist,both
+                 both,sdist,wheel
+
+          Default: both
 
-          Default: wheel
+  --backports
+
+          Prepares backport providers rather than regular ones.
 
   -v, --verbose
           Show verbose information about executed docker, kind, kubectl, helm commands. Useful for
@@ -2103,26 +2107,31 @@ This is the current syntax for  `./breeze <./breeze>`_:
   ####################################################################################################
 
 
-  Detailed usage for command: prepare-provider-readme
+  Detailed usage for command: prepare-provider-documentation
+
 
+  breeze prepare-provider-documentation [FLAGS] [YYYY.MM.DD] [PACKAGE_ID ...]
 
-  breeze prepare-provider-readme [FLAGS] [YYYY.MM.DD] [PACKAGE_ID ...]
+        Prepares documentation files for provider packages.
 
-        Prepares README.md files for backport packages. You can provide (after --) optional version
-        in the form of YYYY.MM.DD, optionally followed by the list of packages to generate readme for.
+        The command is optionally followed by the list of packages to generate readme for.
         If the first parameter is not formatted as a date, then today is regenerated.
         If no packages are specified, readme for all packages are generated.
         If no date is specified, current date + 3 days is used (allowing for PMC votes to pass).
 
+        You can also specify --backport flag to prepare backport providers documentation and in this
+        case you can also optionally specify CALVER version as first parameter.
+
         Examples:
 
-        'breeze prepare-provider-readme' or
-        'breeze prepare-provider-readme 2020.05.10' or
-        'breeze prepare-provider-readme 2020.05.10 https google amazon'
+        'breeze prepare-provider-documentation' or
+        'breeze prepare-provider-documentation --version-suffix-for-pypi rc1' or
+        'breeze prepare-provider-documentation --backports 2020.05.10' or
+        'breeze prepare-provider-documentation --backports 2020.05.10 https google amazon'
 
         General form:
 
-        'breeze prepare-provider-readme YYYY.MM.DD <PACKAGE_ID> ...'
+        'breeze prepare-provider-documentation YYYY.MM.DD <PACKAGE_ID> ...'
 
         * YYYY.MM.DD - is the CALVER version of the package to prepare. Note that this date
           cannot be earlier than the already released version (the script will fail if it
@@ -2134,6 +2143,28 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
   Flags:
 
+  -S, --version-suffix-for-pypi SUFFIX
+          Adds optional suffix to the version in the generated backport package. It can be used
+          to generate rc1/rc2 ... versions of the packages to be uploaded to PyPI.
+
+  -N, --version-suffix-for-svn SUFFIX
+          Adds optional suffix to the generated names of package. It can be used to generate
+          rc1/rc2 ... versions of the packages to be uploaded to SVN.
+
+  --package-format PACKAGE_FORMAT
+
+          Chooses format of packages to prepare.
+
+          One of:
+
+                 both,sdist,wheel
+
+          Default: both
+
+  --backports
+
+          Prepares backport providers rather than regular ones.
+
   -v, --verbose
           Show verbose information about executed docker, kind, kubectl, helm commands. Useful for
           debugging - when you run breeze with --verbose flags you will be able to see the commands
@@ -2165,19 +2196,22 @@ This is the current syntax for  `./breeze <./breeze>`_:
         prepared there so make sure you run prepare-provider-packages first,
         and prepare-airflow-packages second.
 
+        You can also specify --backport flag to prepare backport providers or --package-format to
+        prepare one or both types of supported formats.
+
         Examples:
 
         'breeze prepare-provider-packages' or
         'breeze prepare-provider-packages google' or
-        'breeze prepare-provider-packages --package-format both google' or
+        'breeze prepare-provider-packages --package-format wheel google' or
         'breeze prepare-provider-packages --version-suffix-for-svn rc1 http google amazon' or
         'breeze prepare-provider-packages --version-suffix-for-pypi rc1 http google amazon'
         'breeze prepare-provider-packages --version-suffix-for-pypi a1
                                               --version-suffix-for-svn a1 http google amazon'
 
         General form:
 
-        'breeze prepare-provider-packages [--package-format PACKAGE_FORMAT] \
+        'breeze prepare-provider-packages [--backports] [--package-format PACKAGE_FORMAT] \
               [--version-suffix-for-svn|--version-suffix-for-pypi] <PACKAGE_ID> ...'
 
         * <PACKAGE_ID> is usually directory in the airflow/providers folder (for example
@@ -2192,9 +2226,13 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
           One of:
 
-                 wheel,sdist,both
+                 both,sdist,wheel
+
+          Default: both
+
+  --backports
 
-          Default: wheel
+          Prepares backport providers rather than regular ones.
 
   -S, --version-suffix-for-pypi SUFFIX
           Adds optional suffix to the version in the generated backport package. It can be used
@@ -2667,7 +2705,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
           and you need to be committer to push to Apache Airflow' GitHub registry.
 
   --github-registry GITHUB_REGISTRY
-          Github registry used. GitHub has legacy Packages registry and Public Beta Container
+          GitHub registry used. GitHub has legacy Packages registry and Public Beta Container
           registry.
 
           Default: docker.pkg.github.com.

diff --git a/CI.rst b/CI.rst
@@ -178,7 +178,7 @@ You can use those variables when you try to reproduce the build locally.
 +-----------------------------------------+----------------------------------------+-------------------------------------------------+
 |                                                           Mount variables                                                          |
 +-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+
-| ``MOUNT_LOCAL_SOURCES``                 |     true    |    false    |    false   | Determines whether local sources are            |
+| ``MOUNT_SELECTED_LOCAL_SOURCES``        |     true    |    false    |    false   | Determines whether local sources are            |
 |                                         |             |             |            | mounted to inside the container. Useful for     |
 |                                         |             |             |            | local development, as changes you make          |
 |                                         |             |             |            | locally can be immediately tested in            |
@@ -189,6 +189,15 @@ You can use those variables when you try to reproduce the build locally.
 |                                         |             |             |            | directories) generated locally on the           |
 |                                         |             |             |            | host during development.                        |
 +-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+
+| ``MOUNT_ALL_LOCAL_SOURCES``             |     false   |    false    |    false   | Determines whether all local sources are        |
+|                                         |             |             |            | mounted to inside the container. Useful for     |
+|                                         |             |             |            | local development when you need to access .git  |
+|                                         |             |             |            | folders and other folders excluded when         |
+|                                         |             |             |            | ``MOUNT_SELECTED_LOCAL_SOURCES`` is true.       |
+|                                         |             |             |            | You might need to manually delete egg-info      |
+|                                         |             |             |            | folder when you enter breeze and the folder was |
+|                                         |             |             |            | generated using different python versions.      |
++-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+
 |                                                           Force variables                                                          |
 +-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+
 | ``FORCE_PULL_IMAGES``                   |    true     |    true     |    true    | Determines if images are force-pulled,          |