From 9366f867ca6e76225ded404e249c90a18212bfd1 Mon Sep 17 00:00:00 2001 From: Sajid Alam <90610031+SajidAlamQB@users.noreply.github.com> Date: Fri, 19 May 2023 19:00:51 +0100 Subject: [PATCH] ci: Remove the CircleCI setup (#209) * remove circleci setup files and utils * remove circleci configs in kedro-telemetry * remove redundant .github in kedro-telemetry * Delete continue_config.yml * Update check-release.yml * lint * increase timeout to 40 mins for docker e2e tests --- .circleci/config.yml | 38 -- .circleci/continue_config.yml | 516 ------------------ .github/workflows/merge-gatekeeper.yml | 3 +- kedro-telemetry/.circleci/config.yml | 131 ----- .../.github/ISSUE_TEMPLATE/bug-report.md | 41 -- .../.github/ISSUE_TEMPLATE/feature-request.md | 20 - .../.github/PULL_REQUEST_TEMPLATE.md | 13 - tools/circleci/circleci_release.py | 78 --- tools/circleci/github_release.py | 52 -- tools/circleci/utils/check_no_version_pypi.py | 13 - tools/circleci/utils/package_version.py | 21 - 11 files changed, 1 insertion(+), 925 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 .circleci/continue_config.yml delete mode 100644 kedro-telemetry/.circleci/config.yml delete mode 100644 kedro-telemetry/.github/ISSUE_TEMPLATE/bug-report.md delete mode 100644 kedro-telemetry/.github/ISSUE_TEMPLATE/feature-request.md delete mode 100644 kedro-telemetry/.github/PULL_REQUEST_TEMPLATE.md delete mode 100755 tools/circleci/circleci_release.py delete mode 100755 tools/circleci/github_release.py delete mode 100644 tools/circleci/utils/check_no_version_pypi.py delete mode 100644 tools/circleci/utils/package_version.py diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index b8a27e1c3..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,38 +0,0 @@ -version: 2.1 - -parameters: - release_package: - type: string - default: "" - release_version: - type: string - default: "" - -setup: true - -# the path-filtering orb is required to continue a pipeline based on -# the path of an updated fileset -orbs: - path-filtering: circleci/path-filtering@0.1.1 - -workflows: - always-run: - jobs: - # the path-filtering/filter job determines which pipeline - # parameters to update. - - path-filtering/filter: - name: check-updated-files - # 3-column, whitespace-delimited mapping. One mapping per - # line: - # - mapping: | - kedro-telemetry/.* run-build-kedro-telemetry true - kedro-docker/.* run-build-kedro-docker true - kedro-airflow/.* run-build-kedro-airflow true - kedro-datasets/.* run-build-kedro-datasets true - base-revision: main - # this is the path of the configuration we should trigger once - # path filtering and pipeline parameter value updates are - # complete. In this case, we are using the parent dynamic - # configuration itself. - config-path: .circleci/continue_config.yml diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml deleted file mode 100644 index d339e82c1..000000000 --- a/.circleci/continue_config.yml +++ /dev/null @@ -1,516 +0,0 @@ -version: 2.1 - -orbs: - win: circleci/windows@2.4.1 - -# the default pipeline parameters, which will be updated according to -# the results of the path-filtering orb -parameters: - run-build-kedro-telemetry: - type: boolean - default: false - run-build-kedro-docker: - type: boolean - default: false - run-build-kedro-airflow: - type: boolean - default: false - run-build-kedro-datasets: - type: boolean - default: false - release_package: - type: string - default: "" - release_version: - type: string - default: "" - -commands: - setup_conda: - parameters: - python_version: - type: string - steps: - - run: - name: Cleanup pyenv - command: sudo rm -rf .pyenv/ /opt/circleci/.pyenv/ - - run: - name: Download and install miniconda - command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh > miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - - run: - name: Create conda environment with correct python version - command: | - . /home/circleci/miniconda/etc/profile.d/conda.sh - conda create --name kedro_plugins python=<> -y - - run: - name: Setup bash env to run conda activation at each step - command: | - echo ". /home/circleci/miniconda/etc/profile.d/conda.sh" >> $BASH_ENV - echo "conda deactivate; conda activate kedro_plugins" >> $BASH_ENV - source $BASH_ENV - - setup_requirements: - parameters: - plugin: - type: string - steps: - - run: - name: Install pip setuptools - command: make install-pip-setuptools - - run: - # pytables does not work properly with python 3.9 to handle our HDFDataSet - # if pip-installed, so we install this dependency via conda - name: Install pytables - command: conda install -c conda-forge pytables -y - - run: - name: Install kedro and test requirements - command: | - cd <> - pip install git+https://github.com/kedro-org/kedro@main - pip install . -r test_requirements.txt # TODO(deepyaman): Define `test` extra and `pip install .[test]` - - run: - name: Install pre-commit hooks - command: | - cd <> - pre-commit install --install-hooks - pre-commit install --hook-type pre-push - - run: - # this is needed to fix java cacerts so - # spark can automatically download packages from mvn - # https://stackoverflow.com/a/50103533/1684058 - name: Fix cacerts - command: | - sudo rm /etc/ssl/certs/java/cacerts - sudo update-ca-certificates -f - - run: - # Since recently Spark installation for some reason does not have enough permissions to execute - # /home/circleci/miniconda/envs/kedro_plugins/lib/python3.X/site-packages/pyspark/bin/spark-class. - # So fixing it manually here. - name: Fix Spark permissions - command: sudo chmod -R u+x /home/circleci/miniconda/envs/kedro_plugins/lib/ - - run: - name: Pip freeze - command: pip freeze - - setup: - parameters: - python_version: - type: string - plugin: - type: string - steps: - - checkout - - setup_conda: - python_version: <> - - setup_requirements: - plugin: <> - - # Windows specific commands - win_setup_conda: - # Miniconda3 is pre-installed on the machine: - # https://circleci.com/docs/2.0/hello-world-windows - parameters: - python_version: - type: string - steps: - - run: - name: Initialize conda - command: conda init powershell - - run: - name: Create 'kedro_plugins' conda environment - command: conda create --name kedro_plugins python=<> -y - - - win_setup_env: - steps: - - run: - # Required for Tensorflow tests - name: Install Microsoft Visual C++ Redistributable - command: | - $ProgressPreference = "SilentlyContinue" - Invoke-WebRequest https://aka.ms/vs/16/release/vc_redist.x64.exe -OutFile vc_redist.x64.exe - .\vc_redist.x64.exe /S /v/qn - - run: - name: Install Java 8 - command: | - $ProgressPreference = "SilentlyContinue" - Invoke-WebRequest https://github.com/AdoptOpenJDK/openjdk8-upstream-binaries/releases/download/jdk8u252-b09/OpenJDK8U-jdk_x64_windows_8u252b09.zip -OutFile OpenJDK8U.zip - Expand-Archive .\OpenJDK8U.zip -DestinationPath C:\OpenJDK8U - - run: - name: Create Inbound rules for Java - command: | - New-NetFirewallRule -DisplayName "Allow JDK UDP" -Profile "Public" -Protocol "UDP" -Direction Inbound -Program "C:\OpenJDK8U\openjdk-8u252-b09\bin\java.exe" -Action Allow - New-NetFirewallRule -DisplayName "Allow JDK TCP" -Profile "Public" -Protocol "TCP" -Direction Inbound -Program "C:\OpenJDK8U\openjdk-8u252-b09\bin\java.exe" -Action Allow - - run: - name: Set Java environment variables - command: | - [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable('Path', 'Machine') + ";C:\OpenJDK8U\openjdk-8u252-b09\bin", "Machine") - setx /m JAVA_HOME "C:\OpenJDK8U\openjdk-8u252-b09" - - run: - name: Setup Hadoop binary - command: | - $ProgressPreference = "SilentlyContinue" - Invoke-WebRequest https://github.com/steveloughran/winutils/raw/master/hadoop-2.6.3/bin/winutils.exe -OutFile winutils.exe - New-Item -ItemType directory -Path C:\hadoop\bin - mv .\winutils.exe C:\hadoop\bin - setx /m HADOOP_HOME "C:\hadoop\" - - run: - name: Install 'make' command - command: choco install make - - - win_setup_requirements: - parameters: - plugin: - type: string - python_version: - type: string - steps: - - run: - name: Install GDAL, Fiona and pytables - command: conda activate kedro_plugins; conda install gdal fiona pytables -c conda-forge -y - - run: - name: Install Kedro - command: conda activate kedro_plugins; pip install git+https://github.com/kedro-org/kedro@main - - run: - name: Install all requirements - command: conda activate kedro_plugins; cd <>; pip install . -r test_requirements.txt # TODO(deepyaman): Define `test` extra and `pip install .[test]` - - run: - name: Pip freeze - command: conda activate kedro_plugins; pip freeze - -jobs: - unit_tests: - parameters: - python_version: - type: string - plugin: - type: string - machine: - image: ubuntu-2004:202201-02 - docker_layer_caching: true - steps: - - setup: - python_version: <> - plugin: <> - - run: - name: Run unit tests - command: make plugin=<> test - - e2e_tests: - parameters: - python_version: - type: string - plugin: - type: string - machine: - image: ubuntu-2004:202201-02 - docker_layer_caching: true - steps: - - setup: - python_version: <> - plugin: <> - - run: - name: Run e2e tests - command: make plugin=<> e2e-tests - - lint: - parameters: - plugin: - type: string - machine: - image: ubuntu-2004:202201-02 - docker_layer_caching: true - steps: - - setup: - python_version: "3.8" - plugin: <> - - run: - name: Run pylint and flake8 - command: make plugin=<> lint - - win_unit_tests: - parameters: - python_version: - type: string - plugin: - type: string - executor: - name: win/default - steps: - - checkout - - win_setup_conda: - python_version: <> - - win_setup_env - - win_setup_requirements: - plugin: <> - python_version: <> - # For anything not `kedro-datasets` - - unless: - condition: - equal: ["kedro-datasets", <>] - - # e2e tests are not currently runnable on CircleCI on Windows as - # those require the ability to run Linux containers: - # "The Windows executor currently only supports Windows containers. - # Running Linux containers on Windows is not possible for now" - # (from https://circleci.com/docs/2.0/hello-world-windows/) - steps: - - run: - name: Run unit tests - command: | - conda activate kedro_plugins - cd <> - pytest tests - - - run: - # geopandas and tensorflow conflicts when imported simultaneously. - # The HDF5 header files used to compile this application do not match - # the version used by the HDF5 library to which this application is linked. - # Data corruption or segmentation faults may occur if the application continues. - # This can happen when an application was compiled by one version of HDF5 but - # linked with a different version of static or shared HDF5 library. - # You should recompile the application or check your shared library related - # settings such as 'LD_LIBRARY_PATH'. - # You can, at your own risk, disable this warning by setting the environment - # variable 'HDF5_DISABLE_VERSION_CHECK' to a value of '1'. - # Setting it to 2 or higher will suppress the warning messages totally. - name: Set HDF5_DISABLE_VERSION_CHECK environment variable - command: setx /m HDF5_DISABLE_VERSION_CHECK 1 - - when: - condition: - and: - - not: - equal: [ "3.10", <> ] - - equal: [ "kedro-datasets", <> ] - steps: - - run: - name: Run unit tests without spark in parallel - command: conda activate kedro_plugins; make test-no-spark - - when: - condition: - and: - - equal: [ "3.10", <> ] - - equal: [ "kedro-datasets", <> ] - steps: - - run: - name: Run unit tests without spark sequentially - command: conda activate kedro_plugins; make test-no-spark-sequential - - sync: - parameters: - python_version: - type: string - docker: - # https://circleci.com/docs/2.0/circleci-images/#circleci-base-image - - image: cimg/base:2020.01 - steps: - - checkout - - add_ssh_keys - - run: - name: Set git email and name - command: | - git config --global user.email "kedro@kedro.com" - git config --global user.name "Kedro" - # - run: - # name: Trigger Read The Docs build - # command: ./tools/circleci/rtd-build.sh ${RTD_TOKEN} latest - - setup_conda: - python_version: <> - - run: - name: Maybe trigger the release workflow - command: | - conda activate kedro_plugins - pip install requests - ./tools/circleci/circleci_release.py - - - # This is effectively just a combination of the lint, unit_tests and e2e_tests jobs. - # It's used to check that the nightly docker image is working ok and before publishing a release. - build_package: - parameters: - python_version: - type: string - machine: - image: ubuntu-2004:202201-02 - docker_layer_caching: true - steps: - - setup: - python_version: <> # Just need one Python version here - plugin: <> - - run: - name: Run linters - command: export plugin=<>; make lint - - unless: - condition: - equal: ["3.10", <>] - steps: - - run: - name: Run unit tests in parallel - command: export plugin=<>; make test - - when: - condition: - equal: [ "3.10", <> ] - steps: - - run: - name: Run unit tests sequentially - command: export plugin=<>; make test-sequential - - run: - name: Run e2e tests - command: make plugin=<> e2e-tests - - publish_package: - machine: - image: ubuntu-2004:202201-02 - docker_layer_caching: true - steps: - - run: - name: Print the release package and version - command: | - echo "Release package: <> <>" - - setup: - python_version: "3.8" # Just need one Python version here - plugin: <> # From circle_release.py - - add_ssh_keys - - run: - name: Tag and publish release on Github - command: ./tools/circleci/github_release.py <> <> - - run: - name: Publish to PyPI - command: | - export plugin=<> - make package - make pypi - - -workflows: - # when pipeline parameter, run-build-kedro-telemetry is true, the - # kedro-telemetry job is triggered. - kedro-telemetry: - when: - and: - - <> - - not: <> - - not: <> - jobs: - - unit_tests: - plugin: "kedro-telemetry" - matrix: - parameters: - python_version: ["3.7", "3.8", "3.9", "3.10"] - - win_unit_tests: - plugin: "kedro-telemetry" - matrix: - parameters: - python_version: ["3.7", "3.8", "3.9", "3.10"] - - lint: - plugin: "kedro-telemetry" - # when pipeline parameter, run-build-kedro-docker is true, the - # kedro-docker job is triggered. - kedro-docker: - when: - and: - - <> - - not: <> - - not: <> - jobs: - - unit_tests: - plugin: "kedro-docker" - matrix: - parameters: - python_version: ["3.7", "3.8", "3.9", "3.10"] - - e2e_tests: - plugin: "kedro-docker" - matrix: - parameters: - python_version: ["3.7", "3.8", "3.9", "3.10"] - - win_unit_tests: - plugin: "kedro-docker" - matrix: - parameters: - python_version: ["3.7", "3.8", "3.9", "3.10"] - - lint: - plugin: "kedro-docker" - # when pipeline parameter, run-build-kedro-airflow is true, the - # kedro-airflow job is triggered. - kedro-airflow: - when: - and: - - <> - - not: <> - - not: <> - jobs: - - unit_tests: - plugin: "kedro-airflow" - matrix: - parameters: - python_version: ["3.7", "3.8", "3.9", "3.10"] - pre-steps: - - run: - name: Avoid GPL dependency (unidecode) - command: echo 'export SLUGIFY_USES_TEXT_UNIDECODE=yes' >> $BASH_ENV - - e2e_tests: - plugin: "kedro-airflow" - matrix: - parameters: - python_version: ["3.7", "3.8", "3.9", "3.10"] - - win_unit_tests: - plugin: "kedro-airflow" - matrix: - parameters: - python_version: ["3.7", "3.8", "3.9", "3.10"] - - lint: - plugin: "kedro-airflow" - # when pipeline parameter, run-build-kedro-datasets is true, the - # kedro-datasets job is triggered. - kedro-datasets: - when: - and: - - <> - - not: <> - - not: <> - jobs: - - unit_tests: - plugin: "kedro-datasets" - matrix: - parameters: - python_version: ["3.7", "3.8", "3.9", "3.10"] - - win_unit_tests: - plugin: "kedro-datasets" - matrix: - parameters: - python_version: ["3.7", "3.8", "3.9", "3.10"] - - lint: - plugin: "kedro-datasets" - - # For release - main_updated: - when: - and: - - not: <> - - not: <> - jobs: - - sync: - filters: - branches: - only: main - matrix: - # We just need one Python enviornment to trigger the job - parameters: - python_version: ["3.8"] - - package_release: - when: - and: - - <> - - <> - jobs: - - build_package: - matrix: - parameters: - python_version: ["3.7", "3.8", "3.9", "3.10"] - - publish_package: - requires: - - build_package diff --git a/.github/workflows/merge-gatekeeper.yml b/.github/workflows/merge-gatekeeper.yml index be615ecbd..77ad752de 100644 --- a/.github/workflows/merge-gatekeeper.yml +++ b/.github/workflows/merge-gatekeeper.yml @@ -22,6 +22,5 @@ jobs: uses: upsidr/merge-gatekeeper@v1 with: token: ${{ secrets.GITHUB_TOKEN }} - timeout: 1800 + timeout: 2400 interval: 30 - diff --git a/kedro-telemetry/.circleci/config.yml b/kedro-telemetry/.circleci/config.yml deleted file mode 100644 index d0915db48..000000000 --- a/kedro-telemetry/.circleci/config.yml +++ /dev/null @@ -1,131 +0,0 @@ -version: 2.1 - -orbs: - win: circleci/windows@2.4.0 - -commands: - # Windows-related commands - win_setup_conda: - # Miniconda3 is pre-installed on the machine: - # https://circleci.com/docs/2.0/hello-world-windows/ - description: Setup conda - steps: - - run: - name: Initialize conda - command: conda init powershell - - run: - name: Create 'kedro-telemetry' conda environment - command: | - conda create --name kedro-telemetry python=$env:CONDA_ENV_PY_VERSION -y - - win_setup_requirements: - description: Install kedro-telemetry dependencies - steps: - - run: - name: Install kedro-telemetry dependencies - command: | - conda activate kedro-telemetry - python -m pip install -U pip setuptools wheel - pip install git+https://github.com/kedro-org/kedro@main - pip install -r test_requirements.txt -U - - win_build: - description: Run build on Windows - steps: - - checkout - - win_setup_conda - - win_setup_requirements - - run: - name: Run unit tests - command: | - conda activate kedro-telemetry - pytest .\tests - -jobs: - build_36: &DEFAULT - machine: - # Don't use 2018 image: https://discuss.circleci.com/t/24639/18 - image: circleci/classic:201711-01 - docker_layer_caching: true - environment: - CONDA_ENV_PY_VERSION: "3.6" - steps: - - checkout - - run: - name: Create virtual env - command: | - # Get rid of pyenv stuff - sudo rm -rf .pyenv/ /opt/circleci/.pyenv/ - # Download and install miniconda - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh > miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - # Create an anaconda virtualenv for ${CONDA_ENV_PY_VERSION} and make that the default python interpreter - echo ". /home/circleci/miniconda/etc/profile.d/conda.sh" >> $BASH_ENV - echo "conda deactivate; conda activate kedro-telemetry" >> $BASH_ENV - . /home/circleci/miniconda/etc/profile.d/conda.sh - conda create --name kedro-telemetry python=${CONDA_ENV_PY_VERSION} -y - source $BASH_ENV - - run: - name: Pip install dependencies - command: | - make install-pip-setuptools - pip install git+https://github.com/kedro-org/kedro - pip install -r test_requirements.txt - pre-commit install --install-hooks - pre-commit install --hook-type pre-push - - run: - name: Run pylint and flake8 - command: | - make lint - - run: - name: Run tests - command: | - make test - - build_37: - <<: *DEFAULT - environment: - CONDA_ENV_PY_VERSION: 3.7 - - build_38: - <<: *DEFAULT - environment: - CONDA_ENV_PY_VERSION: 3.8 - - # Windows-related jobs - win_build_36: - executor: - name: win/default - working_directory: ~/repo - environment: - CONDA_ENV_PY_VERSION: "3.6" - steps: [win_build] - - win_build_37: - executor: - name: win/default - working_directory: ~/repo - environment: - CONDA_ENV_PY_VERSION: "3.7" - steps: - - win_build - - win_build_38: - executor: - name: win/default - working_directory: ~/repo - environment: - CONDA_ENV_PY_VERSION: "3.8" - steps: - - win_build - -workflows: - version: 2 - regular: - jobs: - - build_36 - - build_37 - - build_38 - - win_build_36 - - win_build_37 - - win_build_38 diff --git a/kedro-telemetry/.github/ISSUE_TEMPLATE/bug-report.md b/kedro-telemetry/.github/ISSUE_TEMPLATE/bug-report.md deleted file mode 100644 index f89bf88bf..000000000 --- a/kedro-telemetry/.github/ISSUE_TEMPLATE/bug-report.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -name: Bug report -about: If something isn't working -title: '' -labels: 'Issue: Bug Report' -assignees: '' - ---- - -## Description -Short description of the problem here. - -## Context -How has this bug affected you? What were you trying to accomplish? - -## Steps to Reproduce -1. [First Step] -2. [Second Step] -3. [And so on...] - -## Expected Result -Tell us what should happen. - -## Actual Result -Tell us what happens instead. - -``` --- If you received an error, place it here. -``` - -``` --- Separate them if you have more than one. -``` - -## Your Environment -Include as many relevant details about the environment in which you experienced the bug: - -* Kedro-telemetry version used (`pip show kedro-telemetry`): -* Kedro version used (`pip show kedro` or `kedro -V`): -* Python version used (`python -V`): -* Operating system and version: diff --git a/kedro-telemetry/.github/ISSUE_TEMPLATE/feature-request.md b/kedro-telemetry/.github/ISSUE_TEMPLATE/feature-request.md deleted file mode 100644 index a7911c2f1..000000000 --- a/kedro-telemetry/.github/ISSUE_TEMPLATE/feature-request.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Feature request -about: Let us know if you have a feature request or enhancement -title: '<Title>' -labels: 'Issue: Feature Request' -assignees: '' - ---- - -## Description -Is your feature request related to a problem? A clear and concise description of what the problem is: "I'm always frustrated when ..." - -## Context -Why is this change important to you? How would you use it? How can it benefit other users? - -## Possible Implementation -(Optional) Suggest an idea for implementing the addition or change. - -## Possible Alternatives -(Optional) Describe any alternative solutions or features you've considered. diff --git a/kedro-telemetry/.github/PULL_REQUEST_TEMPLATE.md b/kedro-telemetry/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 9efb5746d..000000000 --- a/kedro-telemetry/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,13 +0,0 @@ -## Description -<!-- Why was this PR created? --> - -## Development notes -<!-- What have you changed, and how has this been tested? --> - -## Checklist - -- [ ] Read the [contributing](https://github.com/kedro-org/kedro/blob/main/CONTRIBUTING.md) guidelines -- [ ] Opened this PR as a 'Draft Pull Request' if it is work-in-progress -- [ ] Updated the documentation to reflect the code changes -- [ ] Added a description of this change in the [`RELEASE.md`](https://github.com/kedro-org/kedro/blob/main/RELEASE.md) file -- [ ] Added tests to cover my changes diff --git a/tools/circleci/circleci_release.py b/tools/circleci/circleci_release.py deleted file mode 100755 index dd05d4c5a..000000000 --- a/tools/circleci/circleci_release.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python3 -""" -CircleCI pipeline to check if it needs to trigger a release -""" - -import os -import sys - -import requests -from requests.structures import CaseInsensitiveDict - -from utils.check_no_version_pypi import check_no_version_pypi -from utils.package_version import get_package_version - -PACKAGE_PATHS = ( - "kedro-datasets/kedro_datasets", - "kedro-telemetry/kedro_telemetry", - "kedro-airflow/kedro_airflow", - "kedro-docker/kedro_docker", -) -PROJECT_SLUG = "github/kedro-org/kedro-plugins" -# CIRCLE_BRANCH = "feat/cicd-auto-release" -CIRCLE_BRANCH = os.environ.get("CIRCLE_BRANCH") - - -def circleci_release(project_slug, payload, circle_endpoint, circle_release_token): - """Trigging the CircleCI Release Pipeline""" - # See https://circleci.com/docs/2.0/api-developers-guide - print("Starting the CircleCI Release Pipeline") - CIRCLE_ENDPOINT = f"https://circleci.com/api/v2/project/{project_slug}/pipeline" - - headers = CaseInsensitiveDict() - headers["Content-Type"] = "application/json" - headers["Circle-Token"] = circle_release_token - - resp = requests.post(circle_endpoint, headers=headers, json=payload, timeout=10) - return resp - - -if __name__ == "__main__": - """Trigger the CircleCI Release Process""" - from pathlib import Path - - # Personal API Tokens - https://circleci.com/docs/managing-api-tokens - CIRCLE_RELEASE_TOKEN = os.environ.get("CIRCLE_RELEASE_TOKEN") - if not CIRCLE_RELEASE_TOKEN: - raise ValueError("CIRCLE_RELEASE_TOKEN is not defined as envionrmnet variable.") - - base_path = Path() - # Loop for all 4 repositories - for package_path in PACKAGE_PATHS: - package_name, _ = package_path.split("/") - package_version = get_package_version(base_path, package_path) - pypi_endpoint = f"https://pypi.org/pypi/{package_name}/{package_version}/json/" - circleci_endpoint = ( - f"https://circleci.com/api/v2/project/{PROJECT_SLUG}/pipeline" - ) - payload = { - "branch": CIRCLE_BRANCH, - "parameters": { - "release_package": package_name, - "release_version": package_version, - }, - } - - print(package_name, package_version) - if check_no_version_pypi(pypi_endpoint, package_name, package_version): - res = circleci_release( - PROJECT_SLUG, payload, circleci_endpoint, CIRCLE_RELEASE_TOKEN - ) - print(f"Status Code: {resp.status_code}") - if resp.status_code == 201: - print("Creating CircleCI Pipeline successfully") - else: - print("Failed to create CircleCI Pipeline") - print(resp.content) - if resp.status_code != 201: - sys.exit(1) diff --git a/tools/circleci/github_release.py b/tools/circleci/github_release.py deleted file mode 100755 index d5bc3115c..000000000 --- a/tools/circleci/github_release.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys - -import requests -from requests.structures import CaseInsensitiveDict - -GITHUB_USER = "kedro-org" -GITHUB_REPO = "kedro-plugins" -# On GitHub select "Settings" > "Developer Setting" -> "Personal access Token"" -GITHUB_TAGGING_TOKEN = os.environ.get("GITHUB_TAGGING_TOKEN") - - -def github_release( - package_name, - version, - github_user=GITHUB_USER, - github_repo=GITHUB_REPO, - github_tagging_token=GITHUB_TAGGING_TOKEN, -): - """Trigger the GitHub Release to create artifacts and tags""" - print("Starting GitHub Release") - - github_endpoint = ( - f"https://api.github.com/repos/{github_user}/{github_repo}/releases" - ) - payload = { - "tag_name": f"{package_name}-{version}", # kedro-datasets 0.0.1 - "target_commitish": "main", - "name": f"{version}", - "body": f"Release {version}", - "draft": False, - "prerelease": False, - } - - headers = CaseInsensitiveDict() - headers["Content-Type"] = "application/json" - headers["Authorization"] = f"token {github_tagging_token}" - resp = requests.post(github_endpoint, headers=headers, json=payload, timeout=10) - if resp.status_code == 200: - print("Create GitHub release successfully") - print(resp.content) - else: - print("Failed to create Github release") - print(resp.content) - return resp - - -if __name__ == "__main__": - package_name = sys.argv[1] - package_version = sys.argv[2] - res = github_release(package_name, package_version) diff --git a/tools/circleci/utils/check_no_version_pypi.py b/tools/circleci/utils/check_no_version_pypi.py deleted file mode 100644 index 777f09c9a..000000000 --- a/tools/circleci/utils/check_no_version_pypi.py +++ /dev/null @@ -1,13 +0,0 @@ -import requests - - -def check_no_version_pypi(pypi_endpoint, package_name, package_version): - print("Check if {package_name} {package_version} is on pypi") - response = requests.get(pypi_endpoint, timeout=10) - if response.status_code == 404: - # Not exist on Pypi - do release - print(f"Starting the release of {package_name} {package_version}") - return True - else: - print(f"Skipped: {package_name} {package_version} already exists on PyPI") - return False diff --git a/tools/circleci/utils/package_version.py b/tools/circleci/utils/package_version.py deleted file mode 100644 index 48de594ff..000000000 --- a/tools/circleci/utils/package_version.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 -""" -Get version of Kedro -""" - -import re -from pathlib import Path - -VERSION_MATCHSTR = r'\s*__version__\s*=\s*"(\d+\.\d+\.\d+)"' - - -def get_package_version(base_path, package_path): - init_file_path = Path(base_path) / package_path / "__init__.py" - match_obj = re.search(VERSION_MATCHSTR, Path(init_file_path).read_text()) - return match_obj.group(1) - - -if __name__ == "__main__": - base_path = Path() - package_path = "kedro-datasets/kedro_datasets" - print(get_package_version(base_path, package_path))