diff --git a/.circleci/config.yml b/.circleci/config.yml index a9756501c2..1006ce2e55 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,377 +1,21 @@ version: 2.1 -# add parameters +setup: true orbs: - win: circleci/windows@2.4.1 - -# No windows executor is listed here since windows builds use win/default and modify -# the Python version through the conda environment. -executors: - docker: - parameters: - python_version: - type: string - docker: - - image: cimg/python:<>-node - working_directory: ~/repo - -commands: - setup_python_env: - steps: - - run: - name: Install Python dependencies - command: | - pip install git+https://github.com/kedro-org/kedro@main - pip install -r package/test_requirements.txt -U - - run: - name: Echo package versions - command: | - python -V - pip freeze - - install_node_dependencies: - steps: - - run: - name: Install Node dependencies - command: npm install - - npm_build: - steps: - - run: - name: Build React application - command: make build - - python_package: - steps: - - run: - name: Make Python package - command: make package - - win_setup_python_env: - # Miniconda3 is pre-installed on the machine: - # https://circleci.com/docs/2.0/hello-world-windows/ - parameters: - python_version: - type: string - steps: - - run: - name: Initialize conda - command: conda init powershell - - run: - name: Create 'kedro-viz' conda environment - command: conda create --name kedro-viz python=<> -y - - run: - name: Install Kedro-Viz dependencies - command: | - conda activate kedro-viz - pip install git+https://github.com/kedro-org/kedro@main - pip install -r package/test_requirements.txt -U - - run: - name: Echo package versions - command: | - conda activate kedro-viz - python -V - pip freeze - - run: - name: Install 'make' command - command: choco install make - - setup: - steps: - - checkout - - setup_python_env - - install_node_dependencies - - npm_build - - win_setup: - parameters: - python_version: - type: string - executor: - steps: - - checkout - - win_setup_python_env: - python_version: <> - - install_node_dependencies - - npm_build - -jobs: - e2e_tests: - parameters: - python_version: - type: string - executor: - name: docker - python_version: <> - steps: - - setup - - run: - name: Run all end to end tests - command: make e2e-tests - - win_e2e_tests: - parameters: - python_version: - type: string - executor: win/default - steps: - - win_setup: - python_version: <> - - run: - name: Run all end to end tests on Windows - command: conda activate kedro-viz; make e2e-tests - - unit_tests: - parameters: - python_version: - type: string - executor: - name: docker - python_version: <> - steps: - - setup - - run: - name: Run Python tests - command: make pytest - - win_unit_tests: - parameters: - python_version: - type: string - executor: win/default - steps: - - win_setup: - python_version: <> - - run: - name: Run Python tests on Windows - command: conda activate kedro-viz; make pytest - - lint: - parameters: - python_version: - type: string - executor: - name: docker - python_version: <> - steps: - - checkout - - setup_python_env - - run: - name: Run Python formatters and linters - command: make format-check lint-check - - javascript_lint_and_tests: - executor: - name: docker - python_version: '3.8' - steps: - - checkout - - install_node_dependencies - - run: - name: Test lib transpilation - command: npm run lib - - run: - name: Test JS library imports - command: | - npm run lib-test:setup - cd tools/test-lib/react-app - npm run test:ci - - run: - name: Run Eslint - command: npm run lint - - run: - name: Run JavaScript tests - command: npm run build:css && npm run test:ci - - build_app: - executor: - name: docker - python_version: '3.8' - steps: - - setup - - run: - name: Run secret scan - command: make secret-scan - - run: - name: Run security scan - command: make security-scan - - python_package - - release_to_npm: - executor: - name: docker - python_version: '3.8' - steps: - - checkout - - install_node_dependencies - - npm_build - - run: - name: Authenticate with registry - command: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ~/repo/.npmrc - - run: - name: Publish to npm - command: npm publish - - release_to_pypi: - executor: - name: docker - python_version: '3.8' - steps: - - checkout - - setup_python_env - - install_node_dependencies - - python_package - - run: - name: Install twine - command: python -m pip install -U twine - - run: - name: Publish to PyPI - command: python -m twine upload package/dist/* - - deploy_demo: - executor: - name: docker - python_version: '3.8' - steps: - - checkout - - setup_remote_docker - - run: - name: Setup environment - command: | - cd demo-project - echo "AWS_ECR_URL=public.ecr.aws/g0x0s3o2/kedro-viz-live-demo" >> $BASH_ENV - echo "KEDRO_VIZ_VERSION=$(cat .version)" >> $BASH_ENV - echo "cd demo-project" >> $BASH_ENV - - run: - name: Install AWS CLI - command: pip3 install awscli - - run: - name: Build demo container image - command: | - echo "kedro_viz==$KEDRO_VIZ_VERSION" >> src/docker_requirements.txt - docker build -t $AWS_ECR_URL:$KEDRO_VIZ_VERSION . - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws - docker push $AWS_ECR_URL:$KEDRO_VIZ_VERSION - - run: - name: Create a new lightsail deployment - command: | - # install lightsail cli - # run https://docs.aws.amazon.com/cli/latest/reference/lightsail/create-container-service-deployment.html#create-container-service-deployment - aws lightsail create-container-service-deployment --region eu-west-2 --cli-input-json file://./lightsail.json - - all_circleci_checks_succeeded: - docker: - - image: circleci/python # any light-weight image - steps: - - run: - name: Success! - command: echo "All checks passed" - -release_filter: &release_filter - filters: - tags: - only: /^v[0-9]+\.[0-9]+\.[0-9]+$/ - branches: - ignore: /.*/ + path-filtering: circleci/path-filtering@0.1.1 workflows: - version: 2.1 - build: - jobs: - - e2e_tests: - matrix: - parameters: - python_version: ['3.7', '3.8', '3.9', '3.10'] - - win_e2e_tests: - matrix: - parameters: - python_version: ['3.7', '3.8', '3.9', '3.10'] - filters: - branches: - only: - - main - - demo - - unit_tests: - matrix: - parameters: - python_version: ['3.7', '3.8', '3.9', '3.10'] - - win_unit_tests: - matrix: - parameters: - python_version: ['3.7', '3.8', '3.9', '3.10'] - filters: - branches: - only: - - main - - demo - - lint: - matrix: - parameters: - python_version: ['3.7', '3.8', '3.9', '3.10'] - - javascript_lint_and_tests - - build_app - - deploy_demo: - context: - - kedro-ecr-publish - requires: - - e2e_tests - - win_e2e_tests - - unit_tests - - win_unit_tests - - lint - - javascript_lint_and_tests - - build_app - filters: - branches: - only: demo - - all_circleci_checks_succeeded: - requires: - - e2e_tests - - unit_tests - - lint - - javascript_lint_and_tests - - build_app - - release: - jobs: - - build_app: - <<: *release_filter - - release_to_npm: - <<: *release_filter - - release_to_pypi: - <<: *release_filter - requires: - - build_app - - daily: - triggers: - - schedule: - cron: '0 1 * * *' - filters: - branches: - only: - - main + regular: jobs: - - e2e_tests: - matrix: - parameters: - python_version: ['3.7', '3.8', '3.9', '3.10'] - - win_e2e_tests: - matrix: - parameters: - python_version: ['3.7', '3.8', '3.9', '3.10'] - - unit_tests: - matrix: - parameters: - python_version: ['3.7', '3.8', '3.9', '3.10'] - - win_unit_tests: - matrix: - parameters: - python_version: ['3.7', '3.8', '3.9', '3.10'] - - lint: - matrix: - parameters: - python_version: ['3.7', '3.8', '3.9', '3.10'] - - javascript_lint_and_tests - - build_app + # the path-filtering/filter job determines which pipeline + # parameters to update, i.e. which builds to run. + - path-filtering/filter: + name: check-updated-files + base-revision: main + config-path: .circleci/continue_config.yml + # + mapping: | + package/.* backend_change true + .circleci/.* ci_change true + ^((?!package/).)*$ frontend_change true \ No newline at end of file diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml new file mode 100644 index 0000000000..338e70e0c1 --- /dev/null +++ b/.circleci/continue_config.yml @@ -0,0 +1,379 @@ +version: 2.1 + +# Backend CI jobs are the heavy ones, so we only run them if the backend +# or CI config has changed. +parameters: + backend_change: + type: boolean + default: false + ci_change: + type: boolean + default: false + frontend_change: + type: boolean + default: false + +orbs: + win: circleci/windows@2.4.1 + +# No windows executor is listed here since windows builds use win/default and modify +# the Python version through the conda environment. +executors: + docker: + parameters: + python_version: + type: string + docker: + - image: cimg/python:<>-node + working_directory: ~/repo + +commands: + setup_python_env: + steps: + - run: + name: Install Python dependencies + command: | + pip install git+https://github.com/kedro-org/kedro@main + pip install -r package/test_requirements.txt -U + - run: + name: Echo package versions + command: | + python -V + pip freeze + + install_node_dependencies: + steps: + - run: + name: Install Node dependencies + command: npm install + + npm_build: + steps: + - run: + name: Build React application + command: make build + + win_setup_python_env: + # Miniconda3 is pre-installed on the machine: + # https://circleci.com/docs/2.0/hello-world-windows/ + parameters: + python_version: + type: string + steps: + - run: + name: Initialize conda + command: conda init powershell + - run: + name: Create 'kedro-viz' conda environment + command: conda create --name kedro-viz python=<> -y + - run: + name: Install Kedro-Viz dependencies + command: | + conda activate kedro-viz + pip install git+https://github.com/kedro-org/kedro@main + pip install -r package/test_requirements.txt -U + - run: + name: Echo package versions + command: | + conda activate kedro-viz + python -V + pip freeze + - run: + name: Install 'make' command + command: choco install make + + setup: + steps: + - checkout + - setup_python_env + - install_node_dependencies + - npm_build + + win_setup: + parameters: + python_version: + type: string + executor: + steps: + - checkout + - win_setup_python_env: + python_version: <> + - install_node_dependencies + - npm_build + +jobs: + e2e_tests: + parameters: + python_version: + type: string + executor: + name: docker + python_version: <> + steps: + - setup + - run: + name: Run all end to end tests + command: make e2e-tests + + win_e2e_tests: + parameters: + python_version: + type: string + executor: win/default + steps: + - win_setup: + python_version: <> + - run: + name: Run all end to end tests on Windows + command: conda activate kedro-viz; make e2e-tests + + unit_tests: + parameters: + python_version: + type: string + executor: + name: docker + python_version: <> + steps: + - setup + - run: + name: Run Python tests + command: make pytest + + win_unit_tests: + parameters: + python_version: + type: string + executor: win/default + steps: + - win_setup: + python_version: <> + - run: + name: Run Python tests on Windows + command: conda activate kedro-viz; make pytest + + lint: + parameters: + python_version: + type: string + executor: + name: docker + python_version: <> + steps: + - checkout + - setup_python_env + - run: + name: Run secret scan + command: make secret-scan + - run: + name: Run security scan + command: make security-scan + - run: + name: Run Python formatters and linters + command: make format-check lint-check + + javascript_lint_and_tests: + executor: + name: docker + python_version: '3.8' + steps: + - checkout + - install_node_dependencies + - run: + name: Test lib transpilation + command: npm run lib + - run: + name: Test JS library imports + command: | + npm run lib-test:setup + cd tools/test-lib/react-app + npm run test:ci + - run: + name: Run Eslint + command: npm run lint + - run: + name: Run JavaScript tests + command: npm run build:css && npm run test:ci + + release_to_npm: + executor: + name: docker + python_version: '3.8' + steps: + - checkout + - install_node_dependencies + - npm_build + - run: + name: Authenticate with registry + command: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ~/repo/.npmrc + - run: + name: Publish to npm + command: npm publish + + release_to_pypi: + executor: + name: docker + python_version: '3.8' + steps: + - setup + - run: + name: Make Python package + command: make package + - run: + name: Install twine + command: python -m pip install -U twine + - run: + name: Publish to PyPI + command: python -m twine upload package/dist/* + + deploy_demo: + executor: + name: docker + python_version: '3.8' + steps: + - checkout + - setup_remote_docker + - run: + name: Setup environment + command: | + cd demo-project + echo "AWS_ECR_URL=public.ecr.aws/g0x0s3o2/kedro-viz-live-demo" >> $BASH_ENV + echo "KEDRO_VIZ_VERSION=$(cat .version)" >> $BASH_ENV + echo "cd demo-project" >> $BASH_ENV + - run: + name: Install AWS CLI + command: pip3 install awscli + - run: + name: Build demo container image + command: | + echo "kedro_viz==$KEDRO_VIZ_VERSION" >> src/docker_requirements.txt + docker build -t $AWS_ECR_URL:$KEDRO_VIZ_VERSION . + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws + docker push $AWS_ECR_URL:$KEDRO_VIZ_VERSION + - run: + name: Create a new lightsail deployment + command: | + # install lightsail cli + # run https://docs.aws.amazon.com/cli/latest/reference/lightsail/create-container-service-deployment.html#create-container-service-deployment + aws lightsail create-container-service-deployment --region eu-west-2 --cli-input-json file://./lightsail.json + + all_circleci_checks_succeeded: + docker: + - image: circleci/python # any light-weight image + steps: + - run: + name: Success! + command: echo "All checks passed" + +release_filter: &release_filter + filters: + tags: + only: /^v[0-9]+\.[0-9]+\.[0-9]+$/ + branches: + ignore: /.*/ + +workflows: + version: 2.1 + + build_frontend: + when: + or: + - <> + - <> + jobs: + - javascript_lint_and_tests + - all_circleci_checks_succeeded: + requires: + - javascript_lint_and_tests + + build_backend: + when: + or: + - <> + - <> + jobs: + - e2e_tests: + matrix: + parameters: + python_version: ['3.7', '3.8', '3.9', '3.10'] + - win_e2e_tests: + matrix: + parameters: + python_version: ['3.7', '3.8', '3.9', '3.10'] + filters: + branches: + only: + - main + - demo + - unit_tests: + matrix: + parameters: + python_version: ['3.7', '3.8', '3.9', '3.10'] + - win_unit_tests: + matrix: + parameters: + python_version: ['3.7', '3.8', '3.9', '3.10'] + filters: + branches: + only: + - main + - demo + - lint: + matrix: + parameters: + python_version: ['3.7', '3.8', '3.9', '3.10'] + - deploy_demo: + context: + - kedro-ecr-publish + requires: + - e2e_tests + - win_e2e_tests + - unit_tests + - win_unit_tests + - lint + filters: + branches: + only: demo + - all_circleci_checks_succeeded: + requires: + - e2e_tests + - unit_tests + - lint + + release: + jobs: + - release_to_npm: + <<: *release_filter + - release_to_pypi: + <<: *release_filter + + daily: + triggers: + - schedule: + cron: '0 1 * * *' + filters: + branches: + only: + - main + jobs: + - e2e_tests: + matrix: + parameters: + python_version: ['3.7', '3.8', '3.9', '3.10'] + - win_e2e_tests: + matrix: + parameters: + python_version: ['3.7', '3.8', '3.9', '3.10'] + - unit_tests: + matrix: + parameters: + python_version: ['3.7', '3.8', '3.9', '3.10'] + - win_unit_tests: + matrix: + parameters: + python_version: ['3.7', '3.8', '3.9', '3.10'] + - lint: + matrix: + parameters: + python_version: ['3.7', '3.8', '3.9', '3.10'] + - javascript_lint_and_tests diff --git a/Makefile b/Makefile index 977656304e..d0060cba02 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -package: build +package: find . -regex ".*/__pycache__" -exec rm -rf {} + find . -regex ".*\.egg-info" -exec rm -rf {} + cd package && python setup.py clean --all diff --git a/package.json b/package.json index 3ccdee0078..0d9154e2a5 100644 --- a/package.json +++ b/package.json @@ -143,4 +143,4 @@ "not op_mini all" ], "snyk": true -} \ No newline at end of file +}