Merge branch 'main' into feature/modular_pipeline_tag

.github/actions/install_kedro_and_python_dependencies/action.yml

@@ -0,0 +1,15 @@
+name: Install Kedro and other Python Dependencies
+description: Installs Kedro from the main branch and other Python dependencies, then prints the Python version and installed packages.
+runs:
+  using: composite
+  steps:
+  - name: Install Python dependencies
+    run: |-
+      pip install git+https://github.com/kedro-org/kedro@main
+      pip install -r package/test_requirements.txt -r demo-project/src/docker_requirements.txt -U
+    shell: bash
+  - name: Echo package versions
+    run: |-
+      python -V
+      pip freeze
+    shell: bash

.github/actions/install_node_dependencies/action.yml

@@ -0,0 +1,37 @@
+name: Setup Node.js and Install Dependencies
+description: Sets up a specific Node.js version, caches Node modules, and installs Node dependencies.
+
+inputs:
+  node-version:
+    description: 'Node.js version'
+    required: false
+    default: '18.20.0'
+
+  package-path:
+    description: 'Path to package.json file'
+    required: false
+    default: '.'
+
+runs:
+  using: composite
+  steps:
+    - name: Setup Node.js
+      uses: actions/[email protected]
+      with:
+        node-version: ${{ inputs.node-version }}
+
+    - name: Get NPM Cache Directory
+      id: npm-cache-dir
+      run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
+      shell: bash
+
+    - name: Cache Node.js packages
+      uses: actions/cache@v2
+      with:
+        path: "${{ steps.npm-cache-dir.outputs.dir }}"
+        key: "${{ runner.os }}-node-${{ hashFiles(format('{0}/package-lock.json', inputs.package-path)) }}"
+        restore-keys: "${{ runner.os }}-node-"
+
+    - name: Install Node Dependencies
+      run: npm install
+      shell: bash

.github/actions/setup_tests/action.yml

@@ -0,0 +1,46 @@
+name: Setup Tests
+description: Sets up the testing environment by setting up Python and Node.js, caching Python packages, installing Kedro and other Python dependencies, and building the React application.
+
+inputs:
+  os:
+    description: 'Operating system'
+    required: false
+    default: 'ubuntu-latest'
+  python-version:
+    description: 'Python version'
+    required: false
+    default: '3.9'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python ${{inputs.python-version}}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{inputs.python-version}}      
+
+    - name: Cache python packages for Linux
+      if: inputs.os == 'ubuntu-latest'
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{inputs.os}}-python-${{inputs.python-version}}
+
+    - name: Cache python packages for Windows
+      if: inputs.os == 'windows-latest'
+      uses: actions/cache@v4
+      with:
+        path: ~\AppData\Local\pip\Cache
+        key: ${{inputs.os}}-python-${{inputs.python-version}}
+
+    - name: Install Kedro and other Python Dependencies
+      uses: "./.github/actions/install_kedro_and_python_dependencies"
+
+    - name: Setup Node.js and Install Dependencies
+      uses: "./.github/actions/install_node_dependencies"
+
+    - name: Build React application
+      run: |-
+        node -v
+        make build
+      shell: bash  

.github/workflows/all-checks.yml

@@ -0,0 +1,44 @@
+name: Run all checks on Kedro-Viz
+# Runs end-to-end tests, unit tests, linting and JavaScript 
+# linting & tests on Kedro-Viz for different 
+# operating systems and Python versions.
+
+on:
+  workflow_call:
+  workflow_dispatch:
+  schedule:
+    # Run every day at 1:00 AM(UTC time)
+    - cron: 0 1 * * *  
+jobs:
+  e2e_tests:
+    strategy:
+      matrix:
+        os: [ windows-latest, ubuntu-latest ]
+        python-version: [ "3.9", "3.10", "3.11" ]
+    uses: ./.github/workflows/e2e-tests.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }} 
+
+  unit_tests:
+    strategy:
+      matrix:
+        os: [ windows-latest, ubuntu-latest ]
+        python-version: [ "3.9", "3.10", "3.11" ]
+    uses: ./.github/workflows/unit-tests.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}  
+
+  lint:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.9", "3.10", "3.11" ]
+    uses: ./.github/workflows/lint.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}  
+
+  javascript_lint_and_tests:
+    uses: ./.github/workflows/javascript-lint-and-tests.yml

.github/workflows/build-backend.yml

@@ -0,0 +1,44 @@
+name: Build backend
+# Runs end-to-end tests, unit tests, and linting on the backend code
+# for different operating systems and Python versions.
+
+on:
+  push:
+    paths:
+      - 'package/**'
+      - '.github/**'
+  pull_request:
+    paths:
+      - 'package/**'
+      - '.github/**'
+  workflow_dispatch:
+jobs:
+  e2e_tests:
+    strategy:
+      matrix:
+        os: [ windows-latest, ubuntu-latest ]
+        python-version: [ "3.9", "3.10", "3.11" ]
+    uses: ./.github/workflows/e2e-tests.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}    
+
+  unit_tests:
+    strategy:
+      matrix:
+        os: [ windows-latest, ubuntu-latest ]
+        python-version: [ "3.9", "3.10", "3.11" ]
+    uses: ./.github/workflows/unit-tests.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}      
+
+  lint:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.9", "3.10", "3.11" ]
+    uses: ./.github/workflows/lint.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}

.github/workflows/e2e-tests.yml

@@ -0,0 +1,39 @@
+name: Run e2e tests on Kedro-Viz
+# Runs end-to-end tests on Kedro-Viz for different 
+# operating systems and Python versions.
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        type: string
+      python-version:
+        type: string
+jobs:
+    e2e_tests:
+        runs-on: ${{ inputs.os }}
+
+        # below condition checks if the operating system is Ubuntu, or 
+        # if the operating system is Windows and the branch is main/demo
+        if: >
+          inputs.os == 'ubuntu-latest' || 
+          (
+            (
+              github.ref == 'refs/heads/main' || 
+              github.ref == 'refs/heads/demo'
+            ) && 
+            inputs.os == 'windows-latest'
+          )
+          
+        steps:
+        - name: Checkout code
+          uses: actions/checkout@v4
+
+        - name: Setup Tests
+          uses: "./.github/actions/setup_tests"
+          with:
+            os: ${{ inputs.os }}
+            python-version: ${{ inputs.python-version }}
+
+        - name: Run all end to end tests
+          run: make e2e-tests 

.github/workflows/javascript-lint-and-tests.yml

@@ -0,0 +1,61 @@
+name: Run javascript linters and tests on Kedro-Viz
+# Runs JavaScript linting, unit tests, and end-to-end tests on 
+# Kedro-Viz for ubuntu-latest operating systems and Python 3.9.
+
+on:
+  push:
+    paths-ignore:
+      - 'package/**'
+  pull_request:
+    paths-ignore:
+      - 'package/**'
+  workflow_dispatch:  
+  workflow_call:
+
+jobs:
+    javascript_lint_and_tests:
+        runs-on: ubuntu-latest
+        steps:
+          - name: Checkout code
+            uses: actions/checkout@v4
+
+          - name: Set up Python "3.9"
+            uses: actions/setup-python@v5
+            with:
+              python-version: "3.9"          
+
+          - name: Cache python packages for Linux
+            uses: actions/cache@v4
+            with:
+              path: ~/.cache/pip
+              key: ubuntu-latest-python-3.9            
+
+          - name: Install Kedro and other Python Dependencies
+            uses: "./.github/actions/install_kedro_and_python_dependencies"
+
+          - name: Setup Node.js and Install Dependencies
+            uses: "./.github/actions/install_node_dependencies"
+
+          - name: Setup Cypress requirements
+            run: |-
+              sudo sed -i 's/archive.ubuntu.com/us-east-1.ec2.archive.ubuntu.com/g' /etc/apt/sources.list
+              sudo apt-get update
+              sudo apt-get install libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb          
+
+          - name: Test lib transpilation
+            run: npm run lib
+
+          - name: Test JS library imports
+            run: |-
+              npm run lib-test:setup
+              cd tools/test-lib/react-app
+              npm run test:ci
+
+          - name: Run Eslint
+            run: npm run lint
+
+          - name: Run JavaScript tests
+            run: npm run test:ci
+
+          - name: Run Javascript end to end tests
+            run: npm run cy:ci      

.github/workflows/lint.yml

@@ -0,0 +1,45 @@
+name: Run linters on Kedro-Viz
+# Runs secret scan, security scan, GraphQL schema check, 
+# and Python formatters and linters on Kedro-Viz for 
+# different operating systems and Python versions.
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        type: string
+      python-version:
+        type: string
+jobs:
+    lint:
+        runs-on: ${{ inputs.os }}
+        steps:
+          - name: Checkout code
+            uses: actions/checkout@v4
+
+          - name: Set up Python ${{inputs.python-version}}
+            uses: actions/setup-python@v5
+            with:
+              python-version: ${{inputs.python-version}}          
+
+          - name: Cache python packages for Linux
+            if: inputs.os == 'ubuntu-latest'
+            uses: actions/cache@v4
+            with:
+              path: ~/.cache/pip
+              key: ${{inputs.os}}-python-${{inputs.python-version}}            
+
+          - name: Install Kedro and other Python Dependencies
+            uses: "./.github/actions/install_kedro_and_python_dependencies"
+
+          - name: Run secret scan
+            run: make secret-scan
+
+          - name: Run security scan
+            run: make security-scan
+
+          - name: Verify GraphQL schema is up to date
+            run: make schema-check
+
+          - name: Run Python formatters and linters
+            run: make format-check lint-check          

.github/workflows/merge-gatekeeper.yml

@@ -0,0 +1,26 @@
+name: Merge Gatekeeper
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  merge-gatekeeper:
+    runs-on: ubuntu-latest
+    # Restrict permissions of the GITHUB_TOKEN.
+    # Docs: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
+    permissions:
+      checks: read
+      statuses: read
+    steps:
+      - name: Run Merge Gatekeeper
+        # NOTE: v1 is updated to reflect the latest v1.x.y. Please use any tag/branch that suits your needs:
+        #       https://github.com/upsidr/merge-gatekeeper/tags
+        #       https://github.com/upsidr/merge-gatekeeper/branches
+        uses: upsidr/merge-gatekeeper@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          timeout: 3600
+          interval: 30
+          ignored: 'ci/circleci: win_unit_tests-3.9,ci/circleci: win_unit_tests-3.10,ci/circleci: win_unit_tests-3.8,ci/circleci: unit_tests-3.10,ci/circleci: unit_tests-3.8,ci/circleci: unit_tests-3.9,ci/circleci: win_e2e_tests-3.10,ci/circleci: win_e2e_tests-3.9,ci/circleci: win_e2e_tests-3.8,ci/circleci: e2e_tests-3.8,ci/circleci: e2e_tests-3.9,ci/circleci: e2e_tests-3.10,ci/circleci: lint-3.8,ci/circleci: lint-3.9,ci/circleci: lint-3.10,ci/circleci: javascript_lint_and_tests,ci/circleci: check-updated-files,ci/circleci: all_circleci_checks_succeeded'