From 64da40cd1ee43ffb76c1dcb46fa914fb88fa7130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= Date: Wed, 4 May 2022 15:54:45 +0200 Subject: [PATCH] feat(ci): use GitHub artifacts POC (#469) --- .github/.cache_version | 2 +- .github/actions/cache/action.yml | 606 ------------------ .github/actions/restore-all/action.yml | 242 +++++++ .github/actions/setup/action.yml | 227 +++---- .github/workflows/check.yml | 388 ++++++----- .../package.json | 1 + .../tsconfig.json | 1 - scripts/ci/createMatrix.ts | 136 ---- scripts/ci/githubActions/createMatrix.ts | 187 ++++++ scripts/ci/githubActions/setRunVariables.ts | 89 +++ scripts/ci/githubActions/types.ts | 61 ++ scripts/ci/githubActions/utils.ts | 84 +++ scripts/ci/setRunVariables.ts | 165 ----- scripts/package.json | 4 +- yarn.lock | 1 + 15 files changed, 967 insertions(+), 1227 deletions(-) delete mode 100644 .github/actions/cache/action.yml create mode 100644 .github/actions/restore-all/action.yml delete mode 100644 scripts/ci/createMatrix.ts create mode 100644 scripts/ci/githubActions/createMatrix.ts create mode 100644 scripts/ci/githubActions/setRunVariables.ts create mode 100644 scripts/ci/githubActions/types.ts create mode 100644 scripts/ci/githubActions/utils.ts delete mode 100644 scripts/ci/setRunVariables.ts diff --git a/.github/.cache_version b/.github/.cache_version index d3d9cd8270..8acdd82b76 100644 --- a/.github/.cache_version +++ b/.github/.cache_version @@ -1 +1 @@ -9.2.3 +0.0.1 diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml deleted file mode 100644 index 3213b3a79f..0000000000 --- a/.github/actions/cache/action.yml +++ /dev/null @@ -1,606 +0,0 @@ -name: Cache - -description: Restore cached dependencies. - -inputs: - job: - description: The job that requires this composite. - required: false - language: - description: The language to retrieve dependencies. - required: false - cache_hash: - description: The cache hash from github actions and scripts - required: false - spec: - description: The name of the spec to restore, used for client matrix gen. - required: false - specPath: - description: The path of the spec to hash - required: false - -runs: - using: composite - steps: - - name: Read current GitHub Actions cache version - shell: bash - run: | - echo "CACHE_VERSION_YARN=$(< .github/.cache_version)" >> $GITHUB_ENV - echo "CACHE_VERSION=$(< .github/.cache_version)-${{ inputs.cache_hash }}" >> $GITHUB_ENV - - # JavaScript setup - - name: Get yarn cache directory path - shell: bash - id: yarn-cache-dir - run: echo "::set-output name=dir::$(yarn config get cacheFolder)" - - - name: Restore Yarn - uses: actions/cache@v3 - with: - path: ${{ steps.yarn-cache-dir.outputs.dir || '.yarn/cache' }} - key: node-cache-${{ env.CACHE_VERSION_YARN }}-yarn-${{ hashFiles('yarn.lock') }} - - # Java setup: used during 'java' generation or 'cts' - - name: Download Java formatter - if: ${{ inputs.language == 'java' || inputs.job == 'cts' }} - shell: bash - run: curl -L "https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar" > /tmp/java-formatter.jar - - # Restore bundled specs from input: used during 'client' generation - - name: Restore ${{ inputs.spec }} built spec - if: ${{ inputs.job == 'client' && inputs.spec && inputs.specPath }} - uses: actions/cache@v3 - with: - path: ${{ format('specs/bundled/{0}.yml', inputs.spec) }} - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - format('specs/{0}/**', inputs.specPath), - 'specs/common/**' - )}} - - # Restore bundled specs: used during 'cts' or pushing the 'codegen' - - name: Restore built abtesting spec - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: specs/bundled/abtesting.yml - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/abtesting/**', - 'specs/common/**' - )}} - - - name: Restore built analytics spec - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: specs/bundled/analytics.yml - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/analytics/**', - 'specs/common/**' - )}} - - - name: Restore built insights spec - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: specs/bundled/insights.yml - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/insights/**', - 'specs/common/**' - )}} - - - name: Restore built personalization spec - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: specs/bundled/personalization.yml - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/personalization/**', - 'specs/common/**' - )}} - - - name: Restore built predict spec - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: specs/bundled/predict.yml - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/predict/**', - 'specs/common/**' - )}} - - - name: Restore built query-suggestions spec - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: specs/bundled/query-suggestions.yml - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/query-suggestions/**', - 'specs/common/**' - )}} - - - name: Restore built recommend spec - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: specs/bundled/recommend.yml - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/recommend/**', - 'specs/common/**' - )}} - - - name: Restore built search spec - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: specs/bundled/search.yml - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/search/**', - 'specs/common/**' - )}} - - - name: Restore built algoliasearch-lite spec - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: specs/bundled/algoliasearch-lite.yml - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/search/**', - 'specs/common/**' - )}} - - - name: Restore built sources spec - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: specs/bundled/sources.yml - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/sources/**', - 'specs/common/**' - )}} - - # Restore JavaScript clients utils: used during 'javascript' generation or 'cts' - - name: Restore built JavaScript common client - if: ${{ inputs.language == 'javascript' || inputs.job == 'cts' }} - uses: actions/cache@v3 - with: - path: clients/algoliasearch-client-javascript/packages/client-common - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'clients/algoliasearch-client-javascript/packages/client-common/src/**' - )}} - - - name: Restore built JavaScript node requester - if: ${{ inputs.language == 'javascript' || inputs.job == 'cts' }} - uses: actions/cache@v3 - with: - path: clients/algoliasearch-client-javascript/packages/requester-node-http - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'clients/algoliasearch-client-javascript/packages/requester-node-http/src/**' - )}} - - - name: Restore built JavaScript browser requester - if: ${{ inputs.language == 'javascript' || inputs.job == 'cts' }} - uses: actions/cache@v3 - with: - path: clients/algoliasearch-client-javascript/packages/requester-browser-xhr - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'clients/algoliasearch-client-javascript/packages/requester-browser-xhr/src/**' - )}} - - # Restore JavaScript clients: used during 'cts' or 'codegen' - - name: Restore built JavaScript algoliasearch client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' || inputs.job == 'algoliasearch' }} - uses: actions/cache@v3 - with: - path: clients/algoliasearch-client-javascript/packages/algoliasearch - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/search.yml', - 'specs/bundled/analytics.yml', - 'specs/bundled/personalization.yml', - 'templates/javascript/**', - 'generators/src/**' - )}} - - - name: Restore built JavaScript algoliasearch-lite client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: clients/algoliasearch-client-javascript/packages/algoliasearch-lite - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/algoliasearch-lite.yml', - 'templates/javascript/**', - 'generators/src/**' - )}} - - - name: Restore built JavaScript search client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' || inputs.job == 'algoliasearch' }} - uses: actions/cache@v3 - with: - path: clients/algoliasearch-client-javascript/packages/client-search - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/search.yml', - 'templates/javascript/**', - 'generators/src/**' - )}} - - - name: Restore built JavaScript recommend client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: clients/algoliasearch-client-javascript/packages/recommend - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/recommend.yml', - 'templates/javascript/**', - 'generators/src/**' - )}} - - - name: Restore built JavaScript query-suggestions client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: clients/algoliasearch-client-javascript/packages/client-query-suggestions - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/query-suggestions.yml', - 'templates/javascript/**', - 'generators/src/**' - )}} - - - name: Restore built JavaScript personalization client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' || inputs.job == 'algoliasearch' }} - uses: actions/cache@v3 - with: - path: clients/algoliasearch-client-javascript/packages/client-personalization - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/personalization.yml', - 'templates/javascript/**', - 'generators/src/**' - )}} - - - name: Restore built JavaScript analytics client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' || inputs.job == 'algoliasearch' }} - uses: actions/cache@v3 - with: - path: clients/algoliasearch-client-javascript/packages/client-analytics - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/analytics.yml', - 'templates/javascript/**', - 'generators/src/**' - )}} - - - name: Restore built JavaScript abtesting client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: clients/algoliasearch-client-javascript/packages/client-abtesting - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/abtesting.yml', - 'templates/javascript/**', - 'generators/src/**' - )}} - - - name: Restore built JavaScript insights client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: clients/algoliasearch-client-javascript/packages/client-insights - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/insights.yml', - 'templates/javascript/**', - 'generators/src/**' - )}} - - - name: Restore built JavaScript sources client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: clients/algoliasearch-client-javascript/packages/client-sources - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/sources.yml', - 'templates/javascript/**', - 'generators/src/**' - )}} - - - name: Restore built JavaScript predict client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: clients/algoliasearch-client-javascript/packages/client-predict - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/predict.yml', - 'templates/javascript/**', - 'generators/src/**' - )}} - - # Restore Java clients: used during 'cts' or 'codegen' - - name: Restore Java client gradle files - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' || (inputs.job == 'client' && inputs.language == 'java') }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-java-2/gradle* - clients/algoliasearch-client-java-2/gradle/* - key: | - ${{ env.CACHE_VERSION }}-java-common-${{ - hashFiles( - 'templates/java/**', - 'generators/src/**' - )}} - - - name: Restore built Java search client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/api/SearchClient.java - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/model/search/** - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/search.yml', - 'templates/java/**', - 'generators/src/**' - )}} - - - name: Restore built Java recommend client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/api/RecommendClient.java - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/model/recommend/** - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/recommend.yml', - 'templates/java/**', - 'generators/src/**' - )}} - - - name: Restore built Java personalization client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/api/PersonalizationClient.java - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/model/personalization/** - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/personalization.yml', - 'templates/java/**', - 'generators/src/**' - )}} - - - name: Restore built Java analytics client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/api/AnalyticsClient.java - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/model/analytics/** - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/analytics.yml', - 'templates/java/**', - 'generators/src/**' - )}} - - - name: Restore built Java insights client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/api/InsightsClient.java - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/model/insights/** - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/insights.yml', - 'templates/java/**', - 'generators/src/**' - )}} - - - name: Restore built Java abtesting client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/api/AbtestingClient.java - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/model/abtesting/** - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/abtesting.yml', - 'templates/java/**', - 'generators/src/**' - )}} - - - name: Restore built Java query-suggestions client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/api/QuerySuggestionsClient.java - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/model/querySuggestions/** - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/query-suggestions.yml', - 'templates/java/**', - 'generators/src/**' - )}} - - - name: Restore built Java predict client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/api/PredictClient.java - clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/model/predict/** - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/predict.yml', - 'templates/java/**', - 'generators/src/**' - )}} - - # Restore PHP clients: used during 'cts' or 'codegen' - - name: Restore built PHP search client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-php/lib/Api/SearchClient.php - clients/algoliasearch-client-php/lib/Model/Search/** - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/search.yml', - 'templates/php/**', - 'generators/src/**' - )}} - - - name: Restore built PHP recommend client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-php/lib/Api/RecommendClient.php - clients/algoliasearch-client-php/lib/Model/Recommend/** - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/recommend.yml', - 'templates/php/**', - 'generators/src/**' - )}} - - - name: Restore built PHP personalization client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-php/lib/Api/PersonalizationClient.php - clients/algoliasearch-client-php/lib/Model/Personalization/** - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/personalization.yml', - 'templates/php/**', - 'generators/src/**' - )}} - - - name: Restore built PHP analytics client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-php/lib/Api/AnalyticsClient.php - clients/algoliasearch-client-php/lib/Model/Analytics/** - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/analytics.yml', - 'templates/php/**', - 'generators/src/**' - )}} - - - name: Restore built PHP insights client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-php/lib/Api/InsightsClient.php - clients/algoliasearch-client-php/lib/Model/Insights/** - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/insights.yml', - 'templates/php/**', - 'generators/src/**' - )}} - - - name: Restore built PHP abtesting client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-php/lib/Api/AbtestingClient.php - clients/algoliasearch-client-php/lib/Model/Abtesting/** - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/abtesting.yml', - 'templates/php/**', - 'generators/src/**' - )}} - - - name: Restore built PHP query-suggestions client - if: ${{ inputs.job == 'cts' || inputs.job == 'codegen' }} - uses: actions/cache@v3 - with: - path: | - clients/algoliasearch-client-php/lib/Api/QuerySuggestionsClient.php - clients/algoliasearch-client-php/lib/Model/QuerySuggestions/** - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/query-suggestions.yml', - 'templates/php/**', - 'generators/src/**' - )}} - - - name: Install JavaScript dependencies - shell: bash - run: YARN_ENABLE_IMMUTABLE_INSTALLS=false yarn install diff --git a/.github/actions/restore-all/action.yml b/.github/actions/restore-all/action.yml new file mode 100644 index 0000000000..b7565e9b76 --- /dev/null +++ b/.github/actions/restore-all/action.yml @@ -0,0 +1,242 @@ +name: Restore all artifacts + +description: | + When no input is given, artifacts are restored in the current directory, under their artifact `name`. + + This composite restore all of our artifacts, at their right path. + +runs: + using: composite + steps: + # Bundled specs + - name: spec-abtesting + uses: actions/download-artifact@v3 + with: + name: spec-abtesting + path: specs/bundled/ + + - name: spec-analytics + uses: actions/download-artifact@v3 + with: + name: spec-analytics + path: specs/bundled/ + + - name: spec-insights + uses: actions/download-artifact@v3 + with: + name: spec-insights + path: specs/bundled/ + + - name: spec-personalization + uses: actions/download-artifact@v3 + with: + name: spec-personalization + path: specs/bundled/ + + - name: spec-predict + uses: actions/download-artifact@v3 + with: + name: spec-predict + path: specs/bundled/ + + - name: spec-query-suggestions + uses: actions/download-artifact@v3 + with: + name: spec-query-suggestions + path: specs/bundled/ + + - name: spec-recommend + uses: actions/download-artifact@v3 + with: + name: spec-recommend + path: specs/bundled/ + + - name: spec-search + uses: actions/download-artifact@v3 + with: + name: spec-search + path: specs/bundled/ + + - name: spec-sources + uses: actions/download-artifact@v3 + with: + name: spec-sources + path: specs/bundled/ + + # JavaScript utils + - name: client-javascript-utils-client-common + uses: actions/download-artifact@v3 + with: + name: client-javascript-utils-client-common + path: clients/algoliasearch-client-javascript/packages/client-common/ + + - name: client-javascript-utils-requester-browser-xhr + uses: actions/download-artifact@v3 + with: + name: client-javascript-utils-requester-browser-xhr + path: clients/algoliasearch-client-javascript/packages/requester-browser-xhr/ + + - name: client-javascript-utils-requester-node-http + uses: actions/download-artifact@v3 + with: + name: client-javascript-utils-requester-node-http + path: clients/algoliasearch-client-javascript/packages/requester-node-http/ + + # JavaScript + - name: client-javascript-algoliasearch-lite + uses: actions/download-artifact@v3 + with: + name: client-javascript-algoliasearch-lite + path: clients/algoliasearch-client-javascript/packages/algoliasearch-lite/ + + - name: client-javascript-abtesting + uses: actions/download-artifact@v3 + with: + name: client-javascript-abtesting + path: clients/algoliasearch-client-javascript/packages/client-abtesting/ + + - name: client-javascript-analytics + uses: actions/download-artifact@v3 + with: + name: client-javascript-analytics + path: clients/algoliasearch-client-javascript/packages/client-analytics/ + + - name: client-javascript-insights + uses: actions/download-artifact@v3 + with: + name: client-javascript-insights + path: clients/algoliasearch-client-javascript/packages/client-insights/ + + - name: client-javascript-personalization + uses: actions/download-artifact@v3 + with: + name: client-javascript-personalization + path: clients/algoliasearch-client-javascript/packages/client-personalization/ + + - name: client-javascript-predict + uses: actions/download-artifact@v3 + with: + name: client-javascript-predict + path: clients/algoliasearch-client-javascript/packages/client-predict/ + + - name: client-javascript-query-suggestions + uses: actions/download-artifact@v3 + with: + name: client-javascript-query-suggestions + path: clients/algoliasearch-client-javascript/packages/client-query-suggestions/ + + - name: client-javascript-recommend + uses: actions/download-artifact@v3 + with: + name: client-javascript-recommend + path: clients/algoliasearch-client-javascript/packages/recommend/ + + - name: client-javascript-search + uses: actions/download-artifact@v3 + with: + name: client-javascript-search + path: clients/algoliasearch-client-javascript/packages/client-search/ + + - name: client-javascript-sources + uses: actions/download-artifact@v3 + with: + name: client-javascript-sources + path: clients/algoliasearch-client-javascript/packages/client-sources/ + + # PHP model and API + - name: client-php-abtesting + uses: actions/download-artifact@v3 + with: + name: client-php-abtesting + path: clients/algoliasearch-client-php/lib/ + + - name: client-php-analytics + uses: actions/download-artifact@v3 + with: + name: client-php-analytics + path: clients/algoliasearch-client-php/lib/ + + - name: client-php-insights + uses: actions/download-artifact@v3 + with: + name: client-php-insights + path: clients/algoliasearch-client-php/lib/ + + - name: client-php-personalization + uses: actions/download-artifact@v3 + with: + name: client-php-personalization + path: clients/algoliasearch-client-php/lib/ + + - name: client-php-query-suggestions + uses: actions/download-artifact@v3 + with: + name: client-php-query-suggestions + path: clients/algoliasearch-client-php/lib/ + + - name: client-php-recommend + uses: actions/download-artifact@v3 + with: + name: client-php-recommend + path: clients/algoliasearch-client-php/lib/ + + - name: client-php-search + uses: actions/download-artifact@v3 + with: + name: client-php-search + path: clients/algoliasearch-client-php/lib/ + + # Java model and API + - name: client-java-abtesting + uses: actions/download-artifact@v3 + with: + name: client-java-abtesting + path: clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/ + + - name: client-java-analytics + uses: actions/download-artifact@v3 + with: + name: client-java-analytics + path: clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/ + + - name: client-java-insights + uses: actions/download-artifact@v3 + with: + name: client-java-insights + path: clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/ + + - name: client-java-personalization + uses: actions/download-artifact@v3 + with: + name: client-java-personalization + path: clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/ + + - name: client-java-predict + uses: actions/download-artifact@v3 + with: + name: client-java-predict + path: clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/ + + - name: client-java-query-suggestions + uses: actions/download-artifact@v3 + with: + name: client-java-query-suggestions + path: clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/ + + - name: client-java-recommend + uses: actions/download-artifact@v3 + with: + name: client-java-recommend + path: clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/ + + - name: client-java-search + uses: actions/download-artifact@v3 + with: + name: client-java-search + path: clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/ + + - name: client-java-utils + uses: actions/download-artifact@v3 + with: + name: client-java-utils + path: clients/algoliasearch-client-java-2/ diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 7d84e384db..4bdaa5ffab 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -4,7 +4,7 @@ description: Setup CI environment. inputs: type: - description: Type of setup, `minimal` will only run the JavaScript installation. + description: Type of setup, `minimal` will only setup the JavaScript monorepo, `matrix` will set the run variables. required: false workflow_name: description: Name of the workflow that is executing this action. @@ -13,12 +13,12 @@ inputs: runs: using: composite steps: - - name: Install Node - uses: actions/setup-node@v2 - with: - node-version-file: .nvmrc - cache: yarn + - name: Read current GitHub Actions cache version + shell: bash + run: | + echo "CACHE_VERSION=$(< .github/.cache_version)" >> $GITHUB_ENV + # Java for code generation - name: Install Java if: inputs.type != 'minimal' uses: actions/setup-java@v2 @@ -31,13 +31,42 @@ runs: if: inputs.type != 'minimal' uses: gradle/wrapper-validation-action@v1 - - name: Run yarn install and install java formatter - uses: ./.github/actions/cache + - name: Download Java formatter + if: inputs.type != 'minimal' + shell: bash + run: curl -L "https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar" > /tmp/java-formatter.jar + + # JavaScript for monorepo and tooling + - name: Install Node + uses: actions/setup-node@v2 + with: + node-version-file: .nvmrc + cache: yarn + + - name: Cache node modules + uses: actions/cache@v3 + with: + path: node_modules + key: node-modules-${{ env.CACHE_VERSION }}-${{ hashFiles('yarn.lock') }} + + - name: Get yarn cache directory path + shell: bash + id: yarn-cache-dir + run: echo "::set-output name=dir::$(yarn config get cacheFolder)" + + - name: Restore Yarn + uses: actions/cache@v3 with: - language: java + path: ${{ steps.yarn-cache-dir.outputs.dir || '.yarn/cache' }} + key: yarn-cache-${{ env.CACHE_VERSION }}-${{ hashFiles('yarn.lock') }} + + - name: Install JavaScript dependencies + shell: bash + run: YARN_ENABLE_IMMUTABLE_INSTALLS=false yarn install + # Computing jobs that should run - name: Setting diff outputs variables - if: inputs.type != 'minimal' + if: inputs.type == 'matrix' id: diff shell: bash run: | @@ -47,172 +76,68 @@ runs: yarn workspace scripts setRunVariables "$origin" - - name: Compute cache common hash - if: inputs.type != 'minimal' - shell: bash - run: echo "CACHE_COMMON_HASH=${{ steps.diff.outputs.COMMON_HASH }}" >> $GITHUB_ENV - - name: Compute specs matrix - if: inputs.type != 'minimal' + if: inputs.type == 'matrix' id: spec-matrix shell: bash - run: | - base_changed=${{ - steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || - steps.diff.outputs.SCRIPTS_CHANGED > 0 || - steps.diff.outputs.COMMON_SPECS_CHANGED > 0 - }} - - matrix=$(yarn workspace scripts createMatrix $base_changed ${{ steps.diff.outputs.ORIGIN_BRANCH }}) - - if [[ $matrix == '{"client":["no-run"]}' ]]; then - run="false" - else - run="true" - fi - - echo "Running spec matrix: $run" - echo "Spec matrix: $(echo $matrix | jq .)" + run: yarn workspace scripts createMatrix ${{ steps.diff.outputs.ORIGIN_BRANCH }} - echo "::set-output name=MATRIX::$matrix" - echo "::set-output name=RUN_SPECS::$run" - - - name: Compute the JS client build matrix - if: inputs.type != 'minimal' - id: js-matrix + - name: Compute the client codegen matrix + if: inputs.type == 'matrix' + id: gen-matrix shell: bash run: | - base_changed=${{ - steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || - steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || - steps.diff.outputs.SCRIPTS_CHANGED > 0 || - steps.diff.outputs.GENERATORS_CHANGED > 0 || - steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 || - steps.diff.outputs.JS_COMMON_CHANGED > 0 }} - algoliasearch_changed=${{ steps.diff.outputs.JS_ALGOLIASEARCH_CHANGED > 0 }} - common_changed=${{ steps.diff.outputs.JS_COMMON_CHANGED > 0 }} - - matrix=$(yarn workspace scripts createMatrix $base_changed ${{ steps.diff.outputs.ORIGIN_BRANCH }} javascript) - - if [[ $algoliasearch_changed == 'true' || $base_changed == 'true' ]]; then + if [[ steps.diff.outputs.JS_ALGOLIASEARCH_CHANGED > 0 ]]; then echo "Running algoliasearch: true" - echo "::set-output name=RUN_ALGOLIASEARCH::true" + echo "::set-output name=RUN_JS_ALGOLIASEARCH::true" fi - if [[ $matrix == '{"client":["no-run"]}' ]]; then - run="false" - else - run="true" + if [[ steps.diff.outputs.JS_UTILS_CHANGED > 0 ]]; then + echo "Running JavaScript utils: true" + echo "::set-output name=RUN_JS_UTILS::true" fi - echo "Running javascript matrix: $run" - echo "Javascript matrix: $(echo $matrix | jq .)" - - echo "::set-output name=MATRIX::$matrix" - echo "::set-output name=RUN_CLIENT::$run" - - if [[ $base_changed == 'true' || $common_changed == 'true' ]]; then - echo "Running common javascript: true" - echo "::set-output name=RUN_COMMON::true" - fi - - - name: Compute the Java client build matrix - if: inputs.type != 'minimal' - id: java-matrix - shell: bash - run: | - base_changed=${{ - steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || - steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || - steps.diff.outputs.SCRIPTS_CHANGED > 0 || - steps.diff.outputs.GENERATORS_CHANGED > 0 || - steps.diff.outputs.JAVA_TEMPLATE_CHANGED > 0 }} - - matrix=$(yarn workspace scripts createMatrix $base_changed ${{ steps.diff.outputs.ORIGIN_BRANCH }} java) - - if [[ $matrix == '{"client":["no-run"]}' ]]; then - run="false" - else - run="true" - fi - - echo "Running java matrix: $run" - echo "Java matrix: $(echo $matrix | jq .)" - - echo "::set-output name=MATRIX::$matrix" - echo "::set-output name=RUN_CLIENT::$run" - - - name: Compute the PHP client build matrix - if: inputs.type != 'minimal' - id: php-matrix - shell: bash - run: | - base_changed=${{ - steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || - steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || - steps.diff.outputs.SCRIPTS_CHANGED > 0 || - steps.diff.outputs.GENERATORS_CHANGED > 0 || - steps.diff.outputs.PHP_TEMPLATE_CHANGED > 0 }} - - matrix=$(yarn workspace scripts createMatrix $base_changed ${{ steps.diff.outputs.ORIGIN_BRANCH }} php) - - if [[ $matrix == '{"client":["no-run"]}' ]]; then - run="false" - else - run="true" - fi - - echo "Running php matrix: $run" - echo "PHP matrix: $(echo $matrix | jq .)" - - echo "::set-output name=MATRIX::$matrix" - echo "::set-output name=RUN_CLIENT::$run" + yarn workspace scripts createMatrix ${{ steps.diff.outputs.ORIGIN_BRANCH }} clients outputs: - RUN_SPECS: - description: Determine if the `specs` job should run - value: ${{ steps.spec-matrix.outputs.RUN_SPECS }} - RUN_SCRIPTS: description: Determine if the `scripts` job should run value: ${{ steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 }} + RUN_SPECS: + description: Determine if the `specs` job should run + value: ${{ steps.spec-matrix.outputs.RUN_SPECS }} SPECS_MATRIX: description: The generated `specs` matrix value: ${{ steps.spec-matrix.outputs.MATRIX }} - RUN_JS: - description: Determine if the `client-javascript` job should run - value: ${{ steps.js-matrix.outputs.RUN_CLIENT }} - - RUN_JS_ALGOLIASEARCH: - description: Determine if the `client-javascript-algoliasearch` job should run - value: ${{ steps.js-matrix.outputs.RUN_ALGOLIASEARCH }} - - RUN_JS_COMMON: - description: Whether to build JS client common folders when RUN_JS is false - value: ${{ steps.js-matrix.outputs.RUN_COMMON }} - - JS_MATRIX: - description: The generated `client-javascript` matrix - value: ${{ steps.js-matrix.outputs.MATRIX }} + RUN_JAVASCRIPT: + description: Determine if the `client_javascript` job should run + value: ${{ steps.gen-matrix.outputs.RUN_JAVASCRIPT }} + JAVASCRIPT_MATRIX: + description: The generated `client_javascript` matrix + value: ${{ steps.gen-matrix.outputs.JAVASCRIPT_MATRIX }} RUN_JAVA: - description: Determine if the `client-java` job should run - value: ${{ steps.java-matrix.outputs.RUN_CLIENT }} - + description: Determine if the `client_java` job should run + value: ${{ steps.gen-matrix.outputs.RUN_JAVA }} JAVA_MATRIX: - description: The generated `client-java` matrix - value: ${{ steps.java-matrix.outputs.MATRIX }} + description: The generated `client_java` matrix + value: ${{ steps.gen-matrix.outputs.JAVA_MATRIX }} RUN_PHP: - description: Determine if the `client-php` job should run - value: ${{ steps.php-matrix.outputs.RUN_CLIENT }} - + description: Determine if the `client_php` job should run + value: ${{ steps.gen-matrix.outputs.RUN_PHP }} PHP_MATRIX: - description: The generated `client-php` matrix - value: ${{ steps.php-matrix.outputs.MATRIX }} + description: The generated `client_php` matrix + value: ${{ steps.gen-matrix.outputs.PHP_MATRIX }} + RUN_JS_ALGOLIASEARCH: + description: Determine if the `client-javascript-algoliasearch` job should run + value: ${{ steps.gen-matrix.outputs.RUN_JS_ALGOLIASEARCH }} + RUN_JS_UTILS: + description: Whether to build JS client common folders when RUN_JS is false + value: ${{ steps.gen-matrix.outputs.RUN_JS_UTILS }} RUN_JS_TESTS: description: Determine if the `client_javascript_tests` job should run value: ${{ steps.diff.outputs.JS_COMMON_TESTS_CHANGED > 0 }} @@ -233,6 +158,6 @@ outputs: description: Determine if the `codegen` job should run value: ${{ steps.spec-matrix.outputs.RUN_SPECS == 'true' || - steps.js-matrix.outputs.RUN_CLIENT == 'true' || - steps.java-matrix.outputs.RUN_CLIENT == 'true' || - steps.php-matrix.outputs.RUN_CLIENT == 'true' }} + steps.gen-matrix.outputs.RUN_JAVASCRIPT == 'true' || + steps.gen-matrix.outputs.RUN_JAVA == 'true' || + steps.gen-matrix.outputs.RUN_PHP == 'true' }} diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index dd16cae1ab..6357db4ca2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -11,6 +11,10 @@ concurrency: group: ${{ github.ref }} cancel-in-progress: true +# Uncomment the line below to enable artifacts debugging +# env: +# ACTIONS_RUNNER_DEBUG: true + jobs: setup: runs-on: ubuntu-20.04 @@ -23,6 +27,8 @@ jobs: - name: Setup id: setup uses: ./.github/actions/setup + with: + type: matrix - name: Lint GitHub actions run: yarn github-actions:lint @@ -31,26 +37,24 @@ jobs: run: | yarn cli format java generators diff=$(git status --porcelain ./generators | wc -l) + if [[ $diff > 0 ]]; then echo "Format the generators folder by running 'yarn docker format java generators'" fi + exit $diff - name: Lint json files run: yarn eslint --ext=json . outputs: - CACHE_COMMON_HASH: ${{ env.CACHE_COMMON_HASH }} RUN_SCRIPTS: ${{ steps.setup.outputs.RUN_SCRIPTS }} RUN_SPECS: ${{ steps.setup.outputs.RUN_SPECS }} SPECS_MATRIX: ${{ steps.setup.outputs.SPECS_MATRIX }} - RUN_JS: ${{ steps.setup.outputs.RUN_JS }} - RUN_JS_ALGOLIASEARCH: ${{ steps.setup.outputs.RUN_JS_ALGOLIASEARCH }} - RUN_JS_COMMON: ${{ steps.setup.outputs.RUN_JS_COMMON }} - RUN_JS_TESTS: ${{ steps.setup.outputs.RUN_JS_TESTS }} - JS_MATRIX: ${{ steps.setup.outputs.JS_MATRIX }} + RUN_JAVASCRIPT: ${{ steps.setup.outputs.RUN_JAVASCRIPT }} + JAVASCRIPT_MATRIX: ${{ steps.setup.outputs.JAVASCRIPT_MATRIX }} RUN_JAVA: ${{ steps.setup.outputs.RUN_JAVA }} JAVA_MATRIX: ${{ steps.setup.outputs.JAVA_MATRIX }} @@ -58,6 +62,11 @@ jobs: RUN_PHP: ${{ steps.setup.outputs.RUN_PHP }} PHP_MATRIX: ${{ steps.setup.outputs.PHP_MATRIX }} + RUN_JS_ALGOLIASEARCH: ${{ steps.setup.outputs.RUN_JS_ALGOLIASEARCH }} + RUN_JS_UTILS: ${{ steps.setup.outputs.RUN_JS_UTILS }} + RUN_JS_TESTS: ${{ steps.setup.outputs.RUN_JS_TESTS }} + JS_MATRIX: ${{ steps.setup.outputs.JS_MATRIX }} + RUN_CTS: ${{ steps.setup.outputs.RUN_CTS }} RUN_CODEGEN: ${{ steps.setup.outputs.RUN_CODEGEN }} @@ -70,10 +79,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Restore cache - uses: ./.github/actions/cache + - name: Setup + uses: ./.github/actions/setup with: - cache_hash: ${{ needs.setup.outputs.CACHE_COMMON_HASH }} + type: minimal - name: Check script linting run: yarn scripts:lint @@ -94,32 +103,39 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Restore cache - uses: ./.github/actions/cache - with: - cache_hash: ${{ needs.setup.outputs.CACHE_COMMON_HASH }} - - - name: Cache '${{ matrix.client.name }}' bundled specs + - name: Cache '${{ matrix.client.name }}' bundled spec id: cache uses: actions/cache@v3 with: - path: ${{ format('specs/bundled/{0}.yml', matrix.client.name) }} - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - format('{0}/**', matrix.client.path), - 'specs/common/**' - )}} + key: ${{ matrix.client.cacheKey }} + path: ${{ matrix.client.bundledPath }} - - name: Building '${{ matrix.client.name }}' specs - if: steps.cache.outputs.cache-hit != 'true' + - name: Setup + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + uses: ./.github/actions/setup + with: + type: minimal + + - name: Building '${{ matrix.client.name }}' spec + if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: yarn cli build specs ${{ matrix.client.name }} - client_javascript_common: + - name: Show diff for '${{ matrix.client.name }}' bundled spec + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: git --no-pager diff + + - name: Store '${{ matrix.client.name }}' bundled spec + uses: actions/upload-artifact@v3 + with: + if-no-files-found: error + name: spec-${{ matrix.client.name }} + path: ${{ matrix.client.bundledPath }} + + client_javascript_utils: timeout-minutes: 10 runs-on: ubuntu-20.04 needs: setup - if: ${{ needs.setup.outputs.RUN_JS_COMMON == 'true' }} + if: ${{ needs.setup.outputs.RUN_JS_UTILS == 'true' }} strategy: matrix: client: @@ -129,83 +145,102 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Restore cache - uses: ./.github/actions/cache - with: - cache_hash: ${{ needs.setup.outputs.CACHE_COMMON_HASH }} - - - name: Cache '${{ matrix.client }}' client + - name: Cache '${{ matrix.client }}' client folder id: cache uses: actions/cache@v3 with: - path: clients/algoliasearch-client-javascript/packages/${{ matrix.client }} key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - format('clients/algoliasearch-client-javascript/packages/{0}/src/**', matrix.client) + ${{ matrix.client }}-${{ hashFiles( + format('clients/algoliasearch-client-javascript/packages/{0}/**', matrix.client) )}} + path: clients/algoliasearch-client-javascript/packages/${{ matrix.client }} + + - name: Setup + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + uses: ./.github/actions/setup + with: + type: minimal - name: Build '${{ matrix.client }}' client - if: steps.cache.outputs.cache-hit != 'true' + if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: yarn workspace algoliasearch-client-javascript build ${{ matrix.client }} + - name: Run tests for 'client-common' + if: ${{ + steps.cache.outputs.cache-hit != 'true' && + needs.setup.outputs.RUN_JS_TESTS == 'true' && + matrix.client == 'client-common' }} + run: yarn workspace @experimental-api-clients-automation/client-common test + + - name: Store '${{ matrix.client }}' JavaScript utils package + uses: actions/upload-artifact@v3 + with: + if-no-files-found: error + name: client-javascript-utils-${{ matrix.client }} + path: clients/algoliasearch-client-javascript/packages/${{ matrix.client }} + client_javascript: timeout-minutes: 10 runs-on: ubuntu-20.04 needs: - setup - specs - - client_javascript_common + - scripts + - client_javascript_utils if: | always() && - needs.setup.outputs.RUN_JS == 'true' && + needs.setup.outputs.RUN_JAVASCRIPT == 'true' && !contains(needs.*.result, 'cancelled') && !contains(needs.*.result, 'failure') strategy: - matrix: ${{ fromJSON(needs.setup.outputs.JS_MATRIX) }} + matrix: ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_MATRIX) }} steps: - uses: actions/checkout@v2 - - name: Restore cache - uses: ./.github/actions/cache - with: - job: client - language: javascript - spec: ${{ matrix.client.name }} - specPath: ${{ matrix.client.specPath }} - cache_hash: ${{ needs.setup.outputs.CACHE_COMMON_HASH }} - - - name: Cache '${{ matrix.client.name }}' client + - name: Cache '${{ matrix.client.name }}' client folder id: cache uses: actions/cache@v3 with: + key: ${{ matrix.client.cacheKey }} path: ${{ matrix.client.path }} - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - format('specs/bundled/{0}.yml', matrix.client.name), - 'templates/javascript/**', - 'generators/src/**' - )}} - - name: Generate '${{ matrix.client.name }}' client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn cli generate javascript ${{ matrix.client.name }} + - name: Download '${{ matrix.client.name }}' bundled spec + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + uses: actions/download-artifact@v3 + with: + name: spec-${{ matrix.client.name }} + path: specs/bundled/ - - name: Build '${{ matrix.client.name }}' client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn cli build clients javascript ${{ matrix.client.name }} + - name: Setup + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + uses: ./.github/actions/setup + + - name: Generate '${{ matrix.client.language }}' '${{ matrix.client.name }}' client + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: yarn cli generate ${{ matrix.client.language }} ${{ matrix.client.name }} + + - name: Build '${{ matrix.client.language }}' '${{ matrix.client.name }}' client + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: yarn cli build clients ${{ matrix.client.language }} ${{ matrix.client.name }} - name: Show diff for '${{ matrix.client.name }}' client - if: steps.cache.outputs.cache-hit != 'true' + if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: git --no-pager diff + - name: Store ${{ matrix.client.name }}-${{ matrix.client.language }} client + uses: actions/upload-artifact@v3 + with: + if-no-files-found: error + name: client-${{matrix.client.language }}-${{ matrix.client.name }} + path: ${{ matrix.client.path }} + client_java: - runs-on: ubuntu-20.04 timeout-minutes: 10 + runs-on: ubuntu-20.04 needs: - setup - specs + - scripts if: | always() && needs.setup.outputs.RUN_JAVA == 'true' && @@ -216,47 +251,64 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Restore cache - uses: ./.github/actions/cache - with: - job: client - language: java - spec: ${{ matrix.client.name }} - cache_hash: ${{ needs.setup.outputs.CACHE_COMMON_HASH }} - - - name: Cache '${{ matrix.client.name }}' client + - name: Cache '${{ matrix.client.name }}' client API and Model id: cache uses: actions/cache@v3 with: + key: ${{ matrix.client.cacheKey }} path: | - ${{ format('{0}/algoliasearch-core/src/main/java/com/algolia/api/{1}.java', matrix.client.path, matrix.client.api) }} - ${{ format('{0}/algoliasearch-core/src/main/java/com/algolia/model/{1}/**', matrix.client.path, matrix.client.camelizedName) }} - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - format('specs/bundled/{0}.yml', matrix.client.name), - 'templates/java/**', - 'generators/src/**' - )}} + ${{ matrix.client.apiPath }} + ${{ matrix.client.modelPath }} - - name: Generate '${{ matrix.client.name }}' client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn cli generate java ${{ matrix.client.name }} + - name: Download '${{ matrix.client.name }}' bundled spec + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + uses: actions/download-artifact@v3 + with: + name: spec-${{ matrix.client.name }} + path: specs/bundled/ - - name: Build '${{ matrix.client.name }}' client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn cli build clients java ${{ matrix.client.name }} + - name: Setup + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + uses: ./.github/actions/setup + + - name: Generate '${{ matrix.client.language }}' '${{ matrix.client.name }}' client + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: yarn cli generate ${{ matrix.client.language }} ${{ matrix.client.name }} + + - name: Build '${{ matrix.client.language }}' '${{ matrix.client.name }}' client + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: yarn cli build clients ${{ matrix.client.language }} ${{ matrix.client.name }} - name: Show diff for '${{ matrix.client.name }}' client - if: steps.cache.outputs.cache-hit != 'true' + if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: git --no-pager diff + - name: Store ${{ matrix.client.name }}-${{ matrix.client.language }} client + uses: actions/upload-artifact@v3 + with: + if-no-files-found: error + name: client-${{matrix.client.language }}-${{ matrix.client.name }} + path: | + ${{ matrix.client.modelPath }} + ${{ matrix.client.apiPath }} + + - name: Store Java utils + uses: actions/upload-artifact@v3 + with: + if-no-files-found: error + name: client-java-utils + path: | + clients/algoliasearch-client-java-2/gradle + clients/algoliasearch-client-java-2/gradle* + clients/algoliasearch-client-java-2/algoliasearch-core/src/main/java/com/algolia/*.java + client_php: - runs-on: ubuntu-20.04 timeout-minutes: 10 + runs-on: ubuntu-20.04 needs: - setup - specs + - scripts if: | always() && needs.setup.outputs.RUN_PHP == 'true' && @@ -267,41 +319,43 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Restore cache - uses: ./.github/actions/cache - with: - job: client - language: php - spec: ${{ matrix.client.name }} - cache_hash: ${{ needs.setup.outputs.CACHE_COMMON_HASH }} - - - name: Cache '${{ matrix.client.name }}' client + - name: Cache '${{ matrix.client.name }}' client API and Model id: cache uses: actions/cache@v3 with: + key: ${{ matrix.client.cacheKey }} path: | - ${{ format('{0}/lib/Api/{1}.php', matrix.client.path, matrix.client.api) }} - ${{ format('{0}/lib/Model/{1}/**', matrix.client.path, matrix.client.capitalizedName) }} - key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - format('specs/bundled/{0}.yml', matrix.client.name), - 'templates/php/**', - 'generators/src/**' - )}} + ${{ matrix.client.apiPath }} + ${{ matrix.client.modelPath }} - - name: Generate '${{ matrix.client.name }}' client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn cli generate php ${{ matrix.client.name }} + - name: Download '${{ matrix.client.name }}' bundled spec + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + uses: actions/download-artifact@v3 + with: + name: spec-${{ matrix.client.name }} + path: specs/bundled/ + + - name: Setup + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + uses: ./.github/actions/setup - - name: Build '${{ matrix.client.name }}' client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn cli build clients php ${{ matrix.client.name }} + - name: Generate '${{ matrix.client.language }}' '${{ matrix.client.name }}' client + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: yarn cli generate ${{ matrix.client.language }} ${{ matrix.client.name }} - name: Show diff for '${{ matrix.client.name }}' client - if: steps.cache.outputs.cache-hit != 'true' + if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: git --no-pager diff + - name: Store ${{ matrix.client.name }}-${{ matrix.client.language }} client + uses: actions/upload-artifact@v3 + with: + if-no-files-found: error + name: client-${{matrix.client.language }}-${{ matrix.client.name }} + path: | + ${{ matrix.client.modelPath }} + ${{ matrix.client.apiPath }} + client_javascript_algoliasearch: timeout-minutes: 10 runs-on: ubuntu-20.04 @@ -316,54 +370,46 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Restore cache - uses: ./.github/actions/cache - with: - job: algoliasearch - language: javascript - cache_hash: ${{ needs.setup.outputs.CACHE_COMMON_HASH }} - - - name: Cache 'algoliasearch' client + - name: Cache 'algoliasearch' client folder id: cache uses: actions/cache@v3 with: - path: clients/algoliasearch-client-javascript/packages/algoliasearch key: | - ${{ env.CACHE_VERSION }}-${{ - hashFiles( - 'specs/bundled/search.yml', - 'specs/bundled/analytics.yml', - 'specs/bundled/personalization.yml', - 'templates/javascript/**', - 'generators/src/**' + algoliasearch-${{ hashFiles( + 'clients/algoliasearch-client-javascript/packages/algoliasearch/**' )}} + path: clients/algoliasearch-client-javascript/packages/algoliasearch/ - - name: Build 'algoliasearch' client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn cli build clients javascript algoliasearch + - name: Restore 'analytics' client + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + uses: actions/download-artifact@v3 + with: + name: client-javascript-analytics + path: clients/algoliasearch-client-javascript/packages/client-analytics/ - client_javascript_tests: - runs-on: ubuntu-20.04 - timeout-minutes: 10 - needs: - - setup - - client_javascript - if: | - always() && - needs.setup.outputs.RUN_JS_TESTS == 'true' && - !contains(needs.*.result, 'cancelled') && - !contains(needs.*.result, 'failure') - steps: - - uses: actions/checkout@v2 + - name: Restore 'personalization' client + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + uses: actions/download-artifact@v3 + with: + name: client-javascript-personalization + path: clients/algoliasearch-client-javascript/packages/client-personalization/ - - name: Restore cache - uses: ./.github/actions/cache + - name: Restore 'search' client + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + uses: actions/download-artifact@v3 with: - job: cts - cache_hash: ${{ needs.setup.outputs.CACHE_COMMON_HASH }} + name: client-javascript-search + path: clients/algoliasearch-client-javascript/packages/client-search/ - - name: Run client-common tests - run: yarn workspace @experimental-api-clients-automation/client-common test + - name: Setup + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + uses: ./.github/actions/setup + with: + type: minimal + + - name: Build 'algoliasearch' client + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: yarn cli build clients javascript algoliasearch cts: runs-on: ubuntu-20.04 @@ -385,15 +431,15 @@ jobs: ref: ${{ github.event.pull_request.head.ref }} token: ${{ secrets.TOKEN_GENERATE_BOT }} - - name: Restore cache - uses: ./.github/actions/cache - with: - job: cts - cache_hash: ${{ needs.setup.outputs.CACHE_COMMON_HASH }} + - name: Download all the previously stored artifacts + uses: ./.github/actions/restore-all + + - name: Setup + uses: ./.github/actions/setup - name: Push generated code to generated branch - id: pushGeneratedCode if: github.event_name == 'pull_request' + id: pushGeneratedCode run: yarn workspace scripts pushGeneratedCode env: GITHUB_TOKEN: ${{ secrets.TOKEN_GENERATE_BOT }} @@ -402,10 +448,20 @@ jobs: - name: Check JavaScript client size run: exit $(yarn workspace algoliasearch-client-javascript test:size | echo $?) + # ----> This can be added once the PHP build for the CTS is fixed + # - name: Cache generated CTS + # id: cache + # uses: actions/cache@v3 + # with: + # path: tests/output + # key: ${{ hashFiles( 'tests/output/**' ) }} + - name: Generate + # if: steps.cache.outputs.cache-hit != 'true' run: yarn cli cts generate - name: Check diff with pushed CTS + # if: steps.cache.outputs.cache-hit != 'true' run: | git --no-pager diff exit $(git status --porcelain ./tests/output | wc -l) @@ -431,11 +487,13 @@ jobs: ref: ${{ github.event.pull_request.head.ref }} token: ${{ secrets.TOKEN_GENERATE_BOT }} - - name: Restore cache - uses: ./.github/actions/cache + - name: Download all the previously stored artifacts + uses: ./.github/actions/restore-all + + - name: Setup + uses: ./.github/actions/setup with: - job: codegen - cache_hash: ${{ needs.setup.outputs.CACHE_COMMON_HASH }} + type: minimal - name: Push generated code to main id: pushGeneratedCode @@ -447,4 +505,4 @@ jobs: if: steps.pushGeneratedCode.exitcode == 0 run: yarn workspace scripts spreadGeneration env: - GITHUB_TOKEN: ${{ secrets.TOKEN_RELEASE_BOT }} + GITHUB_TOKEN: ${{ secrets.TOKEN_GENERATE_BOT }} diff --git a/clients/algoliasearch-client-javascript/package.json b/clients/algoliasearch-client-javascript/package.json index 641f81ff55..36c51391c2 100644 --- a/clients/algoliasearch-client-javascript/package.json +++ b/clients/algoliasearch-client-javascript/package.json @@ -24,6 +24,7 @@ "@babel/runtime": "7.17.9", "@rollup/plugin-babel": "5.3.1", "@rollup/plugin-node-resolve": "13.1.3", + "@types/jest": "27.4.1", "@types/rollup-plugin-node-globals": "1.4.1", "@types/semver": "7.3.9", "bundlesize2": "0.0.31", diff --git a/clients/algoliasearch-client-javascript/tsconfig.json b/clients/algoliasearch-client-javascript/tsconfig.json index fb70913461..09bc339dad 100644 --- a/clients/algoliasearch-client-javascript/tsconfig.json +++ b/clients/algoliasearch-client-javascript/tsconfig.json @@ -17,7 +17,6 @@ "strict": true, "suppressImplicitAnyIndexErrors": true, "target": "esnext", - "typeRoots": ["node_modules/@types"], "types": ["node"] }, "include": ["packages/**/*.ts"], diff --git a/scripts/ci/createMatrix.ts b/scripts/ci/createMatrix.ts deleted file mode 100644 index 69c5dc69df..0000000000 --- a/scripts/ci/createMatrix.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { CLIENTS, GENERATORS } from '../common'; -import { camelize, createClientName } from '../cts/utils'; -import type { Language } from '../types'; - -import { getNbGitDiff } from './utils'; - -type CreateMatrix = { - baseChanged: boolean; - baseBranch: string; - forLanguage?: Language; -}; - -type BaseMatrix = { - name: string; - path: string; -}; - -type ClientMatrix = BaseMatrix & { - config: string; - api: string; - capitalizedName: string; - camelizedName: string; - specPath: string; -}; - -type SpecMatrix = BaseMatrix; - -type Matrix = { - client: TMatrix[]; -}; - -// This empty matrix is required by the CI, otherwise it throws -const EMPTY_MATRIX = JSON.stringify({ client: ['no-run'] }); - -async function getClientMatrix({ - forLanguage, - baseBranch, - baseChanged, -}: CreateMatrix): Promise> { - const matrix: Matrix = { client: [] }; - - for (const { language, client, output } of Object.values(GENERATORS)) { - // `algoliasearch` is an aggregation of clients - if (language !== forLanguage || client === 'algoliasearch') { - continue; - } - - const specChanges = await getNbGitDiff({ - branch: baseBranch, - path: `specs/${client}`, - }); - const clientChanges = await getNbGitDiff({ - branch: baseBranch, - path: output, - }); - - if (clientChanges === 0 && specChanges === 0 && !baseChanged) { - continue; - } - - const clientName = createClientName(client, language); - - matrix.client.push({ - name: client, - path: output, - config: `${clientName}Config`, - api: `${clientName}Client`, - capitalizedName: clientName, - camelizedName: camelize(client), - specPath: client === 'algoliasearch-lite' ? 'search' : client, - }); - } - - return matrix; -} - -async function getSpecMatrix({ - baseBranch, - baseChanged, -}: CreateMatrix): Promise> { - const matrix: Matrix = { client: [] }; - - for (const client of CLIENTS) { - const specChanges = await getNbGitDiff({ - branch: baseBranch, - path: `specs/${client}`, - }); - - if (specChanges === 0 && !baseChanged) { - continue; - } - - const spec: SpecMatrix = { - name: client, - path: `specs/${client}`, - }; - - // The `algoliasearch-lite` spec is created by the `search` spec - if (client === 'algoliasearch-lite') { - matrix.client.push({ - ...spec, - path: 'specs/search', - }); - - continue; - } - - matrix.client.push(spec); - } - - return matrix; -} - -/** - * Creates a matrix for the CI jobs based on the files that changed. - */ -async function createMatrix(opts: CreateMatrix): Promise { - const matrix = opts.forLanguage - ? await getClientMatrix(opts) - : await getSpecMatrix(opts); - - // eslint-disable-next-line no-console - console.log( - matrix.client.length === 0 ? EMPTY_MATRIX : JSON.stringify(matrix) - ); -} - -if (require.main === module) { - const args = process.argv.slice(2); - - createMatrix({ - baseChanged: args[0] === 'true', - baseBranch: args[1], - forLanguage: args[2] as Language, - }); -} diff --git a/scripts/ci/githubActions/createMatrix.ts b/scripts/ci/githubActions/createMatrix.ts new file mode 100644 index 0000000000..758cf83fc5 --- /dev/null +++ b/scripts/ci/githubActions/createMatrix.ts @@ -0,0 +1,187 @@ +/* eslint-disable no-console */ +import { CLIENTS, GENERATORS, LANGUAGES } from '../../common'; +import { getLanguageApiFolder, getLanguageModelFolder } from '../../config'; +import { camelize, createClientName } from '../../cts/utils'; +import type { Language } from '../../types'; +import { getNbGitDiff } from '../utils'; + +import { DEPENDENCIES } from './setRunVariables'; +import type { CreateMatrix, ClientMatrix, Matrix, SpecMatrix } from './types'; +import { computeCacheKey, isBaseChanged } from './utils'; + +// This empty matrix is required by the CI, otherwise it throws +const EMPTY_MATRIX = { client: ['no-run'] }; + +/** + * List of dependencies based on the language, inherited from `./setRunVariables.ts` in a more dynamic form. + */ +const MATRIX_DEPENDENCIES = { + common: { + GITHUB_ACTIONS_CHANGED: DEPENDENCIES.GITHUB_ACTIONS_CHANGED, + SCRIPTS_CHANGED: DEPENDENCIES.SCRIPTS_CHANGED, + COMMON_SPECS_CHANGED: DEPENDENCIES.COMMON_SPECS_CHANGED, + }, + clients: { + common: { + GENERATORS_CHANGED: DEPENDENCIES.GENERATORS_CHANGED, + }, + javascript: { + JS_UTILS_CHANGED: DEPENDENCIES.JS_UTILS_CHANGED, + JS_TEMPLATE_CHANGED: DEPENDENCIES.JS_TEMPLATE_CHANGED, + }, + php: { + PHP_TEMPLATE_CHANGED: DEPENDENCIES.PHP_TEMPLATE_CHANGED, + }, + java: { + JAVA_TEMPLATE_CHANGED: DEPENDENCIES.JAVA_TEMPLATE_CHANGED, + }, + }, +}; + +async function getClientMatrix(baseBranch: string): Promise { + const matrix: Record> = { + java: { client: [] }, + php: { client: [] }, + javascript: { client: [] }, + }; + + for (const { language, client, output } of Object.values(GENERATORS)) { + // `algoliasearch` is an aggregation of generated clients. + if (client === 'algoliasearch') { + continue; + } + + const bundledSpec = client === 'algoliasearch-lite' ? 'search' : client; + const specChanges = await getNbGitDiff({ + branch: baseBranch, + path: `specs/${bundledSpec}`, + }); + const clientChanges = await getNbGitDiff({ + branch: baseBranch, + path: output, + }); + const baseChanged = await isBaseChanged(baseBranch, { + ...MATRIX_DEPENDENCIES.common, + ...MATRIX_DEPENDENCIES.clients.common, + ...MATRIX_DEPENDENCIES.clients[language], + }); + + // No changes found, we don't put this job in the matrix + if (clientChanges === 0 && specChanges === 0 && !baseChanged) { + continue; + } + + const clientName = createClientName(client, language); + const camelizedClientName = camelize(client); + const pathToApi = `${output}/${getLanguageApiFolder(language)}`; + const pathToModel = `${output}/${getLanguageModelFolder(language)}`; + + const clientMatrix: ClientMatrix = { + language, + name: client, + path: output, + + configName: `${clientName}Config`, + apiName: `${clientName}Client`, + + apiPath: pathToApi, + modelPath: pathToModel, + + cacheKey: await computeCacheKey(`client-${client}`, [ + `specs/bundled/${bundledSpec}.yml`, + `templates/${language}`, + `generators/src`, + ]), + }; + + // While JavaScript have it's own package per client, other language have + // API and models in folders common to all clients, so we scope it. + switch (language) { + case 'java': + clientMatrix.apiPath = `${pathToApi}/${clientMatrix.apiName}.java`; + clientMatrix.modelPath = `${pathToModel}/${camelizedClientName}/`; + break; + case 'php': + clientMatrix.apiPath = `${pathToApi}/${clientMatrix.apiName}.php`; + clientMatrix.modelPath = `${pathToModel}/${clientName}/`; + break; + default: + break; + } + + matrix[language].client.push(clientMatrix); + } + + // Set output variables for the CI + for (const language of LANGUAGES) { + const lang = language.toLocaleUpperCase(); + const shouldRun = matrix[language].client.length > 0; + + console.log(`::set-output name=RUN_${lang}::${shouldRun}`); + console.log( + `::set-output name=${lang}_MATRIX::${JSON.stringify( + shouldRun ? matrix[language] : EMPTY_MATRIX + )}` + ); + } +} + +async function getSpecMatrix(baseBranch: string): Promise { + const matrix: Matrix = { client: [] }; + + for (const client of CLIENTS) { + // The `algoliasearch-lite` spec is created by the `search` spec + const bundledSpecName = client === 'algoliasearch-lite' ? 'search' : client; + const specChanges = await getNbGitDiff({ + branch: baseBranch, + path: `specs/${bundledSpecName}`, + }); + const baseChanged = await isBaseChanged( + baseBranch, + MATRIX_DEPENDENCIES.common + ); + + // No changes found, we don't put this job in the matrix + if (specChanges === 0 && !baseChanged) { + continue; + } + + const path = `specs/${bundledSpecName}`; + + matrix.client.push({ + name: client, + path, + bundledPath: `specs/bundled/${client}.yml`, + cacheKey: await computeCacheKey(`spec-${client}`, ['specs/common', path]), + }); + } + + const shouldRun = matrix.client.length > 0; + + console.log(`::set-output name=RUN_SPECS::${shouldRun}`); + console.log( + `::set-output name=MATRIX::${JSON.stringify( + shouldRun ? matrix : EMPTY_MATRIX + )}` + ); +} + +/** + * Creates a matrix for the CI jobs based on the files that changed. + */ +async function createMatrix(opts: CreateMatrix): Promise { + if (opts.forClients) { + return await getClientMatrix(opts.baseBranch); + } + + return await getSpecMatrix(opts.baseBranch); +} + +if (require.main === module) { + const args = process.argv.slice(2); + + createMatrix({ + baseBranch: args[0], + forClients: args[1] === 'clients', + }); +} diff --git a/scripts/ci/githubActions/setRunVariables.ts b/scripts/ci/githubActions/setRunVariables.ts new file mode 100644 index 0000000000..a81082083a --- /dev/null +++ b/scripts/ci/githubActions/setRunVariables.ts @@ -0,0 +1,89 @@ +/* eslint-disable no-console */ +import { getLanguageFolder } from '../../config'; + +import { isBaseChanged } from './utils'; + +const JS_CLIENT_FOLDER = getLanguageFolder('javascript'); +const JAVA_CLIENT_FOLDER = getLanguageFolder('java'); +const PHP_CLIENT_FOLDER = getLanguageFolder('php'); + +// Files that are common to every clients +const CLIENTS_COMMON_FILES = [ + 'openapitools.json', + 'config/clients.config.json', +]; + +/** + * Exhaustive list of output variables to use in the CI. + * + * Those variables are used to determine if jobs should run, based on the changes + * made in their respective `path`s. + * + * Negative paths should start with `:!`. + * + * The variable will be accessible in the CI via `steps.diff.outputs.`. + */ +export const DEPENDENCIES = { + GITHUB_ACTIONS_CHANGED: [ + '.github/actions', + '.github/workflows', + '.github/.cache_version', + ], + SPECS_CHANGED: ['specs', ':!specs/bundled'], + COMMON_SPECS_CHANGED: ['specs/common'], + TESTS_CHANGED: ['tests'], + SCRIPTS_CHANGED: ['scripts'], + GENERATORS_CHANGED: ['generators'], + JS_CLIENT_CHANGED: [ + ...CLIENTS_COMMON_FILES, + JS_CLIENT_FOLDER, + `:!${JS_CLIENT_FOLDER}/.github`, + `:!${JS_CLIENT_FOLDER}/README.md`, + ], + JS_ALGOLIASEARCH_CHANGED: [ + `${JS_CLIENT_FOLDER}/packages/algoliasearch`, + `${JS_CLIENT_FOLDER}/packages/client-search`, + `${JS_CLIENT_FOLDER}/packages/client-analytics`, + `${JS_CLIENT_FOLDER}/packages/client-personalization`, + ], + JS_UTILS_CHANGED: [ + `${JS_CLIENT_FOLDER}/packages/client-common`, + `${JS_CLIENT_FOLDER}/packages/requester-browser-xhr`, + `${JS_CLIENT_FOLDER}/packages/requester-node-http`, + ], + JS_COMMON_TESTS_CHANGED: [ + `${JS_CLIENT_FOLDER}/packages/client-common/src/__tests__`, + ], + JS_TEMPLATE_CHANGED: ['templates/javascript'], + JAVA_CLIENT_CHANGED: [...CLIENTS_COMMON_FILES, JAVA_CLIENT_FOLDER], + JAVA_TEMPLATE_CHANGED: ['templates/java'], + PHP_CLIENT_CHANGED: [...CLIENTS_COMMON_FILES, PHP_CLIENT_FOLDER], + PHP_TEMPLATE_CHANGED: ['templates/php'], +}; + +/** + * Outputs variables used in the CI to determine if a job should run. + */ +async function setRunVariables({ + originBranch, +}: { + originBranch: string; +}): Promise { + console.log(`Checking diff between ${originBranch} and HEAD`); + + console.log(`::set-output name=ORIGIN_BRANCH::${originBranch}`); + + await isBaseChanged(originBranch, DEPENDENCIES, true); +} + +if (require.main === module) { + const [originBranch] = process.argv.slice(2); + + if (!originBranch) { + throw new Error( + `Unable to retrieve the origin branch: ${JSON.stringify(originBranch)}` + ); + } + + setRunVariables({ originBranch }); +} diff --git a/scripts/ci/githubActions/types.ts b/scripts/ci/githubActions/types.ts new file mode 100644 index 0000000000..ce8b353462 --- /dev/null +++ b/scripts/ci/githubActions/types.ts @@ -0,0 +1,61 @@ +export type CreateMatrix = { + /** + * The name of the branch of reference. + */ + baseBranch: string; + /** + * `true` means we generated the matrix for the `clients` job, `false` for the specs. + */ + forClients?: boolean; +}; + +type BaseMatrix = { + /** + * Name of the client. + */ + name: string; + /** + * Path to the file/folder being handled. + */ + path: string; + /** + * The computed cache key, used to restore files from the CI. + */ + cacheKey: string; +}; + +export type ClientMatrix = BaseMatrix & { + /** + * The client language. + */ + language: string; + + /** + * The client name plus `Config` appended. With the casing corresponding to the language. + */ + configName: string; + /** + * The client name plus `Client` appended. With the casing corresponding to the language. + */ + apiName: string; + + /** + * Path to the `API` file/folder of the client, based on the language. + */ + apiPath: string; + /** + * Path to the `Model` file/folder of the client, based on the language. + */ + modelPath: string; +}; + +export type SpecMatrix = BaseMatrix & { + /** + * The path of the bundled spec file. + */ + bundledPath: string; +}; + +export type Matrix = { + client: TMatrix[]; +}; diff --git a/scripts/ci/githubActions/utils.ts b/scripts/ci/githubActions/utils.ts new file mode 100644 index 0000000000..84487d3759 --- /dev/null +++ b/scripts/ci/githubActions/utils.ts @@ -0,0 +1,84 @@ +/* eslint-disable no-console */ +import crypto from 'crypto'; + +import { hashElement } from 'folder-hash'; + +import { toAbsolutePath } from '../../common'; +import { getNbGitDiff } from '../utils'; + +/** + * This cache key holds the hash of the common dependencies of all the clients. + */ +const commonCacheKey = (async function (): Promise { + const ghHash = await hashElement(toAbsolutePath('.github'), { + encoding: 'hex', + folders: { exclude: ['ISSUE_TEMPLATE'] }, + files: { include: ['*.yml', '.cache_version'] }, + }); + const scriptsHash = await hashElement(toAbsolutePath('scripts'), { + encoding: 'hex', + folders: { exclude: ['docker', '__tests__'] }, + }); + const configHash = await hashElement(toAbsolutePath('.'), { + encoding: 'hex', + folders: { include: ['config'] }, + files: { include: ['openapitools.json', 'clients.config.json'] }, + }); + + return `${ghHash.hash}-${scriptsHash.hash}-${configHash.hash}`; +})(); + +/** + * Compute a cache key based on the changes in the `paths` array of dependenciy. + * + * The `paths` parameter is an array of string, that needs to be treated as dependencies. + */ +export async function computeCacheKey( + baseName: string, + paths: string[] +): Promise { + let hash = ''; + + for (const path of paths) { + const pathHash = await hashElement(toAbsolutePath(path), { + encoding: 'hex', + files: { + include: ['**'], + }, + }); + + hash += `-${pathHash.hash}`; + } + + return `${baseName}-${crypto + .createHash('sha256') + .update(`${await commonCacheKey}-${hash}`) + .digest('hex')}`; +} + +/** + * Determines if changes have been found in the `dependencies`, compared to the `baseBranch`. + * + * If `setOutput` is true, it will set log the variable values for the CI. + */ +export async function isBaseChanged( + baseBranch: string, + dependencies: Record, + setOutput?: boolean +): Promise { + for (const [key, path] of Object.entries(dependencies)) { + const diff = await getNbGitDiff({ + branch: baseBranch, + path: path.join(' '), + }); + + if (setOutput) { + console.log(`Found ${diff} changes for '${key}'`); + console.log(`::set-output name=${key}::${diff}`); + } else if (diff > 0) { + return true; + } + } + + return false; +} diff --git a/scripts/ci/setRunVariables.ts b/scripts/ci/setRunVariables.ts deleted file mode 100644 index 0625d64d02..0000000000 --- a/scripts/ci/setRunVariables.ts +++ /dev/null @@ -1,165 +0,0 @@ -/* eslint-disable no-console */ -import crypto from 'crypto'; - -import { hashElement } from 'folder-hash'; - -import { toAbsolutePath } from '../common'; -import { getLanguageFolder } from '../config'; - -import { getNbGitDiff } from './utils'; - -const JS_CLIENT_FOLDER = getLanguageFolder('javascript'); -const JAVA_CLIENT_FOLDER = getLanguageFolder('java'); -const PHP_CLIENT_FOLDER = getLanguageFolder('php'); - -// Files that are common to every clients -const CLIENTS_COMMON_FILES = [ - 'openapitools.json', - 'config/clients.config.json', -]; - -/** - * Exhaustive list of output variables to use in the CI. - * - * Those variables are used to determine if jobs should run, based on the changes - * made in their respective `path`s. - * - * Negative paths should start with `:!`. - * - * The variable will be accessible in the CI via `steps.diff.outputs.`. - */ -const VARIABLES_TO_CHECK = [ - { - name: 'GITHUB_ACTIONS_CHANGED', - path: ['.github/actions', '.github/workflows', '.github/.cache_version'], - }, - { - name: 'SPECS_CHANGED', - path: ['specs', ':!specs/bundled'], - }, - { - name: 'COMMON_SPECS_CHANGED', - path: ['specs/common'], - }, - { - name: 'TESTS_CHANGED', - path: ['tests'], - }, - { - name: 'SCRIPTS_CHANGED', - path: ['scripts'], - }, - { - name: 'GENERATORS_CHANGED', - path: ['generators'], - }, - { - name: 'JS_CLIENT_CHANGED', - path: [ - ...CLIENTS_COMMON_FILES, - JS_CLIENT_FOLDER, - `:!${JS_CLIENT_FOLDER}/.github`, - `:!${JS_CLIENT_FOLDER}/README.md`, - ], - }, - { - name: 'JS_ALGOLIASEARCH_CHANGED', - path: [ - `${JS_CLIENT_FOLDER}/packages/algoliasearch`, - `${JS_CLIENT_FOLDER}/packages/client-search`, - `${JS_CLIENT_FOLDER}/packages/client-analytics`, - `${JS_CLIENT_FOLDER}/packages/client-personalization`, - ], - }, - { - name: 'JS_COMMON_CHANGED', - path: [ - `${JS_CLIENT_FOLDER}/packages/client-common`, - `${JS_CLIENT_FOLDER}/packages/requester-browser-xhr`, - `${JS_CLIENT_FOLDER}/packages/requester-node-http`, - ], - }, - { - name: 'JS_COMMON_TESTS_CHANGED', - path: [`${JS_CLIENT_FOLDER}/packages/client-common/src/__tests__`], - }, - { - name: 'JS_TEMPLATE_CHANGED', - path: ['templates/javascript'], - }, - { - name: 'JAVA_CLIENT_CHANGED', - path: [...CLIENTS_COMMON_FILES, JAVA_CLIENT_FOLDER], - }, - { - name: 'JAVA_TEMPLATE_CHANGED', - path: ['templates/java'], - }, - { - name: 'PHP_CLIENT_CHANGED', - path: [...CLIENTS_COMMON_FILES, PHP_CLIENT_FOLDER], - }, - { - name: 'PHP_TEMPLATE_CHANGED', - path: ['templates/php'], - }, -]; - -async function computeCommonHash(): Promise { - const hashGA = await hashElement(toAbsolutePath('.github'), { - encoding: 'hex', - folders: { exclude: ['ISSUE_TEMPLATE'] }, - files: { include: ['*.yml', '.cache_version'] }, - }); - const hashScripts = await hashElement(toAbsolutePath('scripts'), { - encoding: 'hex', - folders: { exclude: ['docker', '__tests__'] }, - }); - const hashConfig = await hashElement(toAbsolutePath('.'), { - encoding: 'hex', - folders: { include: ['config'] }, - files: { include: ['openapitools.json', 'clients.config.json'] }, - }); - - return crypto - .createHash('sha256') - .update(`${hashGA.hash}-${hashScripts.hash}-${hashConfig.hash}`) - .digest('hex'); -} - -/** - * Outputs variables used in the CI to determine if a job should run. - */ -async function setRunVariables({ - originBranch, -}: { - originBranch: string; -}): Promise { - console.log(`Checking diff between ${originBranch} and HEAD`); - - for (const check of VARIABLES_TO_CHECK) { - const diff = await getNbGitDiff({ - branch: originBranch, - path: check.path.join(' '), - }); - - console.log(`Found ${diff} changes for '${check.name}'`); - console.log(`::set-output name=${check.name}::${diff}`); - } - - console.log(`::set-output name=COMMON_HASH::${await computeCommonHash()}`); - - console.log(`::set-output name=ORIGIN_BRANCH::${originBranch}`); -} - -if (require.main === module) { - const [originBranch] = process.argv.slice(2); - - if (!originBranch) { - throw new Error( - `Unable to retrieve the origin branch: ${JSON.stringify(originBranch)}` - ); - } - - setRunVariables({ originBranch }); -} diff --git a/scripts/package.json b/scripts/package.json index ff0262be5c..a88f0e8909 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -3,12 +3,12 @@ "version": "1.0.0", "scripts": { "cleanGeneratedBranch": "ts-node ci/codegen/cleanGeneratedBranch.ts", - "createMatrix": "ts-node ci/createMatrix.ts", + "createMatrix": "ts-node ci/githubActions/createMatrix.ts", "createReleaseIssue": "ts-node release/create-release-issue.ts", "pre-commit": "./ci/husky/pre-commit.js", "processRelease": "ts-node release/process-release.ts", "pushGeneratedCode": "ts-node ci/codegen/pushGeneratedCode.ts", - "setRunVariables": "ts-node ci/setRunVariables.ts", + "setRunVariables": "ts-node ci/githubActions/setRunVariables.ts", "spreadGeneration": "ts-node ci/codegen/spreadGeneration.ts", "test": "jest", "upsertGenerationComment": "ts-node ci/codegen/upsertGenerationComment.ts" diff --git a/yarn.lock b/yarn.lock index 1fc36628f5..df9b8710d6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6580,6 +6580,7 @@ __metadata: "@babel/runtime": 7.17.9 "@rollup/plugin-babel": 5.3.1 "@rollup/plugin-node-resolve": 13.1.3 + "@types/jest": 27.4.1 "@types/rollup-plugin-node-globals": 1.4.1 "@types/semver": 7.3.9 bundlesize2: 0.0.31