forked from OpenLineage/OpenLineage
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
CI and deploy steps for docs (OpenLineage#2961)
Signed-off-by: Pawel Leszczynski <[email protected]>
- Loading branch information
1 parent
1a64c0e
commit 0bcdf5f
Showing
320 changed files
with
178 additions
and
68,901 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| name: Build & Deploy spec to Github Pages | ||
|
|
||
| on: | ||
| push: | ||
| branches: | ||
| - main | ||
| - docs-deploy # testing phase only | ||
|
|
||
| permissions: | ||
| id-token: write | ||
|
|
||
| jobs: | ||
| site-deploy: | ||
| name: Build and Publish To openlineage-site repo | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| path: 'source' | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| repository: 'pawel-big-lebowski/openlineage-site' | ||
| path: 'target' | ||
| token: ${{ secrets.ACTIONS_GITHUB_TOKEN }} | ||
| - uses: actions/setup-node@v4 | ||
| with: | ||
| node-version: 22 | ||
| cache: "npm" | ||
| cache-dependency-path: source/website/package-lock.json | ||
| - uses: actions/setup-java@v4 | ||
| with: | ||
| distribution: 'corretto' | ||
| java-version: '17' | ||
| - uses: shogo82148/actions-setup-perl@v1 | ||
| with: | ||
| perl-version: "5.38" | ||
| - uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: '3.8' | ||
| - name: Regenerate Javadoc to website | ||
| run: | | ||
| cd source/client/java | ||
| ./gradlew javadoc | ||
| mkdir -p ../../website/static/apidocs/javadoc | ||
| cp -r build/docs/javadoc ../../website/static/apidocs | ||
| - name: Regenerate python documentation | ||
| run: | | ||
| cd source/client/python | ||
| pip install -U -e .[docs] | ||
| cd docs && make html && python convert.py | ||
| cp -r converted/* ../../../website/docs/development/developing/python/api-reference | ||
| - name: Install dependencies | ||
| run: | | ||
| cd source/website | ||
| yarn install --frozen-lockfile | ||
| - name: Generate openapi | ||
| run: | | ||
| SPEC_DIR="$(pwd)/source/website/static/spec" | ||
| APIDOC_DIR="$(pwd)/source/website/static/apidocs" | ||
| pushd $SPEC_DIR | ||
| LATEST_VERSION=$(find . -maxdepth 1 | grep -v 'facets' | grep '[0-9]*-[0-9]-[0-9]' | sort -Vr | head -1) | ||
| echo latest version is $LATEST_VERSION | ||
| rm -f ./OpenLineage.json 2>/dev/null | ||
| perl -i -pe"s/version: [[:alnum:]\.-]*/version: ${LATEST_VERSION:2}/g" ./OpenLineage.yml | ||
| echo Versions updated | ||
| mkdir "${LATEST_VERSION}/facets" | ||
| for i in $(ls -d ./facets/* | sort); do cp $i/*.json ${LATEST_VERSION}/facets; done; | ||
| echo "LATEST_VERSION=${LATEST_VERSION}" >> "$GITHUB_ENV" | ||
| pushd $LATEST_VERSION | ||
| ln -sf ../OpenLineage.yml . | ||
| popd | ||
| - name: Generate redoc | ||
| run: | | ||
| npm install @redocly/cli | ||
| npx redocly build-docs -o source/website/static/apidocs/openapi/index.html source/website/static/spec/$LATEST_VERSION/OpenLineage.yml --title "OpenLineage API Docs" | ||
| - name: Clean temp files | ||
| run: | | ||
| pushd source/website/static/spec/$LATEST_VERSION | ||
| rm -rf facets | ||
| rm OpenLineage.yml | ||
| popd | ||
| - name: Push target repo | ||
| run: | | ||
| rm -r target/* | ||
| cp -r source/website/* target | ||
| cd target | ||
| git config user.name "GitHub Action Website Snapshot" | ||
| git config user.email "<>" | ||
| git add . | ||
| if ! git diff-index --quiet HEAD; then | ||
| git commit -m "Refreshing website content from main repo. | ||
| Source commit: | ||
| https://github.com/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}" | ||
| git push | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Copy changed spec JSON files to website static folder | ||
| # This is necessary to keep within repo history of all the spec versions | ||
| # Initialize CHANGE_DONE to 0 (no changes detected by default) | ||
| CHANGE_DONE=0 | ||
|
|
||
| # Use process substitution to avoid subshell problem | ||
| while read -r LINE; do | ||
| # Ignore registry files | ||
| if [[ $LINE =~ "registry.json" ]]; then | ||
| continue | ||
| fi | ||
|
|
||
| # Extract target file name from $id field in spec files using jq | ||
| URL=$(cat "$LINE" | jq -r '.["$id"]') | ||
|
|
||
| # Extract target location in website repo | ||
| LOC="website/static/${URL#*//*/}" | ||
| LOC_DIR="${LOC%/*}" | ||
|
|
||
| # Create dir if necessary, and copy files | ||
| mkdir -p "$LOC_DIR" | ||
| cp "$LINE" "$LOC" | ||
| echo $LOC | ||
| # Check if the file is tracked by Git | ||
| if git ls-files --error-unmatch "$LOC" &>/dev/null; then | ||
| # The file is tracked by Git | ||
| # Check if the copied file differs from the committed version and is not staged | ||
| if ! git diff --quiet HEAD -- "$LOC"; then | ||
| # Check if the differences are not already staged | ||
| if git diff --quiet --cached -- "$LOC"; then | ||
| echo "Change detected in $LINE: $LOC differs from committed version but is not staged" | ||
| CHANGE_DONE=1 # Mark as change detected | ||
| fi | ||
| fi | ||
| else | ||
| # The file is untracked by Git | ||
| echo "Change detected in $LINE: $LOC is untracked" | ||
| CHANGE_DONE=1 # Mark as change detected | ||
| fi | ||
| done < <(git diff --name-only HEAD -- 'spec/*.json' 'spec/OpenLineage.yml') | ||
|
|
||
| # Exit with the value of CHANGE_DONE (0 if no changes, 1 if there were changes) | ||
| exit $CHANGE_DONE |
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.