From 606e9e8f213d7c14b9b16bd0ff57ceb8e5ff7d09 Mon Sep 17 00:00:00 2001 From: Ved misra <47312748+misraved@users.noreply.github.com> Date: Thu, 15 Feb 2024 13:13:00 +0530 Subject: [PATCH] Update Build and Deploy OCI Image and Release Steampipe Anywhere Components workflows to use large runners (#2076) --- .github/workflows/registry-publish.yml | 303 +++++++- .github/workflows/steampipe-anywhere.yml | 940 ++++++++++++++++++++++- 2 files changed, 1235 insertions(+), 8 deletions(-) diff --git a/.github/workflows/registry-publish.yml b/.github/workflows/registry-publish.yml index b6a68535c..07917be1f 100644 --- a/.github/workflows/registry-publish.yml +++ b/.github/workflows/registry-publish.yml @@ -5,9 +5,302 @@ on: tags: - 'v*' +env: + PROJECT_ID: steampipe + ORG: turbot + PLUGIN_REPO: us-docker.pkg.dev/steampipe/plugins + CONFIG_SCHEMA_VERSION: '2020-11-18' + ORAS_VERSION: '0.14.0' + jobs: - registry_publish_workflow: - uses: turbot/steampipe-workflows/.github/workflows/registry-publish.yml@main - secrets: inherit - with: - releaseTimeout: 60m + build: + name: Build + runs-on: ubuntu_8_core + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + - name: Exit if goreleaser file is missing + run: | + test -f .goreleaser.yml + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.21 + + - name: Get latest version tag + run: |- + echo "VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + + - name: Trim tag + run: |- + echo $VERSION + trim=${VERSION#"v"} + echo $trim + echo "VERSION=${trim}" >> $GITHUB_ENV + + - name: Validate Version String + run: |- + if [[ $VERSION =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Version OK: $VERSION" + else + echo "Invalid version: $VERSION" + exit 1 + fi + + - name: Ensure Version Does Not Exist + run: |- + URL=https://$(echo $PLUGIN_REPO | sed 's/\//\/v2\//')/$ORG/$PLUGIN_NAME/tags/list + IDX=$(curl -L $URL | jq ".tags | index(\"$VERSION\")") + if [ $IDX == "null" ]; then + echo "OK - Version does not exist: $VERSION" + else + echo "Version already exists: $VERSION" + exit 1 + fi + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v2 + with: + version: latest + args: release --rm-dist --skip-publish --timeout=${{ inputs.releaseTimeout }} + + - name: List Build Artifacts + run: ls -laR ./dist + + - name: Save Linux Build Artifact - AMD64 + uses: actions/upload-artifact@v2 + with: + name: steampipe-${{ env.PLUGIN_NAME }}_linux_amd64 + path: ./dist/steampipe-plugin-${{ env.PLUGIN_NAME }}_linux_amd64.gz + if-no-files-found: error + + - name: Save Linux Build Artifact - ARM64 + uses: actions/upload-artifact@v2 + with: + name: steampipe-${{ env.PLUGIN_NAME }}_linux_arm64 + path: ./dist/steampipe-plugin-${{ env.PLUGIN_NAME }}_linux_arm64.gz + if-no-files-found: error + + - name: Save MacOS Build Artifact - AMD64 + uses: actions/upload-artifact@v2 + with: + name: steampipe-${{ env.PLUGIN_NAME }}_darwin_amd64 + path: ./dist/steampipe-plugin-${{ env.PLUGIN_NAME }}_darwin_amd64.gz + if-no-files-found: error + + - name: Save MacOS Build Artifact - ARM64 + uses: actions/upload-artifact@v2 + with: + name: steampipe-${{ env.PLUGIN_NAME }}_darwin_arm64 + path: ./dist/steampipe-plugin-${{ env.PLUGIN_NAME }}_darwin_arm64.gz + if-no-files-found: error + + publish-deploy: + name: Publish and Deploy + runs-on: ubuntu-latest + needs: + - build + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + # Setup gcloud CLI + - uses: google-github-actions/setup-gcloud@v0.2.0 + with: + service_account_key: ${{ secrets.STEAMPIPE_REGISTRY_SA_KEY }} + project_id: ${{ env.PROJECT_ID }} + + - run: gcloud config list + + - run: gcloud components install beta + + # Configure Docker to use the gcloud command-line tool as a credential + # helper for authentication + - run: |- + gcloud beta auth configure-docker us-docker.pkg.dev + + # oras is pre-installed on the image, but we should install a specific version + # to protect against unexpected breaking changes + - run: | + curl -LO https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz + sudo tar xzf oras_${ORAS_VERSION}_linux_amd64.tar.gz -C /usr/local/bin oras + oras version + + - name: Download linux-amd64 artifact + uses: actions/download-artifact@v2 + with: + name: steampipe-${{ env.PLUGIN_NAME }}_linux_amd64 + + - name: Download linux_arm64 artifact + uses: actions/download-artifact@v2 + with: + name: steampipe-${{ env.PLUGIN_NAME }}_linux_arm64 + + - name: Download darwin_amd64 artifact + uses: actions/download-artifact@v2 + with: + name: steampipe-${{ env.PLUGIN_NAME }}_darwin_amd64 + + - name: Download darwin_arm64 artifact + uses: actions/download-artifact@v2 + with: + name: steampipe-${{ env.PLUGIN_NAME }}_darwin_arm64 + + - name: List files + run: ls -l + + - name: Get latest version tag + run: |- + echo "VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + + - name: Trim tag + run: |- + echo $VERSION + trim=${VERSION#"v"} + echo $trim + echo "VERSION=${trim}" >> $GITHUB_ENV + + - name: Validate Version String + run: |- + if [[ $VERSION =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Version OK: $VERSION" + else + echo "Invalid version: $VERSION" + exit 1 + fi + + # create the config file + - run: |- + JSON_STRING=$( jq -n \ + --arg name "$PLUGIN_NAME" \ + --arg organization "$ORG" \ + --arg version "$VERSION" \ + --arg schemaVersion "$CONFIG_SCHEMA_VERSION" \ + '{schemaVersion: $schemaVersion, plugin: { name: $name, organization: $organization, version: $version} }' ) + + echo $JSON_STRING > config.json + + - run: cat config.json + + # create the annotations file + - run: |- + JSON_STRING=$( jq -n \ + --arg title "$PLUGIN_NAME" \ + --arg desc "$ORG" \ + --arg version "$VERSION" \ + --arg timestamp "$(date +%FT%T%z)" \ + --arg repo "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" \ + --arg commit "$GITHUB_SHA" \ + --arg vendor "Turbot HQ, Inc." \ + '{ + "$manifest": { + "org.opencontainers.image.title": $title, + "org.opencontainers.image.description": $desc, + "org.opencontainers.image.version": $version, + "org.opencontainers.image.created": $timestamp, + "org.opencontainers.image.source": $repo, + "org.opencontainers.image.revision": $commit, + "org.opencontainers.image.vendor": $vendor + } + }' ) + + echo $JSON_STRING > annotations.json + + - run: cat annotations.json + + - run: cat README.md + + # push to the registry + - run: |- + REF="$PLUGIN_REPO/$ORG/$PLUGIN_NAME:$GITHUB_RUN_ID" + oras push $REF \ + --config config.json:application/vnd.turbot.steampipe.config.v1+json \ + --annotation-file annotations.json \ + steampipe-plugin-${PLUGIN_NAME}_darwin_amd64.gz:application/vnd.turbot.steampipe.plugin.darwin-amd64.layer.v1+gzip \ + steampipe-plugin-${PLUGIN_NAME}_darwin_arm64.gz:application/vnd.turbot.steampipe.plugin.darwin-arm64.layer.v1+gzip \ + steampipe-plugin-${PLUGIN_NAME}_linux_amd64.gz:application/vnd.turbot.steampipe.plugin.linux-amd64.layer.v1+gzip \ + steampipe-plugin-${PLUGIN_NAME}_linux_arm64.gz:application/vnd.turbot.steampipe.plugin.linux-arm64.layer.v1+gzip \ + docs:application/vnd.turbot.steampipe.plugin.docs.layer.v1+tar \ + config:application/vnd.turbot.steampipe.plugin.spc.layer.v1+tar + + tag-versions: + name: Set Version tags + runs-on: ubuntu-latest + needs: + - publish-deploy + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + # Setup gcloud CLI + - uses: google-github-actions/setup-gcloud@v0.2.0 + with: + service_account_key: ${{ secrets.STEAMPIPE_REGISTRY_SA_KEY }} + project_id: ${{ env.PROJECT_ID }} + + - run: gcloud config list --quiet + + - run: gcloud components install alpha --quiet + + - name: Get latest version tag + run: |- + echo "VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + + - name: Trim tag + run: |- + echo $VERSION + trim=${VERSION#"v"} + echo $trim + echo "VERSION=${trim}" >> $GITHUB_ENV + + - name: Validate Version String + run: |- + if [[ $VERSION =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Version OK: $VERSION" + else + echo "Invalid version: $VERSION" + exit 1 + fi + + - name: Ensure Version Does Not Exist + run: |- + URL=https://$(echo $PLUGIN_REPO | sed 's/\//\/v2\//')/$ORG/$PLUGIN_NAME/tags/list + IDX=$(curl -L $URL | jq ".tags | index(\"$VERSION\")") + if [ $IDX == "null" ]; then + echo "OK - Version does not exist: $VERSION" + else + echo "Version already exists: $VERSION" + exit 1 + fi + + - name: Set version tags + uses: turbot/steampipe-workflows/.github/actions/semver-tags@main + id: semver + with: + image-to-tag: '${{ env.PLUGIN_REPO }}/${{ env.ORG }}/${{ env.PLUGIN_NAME }}:${{ github.run_id }}' + image-version: ${{ env.VERSION }} + diff --git a/.github/workflows/steampipe-anywhere.yml b/.github/workflows/steampipe-anywhere.yml index 80c2230f2..7e6261edc 100644 --- a/.github/workflows/steampipe-anywhere.yml +++ b/.github/workflows/steampipe-anywhere.yml @@ -13,6 +13,940 @@ on: jobs: - anywhere_publish_workflow: - uses: turbot/steampipe-workflows/.github/workflows/steampipe-anywhere.yml@main - secrets: inherit + build-postgres-14-fdw-linux-amd64: + name: Build Postgres 14 FDW for Linux - AMD64 + runs-on: ubuntu-20.04 + + steps: + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + - name: Check out steampipe-postgres-fdw + uses: actions/checkout@v2 + with: + repository: "turbot/steampipe-postgres-fdw" + ref: main + + - name: Setup GoLang + uses: actions/setup-go@v4 + with: + go-version: 1.21 + + - name: Setup apt-get + run: |- + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo env ACCEPT_EULA=Y apt-get update + sudo env ACCEPT_EULA=Y apt-get upgrade + + - name: Install PostgreSQL14 Dev + run: |- + sudo apt-get -y install postgresql-server-dev-14 + + - name: Find stuff and set env + run: |- + + which pg_config + pg_config --version + + export PATH=$(pg_config --bindir):$PATH + export PGXS=$(pg_config --pgxs) + + export SERVER_LIB=$(pg_config --includedir)/14/server + export INTERNAL_LIB=$(pg_config --includedir)/internal + + export CFLAGS="$(pg_config --cflags) -I${SERVER_LIB} -I${INTERNAL_LIB} -g" + export PG_CFLAGS="$(pg_config --cflags) -I${SERVER_LIB} -I${INTERNAL_LIB} -g" + + export LDFLAGS=$(pg_config --ldflags) + export PG_LDFLAGS=$(pg_config --ldflags) + + ls -la $SERVER_LIB + ls -la $INTERNAL_LIB + + - name: Build steampipe-postgres-fdw-anywhere + run: | + pwd + ls -ltr + make standalone plugin=${{ env.PLUGIN_NAME }} + ls -ltr + + - name: Rename and zip the build-Linux folder + run: |- + mv build-Linux steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.linux_amd64 + tar -czvf steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.linux_amd64.tar.gz steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.linux_amd64 + + - name: Save Linux Build Artifact - AMD64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.linux_amd64.tar.gz + path: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.linux_amd64.tar.gz + if-no-files-found: error + + build-postgres-14-fdw-linux-arm64: + name: Build Postgres 14 FDW for Linux - ARM64 + runs-on: + group: steampipe-darwin-arm-runners + + steps: + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + - name: Check out steampipe-postgres-fdw-anywhere + uses: actions/checkout@v3 + with: + repository: "turbot/steampipe-postgres-fdw" + ref: self-hosted + + - name: Check docker images available in runner + run: docker images + + - name: Run build script + run: | + chmod +x ./scripts/build-linux-arm-pg14.sh + bash ./scripts/build-linux-arm-pg14.sh ${{ env.PLUGIN_NAME }} + + - name: Check if binaries are available + run: ls -l $GITHUB_WORKSPACE + + - name: Save Linux Build Artifact - AMD64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.linux_arm64.tar.gz + path: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.linux_arm64.tar.gz + if-no-files-found: error + + build-postgres-14-fdw-osx-amd64: + name: Build Postgres 14 FDW for Darwin - AMD64 + runs-on: macos-latest + steps: + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + - name: Install PostgreSQL@14 + run: brew install postgresql@14 + + - name: PGConfig + run: |- + which pg_config + ls -l $(which pg_config) + PGXS=$(pg_config --pgxs) + SERVER_LIB=$(pg_config --includedir)/server + INTERNAL_LIB=$(pg_config --includedir)/internal + + echo $PGXS + echo $SERVER_LIB + echo $INTERNAL_LIB + + ls -l $PGXS + ls -l $SERVER_LIB + ls -l $INTERNAL_LIB + + - name: Check out steampipe-postgres-fdw + uses: actions/checkout@v2 + with: + repository: "turbot/steampipe-postgres-fdw" + ref: main + + - name: Setup Golang + uses: actions/setup-go@v4 + with: + go-version: 1.21 + + - name: Find stuff and set env + run: |- + go version + + which pg_config + pg_config --version + + export PATH=$(pg_config --bindir):$PATH + export PGXS=$(pg_config --pgxs) + + export SERVER_LIB=$(pg_config --includedir)/server + export INTERNAL_LIB=$(pg_config --includedir)/internal + + export CFLAGS="$(pg_config --cflags) -I${SERVER_LIB} -I${INTERNAL_LIB} -g" + export PG_CFLAGS="$(pg_config --cflags) -I${SERVER_LIB} -I${INTERNAL_LIB} -g" + + export LDFLAGS=$(pg_config --ldfalgs) + export PG_LDFLAGS=$(pg_config --ldfalgs) + + - name: Build steampipe-postgres-fdw-anywhere + run: | + pwd + ls -ltr + make standalone plugin=${{ env.PLUGIN_NAME }} + ls -ltr + + - name: Rename and zip the build-Darwin folder + run: |- + mv build-Darwin steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_amd64 + tar -czvf steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_amd64.tar.gz steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_amd64 + + - name: Save MacOS Build Artifact - AMD64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_amd64.tar.gz + path: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_amd64.tar.gz + if-no-files-found: error + + build-postgres-14-fdw-osx-arm64: + name: Build Postgres 14 FDW for Darwin - ARM64 + runs-on: macos-14 + steps: + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + - name: Check architecture + run: uname -a + + - name: Install PostgreSQL@14 + run: brew install postgresql@14 + + - name: PGConfig + run: |- + which pg_config + ls -l $(which pg_config) + PGXS=$(pg_config --pgxs) + SERVER_LIB=$(pg_config --includedir)/server + INTERNAL_LIB=$(pg_config --includedir)/internal + + echo $PGXS + echo $SERVER_LIB + echo $INTERNAL_LIB + + ls -l $PGXS + ls -l $SERVER_LIB + ls -l $INTERNAL_LIB + + - name: Check out steampipe-postgres-fdw + uses: actions/checkout@v2 + with: + repository: "turbot/steampipe-postgres-fdw" + ref: main + + - name: Setup Golang + uses: actions/setup-go@v4 + with: + go-version: 1.21 + + - name: Find stuff and set env + run: |- + go version + + which pg_config + pg_config --version + + export PATH=$(pg_config --bindir):$PATH + export PGXS=$(pg_config --pgxs) + + export SERVER_LIB=$(pg_config --includedir)/server + export INTERNAL_LIB=$(pg_config --includedir)/internal + + export CFLAGS="$(pg_config --cflags) -I${SERVER_LIB} -I${INTERNAL_LIB} -g" + export PG_CFLAGS="$(pg_config --cflags) -I${SERVER_LIB} -I${INTERNAL_LIB} -g" + + export LDFLAGS=$(pg_config --ldfalgs) + export PG_LDFLAGS=$(pg_config --ldfalgs) + + - name: Build steampipe-postgres-fdw-anywhere + run: | + pwd + ls -ltr + make standalone plugin=${{ env.PLUGIN_NAME }} + ls -ltr + + - name: Rename and zip the build-Darwin folder + run: |- + mv build-Darwin steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_arm64 + tar -czvf steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_arm64.tar.gz steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_arm64 + + - name: Save MacOS Build Artifact - AMD64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_arm64.tar.gz + path: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_arm64.tar.gz + if-no-files-found: error + + build-postgres-15-fdw-linux-amd64: + name: Build Postgres 15 FDW for Linux - AMD64 + runs-on: ubuntu-20.04 + + steps: + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + - name: Check out steampipe-postgres-fdw + uses: actions/checkout@v2 + with: + repository: "turbot/steampipe-postgres-fdw" + ref: main + + - name: Setup GoLang + uses: actions/setup-go@v4 + with: + go-version: 1.21 + + - name: Setup apt-get + run: |- + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo env ACCEPT_EULA=Y apt-get update + sudo env ACCEPT_EULA=Y apt-get upgrade + + - name: Install PostgreSQL15 Dev + run: |- + sudo apt-get -y install postgresql-server-dev-15 + + - name: Find stuff and set env + run: |- + + which pg_config + pg_config --version + + export PATH=$(pg_config --bindir):$PATH + export PGXS=$(pg_config --pgxs) + + export SERVER_LIB=$(pg_config --includedir)/15/server + export INTERNAL_LIB=$(pg_config --includedir)/internal + + export CFLAGS="$(pg_config --cflags) -I${SERVER_LIB} -I${INTERNAL_LIB} -g" + export PG_CFLAGS="$(pg_config --cflags) -I${SERVER_LIB} -I${INTERNAL_LIB} -g" + + export LDFLAGS=$(pg_config --ldflags) + export PG_LDFLAGS=$(pg_config --ldflags) + + ls -la $SERVER_LIB + ls -la $INTERNAL_LIB + + - name: Build steampipe-postgres-fdw-anywhere + run: | + pwd + ls -ltr + make standalone plugin=${{ env.PLUGIN_NAME }} + ls -ltr + + - name: Rename and zip the build-Linux folder + run: |- + mv build-Linux steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.linux_amd64 + tar -czvf steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.linux_amd64.tar.gz steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.linux_amd64 + + - name: Save Linux Build Artifact - AMD64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.linux_amd64.tar.gz + path: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.linux_amd64.tar.gz + if-no-files-found: error + + build-postgres-15-fdw-linux-arm64: + name: Build Postgres 15 FDW for Linux - ARM64 + runs-on: + group: steampipe-darwin-arm-runners + + steps: + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + - name: Check out steampipe-postgres-fdw-anywhere + uses: actions/checkout@v3 + with: + repository: "turbot/steampipe-postgres-fdw" + ref: self-hosted + + - name: Check docker images available in runner + run: docker images + + - name: Run build script + run: | + chmod +x ./scripts/build-linux-arm-pg15.sh + bash ./scripts/build-linux-arm-pg15.sh ${{ env.PLUGIN_NAME }} + + - name: Check if binaries are available + run: ls -l $GITHUB_WORKSPACE + + - name: Save Linux Build Artifact - AMD64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.linux_arm64.tar.gz + path: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.linux_arm64.tar.gz + if-no-files-found: error + + build-postgres-15-fdw-osx-amd64: + name: Build Postgres 15 FDW for Darwin - AMD64 + runs-on: macos-latest + steps: + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + - name: Install PostgreSQL@15 + run: brew install postgresql@15 + + - name: PGConfig + run: |- + which pg_config + ls -l $(which pg_config) + PGXS=$(pg_config --pgxs) + SERVER_LIB=$(pg_config --includedir)/server + INTERNAL_LIB=$(pg_config --includedir)/internal + + echo $PGXS + echo $SERVER_LIB + echo $INTERNAL_LIB + + ls -l $PGXS + ls -l $SERVER_LIB + ls -l $INTERNAL_LIB + + - name: Check out steampipe-postgres-fdw + uses: actions/checkout@v2 + with: + repository: "turbot/steampipe-postgres-fdw" + ref: main + + - name: Setup Golang + uses: actions/setup-go@v4 + with: + go-version: 1.21 + + - name: Find stuff and set env + run: |- + go version + + which pg_config + pg_config --version + + export PATH=$(pg_config --bindir):$PATH + export PGXS=$(pg_config --pgxs) + + export SERVER_LIB=$(pg_config --includedir)/server + export INTERNAL_LIB=$(pg_config --includedir)/internal + + export CFLAGS="$(pg_config --cflags) -I${SERVER_LIB} -I${INTERNAL_LIB} -g" + export PG_CFLAGS="$(pg_config --cflags) -I${SERVER_LIB} -I${INTERNAL_LIB} -g" + + export LDFLAGS=$(pg_config --ldfalgs) + export PG_LDFLAGS=$(pg_config --ldfalgs) + + - name: Build steampipe-postgres-fdw-anywhere + run: | + pwd + ls -ltr + make standalone plugin=${{ env.PLUGIN_NAME }} + ls -ltr + + - name: Rename and zip the build-Darwin folder + run: |- + mv build-Darwin steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_amd64 + tar -czvf steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_amd64.tar.gz steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_amd64 + + - name: Save MacOS Build Artifact - AMD64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_amd64.tar.gz + path: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_amd64.tar.gz + if-no-files-found: error + + build-postgres-15-fdw-osx-arm64: + name: Build Postgres 15 FDW for Darwin - ARM64 + runs-on: macos-14 + steps: + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + - name: Check architecture + run: uname -a + + - name: Install PostgreSQL@15 + run: brew install postgresql@15 + + - name: Get the installation directory of the postgresql@15 formula + run: brew --prefix postgresql@15 + + - name: Configure PATH for PostgreSQL + run: echo $(brew --prefix postgresql@15)/bin >> $GITHUB_PATH + + - name: PGConfig + run: |- + which pg_config + ls -l $(which pg_config) + PGXS=$(pg_config --pgxs) + SERVER_LIB=$(pg_config --includedir)/postgresql/server + INTERNAL_LIB=$(pg_config --includedir)/postgresql/internal + + echo $PGXS + echo $SERVER_LIB + echo $INTERNAL_LIB + + ls -l $PGXS + ls -l $SERVER_LIB + ls -l $INTERNAL_LIB + + - name: Check out steampipe-postgres-fdw + uses: actions/checkout@v2 + with: + repository: "turbot/steampipe-postgres-fdw" + ref: main + + - name: Setup Golang + uses: actions/setup-go@v4 + with: + go-version: 1.21 + + - name: Find stuff and set env + run: |- + go version + + which pg_config + pg_config --version + + export PGXS=$(pg_config --pgxs) + + export SERVER_LIB=$(pg_config --includedir)/postgresql/server + export INTERNAL_LIB=$(pg_config --includedir)/postgresql/internal + + export CFLAGS="$(pg_config --cflags) -I${SERVER_LIB} -I${INTERNAL_LIB} -g" + export PG_CFLAGS="$(pg_config --cflags) -I${SERVER_LIB} -I${INTERNAL_LIB} -g" + + export LDFLAGS=$(pg_config --ldfalgs) + export PG_LDFLAGS=$(pg_config --ldfalgs) + + - name: Build steampipe-postgres-fdw-anywhere + run: | + pwd + ls -ltr + make standalone plugin=${{ env.PLUGIN_NAME }} + ls -ltr + + - name: Rename and zip the build-Darwin folder + run: |- + mv build-Darwin steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_arm64 + tar -czvf steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_arm64.tar.gz steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_arm64 + + - name: Save MacOS Build Artifact - ARM64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_arm64.tar.gz + path: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_arm64.tar.gz + if-no-files-found: error + + build-sqlite-extension-linux-amd64: + name: Build SQLite Extension for Linux - AMD64 + runs-on: ubuntu-latest + + steps: + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + - name: Check out steampipe-sqlite-extension + uses: actions/checkout@v2 + with: + repository: "turbot/steampipe-sqlite" + ref: main + + - name: Setup GoLang + uses: actions/setup-go@v4 + with: + go-version: 1.21 + + - name: Build steampipe-sqlite-extension + run: | + pwd + ls -ltr + make build plugin=${{ env.PLUGIN_NAME }} + ls -ltr + + - name: Zip and rename the steampipe_sqlite_${{ env.PLUGIN_NAME }}.so + run: |- + tar -czvf steampipe_sqlite_${{ env.PLUGIN_NAME }}.linux_amd64.tar.gz steampipe_sqlite_${{ env.PLUGIN_NAME }}.so + + - name: Save Linux Build Artifact - AMD64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_sqlite_${{ env.PLUGIN_NAME }}.linux_amd64.tar.gz + path: steampipe_sqlite_${{ env.PLUGIN_NAME }}.linux_amd64.tar.gz + if-no-files-found: error + + build-sqlite-extension-linux-arm64: + name: Build SQLite Extension for Linux - ARM64 + runs-on: + group: steampipe-darwin-arm-runners + + steps: + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + - name: Check out steampipe-sqlite-extension + uses: actions/checkout@v3 + with: + repository: "turbot/steampipe-sqlite" + ref: self-hosted + + - name: Check docker images available in runner + run: docker images + + - name: Run build script + run: | + chmod +x ./scripts/build-linux-arm-sqlite.sh + bash ./scripts/build-linux-arm-sqlite.sh ${{ env.PLUGIN_NAME }} + + - name: Check if binaries are available + run: ls -l $GITHUB_WORKSPACE + + - name: Save Linux Build Artifact - AMD64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_sqlite_${{ env.PLUGIN_NAME }}.linux_arm64.tar.gz + path: steampipe_sqlite_${{ env.PLUGIN_NAME }}.linux_arm64.tar.gz + if-no-files-found: error + + build-sqlite-extension-osx-amd64: + name: Build SQLite Extension for Darwin - AMD64 + runs-on: macos-latest + + steps: + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + - name: Check out steampipe-sqlite-extension + uses: actions/checkout@v2 + with: + repository: "turbot/steampipe-sqlite-extension" + ref: main + + - name: Setup GoLang + uses: actions/setup-go@v4 + with: + go-version: 1.21 + + - name: Build steampipe-sqlite-extension + run: | + pwd + ls -ltr + make build plugin=${{ env.PLUGIN_NAME }} + ls -ltr + + - name: Zip and rename the steampipe_sqlite_${{ env.PLUGIN_NAME }}.so + run: |- + tar -czvf steampipe_sqlite_${{ env.PLUGIN_NAME }}.darwin_amd64.tar.gz steampipe_sqlite_${{ env.PLUGIN_NAME }}.so + + - name: Save Darwin Build Artifact - AMD64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_sqlite_${{ env.PLUGIN_NAME }}.darwin_amd64.tar.gz + path: steampipe_sqlite_${{ env.PLUGIN_NAME }}.darwin_amd64.tar.gz + if-no-files-found: error + + build-sqlite-extension-osx-arm64: + name: Build SQLite Extension for Darwin - ARM64 + runs-on: macos-14 + + steps: + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + - name: Check architecture + run: uname -a + + - name: Check out steampipe-sqlite-extension + uses: actions/checkout@v2 + with: + repository: "turbot/steampipe-sqlite-extension" + ref: main + + - name: Setup GoLang + uses: actions/setup-go@v4 + with: + go-version: 1.21 + + - name: Build steampipe-sqlite-extension + run: | + pwd + ls -ltr + make build plugin=${{ env.PLUGIN_NAME }} + ls -ltr + + - name: Zip and rename the steampipe_sqlite_${{ env.PLUGIN_NAME }}.so + run: |- + tar -czvf steampipe_sqlite_${{ env.PLUGIN_NAME }}.darwin_arm64.tar.gz steampipe_sqlite_${{ env.PLUGIN_NAME }}.so + + - name: Save Darwin Build Artifact - ARM64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_sqlite_${{ env.PLUGIN_NAME }}.darwin_arm64.tar.gz + path: steampipe_sqlite_${{ env.PLUGIN_NAME }}.darwin_arm64.tar.gz + if-no-files-found: error + + build-export-tool: + name: Build Export Tool + runs-on: ubuntu_8_core + + steps: + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + - name: Check out steampipe-table-dump + uses: actions/checkout@v2 + with: + repository: "turbot/steampipe-table-dump" + ref: main + + - name: Setup GoLang + uses: actions/setup-go@v4 + with: + go-version: 1.21 + + - name: Build steampipe-table-dump + run: | + pwd + ls -ltr + make build plugin=${{ env.PLUGIN_NAME }} + ls -ltr + go mod tidy + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v5 + with: + version: latest + args: release --snapshot --skip-publish --rm-dist + + - name: Copy the gzipped binaries from dist + run: |- + ls -al dist/ + cp dist/steampipe_export_${{ env.PLUGIN_NAME }}.linux_amd64.tar.gz steampipe_export_${{ env.PLUGIN_NAME }}.linux_amd64.tar.gz + cp dist/steampipe_export_${{ env.PLUGIN_NAME }}.linux_arm64.tar.gz steampipe_export_${{ env.PLUGIN_NAME }}.linux_arm64.tar.gz + cp dist/steampipe_export_${{ env.PLUGIN_NAME }}.darwin_amd64.tar.gz steampipe_export_${{ env.PLUGIN_NAME }}.darwin_amd64.tar.gz + cp dist/steampipe_export_${{ env.PLUGIN_NAME }}.darwin_arm64.tar.gz steampipe_export_${{ env.PLUGIN_NAME }}.darwin_arm64.tar.gz + ls -al + + - name: Save Linux Build Artifact - AMD64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_export_${{ env.PLUGIN_NAME }}.linux_amd64.tar.gz + path: steampipe_export_${{ env.PLUGIN_NAME }}.linux_amd64.tar.gz + if-no-files-found: error + + - name: Save Linux Build Artifact - ARM64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_export_${{ env.PLUGIN_NAME }}.linux_arm64.tar.gz + path: steampipe_export_${{ env.PLUGIN_NAME }}.linux_arm64.tar.gz + if-no-files-found: error + + - name: Save Darwin Build Artifact - AMD64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_export_${{ env.PLUGIN_NAME }}.darwin_amd64.tar.gz + path: steampipe_export_${{ env.PLUGIN_NAME }}.darwin_amd64.tar.gz + if-no-files-found: error + + - name: Save Darwin Build Artifact - ARM64 + uses: actions/upload-artifact@v3 + with: + name: steampipe_export_${{ env.PLUGIN_NAME }}.darwin_arm64.tar.gz + path: steampipe_export_${{ env.PLUGIN_NAME }}.darwin_arm64.tar.gz + if-no-files-found: error + + build-draft-release: + name: Build Draft Release + runs-on: ubuntu-latest + needs: + # Postgres 14 + - build-postgres-14-fdw-linux-amd64 + - build-postgres-14-fdw-linux-arm64 + - build-postgres-14-fdw-osx-amd64 + - build-postgres-14-fdw-osx-arm64 + # Postgres 15 + - build-postgres-15-fdw-linux-amd64 + - build-postgres-15-fdw-linux-arm64 + - build-postgres-15-fdw-osx-amd64 + - build-postgres-15-fdw-osx-arm64 + # SQLite + - build-sqlite-extension-linux-amd64 + - build-sqlite-extension-linux-arm64 + - build-sqlite-extension-osx-amd64 + - build-sqlite-extension-osx-arm64 + # Export + - build-export-tool + steps: + - name: Set environment variables + run: | + plugin_name=$(echo $GITHUB_REPOSITORY | cut -d'-' -f 3) + echo $plugin_name + echo "PLUGIN_NAME=${plugin_name}" >> $GITHUB_ENV + + - name: Download steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.linux_amd64.tar.gz + id: download_pg14_linux_amd64 + uses: actions/download-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.linux_amd64.tar.gz + + - name: Download steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.linux_arm64.tar.gz + id: download_pg14_linux_arm64 + uses: actions/download-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.linux_arm64.tar.gz + + - name: Download steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_amd64.tar.gz + id: download_pg14_darwin_amd64 + uses: actions/download-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_amd64.tar.gz + + - name: Download steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_arm64.tar.gz + id: download_pg14_darwin_arm64 + uses: actions/download-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_arm64.tar.gz + + - name: Download steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.linux_amd64.tar.gz + id: download_pg15_linux_amd64 + uses: actions/download-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.linux_amd64.tar.gz + + - name: Download steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.linux_arm64.tar.gz + id: download_pg15_linux_arm64 + uses: actions/download-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.linux_arm64.tar.gz + + - name: Download steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_amd64.tar.gz + id: download_pg15_darwin_amd64 + uses: actions/download-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_amd64.tar.gz + + - name: Download steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_arm64.tar.gz + id: download_pg15_darwin_arm64 + uses: actions/download-artifact@v3 + with: + name: steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_arm64.tar.gz + + - name: Download steampipe_sqlite_${{ env.PLUGIN_NAME }}.linux_amd64.tar.gz + id: download_sqlite_linux_amd64 + uses: actions/download-artifact@v3 + with: + name: steampipe_sqlite_${{ env.PLUGIN_NAME }}.linux_amd64.tar.gz + + - name: Download steampipe_sqlite_${{ env.PLUGIN_NAME }}.linux_arm64.tar.gz + id: download_sqlite_linux_arm64 + uses: actions/download-artifact@v3 + with: + name: steampipe_sqlite_${{ env.PLUGIN_NAME }}.linux_arm64.tar.gz + + - name: Download steampipe_sqlite_${{ env.PLUGIN_NAME }}.darwin_amd64.tar.gz + id: download_sqlite_darwin_amd64 + uses: actions/download-artifact@v3 + with: + name: steampipe_sqlite_${{ env.PLUGIN_NAME }}.darwin_amd64.tar.gz + + - name: Download steampipe_sqlite_${{ env.PLUGIN_NAME }}.darwin_arm64.tar.gz + id: download_sqlite_darwin_arm64 + uses: actions/download-artifact@v3 + with: + name: steampipe_sqlite_${{ env.PLUGIN_NAME }}.darwin_arm64.tar.gz + + - name: Download steampipe_export_${{ env.PLUGIN_NAME }}.linux_amd64.tar.gz + id: download_export_linux_amd64 + uses: actions/download-artifact@v3 + with: + name: steampipe_export_${{ env.PLUGIN_NAME }}.linux_amd64.tar.gz + + - name: Download steampipe_export_${{ env.PLUGIN_NAME }}.linux_arm64.tar.gz + id: download_export_linux_arm64 + uses: actions/download-artifact@v3 + with: + name: steampipe_export_${{ env.PLUGIN_NAME }}.linux_arm64.tar.gz + + - name: Download steampipe_export_${{ env.PLUGIN_NAME }}.darwin_amd64.tar.gz + id: download_export_darwin_amd64 + uses: actions/download-artifact@v3 + with: + name: steampipe_export_${{ env.PLUGIN_NAME }}.darwin_amd64.tar.gz + + - name: Download steampipe_export_${{ env.PLUGIN_NAME }}.darwin_arm64.tar.gz + id: download_export_darwin_arm64 + uses: actions/download-artifact@v3 + with: + name: steampipe_export_${{ env.PLUGIN_NAME }}.darwin_arm64.tar.gz + + - name: Check Path + run: |- + ls -la + + - name: Get latest version tag + run: |- + echo "VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + trim=${VERSION#"v"} + echo $trim + echo "VERSION=${trim}" >> $GITHUB_ENV + + - name: Create a draft release + uses: softprops/action-gh-release@v1 + id: create_draft_release + with: + draft: false + prerelease: false + name: ${{ env.VERSION }} + tag_name: ${{ env.VERSION }} + repository: ${{ github.repository }} + files: |- + ${{ steps.download_pg14_darwin_amd64.outputs.download-path }}/steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_amd64.tar.gz + ${{ steps.download_pg14_darwin_arm64.outputs.download-path }}/steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.darwin_arm64.tar.gz + ${{ steps.download_pg14_linux_amd64.outputs.download-path }}/steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.linux_amd64.tar.gz + ${{ steps.download_pg14_linux_arm64.outputs.download-path }}/steampipe_postgres_${{ env.PLUGIN_NAME }}.pg14.linux_arm64.tar.gz + + ${{ steps.download_pg15_darwin_amd64.outputs.download-path }}/steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_amd64.tar.gz + ${{ steps.download_pg15_darwin_arm64.outputs.download-path }}/steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.darwin_arm64.tar.gz + ${{ steps.download_pg15_linux_amd64.outputs.download-path }}/steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.linux_amd64.tar.gz + ${{ steps.download_pg15_linux_arm64.outputs.download-path }}/steampipe_postgres_${{ env.PLUGIN_NAME }}.pg15.linux_arm64.tar.gz + + ${{ steps.download_sqlite_darwin_amd64.outputs.download-path }}/steampipe_sqlite_${{ env.PLUGIN_NAME }}.darwin_amd64.tar.gz + ${{ steps.download_sqlite_darwin_arm64.outputs.download-path }}/steampipe_sqlite_${{ env.PLUGIN_NAME }}.darwin_arm64.tar.gz + ${{ steps.download_sqlite_linux_amd64.outputs.download-path }}/steampipe_sqlite_${{ env.PLUGIN_NAME }}.linux_amd64.tar.gz + ${{ steps.download_sqlite_linux_arm64.outputs.download-path }}/steampipe_sqlite_${{ env.PLUGIN_NAME }}.linux_arm64.tar.gz + + ${{ steps.download_export_darwin_amd64.outputs.download-path }}/steampipe_export_${{ env.PLUGIN_NAME }}.darwin_amd64.tar.gz + ${{ steps.download_export_darwin_arm64.outputs.download-path }}/steampipe_export_${{ env.PLUGIN_NAME }}.darwin_arm64.tar.gz + ${{ steps.download_export_linux_amd64.outputs.download-path }}/steampipe_export_${{ env.PLUGIN_NAME }}.linux_amd64.tar.gz + ${{ steps.download_export_linux_arm64.outputs.download-path }}/steampipe_export_${{ env.PLUGIN_NAME }}.linux_arm64.tar.gz