From 41866cefcebc55a0bf1759469c4038bdeffcdb5e Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Tue, 13 Sep 2022 15:04:55 -0700 Subject: [PATCH 01/10] Reorganize GitHub workflows. * Remove obsoleve workflows, mostly related to releaseing ODFE artifacts. * Update actions in rest workflows. * Limit workflow scopes: * Reduce file scope, for example, JDBC workflow will be triggered on JDBC changes only. * Reduce action scope, so workflows shouldn't be triggered twice on PR. Signed-off-by: Yury-Fridlyand --- .github/workflows/codeql-analysis.yml | 26 ++- .github/workflows/dco.yml | 2 +- .../draft-release-notes-workflow.yml | 2 +- .github/workflows/link-checker.yml | 8 +- .../workflows/sql-cli-release-workflow.yml | 71 ------ .../sql-cli-test-and-build-workflow.yml | 36 ++- .../workflows/sql-jdbc-push-jdbc-maven.yml | 49 ---- .../workflows/sql-jdbc-release-workflow.yml | 58 ----- .../sql-jdbc-test-and-build-workflow.yml | 15 +- .github/workflows/sql-odbc-main.yml | 36 +-- .../workflows/sql-odbc-release-workflow.yml | 212 ------------------ .github/workflows/sql-release-workflow.yml | 58 ----- .../workflows/sql-test-and-build-workflow.yml | 54 ++++- .../sql-workbench-release-workflow.yml | 72 ------ .../sql-workbench-test-and-build-workflow.yml | 48 +++- 15 files changed, 158 insertions(+), 589 deletions(-) delete mode 100644 .github/workflows/sql-cli-release-workflow.yml delete mode 100644 .github/workflows/sql-jdbc-push-jdbc-maven.yml delete mode 100644 .github/workflows/sql-jdbc-release-workflow.yml delete mode 100644 .github/workflows/sql-odbc-release-workflow.yml delete mode 100644 .github/workflows/sql-release-workflow.yml delete mode 100644 .github/workflows/sql-workbench-release-workflow.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 08d827bd45..b1ac364cd5 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -15,19 +15,21 @@ name: "CodeQL" on: pull_request: branches: - - master - - opendistro-* + - main push: - branches: - - master - - opendistro-* - + branches-ignore: + - 'dependabot/**' + paths: + - '**/*.java' + - '.github/workflows/codeql-analysis.yml' jobs: analyze: name: CodeQL-Scan - runs-on: ubuntu-18.04 - + runs-on: ubuntu-latest + permissions: + security-events: write + actions: read strategy: fail-fast: false matrix: @@ -38,11 +40,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -53,7 +55,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -67,4 +69,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml index 53ed5304c2..cf30ea89dc 100644 --- a/.github/workflows/dco.yml +++ b/.github/workflows/dco.yml @@ -15,4 +15,4 @@ jobs: - name: DCO Check uses: tim-actions/dco@v1.1.0 with: - commits: ${{ steps.get-pr-commits.outputs.commits }} \ No newline at end of file + commits: ${{ steps.get-pr-commits.outputs.commits }} diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml index 7f5a124ddd..660a8a1a51 100644 --- a/.github/workflows/draft-release-notes-workflow.yml +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -16,6 +16,6 @@ jobs: with: config-name: draft-release-notes-config.yml tag: (None) - version: 2.4.0.0 + version: 2.3.0.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/link-checker.yml b/.github/workflows/link-checker.yml index b113ef24ec..518af21f3a 100644 --- a/.github/workflows/link-checker.yml +++ b/.github/workflows/link-checker.yml @@ -1,9 +1,13 @@ name: Link Checker + on: push: branches: [ main ] - pull_request: - branches: [ main ] + paths: + - 'docs/**' + - '**/*.md' + - '**/*.txt' + - '**/*.rst' jobs: linkchecker: diff --git a/.github/workflows/sql-cli-release-workflow.yml b/.github/workflows/sql-cli-release-workflow.yml deleted file mode 100644 index 606d848829..0000000000 --- a/.github/workflows/sql-cli-release-workflow.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: Release SQL CLI Artifacts -# This workflows is triggered on creating tags to main -on: - push: - tags: - - 'v*' - -jobs: - build: - - runs-on: ubuntu-latest - defaults: - run: - working-directory: sql-cli - strategy: - matrix: - python-version: [3.8] - - steps: - - name: Checkout SQL CLI - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - - name: Build - run: | - python setup.py sdist bdist_wheel - - - name: Publish to S3 - shell: bash - run: | - tarball=`ls ./dist/*.tar.gz` - wheel=`ls ./dist/*.whl` - - renamed_wheel=`echo $wheel | sed 's/_/-/g'` - mv "$wheel" "$renamed_wheel" - wheel=`ls ./dist/*.whl` - - # Inject the build number before the suffix - tarball_outfile=`basename ${tarball%.tar.gz}-build-${GITHUB_RUN_NUMBER}.tar.gz` - wheel_outfile=`basename ${wheel%.whl}-build-${GITHUB_RUN_NUMBER}.whl` - - s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-clients/sql-cli/" - - echo "Copying ${tarball} to ${s3_prefix}${tarball_outfile}" - aws s3 cp --quiet $tarball ${s3_prefix}${tarball_outfile} - - echo "Copying ${wheel} to ${s3_prefix}${wheel_outfile}" - aws s3 cp --quiet $wheel ${s3_prefix}${wheel_outfile} - - # TODO: Publish to PyPI - # - name: Publish to PyPI - # env: - # TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - # TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - # run: twine upload dist/* diff --git a/.github/workflows/sql-cli-test-and-build-workflow.yml b/.github/workflows/sql-cli-test-and-build-workflow.yml index 3de1ff3aa5..251765e8ac 100644 --- a/.github/workflows/sql-cli-test-and-build-workflow.yml +++ b/.github/workflows/sql-cli-test-and-build-workflow.yml @@ -1,6 +1,12 @@ name: SQL CLI Test and Build -on: [pull_request, push] +on: + push: + branches-ignore: + - 'dependabot/**' + paths: + - 'sql-cli/**' + - '.github/workflows/sql-cli-test-and-build-workflow.yml' jobs: build: @@ -15,10 +21,10 @@ jobs: steps: - name: Checkout SQL CLI - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -28,26 +34,16 @@ jobs: pip install -r requirements-dev.txt pip install setuptools wheel - #TODO: will setup CI for IT once we have OpenSearch and sql plugin release. Not it only runs UT - # It can also be refactored by launching OpenSearch instance with plugin installed from gradle. - - # - name: Set up ES and install SQL plugin - # run: | - # sudo add-apt-repository ppa:openjdk-r/ppa - # sudo apt update - # sudo apt install openjdk-14-jdk - # sudo apt install unzip - # wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.10.0-amd64.deb - # sudo dpkg -i elasticsearch-oss-7.10.0-amd64.deb - # sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/opensearch-sql/opensearch_sql-1.12.0.0.zip - # sudo systemctl start elasticsearch.service + - name: Download and run OpenSearch + run: | + wget -q https://artifacts.opensearch.org/releases/bundle/opensearch/2.2.1/opensearch-2.2.1-linux-x64.tar.gz + tar xf opensearch-2.2.1-linux-x64.tar.gz + opensearch-2.2.1/bin/opensearch-plugin remove opensearch-security + opensearch-2.2.1/bin/opensearch -d - name: Run Tox Testing run: tox - # - name: Stop ES - # run: sudo systemctl stop elasticsearch.service - - name: Build Artifact run: python setup.py sdist bdist_wheel @@ -57,7 +53,7 @@ jobs: cp -r ./dist/*.tar.gz ./dist/*.whl opensearchsql-builds/ - name: Upload Artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: opensearchsql path: sql-cli/opensearchsql-builds diff --git a/.github/workflows/sql-jdbc-push-jdbc-maven.yml b/.github/workflows/sql-jdbc-push-jdbc-maven.yml deleted file mode 100644 index 3c96447233..0000000000 --- a/.github/workflows/sql-jdbc-push-jdbc-maven.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Upload sql-jdbc Jar to Maven - -# This workflow will upload the sql-jdbc jar to maven when a new tag is cut -on: - push: - tags: - - v* - -jobs: - upload-jdbc-jar: - runs-on: ubuntu-latest - defaults: - run: - working-directory: sql-jdbc - name: Upload Jar to Maven - steps: - - name: Checkout Repo - uses: actions/checkout@v2 - - - name: Configure AWS CLI - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - # Since release workflow uses java 10 - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: '10' - - - name: Upload jdbc Jar to Maven - env: - passphrase: ${{ secrets.PASSPHRASE }} - run: | - cd .. - export JAVA10_HOME=$JAVA_HOME - aws s3 cp s3://opendistro-docs/github-actions/pgp-public-key . - aws s3 cp s3://opendistro-docs/github-actions/pgp-private-key . - - gpg --import pgp-public-key - gpg --allow-secret-key-import --import pgp-private-key - - - mkdir /home/runner/.gradle - aws s3 cp s3://opendistro-docs/github-actions/gradle.properties /home/runner/.gradle/ - cd sql-jdbc - ./gradlew publishShadowPublicationToSonatype-stagingRepository -Dcompiler.java=10 -Dbuild.snapshot=false -Djavax.net.ssl.trustStore=$JAVA_HOME/lib/security/cacerts diff --git a/.github/workflows/sql-jdbc-release-workflow.yml b/.github/workflows/sql-jdbc-release-workflow.yml deleted file mode 100644 index 39053615ee..0000000000 --- a/.github/workflows/sql-jdbc-release-workflow.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: Build and Release SQL-JDBC -# This workflow is triggered on creating tags to main or an opensearch release branch -on: - push: - tags: - - "v*" - -jobs: - Release-SQL-JDBC: - strategy: - matrix: - java: [10] - - name: Build and Release JDBC Plugin - runs-on: ubuntu-latest - defaults: - run: - working-directory: sql-jdbc - - steps: - - name: Checkout SQL-JDBC - uses: actions/checkout@v1 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Setup Java ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - - name: Run build - run: | - ./gradlew publishShadowPublicationToInternal-releasesRepository '-Dorg.gradle.jvmargs=--add-modules java.xml.bind' -Dbuild.snapshot=false - - - name: Configure Staging AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Publish to S3 - shell: bash - run: | - jar=`ls -1t build/libs/*.jar | grep -v "SNAPSHOT.jar" | grep -v "sources.jar" | head -1` - - # Inject the build number before the suffix - jar_outfile=`basename ${jar%.jar}-build-${GITHUB_RUN_NUMBER}.jar` - - s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-clients/sql-jdbc/" - - echo "Copying ${jar} to ${s3_prefix}${jar_outfile}" - aws s3 cp --quiet $jar ${s3_prefix}${jar_outfile} diff --git a/.github/workflows/sql-jdbc-test-and-build-workflow.yml b/.github/workflows/sql-jdbc-test-and-build-workflow.yml index 52d0dd68f8..1a5ba12532 100644 --- a/.github/workflows/sql-jdbc-test-and-build-workflow.yml +++ b/.github/workflows/sql-jdbc-test-and-build-workflow.yml @@ -1,6 +1,12 @@ name: SQL JDBC Java CI -on: [push, pull_request] +on: + push: + branches-ignore: + - 'dependabot/**' + paths: + - 'sql-jdbc/**' + - '.github/workflows/sql-jdbc-test-and-build-workflow.yml' jobs: build: @@ -15,11 +21,12 @@ jobs: working-directory: sql-jdbc steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} - name: Build with Gradle @@ -31,7 +38,7 @@ jobs: cp ./build/libs/*.jar sql-jdbc-builds - name: Upload Artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: sql-jdbc path: sql-jdbc/sql-jdbc-builds diff --git a/.github/workflows/sql-odbc-main.yml b/.github/workflows/sql-odbc-main.yml index 0a22780aa9..1fb0625c5c 100644 --- a/.github/workflows/sql-odbc-main.yml +++ b/.github/workflows/sql-odbc-main.yml @@ -1,6 +1,12 @@ name: OpenSearch ODBC Driver -on: [push, pull_request] +on: + push: + branches-ignore: + - 'dependabot/**' + paths: + - 'sql-odbc/**' + - '.github/workflows/sql-odbc-main.yml' env: CI_OUTPUT_PATH: "sql-odbc/ci-output" @@ -16,14 +22,14 @@ jobs: run: working-directory: sql-odbc steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - name: run-cppcheck run: | brew install cppcheck sh run_cppcheck.sh - name: upload-cppcheck-results if: failure() - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: cppcheck-results path: sql-odbc/cppcheck-results.log @@ -61,19 +67,19 @@ jobs: # cp ./bin64/*.log test-output - name: upload-build if: success() - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: mac64-build path: sql-odbc/build-output - name: upload-installer if: success() - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: mac64-installer path: sql-odbc/installer #- name: upload-test-results - # if: success() - # uses: actions/upload-artifact@v1 + # if: always() + # uses: actions/upload-artifact@v3 # with: # name: mac-test-results # path: test-output @@ -83,7 +89,7 @@ jobs: run: working-directory: sql-odbc steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - name: Get specific version CMake, v3.18.3 uses: lukka/get-cmake@v3.18.3 - name: add-msbuild-to-path @@ -106,19 +112,19 @@ jobs: .\scripts\prepare_ci_output.ps1 $Env:ODBC_BIN_PATH $Env:ODBC_LIB_PATH $Env:ODBC_BUILD_PATH - name: upload-build if: always() - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: windows32-build path: sql-odbc/ci-output/build - name: upload-installer if: always() - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: windows32-installer path: sql-odbc/ci-output/installer #- name: upload-test-results # if: always() - # uses: actions/upload-artifact@v1 + # uses: actions/upload-artifact@v3 # with: # name: windows-test-results # path: $CI_OUTPUT_PATH/test @@ -128,7 +134,7 @@ jobs: run: working-directory: sql-odbc steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - name: Get specific version CMake, v3.18.3 uses: lukka/get-cmake@v3.18.3 - name: add-msbuild-to-path @@ -151,19 +157,19 @@ jobs: .\scripts\prepare_ci_output.ps1 $Env:ODBC_BIN_PATH $Env:ODBC_LIB_PATH $Env:ODBC_BUILD_PATH - name: upload-build if: always() - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: windows64-build path: sql-odbc/ci-output/build - name: upload-installer if: always() - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: windows64-installer path: sql-odbc/ci-output/installer #- name: upload-test-results # if: always() - # uses: actions/upload-artifact@v1 + # uses: actions/upload-artifact@v3 # with: # name: windows-test-results # path: sql-odbc/ci-output/test-output diff --git a/.github/workflows/sql-odbc-release-workflow.yml b/.github/workflows/sql-odbc-release-workflow.yml deleted file mode 100644 index 5aea61ed16..0000000000 --- a/.github/workflows/sql-odbc-release-workflow.yml +++ /dev/null @@ -1,212 +0,0 @@ -name: Build and Release SQL-ODBC -# This workflow is triggered on creating tags to main or an opensearch release branch -on: - push: - tags: - - 'v*' - -env: - CI_OUTPUT_PATH: "sql-odbc/ci-output" - ODBC_LIB_PATH: "./build/odbc/lib" - ODBC_BIN_PATH: "./build/odbc/bin" - ODBC_BUILD_PATH: "./build/odbc/build" - AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install" - PLUGIN_NAME: opensearch-sql-odbc - OD_VERSION: 2.4.0.0 - -jobs: - build-mac: - runs-on: macos-10.15 - defaults: - run: - working-directory: sql-odbc - steps: - - uses: actions/checkout@v1 - - name: run-cppcheck - run: | - brew install cppcheck - sh run_cppcheck.sh - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - name: upload-cppcheck-results - if: failure() - uses: actions/upload-artifact@v1 - with: - name: cppcheck-results - path: sql-odbc/cppcheck-results.log - - name: get-dependencies - run: | - brew unlink unixodbc - brew install curl - brew install cmake - brew install libiodbc - - name: configure-and-build-driver - run: | - ./build_mac_release64.sh - - name: build-installer - if: success() - run: | - cd cmake-build64 - cmake ../src - make - cpack . - cd .. - - name: create-output - if: success() - run: | - mkdir build-output - mkdir test-output - mkdir installer - cp ./build/odbc/lib/*.dylib build-output/ - cp ./build/odbc/lib/*.a build-output/ - cp ./cmake-build64/*.pkg installer/ - # cp $(ls -d bin64/* | grep -v "\.") build - - name: upload-build - if: success() - uses: actions/upload-artifact@v1 - with: - name: mac64-build - path: sql-odbc/build-output - - name: upload-installer - if: success() - uses: actions/upload-artifact@v1 - with: - name: mac64-installer - path: sql-odbc/installer - - name: upload-artifacts-s3 - if: success() - run: | - cd installer - - pkg=`ls -1t *.pkg | grep "OpenSearch SQL ODBC Driver" | head -1` - mv "$pkg" "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-macos-x64.pkg" - pkg=`ls -1t *.pkg | grep "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-macos-x64.pkg" | head -1` - - # Inject the build number before the suffix - pkg_outfile=`basename ${pkg%.pkg}-build-${GITHUB_RUN_NUMBER}.pkg` - - s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-clients/sql-odbc/" - - echo "Copying ${pkg} to ${s3_prefix}${pkg_outfile}" - aws s3 cp --quiet $pkg ${s3_prefix}${pkg_outfile} - - build-windows32: - runs-on: windows-latest - defaults: - run: - working-directory: sql-odbc - steps: - - uses: actions/checkout@v1 - - name: Get specific version CMake, v3.18.3 - uses: lukka/get-cmake@v3.18.3 - - name: add-msbuild-to-path - uses: microsoft/setup-msbuild@v1.0.2 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - name: configure-and-build-driver - run: | - .\build_win_release32.ps1 - - name: build-installer - if: success() - run: | - .\scripts\build_installer.ps1 Release Win32 .\src $Env:ODBC_BUILD_PATH $Env:AWS_SDK_INSTALL_PATH - - name: prepare-output - if: always() - run: | - .\scripts\prepare_ci_output.ps1 $Env:ODBC_BIN_PATH $Env:ODBC_LIB_PATH $Env:ODBC_BUILD_PATH - - name: upload-build - if: always() - uses: actions/upload-artifact@v1 - with: - name: windows32-build - path: sql-odbc/ci-output/build - - name: upload-installer - if: always() - uses: actions/upload-artifact@v1 - with: - name: windows32-installer - path: sql-odbc/ci-output/installer - - name: upload-artifacts-s3 - if: success() - shell: bash - run: | - cd ci-output/installer - - msi=`ls -1t *.msi | grep "OpenSearch SQL ODBC Driver" | head -1` - mv "$msi" "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-windows-x86.msi" - msi=`ls -1t *.msi | grep "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-windows-x86.msi" | head -1` - - # Inject the build number before the suffix - msi_outfile=`basename ${msi%.msi}-build-${GITHUB_RUN_NUMBER}.msi` - - s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-clients/sql-odbc/" - - echo "Copying ${msi} to ${s3_prefix}${msi_outfile}" - aws s3 cp --quiet $msi ${s3_prefix}${msi_outfile} - - build-windows64: - runs-on: windows-latest - defaults: - run: - working-directory: sql-odbc - steps: - - uses: actions/checkout@v1 - - name: Get specific version CMake, v3.18.3 - uses: lukka/get-cmake@v3.18.3 - - name: add-msbuild-to-path - uses: microsoft/setup-msbuild@v1.0.2 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - name: configure-and-build-driver - run: | - .\build_win_release64.ps1 - - name: build-installer - if: success() - run: | - .\scripts\build_installer.ps1 Release x64 .\src $Env:ODBC_BUILD_PATH $Env:AWS_SDK_INSTALL_PATH - - name: prepare-output - if: always() - run: | - .\scripts\prepare_ci_output.ps1 $Env:ODBC_BIN_PATH $Env:ODBC_LIB_PATH $Env:ODBC_BUILD_PATH - - name: upload-build - if: always() - uses: actions/upload-artifact@v1 - with: - name: windows64-build - path: sql-odbc/ci-output/build - - name: upload-installer - if: always() - uses: actions/upload-artifact@v1 - with: - name: windows64-installer - path: sql-odbc/ci-output/installer - - name: upload-artifacts-s3 - if: success() - shell: bash - run: | - cd ci-output/installer - - msi=`ls -1t *.msi | grep "OpenSearch SQL ODBC Driver" | head -1` - mv "$msi" "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-windows-x64.msi" - msi=`ls -1t *.msi | grep "${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-windows-x64.msi" | head -1` - - # Inject the build number before the suffix - msi_outfile=`basename ${msi%.msi}-build-${GITHUB_RUN_NUMBER}.msi` - - s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-clients/sql-odbc/" - - echo "Copying ${msi} to ${s3_prefix}${msi_outfile}" - aws s3 cp --quiet $msi ${s3_prefix}${msi_outfile} - diff --git a/.github/workflows/sql-release-workflow.yml b/.github/workflows/sql-release-workflow.yml deleted file mode 100644 index e371723f34..0000000000 --- a/.github/workflows/sql-release-workflow.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: Release SQL Artifacts -# This workflow is triggered on creating tags to main or an opensearch release branch -on: - push: - tags: - - 'v*' - -jobs: - build: - strategy: - matrix: - java: [14] - - name: Build and Release SQL Plugin - runs-on: ubuntu-latest - - steps: - - name: Checkout SQL - uses: actions/checkout@v1 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Setup Java ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - - name: Run build - run: | - ./gradlew build buildDeb buildRpm --no-daemon --refresh-dependencies --console=plain -Dbuild.snapshot=false -x doctest:doctest - - - name: Upload to S3 - shell: bash - run: | - zip=`ls plugin/build/distributions/*.zip` - rpm=`ls plugin/build/distributions/*.rpm` - deb=`ls plugin/build/distributions/*.deb` - - # Inject the build number before the suffix - zip_outfile=`basename ${zip%.zip}-build-${GITHUB_RUN_NUMBER}.zip` - rpm_outfile=`basename ${rpm%.rpm}-build-${GITHUB_RUN_NUMBER}.rpm` - deb_outfile=`basename ${deb%.deb}-build-${GITHUB_RUN_NUMBER}.deb` - - s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-plugins/sql/" - - echo "Copying ${zip} to ${s3_prefix}${zip_outfile}" - aws s3 cp --quiet $zip ${s3_prefix}${zip_outfile} - - echo "Copying ${rpm} to ${s3_prefix}${rpm_outfile}" - aws s3 cp --quiet $rpm ${s3_prefix}${rpm_outfile} - - echo "Copying ${deb} to ${s3_prefix}${deb_outfile}" - aws s3 cp --quiet $deb ${s3_prefix}${deb_outfile} diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index fcc63433a8..65062a2b78 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -1,6 +1,33 @@ name: SQL Java CI -on: [push, pull_request] +on: + push: + branches-ignore: + - 'dependabot/**' + paths: + - 'build/**' + - 'build-tools/**' + - 'build.gradle/**' + - 'buildSrc/**' + - 'common/**' + - 'config/**' + - 'core/**' + - 'doctest/**' + - 'gradle/**' + - 'gradle.properties/**' + - 'gradlew/**' + - 'gradlew.bat/**' + - 'integ-test/**' + - 'legacy/**' + - 'lombok.config/**' + - 'opensearch/**' + - 'plugin/**' + - 'ppl/**' + - 'protocol/**' + - 'scripts/**' + - 'settings.gradle/**' + - 'sql/**' + - '.github/workflows/sql-test-and-build-workflow.yml' jobs: build: @@ -12,11 +39,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: + distribution: 'temurin' java-version: ${{ matrix.java }} - name: Build with Gradle @@ -32,13 +60,29 @@ jobs: # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - name: Upload SQL Coverage Report - uses: codecov/codecov-action@v1 + if: always() + uses: codecov/codecov-action@v3 with: flags: sql-engine token: ${{ secrets.CODECOV_TOKEN }} - name: Upload Artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: opensearch-sql path: opensearch-sql-builds + + - name: Upload test reports + if: always() + uses: actions/upload-artifact@v2 + with: + name: test-reports + path: | + sql/build/reports/** + ppl/build/reports/** + core/build/reports/** + common/build/reports/** + opensearch/build/reports/** + integ-test/build/reports/** + protocol/build/reports/** + legacy/build/reports/** diff --git a/.github/workflows/sql-workbench-release-workflow.yml b/.github/workflows/sql-workbench-release-workflow.yml deleted file mode 100644 index 631eb3049a..0000000000 --- a/.github/workflows/sql-workbench-release-workflow.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: Release SQL-Workbench Artifacts - -on: - push: - tags: - - 'v*' - -env: - PLUGIN_NAME: query-workbench-dashboards - OPENSEARCH_VERSION: 'main' - OPENSEARCH_PLUGIN_VERSION: 2.4.0.0 - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Checkout Plugin - uses: actions/checkout@v1 - - - name: Checkout OpenSearch Dashboards - uses: actions/checkout@v1 - with: - repository: opensearch-project/Opensearch-Dashboards - ref: ${{ env.OPENSEARCH_VERSION }} - path: sql/OpenSearch-Dashboards - - - name: Setup Node - uses: actions/setup-node@v1 - with: - node-version: '14.18.2' - - - name: Move Workbench to Plugins Dir - run: | - mv workbench OpenSearch-Dashboards/plugins - - - name: OpenSearch Dashboards Plugin Bootstrap - uses: nick-invision/retry@v1 - with: - timeout_minutes: 60 - max_attempts: 3 - command: cd OpenSearch-Dashboards/plugins/workbench; yarn osd bootstrap - - - name: Build - run: | - cd OpenSearch-Dashboards/plugins/workbench - yarn build - mv ./build/*.zip ./build/${{ env.PLUGIN_NAME }}-${{ env.OPENSEARCH_PLUGIN_VERSION }}.zip - - - name: Upload to S3 - shell: bash - run: | - cd OpenSearch-Dashboards/plugins/workbench - zip=`ls ./build/*.zip` - - # Inject the build number before the suffix - zip_outfile=`basename ${zip%.zip}-build-${GITHUB_RUN_NUMBER}.zip` - - s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/kibana-plugins/query-workbench/" - - echo "Copying ${zip} to ${s3_prefix}${zip_outfile}" - aws s3 cp --quiet $zip ${s3_prefix}${zip_outfile} diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index a7f788c8af..a6a8222567 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -1,11 +1,39 @@ name: SQL Workbench Test and Build -on: [pull_request, push] +on: + push: + branches-ignore: + - 'dependabot/**' + paths: + - 'build/**' + - 'build-tools/**' + - 'build.gradle/**' + - 'buildSrc/**' + - 'common/**' + - 'config/**' + - 'core/**' + - 'doctest/**' + - 'gradle/**' + - 'gradle.properties/**' + - 'gradlew/**' + - 'gradlew.bat/**' + - 'integ-test/**' + - 'legacy/**' + - 'lombok.config/**' + - 'opensearch/**' + - 'plugin/**' + - 'ppl/**' + - 'protocol/**' + - 'scripts/**' + - 'settings.gradle/**' + - 'sql/**' + - 'workbench/**' + - '.github/workflows/sql-workbench-test-and-build-workflow.yml' env: PLUGIN_NAME: query-workbench-dashboards OPENSEARCH_VERSION: 'main' - OPENSEARCH_PLUGIN_VERSION: 2.4.0.0 + OPENSEARCH_PLUGIN_VERSION: 2.3.0.0 jobs: @@ -15,17 +43,17 @@ jobs: steps: - name: Checkout Plugin - uses: actions/checkout@v1 + uses: actions/checkout@v3 - name: Checkout OpenSearch Dashboards - uses: actions/checkout@v1 + uses: actions/checkout@v1 # can't update to v3 because `setup-node` fails with: repository: opensearch-project/Opensearch-Dashboards ref: ${{ env.OPENSEARCH_VERSION }} path: OpenSearch-Dashboards - + - name: Setup Node - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version-file: "../OpenSearch-Dashboards/.nvmrc" registry-url: 'https://registry.npmjs.org' @@ -35,7 +63,7 @@ jobs: mv workbench ../OpenSearch-Dashboards/plugins - name: OpenSearch Dashboards Plugin Bootstrap - uses: nick-invision/retry@v1 + uses: nick-fields/retry@v2 with: timeout_minutes: 60 max_attempts: 3 @@ -47,7 +75,8 @@ jobs: yarn test:jest --coverage - name: Upload coverage - uses: codecov/codecov-action@v1 + if: always() + uses: codecov/codecov-action@v3 with: flags: query-workbench directory: ../OpenSearch-Dashboards/plugins/workbench @@ -60,7 +89,8 @@ jobs: mv ./build/*.zip ./build/${{ env.PLUGIN_NAME }}-${{ env.OPENSEARCH_PLUGIN_VERSION }}.zip - name: Upload Artifact - uses: actions/upload-artifact@v1 + if: always() + uses: actions/upload-artifact@v1 # can't update to v3 because upload fails with: name: workbench path: ../OpenSearch-Dashboards/plugins/workbench/build From 1d935e6cf205f63dafade3d7937df5f06c207b43 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Wed, 14 Sep 2022 16:24:01 -0700 Subject: [PATCH 02/10] Address PR feedback. Signed-off-by: Yury-Fridlyand --- .github/workflows/draft-release-notes-workflow.yml | 2 +- .github/workflows/sql-cli-test-and-build-workflow.yml | 1 + .github/workflows/sql-workbench-test-and-build-workflow.yml | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml index 660a8a1a51..7f5a124ddd 100644 --- a/.github/workflows/draft-release-notes-workflow.yml +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -16,6 +16,6 @@ jobs: with: config-name: draft-release-notes-config.yml tag: (None) - version: 2.3.0.0 + version: 2.4.0.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sql-cli-test-and-build-workflow.yml b/.github/workflows/sql-cli-test-and-build-workflow.yml index 251765e8ac..5b938ffcb9 100644 --- a/.github/workflows/sql-cli-test-and-build-workflow.yml +++ b/.github/workflows/sql-cli-test-and-build-workflow.yml @@ -34,6 +34,7 @@ jobs: pip install -r requirements-dev.txt pip install setuptools wheel + # tests are designed to run agains http://localhost:9200, so we have to disable/remove security plugin - name: Download and run OpenSearch run: | wget -q https://artifacts.opensearch.org/releases/bundle/opensearch/2.2.1/opensearch-2.2.1-linux-x64.tar.gz diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index a6a8222567..9c3259c235 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -33,7 +33,7 @@ on: env: PLUGIN_NAME: query-workbench-dashboards OPENSEARCH_VERSION: 'main' - OPENSEARCH_PLUGIN_VERSION: 2.3.0.0 + OPENSEARCH_PLUGIN_VERSION: 2.4.0.0 jobs: @@ -51,7 +51,7 @@ jobs: repository: opensearch-project/Opensearch-Dashboards ref: ${{ env.OPENSEARCH_VERSION }} path: OpenSearch-Dashboards - + - name: Setup Node uses: actions/setup-node@v3 with: From 5284e9500f386d532a32a12fd2e33e9ec975e594 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Wed, 14 Sep 2022 19:17:20 -0700 Subject: [PATCH 03/10] Strictly define env version to avoid unexpected failures. Other minor changes. Signed-off-by: Yury-Fridlyand --- .github/workflows/backport.yml | 2 +- .github/workflows/bi-connectors.yml | 2 +- .github/workflows/codeql-analysis.yml | 37 +------------------ .github/workflows/dco.yml | 2 +- .github/workflows/delete_backport_branch.yml | 2 +- .../draft-release-notes-workflow.yml | 2 +- .github/workflows/link-checker.yml | 7 +++- .../sql-cli-test-and-build-workflow.yml | 4 +- .../sql-jdbc-test-and-build-workflow.yml | 2 +- .github/workflows/sql-odbc-main.yml | 2 + .../workflows/sql-test-and-build-workflow.yml | 2 +- .../sql-workbench-test-and-build-workflow.yml | 2 +- 12 files changed, 18 insertions(+), 48 deletions(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index e47d8d88c0..7cb0856a86 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -7,7 +7,7 @@ on: jobs: backport: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 permissions: contents: write pull-requests: write diff --git a/.github/workflows/bi-connectors.yml b/.github/workflows/bi-connectors.yml index d39314f29e..1166bc1d3a 100644 --- a/.github/workflows/bi-connectors.yml +++ b/.github/workflows/bi-connectors.yml @@ -9,7 +9,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: Pack Tableau JDBC connector diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b1ac364cd5..ea87a8b558 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,15 +1,3 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# ******** NOTE ******** - name: "CodeQL" on: @@ -26,7 +14,7 @@ on: jobs: analyze: name: CodeQL-Scan - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 permissions: security-events: write actions: read @@ -34,39 +22,16 @@ jobs: fail-fast: false matrix: language: [ 'java' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] - # Learn more... - # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection steps: - name: Checkout repository uses: actions/checkout@v3 - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v2 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml index cf30ea89dc..fc206f5ae1 100644 --- a/.github/workflows/dco.yml +++ b/.github/workflows/dco.yml @@ -4,7 +4,7 @@ on: [pull_request] jobs: check: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Get PR Commits diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index 387a124b8c..ec69dcaa06 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -6,7 +6,7 @@ on: jobs: delete-branch: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 if: startsWith(github.event.pull_request.head.ref,'backport/') steps: - name: Delete merged branch diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml index 7f5a124ddd..f272fc8646 100644 --- a/.github/workflows/draft-release-notes-workflow.yml +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -8,7 +8,7 @@ on: jobs: update_release_draft: name: Update draft release notes - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: # Drafts your next Release notes as Pull Requests are merged into "develop" - name: Update draft release notes diff --git a/.github/workflows/link-checker.yml b/.github/workflows/link-checker.yml index 518af21f3a..5f5de0687d 100644 --- a/.github/workflows/link-checker.yml +++ b/.github/workflows/link-checker.yml @@ -4,7 +4,10 @@ on: push: branches: [ main ] paths: - - 'docs/**' + - '**/*.pdf' + - '**/*.pnj' + - '**/*.jpg' + - '**/*.json' - '**/*.md' - '**/*.txt' - '**/*.rst' @@ -12,7 +15,7 @@ on: jobs: linkchecker: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/sql-cli-test-and-build-workflow.yml b/.github/workflows/sql-cli-test-and-build-workflow.yml index 5b938ffcb9..0d8776754d 100644 --- a/.github/workflows/sql-cli-test-and-build-workflow.yml +++ b/.github/workflows/sql-cli-test-and-build-workflow.yml @@ -11,7 +11,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 defaults: run: working-directory: sql-cli @@ -34,7 +34,7 @@ jobs: pip install -r requirements-dev.txt pip install setuptools wheel - # tests are designed to run agains http://localhost:9200, so we have to disable/remove security plugin + # tests are designed to run agains http://localhost:9200, so we have to disable/remove security plugin - name: Download and run OpenSearch run: | wget -q https://artifacts.opensearch.org/releases/bundle/opensearch/2.2.1/opensearch-2.2.1-linux-x64.tar.gz diff --git a/.github/workflows/sql-jdbc-test-and-build-workflow.yml b/.github/workflows/sql-jdbc-test-and-build-workflow.yml index 1a5ba12532..5f16f266fe 100644 --- a/.github/workflows/sql-jdbc-test-and-build-workflow.yml +++ b/.github/workflows/sql-jdbc-test-and-build-workflow.yml @@ -15,7 +15,7 @@ jobs: java: - 11 - 17 - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 defaults: run: working-directory: sql-jdbc diff --git a/.github/workflows/sql-odbc-main.yml b/.github/workflows/sql-odbc-main.yml index 1fb0625c5c..e27daca7bc 100644 --- a/.github/workflows/sql-odbc-main.yml +++ b/.github/workflows/sql-odbc-main.yml @@ -15,6 +15,8 @@ env: ODBC_BUILD_PATH: "./build/odbc/build" AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install" +# Tests are disabled (commented out) in all jobs because they are fail and/or outdated +# Keeping them for the brighten future when we can re-activate them jobs: build-mac: runs-on: macos-10.15 diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 65062a2b78..4c20cd2dd9 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -36,7 +36,7 @@ jobs: java: - 11 - 17 - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index 9c3259c235..935d474ccd 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -39,7 +39,7 @@ jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout Plugin From 62acf6abfdfeb6fb5fd1a1441f9954a4f7a4a78f Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 15 Sep 2022 10:09:28 -0700 Subject: [PATCH 04/10] Update .github/workflows/sql-cli-test-and-build-workflow.yml Signed-off-by: Yury-Fridlyand Co-authored-by: Andrew Carbonetto <104530826+acarbonetto@users.noreply.github.com> --- .github/workflows/sql-cli-test-and-build-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sql-cli-test-and-build-workflow.yml b/.github/workflows/sql-cli-test-and-build-workflow.yml index 0d8776754d..7cb0a66fae 100644 --- a/.github/workflows/sql-cli-test-and-build-workflow.yml +++ b/.github/workflows/sql-cli-test-and-build-workflow.yml @@ -34,7 +34,7 @@ jobs: pip install -r requirements-dev.txt pip install setuptools wheel - # tests are designed to run agains http://localhost:9200, so we have to disable/remove security plugin + # tests are designed to run against http://localhost:9200, so we have to disable/remove security plugin - name: Download and run OpenSearch run: | wget -q https://artifacts.opensearch.org/releases/bundle/opensearch/2.2.1/opensearch-2.2.1-linux-x64.tar.gz From 54c897d66b6eda685e46724d86678afaa2ae7c0f Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 15 Sep 2022 11:48:50 -0700 Subject: [PATCH 05/10] Extend workflow trigger criteria. Signed-off-by: Yury-Fridlyand --- .github/workflows/codeql-analysis.yml | 4 +++- .github/workflows/draft-release-notes-workflow.yml | 4 +++- .github/workflows/link-checker.yml | 5 ++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index ea87a8b558..24e4550d89 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -3,7 +3,9 @@ name: "CodeQL" on: pull_request: branches: - - main + - 'main' + - '[1-9]+.[0-9x]+' + - '[1-9]+.[0-9x]+.[0-9x]+' push: branches-ignore: - 'dependabot/**' diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml index f272fc8646..7dddc9eccb 100644 --- a/.github/workflows/draft-release-notes-workflow.yml +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -3,7 +3,9 @@ name: Release Drafter on: push: branches: - - main + - 'main' + - '[1-9]+.[0-9x]+' + - '[1-9]+.[0-9x]+.[0-9x]+' jobs: update_release_draft: diff --git a/.github/workflows/link-checker.yml b/.github/workflows/link-checker.yml index 5f5de0687d..7320cbc7cf 100644 --- a/.github/workflows/link-checker.yml +++ b/.github/workflows/link-checker.yml @@ -2,7 +2,10 @@ name: Link Checker on: push: - branches: [ main ] + branches: + - 'main' + - '[1-9]+.[0-9x]+' + - '[1-9]+.[0-9x]+.[0-9x]+' paths: - '**/*.pdf' - '**/*.pnj' From c4be388bd28b8f4c6be7c5369674af01848cd1c5 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 15 Sep 2022 12:50:35 -0700 Subject: [PATCH 06/10] Make OpenSearch version parametrized for SQL CLI tests. Signed-off-by: Yury-Fridlyand --- .github/workflows/sql-cli-test-and-build-workflow.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sql-cli-test-and-build-workflow.yml b/.github/workflows/sql-cli-test-and-build-workflow.yml index 7cb0a66fae..bdf4f33c2f 100644 --- a/.github/workflows/sql-cli-test-and-build-workflow.yml +++ b/.github/workflows/sql-cli-test-and-build-workflow.yml @@ -18,6 +18,7 @@ jobs: strategy: matrix: python-version: [3.8] + opensearch-version: [ 2.2.1 ] steps: - name: Checkout SQL CLI @@ -37,10 +38,10 @@ jobs: # tests are designed to run against http://localhost:9200, so we have to disable/remove security plugin - name: Download and run OpenSearch run: | - wget -q https://artifacts.opensearch.org/releases/bundle/opensearch/2.2.1/opensearch-2.2.1-linux-x64.tar.gz - tar xf opensearch-2.2.1-linux-x64.tar.gz - opensearch-2.2.1/bin/opensearch-plugin remove opensearch-security - opensearch-2.2.1/bin/opensearch -d + wget -q https://artifacts.opensearch.org/releases/bundle/opensearch/${{ matrix.opensearch-version }}/opensearch-${{ matrix.opensearch-version }}-linux-x64.tar.gz + tar xf opensearch-${{ matrix.opensearch-version }}-linux-x64.tar.gz + opensearch-${{ matrix.opensearch-version }}/bin/opensearch-plugin remove opensearch-security + opensearch-${{ matrix.opensearch-version }}/bin/opensearch -d - name: Run Tox Testing run: tox From 8b5444d3bb2defd930558e6590c19f33c00625f9 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 15 Sep 2022 12:56:31 -0700 Subject: [PATCH 07/10] Make workflows run once only on PRs. Signed-off-by: Yury-Fridlyand --- .github/workflows/bi-connectors.yml | 2 ++ .github/workflows/codeql-analysis.yml | 1 + .github/workflows/draft-release-notes-workflow.yml | 2 ++ .github/workflows/link-checker.yml | 2 ++ .github/workflows/sql-cli-test-and-build-workflow.yml | 2 ++ .github/workflows/sql-jdbc-test-and-build-workflow.yml | 2 ++ .github/workflows/sql-odbc-main.yml | 2 ++ .github/workflows/sql-test-and-build-workflow.yml | 2 ++ .github/workflows/sql-workbench-test-and-build-workflow.yml | 2 ++ 9 files changed, 17 insertions(+) diff --git a/.github/workflows/bi-connectors.yml b/.github/workflows/bi-connectors.yml index 1166bc1d3a..245c568fa9 100644 --- a/.github/workflows/bi-connectors.yml +++ b/.github/workflows/bi-connectors.yml @@ -1,6 +1,8 @@ name: Build connectors for BI tools on: + pull_request: + types: [opened, reopened] push: paths: - 'bi-connectors/PowerBIConnector/**' diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 24e4550d89..e043581de9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -6,6 +6,7 @@ on: - 'main' - '[1-9]+.[0-9x]+' - '[1-9]+.[0-9x]+.[0-9x]+' + types: [opened, reopened] push: branches-ignore: - 'dependabot/**' diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml index 7dddc9eccb..c0a157ee40 100644 --- a/.github/workflows/draft-release-notes-workflow.yml +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -1,6 +1,8 @@ name: Release Drafter on: + pull_request: + types: [opened, reopened] push: branches: - 'main' diff --git a/.github/workflows/link-checker.yml b/.github/workflows/link-checker.yml index 7320cbc7cf..5e42a68947 100644 --- a/.github/workflows/link-checker.yml +++ b/.github/workflows/link-checker.yml @@ -1,6 +1,8 @@ name: Link Checker on: + pull_request: + types: [opened, reopened] push: branches: - 'main' diff --git a/.github/workflows/sql-cli-test-and-build-workflow.yml b/.github/workflows/sql-cli-test-and-build-workflow.yml index bdf4f33c2f..4d939c930c 100644 --- a/.github/workflows/sql-cli-test-and-build-workflow.yml +++ b/.github/workflows/sql-cli-test-and-build-workflow.yml @@ -1,6 +1,8 @@ name: SQL CLI Test and Build on: + pull_request: + types: [opened, reopened] push: branches-ignore: - 'dependabot/**' diff --git a/.github/workflows/sql-jdbc-test-and-build-workflow.yml b/.github/workflows/sql-jdbc-test-and-build-workflow.yml index 5f16f266fe..2342e00cd6 100644 --- a/.github/workflows/sql-jdbc-test-and-build-workflow.yml +++ b/.github/workflows/sql-jdbc-test-and-build-workflow.yml @@ -1,6 +1,8 @@ name: SQL JDBC Java CI on: + pull_request: + types: [opened, reopened] push: branches-ignore: - 'dependabot/**' diff --git a/.github/workflows/sql-odbc-main.yml b/.github/workflows/sql-odbc-main.yml index e27daca7bc..eee65e5b29 100644 --- a/.github/workflows/sql-odbc-main.yml +++ b/.github/workflows/sql-odbc-main.yml @@ -1,6 +1,8 @@ name: OpenSearch ODBC Driver on: + pull_request: + types: [opened, reopened] push: branches-ignore: - 'dependabot/**' diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 4c20cd2dd9..2aa5cfa34f 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -1,6 +1,8 @@ name: SQL Java CI on: + pull_request: + types: [opened, reopened] push: branches-ignore: - 'dependabot/**' diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index 935d474ccd..5dd8206c44 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -1,6 +1,8 @@ name: SQL Workbench Test and Build on: + pull_request: + types: [opened, reopened] push: branches-ignore: - 'dependabot/**' From 17f39a27a093b9a3daac003865588da18246304a Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 15 Sep 2022 13:10:13 -0700 Subject: [PATCH 08/10] Fix and reduce path triggers for Workbench and SQL workflows. Signed-off-by: Yury-Fridlyand --- .../workflows/sql-test-and-build-workflow.yml | 29 +++++-------------- .../sql-workbench-test-and-build-workflow.yml | 29 +++++-------------- 2 files changed, 15 insertions(+), 43 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 2aa5cfa34f..91bee808bf 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -7,28 +7,15 @@ on: branches-ignore: - 'dependabot/**' paths: - - 'build/**' - - 'build-tools/**' - - 'build.gradle/**' - - 'buildSrc/**' - - 'common/**' - - 'config/**' - - 'core/**' - - 'doctest/**' - - 'gradle/**' - - 'gradle.properties/**' - - 'gradlew/**' - - 'gradlew.bat/**' + - '**/*.java' + - '**/*.g4' + - '!sql-jdbc/**' + - '**gradle*' + - '**lombok*' + - '**checkstyle*' - 'integ-test/**' - - 'legacy/**' - - 'lombok.config/**' - - 'opensearch/**' - - 'plugin/**' - - 'ppl/**' - - 'protocol/**' - - 'scripts/**' - - 'settings.gradle/**' - - 'sql/**' + - '**/*.jar' + - '**/*.pom' - '.github/workflows/sql-test-and-build-workflow.yml' jobs: diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index 5dd8206c44..588c2ee3d7 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -7,28 +7,13 @@ on: branches-ignore: - 'dependabot/**' paths: - - 'build/**' - - 'build-tools/**' - - 'build.gradle/**' - - 'buildSrc/**' - - 'common/**' - - 'config/**' - - 'core/**' - - 'doctest/**' - - 'gradle/**' - - 'gradle.properties/**' - - 'gradlew/**' - - 'gradlew.bat/**' - - 'integ-test/**' - - 'legacy/**' - - 'lombok.config/**' - - 'opensearch/**' - - 'plugin/**' - - 'ppl/**' - - 'protocol/**' - - 'scripts/**' - - 'settings.gradle/**' - - 'sql/**' + - '**/*.java' + - '**/*.g4' + - '!sql-jdbc/**' + - '**gradle*' + - '**lombok*' + - '**/*.jar' + - '**/*.pom' - 'workbench/**' - '.github/workflows/sql-workbench-test-and-build-workflow.yml' From 2912b5e1b9e9a929f8ddbed387d25c185c4c0499 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 15 Sep 2022 14:42:39 -0700 Subject: [PATCH 09/10] Use docker OpenSearch image for testing SQL CLI. Signed-off-by: Yury-Fridlyand --- .../sql-cli-test-and-build-workflow.yml | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sql-cli-test-and-build-workflow.yml b/.github/workflows/sql-cli-test-and-build-workflow.yml index 4d939c930c..114813b9e2 100644 --- a/.github/workflows/sql-cli-test-and-build-workflow.yml +++ b/.github/workflows/sql-cli-test-and-build-workflow.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: python-version: [3.8] - opensearch-version: [ 2.2.1 ] + opensearch-version: [ 2.2.1, latest ] steps: - name: Checkout SQL CLI @@ -40,10 +40,14 @@ jobs: # tests are designed to run against http://localhost:9200, so we have to disable/remove security plugin - name: Download and run OpenSearch run: | - wget -q https://artifacts.opensearch.org/releases/bundle/opensearch/${{ matrix.opensearch-version }}/opensearch-${{ matrix.opensearch-version }}-linux-x64.tar.gz - tar xf opensearch-${{ matrix.opensearch-version }}-linux-x64.tar.gz - opensearch-${{ matrix.opensearch-version }}/bin/opensearch-plugin remove opensearch-security - opensearch-${{ matrix.opensearch-version }}/bin/opensearch -d + docker run -p 9200:9200 -e "discovery.type=single-node" -e "DISABLE_SECURITY_PLUGIN=true" --name test -d opensearchproject/opensearch:${{ matrix.opensearch-version }} + + - name: Wait for cluster to start + uses: nick-fields/retry@v2 + with: + timeout_seconds: 1 + max_attempts: 30 + command: curl -q localhost:9200 - name: Run Tox Testing run: tox @@ -61,3 +65,9 @@ jobs: with: name: opensearchsql path: sql-cli/opensearchsql-builds + + - name: Clean up container + if: always() + run: | + docker container stop test + docker container rm test From 6418cf61a8e4eacf84962ef1f8f6362bff6e977a Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 15 Sep 2022 16:10:49 -0700 Subject: [PATCH 10/10] Disable one SQL CLI test. Signed-off-by: Yury-Fridlyand --- sql-cli/tests/test_opensearch_connection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sql-cli/tests/test_opensearch_connection.py b/sql-cli/tests/test_opensearch_connection.py index 1a1aa76ecc..75740881c4 100644 --- a/sql-cli/tests/test_opensearch_connection.py +++ b/sql-cli/tests/test_opensearch_connection.py @@ -39,6 +39,7 @@ def test_query(self, connection): ) @estest + @pytest.mark.skip(reason="Test is not compatible with OpenSearch >= 2.3.0, it returns HTTP/503 instead of HTTP/400") def test_query_nonexistent_index(self, connection): self.load_data_to_es(connection)