Moving CI to GitHub actions (#3669) (#3800)

* test

* test test

* try this again

* test actions in same repo

* nvm revert

* formatting

* fix sh script for building dists

* fix windows build

* add concurrency

* fix random 'Cannot track experimental parser info when active user is None' error

* fix build workflow

* test slim ci

* has changes

* set up postgres for other OS

* update descriptions

* turn off python3.9 unit tests

* add changelog

* clean up todo

* Update .github/workflows/main.yml

* create actions for common code

* temp commit to test

* cosmetic updates

* dev review feedback

* updates

* fix build checks

* rm auto formatting changes

* review feeback: update order of script for setting up postgres on macos runner

* review feedback: add reasoning for not using secrets in workflow

* review feedback: rm unnecessary changes

* more review feedback

* test pull_request_target action

* fix path to cli tool

* split up lint and unit workflows for clear resposibilites

* rm `branches-ignore` filter from pull request trigger

* testing push event

* test

* try this again

* test actions in same repo

* nvm revert

* formatting

* fix windows build

* add concurrency

* fix build workflow

* test slim ci

* has changes

* set up postgres for other OS

* update descriptions

* turn off python3.9 unit tests

* add changelog

* clean up todo

* Update .github/workflows/main.yml

* create actions for common code

* cosmetic updates

* dev review feedback

* updates

* fix build checks

* rm auto formatting changes

* review feedback: add reasoning for not using secrets in workflow

* review feedback: rm unnecessary changes

* more review feedback

* test pull_request_target action

* fix path to cli tool

* split up lint and unit workflows for clear resposibilites

* rm `branches-ignore` filter from pull request trigger

* test dynamic matrix generation

* update label logic

* finishing touches

* align naming

* pass opts to pytest

* slim down push matrix, there are a lot of jobs

* test bump num of proc

* update matrix for all event triggers

* handle case when no changes require integration tests

* dev review feedback

* clean up and add branch name for testing

* Add test results publishing as artifact (#3794)

* Test failures file

* Add testing branch

* Adding upload steps

* Adding date to name

* Adding to integration

* Always upload artifacts

* Adding adapter type

* Always publish unit test results

* Adding comments

* rm unecessary env var

* fix changelog

* update job name

* clean up python deps

Co-authored-by: leahwicz <[email protected]>

Co-authored-by: leahwicz <[email protected]>

.github/actions/setup-postgres-linux/action.yml

@@ -0,0 +1,10 @@
+name: "Set up postgres (linux)"
+description: "Set up postgres service on linux vm for dbt integration tests"
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: |
+        sudo systemctl start postgresql.service
+        pg_isready
+        sudo -u postgres bash ${{ github.action_path }}/setup_db.sh

.github/actions/setup-postgres-linux/setup_db.sh

@@ -0,0 +1 @@
+../../../test/setup_db.sh

.github/actions/setup-postgres-macos/action.yml

@@ -0,0 +1,24 @@
+name: "Set up postgres (macos)"
+description: "Set up postgres service on macos vm for dbt integration tests"
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: |
+        brew services start postgresql
+        echo "Check PostgreSQL service is running"
+        i=10
+        COMMAND='pg_isready'
+        while [ $i -gt -1 ]; do
+            if [ $i == 0 ]; then
+                echo "PostgreSQL service not ready, all attempts exhausted"
+                exit 1
+            fi
+            echo "Check PostgreSQL service status"
+            eval $COMMAND && break
+            echo "PostgreSQL service not ready, wait 10 more sec, attempts left: $i"
+            sleep 10
+            ((i--))
+        done
+        createuser -s postgres
+        bash ${{ github.action_path }}/setup_db.sh

.github/actions/setup-postgres-macos/setup_db.sh

@@ -0,0 +1 @@
+../../../test/setup_db.sh

.github/actions/setup-postgres-windows/action.yml

@@ -0,0 +1,12 @@
+name: "Set up postgres (windows)"
+description: "Set up postgres service on windows vm for dbt integration tests"
+runs:
+  using: "composite"
+  steps:
+    - shell: pwsh
+      run: |
+        $pgService = Get-Service -Name postgresql*
+        Set-Service -InputObject $pgService -Status running -StartupType automatic
+        Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru
+        $env:Path += ";$env:PGBIN"
+        bash ${{ github.action_path }}/setup_db.sh

.github/actions/setup-postgres-windows/setup_db.sh

@@ -0,0 +1 @@
+../../../test/setup_db.sh

.github/pull_request_template.md

@@ -9,14 +9,13 @@ resolves #
     resolves #1234
 -->
 
-
 ### Description
 
 <!--- Describe the Pull Request here -->
 
-
 ### Checklist
- - [ ] I have signed the [CLA](https://docs.getdbt.com/docs/contributor-license-agreements)
- - [ ] I have run this code in development and it appears to resolve the stated issue
- - [ ] This PR includes tests, or tests are not required/relevant for this PR
- - [ ] I have updated the `CHANGELOG.md` and added information about my change to the "dbt next" section.
+
+- [ ] I have signed the [CLA](https://docs.getdbt.com/docs/contributor-license-agreements)
+- [ ] I have run this code in development and it appears to resolve the stated issue
+- [ ] This PR includes tests, or tests are not required/relevant for this PR
+- [ ] I have updated the `CHANGELOG.md` and added information about my change to the "dbt next" section.

.github/scripts/integration-test-matrix.js

@@ -0,0 +1,95 @@
+module.exports = ({ context }) => {
+  const defaultPythonVersion = "3.8";
+  const supportedPythonVersions = ["3.6", "3.7", "3.8", "3.9"];
+  const supportedAdapters = ["snowflake", "postgres", "bigquery", "redshift"];
+
+  // if PR, generate matrix based on files changed and PR labels
+  if (context.eventName.includes("pull_request")) {
+    // `changes` is a list of adapter names that have related
+    // file changes in the PR
+    // ex: ['postgres', 'snowflake']
+    const changes = JSON.parse(process.env.CHANGES);
+    const labels = context.payload.pull_request.labels.map(({ name }) => name);
+    console.log("labels", labels);
+    console.log("changes", changes);
+    const testAllLabel = labels.includes("test all");
+    const include = [];
+
+    for (const adapter of supportedAdapters) {
+      if (
+        changes.includes(adapter) ||
+        testAllLabel ||
+        labels.includes(`test ${adapter}`)
+      ) {
+        for (const pythonVersion of supportedPythonVersions) {
+          if (
+            pythonVersion === defaultPythonVersion ||
+            labels.includes(`test python${pythonVersion}`) ||
+            testAllLabel
+          ) {
+            // always run tests on ubuntu by default
+            include.push({
+              os: "ubuntu-latest",
+              adapter,
+              "python-version": pythonVersion,
+            });
+
+            if (labels.includes("test windows") || testAllLabel) {
+              include.push({
+                os: "windows-latest",
+                adapter,
+                "python-version": pythonVersion,
+              });
+            }
+
+            if (labels.includes("test macos") || testAllLabel) {
+              include.push({
+                os: "macos-latest",
+                adapter,
+                "python-version": pythonVersion,
+              });
+            }
+          }
+        }
+      }
+    }
+
+    console.log("matrix", { include });
+
+    return {
+      include,
+    };
+  }
+  // if not PR, generate matrix of python version, adapter, and operating
+  // system to run integration tests on
+
+  const include = [];
+  // run for all adapters and python versions on ubuntu
+  for (const adapter of supportedAdapters) {
+    for (const pythonVersion of supportedPythonVersions) {
+      include.push({
+        os: 'ubuntu-latest',
+        adapter: adapter,
+        "python-version": pythonVersion,
+      });
+    }
+  }
+
+  // additionally include runs for all adapters, on macos and windows,
+  // but only for the default python version
+  for (const adapter of supportedAdapters) {
+    for (const operatingSystem of ["windows-latest", "macos-latest"]) {
+      include.push({
+        os: operatingSystem,
+        adapter: adapter,
+        "python-version": defaultPythonVersion,
+      });
+    }
+  }
+
+  console.log("matrix", { include });
+
+  return {
+    include,
+  };
+};