From cf596b46839d84cf4f2d9dbe17b0609c0dc5cb0e Mon Sep 17 00:00:00 2001
From: Enno <458526+ennru@users.noreply.github.com>
Date: Sat, 29 Jan 2022 18:25:59 +0100
Subject: [PATCH 01/12] CI checks in GitHub actions

---
 .github/workflows/basic-checks.yml | 62 ++++++++++++++++++++++++++++++
 .travis.yml                        |  8 ----
 2 files changed, 62 insertions(+), 8 deletions(-)
 create mode 100644 .github/workflows/basic-checks.yml

diff --git a/.github/workflows/basic-checks.yml b/.github/workflows/basic-checks.yml
new file mode 100644
index 0000000000..bca3edc684
--- /dev/null
+++ b/.github/workflows/basic-checks.yml
@@ -0,0 +1,62 @@
+name: Basic Checks
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - main
+    tags-ignore:
+      - v*
+
+jobs:
+  check-code-style:
+    name: Compile, Code Style, Binary Compatibility
+    runs-on: ubuntu-20.04
+    env:
+      JAVA_OPTS: -Xms2G -Xmx2G -Xss2M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          # MiMa needs the latest tag, fetch the entire history to make `previousStableVersion` succeed
+          fetch-depth: 0
+
+      - name: Set up JDK 11
+        uses: olafurpg/setup-scala@v13
+        with:
+          java-version: adopt@1.11
+
+      - name: Cache Coursier cache
+        uses: coursier/cache-action@v6.3
+
+      - name: "Code style check and MiMa. Run locally with: sbt verifyCodeStyle; mimaReportBinaryIssues"
+        run: sbt "verifyCodeStyle; mimaReportBinaryIssues"
+
+      - name: "Compile all code with Scala 2.13 and fatal warnings enabled. Run locally with: env CI=true sbt Test/compile"
+        run: sbt "++2.13.8 Test/compile"
+
+  documentation:
+    name: ScalaDoc, Documentation with Paradox
+    runs-on: ubuntu-20.04
+    env:
+      JAVA_OPTS: -Xms2G -Xmx2G -Xss2M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          # MiMa needs the latest tag, fetch the entire history to make `previousStableVersion` succeed
+          fetch-depth: 0
+
+      - name: Set up JDK 11
+        uses: olafurpg/setup-scala@v13
+        with:
+          java-version: adopt@1.11
+
+      - name: Cache Coursier cache
+        uses: coursier/cache-action@v6.3
+
+      - name: "Create all API docs and create site with Paradox"
+        run: sbt "unidoc; docs/paradox"
diff --git a/.travis.yml b/.travis.yml
index 78ec8bd452..11c1d15a49 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -38,14 +38,6 @@ jobs:
         - rm -rf $HOME/.sbt
         - rm -rf $HOME/.jabba
       name: "drop-travis-caches"
-    - stage: check
-      env: CMD="verifyCodeStyle; mimaReportBinaryIssues"
-      name: "Code style check and MiMa. Run locally with: sbt verifyCodeStyle; mimaReportBinaryIssues"
-      if: type != cron
-    - env: CMD="++2.13.8 Test/compile"
-      name: "Compile all code with Scala 2.13 and fatal warnings enabled. Run locally with: env CI=true sbt ++2.13.8 Test/compile"
-    - env: CMD="unidoc; docs/paradox"
-      name: "Create all API docs and create site with Paradox"
 
     - stage: test
       name: amqp

From 5b9ce5adcbc105249d83f4e37cfea18189bbb712 Mon Sep 17 00:00:00 2001
From: Enno <458526+ennru@users.noreply.github.com>
Date: Sun, 30 Jan 2022 14:01:39 +0100
Subject: [PATCH 02/12] More shallow checkout

---
 .github/workflows/basic-checks.yml | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/basic-checks.yml b/.github/workflows/basic-checks.yml
index bca3edc684..de0aa0a503 100644
--- a/.github/workflows/basic-checks.yml
+++ b/.github/workflows/basic-checks.yml
@@ -10,7 +10,7 @@ on:
       - v*
 
 jobs:
-  check-code-style:
+  style-compile-mima:
     name: Compile, Code Style, Binary Compatibility
     runs-on: ubuntu-20.04
     env:
@@ -19,9 +19,11 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-        with:
-          # MiMa needs the latest tag, fetch the entire history to make `previousStableVersion` succeed
-          fetch-depth: 0
+        with: # https://github.com/olafurpg/setup-scala#faster-checkout-of-big-repos
+          fetch-depth: 100
+
+      - name: Fetch tags
+        run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
 
       - name: Set up JDK 11
         uses: olafurpg/setup-scala@v13
@@ -31,11 +33,8 @@ jobs:
       - name: Cache Coursier cache
         uses: coursier/cache-action@v6.3
 
-      - name: "Code style check and MiMa. Run locally with: sbt verifyCodeStyle; mimaReportBinaryIssues"
-        run: sbt "verifyCodeStyle; mimaReportBinaryIssues"
-
-      - name: "Compile all code with Scala 2.13 and fatal warnings enabled. Run locally with: env CI=true sbt Test/compile"
-        run: sbt "++2.13.8 Test/compile"
+      - name: "Code style, compile tests, MiMa. Run locally with: sbt ++2.13.8 verifyCodeStyle; Test/compile; mimaReportBinaryIssues"
+        run: sbt "++2.13.8 verifyCodeStyle; Test/compile; mimaReportBinaryIssues"
 
   documentation:
     name: ScalaDoc, Documentation with Paradox
@@ -46,9 +45,11 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-        with:
-          # MiMa needs the latest tag, fetch the entire history to make `previousStableVersion` succeed
-          fetch-depth: 0
+        with: # https://github.com/olafurpg/setup-scala#faster-checkout-of-big-repos
+          fetch-depth: 100
+
+      - name: Fetch tags
+        run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
 
       - name: Set up JDK 11
         uses: olafurpg/setup-scala@v13

From db451729e5177925bc5f29b261a5c99560228b96 Mon Sep 17 00:00:00 2001
From: Enno <458526+ennru@users.noreply.github.com>
Date: Sun, 30 Jan 2022 14:08:12 +0100
Subject: [PATCH 03/12] quoting

---
 .github/workflows/basic-checks.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/basic-checks.yml b/.github/workflows/basic-checks.yml
index de0aa0a503..cb62225c83 100644
--- a/.github/workflows/basic-checks.yml
+++ b/.github/workflows/basic-checks.yml
@@ -33,8 +33,8 @@ jobs:
       - name: Cache Coursier cache
         uses: coursier/cache-action@v6.3
 
-      - name: "Code style, compile tests, MiMa. Run locally with: sbt ++2.13.8 verifyCodeStyle; Test/compile; mimaReportBinaryIssues"
-        run: sbt "++2.13.8 verifyCodeStyle; Test/compile; mimaReportBinaryIssues"
+      - name: "Code style, compile tests, MiMa. Run locally with: sbt ++2.13.8 \"verifyCodeStyle; Test/compile; mimaReportBinaryIssues\""
+        run: sbt ++2.13.8 "verifyCodeStyle; Test/compile; mimaReportBinaryIssues"
 
   documentation:
     name: ScalaDoc, Documentation with Paradox

From 3849bf4a04d2f6789b333ec7eff6db858664e8d0 Mon Sep 17 00:00:00 2001
From: Enno <458526+ennru@users.noreply.github.com>
Date: Sun, 30 Jan 2022 14:42:21 +0100
Subject: [PATCH 04/12] Matrix for connectors?

---
 .github/workflows/basic-checks.yml |  4 +--
 .github/workflows/connectors.yml   | 54 ++++++++++++++++++++++++++++++
 scripts/gh-test-if-changed.sh      | 37 ++++++++++++++++++++
 3 files changed, 93 insertions(+), 2 deletions(-)
 create mode 100644 .github/workflows/connectors.yml
 create mode 100755 scripts/gh-test-if-changed.sh

diff --git a/.github/workflows/basic-checks.yml b/.github/workflows/basic-checks.yml
index cb62225c83..2d81deab1a 100644
--- a/.github/workflows/basic-checks.yml
+++ b/.github/workflows/basic-checks.yml
@@ -33,8 +33,8 @@ jobs:
       - name: Cache Coursier cache
         uses: coursier/cache-action@v6.3
 
-      - name: "Code style, compile tests, MiMa. Run locally with: sbt ++2.13.8 \"verifyCodeStyle; Test/compile; mimaReportBinaryIssues\""
-        run: sbt ++2.13.8 "verifyCodeStyle; Test/compile; mimaReportBinaryIssues"
+      - name: "Code style, compile tests, MiMa. Run locally with: sbt +~2.13 \"verifyCodeStyle; Test/compile; mimaReportBinaryIssues\""
+        run: sbt +~2.13 "verifyCodeStyle; Test/compile; mimaReportBinaryIssues"
 
   documentation:
     name: ScalaDoc, Documentation with Paradox
diff --git a/.github/workflows/connectors.yml b/.github/workflows/connectors.yml
new file mode 100644
index 0000000000..c7650f0476
--- /dev/null
+++ b/.github/workflows/connectors.yml
@@ -0,0 +1,54 @@
+name: Build and test connectors
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - main
+    tags-ignore:
+      - v*
+
+concurrency:
+  # Only run once for latest commit per ref and cancel other (previous) runs.
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  GH_PULL_REQUEST: ${{github.event.pull_request}}
+
+jobs:
+  connector:
+    name: Alpakka AMQP
+    runs-on: ubuntu-20.04
+
+    strategy:
+      matrix:
+        include:
+          - { connector: amqp,       pre_cmd: 'docker-compose up -d amqp' }
+          - { connector: avroparquet }
+
+    env:
+      JAVA_OPTS: -Xms2G -Xmx2G -Xss2M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          # MiMa needs the latest tag, fetch the entire history to make `previousStableVersion` succeed
+          fetch-depth: 0
+
+      - name: Set up JDK 11
+        uses: olafurpg/setup-scala@v13
+        with:
+          java-version: adopt@1.8
+
+      - name: Cache Coursier cache
+        uses: coursier/cache-action@v6.3
+
+      - name: "amqp"
+        env:
+          CONNECTOR: ${{ matrix.connector }}
+          PRE_CMD: ${{ matrix.pre_cmd }}
+        run: |-
+          ./scripts/gh-test-if-changed.sh "${CONNECTOR}" "${PRE_CMD:=echo NOOP}" "+${CONNECTOR}/testChanged" 
diff --git a/scripts/gh-test-if-changed.sh b/scripts/gh-test-if-changed.sh
new file mode 100755
index 0000000000..d4755c9146
--- /dev/null
+++ b/scripts/gh-test-if-changed.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -x
+
+DIR=$1
+PRE_CMD=$2
+CMD=$3
+
+if [ "$GITHUB_EVENT_NAME" == "pull_request" ]
+then
+  COMPARE_TO="origin/master"
+else
+  # for non-pr (master) builds, comparison for changes is made against the previous commit,
+  # which might be:
+  # * merge commit, then HEAD^ is the last commit before branching out
+  # * squash commit, then HEAD^ is the previous state of master
+  COMPARE_TO="HEAD^"
+fi
+
+git diff "$COMPARE_TO" --exit-code --quiet "$DIR" build.sbt project/ .github/workflows/
+DIFF_EXIT_CODE=$?
+
+if [ "$GITHUB_EVENT_NAME" == "schedule" ]
+then
+  echo "Building everything because nightly"
+elif [ "$DIFF_EXIT_CODE" -eq 1 ]
+then
+  echo "Changes in ${DIR}"
+else
+  echo "No changes in $DIR"
+  exit 0
+fi
+
+CURR_DIR=$(dirname "$(readlink -f "$0")")
+
+$PRE_CMD
+sbt "$CMD"

From 225acc1fe8e136d9a2723b8a90c0c4ac6ad8d377 Mon Sep 17 00:00:00 2001
From: Enno <458526+ennru@users.noreply.github.com>
Date: Sun, 30 Jan 2022 14:44:53 +0100
Subject: [PATCH 05/12] Remove env

---
 .github/workflows/connectors.yml | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/connectors.yml b/.github/workflows/connectors.yml
index c7650f0476..b73704e0cb 100644
--- a/.github/workflows/connectors.yml
+++ b/.github/workflows/connectors.yml
@@ -14,12 +14,9 @@ concurrency:
   group: ci-${{ github.ref }}
   cancel-in-progress: true
 
-env:
-  GH_PULL_REQUEST: ${{github.event.pull_request}}
-
 jobs:
   connector:
-    name: Alpakka AMQP
+    name: Alpakka
     runs-on: ubuntu-20.04
 
     strategy:
@@ -34,9 +31,11 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-        with:
-          # MiMa needs the latest tag, fetch the entire history to make `previousStableVersion` succeed
-          fetch-depth: 0
+        with: # https://github.com/olafurpg/setup-scala#faster-checkout-of-big-repos
+          fetch-depth: 100
+
+      - name: Fetch tags
+        run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
 
       - name: Set up JDK 11
         uses: olafurpg/setup-scala@v13
@@ -46,7 +45,7 @@ jobs:
       - name: Cache Coursier cache
         uses: coursier/cache-action@v6.3
 
-      - name: "amqp"
+      - name: connector
         env:
           CONNECTOR: ${{ matrix.connector }}
           PRE_CMD: ${{ matrix.pre_cmd }}

From e78bdd12d59d6cdbe576b74ef86611d311f6e796 Mon Sep 17 00:00:00 2001
From: Enno <458526+ennru@users.noreply.github.com>
Date: Sun, 30 Jan 2022 14:51:00 +0100
Subject: [PATCH 06/12] fetch whole repo to detect changes from master

---
 .github/workflows/connectors.yml | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/connectors.yml b/.github/workflows/connectors.yml
index b73704e0cb..c794252d65 100644
--- a/.github/workflows/connectors.yml
+++ b/.github/workflows/connectors.yml
@@ -1,4 +1,4 @@
-name: Build and test connectors
+name: Connector
 
 on:
   pull_request:
@@ -16,7 +16,6 @@ concurrency:
 
 jobs:
   connector:
-    name: Alpakka
     runs-on: ubuntu-20.04
 
     strategy:
@@ -31,16 +30,13 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-        with: # https://github.com/olafurpg/setup-scala#faster-checkout-of-big-repos
-          fetch-depth: 100
-
-      - name: Fetch tags
-        run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
+        with:
+          fetch-depth: 0
 
-      - name: Set up JDK 11
+      - name: Set up JDK 8
         uses: olafurpg/setup-scala@v13
         with:
-          java-version: adopt@1.8
+          java-version: adopt@1.8.0-292
 
       - name: Cache Coursier cache
         uses: coursier/cache-action@v6.3
@@ -50,4 +46,4 @@ jobs:
           CONNECTOR: ${{ matrix.connector }}
           PRE_CMD: ${{ matrix.pre_cmd }}
         run: |-
-          ./scripts/gh-test-if-changed.sh "${CONNECTOR}" "${PRE_CMD:=echo NOOP}" "+${CONNECTOR}/testChanged" 
+          ./scripts/gh-test-if-changed.sh "${CONNECTOR}" "${PRE_CMD:=echo NOOP}" "+${CONNECTOR}/test" 

From 6e1e16f0eeb74f0e39aad64279bfd285434244c6 Mon Sep 17 00:00:00 2001
From: Enno <458526+ennru@users.noreply.github.com>
Date: Sun, 30 Jan 2022 15:11:41 +0100
Subject: [PATCH 07/12] List all connectors

---
 .github/workflows/connectors.yml | 56 +++++++++++++++++++++++++++++---
 1 file changed, 52 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/connectors.yml b/.github/workflows/connectors.yml
index c794252d65..9601b6685c 100644
--- a/.github/workflows/connectors.yml
+++ b/.github/workflows/connectors.yml
@@ -1,4 +1,4 @@
-name: Connector
+name: Connectors
 
 on:
   pull_request:
@@ -21,8 +21,56 @@ jobs:
     strategy:
       matrix:
         include:
-          - { connector: amqp,       pre_cmd: 'docker-compose up -d amqp' }
+          - { connector: amqp,                         pre_cmd: 'docker-compose up -d amqp' }
           - { connector: avroparquet }
+          - { connector: awslambda }
+          - { connector: aws-event-bridge,             pre_cmd: 'docker-compose up -d amazoneventbridge' }
+          - { connector: azure-storage-queue }
+          - { connector: cassandra,                    pre_cmd: 'docker-compose up -d cassandra' }
+          - { connector: couchbase,                    pre_cmd: 'docker-compose up -d couchbase_prep' }
+          - { connector: csv }
+          - { connector: dynamodb,                     pre_cmd: 'docker-compose up -d dynamodb' }
+          - { connector: elasticsearch,                pre_cmd: 'docker-compose up -d elasticsearch6 elasticsearch7' }
+          - { connector: file }
+          - { connector: ftp,                          pre_cmd: 'docker-compose up -d ftp sftp squid' }
+          - { connector: geode,                        pre_cmd: 'docker-compose up -d geode' }
+          - { connector: google-cloud-bigquery }
+          - { connector: google-cloud-bigquery-storage }
+          - { connector: google-cloud-pub-sub,         pre_cmd: 'docker-compose up -d gcloud-pubsub-emulator_prep' }
+          - { connector: google-cloud-pub-sub-grpc,    pre_cmd: 'docker-compose up -d gcloud-pubsub-emulator_prep' }
+          - { connector: google-cloud-storage }
+          - { connector: google-common }
+          - { connector: google-fcm }
+#      # hbase disabled until we resolve why new docker image fails our build: https://github.com/akka/alpakka/issues/2185
+#      #    - name: hbase
+#      #      env:
+#      #      - PRE_CMD="docker-compose up -d hbase"
+          - { connector: hdfs,                         pre_cmd: 'file ${HOME}/.cache/coursier/v1/https/repo1.maven.org/maven2/org/typelevel/cats-kernel_2.13/2.0.0/cats-kernel_2.13-2.0.0.jar' }
+          - { connector:  huawei-push-kit }
+          - { connector: influxdb,                     pre_cmd: 'docker-compose up -d influxdb' }
+          - { connector: ironmq,                       pre_cmd: 'docker-compose up -d ironauth ironmq' }
+          - { connector: jms,                          pre_cmd: 'docker-compose up -d ibmmq' }
+          - { connector: json-streaming }
+          - { connector: kinesis }
+          - { connector: kudu,                         pre_cmd: 'docker-compose up -d kudu-master-data kudu-tserver-data kudu-master kudu-tserver' }
+          - { connector: mongodb,                      pre_cmd: 'docker-compose up -d mongo' }
+          - { connector: mqtt,                         pre_cmd: 'docker-compose up -d mqtt' }
+          - { connector: mqtt-streaming,               pre_cmd: 'docker-compose up -d mqtt' }
+          - { connector: orientdb,                     pre_cmd: 'docker-compose up -d orientdb' }
+          - { connector: pravega,                      pre_cmd: 'docker-compose up -d pravega'}
+          - { connector: reference }
+          - { connector: s3,                           pre_cmd: 'docker-compose up -d minio_prep' }
+          - { connector: spring-web }
+          - { connector: simple-codecs }
+          - { connector: slick }
+          - { connector: sns,                          pre_cmd: 'docker-compose up -d amazonsns' }
+          - { connector: solr }
+          - { connector: sqs,                          pre_cmd: 'docker-compose up -d elasticmq' }
+          - { connector: sse }
+          - { connector: text }
+          - { connector: udp }
+          - { connector: unix-domain-socket }
+          - { connector: xml }
 
     env:
       JAVA_OPTS: -Xms2G -Xmx2G -Xss2M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
@@ -30,7 +78,7 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-        with:
+        with: # test-if-changed compares with master branch
           fetch-depth: 0
 
       - name: Set up JDK 8
@@ -41,7 +89,7 @@ jobs:
       - name: Cache Coursier cache
         uses: coursier/cache-action@v6.3
 
-      - name: connector
+      - name: ${{ matrix.connector }}
         env:
           CONNECTOR: ${{ matrix.connector }}
           PRE_CMD: ${{ matrix.pre_cmd }}

From 2a31c25fecf9ef095195020749c8573fb589c233 Mon Sep 17 00:00:00 2001
From: Enno <458526+ennru@users.noreply.github.com>
Date: Sun, 30 Jan 2022 15:25:05 +0100
Subject: [PATCH 08/12] fail slow

---
 .github/workflows/connectors.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/connectors.yml b/.github/workflows/connectors.yml
index 9601b6685c..39b137aecb 100644
--- a/.github/workflows/connectors.yml
+++ b/.github/workflows/connectors.yml
@@ -19,6 +19,7 @@ jobs:
     runs-on: ubuntu-20.04
 
     strategy:
+      fail-fast: false
       matrix:
         include:
           - { connector: amqp,                         pre_cmd: 'docker-compose up -d amqp' }
@@ -41,10 +42,8 @@ jobs:
           - { connector: google-cloud-storage }
           - { connector: google-common }
           - { connector: google-fcm }
-#      # hbase disabled until we resolve why new docker image fails our build: https://github.com/akka/alpakka/issues/2185
-#      #    - name: hbase
-#      #      env:
-#      #      - PRE_CMD="docker-compose up -d hbase"
+          # hbase disabled until we resolve why new docker image fails our build: https://github.com/akka/alpakka/issues/2185
+          - { connector: hbase,                        pre_cmd: 'docker-compose up -d hbase' }
           - { connector: hdfs,                         pre_cmd: 'file ${HOME}/.cache/coursier/v1/https/repo1.maven.org/maven2/org/typelevel/cats-kernel_2.13/2.0.0/cats-kernel_2.13-2.0.0.jar' }
           - { connector:  huawei-push-kit }
           - { connector: influxdb,                     pre_cmd: 'docker-compose up -d influxdb' }
@@ -94,4 +93,4 @@ jobs:
           CONNECTOR: ${{ matrix.connector }}
           PRE_CMD: ${{ matrix.pre_cmd }}
         run: |-
-          ./scripts/gh-test-if-changed.sh "${CONNECTOR}" "${PRE_CMD:=echo NOOP}" "+${CONNECTOR}/test" 
+          ./scripts/gh-test-if-changed.sh "${CONNECTOR}" "${PRE_CMD:=echo NOOP}" "${CONNECTOR}/test"

From e6a4276658c1f45061cbfb50c5673e730d472842 Mon Sep 17 00:00:00 2001
From: Enno <458526+ennru@users.noreply.github.com>
Date: Mon, 31 Jan 2022 09:26:16 +0100
Subject: [PATCH 09/12] collapse into single file

---
 .github/workflows/basic-checks.yml            |  63 ------
 .../{connectors.yml => check-build-test.yml}  |  63 +++++-
 .travis.yml                                   | 196 +++++++++---------
 3 files changed, 158 insertions(+), 164 deletions(-)
 delete mode 100644 .github/workflows/basic-checks.yml
 rename .github/workflows/{connectors.yml => check-build-test.yml} (70%)

diff --git a/.github/workflows/basic-checks.yml b/.github/workflows/basic-checks.yml
deleted file mode 100644
index 2d81deab1a..0000000000
--- a/.github/workflows/basic-checks.yml
+++ /dev/null
@@ -1,63 +0,0 @@
-name: Basic Checks
-
-on:
-  pull_request:
-  push:
-    branches:
-      - master
-      - main
-    tags-ignore:
-      - v*
-
-jobs:
-  style-compile-mima:
-    name: Compile, Code Style, Binary Compatibility
-    runs-on: ubuntu-20.04
-    env:
-      JAVA_OPTS: -Xms2G -Xmx2G -Xss2M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with: # https://github.com/olafurpg/setup-scala#faster-checkout-of-big-repos
-          fetch-depth: 100
-
-      - name: Fetch tags
-        run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
-
-      - name: Set up JDK 11
-        uses: olafurpg/setup-scala@v13
-        with:
-          java-version: adopt@1.11
-
-      - name: Cache Coursier cache
-        uses: coursier/cache-action@v6.3
-
-      - name: "Code style, compile tests, MiMa. Run locally with: sbt +~2.13 \"verifyCodeStyle; Test/compile; mimaReportBinaryIssues\""
-        run: sbt +~2.13 "verifyCodeStyle; Test/compile; mimaReportBinaryIssues"
-
-  documentation:
-    name: ScalaDoc, Documentation with Paradox
-    runs-on: ubuntu-20.04
-    env:
-      JAVA_OPTS: -Xms2G -Xmx2G -Xss2M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with: # https://github.com/olafurpg/setup-scala#faster-checkout-of-big-repos
-          fetch-depth: 100
-
-      - name: Fetch tags
-        run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
-
-      - name: Set up JDK 11
-        uses: olafurpg/setup-scala@v13
-        with:
-          java-version: adopt@1.11
-
-      - name: Cache Coursier cache
-        uses: coursier/cache-action@v6.3
-
-      - name: "Create all API docs and create site with Paradox"
-        run: sbt "unidoc; docs/paradox"
diff --git a/.github/workflows/connectors.yml b/.github/workflows/check-build-test.yml
similarity index 70%
rename from .github/workflows/connectors.yml
rename to .github/workflows/check-build-test.yml
index 39b137aecb..3574c97301 100644
--- a/.github/workflows/connectors.yml
+++ b/.github/workflows/check-build-test.yml
@@ -1,4 +1,4 @@
-name: Connectors
+name: CI
 
 on:
   pull_request:
@@ -15,7 +15,60 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  connector:
+  style-compile-mima:
+    name: Compile, Code Style, Binary Compatibility
+    runs-on: ubuntu-20.04
+    env:
+      JAVA_OPTS: -Xms2G -Xmx2G -Xss2M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with: # https://github.com/olafurpg/setup-scala#faster-checkout-of-big-repos
+          fetch-depth: 100
+
+      - name: Fetch tags
+        run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
+
+      - name: Set up JDK 11
+        uses: olafurpg/setup-scala@v13
+        with:
+          java-version: adopt@1.8
+
+      - name: Cache Coursier cache
+        uses: coursier/cache-action@v6.3
+
+      - name: "Code style, compile tests, MiMa. Run locally with: sbt +~2.13 \"verifyCodeStyle; Test/compile; mimaReportBinaryIssues\""
+        run: sbt +~2.13 "verifyCodeStyle; Test/compile; mimaReportBinaryIssues"
+
+  documentation:
+    name: ScalaDoc, Documentation with Paradox
+    runs-on: ubuntu-20.04
+    env:
+      JAVA_OPTS: -Xms2G -Xmx2G -Xss2M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with: # https://github.com/olafurpg/setup-scala#faster-checkout-of-big-repos
+          fetch-depth: 100
+
+      - name: Fetch tags
+        run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
+
+      - name: Set up JDK 11
+        uses: olafurpg/setup-scala@v13
+        with:
+          java-version: adopt@1.11
+
+      - name: Cache Coursier cache
+        uses: coursier/cache-action@v6.3
+
+      - name: "Create all API docs and create site with Paradox"
+        run: sbt "unidoc; docs/paradox"
+
+  connectors:
+    needs: [style-compile-mima]
     runs-on: ubuntu-20.04
 
     strategy:
@@ -43,7 +96,7 @@ jobs:
           - { connector: google-common }
           - { connector: google-fcm }
           # hbase disabled until we resolve why new docker image fails our build: https://github.com/akka/alpakka/issues/2185
-          - { connector: hbase,                        pre_cmd: 'docker-compose up -d hbase' }
+          # - { connector: hbase,                        pre_cmd: 'docker-compose up -d hbase' }
           - { connector: hdfs,                         pre_cmd: 'file ${HOME}/.cache/coursier/v1/https/repo1.maven.org/maven2/org/typelevel/cats-kernel_2.13/2.0.0/cats-kernel_2.13-2.0.0.jar' }
           - { connector:  huawei-push-kit }
           - { connector: influxdb,                     pre_cmd: 'docker-compose up -d influxdb' }
@@ -94,3 +147,7 @@ jobs:
           PRE_CMD: ${{ matrix.pre_cmd }}
         run: |-
           ./scripts/gh-test-if-changed.sh "${CONNECTOR}" "${PRE_CMD:=echo NOOP}" "${CONNECTOR}/test"
+
+      - name: Print logs on failure
+        if: ${{ failure() }}
+        run: find . -name "*.log" -exec ./scripts/cat-log.sh {} \;
diff --git a/.travis.yml b/.travis.yml
index 11c1d15a49..b06fed4a6c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -39,104 +39,104 @@ jobs:
         - rm -rf $HOME/.jabba
       name: "drop-travis-caches"
 
-    - stage: test
-      name: amqp
-      env:
-      - PRE_CMD="docker-compose up -d amqp"
-    - name: avroparquet
-    - name: awslambda
-    - name: aws-event-bridge
-      env:
-      - PRE_CMD="docker-compose up -d amazoneventbridge"
-    - name: azure-storage-queue
-    - name: cassandra
-      env:
-      - PRE_CMD="docker-compose up -d cassandra"
-    - name: couchbase
-      env:
-      - PRE_CMD="docker-compose up -d couchbase_prep"
-    - name: csv
-    - name: dynamodb
-      env:
-      - PRE_CMD="docker-compose up -d dynamodb"
-    - name: elasticsearch
-      env:
-      - PRE_CMD="docker-compose up -d elasticsearch6 elasticsearch7"
-    - name: file
-    - name: ftp
-      env:
-      - PRE_CMD="docker-compose up -d ftp sftp squid"
-    - name: geode
-      env:
-      - PRE_CMD="docker-compose up -d geode"
-    - name: google-cloud-bigquery
-    - name: google-cloud-bigquery-storage
-    - name: google-cloud-pub-sub
-      env:
-        - PRE_CMD="docker-compose up -d gcloud-pubsub-emulator_prep"
-    - name: google-cloud-pub-sub-grpc
-      env:
-      - PRE_CMD="docker-compose up -d gcloud-pubsub-emulator_prep"
-    - name: google-cloud-storage
-    - name: google-common
-    - name: google-fcm
-# hbase disabled until we resolve why new docker image fails our build: https://github.com/akka/alpakka/issues/2185
-#    - name: hbase
-#      env:
-#      - PRE_CMD="docker-compose up -d hbase"
-    - name: hdfs
-      env:
-      - PRE_CMD="file /home/travis/.cache/coursier/v1/https/repo1.maven.org/maven2/org/typelevel/cats-kernel_2.13/2.0.0/cats-kernel_2.13-2.0.0.jar"
-    - name: huawei-push-kit
-    - name: influxdb
-      env:
-      - PRE_CMD="docker-compose up -d influxdb"
-    - name: ironmq
-      env:
-      - PRE_CMD="docker-compose up -d ironauth ironmq"
-    - name: jms
-      env:
-      - PRE_CMD="docker-compose up -d ibmmq"
-    - name: json-streaming
-    - name: kinesis
-    - name: kudu
-      env:
-      - PRE_CMD="docker-compose up -d kudu-master-data kudu-tserver-data kudu-master kudu-tserver"
-    - name: mongodb
-      env:
-      - PRE_CMD="docker-compose up -d mongo"
-    - name: mqtt
-      env:
-      - PRE_CMD="docker-compose up -d mqtt"
-    - name: mqtt-streaming
-      env:
-      - PRE_CMD="docker-compose up -d mqtt"
-    - name: orientdb
-      env:
-      - PRE_CMD="docker-compose up -d orientdb"
-    - name: pravega
-      env:
-        - PRE_CMD="docker-compose up -d pravega"
-    - name: reference
-    - name: s3
-      env:
-      - PRE_CMD="docker-compose up -d minio_prep"
-    - name: spring-web
-    - name: simple-codecs
-    - name: slick
-    - name: sns
-      env:
-      - PRE_CMD="docker-compose up -d amazonsns"
-    - name: solr
-    - name: sqs
-      env:
-      - PRE_CMD="docker-compose up -d elasticmq"
-    - name: sse
-    - name: text
-    - name: udp
-    - name: unix-domain-socket
-    - name: xml
-
+#    - stage: test
+#      name: amqp
+#      env:
+#      - PRE_CMD="docker-compose up -d amqp"
+#    - name: avroparquet
+#    - name: awslambda
+#    - name: aws-event-bridge
+#      env:
+#      - PRE_CMD="docker-compose up -d amazoneventbridge"
+#    - name: azure-storage-queue
+#    - name: cassandra
+#      env:
+#      - PRE_CMD="docker-compose up -d cassandra"
+#    - name: couchbase
+#      env:
+#      - PRE_CMD="docker-compose up -d couchbase_prep"
+#    - name: csv
+#    - name: dynamodb
+#      env:
+#      - PRE_CMD="docker-compose up -d dynamodb"
+#    - name: elasticsearch
+#      env:
+#      - PRE_CMD="docker-compose up -d elasticsearch6 elasticsearch7"
+#    - name: file
+#    - name: ftp
+#      env:
+#      - PRE_CMD="docker-compose up -d ftp sftp squid"
+#    - name: geode
+#      env:
+#      - PRE_CMD="docker-compose up -d geode"
+#    - name: google-cloud-bigquery
+#    - name: google-cloud-bigquery-storage
+#    - name: google-cloud-pub-sub
+#      env:
+#        - PRE_CMD="docker-compose up -d gcloud-pubsub-emulator_prep"
+#    - name: google-cloud-pub-sub-grpc
+#      env:
+#      - PRE_CMD="docker-compose up -d gcloud-pubsub-emulator_prep"
+#    - name: google-cloud-storage
+#    - name: google-common
+#    - name: google-fcm
+## hbase disabled until we resolve why new docker image fails our build: https://github.com/akka/alpakka/issues/2185
+##    - name: hbase
+##      env:
+##      - PRE_CMD="docker-compose up -d hbase"
+#    - name: hdfs
+#      env:
+#      - PRE_CMD="file /home/travis/.cache/coursier/v1/https/repo1.maven.org/maven2/org/typelevel/cats-kernel_2.13/2.0.0/cats-kernel_2.13-2.0.0.jar"
+#    - name: huawei-push-kit
+#    - name: influxdb
+#      env:
+#      - PRE_CMD="docker-compose up -d influxdb"
+#    - name: ironmq
+#      env:
+#      - PRE_CMD="docker-compose up -d ironauth ironmq"
+#    - name: jms
+#      env:
+#      - PRE_CMD="docker-compose up -d ibmmq"
+#    - name: json-streaming
+#    - name: kinesis
+#    - name: kudu
+#      env:
+#      - PRE_CMD="docker-compose up -d kudu-master-data kudu-tserver-data kudu-master kudu-tserver"
+#    - name: mongodb
+#      env:
+#      - PRE_CMD="docker-compose up -d mongo"
+#    - name: mqtt
+#      env:
+#      - PRE_CMD="docker-compose up -d mqtt"
+#    - name: mqtt-streaming
+#      env:
+#      - PRE_CMD="docker-compose up -d mqtt"
+#    - name: orientdb
+#      env:
+#      - PRE_CMD="docker-compose up -d orientdb"
+#    - name: pravega
+#      env:
+#        - PRE_CMD="docker-compose up -d pravega"
+#    - name: reference
+#    - name: s3
+#      env:
+#      - PRE_CMD="docker-compose up -d minio_prep"
+#    - name: spring-web
+#    - name: simple-codecs
+#    - name: slick
+#    - name: sns
+#      env:
+#      - PRE_CMD="docker-compose up -d amazonsns"
+#    - name: solr
+#    - name: sqs
+#      env:
+#      - PRE_CMD="docker-compose up -d elasticmq"
+#    - name: sse
+#    - name: text
+#    - name: udp
+#    - name: unix-domain-socket
+#    - name: xml
+#
     - stage: licenses
       script: echo "License checking is temporarily disabled"
 

From e43fbd05d8c68fe5371f5b5da946e45adc0c6eac Mon Sep 17 00:00:00 2001
From: Enno <458526+ennru@users.noreply.github.com>
Date: Mon, 31 Jan 2022 16:46:15 +0100
Subject: [PATCH 10/12] use coursier/setup-action; validate links

---
 .github/workflows/check-build-test.yml | 19 +++++++++++--------
 .github/workflows/link-validator.yml   |  1 -
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/check-build-test.yml b/.github/workflows/check-build-test.yml
index 3574c97301..55a4fac733 100644
--- a/.github/workflows/check-build-test.yml
+++ b/.github/workflows/check-build-test.yml
@@ -30,10 +30,10 @@ jobs:
       - name: Fetch tags
         run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
 
-      - name: Set up JDK 11
-        uses: olafurpg/setup-scala@v13
+      - name: Set up JDK 8
+        uses: coursier/setup-action@v1
         with:
-          java-version: adopt@1.8
+          jvm: adopt:8
 
       - name: Cache Coursier cache
         uses: coursier/cache-action@v6.3
@@ -57,15 +57,18 @@ jobs:
         run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
 
       - name: Set up JDK 11
-        uses: olafurpg/setup-scala@v13
+        uses: coursier/setup-action@v1
         with:
-          java-version: adopt@1.11
+          jvm: adopt:11
 
       - name: Cache Coursier cache
         uses: coursier/cache-action@v6.3
 
       - name: "Create all API docs and create site with Paradox"
-        run: sbt "unidoc; docs/paradox"
+        run: sbt docs/makeSite
+
+      - name: Run Link Validator
+        run: cs launch net.runne::site-link-validator:0.2.2 -- scripts/link-validator.conf
 
   connectors:
     needs: [style-compile-mima]
@@ -134,9 +137,9 @@ jobs:
           fetch-depth: 0
 
       - name: Set up JDK 8
-        uses: olafurpg/setup-scala@v13
+        uses: coursier/setup-action@v1
         with:
-          java-version: adopt@1.8.0-292
+          jvm: adopt:8
 
       - name: Cache Coursier cache
         uses: coursier/cache-action@v6.3
diff --git a/.github/workflows/link-validator.yml b/.github/workflows/link-validator.yml
index f0e2761d2a..53850a80bb 100644
--- a/.github/workflows/link-validator.yml
+++ b/.github/workflows/link-validator.yml
@@ -1,7 +1,6 @@
 name: Link Validator
 
 on:
-  pull_request:
   schedule:
     - cron:  '0 6 * * 1'
 

From 2d437fbc1d7f6825cc22471c545c35bca3597b14 Mon Sep 17 00:00:00 2001
From: Enno <458526+ennru@users.noreply.github.com>
Date: Tue, 1 Feb 2022 11:34:52 +0100
Subject: [PATCH 11/12] Coursier setup in link-validator

---
 .github/workflows/link-validator.yml | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/link-validator.yml b/.github/workflows/link-validator.yml
index 53850a80bb..8537f8d016 100644
--- a/.github/workflows/link-validator.yml
+++ b/.github/workflows/link-validator.yml
@@ -10,20 +10,11 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-        with:
-          # we don't know what commit the last tag was it's safer to get entire repo so previousStableVersion resolves
-          fetch-depth: 0
-
-      - name: Checkout GitHub merge
-        if: github.event.pull_request
-        run: |-
-          git fetch origin pull/${{ github.event.pull_request.number }}/merge:scratch
-          git checkout scratch
 
       - name: Set up JDK 11
-        uses: olafurpg/setup-scala@v10
+        uses: coursier/setup-action@v1
         with:
-          java-version: adopt@1.11.0-9
+          jvm: adopt:11
 
       - name: Cache Coursier cache
         uses: coursier/cache-action@v5
@@ -31,8 +22,5 @@ jobs:
       - name: sbt site
         run: sbt docs/makeSite
 
-      - name: Install Coursier command line tool
-        run: curl -fLo cs https://git.io/coursier-cli-linux && chmod +x cs && ./cs
-
       - name: Run Link Validator
-        run: ./cs launch net.runne::site-link-validator:0.2.2 -- scripts/link-validator.conf
+        run: cs launch net.runne::site-link-validator:0.2.2 -- scripts/link-validator.conf

From 545a3855708da6d72ec3e7a9fa40403db7ec8dad Mon Sep 17 00:00:00 2001
From: Enno <458526+ennru@users.noreply.github.com>
Date: Wed, 2 Feb 2022 14:28:01 +0100
Subject: [PATCH 12/12] pin setup-action to 1.1.2

---
 .github/workflows/check-build-test.yml | 9 +++++----
 .github/workflows/link-validator.yml   | 2 +-
 .travis.yml                            | 8 ++++----
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/check-build-test.yml b/.github/workflows/check-build-test.yml
index 55a4fac733..ae6851e960 100644
--- a/.github/workflows/check-build-test.yml
+++ b/.github/workflows/check-build-test.yml
@@ -31,7 +31,7 @@ jobs:
         run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
 
       - name: Set up JDK 8
-        uses: coursier/setup-action@v1
+        uses: coursier/setup-action@v1.1.2
         with:
           jvm: adopt:8
 
@@ -57,7 +57,7 @@ jobs:
         run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
 
       - name: Set up JDK 11
-        uses: coursier/setup-action@v1
+        uses: coursier/setup-action@v1.1.2
         with:
           jvm: adopt:11
 
@@ -89,7 +89,8 @@ jobs:
           - { connector: dynamodb,                     pre_cmd: 'docker-compose up -d dynamodb' }
           - { connector: elasticsearch,                pre_cmd: 'docker-compose up -d elasticsearch6 elasticsearch7' }
           - { connector: file }
-          - { connector: ftp,                          pre_cmd: 'docker-compose up -d ftp sftp squid' }
+          # ftp tests fail consistently, needs work in a separate PR
+#          - { connector: ftp,                          pre_cmd: 'docker-compose up -d ftp sftp squid' }
           - { connector: geode,                        pre_cmd: 'docker-compose up -d geode' }
           - { connector: google-cloud-bigquery }
           - { connector: google-cloud-bigquery-storage }
@@ -137,7 +138,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up JDK 8
-        uses: coursier/setup-action@v1
+        uses: coursier/setup-action@v1.1.2
         with:
           jvm: adopt:8
 
diff --git a/.github/workflows/link-validator.yml b/.github/workflows/link-validator.yml
index 8537f8d016..d434d604b3 100644
--- a/.github/workflows/link-validator.yml
+++ b/.github/workflows/link-validator.yml
@@ -12,7 +12,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Set up JDK 11
-        uses: coursier/setup-action@v1
+        uses: coursier/setup-action@v1.1.2
         with:
           jvm: adopt:11
 
diff --git a/.travis.yml b/.travis.yml
index b06fed4a6c..4bc4ff2cd6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -39,7 +39,7 @@ jobs:
         - rm -rf $HOME/.jabba
       name: "drop-travis-caches"
 
-#    - stage: test
+    - stage: test
 #      name: amqp
 #      env:
 #      - PRE_CMD="docker-compose up -d amqp"
@@ -63,9 +63,9 @@ jobs:
 #      env:
 #      - PRE_CMD="docker-compose up -d elasticsearch6 elasticsearch7"
 #    - name: file
-#    - name: ftp
-#      env:
-#      - PRE_CMD="docker-compose up -d ftp sftp squid"
+    - name: ftp
+      env:
+      - PRE_CMD="docker-compose up -d ftp sftp squid"
 #    - name: geode
 #      env:
 #      - PRE_CMD="docker-compose up -d geode"