fix(ingestion): workaround for Python 3.8/3.9 mypy invalid syntax iss…

…ue with airflow 2.2.0 (datahub-project#3460)
maggiehays · Nov 3, 2021 · 344a797 · 344a797
1 parent 0d8f232
commit 344a797
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 1 deletion.
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -58,11 +58,14 @@ jobs:
 
   metadata-ingestion-general:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.6", "3.9"]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.6"
+          python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Run metadata-ingestion tests

diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle
@@ -29,7 +29,12 @@ task installDev(type: Exec, dependsOn: [install]) {
   commandLine "${venv_name}/bin/pip", 'install', '-e', '.[dev]'
 }
 task lint(type: Exec, dependsOn: installDev) {
+  /*
+    The find/sed combo below is a temporary work-around for the following mypy issue with airflow 2.2.0:
+   "venv/lib/python3.8/site-packages/airflow/_vendor/connexion/spec.py:169: error: invalid syntax".
+   */
   commandLine 'bash', '-x', '-c',
+    "find ${venv_name}/lib -path *airflow/_vendor/connexion/spec.py -exec sed -i.bak -e '169,169s/  # type: List\\[str\\]//g' {} \\; && " +
     "source ${venv_name}/bin/activate && black --check --diff src/ tests/ && isort --check --diff src/ tests/ && flake8 --count --statistics src/ tests/ && mypy src/ tests/"
 }
 task lintFix(type: Exec, dependsOn: installDev) {