From b0a38cd535bd3c90d4de8a53ec5316b86c5d6b14 Mon Sep 17 00:00:00 2001
From: Padraic Slattery <pgoslatara@gmail.com>
Date: Mon, 9 Dec 2024 21:10:21 +0100
Subject: [PATCH] Moving dbt commands to retrying make command

---
 .github/workflows/ci_pipeline.yml           | 12 +++---------
 .github/workflows/dbt_artifact_probes.yml   |  5 +----
 .github/workflows/merge_pipeline.yml        | 10 ++--------
 .github/workflows/post_release_pipeline.yml | 10 ++--------
 makefile                                    |  7 +++++++
 5 files changed, 15 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/ci_pipeline.yml b/.github/workflows/ci_pipeline.yml
index 944766aa..f0c6c58c 100644
--- a/.github/workflows/ci_pipeline.yml
+++ b/.github/workflows/ci_pipeline.yml
@@ -2,7 +2,7 @@
 name: CI pipeline
 
 on:
-  pull_request_target:
+  pull_request: # _target:
       branches:
           - main
 
@@ -108,18 +108,12 @@ jobs:
               if: matrix.operating-system == 'windows-latest'
               run: |
                 $env:Path += ';C:\Users\runneradmin\.local\bin'
-                poetry run dbt deps
-                poetry run dbt build
-                poetry run dbt docs generate
-                poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml
+                (r=3;while ! make build-and-run-dbt-bouncer ; do ((--r))||exit;done)
 
             - name: Regenerate dbt artifacts and run `dbt-bouncer`
               if: matrix.operating-system != 'windows-latest'
               run: |
-                poetry run dbt deps
-                poetry run dbt build
-                poetry run dbt docs generate
-                poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml
+                  (r=3;while ! make build-and-run-dbt-bouncer ; do ((--r))||exit;done)
 
     e2e-tests:
         needs: [pre-commit]
diff --git a/.github/workflows/dbt_artifact_probes.yml b/.github/workflows/dbt_artifact_probes.yml
index 27da9dfd..3bf59d80 100644
--- a/.github/workflows/dbt_artifact_probes.yml
+++ b/.github/workflows/dbt_artifact_probes.yml
@@ -66,7 +66,4 @@
 
                 - name: Regenerate dbt artifacts and run `dbt-bouncer`
                   run: |
-                    poetry run dbt deps
-                    poetry run dbt build
-                    poetry run dbt docs generate
-                    poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml
+                    (r=3;while ! make build-and-run-dbt-bouncer ; do ((--r))||exit;done)
diff --git a/.github/workflows/merge_pipeline.yml b/.github/workflows/merge_pipeline.yml
index 41abaa67..d24e1fd2 100644
--- a/.github/workflows/merge_pipeline.yml
+++ b/.github/workflows/merge_pipeline.yml
@@ -157,15 +157,9 @@
               if: matrix.operating-system == 'windows-latest'
               run: |
                 $env:Path += ';C:\Users\runneradmin\.local\bin'
-                poetry run dbt deps
-                poetry run dbt build
-                poetry run dbt docs generate
-                poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml
+                make build-and-run-dbt-bouncer
 
             - name: Regenerate dbt artifacts and run `dbt-bouncer`
               if: matrix.operating-system != 'windows-latest'
               run: |
-                poetry run dbt deps
-                poetry run dbt build
-                poetry run dbt docs generate
-                poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml
+                (r=3;while ! make build-and-run-dbt-bouncer ; do ((--r))||exit;done)
diff --git a/.github/workflows/post_release_pipeline.yml b/.github/workflows/post_release_pipeline.yml
index 3568ab20..db096d76 100644
--- a/.github/workflows/post_release_pipeline.yml
+++ b/.github/workflows/post_release_pipeline.yml
@@ -119,15 +119,9 @@ jobs:
               if: matrix.operating-system == 'windows-latest'
               run: |
                 $env:Path += ';C:\Users\runneradmin\.local\bin'
-                poetry run dbt deps
-                poetry run dbt build
-                poetry run dbt docs generate
-                poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml
+                (r=3;while ! make build-and-run-dbt-bouncer ; do ((--r))||exit;done)
 
             - name: Regenerate dbt artifacts and run `dbt-bouncer`
               if: matrix.operating-system != 'windows-latest'
               run: |
-                poetry run dbt deps
-                poetry run dbt build
-                poetry run dbt docs generate
-                poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml
+                (r=3;while ! make build-and-run-dbt-bouncer ; do ((--r))||exit;done)
diff --git a/makefile b/makefile
index 2d00f675..cabdacc7 100644
--- a/makefile
+++ b/makefile
@@ -1,3 +1,10 @@
+# On GitHub the `dbt build` command often returns "leaked semaphores" errors.
+build-and-run-dbt-bouncer:
+	poetry run dbt deps
+	poetry run dbt build
+	poetry run dbt docs generate
+	poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml
+
 build-artifacts:
 	poetry run python ./scripts/generate_artifacts.py