feat: make dbt thread count configurable (#190)

* feat: make dbt thread count configurable * chore: update dbt configuration and environment variables --------- Co-authored-by: Maria Lorena Rodriguez Viruel <[email protected]>
medic · Dec 6, 2024 · 17318c1 · 17318c1
1 parent b4eefa6
commit 17318c1
Show file tree

Hide file tree

Showing 6 changed files with 7 additions and 2 deletions.
diff --git a/dbt/.dbt/profiles.yml b/dbt/.dbt/profiles.yml
@@ -2,6 +2,7 @@ default:
   target: default
   outputs:
     default:
+      threads: "{{ env_var('DBT_THREAD_COUNT') | as_number }}"
       type: postgres
       host: "{{ env_var('POSTGRES_HOST') }}"
       user: "{{ env_var('POSTGRES_USER') }}"

diff --git a/deploy/cht_sync/values.yaml.template b/deploy/cht_sync/values.yaml.template
@@ -10,6 +10,7 @@ postgres:
   port: 5432 # if postgres is outside the cluster
 
 cht_pipeline_branch_url: "https://github.com/medic/cht-pipeline.git#main"
+DBT_THREAD_COUNT: 1
 
 # values shared by all couchdb instances
 # can be omitted if couchdb instances do not share any values

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -38,3 +38,4 @@ services:
       - CHT_PIPELINE_BRANCH_URL=${CHT_PIPELINE_BRANCH_URL}
       - DATAEMON_INTERVAL=${DATAEMON_INTERVAL}
       - DBT_PACKAGE_TARBALL_URL=${DBT_PACKAGE_TARBALL_URL}
+      - DBT_THREAD_COUNT=${DBT_THREAD_COUNT}
diff --git a/env.template b/env.template
@@ -10,6 +10,7 @@ POSTGRES_PORT=5432
 # dbt
 CHT_PIPELINE_BRANCH_URL="https://github.com/medic/cht-pipeline.git#main"
 DATAEMON_INTERVAL=5
+DBT_THREAD_COUNT=1
 
 # couchdb
 COUCHDB_USER=medic

diff --git a/package.json b/package.json
@@ -6,7 +6,7 @@
   "scripts": {
     "postinstall": "cd couch2pg && npm ci",
     "test:e2e": "npm run test:e2e-data && npm run test:e2e-stop-containers && npm run test:e2e-containers && npm run test:e2e-mocha && npm run test:e2e-stop-containers ",
-    "test:e2e-mocha": "mocha tests/**/*.spec.js --timeout 50000",
+    "test:e2e-mocha": "mocha tests/**/*.spec.js --timeout 70000",
     "lint": "eslint --color --cache .",
     "test:e2e-stop-containers": "docker compose --env-file ./tests/.e2e-env -f docker-compose.yml -f docker-compose.couchdb.yml -f docker-compose.postgres.yml -f tests/dbt/docker-compose.yml -f docker-compose.bastion.yml kill && docker compose --env-file ./tests/.e2e-env -f docker-compose.yml -f docker-compose.couchdb.yml -f docker-compose.postgres.yml -f tests/dbt/docker-compose.yml -f docker-compose.bastion.yml rm -f",
     "test:e2e-containers": "docker compose --env-file ./tests/.e2e-env -f docker-compose.yml -f docker-compose.couchdb.yml -f docker-compose.postgres.yml -f tests/dbt/docker-compose.yml -f docker-compose.bastion.yml up -d --build --force-recreate && npm run wait-for-couchdb",

diff --git a/tests/.e2e-env b/tests/.e2e-env
@@ -13,4 +13,5 @@ COUCHDB_HOST="host.docker.internal"
 COUCHDB_PORT=5984
 COUCHDB_SECURE=false
 POSTGRES_HOST=postgres
-BASTION_AUTHORIZED_KEYS_FILE=./tests/utils/bastion-ssh-key.pub
+BASTION_AUTHORIZED_KEYS_FILE=./tests/utils/bastion-ssh-key.pub
+DBT_THREAD_COUNT=1