From 06eb331278f4708da1542d497f8412db7e1a1bd3 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 26 Aug 2022 16:04:25 -0400 Subject: [PATCH 1/6] start on local dev dockerfile --- warehouse/Dockerfile.local | 26 ++++++++++++++++++++++++++ warehouse/dbt.sh | 3 +++ 2 files changed, 29 insertions(+) create mode 100644 warehouse/Dockerfile.local create mode 100755 warehouse/dbt.sh diff --git a/warehouse/Dockerfile.local b/warehouse/Dockerfile.local new file mode 100644 index 0000000000..c36a5acca3 --- /dev/null +++ b/warehouse/Dockerfile.local @@ -0,0 +1,26 @@ +FROM python:3.9-buster + +LABEL org.opencontainers.image.source https://github.com/cal-itp/data-infra + +RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - +RUN apt-get update \ + && apt-get install -y nodejs + +RUN npm install -g --unsafe-perm=true --allow-root netlify-cli + +RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - +ENV PATH="${PATH}:/root/.poetry/bin" + +RUN mkdir /app +WORKDIR /app + +COPY ./pyproject.toml /app/pyproject.toml +COPY ./poetry.lock /app/poetry.lock +RUN poetry export -f requirements.txt --without-hashes --output requirements.txt \ + && pip install -r requirements.txt + +COPY ./dbt_project.yml /app/dbt_project.yml +COPY ./packages.yml /app/packages.yml +RUN dbt deps + +CMD ["dbt", "run", "--project-dir", "/app", "--profiles-dir", "/app"] diff --git a/warehouse/dbt.sh b/warehouse/dbt.sh new file mode 100755 index 0000000000..e6b215b633 --- /dev/null +++ b/warehouse/dbt.sh @@ -0,0 +1,3 @@ +#!/bin/bash +docker build -t local-dbt -f Dockerfile.local . +docker run --entrypoint dbt -v $(pwd):/app local-dbt "$@" From 45777f8737024b356d78fb96e3a1a7b9415b01e4 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 26 Aug 2022 16:06:50 -0400 Subject: [PATCH 2/6] handle local profiles dir --- warehouse/dbt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warehouse/dbt.sh b/warehouse/dbt.sh index e6b215b633..0ef7442444 100755 --- a/warehouse/dbt.sh +++ b/warehouse/dbt.sh @@ -1,3 +1,3 @@ #!/bin/bash docker build -t local-dbt -f Dockerfile.local . -docker run --entrypoint dbt -v $(pwd):/app local-dbt "$@" +docker run --entrypoint dbt -v ~/.dbt:/local_dbt -v $(pwd):/app local-dbt "$@" --profiles-dir /local_dbt From 69aeb0505595842ed3a46372386a33ba27b8a0d4 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Mon, 29 Aug 2022 14:36:54 -0400 Subject: [PATCH 3/6] make dbt docker work with local google credentials --- warehouse/dbt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warehouse/dbt.sh b/warehouse/dbt.sh index 0ef7442444..089316bed9 100755 --- a/warehouse/dbt.sh +++ b/warehouse/dbt.sh @@ -1,3 +1,3 @@ #!/bin/bash docker build -t local-dbt -f Dockerfile.local . -docker run --entrypoint dbt -v ~/.dbt:/local_dbt -v $(pwd):/app local-dbt "$@" --profiles-dir /local_dbt +docker run --entrypoint dbt -e GOOGLE_APPLICATION_CREDENTIALS=/gcloud_config/application_default_credentials.json -v ~/.dbt:/local_dbt -v ~/.config/gcloud:/gcloud_config -v $(pwd):/app local-dbt "$@" --profiles-dir /local_dbt From 5442c059961264f3dafd2a366daaf1e70c08214c Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Mon, 29 Aug 2022 15:20:56 -0400 Subject: [PATCH 4/6] add build-essentials per recommendation --- warehouse/Dockerfile.local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warehouse/Dockerfile.local b/warehouse/Dockerfile.local index c36a5acca3..44df8a3fed 100644 --- a/warehouse/Dockerfile.local +++ b/warehouse/Dockerfile.local @@ -4,7 +4,7 @@ LABEL org.opencontainers.image.source https://github.com/cal-itp/data-infra RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - RUN apt-get update \ - && apt-get install -y nodejs + && apt-get install -y nodejs build-essential RUN npm install -g --unsafe-perm=true --allow-root netlify-cli From b92741f6da0492873cc4a0e71dceee06a8c1cddf Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 31 Aug 2022 10:32:16 -0400 Subject: [PATCH 5/6] update poetry install method and add libgdal-dev --- warehouse/Dockerfile | 4 ++-- warehouse/Dockerfile.local | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/warehouse/Dockerfile b/warehouse/Dockerfile index b1da0a063f..f32ef0fd4b 100644 --- a/warehouse/Dockerfile +++ b/warehouse/Dockerfile @@ -4,11 +4,11 @@ LABEL org.opencontainers.image.source https://github.com/cal-itp/data-infra RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - RUN apt-get update \ - && apt-get install -y nodejs + && apt-get install -y nodejs libgdal-dev RUN npm install -g --unsafe-perm=true --allow-root netlify-cli -RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - +RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python - ENV PATH="${PATH}:/root/.poetry/bin" RUN mkdir /app diff --git a/warehouse/Dockerfile.local b/warehouse/Dockerfile.local index 44df8a3fed..76ac3ea97c 100644 --- a/warehouse/Dockerfile.local +++ b/warehouse/Dockerfile.local @@ -4,11 +4,11 @@ LABEL org.opencontainers.image.source https://github.com/cal-itp/data-infra RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - RUN apt-get update \ - && apt-get install -y nodejs build-essential + && apt-get install -y nodejs build-essential libgdal-dev RUN npm install -g --unsafe-perm=true --allow-root netlify-cli -RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - +RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python - ENV PATH="${PATH}:/root/.poetry/bin" RUN mkdir /app From 8250b0fc035b42223bd9dc28d42412aa0b17e972 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 31 Aug 2022 12:41:01 -0400 Subject: [PATCH 6/6] poetry changed its bin location --- warehouse/Dockerfile | 2 +- warehouse/Dockerfile.local | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/warehouse/Dockerfile b/warehouse/Dockerfile index f32ef0fd4b..2797880c54 100644 --- a/warehouse/Dockerfile +++ b/warehouse/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update \ RUN npm install -g --unsafe-perm=true --allow-root netlify-cli RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python - -ENV PATH="${PATH}:/root/.poetry/bin" +ENV PATH="/root/.local/bin:${PATH}" RUN mkdir /app WORKDIR /app diff --git a/warehouse/Dockerfile.local b/warehouse/Dockerfile.local index 76ac3ea97c..2e5c6e4f8e 100644 --- a/warehouse/Dockerfile.local +++ b/warehouse/Dockerfile.local @@ -9,7 +9,7 @@ RUN apt-get update \ RUN npm install -g --unsafe-perm=true --allow-root netlify-cli RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python - -ENV PATH="${PATH}:/root/.poetry/bin" +ENV PATH="/root/.local/bin:${PATH}" RUN mkdir /app WORKDIR /app