Fix docker-based testing for Linux users

See dbt-labs#2739

This change enables Linux users to run the dbt tests via the docker
image. It follows the recommendations from this article:
https://jtreminio.com/blog/running-docker-containers-as-current-host-user/

Notable changes:
*  Added new Makefile rule to generate a .env file that contains USER_ID
and GROUP_ID environment variables to the ID of the current user. This
is in turn used by docker-compose and the Dockerfile to make the Docker
image run as the current user. (Note that on Windows or Mac, this
behavior happens by default).
*  Reordered Dockerfile to allow for better caching of intermediate
images (i.e., put things that don't depend on ARGS earlier).
*  Bumped CircleCI's Dockerfile from version 7 to 9.  Jake rebuilt
9 off of this PR.

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2.1
 jobs:
   unit:
     docker: &test_only
-      - image: fishtownanalytics/test-container:7
+      - image: fishtownanalytics/test-container:9
         environment:
           DBT_INVOCATION_ENV: circle
     steps:
@@ -30,7 +30,7 @@ jobs:
           destination: dist
   integration-postgres-py36:
     docker: &test_and_postgres
-      - image: fishtownanalytics/test-container:7
+      - image: fishtownanalytics/test-container:9
         environment:
           DBT_INVOCATION_ENV: circle
       - image: postgres

.gitignore

@@ -43,6 +43,7 @@ htmlcov/
 .coverage
 .coverage.*
 .cache
+.env
 nosetests.xml
 coverage.xml
 *,cover

CHANGELOG.md

@@ -2,12 +2,14 @@
 
 ### Under the hood
 - If column config says quote, use quoting in SQL for adding a comment. ([#2539](https://github.com/fishtown-analytics/dbt/issues/2539), [#2733](https://github.com/fishtown-analytics/dbt/pull/2733))
+- Added support for running docker-based tests under Linux. ([#2739](https://github.com/fishtown-analytics/dbt/issues/2739))
 
 ### Features
 - Specify all three logging levels (`INFO`, `WARNING`, `ERROR`) in result logs for commands `test`, `seed`, `run`, `snapshot` and `source snapshot-freshness` ([#2680](https://github.com/fishtown-analytics/dbt/pull/2680), [#2723](https://github.com/fishtown-analytics/dbt/pull/2723))
 
 Contributors:
 - [@tpilewicz](https://github.com/tpilewicz) ([#2723](https://github.com/fishtown-analytics/dbt/pull/2723))
+- [@heisencoder](https://github.com/heisencoder) ([#2739](https://github.com/fishtown-analytics/dbt/issues/2739))
 
 
 ## dbt 0.18.0 (September 03, 2020)

Dockerfile

@@ -1,7 +1,6 @@
 FROM ubuntu:18.04
 
 ENV DEBIAN_FRONTEND noninteractive
-ARG DOCKERIZE_VERSION=v0.6.1
 
 RUN apt-get update && \
     apt-get dist-upgrade -y && \
@@ -19,20 +18,33 @@ RUN apt-get update && \
         python3.9 python3.9-dev python3.9-venv && \
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-RUN useradd -mU dbt_test_user
-RUN mkdir /usr/app && chown dbt_test_user /usr/app
-RUN mkdir /home/tox && chown dbt_test_user /home/tox
+ARG DOCKERIZE_VERSION=v0.6.1
 RUN curl -LO https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \
     tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \
     rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
 
-WORKDIR /usr/app
-VOLUME /usr/app
-
 RUN pip3 install -U "tox==3.14.4" wheel "six>=1.14.0,<1.15.0" "virtualenv==20.0.3" setuptools
 # tox fails if the 'python' interpreter (python2) doesn't have `tox` installed
 RUN pip install -U "tox==3.14.4" "six>=1.14.0,<1.15.0" "virtualenv==20.0.3" setuptools
 
+# These args are passed in via docker-compose, which reads then from the .env file.
+# On Linux, run `make .env` to create the .env file for the current user.
+# On MacOS and Windows, these can stay unset.
+ARG USER_ID
+ARG GROUP_ID
+
+RUN if [ ${USER_ID:-0} -ne 0 ] && [ ${GROUP_ID:-0} -ne 0 ]; then \
+        groupadd -g ${GROUP_ID} dbt_test_user && \
+        useradd -m -l -u ${USER_ID} -g ${GROUP_ID} dbt_test_user; \
+    else \
+        useradd -mU -l dbt_test_user; \
+    fi
+RUN mkdir /usr/app && chown dbt_test_user /usr/app
+RUN mkdir /home/tox && chown dbt_test_user /home/tox
+
+WORKDIR /usr/app
+VOLUME /usr/app
+
 USER dbt_test_user
 
 ENV PYTHONIOENCODING=utf-8

Makefile

@@ -5,25 +5,33 @@ changed_tests := `git status --porcelain | grep '^\(M\| M\|A\| A\)' | awk '{ pri
 install:
 	pip install -e .
 
-test:
+test: .env
 	@echo "Full test run starting..."
 	@time docker-compose run test tox
 
-test-unit:
+test-unit: .env
 	@echo "Unit test run starting..."
 	@time docker-compose run test tox -e unit-py36,flake8
 
-test-integration:
+test-integration: .env
 	@echo "Integration test run starting..."
 	@time docker-compose run test tox -e integration-postgres-py36,integration-redshift-py36,integration-snowflake-py36,integration-bigquery-py36
 
-test-quick:
+test-quick: .env
 	@echo "Integration test run starting..."
 	@time docker-compose run test tox -e integration-postgres-py36 -- -x
 
+# This rule creates a file named .env that is used by docker-compose for passing
+# the USER_ID and GROUP_ID arguments to the Docker image.
+.env:
+	@echo USER_ID=$(shell id -u) > .env
+	@echo GROUP_ID=$(shell id -g) >> .env
+	@time docker-compose build
+
 clean:
 	rm -f .coverage
 	rm -rf .eggs/
+	rm -f .env
 	rm -rf .tox/
 	rm -rf build/
 	rm -rf dbt.egg-info/

docker-compose.yml

@@ -14,6 +14,10 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
+      args:
+        # Run `make .env` to set $USER_ID and $GROUP_ID
+        USER_ID: ${USER_ID}
+        GROUP_ID: ${GROUP_ID}
     command: "/root/.virtualenvs/dbt/bin/pytest"
     env_file:
       - ./test.env