Consolidate local volume mounts to dev/containers (#5215)

.gitignore

@@ -108,6 +108,8 @@ ENV/
 # Local Postgres data
 pg_data/
 pg_data.bak*/
+dev/containers/postgresql/data/
+dev/containers/postgresql/data.bak*/
 
 # static files
 koku/staticfiles/

Makefile

@@ -67,7 +67,7 @@ help:
 	@echo "--- Commands using local services ---"
 	@echo "  delete-testing                        Delete stale files/subdirectories from the testing directory."
 	@echo "  delete-trino                          Delete stale files/subdirectories from the trino data directory."
-	@echo "  delete-trino-data                     Delete old trino data from .trino/parquet_data/koku-bucket/."
+	@echo "  delete-trino-data                     Delete old trino data from the Minio koku-bucket bucket."
 	@echo "  delete-redis-cache                    Flushes cache keys inside of the redis container."
 	@echo "  create-test-customer                  create a test customer and tenant in the database"
 	@echo "  create-test-customer-no-sources       create a test customer and tenant in the database without test sources"
@@ -97,7 +97,7 @@ help:
 	@echo "  make-migrations                       make migrations for the database"
 	@echo "  requirements                          generate Pipfile.lock"
 	@echo "  clowdapp                              generates a new clowdapp.yaml"
-	@echo "  delete-db                             delete local directory $(TOPDIR)/pg_data"
+	@echo "  delete-db                             delete local directory $(TOPDIR)/dev/containers/postgresql/data"
 	@echo "  delete-test-db                        delete the django test db"
 	@echo "  reset-db-statistics                   clear the pg_stat_statements statistics"
 	@echo "  run-migrations                        run migrations against database"
@@ -167,10 +167,11 @@ delete-testing:
 	@$(PREFIX) $(PYTHON) $(SCRIPTDIR)/clear_testing.py -p $(TOPDIR)/testing
 
 delete-trino:
-	@$(PREFIX) rm -rf $(TOPDIR)/.trino/trino/*
+	@$(PREFIX) rm -rf $(TOPDIR)/dev/containers/trino/data/*
+	@$(PREFIX) rm -rf $(TOPDIR)/dev/containers/trino/logs/*
 
 delete-trino-data:
-	@$(PREFIX) rm -rf $(TOPDIR)/.trino/parquet_data/koku-bucket/*
+	@$(PREFIX) rm -rf $(TOPDIR)/dev/containers/minio/koku-bucket/*
 
 delete-redis-cache:
 	$(DOCKER) exec -it koku_redis redis-cli -n 1 flushall
@@ -213,7 +214,7 @@ make-migrations:
 	$(DJANGO_MANAGE) makemigrations api reporting reporting_common cost_models key_metrics
 
 delete-db:
-	@$(PREFIX) rm -rf $(TOPDIR)/pg_data/data/*
+	@$(PREFIX) rm -rf $(TOPDIR)/dev/containers/postgresql/data/
 
 delete-test-db:
 	@PGPASSWORD=$$DATABASE_PASSWORD psql -h $$POSTGRES_SQL_SERVICE_HOST \
@@ -382,7 +383,7 @@ docker-iqe-api-tests: docker-reinitdb _set-test-dir-permissions delete-testing
 docker-iqe-vortex-tests: docker-reinitdb _set-test-dir-permissions delete-testing
 	./testing/run_vortex_api_tests.sh
 
-CONTAINER_DIRS = $(TOPDIR)/pg_data/data $(TOPDIR)/.trino/{parquet_data,trino}
+CONTAINER_DIRS = $(TOPDIR)/dev/containers/postgresql/data $(TOPDIR)/dev/containers/minio $(TOPDIR)/dev/containers/trino/{data,logs}
 docker-host-dir-setup:
 	@mkdir -p -m 0755 $(CONTAINER_DIRS) 2>&1 > /dev/null
 	@chown $(USER_ID):$(GROUP_ID) $(CONTAINER_DIRS)
@@ -564,7 +565,7 @@ backup-local-db-dir:
 	$(DOCKER_COMPOSE) stop db
 	@cd $(TOPDIR)
 	@echo "Copying pg_data to pg_data.bak..."
-	@$(PREFIX) cp -rp ./pg_data ./pg_data.bak
+	@$(PREFIX) cp -rp ./dev/containers/postgresql/data ./dev/containers/postgresql/data.bak
 	@cd - >/dev/null
 	$(DOCKER_COMPOSE) start db
 
@@ -577,12 +578,12 @@ local-upload-data:
 
 restore-local-db-dir:
 	@cd $(TOPDIR)
-	@if [ -d ./pg_data.bak ] ; then \
+	@if [ -d ./dev/containers/postgresql/data.bak ] ; then \
 	    $(DOCKER_COMPOSE) stop db ; \
 	    echo "Removing pg_data..." ; \
-	    $(PREFIX) rm -rf ./pg_data ; \
+	    $(PREFIX) rm -rf ./dev/containers/postgresql/data ; \
 	    echo "Renaming pg_data.bak to pg_data..." ; \
-	    $(PREFIX) mv -f ./pg_data.bak ./pg_data ; \
+	    $(PREFIX) mv -f ./dev/containers/postgresql/data.bak ./dev/containers/postgresql/data ; \
 	    $(DOCKER_COMPOSE) start db ; \
 	    echo "NOTE :: Migrations may need to be run." ; \
 	else \

dev/containers/minio/.gitignore

@@ -0,0 +1,5 @@
+# Ignore all files in this dir...
+*
+
+# ... except for this one.
+!.gitignore

dev/containers/trino/data/.gitignore

@@ -0,0 +1,5 @@
+# Ignore all files in this dir...
+*
+
+# ... except for this one.
+!.gitignore

dev/containers/trino/logs/.gitignore

@@ -0,0 +1,5 @@
+# Ignore all files in this dir...
+*
+
+# ... except for this one.
+!.gitignore

docker-compose.yml

@@ -463,7 +463,7 @@ services:
     ports:
       - ${POSTGRES_SQL_SERVICE_PORT-15432}:5432
     volumes:
-      - ./pg_data/data:/var/lib/${DATABASE_DATA_DIR-postgresql}/data
+      - ./dev/containers/postgresql/data:/var/lib/${DATABASE_DATA_DIR-postgresql}/data
     command:
       # This command give more precise control over the parameter settings
       # Included are loading the pg_stat_statements lib
@@ -605,7 +605,7 @@ services:
       - 9090:9090
     command: server /home/kminio/data --console-address ":9090"
     volumes:
-      - ./.trino/parquet_data:/home/kminio/data
+      - ./dev/containers/minio:/home/kminio/data
 
   create-parquet-bucket:
     image: minio/mc:latest
@@ -676,8 +676,8 @@ services:
       - ./dev/containers/trino/etc/catalog/hive.properties:/etc/trino/catalog/hive.properties
       - ./dev/containers/trino/etc/catalog/postgres.properties:/etc/trino/catalog/postgres.properties
       - ./dev/containers/trino/etc/hadoop-config/core-site.xml:/etc/trino/hadoop-config/core-site.xml
-      - ./.trino/trino/trino-data:/data/trino/data
-      - ./.trino/trino/trino-logs:/data/trino/logs
+      - ./dev/containers/trino/data:/data/trino/data
+      - ./dev/containers/trino/logs:/data/trino/logs
     links:
       - hive-metastore
     depends_on: