Fix release flow (#119)

.flake8

@@ -1,3 +1,2 @@
 [flake8]
-
-max-line-length = 120
+max-line-length = 120

.github/workflows/release.yml

@@ -1,7 +1,6 @@
 name: Publish
 
 on:
-  workflow_dispatch:
   push:
     tags:
       - '*'
@@ -24,8 +23,10 @@ jobs:
       id: meta
       uses: docker/metadata-action@v5
       with:
-        images: datamindedbe/cookiecutter
-        tags: type=ref
+        images: |
+          datamindedbe/cookiecutter
+        tags: |
+          type=ref,event=tag
     - name: Login to Docker Hub
       uses: docker/login-action@v3
       with:

.github/workflows/test.yml

@@ -13,22 +13,16 @@ jobs:
     steps:
       - name: checkout
         uses: actions/checkout@v4
-      - name: Set up python
-        uses: actions/setup-python@v5
-        with:
-          python-version-file: ".python-version"
-          architecture: x64
       - uses: actions/setup-java@v4
         with:
           distribution: 'adopt'
           java-version: '11'
-      - name: Cache python dependencies
-        uses: actions/cache@v4
+      - name: Set up python
+        uses: actions/setup-python@v5
         with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
+          python-version-file: ".python-version"
+          cache: 'pip'
+          architecture: x64
       - name: install dependencies
         run: pip install -r requirements.txt
       - name: install dev dependencies

dev-requirements.in

@@ -1,6 +1,7 @@
 -c requirements.txt
-pytest==7.4.0
-pytest-cookies==0.7.0
-flake8==5.0.4
-black==22.12.0
-pip-tools==6.12.1
+
+pytest>=7.4.0
+pytest-cookies>=0.7.0
+flake8
+black
+pip-tools

dev-requirements.txt

@@ -12,7 +12,7 @@ binaryornot==0.4.4
     # via
     #   -c requirements.txt
     #   cookiecutter
-black==22.12.0
+black==24.4.0
     # via -r dev-requirements.in
 build==1.2.1
     # via pip-tools
@@ -38,7 +38,7 @@ cookiecutter==2.3.1
     # via
     #   -c requirements.txt
     #   pytest-cookies
-flake8==5.0.4
+flake8==7.0.0
     # via -r dev-requirements.in
 idna==3.7
     # via
@@ -69,31 +69,34 @@ mypy-extensions==1.0.0
 packaging==24.0
     # via
     #   -c requirements.txt
+    #   black
     #   build
     #   pytest
 pathspec==0.11.2
     # via
     #   -c requirements.txt
     #   black
-pip-tools==6.12.1
+pip-tools==7.4.1
     # via -r dev-requirements.in
 platformdirs==4.2.0
     # via
     #   -c requirements.txt
     #   black
 pluggy==1.5.0
     # via pytest
-pycodestyle==2.9.1
+pycodestyle==2.11.1
     # via flake8
-pyflakes==2.5.0
+pyflakes==3.2.0
     # via flake8
 pygments==2.17.2
     # via
     #   -c requirements.txt
     #   rich
 pyproject-hooks==1.0.0
-    # via build
-pytest==7.4.0
+    # via
+    #   build
+    #   pip-tools
+pytest==8.1.1
     # via
     #   -r dev-requirements.in
     #   pytest-cookies

project/dbt/hooks/post_gen_project.py

@@ -39,7 +39,15 @@ def initialize_dbt():
 def initialize_dbt_in_dir(dir: str):
     os.chdir(dir)
     try:
-        res = dbtRunner().invoke(["init", "--skip-profile-setup", f"--project-dir={dir}", f"--profiles-dir={dir}", 'temp'])
+        res = dbtRunner().invoke(
+            [
+                "init",
+                "--skip-profile-setup",
+                f"--project-dir={dir}",
+                f"--profiles-dir={dir}",
+                "temp",
+            ]
+        )
         if not res.success:
             raise Exception(res.exception)
     finally:
@@ -76,19 +84,19 @@ def cleanup_development_environment():
 
 def fix_dbt_project():
     root_dir = os.getcwd()
-    dbt_dir = os.path.join(root_dir,'temp')
+    dbt_dir = os.path.join(root_dir, "temp")
     files = os.listdir(dbt_dir)
     for file in files:
-        if file == 'README.md' or file == '.gitignore':
+        if file == "README.md" or file == ".gitignore":
             continue
         file_name = os.path.join(dbt_dir, file)
         shutil.move(file_name, root_dir)
     shutil.rmtree(dbt_dir)
-    with open(f'./dbt_project.yml', "rt") as f:
+    with open(f"./dbt_project.yml", "rt") as f:
         data = f.read()
         data = data.replace(f"profile: 'temp'", "profile: 'default'")
         data = data.replace("temp", f"{project_name}")
-    with open(f'./dbt_project.yml', "wt") as f:
+    with open(f"./dbt_project.yml", "wt") as f:
         f.write(data)
 
 

project/python/hooks/post_gen_project.py

@@ -33,5 +33,3 @@ def cleanup_development_environment():
 if __name__ == "__main__":
     cleanup_resources()
     cleanup_development_environment()
-
-

tests/projects/test_dbt.py

@@ -4,9 +4,9 @@
 from project.dbt.hooks.post_gen_project import initialize_dbt_in_dir
 
 
-def assertFileContainsContent(file, content):
-    fileContent = open(file).read()
-    assert fileContent.__contains__(content)
+def assert_file_contains_content(file, content):
+    file_content = open(file).read()
+    assert content in file_content
 
 
 def test_dbt_template(cookies):
@@ -26,8 +26,8 @@ def test_dbt_duckdb_template(cookies):
     )
     assert 0 == result.exit_code
     assert result.exception is None
-    assertFileContainsContent(
-        result.project + "/dags/duckdb_test.py", "ConveyorContainerOperatorV2("
+    assert_file_contains_content(
+        result.project_path / "dags/duckdb_test.py", "ConveyorContainerOperatorV2("
     )
     assert result.project_path.joinpath("query_duckdb.python").exists()
 
@@ -43,9 +43,9 @@ def test_dbt_duckdb_not_conveyor_managed_template(cookies):
     )
     assert 0 == result.exit_code
     assert result.exception is None
-    file = result.project + "/dags/duckdb_test.py"
-    file_content = file.read()
-    assert not file_content.__contains__("aws_role")
+    with open(result.project_path / "dags/duckdb_test.py") as file:
+        file_content = file.read()
+    assert "aws_role" not in file_content
 
 
 def test_dbt_duckdb_conveyor_managed_template(cookies):
@@ -59,9 +59,9 @@ def test_dbt_duckdb_conveyor_managed_template(cookies):
     )
     assert 0 == result.exit_code
     assert result.exception is None
-    file = result.project + "/dags/duckdb_test.py"
-    file_content = file.read()
-    assert file_content.__contains__("aws_role")
+    with open(result.project_path / "dags/duckdb_test.py") as file:
+        file_content = file.read()
+    assert "aws_role" in file_content
 
 
 def test_dbt_postgres_template(cookies):
@@ -71,8 +71,8 @@ def test_dbt_postgres_template(cookies):
     )
     assert 0 == result.exit_code
     assert result.exception is None
-    assertFileContainsContent(
-        result.project + "/dags/postgres_test.py", "factory.add_tasks_to_dag("
+    assert_file_contains_content(
+        result.project_path / "dags/postgres_test.py", "factory.add_tasks_to_dag("
     )
     assert not result.project_path.joinpath("query_duckdb.python").exists()
 

tests/projects/test_pyspark.py

@@ -5,13 +5,13 @@
 def assert_template_succeeds(result):
     assert 0 == result.exit_code, result.exception
     assert result.exception is None
-    assert result.project.isdir()
+    assert result.project_path.is_dir()
 
 
 def assert_first_line(file, content):
     with open(file) as f:
-        firstline = f.readline().rstrip()
-        assert firstline.__contains__(content)
+        first_line = f.readline().rstrip()
+        assert content in first_line
 
 
 def test_pyspark_template(cookies):
@@ -21,7 +21,7 @@ def test_pyspark_template(cookies):
     )
     assert_template_succeeds(result)
     assert_first_line(
-        result.project + "/Dockerfile",
+        result.project_path / "Dockerfile",
         "FROM public.ecr.aws/dataminded/spark-k8s-glue",
     )
 
@@ -32,7 +32,7 @@ def test_pyspark_template_azure(cookies):
         extra_context={"cloud": "azure", "conveyor_managed_role": True},
     )
     assert_template_succeeds(result)
-    assert not (result.project + "/resources").isdir()
+    assert not (result.project_path / "resources").is_dir()
     assert_batch_files(result)
 
 
@@ -42,14 +42,14 @@ def test_pyspark_template_no_role(cookies):
         extra_context={"conveyor_managed_role": False},
     )
     assert_template_succeeds(result)
-    assert not (result.project + "/resources").isdir()
+    assert not (result.project_path / "resources").is_dir()
 
 
 def assert_project_can_be_build(result):
     assert_template_succeeds(result)
     process = subprocess.Popen(
         ["docker", "build", "."],
-        cwd=result.project,
+        cwd=result.project_path,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
     )
@@ -85,14 +85,14 @@ def test_pyspark_template_spark_pipenv(cookies):
 
 
 def assert_streaming_files(result, exist: bool = True):
-    assert (result.project + "/streaming.yaml").isfile() == exist
-    assert (result.project + "/src/pyspark/streaming_app.py").isfile() == exist
+    assert (result.project_path / "streaming.yaml").is_file() == exist
+    assert (result.project_path / "src/pyspark/streaming_app.py").is_file() == exist
 
 
 def assert_batch_files(result, exist: bool = True):
-    assert (result.project + "/dags").isdir() == exist
-    assert (result.project + "/src/pyspark/transformations").isdir() == exist
-    assert (result.project + "/src/pyspark/app.py").isfile() == exist
+    assert (result.project_path / "dags").is_dir() == exist
+    assert (result.project_path / "src/pyspark/transformations").is_dir() == exist
+    assert (result.project_path / "src/pyspark/app.py").is_file() == exist
 
 
 def test_pyspark_template_only_batch(cookies):

tests/projects/test_python.py

@@ -8,13 +8,13 @@ def test_python_template(cookies):
     )
     assert 0 == result.exit_code
     assert result.exception is None
-    assert result.project.isdir()
-    assert not (result.project + "/Pipfile").exists()
-    assert not (result.project + "/Pipfile.lock").exists()
-    assert (result.project + "/requirements.in").exists()
-    assert (result.project + "/requirements.txt").exists()
-    assert (result.project + "/dev-requirements.in").exists()
-    assert (result.project + "/dev-requirements.txt").exists()
+    assert result.project_path.is_dir()
+    assert not (result.project_path / "Pipfile").exists()
+    assert not (result.project_path / "Pipfile.lock").exists()
+    assert (result.project_path / "requirements.in").exists()
+    assert (result.project_path / "requirements.txt").exists()
+    assert (result.project_path / "dev-requirements.in").exists()
+    assert (result.project_path / "dev-requirements.txt").exists()
 
 
 def test_python_template_no_role(cookies):
@@ -24,8 +24,8 @@ def test_python_template_no_role(cookies):
     )
     assert 0 == result.exit_code
     assert result.exception is None
-    assert result.project.isdir()
-    assert not (result.project + "/resources").isdir()
+    assert result.project_path.is_dir()
+    assert not (result.project_path / "resources").is_dir()
 
 
 def test_python_template_azure(cookies):
@@ -35,5 +35,5 @@ def test_python_template_azure(cookies):
     )
     assert 0 == result.exit_code
     assert result.exception is None
-    assert result.project.isdir()
-    assert not (result.project + "/resources").isdir()
+    assert result.project_path.is_dir()
+    assert not (result.project_path / "resources").is_dir()