Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Changed test environment (nox -s fides_env) to run fides deploy for local testing #3017

Merged
merged 37 commits into from
Apr 12, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
fe16f83
Replace the test environment with 'fides deploy up'
NevilleS Apr 7, 2023
45651e6
Working version of using 'fides deploy up for test environment
NevilleS Apr 8, 2023
db71ce3
Cleanup unused test_env files, update docs
NevilleS Apr 8, 2023
d7ac359
Remove 'fides deploy' from release checklist
NevilleS Apr 8, 2023
e7541f8
Formatting, linting, etc.
NevilleS Apr 8, 2023
397ce97
Tidy up some nox loose ends
NevilleS Apr 8, 2023
61d3e98
Change redis enabled to default to True
NevilleS Apr 9, 2023
078d1b8
Use explicit docker container name instead of "fides-fides-1"
NevilleS Apr 9, 2023
6727530
Formatting
NevilleS Apr 9, 2023
fb7a0d4
Update sample app privacy center with GPC defaults
NevilleS Apr 9, 2023
5f15481
Change 'fides deploy up --image' to override tag, too
NevilleS Apr 9, 2023
a930d3b
Remove user creation from test env setup scripts
NevilleS Apr 9, 2023
0e53b9d
Refactor check_required_webserver_config_values
NevilleS Apr 9, 2023
b57e237
Add 'fides user login' to deploy command
NevilleS Apr 9, 2023
7f6640a
Enable hot-reloading in 'fides deploy' with a custom command
NevilleS Apr 9, 2023
45c0249
Fix fides user login for deploy
NevilleS Apr 10, 2023
8658aba
Formatting
NevilleS Apr 10, 2023
3f8750b
Pylint
NevilleS Apr 10, 2023
52ebc4d
Update cli_checks.yml
NevilleS Apr 10, 2023
fb82820
Update LOGIN command to use fides container name
NevilleS Apr 10, 2023
9df820c
Update smoke test to expect only email
NevilleS Apr 10, 2023
b03340b
Add build-time report
NevilleS Apr 10, 2023
36e3af8
Add TODO for teardown
NevilleS Apr 10, 2023
d9ff8d8
Remove (failed) attempt at hot-reloading
NevilleS Apr 10, 2023
9e2d20a
Fix accidental leftover click.option
NevilleS Apr 10, 2023
8689730
fix: seed_example_data docker command
ThomasLaPiana Apr 11, 2023
12e60e6
Fix janky teardown code
NevilleS Apr 10, 2023
266aad0
fix: teardown doesn't teardown the test env
ThomasLaPiana Apr 11, 2023
e1052b8
Merge branch 'ns-new-test-env' of https://github.com/ethyca/fides int…
ThomasLaPiana Apr 11, 2023
8f5bd1e
refactor: slight code cleanup
ThomasLaPiana Apr 11, 2023
d9be6b6
refactor: add proper names to the sample app services
ThomasLaPiana Apr 11, 2023
8665f5f
feat: another slight tweak to the pip install step
ThomasLaPiana Apr 11, 2023
6de259a
Merge branch 'main' of github.com:ethyca/fides into ns-new-test-env
NevilleS Apr 11, 2023
e4e4d2c
fix: external test compose file invalid
ThomasLaPiana Apr 11, 2023
235c81f
Merge branch 'ns-new-test-env' of https://github.com/ethyca/fides int…
ThomasLaPiana Apr 11, 2023
10b24db
Update CHANGELOG.md
NevilleS Apr 12, 2023
8dcb946
Remove the 'fides deploy' CI check (redundant with cypress_e2e.yml wh…
NevilleS Apr 12, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions noxfiles/constants_nox.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@

# Files
COMPOSE_FILE = "docker-compose.yml"
INTEGRATION_COMPOSE_FILE = "docker-compose.integration-tests.yml"
INTEGRATION_POSTGRES_COMPOSE_FILE = "docker/docker-compose.integration-postgres.yml"
INTEGRATION_COMPOSE_FILE = "./docker-compose.integration-tests.yml"
INTEGRATION_POSTGRES_COMPOSE_FILE = "./docker/docker-compose.integration-postgres.yml"
REMOTE_DEBUG_COMPOSE_FILE = "docker-compose.remote-debug.yml"
SAMPLE_PROJECT_COMPOSE_FILE = "src/fides/data/sample_project/docker-compose.yml"
SAMPLE_PROJECT_COMPOSE_FILE = "./src/fides/data/sample_project/docker-compose.yml"
WITH_TEST_CONFIG = ("-f", "tests/ctl/test_config.toml")

# Image Names & Tags
Expand Down
60 changes: 27 additions & 33 deletions noxfiles/utils_nox.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,35 +9,16 @@
)
from run_infrastructure import run_infrastructure

COMPOSE_DOWN = (
"docker",
"compose",
"-f",
COMPOSE_FILE_LIST = {
COMPOSE_FILE,
"-f",
SAMPLE_PROJECT_COMPOSE_FILE,
"-f",
INTEGRATION_COMPOSE_FILE,
"-f",
"docker/docker-compose.integration-mariadb.yml",
"-f",
"docker/docker-compose.integration-mongodb.yml",
"-f",
"docker/docker-compose.integration-mysql.yml",
"-f",
"docker/docker-compose.integration-postgres.yml",
"-f",
"docker/docker-compose.integration-mssql.yml",
"down",
"--remove-orphans",
)
COMPOSE_DOWN_VOLUMES = COMPOSE_DOWN + ("--volumes",)

# TODO: this doesn't actually work to teardown active containers. To see this:
# 1) Run `nox -s "fides_env(test) -- keep_alive`
# 2) Run `nox -s teardown`
# 3) Note that the docker compose project is still up & running.
# ...so, either get this to work as expected, or add a new session to teardown fides_env
}

# NOTE: The SAMPLE_PROJECT_COMPOSE_FILE expects to be run from it's normal
ThomasLaPiana marked this conversation as resolved.
Show resolved Hide resolved
# working directory, so when we reference it from the root directory we'll get
Expand All @@ -46,9 +27,6 @@
# To workaround this, we need to set the ENV variable below to point to *any*
# valid file to keep 'docker compose' happy. Sorry for the fragility of this -
# see comment in src/fides/data/sample_project/docker-compose.yml for details!
COMPOSE_DOWN_ENV = {
"FIDES_DEPLOY_ENV_FILE": ".env",
}


@nox.session()
Expand All @@ -63,20 +41,36 @@ def clean(session: nox.Session) -> None:
Clean up docker containers, remove orphans, remove volumes
and prune images related to this project.
"""
clean_command = (*COMPOSE_DOWN, "--volumes", "--rmi", "all")
session.run(*clean_command, external=True, env=COMPOSE_DOWN_ENV)
teardown(session, volumes=True, images=True)
session.run("docker", "system", "prune", "--force", "--all", external=True)
print("Clean Complete!")


@nox.session()
def teardown(session: nox.Session) -> None:
"""Tear down the docker dev environment."""
if "volumes" in session.posargs:
session.run(*COMPOSE_DOWN_VOLUMES, external=True, env=COMPOSE_DOWN_ENV)
else:
session.run(*COMPOSE_DOWN, external=True, env=COMPOSE_DOWN_ENV)
print("Teardown complete")
def teardown(session: nox.Session, volumes: bool = False, images: bool = False) -> None:
"""Tear down all docker environments."""
for compose_file in COMPOSE_FILE_LIST:
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh yeah, I prefer this. Good idea.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it also showed me that one of those compose files doesn't seem to be valid, I'll double check on it

teardown_command = (
"docker",
"compose",
"-f",
compose_file,
"down",
"--remove-orphans",
)

if volumes or "volumes" in session.posargs:
teardown_command = (*teardown_command, "--volumes")

if images:
teardown_command = (*teardown_command, "--rmi", "all")

try:
session.run(*teardown_command, external=True)
except nox.command.CommandFailed:
session.warn(f"Teardown failed: '{teardown_command}'")

session.log("Teardown complete")


def install_requirements(session: nox.Session) -> None:
Expand Down