diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index 39d1c204da..5e0a21a96b 100644 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -2,8 +2,6 @@ name: Code Checks on: pull_request: - branches: - - main paths-ignore: - "**.md" diff --git a/.github/workflows/frontend_pr_checks.yml b/.github/workflows/frontend_pr_checks.yml index aadcf825e6..76f4c1cf3f 100644 --- a/.github/workflows/frontend_pr_checks.yml +++ b/.github/workflows/frontend_pr_checks.yml @@ -2,8 +2,6 @@ name: Frontend PR Checks on: pull_request: - branches: - - main paths: - "clients/**" - ".github/workflows/frontend_pr_checks.yml" diff --git a/.github/workflows/publish_package.yaml b/.github/workflows/publish_package.yaml index c1118eb6d2..4303ae55c7 100644 --- a/.github/workflows/publish_package.yaml +++ b/.github/workflows/publish_package.yaml @@ -38,8 +38,15 @@ jobs: - name: Install Twine and wheel run: pip install twine wheel - - name: Build the package - run: python setup.py sdist bdist_wheel + # The git reset is required here because the build modifies + # egg-info and the wheel becomes a dirty version + - name: Build the sdist + run: | + python setup.py sdist + git reset --hard + + - name: Build the wheel + run: python setup.py bdist_wheel - name: Upload to test pypi run: twine upload --repository testpypi dist/* @@ -53,7 +60,6 @@ jobs: if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo ::set-output name=match::true fi - - name: Upload to pypi if: steps.check-tag.outputs.match == 'true' run: twine upload dist/* diff --git a/.gitignore b/.gitignore index 85520d2b05..157fe84d09 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ -docs/fides/docs/api/openapi.json -docs/fides/docs/schemas/config_schema.json +# frontend ui-build/ ## generic files to ignore @@ -20,6 +19,8 @@ tmp/* # docs docs/fides/site/ +docs/fides/docs/api/openapi.json +docs/fides/docs/schemas/config_schema.json # python specific *.pyc diff --git a/CHANGELOG.md b/CHANGELOG.md index 7105fa2595..a6b93ba3fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,29 @@ The types of changes are: ### Changed +* The following environment variables have been deprecated, and replaced with the new environment variable names indicated below. To avoid breaking existing workflows, the deprecated variables are still respected in v1.8.1. They will be removed in a future release. + * `FIDESCTL__API__DATABASE_HOST` --> `FIDESCTL__DATABASE__SERVER` + * `FIDESCTL__API__DATABASE_NAME` --> `FIDESCTL__DATABASE__DB` + * `FIDESCTL__API__DATABASE_PASSWORD` --> `FIDESCTL__DATABASE__PASSWORD` + * `FIDESCTL__API__DATABASE_PORT` --> `FIDESCTL__DATABASE__PORT` + * `FIDESCTL__API__DATABASE_TEST_DATABASE_NAME` --> `FIDESCTL__DATABASE__TEST_DB` + * `FIDESCTL__API__DATABASE_USER` --> `FIDESCTL__DATABASE__USER` + +### Developer Experience + +* The included `docker-compose.yml` no longer references outdated ENV variables [#964](https://github.com/ethyca/fides/pull/964) + +### Docs + +* Minor release documentation now reflects the desired patch release process [#955](https://github.com/ethyca/fides/pull/955) +* Updated references to ENV variables [#964](https://github.com/ethyca/fides/pull/964) + +### Fixed + +* Deprecated config options will continue to be respected when set via environment variables [#965](https://github.com/ethyca/fides/pull/965) +* The git cache is rebuilt within the Docker container [#962](https://github.com/ethyca/fides/pull/962) +* The `wheel` pypi build no longer has a dirty version tag [#962](https://github.com/ethyca/fides/pull/962) + ## [1.8.0](https://github.com/ethyca/fides/compare/1.7.1...1.8.0) - 2022-08-04 ### Added @@ -67,8 +90,6 @@ The types of changes are: * Remove `API_PREFIX` from fidesctl/core/utils.py and change references to `API_PREFIX` in fidesctl/api/reoutes/util.py [922](https://github.com/ethyca/fides/pull/922) -### Docs - ### Fixed * Dataset field columns show all columns by default in the UI [#898](https://github.com/ethyca/fides/pull/898) diff --git a/Dockerfile b/Dockerfile index e0ee598176..77a3b9fb3c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -83,6 +83,10 @@ RUN pip install -r optional-requirements.txt COPY . /fides WORKDIR /fides +# Reset the busted git cache +RUN git rm --cached -r . +RUN git reset --hard + # Immediately flush to stdout, globally ENV PYTHONUNBUFFERED=TRUE @@ -99,7 +103,7 @@ CMD ["fidesctl", "webserver"] FROM builder as dev # Install fidesctl as a symlink -RUN pip install -e ".[all,mssql]" +RUN pip install --no-deps -e ".[all,mssql]" ################################## ## Production Application Setup ## @@ -112,4 +116,4 @@ COPY --from=frontend /fides/clients/admin-ui/out/ /fides/src/fidesctl/ui-build/s # Install without a symlink RUN python setup.py bdist_wheel -RUN pip install dist/fidesctl-*.whl +RUN pip install --no-deps dist/fidesctl-*.whl diff --git a/docker-compose.yml b/docker-compose.yml index 9e15d7522f..d79c5f308f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,12 +26,12 @@ services: target: /fides read_only: False environment: - FIDESCTL_TEST_MODE: "True" FIDES__CONFIG_PATH: "/fides/.fides/" + FIDESCTL__CLI__ANALYTICS_ID: ${FIDESCTL__CLI__ANALYTICS_ID} FIDESCTL__CLI__SERVER_HOST: "fidesctl" FIDESCTL__CLI__SERVER_PORT: "8080" - FIDESCTL__CLI__ANALYTICS_ID: ${FIDESCTL__CLI__ANALYTICS_ID} - FIDESCTL__API__DATABASE_HOST: "fidesctl-db" + FIDESCTL__DATABASE__SERVER: "fidesctl-db" + FIDESCTL_TEST_MODE: "True" fidesctl-ui: image: ethyca/fidesctl:local-ui diff --git a/docs/fides/docs/development/releases.md b/docs/fides/docs/development/releases.md index 5fb4c27cfa..ffe22ea813 100644 --- a/docs/fides/docs/development/releases.md +++ b/docs/fides/docs/development/releases.md @@ -26,7 +26,8 @@ Fidesctl uses continuous delivery with a single `main` branch. All code changes We use GitHub’s `release` feature to tag releases that then get automatically deployed to DockerHub & PyPi via GitHub Actions pipelines. We also use a `CHANGELOG.md` to make sure that our users are never surprised about an upcoming change and can plan upgrades accordingly. The release steps are as follows: -### Major +### Major and Minor + 1. Open a PR that is titled the version of the release (i.e. `1.6.0`) * Rename the `Unreleased` section of `CHANGELOG.md` to the new version number and put a date next to it * Update the `compare` links for both the new version and for the new `Unreleased` section @@ -38,21 +39,131 @@ We use GitHub’s `release` feature to tag releases that then get automatically 1. Auto-populate the release notes 1. Publish the release -### Minor -1. Create a new branch from the latest tagged release being patched (i.e. `1.6.0`) -1. Use `git cherry-pick ` to incorporate specific changes required for the minor release +### Patch + +It may be desired for a patch release to contain only select commits to the `main` branch since the last major or minor release. To create a release with only the desired changes, follow the steps below: + +1. Checkout the most recent release's tag + 1. To fetch the most recent tag's name, run: + + ```sh + # fides on main + git describe --abbrev=0 --tags + + #=> 1.2.3 + ``` + + 1. To checkout the most recent tag, run: + + ```sh + #fides on main + git checkout 1.2.3 + + #=> Note: switching to '1.2.3'. + # + # You are in 'detached HEAD' state. You can look around, make experimental + # changes and commit them, and you can discard any commits you make in this + # state without impacting any branches by switching back to a branch. + # + # If you want to create a new branch to retain commits you create, you may + # do so (now or later) by using -c with the switch command. Example: + # + # git switch -c + # + # Or undo this operation with: + # + # git switch - + # + # Turn off this advice by setting config variable advice.detachedHead to false + # + # HEAD is now at 0123abcd Commit Message + ``` + + !!! tip + This can be combined into a single command: + + ```sh + # fides on main + git checkout $(git describe --abbrev=0 --tags) + + #=> Note: switching to '1.2.3'. + # + # You are in 'detached HEAD' state. You can look around, make experimental + # changes and commit them, and you can discard any commits you make in this + # state without impacting any branches by switching back to a branch. + # + # If you want to create a new branch to retain commits you create, you may + # do so (now or later) by using -c with the switch command. Example: + # + # git switch -c + # + # Or undo this operation with: + # + # git switch - + # + # Turn off this advice by setting config variable advice.detachedHead to false + # + # HEAD is now at 0123abcd Commit Message + ``` + +1. Create a new branch from the `HEAD` commit of the most recent release's tag, called `release-v` + + ```sh + # fides on tags/1.2.3 + git checkout -b release-v1.2.4 + + #=> Switched to a new branch 'release-v1.2.4' + ``` + +1. If the changes to be included in the patch release are contained in one or more unmerged pull requests, change the base branch of the pull request(s) to the release branch created in the previous step +1. Once approved, merge the pull request(s) into the release branch +1. Create a new branch off of the release branch by running: + + ```sh + # fides on release-v1.2.4 + git checkout -b prepare-release-v1.2.4 + + #=> Switched to a new branch 'prepare-release-v1.2.4' + ``` + +1. **Optional:** Incorporate any additional specific changes required for the patch release by running: + + ```sh + # fides on prepare-release-v1.2.4 + git cherry-pick ... + ``` + 1. Copy the `Unreleased` section of `CHANGELOG.md` and paste above the release being patched - * Rename `Unreleased` to the new version number and put a date next to it - * Cut and paste the changes documented that are now included in the minor release in the correct section -1. Open a PR that is titled the version of the release (i.e. `1.6.1`), add the `do not merge` label -1. Due to potential merge conflicts when opening against main, checks may need to be run manually -1. Create a new release, using the branch created in step 1 -1. Add the new version as the tag (i.e. `1.6.1`) -1. Make the title the version with a `v` in front of it (i.e. `v1.6.1`) -1. Add a link to the `CHANGELOG.md` (the changes are not merged yet but can be planned for) -1. Auto-populate the release notes + 1. Rename `Unreleased` to the new version number and put a date next to it + 1. Cut and paste the documented changes that are now included in the patch release to the correct section + 1. Commit these changes +1. Open a pull request to incorporate any cherry-picked commits and the `CHANGELOG.md` changes into the release branch + 1. Set the base branch of this pull request to the release branch + 1. Once approved, merge the pull request into the release branch +1. Create a new release from the release branch + 1. Add the new version as the tag (i.e. `1.2.4`) + 1. Title the release with the version number, prefixed with a `v` (i.e. `v1.2.4`) + 1. Add a link to the `CHANGELOG.md` + 1. Auto-populate the release notes 1. Publish the release -1. Create a new branch from `main` -1. Copy the changes from `CHANGELOG.md` for the minor release into the new branch -1. Open a separate PR for the `CHANGELOG.md` updates for the minor release, once approved merge the PR -1. Close the original PR made for the release (labeled `do not merge`) without merging +1. Merge the new release tag into `main` + + !!! warning + Pushing commits (including merge commits) to the `main` branch requires admin-level repository permissions. + + 1. Checkout the `main` branch, and update the local repository: + + ```sh + git checkout main + #=> Switched to branch 'main'... + + git pull + ``` + + 1. Merge the new release tag into `main`: + + ```sh + git merge tags/1.2.4 + + ``` + 1. Handle any merge conflicts, and push to the remote `main` branch diff --git a/docs/fides/docs/quickstart/docker.md b/docs/fides/docs/quickstart/docker.md index 7b322ea087..bcea1feb14 100644 --- a/docs/fides/docs/quickstart/docker.md +++ b/docs/fides/docs/quickstart/docker.md @@ -12,7 +12,7 @@ Docker and Docker-Compose are the only requirements here. ## Docker Setup -This is a reference file that you can copy/paste into a local `docker-compose.yml` file, which should be created in your project's root folder. It will create a database and spin up the fidesctl webserver. +This is a reference file that you can copy/paste into a local `docker-compose.yml` file, which should be created in your project's root folder. It will create a database and spin up the fidesctl webserver. Make sure that you don't have anything else running on port `5432` or `8080` before using this file. @@ -34,10 +34,10 @@ services: ports: - "8080:8080" environment: - FIDESCTL_TEST_MODE: "True" FIDESCTL__CLI__SERVER_HOST: "fidesctl" FIDESCTL__CLI__SERVER_PORT: "8080" - FIDESCTL__API__DATABASE_HOST: "fidesctl-db" + FIDESCTL__DATABASE__SERVER: "fidesctl-db" + FIDESCTL_TEST_MODE: "True" volumes: - type: bind source: . @@ -86,7 +86,7 @@ Now you can start interacting with your installation. Run the following commands ``` 1. `fidesctl status` -> This confirms that your `fidesctl` CLI can reach the server and everything is ready to go! - + Once your installation is running, you can use `fidesctl` from the shell to get a list of all the possible [CLI commands](../cli.md). diff --git a/docs/fides/docs/tutorial/add.md b/docs/fides/docs/tutorial/add.md index dacf1f70c8..991323dd51 100644 --- a/docs/fides/docs/tutorial/add.md +++ b/docs/fides/docs/tutorial/add.md @@ -57,7 +57,11 @@ fidesctl: ports: - "8080:8080" environment: - - FIDESCTL__API__DATABASE_URL=postgres:postgres@db:5432/fidesctl + FIDESCTL__DATABASE__DB: "fidesctl" + FIDESCTL__DATABASE__PASSWORD: "postgres" + FIDESCTL__DATABASE__PORT: 5432 + FIDESCTL__DATABASE__SERVER: "db" + FIDESCTL__DATABASE__USER: "postgres" ``` > See [the fidesctl installation guide](../installation/installation.md) for a more detailed fidesctl server setup walkthrough, and [the `docker-compose` documentation](https://docs.docker.com/compose/compose-file/) for an explanation of the above configuration options. diff --git a/src/fidesctl/ctl/core/config/__init__.py b/src/fidesctl/ctl/core/config/__init__.py index c95a25b924..d37a189987 100644 --- a/src/fidesctl/ctl/core/config/__init__.py +++ b/src/fidesctl/ctl/core/config/__init__.py @@ -3,6 +3,8 @@ config sections into a single config. """ from functools import lru_cache +from os import environ +from re import compile as regex from typing import Dict, MutableMapping import toml @@ -49,6 +51,30 @@ def handle_deprecated_fields(settings: MutableMapping) -> MutableMapping: return settings +def handle_deprecated_env_variables(settings: MutableMapping) -> MutableMapping: + """ + Custom logic for handling deprecated ENV variable configuration. + """ + + deprecated_env_vars = regex(r"FIDESCTL__API__(\w+)") + + for key, val in environ.items(): + match = deprecated_env_vars.search(key) + if match: + setting = match.group(1).lower() + setting = setting[setting.startswith("database_") and len("database_") :] + if setting == "host": + setting = "server" + if setting == "name": + setting = "db" + if setting == "test_database_name": + setting = "test_db" + + settings["database"][setting] = val + + return settings + + @lru_cache(maxsize=1) def get_config(config_path_override: str = "") -> FidesctlConfig: """ @@ -69,6 +95,10 @@ def get_config(config_path_override: str = "") -> FidesctlConfig: # credentials specific logic for populating environment variable configs. # this is done to allow overrides without hard typed pydantic models settings = handle_deprecated_fields(settings) + + # Called after `handle_deprecated_fields` to ensure ENV vars are respected + settings = handle_deprecated_env_variables(settings) + config_environment_dict = settings.get("credentials", dict()) settings["credentials"] = merge_credentials_environment( credentials_dict=config_environment_dict diff --git a/tests/ctl/core/test_config.py b/tests/ctl/core/test_config.py index 3566753872..e2cb4be367 100644 --- a/tests/ctl/core/test_config.py +++ b/tests/ctl/core/test_config.py @@ -34,7 +34,7 @@ def test_get_config(test_config_path: str) -> None: clear=True, ) @pytest.mark.unit -def test_get_deprecated_api_config(test_deprecated_config_path: str) -> None: +def test_get_deprecated_api_config_from_file(test_deprecated_config_path: str) -> None: """ Test that the deprecated API config values get written as database values. """ @@ -46,6 +46,34 @@ def test_get_deprecated_api_config(test_deprecated_config_path: str) -> None: assert config.database.test_db == "fidesctl_test_deprecated" +@patch.dict( + os.environ, + { + "FIDESCTL__API__DATABASE_HOST": "test_host", + "FIDESCTL__API__DATABASE_NAME": "test_db_name", + "FIDESCTL__API__DATABASE_PASSWORD": "test_password", + "FIDESCTL__API__DATABASE_PORT": "1234", + "FIDESCTL__API__DATABASE_TEST_DATABASE_NAME": "test_test_db_name", + "FIDESCTL__API__DATABASE_USER": "phil_rules", + }, + clear=True, +) +@pytest.mark.unit +def test_get_deprecated_api_config_from_env(test_config_path: str) -> None: + """ + Test that the deprecated API config values get written as database values, + when set via ENV variables. + """ + + config = get_config(test_config_path) + assert config.database.db == "test_db_name" + assert config.database.password == "test_password" + assert config.database.port == "1234" + assert config.database.server == "test_host" + assert config.database.test_db == "test_test_db_name" + assert config.database.user == "phil_rules" + + @patch.dict( os.environ, {"FIDESCTL_CONFIG_PATH": ""},