diff --git a/.fides/systems.yml b/.fides/systems.yml index e5d36896f2..429873ca8f 100644 --- a/.fides/systems.yml +++ b/.fides/systems.yml @@ -28,7 +28,7 @@ system: - fides_db # System Info - - fides_key: privacy_request_fullfillment + - fides_key: privacy_request_fulfillment name: Fides Privacy Request Fulfillment organization_fides_key: default_organization description: Privacy request fufillment. diff --git a/CHANGELOG.md b/CHANGELOG.md index 56cfd6c1c5..8d66dfed1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ The types of changes are: * Add warning to 'fides deploy' when installed outside of a virtual environment [#2641](https://github.com/ethyca/fides/pull/2641) * Redesigned the default/init config file to be auto-documented. Also updates the `fides init` logic and analytics consent logic [#2694](https://github.com/ethyca/fides/pull/2694) * Change how config creation/import is handled across the application [#2622](https://github.com/ethyca/fides/pull/2622) +* Update the CLI aesthetics & docstrings [#2703](https://github.com/ethyca/fides/pull/2703) * Updates Roles->Scopes Mapping [#2744](https://github.com/ethyca/fides/pull/2744) * Return user scopes as an enum, as well as total scopes [#2741](https://github.com/ethyca/fides/pull/2741) diff --git a/requirements.txt b/requirements.txt index d4be59d3b8..ad520d2842 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ APScheduler==3.9.1.post1 asyncpg==0.25.0 boto3==1.26.1 celery[pytest]==5.2.7 -click==8.1.3 colorama>=0.4.3 cryptography==38.0.3 dask==2022.9.2 @@ -35,6 +34,7 @@ PyMySQL==1.0.2 python-jose[cryptography]==3.3.0 pyyaml>=5,<6 redis==3.5.3 +rich-click==1.6.1 sendgrid==6.9.7 slowapi==0.1.5 snowflake-sqlalchemy==1.4.3 diff --git a/src/fides/cli/__init__.py b/src/fides/cli/__init__.py index efe8d2714f..fe02a594fc 100644 --- a/src/fides/cli/__init__.py +++ b/src/fides/cli/__init__.py @@ -1,14 +1,17 @@ -"""Contains the groups and setup for the CLI.""" +""" +Entrypoint for the Fides command-line. +""" from importlib.metadata import version from platform import system -import click +import rich_click as click from fideslog.sdk.python.client import AnalyticsClient import fides from fides.cli.utils import check_server from fides.core.config import get_config +from . import cli_formatting from .commands.annotate import annotate from .commands.core import evaluate, parse, pull, push from .commands.crud import delete, get_resource, list_resources @@ -58,24 +61,28 @@ "--config-path", "-f", "config_path", - default="", - help="Path to a configuration file. Use 'fides view-config' to print the config. Not compatible with the 'fides webserver' subcommand.", + show_default=True, + help="Path to a Fides config file. _Defaults to `.fides/fides.toml`._", ) @click.option( "--local", is_flag=True, - help="Run in 'local_mode'. This mode doesn't make API calls and can be used without the API server/database.", + help="Run in `local_mode`. This mode doesn't make API calls and can be used without the API server/database.", ) @click.pass_context def cli(ctx: click.Context, config_path: str, local: bool) -> None: """ - The parent group for the Fides CLI. + __Command-line tool for the Fides privacy engineering platform.__ + + --- + + _Note: The common MANIFESTS_DIR argument _always_ defaults to ".fides/" if not specified._ """ ctx.ensure_object(dict) config = get_config(config_path, verbose=True) - # Dyanmically add commands to the CLI + # Dynamically add commands to the CLI cli.commands = LOCAL_COMMAND_DICT if not (local or config.cli.local_mode): diff --git a/src/fides/cli/cli_formatting.py b/src/fides/cli/cli_formatting.py new file mode 100644 index 0000000000..1da85f45f2 --- /dev/null +++ b/src/fides/cli/cli_formatting.py @@ -0,0 +1,114 @@ +""" +This is a configuration file for `rich_click`, used to customize the +visual aesthetic and output of the CLI. + +This is the list of all documented configuration options for `rich_click`. + +You can set a style attribute by adding one or more of the following words: + +- "bold" or "b" for bold text. +- "blink" for text that flashes (use this one sparingly). +- "blink2" for text that flashes rapidly (not supported by most terminals). +- "conceal" for concealed text (not supported by most terminals). +- "italic" or "i" for italic text (not supported on Windows). +- "reverse" or "r" for text with foreground and background colors reversed. +- "strike" or "s" for text with a line through it. +- "underline" or "u" for underlined text. +- "underline2" or "uu" for doubly underlined text. +- "frame" for framed text. +- "encircle" for encircled text. +- "overline" or "o" for overlined text. + +The list of valid colors is here: +- https://rich.readthedocs.io/en/stable/appendix/colors.html +""" + +from rich_click import rich_click + +# Default styles +rich_click.STYLE_OPTION = "bold #fca311" +rich_click.STYLE_SWITCH = "bold #fca311" +rich_click.STYLE_ARGUMENT = "bold #00ff5f" +rich_click.STYLE_METAVAR = "bold #8700af" +rich_click.STYLE_METAVAR_APPEND = "dim yellow" +rich_click.STYLE_METAVAR_SEPARATOR = "dim" +rich_click.STYLE_HEADER_TEXT = "" +rich_click.STYLE_FOOTER_TEXT = "" +rich_click.STYLE_USAGE = "yellow" +rich_click.STYLE_USAGE_COMMAND = "bold" +rich_click.STYLE_DEPRECATED = "red" +rich_click.STYLE_HELPTEXT_FIRST_LINE = "" +rich_click.STYLE_HELPTEXT = "" +rich_click.STYLE_OPTION_HELP = "" +rich_click.STYLE_OPTION_DEFAULT = "bold #8700af" +rich_click.STYLE_OPTION_ENVVAR = "dim yellow" +rich_click.STYLE_REQUIRED_SHORT = "red" +rich_click.STYLE_REQUIRED_LONG = "dim red" +rich_click.STYLE_OPTIONS_PANEL_BORDER = "dim" +rich_click.ALIGN_OPTIONS_PANEL = "left" +rich_click.STYLE_OPTIONS_TABLE_SHOW_LINES = False +rich_click.STYLE_OPTIONS_TABLE_LEADING = 0 +rich_click.STYLE_OPTIONS_TABLE_PAD_EDGE = False +rich_click.STYLE_OPTIONS_TABLE_PADDING = (0, 1) +rich_click.STYLE_OPTIONS_TABLE_BOX = "" +rich_click.STYLE_OPTIONS_TABLE_ROW_STYLES = None +rich_click.STYLE_OPTIONS_TABLE_BORDER_STYLE = None +rich_click.STYLE_COMMANDS_PANEL_BORDER = "dim" +rich_click.ALIGN_COMMANDS_PANEL = "left" +rich_click.STYLE_COMMANDS_TABLE_SHOW_LINES = False +rich_click.STYLE_COMMANDS_TABLE_LEADING = 0 +rich_click.STYLE_COMMANDS_TABLE_PAD_EDGE = False +rich_click.STYLE_COMMANDS_TABLE_PADDING = (0, 1) +rich_click.STYLE_COMMANDS_TABLE_BOX = "" +rich_click.STYLE_COMMANDS_TABLE_ROW_STYLES = None +rich_click.STYLE_COMMANDS_TABLE_BORDER_STYLE = None +rich_click.STYLE_ERRORS_PANEL_BORDER = "red" +rich_click.ALIGN_ERRORS_PANEL = "left" +rich_click.STYLE_ERRORS_SUGGESTION = "dim" +rich_click.STYLE_ABORTED = "red" +rich_click.MAX_WIDTH = None # Set to an int to limit to that many characters +rich_click.COLOR_SYSTEM = "auto" # Set to None to disable colors + +# Fixed strings +rich_click.HEADER_TEXT = None +rich_click.FOOTER_TEXT = None +rich_click.DEPRECATED_STRING = "(Deprecated) " +rich_click.DEFAULT_STRING = "[default: {}]" +rich_click.ENVVAR_STRING = "[env var: {}]" +rich_click.REQUIRED_SHORT_STRING = "*" +rich_click.REQUIRED_LONG_STRING = "[required]" +rich_click.RANGE_STRING = " [{}]" +rich_click.APPEND_METAVARS_HELP_STRING = "({})" +rich_click.ARGUMENTS_PANEL_TITLE = "Arguments" +rich_click.OPTIONS_PANEL_TITLE = "Options" +rich_click.COMMANDS_PANEL_TITLE = "Commands" +rich_click.ERRORS_PANEL_TITLE = "Error" +rich_click.ERRORS_SUGGESTION = ( + None # Default: Try 'cmd -h' for help. Set to False to disable. +) +rich_click.ERRORS_EPILOGUE = None +rich_click.ABORTED_TEXT = "Aborted." + +# Behaviours +rich_click.SHOW_ARGUMENTS = True # Show positional arguments +rich_click.SHOW_METAVARS_COLUMN = ( + True # Show a column with the option metavar (eg. INTEGER) +) +rich_click.APPEND_METAVARS_HELP = ( + False # Append metavar (eg. [TEXT]) after the help text +) +rich_click.GROUP_ARGUMENTS_OPTIONS = ( + False # Show arguments with options instead of in own panel +) +rich_click.USE_MARKDOWN = True # Parse help strings as markdown +rich_click.USE_MARKDOWN_EMOJI = True # Parse emoji codes in markdown :smile: +rich_click.USE_RICH_MARKUP = ( + False # Parse help strings for rich markup (eg. [red]my text[/]) +) +rich_click.COMMAND_GROUPS = {} # Define sorted groups of panels to display subcommands +rich_click.OPTION_GROUPS = ( + {} +) # Define sorted groups of panels to display options and arguments +rich_click.USE_CLICK_SHORT_HELP = ( + False # Use click's default function to truncate help text +) diff --git a/src/fides/cli/commands/annotate.py b/src/fides/cli/commands/annotate.py index 4c6e390bee..95460b82d4 100644 --- a/src/fides/cli/commands/annotate.py +++ b/src/fides/cli/commands/annotate.py @@ -1,6 +1,6 @@ """Contains the annotate group of CLI commands for fides.""" -import click +import rich_click as click from fides.cli.utils import with_analytics from fides.core import annotate_dataset as _annotate_dataset @@ -10,7 +10,7 @@ @click.pass_context def annotate(ctx: click.Context) -> None: """ - Annotate fides resource types + Interactively annotate Fides resources. """ @@ -21,21 +21,21 @@ def annotate(ctx: click.Context) -> None: "-a", "--all-members", is_flag=True, - help="Annotate all dataset members, not just fields", + help="Annotate all parts of the dataset including schemas and tables.", ) @click.option( "-v", "--validate", is_flag=True, default=False, - help="Strictly validate annotation inputs.", + help="Validate annotation inputs.", ) @with_analytics def annotate_dataset( ctx: click.Context, input_filename: str, all_members: bool, validate: bool ) -> None: """ - Guided flow for annotating datasets. The dataset file will be edited in-place. + Interactively annotate a dataset file in-place. """ config = ctx.obj["CONFIG"] _annotate_dataset.annotate_dataset( diff --git a/src/fides/cli/commands/core.py b/src/fides/cli/commands/core.py index e929322fc7..7c80437d6e 100644 --- a/src/fides/cli/commands/core.py +++ b/src/fides/cli/commands/core.py @@ -1,14 +1,9 @@ """Contains all of the core CLI commands for fides.""" from typing import Optional -import click +import rich_click as click -from fides.cli.options import ( - dry_flag, - fides_key_option, - manifests_dir_argument, - verbose_flag, -) +from fides.cli.options import dry_flag, manifests_dir_argument, verbose_flag from fides.cli.utils import pretty_echo, print_divider, with_analytics from fides.core import audit as _audit from fides.core import evaluate as _evaluate @@ -24,13 +19,13 @@ @click.option( "--diff", is_flag=True, - help="Include any changes between server and local resources in the command output", + help="Print any diffs between the local & server objects", ) @manifests_dir_argument @with_analytics def push(ctx: click.Context, dry: bool, diff: bool, manifests_dir: str) -> None: """ - Validate local manifest files and persist any changes via the API server. + Parse local manifest files and upload them to the server. """ config = ctx.obj["CONFIG"] @@ -47,19 +42,28 @@ def push(ctx: click.Context, dry: bool, diff: bool, manifests_dir: str) -> None: @click.command() @click.pass_context @manifests_dir_argument -@fides_key_option +@click.option( + "--fides-key", + "-k", + help="The fides_key of a specific policy to evaluate.", + default="", +) @click.option( "-m", "--message", - help="A message that you can supply to describe the context of this evaluation.", + help="Describe the context of this evaluation.", ) @click.option( "-a", "--audit", is_flag=True, - help="Raise errors if resources are missing attributes required for building a data map.", + help="Validate that the objects in this evaluation produce a valid data map.", +) +@click.option( + "--dry", + is_flag=True, + help="Do not upload objects or results to the Fides webserver.", ) -@dry_flag @with_analytics def evaluate( ctx: click.Context, @@ -70,12 +74,7 @@ def evaluate( dry: bool, ) -> None: """ - Compare your System's Privacy Declarations with your Organization's Policy Rules. - - All local resources are applied to the server before evaluation. - - If your policy evaluation fails, it is expected that you will need to - either adjust your Privacy Declarations, Datasets, or Policies before trying again. + Evaluate System-level Privacy Declarations against Organization-level Policy Rules. """ config = ctx.obj["CONFIG"] @@ -127,10 +126,7 @@ def evaluate( @with_analytics def parse(ctx: click.Context, manifests_dir: str, verbose: bool = False) -> None: """ - Reads the resource files that are stored in MANIFESTS_DIR and its subdirectories to verify - the validity of all manifest files. - - If the taxonomy is invalid, this command prints the error messages and triggers a non-zero exit code. + Parse all Fides objects located in the supplied directory. """ taxonomy = _parse.parse(manifests_dir=manifests_dir) if verbose: @@ -149,12 +145,7 @@ def parse(ctx: click.Context, manifests_dir: str, verbose: bool = False) -> None @with_analytics def pull(ctx: click.Context, manifests_dir: str, all_resources: Optional[str]) -> None: """ - Update local resource files by their fides_key to match their server versions. - - Alternatively, with the "--all" flag all resources from the server will be pulled - down into a local file. - - The pull is aborted if there are unstaged or untracked files in the manifests dir. + Update local resource files based on the state of the objects on the server. """ # Make the resources that are pulled configurable diff --git a/src/fides/cli/commands/crud.py b/src/fides/cli/commands/crud.py index a2826fbcbc..cdd0578c67 100644 --- a/src/fides/cli/commands/crud.py +++ b/src/fides/cli/commands/crud.py @@ -1,12 +1,12 @@ """Contains all of the CRUD-type CLI commands for fides.""" -import click +import rich_click as click import yaml from fides.cli.options import fides_key_argument, resource_type_argument from fides.cli.utils import handle_cli_response, print_divider, with_analytics from fides.core import api as _api from fides.core.api_helpers import get_server_resource, list_server_resources -from fides.core.utils import echo_green +from fides.core.utils import echo_green, echo_red @click.command() @@ -16,7 +16,7 @@ @with_analytics def delete(ctx: click.Context, resource_type: str, fides_key: str) -> None: """ - Delete a resource on the server. + Delete an object from the server. """ config = ctx.obj["CONFIG"] handle_cli_response( @@ -25,7 +25,11 @@ def delete(ctx: click.Context, resource_type: str, fides_key: str) -> None: resource_type=resource_type, resource_id=fides_key, headers=config.user.auth_header, - ) + ), + verbose=False, + ) + echo_green( + f"{resource_type.capitalize()} with fides_key '{fides_key}' successfully deleted." ) @@ -36,7 +40,7 @@ def delete(ctx: click.Context, resource_type: str, fides_key: str) -> None: @with_analytics def get_resource(ctx: click.Context, resource_type: str, fides_key: str) -> None: """ - View a resource from the server as a YAML object. + View an object from the server. """ config = ctx.obj["CONFIG"] resource = get_server_resource( @@ -54,9 +58,12 @@ def get_resource(ctx: click.Context, resource_type: str, fides_key: str) -> None @click.pass_context @resource_type_argument @with_analytics -def list_resources(ctx: click.Context, resource_type: str) -> None: +@click.option( + "--verbose", "-v", is_flag=True, help="Displays the entire object list as YAML." +) +def list_resources(ctx: click.Context, verbose: bool, resource_type: str) -> None: """ - Get a list of all resources of this type from the server and display them as YAML. + View all objects of a single type from the server. """ config = ctx.obj["CONFIG"] resources = list_server_resources( @@ -67,4 +74,14 @@ def list_resources(ctx: click.Context, resource_type: str) -> None: raw=True, ) print_divider() - echo_green(yaml.dump({resource_type: resources})) + if verbose: + echo_green(yaml.dump({resource_type: resources})) + else: + if resources: + sorted_fides_keys = sorted( + {resource["fides_key"] for resource in resources if resource} + ) + formatted_fides_keys = "\n ".join(sorted_fides_keys) + echo_green(f"{resource_type.capitalize()} list:\n {formatted_fides_keys}") + else: + echo_red(f"No {resource_type.capitalize()} resources found!") diff --git a/src/fides/cli/commands/db.py b/src/fides/cli/commands/db.py index 7f27a466a7..0d8d45faeb 100644 --- a/src/fides/cli/commands/db.py +++ b/src/fides/cli/commands/db.py @@ -1,5 +1,5 @@ """Contains the db group of the commands for fides.""" -import click +import rich_click as click from fides.cli.options import yes_flag from fides.cli.utils import handle_cli_response, with_analytics @@ -11,7 +11,7 @@ @click.pass_context def database(ctx: click.Context) -> None: """ - Database utility commands + Run actions against the application database. """ @@ -20,7 +20,7 @@ def database(ctx: click.Context) -> None: @with_analytics def db_init(ctx: click.Context) -> None: """ - Initialize the fides database. + Initialize the Fides database. """ config = ctx.obj["CONFIG"] handle_cli_response( @@ -38,7 +38,7 @@ def db_init(ctx: click.Context) -> None: @with_analytics def db_reset(ctx: click.Context, yes: bool) -> None: """ - Wipes all user-created data and resets the database back to its freshly initialized state. + Reset the database back to its initial state. """ config = ctx.obj["CONFIG"] if yes: diff --git a/src/fides/cli/commands/export.py b/src/fides/cli/commands/export.py index cc6daebb39..5d6879f944 100644 --- a/src/fides/cli/commands/export.py +++ b/src/fides/cli/commands/export.py @@ -1,5 +1,5 @@ """Contains the export group of CLI commands for fides.""" -import click +import rich_click as click from fides.cli.options import ( dry_flag, @@ -16,7 +16,7 @@ @click.pass_context def export(ctx: click.Context) -> None: """ - Export fides resource types + Export Fides data maps. """ @@ -31,7 +31,7 @@ def export_system( dry: bool, ) -> None: """ - Export a system in a data map format. + Export systems in a data map format. """ config = ctx.obj["CONFIG"] taxonomy = _parse.parse(manifests_dir) @@ -55,7 +55,7 @@ def export_dataset( dry: bool, ) -> None: """ - Export a dataset in a data map format. + Export datasets in a data map format. """ config = ctx.obj["CONFIG"] taxonomy = _parse.parse(manifests_dir) @@ -79,7 +79,7 @@ def export_organization( dry: bool, ) -> None: """ - Export an organization in a data map format. + Export organizations in a data map format. """ config = ctx.obj["CONFIG"] taxonomy = _parse.parse(manifests_dir) @@ -111,17 +111,9 @@ def export_datamap( csv: bool, ) -> None: """ - Export a formatted data map to excel using the fides template. + Export a data map using the standard Fides template. The data map is comprised of an Organization, Systems, and Datasets. - - The default organization is used, however a custom one can be - passed if required. - - A custom manifest directory can be provided for the output location. - - The csv flag can be used to output data as csv, while the dry - flag can be used to return data to the console instead. """ config = ctx.obj["CONFIG"] _export.export_datamap( diff --git a/src/fides/cli/commands/generate.py b/src/fides/cli/commands/generate.py index edb5036853..5cd8d64392 100644 --- a/src/fides/cli/commands/generate.py +++ b/src/fides/cli/commands/generate.py @@ -1,5 +1,5 @@ """Contains the generate group of CLI commands for fides.""" -import click +import rich_click as click from fides.cli.options import ( aws_access_key_id_option, @@ -27,7 +27,7 @@ @click.pass_context def generate(ctx: click.Context) -> None: """ - Generate fides resource types + Programmatically generate Fides objects. """ @@ -35,7 +35,7 @@ def generate(ctx: click.Context) -> None: @click.pass_context def generate_dataset(ctx: click.Context) -> None: """ - Generate fides Dataset resources + Generate Fides datasets. """ @@ -54,13 +54,7 @@ def generate_dataset_db( include_null: bool, ) -> None: """ - Connect to a database directly via a SQLAlchemy-style connection string and - generate a dataset manifest file that consists of every schema/table/field. - Connection string can be supplied as an option or a credentials reference - to fides config. - - This is a one-time operation that does not track the state of the database. - It will need to be run again if the database schema changes. + Generate a Fides dataset by walking a database and recording every schema/table/field. """ actual_connection_string = handle_database_credentials_options( fides_config=ctx.obj["CONFIG"], @@ -79,7 +73,7 @@ def generate_dataset_db( @click.pass_context def generate_dataset_gcp(ctx: click.Context) -> None: """ - Generate fides Dataset resources for Google Cloud Platform + Generate Fides datasets from Google Cloud Platform. """ @@ -100,13 +94,7 @@ def generate_dataset_bigquery( include_null: bool, ) -> None: """ - Connect to a BigQuery dataset directly via a SQLAlchemy connection and - generate a dataset manifest file that consists of every schema/table/field. - A path to a google authorization keyfile can be supplied as an option, or a - credentials reference to fides config. - - This is a one-time operation that does not track the state of the dataset. - It will need to be run again if the dataset schema changes. + Generate a dataset object from BigQuery using a SQLAlchemy connection string. """ bigquery_config = handle_bigquery_config_options( @@ -127,7 +115,7 @@ def generate_dataset_bigquery( @click.pass_context def generate_system(ctx: click.Context) -> None: """ - Generate fides System resources + Generate Fides systems. """ @@ -150,13 +138,8 @@ def generate_system_okta( org_key: str, ) -> None: """ - Generates systems for your Okta applications. Connect to an Okta admin - account by providing an organization url and auth token or a credentials - reference to fides config. Auth token and organization url can also - be supplied by setting environment variables as defined by the okta python sdk. - - This is a one-time operation that does not track the state of the okta resources. - It will need to be run again if the tracked resources change. + Generates systems from your Okta applications. Connects via + an Okta admin account. """ config = ctx.obj["CONFIG"] okta_config = handle_okta_credentials_options( @@ -199,12 +182,6 @@ def generate_system_aws( """ Connect to an aws account and generate a system manifest file that consists of every tracked resource. - Credentials can be supplied as options, a credentials - reference to fides config, or boto3 environment configuration. - Tracked resources: [Redshift, RDS, DynamoDb, S3] - - This is a one-time operation that does not track the state of the aws resources. - It will need to be run again if the tracked resources change. """ config = ctx.obj["CONFIG"] aws_config = handle_aws_credentials_options( diff --git a/src/fides/cli/commands/scan.py b/src/fides/cli/commands/scan.py index 5fa89e3f84..9782069b01 100644 --- a/src/fides/cli/commands/scan.py +++ b/src/fides/cli/commands/scan.py @@ -1,6 +1,6 @@ """Contains the scan group of the commands for fides.""" -import click +import rich_click as click from fides.cli.options import ( aws_access_key_id_option, @@ -28,7 +28,7 @@ @click.pass_context def scan(ctx: click.Context) -> None: """ - Scan external resource coverage against fides resources + Scan and report on discrepancies between Fides resource files and real infrastructure. """ @@ -36,7 +36,7 @@ def scan(ctx: click.Context) -> None: @click.pass_context def scan_dataset(ctx: click.Context) -> None: """ - Scan fides Dataset resources + Scan and report on Fides Dataset resources. """ @@ -55,15 +55,10 @@ def scan_dataset_db( coverage_threshold: int, ) -> None: """ - Connect to a database directly via a SQLAlchemy-style connection string and - compare the database objects to existing datasets. Connection string can be - supplied as an option or a credentials reference to fides config. + Scan a database directly using a SQLAlchemy-style connection string. - If there are fields within the database that aren't listed and categorized - within one of the datasets, this counts as lacking coverage. - - Outputs missing fields and has a non-zero exit if coverage is - under the stated threshold. + _If there are fields within the database that aren't listed and categorized + within one of the datasets, this counts as lacking coverage._ """ config = ctx.obj["CONFIG"] actual_connection_string = handle_database_credentials_options( @@ -85,7 +80,7 @@ def scan_dataset_db( @click.pass_context def scan_system(ctx: click.Context) -> None: """ - Scan fides System resources + Scan and report on Fides System resources. """ @@ -108,14 +103,7 @@ def scan_system_okta( coverage_threshold: int, ) -> None: """ - Scans your existing systems and compares them to found Okta applications. - Connect to an Okta admin account by providing an organization url and - auth token or a credentials reference to fides config. Auth token and - organization url can also be supplied by setting environment variables - as defined by the okta python sdk. - - Outputs missing resources and has a non-zero exit if coverage is - under the stated threshold. + Scan an Okta account and compare applications with annotated Fides Systems. """ config = ctx.obj["CONFIG"] @@ -154,13 +142,9 @@ def scan_system_aws( coverage_threshold: int, ) -> None: """ - Connect to an aws account and compares tracked resources to existing systems. - Credentials can be supplied as options, a credentials reference to fides - config, or boto3 environment configuration. - Tracked resources: [Redshift, RDS, DynamoDb, S3] + Scan an AWS account and compare objects with annotated Fides Systems. - Outputs missing resources and has a non-zero exit if coverage is - under the stated threshold. + _Scannable resources: [Redshift, RDS, DynamoDb, S3]_ """ config = ctx.obj["CONFIG"] aws_config = handle_aws_credentials_options( diff --git a/src/fides/cli/commands/user.py b/src/fides/cli/commands/user.py index 773712e547..395b1b7759 100644 --- a/src/fides/cli/commands/user.py +++ b/src/fides/cli/commands/user.py @@ -1,5 +1,5 @@ """Contains the user command group for the fides CLI.""" -import click +import rich_click as click from fides.cli.options import ( first_name_option, @@ -28,9 +28,7 @@ def create( ctx: click.Context, username: str, password: str, first_name: str, last_name: str ) -> None: """ - Use credentials from the credentials file to create a new user. - - Gives full permissions to the new user. + Use the credentials file to create a new user. Gives full permissions to the new user. """ config = ctx.obj["CONFIG"] server_url = config.cli.server_url @@ -49,7 +47,7 @@ def create( @password_option def login(ctx: click.Context, username: str, password: str) -> None: """ - Use credentials to get a user access token and write it to a credentials file. + Generate a user access token and write it to a credentials file. """ config = ctx.obj["CONFIG"] server_url = config.cli.server_url @@ -59,7 +57,9 @@ def login(ctx: click.Context, username: str, password: str) -> None: @user.command(name="permissions") @click.pass_context def get_permissions(ctx: click.Context) -> None: - """List the directly-assigned scopes and roles available to the current user.""" + """ + List the directly-assigned scopes and roles available to the current user. + """ config = ctx.obj["CONFIG"] server_url = config.cli.server_url get_permissions_command(server_url=server_url) diff --git a/src/fides/cli/commands/util.py b/src/fides/cli/commands/util.py index 2c1acc2475..f0e10df194 100644 --- a/src/fides/cli/commands/util.py +++ b/src/fides/cli/commands/util.py @@ -2,7 +2,7 @@ from datetime import datetime, timezone from subprocess import CalledProcessError -import click +import rich_click as click import fides from fides.cli.utils import ( @@ -28,16 +28,14 @@ @click.command() @click.pass_context -@click.argument("fides_directory_location", default=".", type=click.Path(exists=True)) +@click.argument("fides_dir", default=".", type=click.Path(exists=True)) @click.option( "--opt-in", is_flag=True, help="Automatically opt-in to anonymous usage analytics." ) -def init(ctx: click.Context, fides_directory_location: str, opt_in: bool) -> None: +def init(ctx: click.Context, fides_dir: str, opt_in: bool) -> None: """ - Initializes a fides instance, creating the default directory (`.fides/`) and - the configuration file (`fides.toml`) if necessary. - - Additionally, requests the ability to respectfully collect anonymous usage data. + Initializes a Fides instance by creating the default directory and + configuration file if not present. """ executed_at = datetime.now(timezone.utc) @@ -47,7 +45,7 @@ def init(ctx: click.Context, fides_directory_location: str, opt_in: bool) -> Non click.echo("Initializing fides...") config, config_path = create_and_update_config_file( - config, fides_directory_location, opt_in=opt_in + config, fides_dir, opt_in=opt_in ) print_divider() @@ -61,7 +59,7 @@ def init(ctx: click.Context, fides_directory_location: str, opt_in: bool) -> Non @with_analytics def status(ctx: click.Context) -> None: """ - Sends a request to the fides API healthcheck endpoint and prints the response. + Check Fides server availability. """ config = ctx.obj["CONFIG"] cli_version = fides.__version__ @@ -78,7 +76,9 @@ def status(ctx: click.Context) -> None: @click.option("--port", "-p", type=int, default=8080) def webserver(ctx: click.Context, port: int = 8080) -> None: """ - Starts the fides API server using Uvicorn. + Start the Fides webserver. + + _Requires Redis and Postgres to be configured and running_ """ # This has to be here to avoid a circular dependency from fides.api.main import start_webserver @@ -91,7 +91,7 @@ def webserver(ctx: click.Context, port: int = 8080) -> None: @with_analytics def worker(ctx: click.Context) -> None: """ - Starts a celery worker. + Start a Celery worker for the Fides webserver. """ # This has to be here to avoid a circular dependency from fides.api.ops.worker import start_worker @@ -123,9 +123,11 @@ def deploy(ctx: click.Context) -> None: is_flag=True, help="Disable the initialization of the Fides CLI, to run in headless mode.", ) -def up(ctx: click.Context, no_pull: bool = False, no_init: bool = False) -> None: +def up( + ctx: click.Context, no_pull: bool = False, no_init: bool = False +) -> None: # pragma: no cover """ - Starts the sample project via docker compose. + Starts a sample project via docker compose. """ check_virtualenv() @@ -138,7 +140,9 @@ def up(ctx: click.Context, no_pull: bool = False, no_init: bool = False) -> None try: check_fides_uploads_dir() + print("> Starting application...") start_application() + print("> Seeding data...") seed_example_data() click.clear() diff --git a/src/fides/cli/commands/view.py b/src/fides/cli/commands/view.py index fde0ab432c..c8711f53a3 100644 --- a/src/fides/cli/commands/view.py +++ b/src/fides/cli/commands/view.py @@ -1,9 +1,10 @@ """Contains the view group of the commands for fides.""" -import click -from toml import dumps as toml_dumps +import rich_click as click +from toml import dumps from fides.cli.utils import print_divider, with_analytics +from fides.core.utils import echo_red, get_credentials_path, read_credentials_file @click.group(name="view") @@ -27,7 +28,9 @@ def view_config( ctx: click.Context, section: str = "", exclude_unset: bool = False ) -> None: """ - Prints the configuration values being used. + Prints the configuration values being used for this command-line instance. + + _Note: To see the configuration values being used by the webserver, `GET` the `/api/v1/config` endpoint._ """ config = ctx.obj["CONFIG"] config_dict = config.dict(exclude_unset=exclude_unset) @@ -35,4 +38,22 @@ def view_config( config_dict = config_dict[section] print_divider() - print(toml_dumps(config_dict)) + print(dumps(config_dict)) + + +@view.command(name="credentials") +@click.pass_context +@with_analytics +def view_credentials(ctx: click.Context) -> None: + """ + Prints the credentials file. + """ + credentials_path = get_credentials_path() + try: + credentials = read_credentials_file(credentials_path) + except FileNotFoundError: + echo_red(f"No credentials file found at path: {credentials_path}") + raise SystemExit(1) + + print_divider() + print(dumps(credentials.dict())) diff --git a/src/fides/cli/options.py b/src/fides/cli/options.py index 5700f3ec83..6da30ac6c9 100644 --- a/src/fides/cli/options.py +++ b/src/fides/cli/options.py @@ -1,17 +1,21 @@ """ -Contains all of the options/arguments used by the CLI commands. +Reusable command-line arguments and options. """ from typing import Callable -import click +import rich_click as click from fideslang import model_list def coverage_threshold_option(command: Callable) -> Callable: - "Add a flag that assumes yes." + """An option decorator that sets a required coverage percentage.""" command = click.option( - "--coverage-threshold", "-c", type=click.IntRange(0, 100), default=100 + "--coverage-threshold", + "-c", + type=click.IntRange(0, 100), + default=100, + help="Set the coverage percentage for a passing scan.", )(command) return command @@ -40,7 +44,6 @@ def fides_key_option(command: Callable) -> Callable: "-k", "--fides-key", default="", - help="The fides_key of the single policy that you wish to evaluate.", )(command) return command @@ -48,9 +51,7 @@ def fides_key_option(command: Callable) -> Callable: def manifests_dir_argument(command: Callable) -> Callable: "Add the manifests_dir argument." command = click.argument( - "manifests_dir", - default=".fides/", - type=click.Path(exists=True), + "manifests_dir", type=click.Path(exists=True), default=".fides/" )(command) return command @@ -58,7 +59,7 @@ def manifests_dir_argument(command: Callable) -> Callable: def dry_flag(command: Callable) -> Callable: "Add a flag that prevents side-effects." command = click.option( - "--dry", is_flag=True, help="Prevent the persistance of any changes." + "--dry", is_flag=True, help="Do not upload results to the Fides webserver." )(command) return command @@ -69,7 +70,7 @@ def yes_flag(command: Callable) -> Callable: "--yes", "-y", is_flag=True, - help="Automatically responds 'yes' to any prompts.", + help="Automatically responds `yes` to any prompts.", )(command) return command @@ -90,7 +91,7 @@ def include_null_flag(command: Callable) -> Callable: command = click.option( "--include-null", is_flag=True, - help="Includes attributes that would otherwise be null.", + help="Include null attributes.", )(command) return command @@ -101,7 +102,8 @@ def organization_fides_key_option(command: Callable) -> Callable: "--org-key", "-k", default="default_organization", - help="The organization_fides_key you wish to export resources for.", + show_default=True, + help="The `organization_fides_key` of the `Organization` you want to specify.", )(command) return command @@ -112,6 +114,7 @@ def output_directory_option(command: Callable) -> Callable: "--output-dir", "-d", default=".fides/", + show_default=True, help="The output directory for the data map to be exported to.", )(command) return command @@ -122,7 +125,7 @@ def credentials_id_option(command: Callable) -> Callable: command = click.option( "--credentials-id", type=str, - help="Use credentials defined within fides config", + help="Use credentials keys defined within Fides config.", )(command) return command @@ -132,7 +135,7 @@ def connection_string_option(command: Callable) -> Callable: command = click.option( "--connection-string", type=str, - help="Use connection string option to connect to a database", + help="Use the connection string option to connect to a database.", )(command) return command @@ -142,7 +145,7 @@ def okta_org_url_option(command: Callable) -> Callable: command = click.option( "--org-url", type=str, - help="Use org url option to connect to okta. Requires options --org-url and --token", + help="Connect to Okta using an 'Org URL'. _Requires options `--org-url` & `--token`._", )(command) return command @@ -152,7 +155,7 @@ def okta_token_option(command: Callable) -> Callable: command = click.option( "--token", type=str, - help="Use token option to connect to okta. Requires options --org-url and --token", + help="Connect to Okta using a token. _Requires options `--org-url` and `--token`._", )(command) return command @@ -162,7 +165,7 @@ def aws_access_key_id_option(command: Callable) -> Callable: command = click.option( "--access_key_id", type=str, - help="Use access key id option to connect to aws. Requires options --access_key_id, --secret_access_key and --region", + help="Connect to AWS using an `Access Key ID`. _Requires options `--access_key_id`, `--secret_access_key` & `--region`._", )(command) return command @@ -172,7 +175,7 @@ def aws_secret_access_key_option(command: Callable) -> Callable: command = click.option( "--secret_access_key", type=str, - help="Use access key option to connect to aws. Requires options --access_key_id, --secret_access_key and --region", + help="Connect to AWS using an `Access Key`. _Requires options `--access_key_id`, `--secret_access_key` & `--region`._", )(command) return command @@ -182,7 +185,7 @@ def aws_region_option(command: Callable) -> Callable: command = click.option( "--region", type=str, - help="Use region option to connect to aws. Requires options --access_key_id, --secret_access_key and --region", + help="Connect to AWS using a specific `Region`. _Requires options `--access_key_id`, `--secret_access_key` & `--region`._", )(command) return command diff --git a/src/fides/cli/utils.py b/src/fides/cli/utils.py index c0efbe22a6..5c7ec76e7a 100644 --- a/src/fides/cli/utils.py +++ b/src/fides/cli/utils.py @@ -10,8 +10,8 @@ from platform import system from typing import Any, Callable, Dict, Optional, Union -import click import requests +import rich_click as click from fideslog.sdk.python.client import AnalyticsClient from fideslog.sdk.python.event import AnalyticsEvent from fideslog.sdk.python.exceptions import AnalyticsError diff --git a/tests/ctl/cli/test_cli.py b/tests/ctl/cli/test_cli.py index 5de3f9841e..53e2b2e241 100644 --- a/tests/ctl/cli/test_cli.py +++ b/tests/ctl/cli/test_cli.py @@ -51,13 +51,22 @@ def test_init_opt_in(test_cli_runner: CliRunner) -> None: assert result.exit_code == 0 -@pytest.mark.unit -def test_view_config(test_cli_runner: CliRunner) -> None: - result = test_cli_runner.invoke( - cli, ["view", "config"], env={"FIDES__USER__ANALYTICS_OPT_OUT": "true"} - ) - print(result.output) - assert result.exit_code == 0 +class TestView: + @pytest.mark.unit + def test_view_config(self, test_cli_runner: CliRunner) -> None: + result = test_cli_runner.invoke( + cli, ["view", "config"], env={"FIDES__USER__ANALYTICS_OPT_OUT": "true"} + ) + print(result.output) + assert result.exit_code == 0 + + @pytest.mark.unit + def test_view_credentials(self, test_cli_runner: CliRunner) -> None: + result = test_cli_runner.invoke( + cli, ["view", "credentials"], env={"FIDES__USER__ANALYTICS_OPT_OUT": "true"} + ) + print(result.output) + assert result.exit_code == 0 @pytest.mark.unit @@ -197,8 +206,8 @@ def test_audit(test_config_path: str, test_cli_runner: CliRunner) -> None: assert result.exit_code == 0 +@pytest.mark.integration class TestCRUD: - @pytest.mark.integration def test_get(self, test_config_path: str, test_cli_runner: CliRunner) -> None: result = test_cli_runner.invoke( cli, @@ -207,12 +216,36 @@ def test_get(self, test_config_path: str, test_cli_runner: CliRunner) -> None: print(result.output) assert result.exit_code == 0 - @pytest.mark.integration + def test_delete(self, test_config_path: str, test_cli_runner: CliRunner) -> None: + result = test_cli_runner.invoke( + cli, + ["-f", test_config_path, "delete", "system", "demo_marketing_system"], + ) + print(result.output) + assert result.exit_code == 0 + def test_ls(self, test_config_path: str, test_cli_runner: CliRunner) -> None: result = test_cli_runner.invoke(cli, ["-f", test_config_path, "ls", "system"]) print(result.output) assert result.exit_code == 0 + def test_ls_verbose( + self, test_config_path: str, test_cli_runner: CliRunner + ) -> None: + result = test_cli_runner.invoke( + cli, ["-f", test_config_path, "ls", "system", "--verbose"] + ) + print(result.output) + assert result.exit_code == 0 + + def test_ls_no_resources_found( + self, test_config_path: str, test_cli_runner: CliRunner + ) -> None: + """This test only workss because we don't have any registry resources by default.""" + result = test_cli_runner.invoke(cli, ["-f", test_config_path, "ls", "registry"]) + print(result.output) + assert result.exit_code == 0 + class TestEvaluate: @pytest.mark.integration