From 7cca0597c3f46837377dde580ad5b16413f201e2 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Fri, 2 Aug 2024 19:31:54 +0530 Subject: [PATCH] feat(ingest/mode): add option to exclude restricted --- .../src/datahub/ingestion/source/mode.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/mode.py b/metadata-ingestion/src/datahub/ingestion/source/mode.py index 4b4822bcb98cae..87fa4013c8bc6c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/mode.py +++ b/metadata-ingestion/src/datahub/ingestion/source/mode.py @@ -135,9 +135,14 @@ class ModeConfig(StatefulIngestionConfigBase, DatasetLineageProviderConfigBase): connect_uri: str = Field( default="https://app.mode.com", description="Mode host URL." ) - token: str = Field(description="Mode user token.") + token: str = Field( + description="When creating workspace API key this is the 'Key ID'." + ) password: pydantic.SecretStr = Field( - description="Mode password for authentication." + description="When creating workspace API key this is the 'Secret'." + ) + exclude_restricted: bool = Field( + default=False, description="Exclude restricted collections" ) workspace: str = Field( @@ -522,6 +527,11 @@ def _get_space_name_and_tokens(self) -> dict: for s in spaces: logger.debug(f"Space: {s.get('name')}") space_name = s.get("name", "") + if s.get("restricted") and self.config.exclude_restricted: + logging.debug( + f"Skipping space {space_name} due to exclude restricted" + ) + continue if not self.config.space_pattern.allowed(space_name): self.report.report_dropped_space(space_name) logging.debug(f"Skipping space {space_name} due to space pattern")