✨ add support for security-related objects (#58)

* finish work on tokens and secrets

* add support for secrets and verify tests

README.md

@@ -77,9 +77,9 @@ SQL:
 
 Security:
 
-- [ ] Tokens
-- [ ] Passwords (for AWS)
-- [ ] Secrets
+- [x] Tokens (token permissions are set on the workspace level. It basically says "this group can use tokens or not")
+- [x] Passwords (only for AWS, it defines which groups can log in with passwords)
+- [x] Secrets
 
 Workspace:
 

pyproject.toml

@@ -104,7 +104,7 @@ skip_glob = [
 profile = "black"
 
 [tool.pytest.ini_options]
-addopts = "-s -p no:warnings"
+addopts = "-s -p no:warnings -vv"
 log_cli = true
 filterwarnings = [
     "ignore:::.*pyspark.broadcast*",

src/uc_migration_toolkit/cli/app.py

@@ -14,13 +14,12 @@ def migrate_groups(config_file: Annotated[Path, typer.Argument(help="Path to con
 
     config = get_migration_config(config_file)
     toolkit = GroupMigrationToolkit(config)
+    toolkit.prepare_groups_in_environment()
 
-    toolkit.validate_groups()
     toolkit.cleanup_inventory_table()
     toolkit.inventorize_permissions()
-    toolkit.create_or_update_backup_groups()
-    toolkit.apply_backup_group_permissions()
+    toolkit.apply_permissions_to_backup_groups()
     toolkit.replace_workspace_groups_with_account_groups()
-    # toolkit.apply_account_group_permissions()
-    # toolkit.delete_backup_groups()
-    # toolkit.cleanup_inventory_table()
+    toolkit.apply_permissions_to_account_groups()
+    toolkit.delete_backup_groups()
+    toolkit.cleanup_inventory_table()

src/uc_migration_toolkit/managers/inventory/inventorizer.py

@@ -1,10 +1,15 @@
+import json
+from abc import ABC, abstractmethod
 from collections.abc import Callable, Iterator
 from functools import partial
 from typing import Generic, TypeVar
 
-from databricks.sdk.service.iam import ObjectPermissions
+from databricks.sdk.core import DatabricksError
+from databricks.sdk.service.iam import AccessControlResponse, ObjectPermissions
+from databricks.sdk.service.workspace import AclItem, SecretScope
 
 from uc_migration_toolkit.managers.inventory.types import (
+    AclItemsContainer,
     LogicalObjectType,
     PermissionsInventoryItem,
     RequestObjectType,
@@ -17,7 +22,17 @@
 InventoryObject = TypeVar("InventoryObject")
 
 
-class StandardInventorizer(Generic[InventoryObject]):
+class BaseInventorizer(ABC, Generic[InventoryObject]):
+    @abstractmethod
+    def preload(self):
+        """Any preloading activities should happen here"""
+
+    @abstractmethod
+    def inventorize(self) -> list[PermissionsInventoryItem]:
+        """Any inventorization activities should happen here"""
+
+
+class StandardInventorizer(BaseInventorizer[InventoryObject]):
     """
     Standard means that it can collect using the default listing/permissions function without any additional logic.
     """
@@ -54,7 +69,7 @@ def _process_single_object(self, _object: InventoryObject) -> PermissionsInvento
             object_id=object_id,
             logical_object_type=self._logical_object_type,
             request_object_type=self._request_object_type,
-            object_permissions=permissions.as_dict(),
+            raw_object_permissions=json.dumps(permissions.as_dict()),
         )
         return inventory_item
 
@@ -66,3 +81,92 @@ def inventorize(self):
         collected = threaded_execution.run()
         logger.info(f"Permissions fetched for {len(collected)} objects of type {self._request_object_type}")
         return collected
+
+
+class TokensAndPasswordsInventorizer(BaseInventorizer[InventoryObject]):
+    def __init__(self):
+        self._tokens_acl = []
+        self._passwords_acl = []
+
+    @staticmethod
+    def _preload_tokens():
+        try:
+            return provider.ws.get_tokens().get("access_control_list", [])
+        except DatabricksError as e:
+            logger.warning("Cannot load token permissions due to error:")
+            logger.warning(e)
+            return []
+
+    @staticmethod
+    def _preload_passwords():
+        try:
+            return provider.ws.get_passwords().get("access_control_list", [])
+        except DatabricksError as e:
+            logger.error("Cannot load password permissions due to error:")
+            logger.error(e)
+            return []
+
+    def preload(self):
+        self._tokens_acl = [AccessControlResponse.from_dict(acl) for acl in self._preload_tokens()]
+        self._passwords_acl = [AccessControlResponse.from_dict(acl) for acl in self._preload_passwords()]
+
+    def inventorize(self) -> list[PermissionsInventoryItem]:
+        results = []
+
+        if self._passwords_acl:
+            results.append(
+                PermissionsInventoryItem(
+                    object_id="passwords",
+                    logical_object_type=LogicalObjectType.PASSWORD,
+                    request_object_type=RequestObjectType.AUTHORIZATION,
+                    raw_object_permissions=json.dumps(
+                        ObjectPermissions(
+                            object_id="passwords", object_type="authorization", access_control_list=self._passwords_acl
+                        ).as_dict()
+                    ),
+                )
+            )
+
+        if self._tokens_acl:
+            results.append(
+                PermissionsInventoryItem(
+                    object_id="tokens",
+                    logical_object_type=LogicalObjectType.TOKEN,
+                    request_object_type=RequestObjectType.AUTHORIZATION,
+                    raw_object_permissions=json.dumps(
+                        ObjectPermissions(
+                            object_id="tokens", object_type="authorization", access_control_list=self._tokens_acl
+                        ).as_dict()
+                    ),
+                )
+            )
+        return results
+
+
+class SecretScopeInventorizer(BaseInventorizer[InventoryObject]):
+    def __init__(self):
+        self._scopes = provider.ws.secrets.list_scopes()
+
+    @staticmethod
+    def _get_acls_for_scope(scope: SecretScope) -> Iterator[AclItem]:
+        return provider.ws.secrets.list_acls(scope.name)
+
+    def _prepare_permissions_inventory_item(self, scope: SecretScope) -> PermissionsInventoryItem:
+        acls = self._get_acls_for_scope(scope)
+        acls_container = AclItemsContainer.from_sdk(list(acls))
+
+        return PermissionsInventoryItem(
+            object_id=scope.name,
+            logical_object_type=LogicalObjectType.SECRET_SCOPE,
+            request_object_type=None,
+            raw_object_permissions=json.dumps(acls_container.model_dump(mode="json")),
+        )
+
+    def inventorize(self) -> list[PermissionsInventoryItem]:
+        executables = [partial(self._prepare_permissions_inventory_item, scope) for scope in self._scopes]
+        results = ThreadedExecution[PermissionsInventoryItem](executables).run()
+        logger.info(f"Permissions fetched for {len(results)} objects of type {LogicalObjectType.SECRET_SCOPE}")
+        return results
+
+    def preload(self):
+        pass