Vault abstraction for Galaxy

doc/source/admin/galaxy_options.rst

@@ -4809,5 +4809,13 @@
 :Default: ``plugins/welcome_page/new_user/static/topics/``
 :Type: str
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``vault_config_file``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-
+:Description:
+    Vault configuration
+    The value of this option will be resolved with respect to
+    <config_dir>.
+:Default: ``vault_conf.yml``
+:Type: str

lib/galaxy/app.py

@@ -45,6 +45,10 @@
 )
 from galaxy.quota import get_quota_agent, QuotaAgent
 from galaxy.security.idencoding import IdEncodingHelper
+from galaxy.security.vault import (
+    Vault,
+    VaultFactory
+)
 from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager
 from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager
 from galaxy.tool_util.deps.views import DependencyResolversView
@@ -183,6 +187,8 @@ def __init__(self, **kwargs):
         # ConfiguredFileSources
         self.file_sources = self._register_singleton(ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config))
 
+        self.vault = self._register_singleton(Vault, VaultFactory.from_app_config(self.config))
+
         # We need the datatype registry for running certain tasks that modify HDAs, and to build the registry we need
         # to setup the installed repositories ... this is not ideal
         self._configure_tool_config_files()

lib/galaxy/config/sample/galaxy.yml.sample

@@ -2353,3 +2353,7 @@ galaxy:
   # is relative to galaxy/static
   #welcome_directory: plugins/welcome_page/new_user/static/topics/
 
+  # Vault config file
+  # The value of this option will be resolved with respect to
+  # <config_dir>.
+  vault_config_file: vault_conf.yml

lib/galaxy/dependencies/__init__.py

@@ -33,6 +33,7 @@ def __init__(self, config_file, config=None):
         self.container_interface_types = []
         self.job_rule_modules = []
         self.error_report_modules = []
+        self.vault_type = None
         if config is None:
             self.config = load_app_properties(config_file=self.config_file)
         else:
@@ -151,6 +152,17 @@ def collect_types(from_dict):
             file_sources_conf = []
         self.file_sources = [c.get('type', None) for c in file_sources_conf]
 
+        # Parse vault config
+        vault_conf_yml = self.config.get(
+            "vault_config_file",
+            join(dirname(self.config_file), 'vault_conf.yml'))
+        if exists(vault_conf_yml):
+            with open(vault_conf_yml) as f:
+                vault_conf = yaml.safe_load(f)
+        else:
+            vault_conf = {}
+        self.vault_type = vault_conf.get('type', '').lower()
+
     def get_conditional_requirements(self):
         crfile = join(dirname(__file__), 'conditional-requirements.txt')
         for req in pkg_resources.parse_requirements(open(crfile).readlines()):
@@ -261,6 +273,12 @@ def check_weasyprint(self):
         # See notes in ./conditional-requirements.txt for more information.
         return os.environ.get("GALAXY_DEPENDENCIES_INSTALL_WEASYPRINT") == "1"
 
+    def check_custos_sdk(self):
+        return 'custos' == self.vault_type
+
+    def check_hvac(self):
+        return 'hashicorp' == self.vault_type
+
 
 def optional(config_file=None):
     if not config_file:

lib/galaxy/files/__init__.py

@@ -263,6 +263,11 @@ def is_admin(self):
         """Whether this user is an administrator."""
         return self.trans.user_is_admin
 
+    @property
+    def vault(self):
+        user = self.trans.user
+        return user and user.personal_vault or defaultdict(lambda: None)
+
 
 class DictFileSourcesUserContext:
 
@@ -296,3 +301,7 @@ def group_names(self):
     @property
     def is_admin(self):
         return self._kwd.get("is_admin")
+
+    @property
+    def vault(self):
+        return self._kwd.get("vault")

lib/galaxy/managers/context.py

@@ -50,6 +50,7 @@
 )
 from galaxy.model.base import ModelMapping
 from galaxy.security.idencoding import IdEncodingHelper
+from galaxy.security.vault import UserVaultWrapper
 from galaxy.structured_app import MinimalManagerApp
 from galaxy.util import bunch
 
@@ -198,6 +199,11 @@ class ProvidesUserContext(ProvidesAppContext):
     def user(self):
         """Provide access to the user object."""
 
+    @property
+    def user_vault(self):
+        """Provide access to a user's personal vault."""
+        return UserVaultWrapper(self.app.vault, self.user)
+
     @property
     def anonymous(self) -> bool:
         return self.user is None

lib/galaxy/model/__init__.py

@@ -8420,6 +8420,16 @@ class LibraryDatasetCollectionAnnotationAssociation(Base, RepresentById):
     user = relationship('User')
 
 
+class Vault(Base, RepresentById):
+    __tablename__ = 'vault'
+
+    id = Column(Integer, primary_key=True)
+    create_time = Column(DateTime, default=now)
+    update_time = Column(DateTime, default=now, onupdate=now)
+    key = Column(Text, index=True, unique=True)
+    value = Column(Text)
+
+
 # Item rating classes.
 class ItemRatingAssociation(Base):
     __abstract__ = True

lib/galaxy/model/migrate/versions/180_add_vault_table.py

@@ -0,0 +1,25 @@
+import datetime
+
+from sqlalchemy import Column, DateTime, Integer, MetaData, Table, Text
+
+now = datetime.datetime.utcnow
+meta = MetaData()
+
+vault = Table(
+    'vault', meta,
+    Column('id', Integer, primary_key=True),
+    Column("create_time", DateTime, default=now),
+    Column("update_time", DateTime, default=now, onupdate=now),
+    Column('key', Text, index=True, unique=True),
+    Column('value', Text),
+)
+
+
+def upgrade(migrate_engine):
+    meta.bind = migrate_engine
+    vault.create()
+
+
+def downgrade(migrate_engine):
+    meta.bind = migrate_engine
+    vault.drop()

lib/galaxy/security/vault.py

@@ -0,0 +1,159 @@
+from abc import ABC
+import json
+import os
+import yaml
+
+from cryptography.fernet import Fernet, MultiFernet
+
+try:
+    from custos.clients.resource_secret_management_client import ResourceSecretManagementClient
+    from custos.transport.settings import CustosServerClientSettings
+    import custos.clients.utils.utilities as custos_util
+    custos_sdk_available = True
+except ImportError:
+    custos_sdk_available = False
+
+try:
+    import hvac
+except ImportError:
+    hvac = None
+
+from galaxy import model
+
+
+class UnknownVaultTypeException(Exception):
+    pass
+
+
+class Vault(ABC):
+
+    def read_secret(self, key: str) -> str:
+        pass
+
+    def write_secret(self, key: str, value: str) -> None:
+        pass
+
+
+class HashicorpVault(Vault):
+
+    def __init__(self, config):
+        if not hvac:
+            raise UnknownVaultTypeException("Hashicorp vault library 'hvac' is not available. Make sure hvac is installed.")
+        self.vault_address = config.get('vault_address')
+        self.vault_token = config.get('vault_token')
+        self.client = hvac.Client(url=self.vault_address, token=self.vault_token)
+
+    def read_secret(self, key: str) -> str:
+        try:
+            response = self.client.secrets.kv.read_secret_version(path=key)
+            return response['data']['data'].get('value')
+        except hvac.exceptions.InvalidPath:
+            return None
+
+    def write_secret(self, key: str, value: str) -> None:
+        self.client.secrets.kv.v2.create_or_update_secret(path=key, secret={'value': value})
+
+
+class DatabaseVault(Vault):
+
+    def __init__(self, sa_session, config):
+        self.sa_session = sa_session
+        self.encryption_keys = config.get('encryption_keys')
+        self.fernet_keys = [Fernet(key.encode('utf-8')) for key in self.encryption_keys]
+
+    def _get_multi_fernet(self) -> MultiFernet:
+        return MultiFernet(self.fernet_keys)
+
+    def _update_or_create(self, key: str, value: str):
+        vault_entry = self.sa_session.query(model.Vault).filter_by(key=key).first()
+        if vault_entry:
+            vault_entry.value = value
+        else:
+            vault_entry = model.Vault(key=key, value=value)
+        self.sa_session.merge(vault_entry)
+        self.sa_session.flush()
+
+    def read_secret(self, key: str) -> str:
+        key_obj = self.sa_session.query(model.Vault).filter_by(key=key).first()
+        if key_obj:
+            f = self._get_multi_fernet()
+            return f.decrypt(key_obj.value.encode('utf-8')).decode('utf-8')
+        return None
+
+    def write_secret(self, key: str, value: str) -> None:
+        f = self._get_multi_fernet()
+        token = f.encrypt(value.encode('utf-8'))
+        self._update_or_create(key=key, value=token.decode('utf-8'))
+
+
+class CustosVault(Vault):
+
+    def __init__(self, config):
+        if not custos_sdk_available:
+            raise UnknownVaultTypeException("Custos sdk library 'custos-sdk' is not available. Make sure the custos-sdk is installed.")
+        self.custos_settings = CustosServerClientSettings(custos_host=config.get('custos_host'),
+                                                          custos_port=config.get('custos_port'),
+                                                          custos_client_id=config.get('custos_client_id'),
+                                                          custos_client_sec=config.get('custos_client_sec'))
+        self.b64_encoded_custos_token = custos_util.get_token(custos_settings=self.custos_settings)
+        self.client = ResourceSecretManagementClient(self.custos_settings)
+
+    def read_secret(self, key: str) -> str:
+        try:
+            response = self.client.get_KV_credential(token=self.b64_encoded_custos_token,
+                                                     client_id=self.custos_settings.CUSTOS_CLIENT_ID,
+                                                     key=key)
+            return json.loads(response).get('value')
+        except Exception:
+            return None
+
+    def write_secret(self, key: str, value: str) -> None:
+        if self.read_secret(key):
+            self.client.update_KV_credential(token=self.b64_encoded_custos_token,
+                                             client_id=self.custos_settings.CUSTOS_CLIENT_ID,
+                                             key=key, value=value)
+        else:
+            self.client.set_KV_credential(token=self.b64_encoded_custos_token,
+                                          client_id=self.custos_settings.CUSTOS_CLIENT_ID,
+                                          key=key, value=value)
+
+
+class UserVaultWrapper(Vault):
+
+    def __init__(self, vault: Vault, user):
+        self.vault = vault
+        self.user = user
+
+    def read_secret(self, key: str) -> str:
+        return self.vault.read_secret(f"user/{self.user.id}/{key}")
+
+    def write_secret(self, key: str, value: str) -> None:
+        return self.vault.write_secret(f"user/{self.user.id}/{key}", value)
+
+
+class VaultFactory(object):
+
+    @staticmethod
+    def load_vault_config(vault_conf_yml: str) -> dict:
+        if os.path.exists(vault_conf_yml):
+            with open(vault_conf_yml) as f:
+                return yaml.safe_load(f)
+        return None
+
+    @staticmethod
+    def from_vault_type(app, vault_type: str, cfg: dict) -> Vault:
+        if vault_type == "hashicorp":
+            return HashicorpVault(cfg)
+        elif vault_type == "database":
+            return DatabaseVault(app.model.context, cfg)
+        elif vault_type == "custos":
+            return CustosVault(cfg)
+        else:
+            raise UnknownVaultTypeException(f"Unknown vault type: {vault_type}")
+
+    @staticmethod
+    def from_app_config(app) -> Vault:
+        vault_config = VaultFactory.load_vault_config(app.config.vault_config_file)
+        if vault_config:
+            return VaultFactory.from_vault_type(app, vault_config.get('type'), vault_config)
+        return None

lib/galaxy/structured_app.py

@@ -16,6 +16,7 @@
 from galaxy.objectstore import ObjectStore
 from galaxy.quota import QuotaAgent
 from galaxy.security.idencoding import IdEncodingHelper
+from galaxy.security.vault import Vault
 from galaxy.tool_util.deps.views import DependencyResolversView
 from galaxy.tool_util.verify import test_data
 from galaxy.util.dbkeys import GenomeBuilds
@@ -98,6 +99,7 @@ class StructuredApp(MinimalManagerApp):
     security_agent: GalaxyRBACAgent
     host_security_agent: HostAgent
     trs_proxy: TrsProxy
+    vault: Vault
     webhooks_registry: WebhooksRegistry
 
     queue_worker: Any  # 'galaxy.queue_worker.GalaxyQueueWorker'

lib/galaxy/webapps/galaxy/config_schema.yml

@@ -3468,4 +3468,12 @@ mapping:
         desc: |
           Location of New User Welcome data, a single directory containing the
           images and JSON of Topics/Subtopics/Slides as export. This location
-          is relative to galaxy/static
+          is relative to galaxy/static
+
+      vault_config_file:
+        type: str
+        default: vault_conf.yml
+        path_resolves_to: config_dir
+        required: false
+        desc: |
+          Vault config file.

test/integration/test_config_defaults.py

@@ -94,6 +94,7 @@
     'tool_sheds_config_file',
     'trs_servers_config_file',
     'user_preferences_extra_conf_path',
+    'vault_config_file',
     'webhooks_dir',
     'workflow_resource_params_file',
     'workflow_resource_params_mapper',
@@ -131,6 +132,7 @@
     'tool_path': 'root_dir',
     'tool_sheds_config_file': 'config_dir',
     'user_preferences_extra_conf_path': 'config_dir',
+    'vault_config_file': 'config_dir',
     'workflow_resource_params_file': 'config_dir',
     'workflow_schedulers_config_file': 'config_dir',
 }