diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index fd776613867c..a86b3a410e3e 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -4781,5 +4781,13 @@ :Default: ``plugins/welcome_page/new_user/static/topics/`` :Type: str +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``vault_config_file`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - +:Description: + Vault configuration + The value of this option will be resolved with respect to + . +:Default: ``vault_conf.yml`` +:Type: str diff --git a/doc/source/admin/special_topics/index.rst b/doc/source/admin/special_topics/index.rst index a49416a3785e..2f904d28fa55 100644 --- a/doc/source/admin/special_topics/index.rst +++ b/doc/source/admin/special_topics/index.rst @@ -13,6 +13,7 @@ Special Topics gtn job_metrics webhooks + vault performance_tracking bug_reports gdpr_compliance diff --git a/doc/source/admin/special_topics/vault.md b/doc/source/admin/special_topics/vault.md new file mode 100644 index 000000000000..6dee3563ec57 --- /dev/null +++ b/doc/source/admin/special_topics/vault.md @@ -0,0 +1,124 @@ +# Storing secrets in the vault + +Galaxy can be configured to store secrets in an external vault, which is useful for secure handling and centralization of secrets management. +In particular, information fields in the "Manage information" section of the user profile, such as AWS credentials, can be configured to be stored +in an encrypted vault instead of the database. Vault keys are generally stored as key-value pairs in a hierarchical fashion, for example: +`/galaxy/user/2/preferences/aws/client_secret`. + +## Vault backends + +There are currently 3 supported backends. + +| Backend | Description | +|-------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| hashicorp | Hashicorp Vault is a secrets and encryption management system. https://www.vaultproject.io/ | +| custos | Custos is an NSF-funded project, backed by open source software that provides science gateways such as Galaxy with single sign-on, group management, and management of secrets such as access keys and OAuth2 access tokens. Custos secrets management is backed by Hashicorp's vault, but provides a convenient, always-on ReST API service. | +| database | The database backend stores secrets in an encrypted table in the Galaxy database itself. It is a convenient way to get started with a vault, and while it supports basic key rotation, we recommend using one of the other options in production. | + +## Configuring Galaxy + +The first step to using a vault is to configure the `vault_config_file` setting in `galaxy.yml`. + +```yaml +galaxy: + # Vault config file + # The value of this option will be resolved with respect to + # . + vault_config_file: vault_conf.yml +``` + +## Configuring vault_conf.yml + +The vault_conf.yml file itself has two basic fields: +```yaml +type: hashicorp # required +path_prefix: /galaxy # optional +... +``` + +The `type` must be a valid backend type: `hashicorp`, `custos`, or `database`. At present, only a single vault backend +is supported. The `path_prefix` property indicates the root path under which to store all vault keys. If multiple +Galaxy instances are using the same vault, a prefix can be used to uniquely identify the Galaxy instance. +If no path_prefix is provided, the prefix defaults to `/galaxy`. + +## Vault configuration for Hashicorp Vault + +```yaml +type: hashicorp +path_prefix: /my_galaxy_instance +vault_address: http://localhost:8200 +vault_token: vault_application_token +``` + +## Vault configuration for Custos + +```yaml +type: custos +custos_host: service.staging.usecustos.org +custos_port: 30170 +custos_client_id: custos-jeREDACTEDye-10000001 +custos_client_sec: OGREDACTEDBSUDHn +``` + +Obtaining the Custos client id and client secret requires first registering your Galaxy instance with Custos. +Visit [usecustos.org](http://usecustos.org/) for more information. + +## Vault configuration for database + +```yaml +type: database +path_prefix: /galaxy +# Encryption keys must be valid fernet keys +# To generate a valid key: +# +# Use the ascii string value as a key +# For more details, see: https://cryptography.io/en/latest/fernet/# +encryption_keys: + - 5RrT94ji178vQwha7TAmEix7DojtsLlxVz8Ef17KWgg= + - iNdXd7tRjLnSqRHxuhqQ98GTLU8HUbd5_Xx38iF8nZ0= + - IK83IXhE4_7W7xCFEtD9op0BAs11pJqYN236Spppp7g= +``` + +Secrets stored in the database are encrypted using Fernet keys. Therefore, all listed encryption keys must be valid fernet keys. To generate a new +Fernet key, use the following Python code: + +```python +from cryptography.fernet import Fernet +Fernet.generate_key().decode('utf-8') + +If multiple encryption keys are defined, only the first key is used to encrypt secrets. The remaining keys are tried in turn during decryption. This is useful for key rotation. +We recommend periodically generating a new fernet key and rotating old keys. However, before removing an old key, make sure that any data encrypted using that old key is no longer +present or the data will no longer be decryptable. + +## Configuring user preferences to use the vault + +The `user_preferences_extra_conf.yml` can be used to automatically route secrets to a vault. An example configuration follows: + +```yaml +preferences: + googledrive: + description: Your Google Drive account + inputs: + - name: client_id + label: Client ID + type: text + required: True + - name: client_secret + label: Client Secret + type: secret + store: vault + required: True + - name: access_token + label: Access token + type: password + store: vault + required: True + - name: refresh_token + label: Refresh Token + type: secret + store: vault + required: True +``` + +Note the `store: vault` property, which results in the property being stored in the vault. Note also that if you use `type: password`, the secret is sent to the client front-end, +but specifying `type: secret` would mean that the values cannot be retrieved by the client, only written to, providing an extra layer of security. diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 82734ad3b26f..7dba58de06eb 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -45,6 +45,10 @@ ) from galaxy.quota import get_quota_agent, QuotaAgent from galaxy.security.idencoding import IdEncodingHelper +from galaxy.security.vault import ( + Vault, + VaultFactory +) from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager from galaxy.tool_util.deps.views import DependencyResolversView @@ -207,6 +211,8 @@ def __init__(self, **kwargs): # ConfiguredFileSources self.file_sources = self._register_singleton(ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config)) + self.vault = self._register_singleton(Vault, VaultFactory.from_app(self)) + # We need the datatype registry for running certain tasks that modify HDAs, and to build the registry we need # to setup the installed repositories ... this is not ideal self._configure_tool_config_files() diff --git a/lib/galaxy/app_unittest_utils/galaxy_mock.py b/lib/galaxy/app_unittest_utils/galaxy_mock.py index 7a10bef48544..1fb9dc006019 100644 --- a/lib/galaxy/app_unittest_utils/galaxy_mock.py +++ b/lib/galaxy/app_unittest_utils/galaxy_mock.py @@ -188,6 +188,7 @@ def __init__(self, **kwargs): self.enable_tool_shed_check = False self.monitor_thread_join_timeout = 1 self.integrated_tool_panel_config = None + self.vault_config_file = kwargs.get('vault_config_file') @property def config_dict(self): diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index b7cef22b3c37..aacdb8a71002 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -2332,3 +2332,7 @@ galaxy: # is relative to galaxy/static #welcome_directory: plugins/welcome_page/new_user/static/topics/ + # Vault config file + # The value of this option will be resolved with respect to + # . + vault_config_file: vault_conf.yml diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index c8c3f781d31b..8f98f0075a64 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -33,6 +33,7 @@ def __init__(self, config_file, config=None): self.container_interface_types = [] self.job_rule_modules = [] self.error_report_modules = [] + self.vault_type = None if config is None: self.config = load_app_properties(config_file=self.config_file) else: @@ -151,6 +152,17 @@ def collect_types(from_dict): file_sources_conf = [] self.file_sources = [c.get('type', None) for c in file_sources_conf] + # Parse vault config + vault_conf_yml = self.config.get( + "vault_config_file", + join(dirname(self.config_file), 'vault_conf.yml')) + if exists(vault_conf_yml): + with open(vault_conf_yml) as f: + vault_conf = yaml.safe_load(f) + else: + vault_conf = {} + self.vault_type = vault_conf.get('type', '').lower() + def get_conditional_requirements(self): crfile = join(dirname(__file__), 'conditional-requirements.txt') for req in pkg_resources.parse_requirements(open(crfile).readlines()): @@ -273,6 +285,12 @@ def check_weasyprint(self): # See notes in ./conditional-requirements.txt for more information. return os.environ.get("GALAXY_DEPENDENCIES_INSTALL_WEASYPRINT") == "1" + def check_custos_sdk(self): + return 'custos' == self.vault_type + + def check_hvac(self): + return 'hashicorp' == self.vault_type + def optional(config_file=None): if not config_file: diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index defa88a033f0..8356a4699d8e 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -25,6 +25,10 @@ fs-gcsfs # type: googlecloudstorage fs-onedatafs # type: onedata fs-basespace # type: basespace +# Vault backend +hvac +custos-sdk + # Chronos client chronos-python==1.2.1 diff --git a/lib/galaxy/files/__init__.py b/lib/galaxy/files/__init__.py index 560cdbe1f64f..ebcac4c9942b 100644 --- a/lib/galaxy/files/__init__.py +++ b/lib/galaxy/files/__init__.py @@ -282,6 +282,18 @@ def is_admin(self): """Whether this user is an administrator.""" return self.trans.user_is_admin + @property + def user_vault(self): + """User vault namespace""" + user_vault = self.trans.user_vault + return user_vault or defaultdict(lambda: None) + + @property + def app_vault(self): + """App vault namespace""" + vault = self.trans.app.vault + return vault or defaultdict(lambda: None) + class DictFileSourcesUserContext(FileSourceDictifiable): @@ -315,3 +327,11 @@ def group_names(self): @property def is_admin(self): return self._kwd.get("is_admin") + + @property + def user_vault(self): + return self._kwd.get("user_vault") + + @property + def app_vault(self): + return self._kwd.get("app_vault") diff --git a/lib/galaxy/managers/context.py b/lib/galaxy/managers/context.py index 3bfb65365f9e..d4c850e70c8d 100644 --- a/lib/galaxy/managers/context.py +++ b/lib/galaxy/managers/context.py @@ -49,6 +49,7 @@ from galaxy.model.base import ModelMapping from galaxy.model.scoped_session import galaxy_scoped_session from galaxy.security.idencoding import IdEncodingHelper +from galaxy.security.vault import UserVaultWrapper from galaxy.structured_app import MinimalManagerApp from galaxy.util import bunch @@ -197,6 +198,11 @@ class ProvidesUserContext(ProvidesAppContext): def user(self): """Provide access to the user object.""" + @property + def user_vault(self): + """Provide access to a user's personal vault.""" + return UserVaultWrapper(self.app.vault, self.user) + @property def anonymous(self) -> bool: return self.user is None diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 848810a09325..cdca2dfafc55 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -77,6 +77,7 @@ from sqlalchemy.ext.orderinglist import ordering_list from sqlalchemy.orm import ( aliased, + backref, column_property, deferred, joinedload, @@ -8479,6 +8480,17 @@ class LibraryDatasetCollectionAnnotationAssociation(Base, RepresentById): user = relationship('User') +class Vault(Base): + __tablename__ = 'vault' + + key = Column(Text, primary_key=True) + parent_key = Column(Text, ForeignKey(key), index=True, nullable=True) + children = relationship('Vault', backref=backref('parent', remote_side=[key])) + value = Column(Text, nullable=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + + # Item rating classes. class ItemRatingAssociation(Base): __abstract__ = True diff --git a/lib/galaxy/model/migrate/versions/0180_add_vault_table.py b/lib/galaxy/model/migrate/versions/0180_add_vault_table.py new file mode 100644 index 000000000000..3006e90c74bc --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0180_add_vault_table.py @@ -0,0 +1,25 @@ +import datetime + +from sqlalchemy import Column, DateTime, ForeignKey, MetaData, Table, Text + +now = datetime.datetime.utcnow +meta = MetaData() + +vault = Table( + 'vault', meta, + Column('key', Text, primary_key=True), + Column('parent_key', Text, ForeignKey('vault.key'), index=True, nullable=True), + Column('value', Text, nullable=True), + Column("create_time", DateTime, default=now), + Column("update_time", DateTime, default=now, onupdate=now) +) + + +def upgrade(migrate_engine): + meta.bind = migrate_engine + vault.create() + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + vault.drop() diff --git a/lib/galaxy/security/vault.py b/lib/galaxy/security/vault.py new file mode 100644 index 000000000000..cf2e3a928de6 --- /dev/null +++ b/lib/galaxy/security/vault.py @@ -0,0 +1,279 @@ +import abc +import logging +import os +import re +from typing import List, Optional + +import yaml +from cryptography.fernet import Fernet, MultiFernet + +try: + from custos.clients.resource_secret_management_client import ResourceSecretManagementClient + from custos.clients.utils.exceptions.CustosExceptions import KeyDoesNotExist + from custos.transport.settings import CustosServerClientSettings + custos_sdk_available = True +except ImportError: + custos_sdk_available = False + +try: + import hvac +except ImportError: + hvac = None + +from galaxy import model + +log = logging.getLogger(__name__) + +VAULT_KEY_INVALID_REGEX = re.compile(r"\s\/|\/\s|\/\/") + + +class InvalidVaultConfigException(Exception): + pass + + +class InvalidVaultKeyException(Exception): + pass + + +class Vault(abc.ABC): + """ + A simple abstraction for reading/writing from external vaults. + """ + + @abc.abstractmethod + def read_secret(self, key: str) -> Optional[str]: + """ + Reads a secret from the vault. + + :param key: The key to read. Typically a hierarchical path such as `/galaxy/user/1/preferences/editor` + :return: The string value stored at the key, such as 'ace_editor'. + """ + pass + + @abc.abstractmethod + def write_secret(self, key: str, value: str) -> None: + """ + Write a secret to the vault. + + :param key: The key to write to. Typically a hierarchical path such as `/galaxy/user/1/preferences/editor` + :param value: The value to write, such as 'vscode' + :return: + """ + pass + + @abc.abstractmethod + def list_secrets(self, key: str) -> List[str]: + """ + Lists secrets at a given path. + + :param key: The key prefix to list. e.g. `/galaxy/user/1/preferences`. A trailing slash is optional. + :return: The list of subkeys at path. e.g. + ['/galaxy/user/1/preferences/editor`, '/galaxy/user/1/preferences/storage`] + Note that only immediate subkeys are returned. + """ + pass + + +class NullVault(Vault): + + def read_secret(self, key: str) -> Optional[str]: + raise InvalidVaultConfigException("No vault configured. Make sure the vault_config_file setting is defined in galaxy.yml") + + def write_secret(self, key: str, value: str) -> None: + raise InvalidVaultConfigException("No vault configured. Make sure the vault_config_file setting is defined in galaxy.yml") + + def list_secrets(self, key: str) -> List[str]: + raise NotImplementedError() + + +class HashicorpVault(Vault): + + def __init__(self, config): + if not hvac: + raise InvalidVaultConfigException("Hashicorp vault library 'hvac' is not available. Make sure hvac is installed.") + self.vault_address = config.get('vault_address') + self.vault_token = config.get('vault_token') + self.client = hvac.Client(url=self.vault_address, token=self.vault_token) + + def read_secret(self, key: str) -> Optional[str]: + try: + response = self.client.secrets.kv.read_secret_version(path=key) + return response['data']['data'].get('value') + except hvac.exceptions.InvalidPath: + return None + + def write_secret(self, key: str, value: str) -> None: + self.client.secrets.kv.v2.create_or_update_secret(path=key, secret={'value': value}) + + def list_secrets(self, key: str) -> List[str]: + raise NotImplementedError() + + +class DatabaseVault(Vault): + + def __init__(self, sa_session, config): + self.sa_session = sa_session + self.encryption_keys = config.get('encryption_keys') + self.fernet_keys = [Fernet(key.encode('utf-8')) for key in self.encryption_keys] + + def _get_multi_fernet(self) -> MultiFernet: + return MultiFernet(self.fernet_keys) + + def _update_or_create(self, key: str, value: Optional[str]) -> model.Vault: + vault_entry = self.sa_session.query(model.Vault).filter_by(key=key).first() + if vault_entry: + if value: + vault_entry.value = value + self.sa_session.merge(vault_entry) + self.sa_session.flush() + else: + # recursively create parent keys + parent_key, _, _ = key.rpartition("/") + if parent_key: + self._update_or_create(parent_key, None) + vault_entry = model.Vault(key=key, value=value, parent_key=parent_key or None) + self.sa_session.merge(vault_entry) + self.sa_session.flush() + return vault_entry + + def read_secret(self, key: str) -> Optional[str]: + key_obj = self.sa_session.query(model.Vault).filter_by(key=key).first() + if key_obj and key_obj.value: + f = self._get_multi_fernet() + return f.decrypt(key_obj.value.encode('utf-8')).decode('utf-8') + return None + + def write_secret(self, key: str, value: str) -> None: + f = self._get_multi_fernet() + token = f.encrypt(value.encode('utf-8')) + self._update_or_create(key=key, value=token.decode('utf-8')) + + def list_secrets(self, key: str) -> List[str]: + raise NotImplementedError() + + +class CustosVault(Vault): + + def __init__(self, config): + if not custos_sdk_available: + raise InvalidVaultConfigException("Custos sdk library 'custos-sdk' is not available. Make sure the custos-sdk is installed.") + custos_settings = CustosServerClientSettings(custos_host=config.get('custos_host'), + custos_port=config.get('custos_port'), + custos_client_id=config.get('custos_client_id'), + custos_client_sec=config.get('custos_client_sec')) + self.client = ResourceSecretManagementClient(custos_settings) + + def read_secret(self, key: str) -> Optional[str]: + try: + response = self.client.get_kv_credential(key=key) + return response.get('value') + except KeyDoesNotExist: + return None + + def write_secret(self, key: str, value: str) -> None: + self.client.set_kv_credential(key=key, value=value) + + def list_secrets(self, key: str) -> List[str]: + raise NotImplementedError() + + +class UserVaultWrapper(Vault): + + def __init__(self, vault: Vault, user): + self.vault = vault + self.user = user + + def read_secret(self, key: str) -> Optional[str]: + return self.vault.read_secret(f"user/{self.user.id}/{key}") + + def write_secret(self, key: str, value: str) -> None: + return self.vault.write_secret(f"user/{self.user.id}/{key}", value) + + def list_secrets(self, key: str) -> List[str]: + raise NotImplementedError() + + +class VaultKeyValidationWrapper(Vault): + """ + A decorator to standardize and validate vault key paths + """ + + def __init__(self, vault: Vault): + self.vault = vault + + @staticmethod + def validate_key(key): + if not key: + return False + return not VAULT_KEY_INVALID_REGEX.search(key) + + def normalize_key(self, key): + # remove leading and trailing slashes + key = key.strip("/") + if not self.validate_key(key): + raise InvalidVaultKeyException( + f"Vault key: {key} is invalid. Make sure that it is not empty, contains double slashes or contains" + "whitespace before or after the separator.") + return key + + def read_secret(self, key: str) -> Optional[str]: + key = self.normalize_key(key) + return self.vault.read_secret(key) + + def write_secret(self, key: str, value: str) -> None: + key = self.normalize_key(key) + return self.vault.write_secret(key, value) + + def list_secrets(self, key: str) -> List[str]: + raise NotImplementedError() + + +class VaultKeyPrefixWrapper(Vault): + """ + Adds a prefix to all vault keys, such as the galaxy instance id + """ + + def __init__(self, vault: Vault, prefix: str): + self.vault = vault + self.prefix = prefix.strip("/") + + def read_secret(self, key: str) -> Optional[str]: + return self.vault.read_secret(f"/{self.prefix}/{key}") + + def write_secret(self, key: str, value: str) -> None: + return self.vault.write_secret(f"/{self.prefix}/{key}", value) + + def list_secrets(self, key: str) -> List[str]: + raise NotImplementedError() + + +class VaultFactory(object): + + @staticmethod + def load_vault_config(vault_conf_yml: str) -> Optional[dict]: + if os.path.exists(vault_conf_yml): + with open(vault_conf_yml) as f: + return yaml.safe_load(f) + return None + + @staticmethod + def from_vault_type(app, vault_type: Optional[str], cfg: dict) -> Vault: + vault: Vault + if vault_type == "hashicorp": + vault = HashicorpVault(cfg) + elif vault_type == "database": + vault = DatabaseVault(app.model.context, cfg) + elif vault_type == "custos": + vault = CustosVault(cfg) + else: + raise InvalidVaultConfigException(f"Unknown vault type: {vault_type}") + vault_prefix = cfg.get('path_prefix') or "/galaxy" + return VaultKeyValidationWrapper(VaultKeyPrefixWrapper(vault, prefix=vault_prefix)) + + @staticmethod + def from_app(app) -> Vault: + vault_config = VaultFactory.load_vault_config(app.config.vault_config_file) + if vault_config: + return VaultFactory.from_vault_type(app, vault_config.get('type', None), vault_config) + log.warning("No vault configured. We recommend defining the vault_config_file setting in galaxy.yml") + return NullVault() diff --git a/lib/galaxy/structured_app.py b/lib/galaxy/structured_app.py index fc684d56def1..7aba729a5fd5 100644 --- a/lib/galaxy/structured_app.py +++ b/lib/galaxy/structured_app.py @@ -18,6 +18,7 @@ from galaxy.objectstore import ObjectStore from galaxy.quota import QuotaAgent from galaxy.security.idencoding import IdEncodingHelper +from galaxy.security.vault import Vault from galaxy.tool_util.deps.views import DependencyResolversView from galaxy.tool_util.verify import test_data from galaxy.util.dbkeys import GenomeBuilds @@ -115,6 +116,7 @@ class StructuredApp(MinimalManagerApp): security_agent: GalaxyRBACAgent host_security_agent: HostAgent trs_proxy: TrsProxy + vault: Vault webhooks_registry: WebhooksRegistry queue_worker: Any # 'galaxy.queue_worker.GalaxyQueueWorker' diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index dfc0f65a813b..6c6ba8187b64 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -30,6 +30,7 @@ validate_password, validate_publicname ) +from galaxy.security.vault import UserVaultWrapper from galaxy.tool_util.toolbox.filters import FilterFactory from galaxy.util import ( docstring_trim, @@ -288,7 +289,7 @@ def _get_extra_user_preferences(self, trans): """ return trans.app.config.user_preferences_extra['preferences'] - def _build_extra_user_pref_inputs(self, preferences, user): + def _build_extra_user_pref_inputs(self, trans, preferences, user): """ Build extra user preferences inputs list. Add values to the fields if present @@ -297,6 +298,7 @@ def _build_extra_user_pref_inputs(self, preferences, user): return [] extra_pref_inputs = list() # Build sections for different categories of inputs + user_vault = UserVaultWrapper(trans.app.vault, user) for item, value in preferences.items(): if value is not None: input_fields = copy.deepcopy(value["inputs"]) @@ -307,10 +309,19 @@ def _build_extra_user_pref_inputs(self, preferences, user): input['help'] = f"{help} {required}" else: input['help'] = required - field = f"{item}|{input['name']}" - for data_item in user.extra_preferences: - if field in data_item: - input['value'] = user.extra_preferences[data_item] + if input.get('store') == 'vault': + field = f"{item}/{input['name']}" + input['value'] = user_vault.read_secret(f'preferences/{field}') + else: + field = f"{item}|{input['name']}" + for data_item in user.extra_preferences: + if field in data_item: + input['value'] = user.extra_preferences[data_item] + # regardless of the store, do not send secret type values to client + if input.get('type') == 'secret': + input['value'] = "__SECRET_PLACEHOLDER__" + # let the client treat it as a password field + input['type'] = "password" extra_pref_inputs.append({'type': 'section', 'title': value['description'], 'name': item, 'expanded': True, 'inputs': input_fields}) return extra_pref_inputs @@ -382,7 +393,7 @@ def get_information(self, trans, id, **kwd): inputs.append(address_repeat) # Build input sections for extra user preferences - extra_user_pref = self._build_extra_user_pref_inputs(self._get_extra_user_preferences(trans), user) + extra_user_pref = self._build_extra_user_pref_inputs(trans, self._get_extra_user_preferences(trans), user) for item in extra_user_pref: inputs.append(item) else: @@ -460,20 +471,26 @@ def set_information(self, trans, id, payload=None, **kwd): # Update values for extra user preference items extra_user_pref_data = dict() extra_pref_keys = self._get_extra_user_preferences(trans) + user_vault = UserVaultWrapper(trans.app.vault, user) if extra_pref_keys is not None: for key in extra_pref_keys: key_prefix = f"{key}|" for item in payload: if item.startswith(key_prefix): - # Show error message if the required field is empty - if payload[item] == "": - # Raise an exception when a required field is empty while saving the form - keys = item.split("|") - section = extra_pref_keys[keys[0]] - for input in section['inputs']: - if input['name'] == keys[1] and input['required']: - raise exceptions.ObjectAttributeMissingException("Please fill the required field") - extra_user_pref_data[item] = payload[item] + keys = item.split("|") + section = extra_pref_keys[keys[0]] + matching_input = [input for input in section['inputs'] if input['name'] == keys[1]] + if matching_input: + input = matching_input[0] + if input.get('required') and payload[item] == "": + raise exceptions.ObjectAttributeMissingException("Please fill the required field") + if not (input.get('type') == 'secret' and payload[item] == "__SECRET_PLACEHOLDER__"): + if input.get('store') == 'vault': + user_vault.write_secret(f'preferences/{keys[0]}/{keys[1]}', str(payload[item])) + else: + extra_user_pref_data[item] = payload[item] + else: + extra_user_pref_data[item] = payload[item] user.preferences["extra_user_preferences"] = json.dumps(extra_user_pref_data) # Update user addresses diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 5c5894219f8c..3d7452367334 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -3482,3 +3482,11 @@ mapping: Location of New User Welcome data, a single directory containing the images and JSON of Topics/Subtopics/Slides as export. This location is relative to galaxy/static + + vault_config_file: + type: str + default: vault_conf.yml + path_resolves_to: config_dir + required: false + desc: | + Vault config file. diff --git a/test/integration/file_sources_conf_vault.yml b/test/integration/file_sources_conf_vault.yml new file mode 100644 index 000000000000..93e510b9e0af --- /dev/null +++ b/test/integration/file_sources_conf_vault.yml @@ -0,0 +1,8 @@ +- type: posix + id: test_user_vault + root: ${user.user_vault.read_secret('posix/root_path')} + label: a user level test path +- type: posix + id: test_app_vault + root: ${user.app_vault.read_secret('posix/root_path')} + label: an app level test path diff --git a/test/integration/test_config_defaults.py b/test/integration/test_config_defaults.py index 9359417ab460..ce52047eaf17 100644 --- a/test/integration/test_config_defaults.py +++ b/test/integration/test_config_defaults.py @@ -94,6 +94,7 @@ 'tool_sheds_config_file', 'trs_servers_config_file', 'user_preferences_extra_conf_path', + 'vault_config_file', 'webhooks_dir', 'workflow_resource_params_file', 'workflow_resource_params_mapper', @@ -131,6 +132,7 @@ 'tool_path': 'root_dir', 'tool_sheds_config_file': 'config_dir', 'user_preferences_extra_conf_path': 'config_dir', + 'vault_config_file': 'config_dir', 'workflow_resource_params_file': 'config_dir', 'workflow_schedulers_config_file': 'config_dir', } diff --git a/test/integration/test_vault_extra_prefs.py b/test/integration/test_vault_extra_prefs.py new file mode 100644 index 000000000000..62706ed0be81 --- /dev/null +++ b/test/integration/test_vault_extra_prefs.py @@ -0,0 +1,99 @@ +import json +import os +from typing import Any, cast + +from requests import ( + get, + put +) + +from galaxy_test.driver import integration_util + +TEST_USER_EMAIL = "vault_test_user@bx.psu.edu" + + +class ExtraUserPreferencesTestCase(integration_util.IntegrationTestCase): + + @classmethod + def handle_galaxy_config_kwds(cls, config): + config["vault_config_file"] = os.path.join(os.path.dirname(__file__), "vault_conf.yml") + config["user_preferences_extra_conf_path"] = os.path.join(os.path.dirname(__file__), "user_preferences_extra_conf.yml") + + def test_extra_prefs_vault_storage(self): + user = self._setup_user(TEST_USER_EMAIL) + url = self.__url("information/inputs", user) + app = cast(Any, self._test_driver.app if self._test_driver else None) + db_user = app.model.context.query(app.model.User).filter(app.model.User.email == user['email']).first() + + # create some initial data + put(url, data=json.dumps({ + "vaulttestsection|client_id": "hello_client_id", + "vaulttestsection|client_secret": "hello_client_secret", + "vaulttestsection|refresh_token": "a_super_secret_value", + })) + + # retrieve saved data + response = get(url).json() + + def get_input_by_name(inputs, name): + return [input for input in inputs if input['name'] == name][0] + + inputs = [section for section in response["inputs"] if section['name'] == 'vaulttestsection'][0]["inputs"] + + # value should be what we saved + input_client_id = get_input_by_name(inputs, 'client_id') + self.assertEqual(input_client_id['value'], "hello_client_id") + + # however, this value should not be in the vault + self.assertIsNone(app.vault.read_secret(f"user/{db_user.id}/preferences/vaulttestsection/client_id")) + # it should be in the user preferences model + self.assertEqual(db_user.extra_preferences['vaulttestsection|client_id'], "hello_client_id") + + # the secret however, was configured to be stored in the vault + input_client_secret = get_input_by_name(inputs, 'client_secret') + self.assertEqual(input_client_secret['value'], "hello_client_secret") + self.assertEqual(app.vault.read_secret( + f"user/{db_user.id}/preferences/vaulttestsection/client_secret"), "hello_client_secret") + # it should not be stored in the user preferences model + self.assertIsNone(db_user.extra_preferences['vaulttestsection|client_secret']) + + # secret type values should not be retrievable by the client + input_refresh_token = get_input_by_name(inputs, 'refresh_token') + self.assertNotEqual(input_refresh_token['value'], "a_super_secret_value") + self.assertEqual(input_refresh_token['value'], "__SECRET_PLACEHOLDER__") + + # however, that secret value should be correctly stored on the server + self.assertEqual(app.vault.read_secret( + f"user/{db_user.id}/preferences/vaulttestsection/refresh_token"), "a_super_secret_value") + + def test_extra_prefs_vault_storage_update_secret(self): + user = self._setup_user(TEST_USER_EMAIL) + url = self.__url("information/inputs", user) + app = cast(Any, self._test_driver.app if self._test_driver else None) + db_user = app.model.context.query(app.model.User).filter(app.model.User.email == user['email']).first() + + # write the initial secret value + put(url, data=json.dumps({ + "vaulttestsection|refresh_token": "a_new_secret_value", + })) + + # attempt to overwrite it with placeholder + put(url, data=json.dumps({ + "vaulttestsection|refresh_token": "__SECRET_PLACEHOLDER__", + })) + + # value should not have been overwritten + self.assertEqual(app.vault.read_secret( + f"user/{db_user.id}/preferences/vaulttestsection/refresh_token"), "a_new_secret_value") + + # write a new value + put(url, data=json.dumps({ + "vaulttestsection|refresh_token": "an_updated_secret_value", + })) + + # value should now be overwritten + self.assertEqual(app.vault.read_secret( + f"user/{db_user.id}/preferences/vaulttestsection/refresh_token"), "an_updated_secret_value") + + def __url(self, action, user): + return self._api_url(f"users/{user['id']}/{action}", params=dict(key=self.master_api_key)) diff --git a/test/integration/test_vault_file_source.py b/test/integration/test_vault_file_source.py new file mode 100644 index 000000000000..f8f6a9c330de --- /dev/null +++ b/test/integration/test_vault_file_source.py @@ -0,0 +1,126 @@ +import json +import os +import tempfile + +from galaxy.security.vault import UserVaultWrapper +from galaxy_test.base import api_asserts +from galaxy_test.base.populators import DatasetPopulator +from galaxy_test.driver import integration_util + + +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +FILE_SOURCES_VAULT_CONF = os.path.join(SCRIPT_DIRECTORY, "file_sources_conf_vault.yml") +VAULT_CONF = os.path.join(SCRIPT_DIRECTORY, "vault_conf.yml") + + +class VaultFileSourceIntegrationTestCase(integration_util.IntegrationTestCase): + USER_1_APP_VAULT_ENTRY = "randomvaultuser1@universe.com" + USER_2_APP_VAULT_ENTRY = "randomvaultuser2@universe.com" + + @classmethod + def handle_galaxy_config_kwds(cls, config): + config["file_sources_config_file"] = FILE_SOURCES_VAULT_CONF + config["vault_config_file"] = VAULT_CONF + config["user_library_import_symlink_allowlist"] = os.path.realpath(tempfile.mkdtemp()) + + def setUp(self): + super().setUp() + self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + + def test_vault_secret_per_user_in_file_source(self): + """ + This file source performs a user vault lookup. The secret stored for the first user is a + valid path and should succeed, while the second user's stored secret should fail. + """ + with self._different_user(email=self.USER_1_APP_VAULT_ENTRY): + app = self._app + user = app.model.context.query(app.model.User).filter( + app.model.User.email == self.USER_1_APP_VAULT_ENTRY).first() + user_vault = UserVaultWrapper(self._app.vault, user) + # use a valid symlink path so the posix list succeeds + user_vault.write_secret('posix/root_path', app.config.user_library_import_symlink_allowlist[0]) + + data = {"target": "gxfiles://test_user_vault"} + list_response = self.galaxy_interactor.get("remote_files", data) + api_asserts.assert_status_code_is_ok(list_response) + remote_files = list_response.json() + print(remote_files) + + with self._different_user(email=self.USER_2_APP_VAULT_ENTRY): + app = self._app + user = app.model.context.query(app.model.User).filter( + app.model.User.email == self.USER_2_APP_VAULT_ENTRY).first() + user_vault = UserVaultWrapper(self._app.vault, user) + # use an invalid symlink path so the posix list fails + user_vault.write_secret('posix/root_path', '/invalid/root') + + data = {"target": "gxfiles://test_user_vault"} + list_response = self.galaxy_interactor.get("remote_files", data) + api_asserts.assert_status_code_is(list_response, 404) + + def test_vault_secret_per_app_in_file_source(self): + """ + This file source performs an app level vault lookup. Although the secret stored for the first user is a + valid path and the second user's stored secret is invalid, we are performing an app level lookup + which should succeed for both users. + """ + # write app level secret + app = self._app + # use a valid symlink path so the posix list succeeds + app.vault.write_secret('posix/root_path', app.config.user_library_import_symlink_allowlist[0]) + + with self._different_user(email=self.USER_1_APP_VAULT_ENTRY): + user = app.model.context.query(app.model.User).filter( + app.model.User.email == self.USER_1_APP_VAULT_ENTRY).first() + user_vault = UserVaultWrapper(self._app.vault, user) + # use a valid symlink path so the posix list succeeds + user_vault.write_secret('posix/root_path', app.config.user_library_import_symlink_allowlist[0]) + + data = {"target": "gxfiles://test_app_vault"} + list_response = self.galaxy_interactor.get("remote_files", data) + api_asserts.assert_status_code_is_ok(list_response) + remote_files = list_response.json() + print(remote_files) + + with self._different_user(email=self.USER_2_APP_VAULT_ENTRY): + user = app.model.context.query(app.model.User).filter( + app.model.User.email == self.USER_2_APP_VAULT_ENTRY).first() + user_vault = UserVaultWrapper(self._app.vault, user) + # use an invalid symlink path so the posix list would fail if used + user_vault.write_secret('posix/root_path', '/invalid/root') + + data = {"target": "gxfiles://test_app_vault"} + list_response = self.galaxy_interactor.get("remote_files", data) + api_asserts.assert_status_code_is_ok(list_response) + remote_files = list_response.json() + print(remote_files) + + def test_upload_file_from_remote_source(self): + with self._different_user(email=self.USER_1_APP_VAULT_ENTRY): + app = self._app + user = app.model.context.query(app.model.User).filter( + app.model.User.email == self.USER_1_APP_VAULT_ENTRY).first() + user_vault = UserVaultWrapper(self._app.vault, user) + # use a valid symlink path so the posix list succeeds + user_vault.write_secret('posix/root_path', app.config.user_library_import_symlink_allowlist[0]) + data = {"target": "gxfiles://test_user_vault"} + with open(os.path.join(app.config.user_library_import_symlink_allowlist[0], 'a_file'), 'w') as fh: + fh.write('I require access to the vault') + list_response = self.galaxy_interactor.get("remote_files", data) + api_asserts.assert_status_code_is_ok(list_response) + remote_files = list_response.json() + assert len(remote_files) == 1 + with self.dataset_populator.test_history() as history_id: + element = dict(src="url", url="gxfiles://test_user_vault/a_file") + target = { + "destination": {"type": "hdas"}, + "elements": [element], + } + targets = json.dumps([target]) + payload = { + "history_id": history_id, + "targets": targets, + } + new_dataset = self.dataset_populator.fetch(payload, assert_ok=True).json()["outputs"][0] + content = self.dataset_populator.get_history_dataset_content(history_id, dataset=new_dataset) + assert content == "I require access to the vault", content diff --git a/test/integration/user_preferences_extra_conf.yml b/test/integration/user_preferences_extra_conf.yml new file mode 100644 index 000000000000..b678edeed82c --- /dev/null +++ b/test/integration/user_preferences_extra_conf.yml @@ -0,0 +1,23 @@ +preferences: + vaulttestsection: + description: A dummy extra prefs section + inputs: + - name: client_id + label: Client ID + type: text + required: True + - name: client_secret + label: Client Secret + type: password + store: vault + required: True + - name: access_token + label: Access token + type: password + store: vault + required: True + - name: refresh_token + label: Refresh Token + type: secret + store: vault + required: True diff --git a/test/integration/vault_conf.yml b/test/integration/vault_conf.yml new file mode 100644 index 000000000000..9064f7a4734e --- /dev/null +++ b/test/integration/vault_conf.yml @@ -0,0 +1,13 @@ +type: database +# Encryption keys must be valid fernet keys +# To generate a valid key: +# >>> from cryptography.fernet import Fernet +# >>> Fernet.generate_key() +# b'pZDP8_baVs3oWT4597HJWCysm49j-XELONQ-EdoU0DE=' +# +# Use the ascii string value as a key +# For more details, see: https://cryptography.io/en/latest/fernet/# +encryption_keys: + - 5RrT94ji178vQwha7TAmEix7DojtsLlxVz8Ef17KWgg= + - iNdXd7tRjLnSqRHxuhqQ98GTLU8HUbd5_Xx38iF8nZ0= + - IK83IXhE4_7W7xCFEtD9op0BAs11pJqYN236Spppp7g= diff --git a/test/unit/app/dependencies/test_deps.py b/test/unit/app/dependencies/test_deps.py index c97f81a18e08..216cd0308b72 100644 --- a/test/unit/app/dependencies/test_deps.py +++ b/test/unit/app/dependencies/test_deps.py @@ -29,6 +29,12 @@ runner1: load: job_runner_A """ +VAULT_CONF_CUSTOS = """ +type: custos +""" +VAULT_CONF_HASHICORP = """ +type: hashicorp +""" def test_default_objectstore(): @@ -95,6 +101,28 @@ def test_yaml_jobconf_runners(): assert 'job_runner_A' in cds.job_runners +def test_vault_custos_configured(): + with _config_context() as cc: + vault_conf = cc.write_config("vault_conf.yml", VAULT_CONF_CUSTOS) + config = { + "vault_config_file": vault_conf, + } + cds = cc.get_cond_deps(config=config) + assert cds.check_custos_sdk() + assert not cds.check_hvac() + + +def test_vault_hashicorp_configured(): + with _config_context() as cc: + vault_conf = cc.write_config("vault_conf.yml", VAULT_CONF_HASHICORP) + config = { + "vault_config_file": vault_conf, + } + cds = cc.get_cond_deps(config=config) + assert cds.check_hvac() + assert not cds.check_custos_sdk() + + @contextmanager def _config_context(): config_dir = mkdtemp() diff --git a/test/unit/config/test_config_values.py b/test/unit/config/test_config_values.py index 259f9afedacf..73ee753583e6 100644 --- a/test/unit/config/test_config_values.py +++ b/test/unit/config/test_config_values.py @@ -157,6 +157,7 @@ def _load_paths(self): 'tool_test_data_directories': self._in_root_dir('test-data'), 'trs_servers_config_file': self._in_config_dir('trs_servers_conf.yml'), 'user_preferences_extra_conf_path': self._in_config_dir('user_preferences_extra_conf.yml'), + 'vault_config_file': self._in_config_dir('vault_conf.yml'), 'workflow_resource_params_file': self._in_config_dir('workflow_resource_params_conf.xml'), 'workflow_schedulers_config_file': self._in_config_dir('workflow_schedulers_conf.xml'), } diff --git a/test/unit/data/model/mapping/test_model_mapping.py b/test/unit/data/model/mapping/test_model_mapping.py index b381093fb737..8793c46df297 100644 --- a/test/unit/data/model/mapping/test_model_mapping.py +++ b/test/unit/data/model/mapping/test_model_mapping.py @@ -5447,6 +5447,26 @@ def test_relationships(self, session, cls_, user, role): assert stored_obj.role.id == role.id +class TestVault(BaseTest): + def test_table(self, cls_): + assert cls_.__tablename__ == "vault" + + def test_columns(self, session, cls_): + create_time = update_time = datetime.now() + key = '/some/path' + parent_key = '/some' + value = 'helloworld' + obj = cls_(create_time=create_time, update_time=update_time, key=key, parent_key=parent_key, value=value) + + with dbcleanup(session, obj, where_clause=cls_.key == key): + stored_obj = get_stored_obj(session, cls_, where_clause=cls_.key == key) + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.key == key + assert stored_obj.parent_key == parent_key + assert stored_obj.value == value + + class TestVisualization(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == "visualization" diff --git a/test/unit/security/fixtures/vault_conf_custos.yml b/test/unit/security/fixtures/vault_conf_custos.yml new file mode 100644 index 000000000000..ef1b0d91a0df --- /dev/null +++ b/test/unit/security/fixtures/vault_conf_custos.yml @@ -0,0 +1,6 @@ +type: custos +path_prefix: /my_galaxy_instance +custos_host: service.staging.usecustos.org +custos_port: 30170 +custos_client_id: ${custos_client_id} +custos_client_sec: ${custos_client_secret} diff --git a/test/unit/security/fixtures/vault_conf_database.yml b/test/unit/security/fixtures/vault_conf_database.yml new file mode 100644 index 000000000000..bebc81a3b693 --- /dev/null +++ b/test/unit/security/fixtures/vault_conf_database.yml @@ -0,0 +1,14 @@ +type: database +path_prefix: /my_galaxy_instance +# Encryption keys must be valid fernet keys +# To generate a valid key: +# >>> from cryptography.fernet import Fernet +# >>> Fernet.generate_key() +# b'pZDP8_baVs3oWT4597HJWCysm49j-XELONQ-EdoU0DE=' +# +# Use the ascii string value as a key +# For more details, see: https://cryptography.io/en/latest/fernet/# +encryption_keys: + - 5RrT94ji178vQwha7TAmEix7DojtsLlxVz8Ef17KWgg= + - iNdXd7tRjLnSqRHxuhqQ98GTLU8HUbd5_Xx38iF8nZ0= + - IK83IXhE4_7W7xCFEtD9op0BAs11pJqYN236Spppp7g= diff --git a/test/unit/security/fixtures/vault_conf_database_invalid_keys.yml b/test/unit/security/fixtures/vault_conf_database_invalid_keys.yml new file mode 100644 index 000000000000..925e065a0153 --- /dev/null +++ b/test/unit/security/fixtures/vault_conf_database_invalid_keys.yml @@ -0,0 +1,13 @@ +type: database +path_prefix: /my_galaxy_instance +# Encryption keys must be valid fernet keys +# To generate a valid key: +# >>> from cryptography.fernet import Fernet +# >>> Fernet.generate_key() +# b'pZDP8_baVs3oWT4597HJWCysm49j-XELONQ-EdoU0DE=' +# +# Use the ascii string value as a key +# For more details, see: https://cryptography.io/en/latest/fernet/# +encryption_keys: + - aXOrK0R8eXRztiy8CHo-fNnwKhBBhMSS8cPv4JY4jOQ= + - 4KSQIhUU5W8oK36XxVBxXanA2Ge9Yu4ofe4-328E11o= diff --git a/test/unit/security/fixtures/vault_conf_database_rotated.yml b/test/unit/security/fixtures/vault_conf_database_rotated.yml new file mode 100644 index 000000000000..d11af0e2e5df --- /dev/null +++ b/test/unit/security/fixtures/vault_conf_database_rotated.yml @@ -0,0 +1,15 @@ +type: database +path_prefix: /my_galaxy_instance +# Encryption keys must be valid fernet keys +# To generate a valid key: +# >>> from cryptography.fernet import Fernet +# >>> Fernet.generate_key() +# b'pZDP8_baVs3oWT4597HJWCysm49j-XELONQ-EdoU0DE=' +# +# Use the ascii string value as a key +# For more details, see: https://cryptography.io/en/latest/fernet/# +encryption_keys: + - 5peww5oT8NMpxE31LpTiEE8qhccy_pPl4GW8iYu9gzU= + - 5RrT94ji178vQwha7TAmEix7DojtsLlxVz8Ef17KWgg= + - iNdXd7tRjLnSqRHxuhqQ98GTLU8HUbd5_Xx38iF8nZ0= + - IK83IXhE4_7W7xCFEtD9op0BAs11pJqYN236Spppp7g= diff --git a/test/unit/security/fixtures/vault_conf_hashicorp.yml b/test/unit/security/fixtures/vault_conf_hashicorp.yml new file mode 100644 index 000000000000..e578a15f8f6a --- /dev/null +++ b/test/unit/security/fixtures/vault_conf_hashicorp.yml @@ -0,0 +1,10 @@ +# to run tests, start vault with: +# $ vault server -dev -dev-root-token-id=vault_application_token +# The Vault application token can is obtained using command `vault token create`. +# If running the vault server in -dev mode, a token is displayed at startup as a root token. +# For more info on how to obtain Vault tokens, see the +# [Vault documentation](https://learn.hashicorp.com/tutorials/vault/getting-started-authentication?in=vault/getting-started#token-authentication). +type: hashicorp +path_prefix: /my_galaxy_instance +vault_address: ${vault_address} +vault_token: ${vault_token} diff --git a/test/unit/security/test_vault.py b/test/unit/security/test_vault.py new file mode 100644 index 000000000000..6b694472c124 --- /dev/null +++ b/test/unit/security/test_vault.py @@ -0,0 +1,116 @@ +import os +import string +import tempfile +import unittest +from abc import ABC + +from cryptography.fernet import InvalidToken + +from galaxy.model.unittest_utils.data_app import GalaxyDataTestApp, GalaxyDataTestConfig +from galaxy.security.vault import InvalidVaultKeyException, Vault, VaultFactory + + +class VaultTestBase(ABC): + vault: Vault + + def test_read_write_secret(self): + self.vault.write_secret("my/test/secret", "hello world") + self.assertEqual(self.vault.read_secret("my/test/secret"), "hello world") # type: ignore + + def test_overwrite_secret(self): + self.vault.write_secret("my/new/secret", "hello world") + self.vault.write_secret("my/new/secret", "hello overwritten") + self.assertEqual(self.vault.read_secret("my/new/secret"), "hello overwritten") # type: ignore + + def test_valid_paths(self): + with self.assertRaises(InvalidVaultKeyException): # type: ignore + self.vault.write_secret("", "hello world") + with self.assertRaises(InvalidVaultKeyException): # type: ignore + self.vault.write_secret("my//new/secret", "hello world") + with self.assertRaises(InvalidVaultKeyException): # type: ignore + self.vault.write_secret("my/ /new/secret", "hello world") + # leading and trailing slashes should be ignored + self.vault.write_secret("/my/new/secret with space/", "hello overwritten") + self.assertEqual(self.vault.read_secret("my/new/secret with space"), "hello overwritten") # type: ignore + + +VAULT_CONF_HASHICORP = os.path.join(os.path.dirname(__file__), "fixtures/vault_conf_hashicorp.yml") + + +@unittest.skipIf(not os.environ.get('VAULT_ADDRESS') or not os.environ.get('VAULT_TOKEN'), + "VAULT_ADDRESS and VAULT_TOKEN env vars not set") +class TestHashicorpVault(VaultTestBase, unittest.TestCase): + + def setUp(self) -> None: + with tempfile.NamedTemporaryFile( + mode="w", prefix="vault_hashicorp", delete=False) as tempconf, open(VAULT_CONF_HASHICORP) as f: + content = string.Template(f.read()).safe_substitute( + vault_address=os.environ.get('VAULT_ADDRESS'), + vault_token=os.environ.get('VAULT_TOKEN')) + tempconf.write(content) + self.vault_temp_conf = tempconf.name + config = GalaxyDataTestConfig(vault_config_file=self.vault_temp_conf) + app = GalaxyDataTestApp(config=config) + self.vault = VaultFactory.from_app(app) + + def tearDown(self) -> None: + os.remove(self.vault_temp_conf) + + +VAULT_CONF_DATABASE = os.path.join(os.path.dirname(__file__), "fixtures/vault_conf_database.yml") +VAULT_CONF_DATABASE_ROTATED = os.path.join(os.path.dirname(__file__), "fixtures/vault_conf_database_rotated.yml") +VAULT_CONF_DATABASE_INVALID = os.path.join(os.path.dirname(__file__), "fixtures/vault_conf_database_invalid_keys.yml") + + +class TestDatabaseVault(VaultTestBase, unittest.TestCase): + + def setUp(self) -> None: + config = GalaxyDataTestConfig(vault_config_file=VAULT_CONF_DATABASE) + app = GalaxyDataTestApp(config=config) + self.vault = VaultFactory.from_app(app) + + def test_rotate_keys(self): + config = GalaxyDataTestConfig(vault_config_file=VAULT_CONF_DATABASE) + app = GalaxyDataTestApp(config=config) + vault = VaultFactory.from_app(app) + vault.write_secret("my/rotated/secret", "hello rotated") + + # should succeed after rotation + app.config.vault_config_file = VAULT_CONF_DATABASE_ROTATED # type: ignore + vault = VaultFactory.from_app(app) + self.assertEqual(vault.read_secret("my/rotated/secret"), "hello rotated") + + def test_wrong_keys(self): + config = GalaxyDataTestConfig(vault_config_file=VAULT_CONF_DATABASE) + app = GalaxyDataTestApp(config=config) + vault = VaultFactory.from_app(app) + vault.write_secret("my/incorrect/secret", "hello incorrect") + + # should fail because decryption keys are the wrong + app.config.vault_config_file = VAULT_CONF_DATABASE_INVALID # type: ignore + vault = VaultFactory.from_app(app) + with self.assertRaises(InvalidToken): + vault.read_secret("my/incorrect/secret") + + +VAULT_CONF_CUSTOS = os.path.join(os.path.dirname(__file__), "fixtures/vault_conf_custos.yml") + + +@unittest.skipIf(not os.environ.get('CUSTOS_CLIENT_ID') or not os.environ.get('CUSTOS_CLIENT_SECRET'), + "CUSTOS_CLIENT_ID and CUSTOS_CLIENT_SECRET env vars not set") +class TestCustosVault(VaultTestBase, unittest.TestCase): + + def setUp(self) -> None: + with tempfile.NamedTemporaryFile( + mode="w", prefix="vault_custos", delete=False) as tempconf, open(VAULT_CONF_CUSTOS) as f: + content = string.Template(f.read()).safe_substitute( + custos_client_id=os.environ.get('CUSTOS_CLIENT_ID'), + custos_client_secret=os.environ.get('CUSTOS_CLIENT_SECRET')) + tempconf.write(content) + self.vault_temp_conf = tempconf.name + config = GalaxyDataTestConfig(vault_config_file=self.vault_temp_conf) + app = GalaxyDataTestApp(config=config) + self.vault = VaultFactory.from_app(app) + + def tearDown(self) -> None: + os.remove(self.vault_temp_conf)