Skip to content

Commit

Permalink
Merge branch 'master' of github.com:quiltdata/quilt into crossbucket-…
Browse files Browse the repository at this point in the history 
…push-frontend
  • Loading branch information
@fiskus
fiskus committed Dec 2, 2020
2 parents ed3f22f + e22cdcc commit dbd68c3
Show file tree
Hide file tree
Showing 6 changed files with 280 additions and 125 deletions.
4 changes: 4 additions & 0 deletions api/python/quilt3/backends/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,10 @@ def shorten_top_hash(self, pkg_name: str, top_hash: str) -> str:
def workflow_conf_pk(self) -> PhysicalKey:
return self.base.join(self.workflow_conf_path)

def get_workflow_validator(self):
from quilt3.workflows import WorkflowValidator
return WorkflowValidator.load(self.workflow_conf_pk)


class PackageRegistryV1(PackageRegistry):
root_path = '.quilt'
Expand Down
174 changes: 98 additions & 76 deletions api/python/quilt3/workflows/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,89 +32,111 @@ def _schema_load_object_hook(o):
_load_schema_json = json.JSONDecoder(object_hook=_schema_load_object_hook).decode


class WorkflowValidator:
def __init__(self, config: dict, physical_key: util.PhysicalKey):
"""
Args:
config: validated workflow config or `None` if there is no config
physical_key: from where config was loaded
"""
self.config = config
self.physical_key = physical_key

@classmethod
def load(cls, pk: util.PhysicalKey):
data = None
try:
data, pk = get_bytes_and_effective_pk(pk)
except FileNotFoundError:
pass
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] != 'NoSuchKey':
raise util.QuiltException(f"Couldn't load workflows config. {e}.")
if data is None:
return

try:
# TODO: raise if objects contain duplicate properties
config = yaml.safe_load(data.decode())
except yaml.YAMLError as e:
raise util.QuiltException("Couldn't parse workflows config as YAML.") from e
conf_validator = _get_conf_validator()
try:
conf_validator(config)
except jsonschema.ValidationError as e:
raise util.QuiltException(f'Workflows config failed validation: {e.message}.') from e

return cls(config, pk)

def validate(self, workflow, meta, message):
if workflow is ...:
workflow = self.config.get('default_workflow')

result = {
'id': workflow,
'config': str(self.physical_key),
}
if workflow is None:
if self.config.get('is_workflow_required', True):
raise util.QuiltException('Workflow required, but none specified.')
return result

workflows_data = self.config['workflows']
if workflow not in workflows_data:
raise util.QuiltException(f'There is no {workflow!r} workflow in config.')
workflow_data = workflows_data[workflow]
metadata_schema_id = workflow_data.get('metadata_schema')
if metadata_schema_id:
schemas = self.config.get('schemas', {})
if metadata_schema_id not in schemas:
raise util.QuiltException(f'There is no {metadata_schema_id!r} in schemas.')
schema_url = schemas[metadata_schema_id]['url']
try:
schema_pk = util.PhysicalKey.from_url(schema_url)
except util.URLParseError as e:
raise util.QuiltException(f"Couldn't parse URL {schema_url!r}. {e}.")
if schema_pk.is_local() and not self.physical_key.is_local():
raise util.QuiltException(f"Local schema {str(schema_pk)!r} can't be used on the remote registry.")

handled_exception = (OSError if schema_pk.is_local() else botocore.exceptions.ClientError)
try:
schema_data, schema_pk_to_store = get_bytes_and_effective_pk(schema_pk)
except handled_exception as e:
raise util.QuiltException(f"Couldn't load schema at {schema_pk}. {e}.")
try:
schema = _load_schema_json(schema_data.decode())
except json.JSONDecodeError as e:
raise util.QuiltException(f"Couldn't parse {schema_pk} as JSON. {e}.")

validator_cls = jsonschema.Draft7Validator
if isinstance(schema, dict) and '$schema' in schema:
meta_schema = schema['$schema']
if not isinstance(meta_schema, str):
raise util.QuiltException('$schema must be a string.')
validator_cls = SUPPORTED_META_SCHEMAS.get(meta_schema)
if validator_cls is None:
raise util.QuiltException(f"Unsupported meta-schema: {meta_schema}.")
try:
jsonschema.validate(meta, schema, cls=validator_cls)
except jsonschema.ValidationError as e:
raise util.QuiltException(f"Metadata failed validation: {e.message}.")
result['schemas'] = {metadata_schema_id: str(schema_pk_to_store)}
if workflow_data.get('is_message_required', False) and not message:
raise util.QuiltException('Commit message is required by workflow, but none was provided.')

return result


def validate(*, registry: PackageRegistry, workflow, meta, message):
# workflow is ... => no workflow provided by user;
# workflow is None => don't use any workflow.
if not (workflow in (None, ...) or isinstance(workflow, str)):
raise TypeError

conf_info = None
try:
conf_info = get_bytes_and_effective_pk(registry.workflow_conf_pk)
except FileNotFoundError:
pass
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] != 'NoSuchKey':
raise util.QuiltException(f"Couldn't load workflows config. {e}.")
if conf_info is None:
workflow_validator = registry.get_workflow_validator()
if workflow_validator is None:
if workflow is ...:
return
raise util.QuiltException(f'{workflow!r} workflow is specified, but no workflows config exist.')
conf_data, conf_pk = conf_info
try:
# TODO: raise if objects contain duplicate properties
conf_data = yaml.safe_load(conf_data.decode())
except yaml.YAMLError as e:
raise util.QuiltException("Couldn't parse workflows config as YAML.") from e
conf_validator = _get_conf_validator()
try:
conf_validator(conf_data)
except jsonschema.ValidationError as e:
raise util.QuiltException(f'Workflows config failed validation: {e.message}.') from e

if workflow is ...:
workflow = conf_data.get('default_workflow')

result = {
'id': workflow,
'config': str(conf_pk),
}
if workflow is None:
if conf_data.get('is_workflow_required', True):
raise util.QuiltException('Workflow required, but none specified.')
return result

workflows_data = conf_data['workflows']
if workflow not in workflows_data:
raise util.QuiltException(f'There is no {workflow!r} workflow in config.')
workflow_data = workflows_data[workflow]
metadata_schema_id = workflow_data.get('metadata_schema')
if metadata_schema_id:
schemas = conf_data.get('schemas', {})
if metadata_schema_id not in schemas:
raise util.QuiltException(f'There is no {metadata_schema_id!r} in schemas.')
schema_url = schemas[metadata_schema_id]['url']
try:
schema_pk = util.PhysicalKey.from_url(schema_url)
except util.URLParseError as e:
raise util.QuiltException(f"Couldn't parse URL {schema_url!r}. {e}.")
if schema_pk.is_local() and not registry.is_local:
raise util.QuiltException(f"Local schema {str(schema_pk)!r} can't be used on the remote registry.")

handled_exception = (OSError if schema_pk.is_local() else botocore.exceptions.ClientError)
try:
schema_data, schema_pk_to_store = get_bytes_and_effective_pk(schema_pk)
except handled_exception as e:
raise util.QuiltException(f"Couldn't load schema at {schema_pk}. {e}.")
try:
schema = _load_schema_json(schema_data.decode())
except json.JSONDecodeError as e:
raise util.QuiltException(f"Couldn't parse {schema_pk} as JSON. {e}.")

validator_cls = jsonschema.Draft7Validator
if isinstance(schema, dict) and '$schema' in schema:
meta_schema = schema['$schema']
if not isinstance(meta_schema, str):
raise util.QuiltException('$schema must be a string.')
validator_cls = SUPPORTED_META_SCHEMAS.get(meta_schema)
if validator_cls is None:
raise util.QuiltException(f"Unsupported meta-schema: {meta_schema}.")
try:
jsonschema.validate(meta, schema, cls=validator_cls)
except jsonschema.ValidationError as e:
raise util.QuiltException(f"Metadata failed validation: {e.message}.")
result['schemas'] = {metadata_schema_id: str(schema_pk_to_store)}
if workflow_data.get('is_message_required', False) and not message:
raise util.QuiltException('Commit message is required by workflow, but none was provided.')

return result
return workflow_validator.validate(workflow, meta, message)
18 changes: 14 additions & 4 deletions api/python/quilt3/workflows/config-1.schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,11 +78,16 @@
"description": "Buckets usable as destination with \"Push to Bucket\" in web UI.",
"examples": [
{
"s3://some-bucket-1": {
"title": "Staging"
"s3://bucket1": {
"title": "Bucket 1 Title",
"copy_data": true
},
"s3://some-bucket-2": {
"title": "Production"
"s3://bucket2": {
"title": "Bucket 2 Title",
"copy_data": false
},
"s3://bucket3": {
"title": "Bucket 3 Title (`copy_data` defaults to `true`)"
}
}
],
Expand All @@ -99,6 +104,11 @@
"title": {
"type": "string",
"minLength": 1
},
"copy_data": {
"type": "boolean",
"default": true,
"description": "If true, all package entries will be copied to the destination bucket. If false, all entries will remain in their current locations."
}
}
}
Expand Down
5 changes: 2 additions & 3 deletions catalog/app/containers/Bucket/PackageCreateDialog.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -473,9 +473,7 @@ function PackageCreateDialog({ bucket, open, workflowsConfig, onClose, refresh }
scroll="body"
onExited={reset(form)}
>
<M.DialogTitle>
{success ? 'Package created' : 'Create package'}
</M.DialogTitle>
<M.DialogTitle>{success ? 'Package created' : 'Create package'}</M.DialogTitle>
{success ? (
<>
<M.DialogContent style={{ paddingTop: 0 }}>
Expand Down Expand Up @@ -564,6 +562,7 @@ function PackageCreateDialog({ bucket, open, workflowsConfig, onClose, refresh }
validate={validate}
validateFields={['meta']}
isEqual={R.equals}
initialValue={PD.EMPTY_META_VALUE}
/>
),
_: () => <PD.MetaInputSkeleton />,
Expand Down
Loading

0 comments on commit dbd68c3

Please sign in to comment.