Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Revert highlevel exceptions #1426

Merged
merged 1 commit into from
Dec 1, 2023
Merged

Revert highlevel exceptions #1426

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 16 additions & 37 deletions superduperdb/backends/local/artifacts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@

from superduperdb import logging
from superduperdb.backends.base.artifact import ArtifactStore
from superduperdb.base import exceptions
from superduperdb.misc.colors import Colors


Expand Down Expand Up @@ -38,51 +37,31 @@ def delete(self, file_id: str):
Delete artifact from artifact store
:param file_id: File id uses to identify artifact in store
"""
try:
os.remove(f'{self.conn}/{file_id}')
except Exception as e:
raise exceptions.ArtifactStoreDeleteException(
f'Error while deleting {file_id}'
) from e
os.remove(f'{self.conn}/{file_id}')

def drop(self, force: bool = False):
"""
Drop the artifact store.
"""
try:
if not force:
if not click.confirm(
f'{Colors.RED}[!!!WARNING USE WITH CAUTION AS YOU '
f'WILL LOSE ALL DATA!!!]{Colors.RESET} '
'Are you sure you want to drop all artifacts? ',
default=False,
):
logging.warn('Aborting...')
shutil.rmtree(self.conn, ignore_errors=force)
except Exception as e:
raise exceptions.ArtifactStoreDeleteException(
'Error while dropping in artifact store'
) from e
if not force:
if not click.confirm(
f'{Colors.RED}[!!!WARNING USE WITH CAUTION AS YOU '
f'WILL LOSE ALL DATA!!!]{Colors.RESET} '
'Are you sure you want to drop all artifacts? ',
default=False,
):
logging.warn('Aborting...')
shutil.rmtree(self.conn, ignore_errors=force)

def save_artifact(self, serialized: bytes) -> t.Any:
try:
h = uuid.uuid4().hex
with open(os.path.join(self.conn, h), 'wb') as f:
f.write(serialized)
return h
except Exception as e:
raise exceptions.ArtifactStoreSaveException(
'Error while saving artifacts'
) from e
h = uuid.uuid4().hex
with open(os.path.join(self.conn, h), 'wb') as f:
f.write(serialized)
return h

def load_bytes(self, file_id: str) -> bytes:
try:
with open(os.path.join(self.conn, file_id), 'rb') as f:
return f.read()
except Exception as e:
raise exceptions.ArtifactStoreLoadException(
'Error while loading artifacts'
) from e
with open(os.path.join(self.conn, file_id), 'rb') as f:
return f.read()

def disconnect(self):
"""
Expand Down
45 changes: 12 additions & 33 deletions superduperdb/backends/mongodb/artifacts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@

from superduperdb import logging
from superduperdb.backends.base.artifact import ArtifactStore
from superduperdb.base import exceptions
from superduperdb.misc.colors import Colors


Expand All @@ -24,44 +23,24 @@ def url(self):
return self.conn.HOST + ':' + str(self.conn.PORT) + '/' + self.name

def drop(self, force: bool = False):
try:
if not force:
if not click.confirm(
f'{Colors.RED}[!!!WARNING USE WITH CAUTION AS YOU '
f'WILL LOSE ALL DATA!!!]{Colors.RESET} '
'Are you sure you want to drop all artifacts? ',
default=False,
):
logging.warn('Aborting...')
return self.db.client.drop_database(self.db.name)
except Exception as e:
raise exceptions.ArtifactStoreDeleteException(
'Error while dropping in artifact store'
) from e
if not force:
if not click.confirm(
f'{Colors.RED}[!!!WARNING USE WITH CAUTION AS YOU '
f'WILL LOSE ALL DATA!!!]{Colors.RESET} '
'Are you sure you want to drop all artifacts? ',
default=False,
):
logging.warn('Aborting...')
return self.db.client.drop_database(self.db.name)

def delete(self, file_id: str):
try:
return self.filesystem.delete(file_id)
except Exception as e:
raise exceptions.ArtifactStoreDeleteException(
'Error while dropping in artifact store'
) from e
return self.filesystem.delete(file_id)

def load_bytes(self, file_id: str):
try:
return self.filesystem.get(file_id).read()
except Exception as e:
raise exceptions.ArtifactStoreLoadException(
'Error while saving artifacts'
) from e
return self.filesystem.get(file_id).read()

def save_artifact(self, serialized: bytes):
try:
return self.filesystem.put(serialized)
except Exception as e:
raise exceptions.ArtifactStoreSaveException(
'Error while saving artifacts'
) from e
return self.filesystem.put(serialized)

def disconnect(self):
"""
Expand Down
95 changes: 26 additions & 69 deletions superduperdb/backends/mongodb/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

from superduperdb import logging
from superduperdb.backends.base.metadata import MetaDataStore
from superduperdb.base import exceptions
from superduperdb.components.component import Component
from superduperdb.misc.colors import Colors

Expand Down Expand Up @@ -37,54 +36,34 @@ def url(self):
return self.conn.HOST + ':' + str(self.conn.PORT) + '/' + self.name

def drop(self, force: bool = False):
try:
if not force:
if not click.confirm(
f'{Colors.RED}[!!!WARNING USE WITH CAUTION AS YOU '
f'WILL LOSE ALL DATA!!!]{Colors.RESET} '
'Are you sure you want to drop all meta-data? ',
default=False,
):
logging.warn('Aborting...')
self.db.drop_collection(self.meta_collection.name)
self.db.drop_collection(self.component_collection.name)
self.db.drop_collection(self.job_collection.name)
self.db.drop_collection(self.parent_child_mappings.name)
except Exception as e:
raise exceptions.MetaDataStoreDeleteException(
'Error while dropping in metadata store'
) from e
if not force:
if not click.confirm(
f'{Colors.RED}[!!!WARNING USE WITH CAUTION AS YOU '
f'WILL LOSE ALL DATA!!!]{Colors.RESET} '
'Are you sure you want to drop all meta-data? ',
default=False,
):
logging.warn('Aborting...')
self.db.drop_collection(self.meta_collection.name)
self.db.drop_collection(self.component_collection.name)
self.db.drop_collection(self.job_collection.name)
self.db.drop_collection(self.parent_child_mappings.name)

def create_parent_child(self, parent: str, child: str) -> None:
try:
self.parent_child_mappings.insert_one(
{
'parent': parent,
'child': child,
}
)
except Exception as e:
raise exceptions.MetaDataStoreDeleteException(
'Error while creating parent child'
) from e
self.parent_child_mappings.insert_one(
{
'parent': parent,
'child': child,
}
)

def create_component(self, info: t.Dict) -> InsertOneResult:
try:
if 'hidden' not in info:
info['hidden'] = False
return self.component_collection.insert_one(info)
except Exception as e:
raise exceptions.MetaDataStoreCreateException(
'Error while creating component in metadata store'
) from e
if 'hidden' not in info:
info['hidden'] = False
return self.component_collection.insert_one(info)

def create_job(self, info: t.Dict) -> InsertOneResult:
try:
return self.job_collection.insert_one(info)
except Exception as e:
raise exceptions.MetaDataStoreJobException(
'Error while creating job in metadata store'
) from e
return self.job_collection.insert_one(info)

def get_parent_child_relations(self):
c = self.parent_child_mappings.find()
Expand All @@ -94,38 +73,16 @@ def get_component_version_children(self, unique_id: str):
return self.parent_child_mappings.distinct('child', {'parent': unique_id})

def get_job(self, identifier: str):
try:
return self.job_collection.find_one({'identifier': identifier})
except Exception as e:
raise exceptions.MetaDataStoreJobException(
'Error while getting job in metadata store'
) from e
return self.job_collection.find_one({'identifier': identifier})

def create_metadata(self, key: str, value: str):
try:
return self.meta_collection.insert_one({'key': key, 'value': value})
except Exception as e:
raise exceptions.MetaDataStoreCreateException(
'Error while creating metadata in metadata store'
) from e
return self.meta_collection.insert_one({'key': key, 'value': value})

def get_metadata(self, key: str):
try:
return self.meta_collection.find_one({'key': key})['value']
except Exception as e:
raise exceptions.MetadatastoreException(
'Error while getting metadata in metadata store'
) from e
return self.meta_collection.find_one({'key': key})['value']

def update_metadata(self, key: str, value: str):
try:
return self.meta_collection.update_one(
{'key': key}, {'$set': {'value': value}}
)
except Exception as e:
raise exceptions.MetaDataStoreUpdateException(
'Error while updating metadata in metadata store'
) from e
return self.meta_collection.update_one({'key': key}, {'$set': {'value': value}})

def get_latest_version(
self, type_id: str, identifier: str, allow_hidden: bool = False
Expand Down
Loading