From 29153107f6816cbee6baf07ac24be66d4a2ecdcc Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 7 Sep 2021 13:05:57 +0100 Subject: [PATCH 1/3] Add test Loom file which fails metadata setting --- .../test/test_binary_loom_spec_version.loom | Bin 0 -> 15876 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lib/galaxy/datatypes/test/test_binary_loom_spec_version.loom diff --git a/lib/galaxy/datatypes/test/test_binary_loom_spec_version.loom b/lib/galaxy/datatypes/test/test_binary_loom_spec_version.loom new file mode 100644 index 0000000000000000000000000000000000000000..9c80e4dae8e716ac6cd21bf5fdfbf948e4be8321 GIT binary patch literal 15876 zcmeHOK~Gab5T5r6ic%CbA{wKO9wbp4Kty9uYoG=rv;>TJgQk52lG2h=wDDx(1w9xK z9*t*@+)RwWK`$Ony!i)Aj0Y0j*_kQw^!bE940K+@&hG5&?7pw_9y8P3r$d9cTJ|5` z4|p{-K?AhfsuZhyHC!i-w55NFD-P2orf*!TIIst>uY>umtUuRBjCme-cPyC%BPsfY z@-XOXpjtv0O6v-Nq2$d`yy0cOYuhmOyVsb(WG+)F&sG7lrNU&UQYkMmzg&8_u8sZE z<;?sH3yA%Sor+cmnymd&ffm|nyBwEBG_oK)Kmh)B#8^H`w#9B3O?V}sw;6g@Ber4W zNxvJR8JgjWtvLMrxVGPcg_*^CzTmMATnCPGAm`-^nTltgGP;`hNsG9VDz60zF3(Oc z&)@Pvk{28SV#Ek3$5ZJ+$E>=Q#d4o6Q$^%fxP9U(@d+EV3dV(p z{YGNU`M~-`-*;}eA0dGR%e`d%BKMx#pZi(hKM{Q2;rA!+IiUx_)+4~PoZsFA$1pY8 zuLvjtihv@Z2q*%IfFiIn5QxH%U8n<5^XY){OfG-vLh5EbeaG@!jRQ!ba*X60@_fbr z_0#IB>{LhNlYJ(lcmYNhKWou?8~gcwuRbhVf8A~;uSF}=(z0F2OOmd z4kijOX6C$w$$TbTDU~s5w{7VcYn3Qc_(3$te*Mv5nEie^PcvvV6@>Ke=KJkevO;~F zU68vsS*1VPYeEDzNW-#Cl4X+T!rzXV_ciwU{|jt1%f!i6qboGa^z|s+BQ6D}wnHV$ zbn-NHjuJ7Yya{TIzg8&Q`1hymF6vLl5J2BU^*9Ht!VDLaumctn<@im1;Y%BtB*hCrl z8n8XSXHcueCyJ42#4wG1=awjs;is>~bzAWt)yGRQ{{3T57%s|~dk!Kt?3_1)T1 z1QY>9KoL*`6oFlUK=swd-k!ehOBdfR_vRiu5jgeDy%FYA$#yZJzdBWzz1^r&g=6sg z2dC<6oJ4VI8Mf_C6(x{yb##67IZez~5`sXGbJ`LxZi2UWOSjlt?TeQVrHX(epa>`e zihv@pYY<@Pv=Q7N4#+ODCB8j58@85FJ7gTR4j4|SM<&NdlLM3Yl4Iky(@ zk<6et5kh<7UGZ+y9|hWHfEhZ}H&FJ>(U{!CY-Pr3LOr-KElAIBsWg|Y&U@wAIj>l` z&>im*cbA%z`~dg!+&DMgRD0ak}6Qc!k3HnXGbCL?L0T4m8_&M(kOp zl-VFf5~zQ#`3qWqwzOERn3V+Wg7Tlap Date: Tue, 7 Sep 2021 13:13:51 +0100 Subject: [PATCH 2/3] Add dict to exception raised when encoding metadata fails Add the second part in the following traceback: ``` DEBUG galaxy.metadata:__init__.py:93 setting metadata externally failed for HistoryDatasetAssociation 1: Traceback (most recent call last): File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/lib/galaxy/model/metadata.py", line 246, in to_JSON_dict encoded_meta_dict = galaxy.model.custom_types.json_encoder.encode(meta_dict) File "/usr/lib/python3.8/json/encoder.py", line 199, in encode chunks = self.iterencode(o, _one_shot=True) File "/usr/lib/python3.8/json/encoder.py", line 257, in iterencode return _iterencode(o, 0) File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/lib/galaxy/model/custom_types.py", line 39, in default return json.JSONEncoder.default(self, obj) File "/usr/lib/python3.8/json/encoder.py", line 179, in default raise TypeError(f'Object of type {o.__class__.__name__} ' TypeError: Object of type ndarray is not JSON serializable The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/lib/galaxy/metadata/set_metadata.py", line 334, in set_metadata_portable dataset.metadata.to_JSON_dict(filename_out) # write out results of set_meta File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/lib/galaxy/model/metadata.py", line 248, in to_JSON_dict raise Exception(f"Failed encoding metadata dictionary: {meta_dict}") from e Exception: Failed encoding metadata dictionary: {'dbkey': '?', 'title': None, 'description': None, 'url': None, 'doi': None, 'loom_spec_version': array([b'2.0.1'], dtype='|S6'), 'shape': (5, 10), 'layers _count': 0, 'layers_names': [], 'row_attrs_count': 1, 'row_attrs_names': ['rownames'], 'col_attrs_count': 2, 'col_attrs_names': ['colnames', 'colnames_factor'], 'col_graphs_count': 0, 'col_graphs_names': [], 'row_graphs_count': 0, 'row_graphs_names': []} ``` --- lib/galaxy/model/metadata.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/model/metadata.py b/lib/galaxy/model/metadata.py index d9f1816aaf47..bdd3d47fa218 100644 --- a/lib/galaxy/model/metadata.py +++ b/lib/galaxy/model/metadata.py @@ -239,7 +239,10 @@ def to_JSON_dict(self, filename=None): meta_dict['__validated_state__'] = dataset_meta_dict['__validated_state__'] if '__validated_state_message__' in dataset_meta_dict: meta_dict['__validated_state_message__'] = dataset_meta_dict['__validated_state_message__'] - encoded_meta_dict = galaxy.model.custom_types.json_encoder.encode(meta_dict) + try: + encoded_meta_dict = galaxy.model.custom_types.json_encoder.encode(meta_dict) + except Exception as e: + raise Exception(f"Failed encoding metadata dictionary: {meta_dict}") from e if filename is None: return encoded_meta_dict with open(filename, 'wt+') as fh: From 8a7ca520aa32e79d8cc001ab37290247a66843e0 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Sat, 14 Aug 2021 17:40:05 +0100 Subject: [PATCH 3/3] Fix setting metadata for some Loom files Loom files created by recent versions of LoomExperiment Bioconductor package encode the Loom version in an binary array, which breaks metadata setting in Galaxy. --- lib/galaxy/datatypes/binary.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 8c4e255046db..ac816e7e4db1 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -16,6 +16,7 @@ from typing import Optional import h5py +import numpy as np import pysam import pysam.bcftools from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP @@ -928,7 +929,12 @@ def set_meta(self, dataset, overwrite=True, **kwd): dataset.metadata.description = loom_file.attrs.get('description') dataset.metadata.url = loom_file.attrs.get('url') dataset.metadata.doi = loom_file.attrs.get('doi') - dataset.metadata.loom_spec_version = loom_file.attrs.get('LOOM_SPEC_VERSION') + loom_spec_version = loom_file.attrs.get('LOOM_SPEC_VERSION') + if isinstance(loom_spec_version, np.ndarray): + loom_spec_version = loom_spec_version[0] + if isinstance(loom_spec_version, bytes): + loom_spec_version = loom_spec_version.decode() + dataset.metadata.loom_spec_version = loom_spec_version dataset.creation_date = loom_file.attrs.get('creation_date') dataset.metadata.shape = tuple(loom_file['matrix'].shape)