From 1985e3ef3b464089ce945551a818626ef4204d21 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Tue, 8 Jun 2021 13:02:48 +0300 Subject: [PATCH 1/5] add fixes --- datumaro/plugins/cifar_format.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/datumaro/plugins/cifar_format.py b/datumaro/plugins/cifar_format.py index b6f3a01a23..60243bc99c 100644 --- a/datumaro/plugins/cifar_format.py +++ b/datumaro/plugins/cifar_format.py @@ -69,7 +69,7 @@ def _load_items(self, path): # 'filenames': list # 'labels': list with open(path, 'rb') as anno_file: - annotation_dict = pickle.load(anno_file) + annotation_dict = pickle.load(anno_file, encoding='latin1') labels = annotation_dict.get('labels', []) filenames = annotation_dict.get('filenames', []) @@ -94,11 +94,13 @@ def _load_items(self, path): if 0 < len(images_data): image = images_data[i] if size is not None and image is not None: - image = image.reshape(size[i][0], - size[i][1], 3).astype(np.uint8) + image = image.reshape(3, size[i][0], + size[i][1]).astype(np.uint8) + image = np.transpose(image, (1, 2, 0)) elif image is not None: - image = image.reshape(CifarPath.IMAGE_SIZE, - CifarPath.IMAGE_SIZE, 3).astype(np.uint8) + image = image.reshape(3, CifarPath.IMAGE_SIZE, + CifarPath.IMAGE_SIZE).astype(np.uint8) + image = np.transpose(image, (1, 2, 0)) items[item_id] = DatasetItem(id=item_id, subset=self._subset, image=image, annotations=annotations) @@ -150,7 +152,8 @@ def apply(self): data.append(None) else: image = image.data - data.append(image.reshape(-1).astype(np.uint8)) + data.append(np.transpose(image, \ + (2, 0, 1)).reshape(-1).astype(np.uint8)) if image.shape[0] != CifarPath.IMAGE_SIZE or \ image.shape[1] != CifarPath.IMAGE_SIZE: image_sizes[len(data) - 1] = (image.shape[0], image.shape[1]) From e6021355d560a42d12465554708b1daaea3ca5f2 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Tue, 8 Jun 2021 16:41:52 +0300 Subject: [PATCH 2/5] update Changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1f8232d04..a8df3a137e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ### Fixed -- +- Image processing and file upload in CIFAR () ### Security - From b626e9db5f2baa62a53924482afe394cba2572ea Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 9 Jun 2021 10:32:19 +0300 Subject: [PATCH 3/5] add another image to test --- tests/assets/cifar_dataset/data_batch_1 | Bin 3317 -> 6393 bytes tests/assets/cifar_dataset/test_batch | Bin 9494 -> 9717 bytes tests/test_cifar_format.py | 5 +++++ 3 files changed, 5 insertions(+) diff --git a/tests/assets/cifar_dataset/data_batch_1 b/tests/assets/cifar_dataset/data_batch_1 index e4ed1edc0b8f4b129ff28008e584bf6f04e1a70b..361d15a734de4bdb23cf712b6a38b5064f7e3bbd 100644 GIT binary patch literal 6393 zcmeI%&r8EF6bJBDH$}Ir;IYS@3OfkiJb1HM!TTt$ZPo?av}Ea_py0(HgWUG~H~kk} zoBk6{z6s=zygb5(KmzY=^8It|x%SEnd$E?KP*jO6uh`CHMTJO~VWSErdsb2^%3&q) z$WC%H#P&+HRnvu8twmV1rIDmHb!_K}7Fn&fMz@*aJ=>`ZWP7giF=XGTdFOjS*VQ{cVA<`sr6qtj^l^lzPTZ~Go0>L>*L+;`x3&ySNs00Izz00bZ~2;fD>J%9iNAOHaf{0D)*_o9ow8enHBnQW7|T18); UFq)*2stDY4ung*TaKmkQ02&waB>(^b literal 3317 zcmeI#zfQw25C(7?S`aiVd&W+YGVlVh5s`r%MVG`)EhSFm+CwCeSol-vwjM9ZOMvS> z2KNmf?9Z0`+wN`g{ga58S6buMM)%6Vpmy|3xY7f3s>-v;bOd& zF@BiltQoD!2H!gk*1}AfR8}>k**WVQM+d@XED~Ot_Eb59XWr>v*7QkFv@T>JA4T#( z+iP65Zl2Wf)|t;~|8**yqkN&VEFXSD;!+4l_5MzTA4m=humB6N01L1H3$OqSu)x0& hpfCyGss`6=t2n2G(ZpHMMw>2;+i9MToAj1^_6MrCYlQ#+ diff --git a/tests/assets/cifar_dataset/test_batch b/tests/assets/cifar_dataset/test_batch index e3776023189c3f89998c42f2d7ff9c0aa6e7a34d..1f494f9c366cf2e5a1cbba6920d5aef6201de420 100644 GIT binary patch delta 348 zcmbQ{_0^lDfo1Bei7d{_J=~eOiRr2Frg{Z==~GgB*m4q+Qge!@#7@!hX7pzEX7Xm9 z7!_L2(mExBw}&;Qq_QA&%H%0MO#T*AIy>5@1Ob&X_b}E>@$>WZ`VRzP!rQ)NN>V4# zhzy<#es2b#UO|XbMqdUVAWy7^snigmKSK&tHv>qw69W$e10xh{{P}}hX$nvnhk?ov z8m0x>M0iZiQHA;F{ItA|Xp~4fhLT#9R8F<>JWbpK`rj%3`q)wSUrH851a7t%K`;?$5 z8s5x3j5SmI{QSKB0|A)uW+<7G)ak^a!oaY(QSvAE<_VIoAjISVMq`#KK-tOXRMi3G C6)VC3 diff --git a/tests/test_cifar_format.py b/tests/test_cifar_format.py index 480d795954..bf7a220953 100644 --- a/tests/test_cifar_format.py +++ b/tests/test_cifar_format.py @@ -146,6 +146,11 @@ def test_can_import(self): DatasetItem(id='image_4', subset='test', image=np.ones((32, 32, 3)), annotations=[Label(2)] + ), + DatasetItem(id='image_5', subset='test', + image=np.array([[[1., 2., 3.], [4., 5., 6.]], + [[1., 2., 3.], [4., 5., 6.]]]), + annotations=[Label(3)] ) ], categories=['airplane', 'automobile', 'bird', 'cat']) From 4e80c7a726edcfefd89ff0b5362c07cb47f178b0 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Thu, 10 Jun 2021 09:47:03 +0300 Subject: [PATCH 4/5] review fixes --- CHANGELOG.md | 2 +- tests/test_cifar_format.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8df3a137e..15b741d562 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ### Fixed -- Image processing and file upload in CIFAR () +- Changed image saving in R...RG...GB...B to RGB and added latin1 encoding for loading pickle files () ### Security - diff --git a/tests/test_cifar_format.py b/tests/test_cifar_format.py index bf7a220953..e8c09b5b55 100644 --- a/tests/test_cifar_format.py +++ b/tests/test_cifar_format.py @@ -148,6 +148,7 @@ def test_can_import(self): annotations=[Label(2)] ), DatasetItem(id='image_5', subset='test', + # in the images file: 1, 4, 1, 4, 2, 5, 2, 5, 3, 6, 3, 6 image=np.array([[[1., 2., 3.], [4., 5., 6.]], [[1., 2., 3.], [4., 5., 6.]]]), annotations=[Label(3)] From 9dc9b8e9b30903471eaaa95d9e0fbc578e929392 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 10 Jun 2021 13:52:28 +0300 Subject: [PATCH 5/5] Fixes --- CHANGELOG.md | 2 +- datumaro/plugins/cifar_format.py | 2 +- tests/test_cifar_format.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15b741d562..76974b9ed1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ### Fixed -- Changed image saving in R...RG...GB...B to RGB and added latin1 encoding for loading pickle files () +- Incorrect image layout on saving and a problem with ecoding on loading () ### Security - diff --git a/datumaro/plugins/cifar_format.py b/datumaro/plugins/cifar_format.py index 60243bc99c..101a0c40c9 100644 --- a/datumaro/plugins/cifar_format.py +++ b/datumaro/plugins/cifar_format.py @@ -152,7 +152,7 @@ def apply(self): data.append(None) else: image = image.data - data.append(np.transpose(image, \ + data.append(np.transpose(image, (2, 0, 1)).reshape(-1).astype(np.uint8)) if image.shape[0] != CifarPath.IMAGE_SIZE or \ image.shape[1] != CifarPath.IMAGE_SIZE: diff --git a/tests/test_cifar_format.py b/tests/test_cifar_format.py index e8c09b5b55..1983b1a875 100644 --- a/tests/test_cifar_format.py +++ b/tests/test_cifar_format.py @@ -148,7 +148,6 @@ def test_can_import(self): annotations=[Label(2)] ), DatasetItem(id='image_5', subset='test', - # in the images file: 1, 4, 1, 4, 2, 5, 2, 5, 3, 6, 3, 6 image=np.array([[[1., 2., 3.], [4., 5., 6.]], [[1., 2., 3.], [4., 5., 6.]]]), annotations=[Label(3)] @@ -157,7 +156,7 @@ def test_can_import(self): dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'cifar') - compare_datasets(self, expected_dataset, dataset) + compare_datasets(self, expected_dataset, dataset, require_images=True) @mark_requirement(Requirements.DATUM_GENERAL_REQ) def test_can_detect(self):