From 02d64ed957542e4a2b866a0f8a6d617511a67544 Mon Sep 17 00:00:00 2001 From: duhaime Date: Mon, 18 Nov 2019 13:17:33 -0500 Subject: [PATCH 01/12] use with block when opening images to autoclose filehandlers --- keras_preprocessing/image/utils.py | 50 +++++++++++++++--------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/keras_preprocessing/image/utils.py b/keras_preprocessing/image/utils.py index b764caad..a62c1a24 100644 --- a/keras_preprocessing/image/utils.py +++ b/keras_preprocessing/image/utils.py @@ -109,31 +109,31 @@ def load_img(path, grayscale=False, color_mode='rgb', target_size=None, if pil_image is None: raise ImportError('Could not import PIL.Image. ' 'The use of `load_img` requires PIL.') - img = pil_image.open(path) - if color_mode == 'grayscale': - # if image is not already an 8-bit, 16-bit or 32-bit grayscale image - # convert it to an 8-bit grayscale image. - if img.mode not in ('L', 'I;16', 'I'): - img = img.convert('L') - elif color_mode == 'rgba': - if img.mode != 'RGBA': - img = img.convert('RGBA') - elif color_mode == 'rgb': - if img.mode != 'RGB': - img = img.convert('RGB') - else: - raise ValueError('color_mode must be "grayscale", "rgb", or "rgba"') - if target_size is not None: - width_height_tuple = (target_size[1], target_size[0]) - if img.size != width_height_tuple: - if interpolation not in _PIL_INTERPOLATION_METHODS: - raise ValueError( - 'Invalid interpolation method {} specified. Supported ' - 'methods are {}'.format( - interpolation, - ", ".join(_PIL_INTERPOLATION_METHODS.keys()))) - resample = _PIL_INTERPOLATION_METHODS[interpolation] - img = img.resize(width_height_tuple, resample) + with pil_image.open(path) as img: + if color_mode == 'grayscale': + # if image is not already an 8-bit, 16-bit or 32-bit grayscale image + # convert it to an 8-bit grayscale image. + if img.mode not in ('L', 'I;16', 'I'): + img = img.convert('L') + elif color_mode == 'rgba': + if img.mode != 'RGBA': + img = img.convert('RGBA') + elif color_mode == 'rgb': + if img.mode != 'RGB': + img = img.convert('RGB') + else: + raise ValueError('color_mode must be "grayscale", "rgb", or "rgba"') + if target_size is not None: + width_height_tuple = (target_size[1], target_size[0]) + if img.size != width_height_tuple: + if interpolation not in _PIL_INTERPOLATION_METHODS: + raise ValueError( + 'Invalid interpolation method {} specified. Supported ' + 'methods are {}'.format( + interpolation, + ", ".join(_PIL_INTERPOLATION_METHODS.keys()))) + resample = _PIL_INTERPOLATION_METHODS[interpolation] + img = img.resize(width_height_tuple, resample) return img From 355f82ded3469cce244a43f2f88fde769ac2e277 Mon Sep 17 00:00:00 2001 From: duhaime Date: Mon, 18 Nov 2019 17:55:15 -0500 Subject: [PATCH 02/12] correct indentation --- keras_preprocessing/image/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keras_preprocessing/image/utils.py b/keras_preprocessing/image/utils.py index a62c1a24..b650b9ae 100644 --- a/keras_preprocessing/image/utils.py +++ b/keras_preprocessing/image/utils.py @@ -134,7 +134,7 @@ def load_img(path, grayscale=False, color_mode='rgb', target_size=None, ", ".join(_PIL_INTERPOLATION_METHODS.keys()))) resample = _PIL_INTERPOLATION_METHODS[interpolation] img = img.resize(width_height_tuple, resample) - return img + return img def list_pictures(directory, ext=('jpg', 'jpeg', 'bmp', 'png', 'ppm', 'tif', From fa8af4702062cee4e5c9563d279b11bbf12b5913 Mon Sep 17 00:00:00 2001 From: duhaime Date: Mon, 18 Nov 2019 18:16:56 -0500 Subject: [PATCH 03/12] use io bytes to read from file handler --- keras_preprocessing/image/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/keras_preprocessing/image/utils.py b/keras_preprocessing/image/utils.py index b650b9ae..9715bce0 100644 --- a/keras_preprocessing/image/utils.py +++ b/keras_preprocessing/image/utils.py @@ -4,6 +4,7 @@ from __future__ import division from __future__ import print_function +import io import os import warnings @@ -109,7 +110,8 @@ def load_img(path, grayscale=False, color_mode='rgb', target_size=None, if pil_image is None: raise ImportError('Could not import PIL.Image. ' 'The use of `load_img` requires PIL.') - with pil_image.open(path) as img: + with open(path, 'rb') as f: + img = pil_image.open(io.BytesIO(f.read())) if color_mode == 'grayscale': # if image is not already an 8-bit, 16-bit or 32-bit grayscale image # convert it to an 8-bit grayscale image. From 06c1856f40ab597bb6855da39a0f7d559776a679 Mon Sep 17 00:00:00 2001 From: duhaime Date: Mon, 18 Nov 2019 18:30:16 -0500 Subject: [PATCH 04/12] test that images opened by load_img are closed --- tests/image/utils_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/image/utils_test.py b/tests/image/utils_test.py index 8d17314f..b9f032c3 100644 --- a/tests/image/utils_test.py +++ b/tests/image/utils_test.py @@ -1,5 +1,6 @@ import numpy as np import pytest +import resource from keras_preprocessing.image import utils @@ -326,5 +327,12 @@ def test_array_to_img_and_img_to_array(): img = utils.img_to_array(x, data_format='channels_last') +def test_image_file_handlers_close(tmpdir): + im = utils.array_to_img(np.random.rand(1,1,3)) + path = str(tmpdir / 'sample_image.png') + utils.save_img(path, im) + max_open_files, _ = soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + images = [utils.load_img(path) for i in range(max_open_files+1)] + if __name__ == '__main__': pytest.main([__file__]) From 30b4f4e452cd49aba2316da807e70da3f067ed5d Mon Sep 17 00:00:00 2001 From: duhaime Date: Mon, 18 Nov 2019 19:26:15 -0500 Subject: [PATCH 05/12] make flake8 -v --count return 0 --- keras_preprocessing/text.py | 7 +++++-- tests/image/utils_test.py | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/keras_preprocessing/text.py b/keras_preprocessing/text.py index 573e4110..0d9d5917 100644 --- a/keras_preprocessing/text.py +++ b/keras_preprocessing/text.py @@ -7,6 +7,7 @@ import string import sys +import six import warnings from collections import OrderedDict from collections import defaultdict @@ -43,8 +44,10 @@ def text_to_word_sequence(text, text = text.lower() if sys.version_info < (3,): - if isinstance(text, unicode): - translate_map = dict((ord(c), unicode(split)) for c in filters) + # in Python 2.x six.string_types == basestring + # and basestring is a superclass for unicode and str + if isinstance(text, six.string_types) and not isinstance(text, str): + translate_map = dict((ord(c), six.u(split)) for c in filters) text = text.translate(translate_map) elif len(split) == 1: translate_map = maketrans(filters, split * len(filters)) diff --git a/tests/image/utils_test.py b/tests/image/utils_test.py index b9f032c3..a95b5b91 100644 --- a/tests/image/utils_test.py +++ b/tests/image/utils_test.py @@ -328,11 +328,13 @@ def test_array_to_img_and_img_to_array(): def test_image_file_handlers_close(tmpdir): - im = utils.array_to_img(np.random.rand(1,1,3)) + im = utils.array_to_img(np.random.rand(1, 1, 3)) path = str(tmpdir / 'sample_image.png') utils.save_img(path, im) max_open_files, _ = soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) - images = [utils.load_img(path) for i in range(max_open_files+1)] + for i in range(max_open_files+1): + utils.load_img(path) + if __name__ == '__main__': pytest.main([__file__]) From 24950ddc80218aee8dc02d17aae678a1b954207d Mon Sep 17 00:00:00 2001 From: duhaime Date: Mon, 25 Nov 2019 09:19:28 -0500 Subject: [PATCH 06/12] use with pil_image.load as context handler for image lading --- keras_preprocessing/image/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/keras_preprocessing/image/utils.py b/keras_preprocessing/image/utils.py index 9715bce0..5ef47e16 100644 --- a/keras_preprocessing/image/utils.py +++ b/keras_preprocessing/image/utils.py @@ -4,7 +4,6 @@ from __future__ import division from __future__ import print_function -import io import os import warnings @@ -110,8 +109,7 @@ def load_img(path, grayscale=False, color_mode='rgb', target_size=None, if pil_image is None: raise ImportError('Could not import PIL.Image. ' 'The use of `load_img` requires PIL.') - with open(path, 'rb') as f: - img = pil_image.open(io.BytesIO(f.read())) + with pil_image.load(path) as img: if color_mode == 'grayscale': # if image is not already an 8-bit, 16-bit or 32-bit grayscale image # convert it to an 8-bit grayscale image. From 465f2e9be85cd2bdec8c522fd517d55695f2dd5d Mon Sep 17 00:00:00 2001 From: duhaime Date: Mon, 25 Nov 2019 09:19:47 -0500 Subject: [PATCH 07/12] test that load_img returns an image in memory --- tests/image/utils_test.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/image/utils_test.py b/tests/image/utils_test.py index a95b5b91..9d03c2b2 100644 --- a/tests/image/utils_test.py +++ b/tests/image/utils_test.py @@ -327,14 +327,26 @@ def test_array_to_img_and_img_to_array(): img = utils.img_to_array(x, data_format='channels_last') -def test_image_file_handlers_close(tmpdir): +def write_sample_image(tmpdir): im = utils.array_to_img(np.random.rand(1, 1, 3)) path = str(tmpdir / 'sample_image.png') utils.save_img(path, im) + return path + + +def test_image_file_handlers_close(tmpdir): + path = write_sample_image(tmpdir) max_open_files, _ = soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) for i in range(max_open_files+1): utils.load_img(path) +def test_load_img_returns_image(tmpdir): + path = write_sample_image(tmpdir) + im = utils.load_img(path) + assert hasattr(im, 'height') + assert hasattr(im, 'width') + + if __name__ == '__main__': pytest.main([__file__]) From e719378a6c189a0c5840f75b1f71726110d82dbb Mon Sep 17 00:00:00 2001 From: duhaime Date: Mon, 25 Nov 2019 11:24:49 -0500 Subject: [PATCH 08/12] restore io method to clear tests --- keras_preprocessing/image/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/keras_preprocessing/image/utils.py b/keras_preprocessing/image/utils.py index 5ef47e16..9715bce0 100644 --- a/keras_preprocessing/image/utils.py +++ b/keras_preprocessing/image/utils.py @@ -4,6 +4,7 @@ from __future__ import division from __future__ import print_function +import io import os import warnings @@ -109,7 +110,8 @@ def load_img(path, grayscale=False, color_mode='rgb', target_size=None, if pil_image is None: raise ImportError('Could not import PIL.Image. ' 'The use of `load_img` requires PIL.') - with pil_image.load(path) as img: + with open(path, 'rb') as f: + img = pil_image.open(io.BytesIO(f.read())) if color_mode == 'grayscale': # if image is not already an 8-bit, 16-bit or 32-bit grayscale image # convert it to an 8-bit grayscale image. From b691f540d6e0a6b850bcb0d2f84c80a5c60f3437 Mon Sep 17 00:00:00 2001 From: duhaime Date: Tue, 26 Nov 2019 10:18:13 -0500 Subject: [PATCH 09/12] test object type returned by load_img is PIL.Image.Image --- tests/image/utils_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/image/utils_test.py b/tests/image/utils_test.py index 9d03c2b2..92998a12 100644 --- a/tests/image/utils_test.py +++ b/tests/image/utils_test.py @@ -1,6 +1,7 @@ import numpy as np import pytest import resource +import PIL from keras_preprocessing.image import utils @@ -344,8 +345,7 @@ def test_image_file_handlers_close(tmpdir): def test_load_img_returns_image(tmpdir): path = write_sample_image(tmpdir) im = utils.load_img(path) - assert hasattr(im, 'height') - assert hasattr(im, 'width') + assert isinstance(im, PIL.Image.Image) if __name__ == '__main__': From 946b686cebb52822c3372f412ba69d1fd03dc686 Mon Sep 17 00:00:00 2001 From: duhaime Date: Tue, 26 Nov 2019 10:18:41 -0500 Subject: [PATCH 10/12] use single assignment when testing file handlers closed --- tests/image/utils_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image/utils_test.py b/tests/image/utils_test.py index 92998a12..d954e1e2 100644 --- a/tests/image/utils_test.py +++ b/tests/image/utils_test.py @@ -337,7 +337,7 @@ def write_sample_image(tmpdir): def test_image_file_handlers_close(tmpdir): path = write_sample_image(tmpdir) - max_open_files, _ = soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + max_open_files, _ = resource.getrlimit(resource.RLIMIT_NOFILE) for i in range(max_open_files+1): utils.load_img(path) From e88b7c9434ce8c52fe1ad69470362f8df3bc48c8 Mon Sep 17 00:00:00 2001 From: duhaime Date: Tue, 26 Nov 2019 10:20:37 -0500 Subject: [PATCH 11/12] restore unicode text handling on master --- keras_preprocessing/text.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/keras_preprocessing/text.py b/keras_preprocessing/text.py index 0d9d5917..9fecf7a8 100644 --- a/keras_preprocessing/text.py +++ b/keras_preprocessing/text.py @@ -7,7 +7,6 @@ import string import sys -import six import warnings from collections import OrderedDict from collections import defaultdict @@ -44,10 +43,8 @@ def text_to_word_sequence(text, text = text.lower() if sys.version_info < (3,): - # in Python 2.x six.string_types == basestring - # and basestring is a superclass for unicode and str - if isinstance(text, six.string_types) and not isinstance(text, str): - translate_map = dict((ord(c), six.u(split)) for c in filters) + if isinstance(text, unicode): + translate_map = dict((ord(c), unicode(split)) for c in filters) text = text.translate(translate_map) elif len(split) == 1: translate_map = maketrans(filters, split * len(filters)) @@ -517,4 +514,4 @@ def tokenizer_from_json(json_string): tokenizer.word_index = word_index tokenizer.index_word = index_word - return tokenizer + return tokenizer \ No newline at end of file From d669fd2e8ac1dedd0b77a4da86f4af492985612b Mon Sep 17 00:00:00 2001 From: duhaime Date: Tue, 26 Nov 2019 16:24:36 -0500 Subject: [PATCH 12/12] fast-forward text module --- keras_preprocessing/text.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/keras_preprocessing/text.py b/keras_preprocessing/text.py index 9fecf7a8..bad9bcdd 100644 --- a/keras_preprocessing/text.py +++ b/keras_preprocessing/text.py @@ -43,8 +43,10 @@ def text_to_word_sequence(text, text = text.lower() if sys.version_info < (3,): - if isinstance(text, unicode): - translate_map = dict((ord(c), unicode(split)) for c in filters) + if isinstance(text, unicode): # noqa: F821 + translate_map = dict( + (ord(c), unicode(split)) for c in filters # noqa: F821 + ) text = text.translate(translate_map) elif len(split) == 1: translate_map = maketrans(filters, split * len(filters)) @@ -514,4 +516,4 @@ def tokenizer_from_json(json_string): tokenizer.word_index = word_index tokenizer.index_word = index_word - return tokenizer \ No newline at end of file + return tokenizer