From 2886ec76dc9d414af350c9ffc9dfcc319982113e Mon Sep 17 00:00:00 2001 From: Andrey Bienkowski Date: Fri, 19 Feb 2021 13:40:38 +0300 Subject: [PATCH] Stop throwing if saving a request fails Saving a cached request may fail for any number of reasons: * Running out of memory * Running out of disk space * Other IO-related errors Since we have already made the request and received the data we might as well use it instead of dropping it on the floor just because we can't save it to satisfy future requests. --- cachecontrol/filewrapper.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/cachecontrol/filewrapper.py b/cachecontrol/filewrapper.py index dd91334c..74de86a6 100644 --- a/cachecontrol/filewrapper.py +++ b/cachecontrol/filewrapper.py @@ -3,6 +3,10 @@ # SPDX-License-Identifier: Apache-2.0 from io import BytesIO +import logging + + +logger = logging.getLogger(__name__) class CallbackFileWrapper(object): @@ -51,9 +55,26 @@ def __is_fp_closed(self): # TODO: Add some logging here... return False + def __call_callback(self): + if self.__callback is not None and self.__buf is not None: + try: + self.__callback(self.__buf.getvalue()) + except Exception as e: + self.__callback = None + self.__buf = None + logger.exception('Failed to save a cached request', exc_info=True) + + def __append(self, data): + if self.__callback is not None and self.__buf is not None: + try: + self.__buf.write(data) + except Exception as e: + self.__callback = None + self.__buf = None + logger.exception('Failed to save a cached request', exc_info=True) + def _close(self): - if self.__callback: - self.__callback(self.__buf.getvalue()) + self.__call_callback() # We assign this to None here, because otherwise we can get into # really tricky problems where the CPython interpreter dead locks @@ -64,7 +85,7 @@ def _close(self): def read(self, amt=None): data = self.__fp.read(amt) - self.__buf.write(data) + self.__append(data) if self.__is_fp_closed(): self._close() @@ -77,7 +98,7 @@ def _safe_read(self, amt): # of the chunk. return data - self.__buf.write(data) + self.__append(data) if self.__is_fp_closed(): self._close()