diff --git a/src/icloudpd/authentication.py b/src/icloudpd/authentication.py index aa94cdf2f..e6ee5007a 100644 --- a/src/icloudpd/authentication.py +++ b/src/icloudpd/authentication.py @@ -1,6 +1,5 @@ """Handles username/password authentication and two-step authentication""" -import logging import sys import click import pyicloud_ipd @@ -23,7 +22,7 @@ def authenticate_( client_id=None, ): """Authenticate with iCloud username and password""" - logger.tqdm_write("Authenticating...", logging.DEBUG) + logger.debug("Authenticating...") while True: try: # If password not provided on command line variable will be set to None @@ -42,9 +41,9 @@ def authenticate_( if icloud.requires_2sa: if raise_error_on_2sa: raise TwoStepAuthRequiredError( - "Two-step/two-factor authentication is required!" + "Two-step/two-factor authentication is required" ) - logger.info("Two-step/two-factor authentication is required!") + logger.info("Two-step/two-factor authentication is required") request_2sa(icloud, logger) return icloud return authenticate_ diff --git a/src/icloudpd/autodelete.py b/src/icloudpd/autodelete.py index 4a97b24e4..eeee84df1 100644 --- a/src/icloudpd/autodelete.py +++ b/src/icloudpd/autodelete.py @@ -2,7 +2,6 @@ Delete any files found in "Recently Deleted" """ import os -import logging from tzlocal import get_localzone from icloudpd.paths import local_download_path @@ -10,12 +9,12 @@ def delete_file(logger, path) -> bool: """ Actual deletion of files """ os.remove(path) - logger.tqdm_write(f"Deleted {path}", logging.INFO) + logger.info("Deleted %s", path) return True def delete_file_dry_run(logger, path) -> bool: """ Dry run deletion of files """ - logger.tqdm_write(f"[DRY RUN] Would delete {path}", logging.INFO) + logger.info("[DRY RUN] Would delete %s", path) return True def autodelete_photos(logger, dry_run, icloud, folder_structure, directory): @@ -24,7 +23,7 @@ def autodelete_photos(logger, dry_run, icloud, folder_structure, directory): from the download directory. (I.e. If you delete a photo on your phone, it's also deleted on your computer.) """ - logger.tqdm_write("Deleting any files found in 'Recently Deleted'...", logging.INFO) + logger.info("Deleting any files found in 'Recently Deleted'...") recently_deleted = icloud.photos.albums["Recently Deleted"] @@ -32,9 +31,9 @@ def autodelete_photos(logger, dry_run, icloud, folder_structure, directory): try: created_date = media.created.astimezone(get_localzone()) except (ValueError, OSError): - logger.tqdm_write( - f"Could not convert media created date to local timezone {media.created}", - logging.ERROR) + logger.error( + "Could not convert media created date to local timezone %s", + media.created) created_date = media.created date_path = folder_structure.format(created_date) @@ -45,6 +44,6 @@ def autodelete_photos(logger, dry_run, icloud, folder_structure, directory): local_download_path( media, size, download_dir)) if os.path.exists(path): - logger.tqdm_write(f"Deleting {path}...", logging.DEBUG) + logger.debug("Deleting %s...", path) delete_local = delete_file_dry_run if dry_run else delete_file delete_local(logger, path) diff --git a/src/icloudpd/base.py b/src/icloudpd/base.py index 630abda5c..7bae21b05 100644 --- a/src/icloudpd/base.py +++ b/src/icloudpd/base.py @@ -14,11 +14,11 @@ import click from tqdm import tqdm +from tqdm.contrib.logging import logging_redirect_tqdm from tzlocal import get_localzone from pyicloud_ipd.exceptions import PyiCloudAPIResponseError -from icloudpd.logger import setup_logger from icloudpd.authentication import authenticator, TwoStepAuthRequiredError from icloudpd import download from icloudpd.email_notifications import send_2sa_notification @@ -260,7 +260,11 @@ def main( ): """Download all iCloud photos to a local directory""" - logger = setup_logger() + logging.basicConfig( + format="%(asctime)s %(levelname)-8s %(message)s", + datefmt="%Y-%m-%d %H:%M:%S" + ) + logger = logging.getLogger("icloudpd") if only_print_filenames: logger.disabled = True else: @@ -274,62 +278,66 @@ def main( elif log_level == "error": logger.setLevel(logging.ERROR) - # check required directory param only if not list albums - if not list_albums and not directory: - print('--directory or --list-albums are required') - sys.exit(2) + with logging_redirect_tqdm(): - if auto_delete and delete_after_download: - print('--auto-delete and --delete-after-download are mutually exclusive') - sys.exit(2) + # check required directory param only if not list albums + if not list_albums and not directory: + print('--directory or --list-albums are required') + sys.exit(2) - if watch_with_interval and (list_albums or only_print_filenames): # pragma: no cover - print('--watch_with_interval is not compatible with --list_albums, --only_print_filenames') - sys.exit(2) + if auto_delete and delete_after_download: + print('--auto-delete and --delete-after-download are mutually exclusive') + sys.exit(2) - sys.exit( - core( - download_builder( - logger, - skip_videos, - folder_structure, + if watch_with_interval and (list_albums or only_print_filenames): # pragma: no cover + print( + '--watch_with_interval is not compatible with --list_albums, --only_print_filenames' + ) + sys.exit(2) + + sys.exit( + core( + download_builder( + logger, + skip_videos, + folder_structure, + directory, + size, + force_size, + only_print_filenames, + set_exif_datetime, + skip_live_photos, + live_photo_size, + dry_run), directory, + username, + password, + cookie_directory, size, - force_size, + recent, + until_found, + album, + list_albums, + skip_videos, + auto_delete, only_print_filenames, - set_exif_datetime, - skip_live_photos, - live_photo_size, - dry_run), - directory, - username, - password, - cookie_directory, - size, - recent, - until_found, - album, - list_albums, - skip_videos, - auto_delete, - only_print_filenames, - folder_structure, - smtp_username, - smtp_password, - smtp_host, - smtp_port, - smtp_no_tls, - notification_email, - notification_email_from, - no_progress_bar, - notification_script, - delete_after_download, - domain, - logger, - watch_with_interval, - dry_run + folder_structure, + smtp_username, + smtp_password, + smtp_host, + smtp_port, + smtp_no_tls, + notification_email, + notification_email_from, + no_progress_bar, + notification_script, + delete_after_download, + domain, + logger, + watch_with_interval, + dry_run + ) ) - ) # pylint: disable-msg=too-many-arguments,too-many-statements # pylint: disable-msg=too-many-branches,too-many-locals @@ -353,25 +361,27 @@ def download_photo_(counter, photo) -> bool: """internal function for actually downloading the photos""" filename = clean_filename(photo.filename) if skip_videos and photo.item_type != "image": - logger.tqdm_write( - (f"Skipping {filename}, only downloading photos." - f"(Item type was: {photo.item_type})"), - logging.DEBUG + logger.debug( + "Skipping %s, only downloading photos." + + "(Item type was: %s)", + filename, + photo.item_type ) return False if photo.item_type not in ("image", "movie"): - logger.tqdm_write( - (f"Skipping {filename}, only downloading photos and videos. " - f"(Item type was: {photo.item_type})"), - logging.DEBUG + logger.debug( + "Skipping %s, only downloading photos and videos. " + + "(Item type was: %s)", + filename, + photo.item_type ) return False try: created_date = photo.created.astimezone(get_localzone()) except (ValueError, OSError): - logger.tqdm_write( - f"Could not convert photo created date to local timezone ({photo.created})", - logging.ERROR) + logger.error( + "Could not convert photo created date to local timezone (%s)", + photo.created) created_date = photo.created try: @@ -381,8 +391,8 @@ def download_photo_(counter, photo) -> bool: date_path = folder_structure.format(created_date) except ValueError: # pragma: no cover # This error only seems to happen in Python 2 - logger.tqdm_write( - f"Photo created date was not valid ({photo.created})", logging.ERROR) + logger.error( + "Photo created date was not valid (%s)", photo.created) # e.g. ValueError: year=5 is before 1900 # (https://github.com/icloud-photos-downloader/icloud_photos_downloader/issues/122) # Just use the Unix epoch @@ -397,7 +407,7 @@ def download_photo_(counter, photo) -> bool: versions = photo.versions except KeyError as ex: print( - f"KeyError: {ex} attribute was not found in the photo fields!" + f"KeyError: {ex} attribute was not found in the photo fields." ) with open(file='icloudpd-photo-error.json', mode='w', encoding='utf8') as outfile: # pylint: disable=protected-access @@ -420,9 +430,10 @@ def download_photo_(counter, photo) -> bool: if size not in versions and size != "original": if force_size: - logger.tqdm_write( - f"{size} size does not exist for {filename}. Skipping...", - logging.ERROR + logger.error( + "%s size does not exist for %s. Skipping...", + size, + filename ) return False download_size = "original" @@ -451,16 +462,16 @@ def download_photo_(counter, photo) -> bool: download_path = (f"-{photo_size}.").join( download_path.rsplit(".", 1) ) - logger.tqdm_write( - f"{truncate_middle(download_path, 96)} deduplicated.", - logging.INFO + logger.info( + "%s deduplicated.", + truncate_middle(download_path, 96) ) file_exists = os.path.isfile(download_path) if file_exists: counter.increment() - logger.tqdm_write( - f"{truncate_middle(download_path, 96)} already exists.", - logging.INFO + logger.info( + "%s already exists.", + truncate_middle(download_path, 96) ) if not file_exists: @@ -469,9 +480,9 @@ def download_photo_(counter, photo) -> bool: print(download_path) else: truncated_path = truncate_middle(download_path, 96) - logger.tqdm_write( - f"Downloading {truncated_path}...", - logging.DEBUG + logger.debug( + "Downloading %s...", + truncated_path ) download_result = download.download_media( @@ -489,9 +500,10 @@ def download_photo_(counter, photo) -> bool: # %Y:%m:%d looks wrong, but it's the correct format date_str = created_date.strftime( "%Y-%m-%d %H:%M:%S%z") - logger.tqdm_write( - f"Setting EXIF timestamp for {download_path}: {date_str}", - logging.DEBUG + logger.debug( + "Setting EXIF timestamp for %s: %s", + download_path, + date_str ) exif_datetime.set_photo_exif( logger, @@ -500,9 +512,9 @@ def download_photo_(counter, photo) -> bool: ) if not dry_run: download.set_utime(download_path, created_date) - logger.tqdm_write( - f"Downloaded {truncated_path}", - logging.INFO + logger.info( + "Downloaded %s", + truncated_path ) # Also download the live photo if present @@ -530,32 +542,32 @@ def download_photo_(counter, photo) -> bool: lp_download_path = (f"-{lp_photo_size}.").join( lp_download_path.rsplit(".", 1) ) - logger.tqdm_write( - f"{truncate_middle(lp_download_path, 96)} deduplicated.", - logging.DEBUG + logger.debug( + "%s deduplicated.", + truncate_middle(lp_download_path, 96) ) lp_file_exists = os.path.isfile( lp_download_path) if lp_file_exists: - logger.tqdm_write( - f"{truncate_middle(lp_download_path, 96)} already exists.", - logging.INFO + logger.info( + "%s already exists.", + truncate_middle(lp_download_path, 96) ) if not lp_file_exists: truncated_path = truncate_middle( lp_download_path, 96) - logger.tqdm_write( - f"Downloading {truncated_path}...", - logging.DEBUG + logger.debug( + "Downloading %s...", + truncated_path ) download_result = download.download_media( logger, dry_run, icloud, photo, lp_download_path, lp_size ) success = download_result and success if download_result: - logger.tqdm_write( - f"Downloaded {truncated_path}", - logging.INFO + logger.info( + "Downloaded %s", + truncated_path ) return success return download_photo_ @@ -565,8 +577,8 @@ def download_photo_(counter, photo) -> bool: def delete_photo(logger, icloud, photo): """Delete a photo from the iCloud account.""" clean_filename_local = clean_filename(photo.filename) - logger.tqdm_write( - f"Deleting {clean_filename_local} in iCloud...", logging.DEBUG) + logger.debug( + "Deleting %s in iCloud...", clean_filename_local) # pylint: disable=W0212 url = f"{icloud.photos._service_endpoint}/records/modify?"\ f"{urllib.parse.urlencode(icloud.photos.params)}" @@ -589,15 +601,16 @@ def delete_photo(logger, icloud, photo): icloud.photos.session.post( url, data=post_data, headers={ "Content-type": "application/json"}) - logger.tqdm_write( - f"Deleted {clean_filename_local} in iCloud", logging.INFO) + logger.info( + "Deleted %s in iCloud", clean_filename_local) def delete_photo_dry_run(logger, _icloud, photo): """Dry run for deleting a photo from the iCloud""" - logger.tqdm_write( - f"[DRY RUN] Would delete {clean_filename(photo.filename)} in iCloud", - logging.INFO) + logger.info( + "[DRY RUN] Would delete %s in iCloud", + clean_filename(photo.filename) + ) RetrierT = TypeVar('RetrierT') @@ -625,13 +638,11 @@ def session_error_handler(ex, attempt): """Handles session errors in the PhotoAlbum photos iterator""" if "Invalid global session" in str(ex): if attempt > constants.MAX_RETRIES: - logger.tqdm_write( - "iCloud re-authentication failed! Please try again later." + logger.error( + "iCloud re-authentication failed. Please try again later." ) raise ex - logger.tqdm_write( - "Session error, re-authenticating...", - logging.ERROR) + logger.error("Session error, re-authenticating...") if attempt > 1: # If the first re-authentication attempt failed, # start waiting a few seconds before retrying in case @@ -647,13 +658,11 @@ def internal_error_handler(ex, attempt): """Handles session errors in the PhotoAlbum photos iterator""" if "INTERNAL_ERROR" in str(ex): if attempt > constants.MAX_RETRIES: - logger.tqdm_write( + logger.error( "Internal Error at Apple." ) raise ex - logger.tqdm_write( - "Internal Error at Apple, retrying...", - logging.ERROR) + logger.error("Internal Error at Apple, retrying...") # start waiting a few seconds before retrying in case # there are some issues with the Apple servers time.sleep(constants.WAIT_SECONDS * attempt) @@ -757,9 +766,10 @@ def core( directory = os.path.normpath(directory) videos_phrase = "" if skip_videos else " and videos" - logger.tqdm_write( - f"Looking up all photos{videos_phrase} from album {album}...", - logging.DEBUG + logger.debug( + "Looking up all photos%s from album %s...", + videos_phrase, + album ) session_exception_handler = session_error_handle_builder( @@ -789,6 +799,9 @@ def core( # Use only ASCII characters in progress bar tqdm_kwargs["ascii"] = True + tqdm_kwargs["leave"] = False + tqdm_kwargs["dynamic_ncols"] = True + # Skip the one-line progress bar if we're only printing the filenames, # or if the progress bar is explicitly disabled, # or if this is not a terminal (e.g. cron or piping output to file) @@ -796,20 +809,24 @@ def core( only_print_filenames or no_progress_bar or not sys.stdout.isatty()) if skip_bar: photos_enumerator = photos - logger.set_tqdm(None) + # logger.set_tqdm(None) else: photos_enumerator = tqdm(photos, **tqdm_kwargs) - logger.set_tqdm(photos_enumerator) + # logger.set_tqdm(photos_enumerator) plural_suffix = "" if photos_count == 1 else "s" video_suffix = "" photos_count_str = "the first" if photos_count == 1 else photos_count if not skip_videos: video_suffix = " or video" if photos_count == 1 else " and videos" - logger.tqdm_write( - (f"Downloading {photos_count_str} {size}" - f" photo{plural_suffix}{video_suffix} to {directory} ..."), - logging.INFO + logger.info( + ("Downloading %s %s" + + " photo%s%s to %s ..."), + photos_count_str, + size, + plural_suffix, + video_suffix, + directory ) consecutive_files_found = Counter(0) @@ -822,9 +839,9 @@ def should_break(counter): while True: try: if should_break(consecutive_files_found): - logger.tqdm_write( - f"Found {until_found} consecutive previously downloaded photos. Exiting", - logging.INFO + logger.info( + "Found %s consecutive previously downloaded photos. Exiting", + until_found ) break item = next(photos_iterator) @@ -844,7 +861,7 @@ def delete_cmd(): if only_print_filenames: return 0 - logger.info("All photos have been downloaded!") + logger.info("All photos have been downloaded") if auto_delete: autodelete_photos(logger, dry_run, icloud, @@ -854,7 +871,12 @@ def delete_cmd(): logger.info(f"Waiting for {watch_interval} sec...") interval = range(1, watch_interval) for _ in interval if skip_bar else tqdm( - interval, desc="Waiting...", ascii=True): + interval, + desc="Waiting...", + ascii=True, + leave=False, + dynamic_ncols=True + ): time.sleep(1) else: break diff --git a/src/icloudpd/download.py b/src/icloudpd/download.py index 4ff466950..627939cdc 100644 --- a/src/icloudpd/download.py +++ b/src/icloudpd/download.py @@ -3,7 +3,6 @@ import os import socket import time -import logging import datetime from tzlocal import get_localzone from requests.exceptions import ConnectionError # pylint: disable=redefined-builtin @@ -41,9 +40,9 @@ def mkdirs_for_path(logger, download_path: str) -> bool: os.makedirs(name = download_dir, exist_ok=True) return True except OSError: - logger.tqdm_write( - f"Could not create folder {download_dir}", - logging.ERROR, + logger.error( + "Could not create folder %s", + download_dir, ) return False @@ -51,9 +50,9 @@ def mkdirs_for_path_dry_run(logger, download_path: str) -> bool: """ DRY Run for Creating hierarchy of folders for file path """ download_dir = os.path.dirname(download_path) if not os.path.exists(download_dir): - logger.tqdm_write( - f"[DRY RUN] Would create folder hierarchy {download_dir}", - logging.DEBUG, + logger.debug( + "[DRY RUN] Would create folder hierarchy %s", + download_dir, ) return True @@ -78,9 +77,9 @@ def download_response_to_path_dry_run( download_path: str, _created_date: datetime.datetime) -> bool: """ Pretends to save response content into a file with desired created date """ - logger.tqdm_write( - f"[DRY RUN] Would download {download_path}", - logging.INFO, + logger.info( + "[DRY RUN] Would download %s", + download_path, ) return True @@ -100,17 +99,17 @@ def download_media(logger, dry_run, icloud, photo, download_path, size) -> bool: if photo_response: return download_local(logger, photo_response, download_path, photo.created) - logger.tqdm_write( - f"Could not find URL to download {photo.filename} for size {size}!", - logging.ERROR, + logger.error( + "Could not find URL to download %s for size %s", + photo.filename, + size ) break except (ConnectionError, socket.timeout, PyiCloudAPIResponseError) as ex: if "Invalid global session" in str(ex): - logger.tqdm_write( - "Session error, re-authenticating...", - logging.ERROR) + logger.error( + "Session error, re-authenticating...") if retries > 0: # If the first re-authentication attempt failed, # start waiting a few seconds before retrying in case @@ -121,25 +120,26 @@ def download_media(logger, dry_run, icloud, photo, download_path, size) -> bool: else: # you end up here when p.e. throttling by Apple happens wait_time = (retries + 1) * constants.WAIT_SECONDS - logger.tqdm_write( - f"Error downloading {photo.filename}, retrying after {wait_time} seconds...", - logging.ERROR, + logger.error( + "Error downloading %s, retrying after %s seconds...", + photo.filename, + wait_time ) time.sleep(wait_time) except IOError: - logger.tqdm_write( - f"IOError while writing file to {download_path}! " + + logger.error( + "IOError while writing file to %s. " + "You might have run out of disk space, or the file " + "might be too large for your OS. " + "Skipping this file...", - logging.ERROR + download_path ) break else: - logger.tqdm_write( - f"Could not download {photo.filename}! Please try again later.", - logging.ERROR, + logger.error( + "Could not download %s. Please try again later.", + photo.filename, ) return False diff --git a/src/icloudpd/exif_datetime.py b/src/icloudpd/exif_datetime.py index 22eec5fb7..b4897e5aa 100644 --- a/src/icloudpd/exif_datetime.py +++ b/src/icloudpd/exif_datetime.py @@ -1,6 +1,5 @@ """Get/set EXIF dates from photos""" -import logging import piexif from piexif._exceptions import InvalidImageDataError @@ -11,7 +10,7 @@ def get_photo_exif(logger, path): exif_dict = piexif.load(path) return exif_dict.get("Exif").get(36867) except (ValueError, InvalidImageDataError): - logger.tqdm_write(f"Error fetching EXIF data for {path}", logging.DEBUG) + logger.debug("Error fetching EXIF data for %s", path) return None @@ -25,5 +24,5 @@ def set_photo_exif(logger, path, date): exif_bytes = piexif.dump(exif_dict) piexif.insert(exif_bytes, path) except (ValueError, InvalidImageDataError): - logger.tqdm_write(f"Error setting EXIF data for {path}", logging.DEBUG) + logger.debug("Error setting EXIF data for %s", path) return diff --git a/tests/test_authentication.py b/tests/test_authentication.py index 3166b2cbe..ce83a6aa1 100644 --- a/tests/test_authentication.py +++ b/tests/test_authentication.py @@ -53,7 +53,7 @@ def test_2sa_required(self): ) self.assertTrue( - "Two-step/two-factor authentication is required!" + "Two-step/two-factor authentication is required" in str(context.exception) ) @@ -92,7 +92,7 @@ def test_password_prompt(self): self._caplog.text ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) assert result.exit_code == 0 diff --git a/tests/test_autodelete_photos.py b/tests/test_autodelete_photos.py index e19399d03..09354c0d9 100644 --- a/tests/test_autodelete_photos.py +++ b/tests/test_autodelete_photos.py @@ -84,7 +84,7 @@ def astimezone(self, tz=None): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) # check files @@ -115,7 +115,7 @@ def astimezone(self, tz=None): self._caplog.text, ) self.assertIn( - f"INFO All photos have been downloaded!", self._caplog.text + f"INFO All photos have been downloaded", self._caplog.text ) self.assertIn( f"INFO Deleting any files found in 'Recently Deleted'...", @@ -174,7 +174,7 @@ def test_download_autodelete_photos(self): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) # check files @@ -204,7 +204,7 @@ def test_download_autodelete_photos(self): self._caplog.text, ) self.assertIn( - f"INFO All photos have been downloaded!", self._caplog.text + f"INFO All photos have been downloaded", self._caplog.text ) self.assertIn( f"INFO Deleting any files found in 'Recently Deleted'...", @@ -274,7 +274,7 @@ def test_autodelete_photos(self): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) self.assertIn( "INFO Deleting any files found in 'Recently Deleted'...", @@ -692,7 +692,7 @@ def test_autodelete_photos_dry_run(self): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) self.assertIn( "INFO Deleting any files found in 'Recently Deleted'...", diff --git a/tests/test_download_live_photos.py b/tests/test_download_live_photos.py index 54843720f..52c8aa390 100644 --- a/tests/test_download_live_photos.py +++ b/tests/test_download_live_photos.py @@ -77,7 +77,7 @@ def test_skip_existing_downloads_for_live_photos(self): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) assert result.exit_code == 0 @@ -158,7 +158,7 @@ def test_skip_existing_live_photodownloads(self): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) assert result.exit_code == 0 diff --git a/tests/test_download_photos.py b/tests/test_download_photos.py index 11270ea38..3113180b4 100644 --- a/tests/test_download_photos.py +++ b/tests/test_download_photos.py @@ -106,7 +106,7 @@ def test_download_and_skip_existing_photos(self): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) assert result.exit_code == 0 @@ -213,7 +213,7 @@ def mocked_download(self, size): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) assert result.exit_code == 0 @@ -283,7 +283,7 @@ def test_download_photos_and_get_exif_exceptions(self): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) assert result.exit_code == 0 @@ -354,7 +354,7 @@ def test_skip_existing_downloads(self): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) assert result.exit_code == 0 @@ -518,7 +518,7 @@ def test_handle_io_error(self): ) self.assertIn( "ERROR IOError while writing file to " - f"{os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}! " + f"{os.path.join(base_dir, os.path.normpath('2018/07/31/IMG_7409.JPG'))}. " "You might have run out of disk space, or the file might " "be too large for your OS. Skipping this file...", self._caplog.text, @@ -588,7 +588,7 @@ def mocked_authenticate(self): ) self.assertIn( - "ERROR Could not download IMG_7409.JPG! Please try again later.", + "ERROR Could not download IMG_7409.JPG. Please try again later.", self._caplog.text, ) @@ -659,7 +659,7 @@ def mocked_authenticate(self): ) self.assertIn( - "INFO iCloud re-authentication failed! Please try again later.", + "ERROR iCloud re-authentication failed. Please try again later.", self._caplog.text, ) # Make sure we only call sleep 4 times (skip the first retry) @@ -730,7 +730,7 @@ def mocked_authenticate(self): ) self.assertIn( - "ERROR Could not download IMG_7409.JPG! Please try again later.", + "ERROR Could not download IMG_7409.JPG. Please try again later.", self._caplog.text, ) assert result.exit_code == 0 @@ -834,35 +834,26 @@ def test_missing_size(self): self._caplog.text, ) - # These error messages should not be repeated more than once - assert ( - self._caplog.text.count( - "ERROR Could not find URL to download IMG_7409.JPG for size original!" - ) - == 1 - ) - assert ( - self._caplog.text.count( - "ERROR Could not find URL to download IMG_7408.JPG for size original!" - ) - == 1 - ) - assert ( - self._caplog.text.count( - "ERROR Could not find URL to download IMG_7407.JPG for size original!" - ) - == 1 - ) + # These error messages should not be repeated more than once for each size + for filename in ["IMG_7409.JPG", "IMG_7408.JPG", "IMG_7407.JPG"]: + for size in ["original", "originalVideo"]: + self.assertEqual( + sum(1 for line in self._caplog.text.splitlines() if line == + f"ERROR Could not find URL to download {filename} for size {size}" + ), + 1, + f"Errors for {filename} size {size}" + ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) - assert result.exit_code == 0 + self.assertEqual(result.exit_code, 0, "Exit code") files_in_result = glob.glob(os.path.join( base_dir, "**/*.*"), recursive=True) - assert sum(1 for _ in files_in_result) == 0 + self.assertEqual(sum(1 for _ in files_in_result), 0, "Files in result") def test_size_fallback_to_original(self): base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3]) @@ -914,7 +905,7 @@ def test_size_fallback_to_original(self): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) dp_patched.assert_called_once_with( ANY, @@ -981,7 +972,7 @@ def test_force_size(self): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) dp_patched.assert_not_called @@ -1049,7 +1040,7 @@ def astimezone(self, tz=None): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) assert result.exit_code == 0 @@ -1123,7 +1114,7 @@ def astimezone(self, tz=None): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) assert result.exit_code == 0 @@ -1182,7 +1173,7 @@ def test_unknown_item_type(self): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) dp_patched.assert_not_called @@ -1281,7 +1272,7 @@ def mocked_download(self, size): "DEBUG Skipping IMG_7404.MOV, only downloading photos.", self._caplog.text ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) # Check that file was downloaded @@ -1369,7 +1360,7 @@ def test_download_photos_and_set_exif_exceptions(self): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) assert result.exit_code == 0 @@ -1383,7 +1374,8 @@ def test_download_photos_and_set_exif_exceptions(self): file_name))), f"File {file_name} expected, but does not exist" def test_download_chinese(self): - base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3], "中文") + base_dir = os.path.join( + self.fixtures_path, inspect.stack()[0][3], "中文") recreate_path(base_dir) files_to_download = [ @@ -1431,7 +1423,7 @@ def test_download_chinese(self): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) # Check that file was downloaded @@ -1511,7 +1503,7 @@ def test_download_after_delete(self): "INFO Deleted IMG_7409.JPG in iCloud", self._caplog.text ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) assert cass.all_played assert result.exit_code == 0 @@ -1569,7 +1561,7 @@ def test_download_after_delete_fail(self): "INFO Deleted IMG_7409.JPG in iCloud", self._caplog.text ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) assert cass.all_played assert result.exit_code == 0 @@ -1654,7 +1646,7 @@ def test_download_over_old_original_photos(self): self._caplog.text, ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) # Check that file was downloaded @@ -1858,7 +1850,7 @@ def mock_raise_response_error(arg): # ) self.assertIn( - "ERROR Could not download IMG_7409.JPG! Please try again later.", + "ERROR Could not download IMG_7409.JPG. Please try again later.", self._caplog.text, ) @@ -1917,7 +1909,7 @@ def mock_raise_response_error(offset): ) self.assertIn( - "INFO Internal Error at Apple.", + "ERROR Internal Error at Apple.", self._caplog.text, ) @@ -1978,7 +1970,8 @@ def test_handle_io_error_mkdir(self): files_in_result = glob.glob(os.path.join( base_dir, "**/*.*"), recursive=True) - self.assertEqual(sum(1 for _ in files_in_result), 0, "Files at the end") + self.assertEqual(sum(1 for _ in files_in_result), + 0, "Files at the end") def test_dry_run(self): base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3]) @@ -2045,7 +2038,7 @@ def test_dry_run(self): # self._caplog.text, # ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) assert result.exit_code == 0 @@ -2053,7 +2046,8 @@ def test_dry_run(self): files_in_result = glob.glob(os.path.join( base_dir, "**/*.*"), recursive=True) - self.assertEqual(sum(1 for _ in files_in_result), 0, "Files in the result") + self.assertEqual(sum(1 for _ in files_in_result), + 0, "Files in the result") def test_download_after_delete_dry_run(self): base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3]) @@ -2114,12 +2108,14 @@ def raise_response_error(a0_, a1_, a2_): "INFO [DRY RUN] Would delete IMG_7409.JPG in iCloud", self._caplog.text ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) - self.assertEqual(cass.all_played, False, "All mocks played") + self.assertEqual( + cass.all_played, False, "All mocks played") self.assertEqual(result.exit_code, 0, "Exit code") files_in_result = glob.glob(os.path.join( base_dir, "**/*.*"), recursive=True) - self.assertEqual( sum(1 for _ in files_in_result), 0, "Files in the result") + self.assertEqual(sum(1 for _ in files_in_result), + 0, "Files in the result") diff --git a/tests/test_listing_recent_photos.py b/tests/test_listing_recent_photos.py index 1b96ed9b2..e446cd3ae 100644 --- a/tests/test_listing_recent_photos.py +++ b/tests/test_listing_recent_photos.py @@ -224,7 +224,7 @@ def test_listing_recent_photos_with_missing_downloadURL(self): self.assertEqual.__self__.maxDiff = None self.assertEqual("""\ -KeyError: 'downloadURL' attribute was not found in the photo fields! +KeyError: 'downloadURL' attribute was not found in the photo fields. icloudpd has saved the photo record to: ./icloudpd-photo-error.json Please create a Gist with the contents of this file: https://gist.github.com Then create an issue on GitHub: https://github.com/icloud-photos-downloader/icloud_photos_downloader/issues diff --git a/tests/test_two_step_auth.py b/tests/test_two_step_auth.py index 2c644e0a7..a0c3bd3ed 100644 --- a/tests/test_two_step_auth.py +++ b/tests/test_two_step_auth.py @@ -83,7 +83,7 @@ def test_2sa_flow_device_2fa(self): ) self.assertIn("DEBUG Authenticating...", self._caplog.text) self.assertIn( - "INFO Two-step/two-factor authentication is required!", + "INFO Two-step/two-factor authentication is required", self._caplog.text, ) self.assertIn(" 0: SMS to *******03", result.output) @@ -101,7 +101,7 @@ def test_2sa_flow_device_2fa(self): "DEBUG Looking up all photos and videos from album All Photos...", self._caplog.text ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) assert result.exit_code == 0 @@ -134,7 +134,7 @@ def test_2sa_flow_sms(self): ) self.assertIn("DEBUG Authenticating...", self._caplog.text) self.assertIn( - "INFO Two-step/two-factor authentication is required!", + "INFO Two-step/two-factor authentication is required", self._caplog.text, ) self.assertIn(" 0: SMS to *******03", result.output) @@ -152,7 +152,7 @@ def test_2sa_flow_sms(self): "DEBUG Looking up all photos and videos from album All Photos...", self._caplog.text ) self.assertIn( - "INFO All photos have been downloaded!", self._caplog.text + "INFO All photos have been downloaded", self._caplog.text ) assert result.exit_code == 0 @@ -189,7 +189,7 @@ def test_2sa_flow_sms_failed(self): ) self.assertIn("DEBUG Authenticating...", self._caplog.text) self.assertIn( - "INFO Two-step/two-factor authentication is required!", + "INFO Two-step/two-factor authentication is required", self._caplog.text, ) self.assertIn(" 0: SMS to *******03", result.output)