From 8ba6cc21df0c19d79007881fffe0eed17020ca2c Mon Sep 17 00:00:00 2001 From: Abhishek Sriraman Date: Sat, 27 Jan 2024 19:13:47 +0000 Subject: [PATCH 1/4] Add command line arguments to filter by date. With the new argments --date-before and --date-after, user can choose to only download items that are created either before, after, or before and after the specified dates. Dates are assumed to be specified in the local timezone. --- README.md | 1 + src/icloudpd/base.py | 48 +++++++++++++++++++ tests/test_cli.py | 34 ++++++++++++++ tests/test_download_photos.py | 88 +++++++++++++++++++++++++++++++++++ 4 files changed, 171 insertions(+) diff --git a/README.md b/README.md index 164783361..979784e29 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ There are three ways to run `icloudpd`: - One time download and an option to monitor for iCloud changes continuously (`--watch-with-interval` option) - Optimizations for incremental runs (`--until-found` and `--recent` options) - Photo meta data (EXIF) updates (`--set-exif-datetime` option) +- Filter items to download by creation date (`--date-before` and/or `--date-after`) - ... and many more (use `--help` option to get full list) ## Experimental Mode diff --git a/src/icloudpd/base.py b/src/icloudpd/base.py index 2b1a5f3c7..81b19bcc0 100644 --- a/src/icloudpd/base.py +++ b/src/icloudpd/base.py @@ -233,6 +233,16 @@ type=click.Choice(["com", "cn"]), default="com", ) +@click.option("--date-before", + help="Latest date with which to filter which photo/videos will be downloaded, " + "specified in the format YYYY-MM-DD.", + default=None, + ) +@click.option("--date-after", + help="Earliest date with which to filter which photo/videos will be downloaded, " + "specified in the format YYYY-MM-DD.", + default=None, + ) @click.option("--watch-with-interval", help="Run downloading in a infinite cycle, waiting specified seconds between runs", type=click.IntRange(1), @@ -281,6 +291,8 @@ def main( threads_num: int, # pylint: disable=W0613 delete_after_download: bool, domain: str, + date_before: str, + date_after: str, watch_with_interval: Optional[int], dry_run: bool ): @@ -324,6 +336,25 @@ def main( ) sys.exit(2) + if date_before: + try: + date_before = datetime.datetime.strptime( + date_before, "%Y-%m-%d") + except ValueError: + print("Given --date-before does not match required format YYYY-MM-DD.") + sys.exit(2) + + date_before = date_before.replace(tzinfo=get_localzone()) + + if date_after: + try: + date_after = datetime.datetime.strptime(date_after, "%Y-%m-%d") + except ValueError: + print("Given --date-after does not match required format YYYY-MM-DD.") + sys.exit(2) + + date_after = date_after.replace(tzinfo=get_localzone()) + sys.exit( core( download_builder( @@ -337,6 +368,8 @@ def main( set_exif_datetime, skip_live_photos, live_photo_size, + date_before, + date_after, dry_run) if directory is not None else ( lambda _s: lambda _c, _p: False), @@ -387,6 +420,8 @@ def download_builder( set_exif_datetime: bool, skip_live_photos: bool, live_photo_size: str, + date_before: datetime.datetime, + date_after: datetime.datetime, dry_run: bool) -> Callable[[PyiCloudService], Callable[[Counter, PhotoAsset], bool]]: """factory for downloader""" def state_( @@ -394,6 +429,7 @@ def state_( def download_photo_(counter: Counter, photo: PhotoAsset) -> bool: """internal function for actually downloading the photos""" filename = clean_filename(photo.filename) + if skip_videos and photo.item_type != "image": logger.debug( "Skipping %s, only downloading photos." + @@ -418,6 +454,18 @@ def download_photo_(counter: Counter, photo: PhotoAsset) -> bool: photo.created) created_date = photo.created + if date_before and created_date > date_before: + logger.debug( + "Skipping %s, date is after the given latest date.", + filename) + return False + + if date_after and created_date < date_after: + logger.debug( + "Skipping %s, date is before the given earliest date.", + filename) + return False + try: if folder_structure.lower() == "none": date_path = "" diff --git a/tests/test_cli.py b/tests/test_cli.py index 8f4865ec8..3dddb17ab 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -209,3 +209,37 @@ def test_conflict_options_delete_after_download_and_auto_delete(self): ], ) assert result.exit_code == 2 + + def test_bad_date_before_format(self): + runner = CliRunner() + result = runner.invoke( + main, + [ + "--username", + "jdoe@gmail.com", + "--password", + "password1", + "-d", + "/tmp", + "--date-before", + "alpha" + ], + ) + assert result.exit_code == 2 + + def test_bad_date_after_format(self): + runner = CliRunner() + result = runner.invoke( + main, + [ + "--username", + "jdoe@gmail.com", + "--password", + "password1", + "-d", + "/tmp", + "--date-after", + "alpha" + ], + ) + assert result.exit_code == 2 \ No newline at end of file diff --git a/tests/test_download_photos.py b/tests/test_download_photos.py index 5495b0bac..ac994305c 100644 --- a/tests/test_download_photos.py +++ b/tests/test_download_photos.py @@ -2293,3 +2293,91 @@ def raise_response_error(a0_, a1_, a2_): self.assertEqual(sum(1 for _ in files_in_result), 0, "Files in the result") + + def test_skip_by_date_before(self): + base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3]) + cookie_dir = os.path.join(base_dir, "cookie") + data_dir = os.path.join(base_dir, "data") + + for dir in [base_dir, cookie_dir, data_dir]: + recreate_path(dir) + + default_args = [ + "--username", + "jdoe@gmail.com", + "--password", + "password1", + "--recent", + "1", + "--skip-live-photos", + "--skip-videos", + "--dry-run", + "--no-progress-bar", + "--threads-num", + 1, + "-d", + data_dir, + "--cookie-directory", + cookie_dir, + ] + + with vcr.use_cassette(os.path.join(self.vcr_path, "listing_photos.yml")): + runner = CliRunner(env={ + "CLIENT_ID": "DE309E26-942E-11E8-92F5-14109FE0B321" + }) + + result = runner.invoke( + main, + default_args + ["--date-before", "2000-01-01"], + ) + print_result_exception(result) + + assert result.exit_code == 0 + self.assertIn( + "DEBUG Looking up all photos from album All Photos...", self._caplog.text) + self.assertIn( + f"DEBUG Skipping IMG_7409.JPG, date is after", self._caplog.text + ) + + def test_skip_by_date_after(self): + base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3]) + cookie_dir = os.path.join(base_dir, "cookie") + data_dir = os.path.join(base_dir, "data") + + for dir in [base_dir, cookie_dir, data_dir]: + recreate_path(dir) + + default_args = [ + "--username", + "jdoe@gmail.com", + "--password", + "password1", + "--recent", + "1", + "--skip-live-photos", + "--skip-videos", + "--dry-run", + "--no-progress-bar", + "--threads-num", + 1, + "-d", + data_dir, + "--cookie-directory", + cookie_dir, + ] + with vcr.use_cassette(os.path.join(self.vcr_path, "listing_photos.yml")): + runner = CliRunner(env={ + "CLIENT_ID": "DE309E26-942E-11E8-92F5-14109FE0B321" + }) + + result = runner.invoke( + main, + default_args + ["--date-after", "2020-01-01"], + ) + print_result_exception(result) + assert result.exit_code == 0 + self.assertIn( + "DEBUG Looking up all photos from album All Photos...", self._caplog.text) + self.assertIn( + f"DEBUG Skipping IMG_7409.JPG, date is before", self._caplog.text + ) \ No newline at end of file From 7aea1d68fe51d883bc915e525ac86885466a4d4b Mon Sep 17 00:00:00 2001 From: Abhishek Sriraman Date: Sun, 4 Feb 2024 16:07:32 +0000 Subject: [PATCH 2/4] Address comments. --- README.md | 2 +- src/icloudpd/base.py | 44 ++++++++++++++++------------------ tests/test_cli.py | 8 +++---- tests/test_download_photos.py | 45 ++++++++++++++++++----------------- 4 files changed, 49 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 979784e29..790e5b5f2 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ There are three ways to run `icloudpd`: - One time download and an option to monitor for iCloud changes continuously (`--watch-with-interval` option) - Optimizations for incremental runs (`--until-found` and `--recent` options) - Photo meta data (EXIF) updates (`--set-exif-datetime` option) -- Filter items to download by creation date (`--date-before` and/or `--date-after`) +- Filter items to download by creation date (`--created-before` and/or `--created-after`) - ... and many more (use `--help` option to get full list) ## Experimental Mode diff --git a/src/icloudpd/base.py b/src/icloudpd/base.py index 81b19bcc0..6f0756b19 100644 --- a/src/icloudpd/base.py +++ b/src/icloudpd/base.py @@ -233,14 +233,12 @@ type=click.Choice(["com", "cn"]), default="com", ) -@click.option("--date-before", - help="Latest date with which to filter which photo/videos will be downloaded, " - "specified in the format YYYY-MM-DD.", +@click.option("--created-before", + help="Only download pictures/videos created before specified date in YYYY-MM-DD format.", default=None, ) -@click.option("--date-after", - help="Earliest date with which to filter which photo/videos will be downloaded, " - "specified in the format YYYY-MM-DD.", +@click.option("--created-after", + help="Only download pictures/videos created after specified date in YYYY-MM-DD format.", default=None, ) @click.option("--watch-with-interval", @@ -291,8 +289,8 @@ def main( threads_num: int, # pylint: disable=W0613 delete_after_download: bool, domain: str, - date_before: str, - date_after: str, + created_before: str, + created_after: str, watch_with_interval: Optional[int], dry_run: bool ): @@ -336,24 +334,24 @@ def main( ) sys.exit(2) - if date_before: + if created_before: try: - date_before = datetime.datetime.strptime( - date_before, "%Y-%m-%d") + created_before = datetime.datetime.strptime( + created_before, "%Y-%m-%d") except ValueError: - print("Given --date-before does not match required format YYYY-MM-DD.") + print("Given --created-before does not match required format YYYY-MM-DD.") sys.exit(2) - date_before = date_before.replace(tzinfo=get_localzone()) + created_before = created_before.replace(tzinfo=get_localzone()) - if date_after: + if created_after: try: - date_after = datetime.datetime.strptime(date_after, "%Y-%m-%d") + created_after = datetime.datetime.strptime(created_after, "%Y-%m-%d") except ValueError: - print("Given --date-after does not match required format YYYY-MM-DD.") + print("Given --created-after does not match required format YYYY-MM-DD.") sys.exit(2) - date_after = date_after.replace(tzinfo=get_localzone()) + created_after = created_after.replace(tzinfo=get_localzone()) sys.exit( core( @@ -368,8 +366,8 @@ def main( set_exif_datetime, skip_live_photos, live_photo_size, - date_before, - date_after, + created_before, + created_after, dry_run) if directory is not None else ( lambda _s: lambda _c, _p: False), @@ -420,8 +418,8 @@ def download_builder( set_exif_datetime: bool, skip_live_photos: bool, live_photo_size: str, - date_before: datetime.datetime, - date_after: datetime.datetime, + created_before: datetime.datetime, + created_after: datetime.datetime, dry_run: bool) -> Callable[[PyiCloudService], Callable[[Counter, PhotoAsset], bool]]: """factory for downloader""" def state_( @@ -454,13 +452,13 @@ def download_photo_(counter: Counter, photo: PhotoAsset) -> bool: photo.created) created_date = photo.created - if date_before and created_date > date_before: + if created_before and created_date > created_before: logger.debug( "Skipping %s, date is after the given latest date.", filename) return False - if date_after and created_date < date_after: + if created_after and created_date < created_after: logger.debug( "Skipping %s, date is before the given earliest date.", filename) diff --git a/tests/test_cli.py b/tests/test_cli.py index 3dddb17ab..535c3ea48 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -210,7 +210,7 @@ def test_conflict_options_delete_after_download_and_auto_delete(self): ) assert result.exit_code == 2 - def test_bad_date_before_format(self): + def test_bad_created_before_format(self): runner = CliRunner() result = runner.invoke( main, @@ -221,13 +221,13 @@ def test_bad_date_before_format(self): "password1", "-d", "/tmp", - "--date-before", + "--created-before", "alpha" ], ) assert result.exit_code == 2 - def test_bad_date_after_format(self): + def test_bad_created_after_format(self): runner = CliRunner() result = runner.invoke( main, @@ -238,7 +238,7 @@ def test_bad_date_after_format(self): "password1", "-d", "/tmp", - "--date-after", + "--created-after", "alpha" ], ) diff --git a/tests/test_download_photos.py b/tests/test_download_photos.py index ac994305c..25081324e 100644 --- a/tests/test_download_photos.py +++ b/tests/test_download_photos.py @@ -2294,15 +2294,22 @@ def raise_response_error(a0_, a1_, a2_): self.assertEqual(sum(1 for _ in files_in_result), 0, "Files in the result") - def test_skip_by_date_before(self): + def test_skip_by_created_before(self): base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3]) cookie_dir = os.path.join(base_dir, "cookie") data_dir = os.path.join(base_dir, "data") for dir in [base_dir, cookie_dir, data_dir]: recreate_path(dir) + + with vcr.use_cassette(os.path.join(self.vcr_path, "listing_photos.yml")): + runner = CliRunner(env={ + "CLIENT_ID": "DE309E26-942E-11E8-92F5-14109FE0B321" + }) - default_args = [ + result = runner.invoke( + main, + [ "--username", "jdoe@gmail.com", "--password", @@ -2319,16 +2326,9 @@ def test_skip_by_date_before(self): data_dir, "--cookie-directory", cookie_dir, + "--created-before", + "2000-01-01" ] - - with vcr.use_cassette(os.path.join(self.vcr_path, "listing_photos.yml")): - runner = CliRunner(env={ - "CLIENT_ID": "DE309E26-942E-11E8-92F5-14109FE0B321" - }) - - result = runner.invoke( - main, - default_args + ["--date-before", "2000-01-01"], ) print_result_exception(result) @@ -2339,7 +2339,7 @@ def test_skip_by_date_before(self): f"DEBUG Skipping IMG_7409.JPG, date is after", self._caplog.text ) - def test_skip_by_date_after(self): + def test_skip_by_created_after(self): base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3]) cookie_dir = os.path.join(base_dir, "cookie") data_dir = os.path.join(base_dir, "data") @@ -2347,7 +2347,14 @@ def test_skip_by_date_after(self): for dir in [base_dir, cookie_dir, data_dir]: recreate_path(dir) - default_args = [ + with vcr.use_cassette(os.path.join(self.vcr_path, "listing_photos.yml")): + runner = CliRunner(env={ + "CLIENT_ID": "DE309E26-942E-11E8-92F5-14109FE0B321" + }) + + result = runner.invoke( + main, + [ "--username", "jdoe@gmail.com", "--password", @@ -2364,15 +2371,9 @@ def test_skip_by_date_after(self): data_dir, "--cookie-directory", cookie_dir, - ] - with vcr.use_cassette(os.path.join(self.vcr_path, "listing_photos.yml")): - runner = CliRunner(env={ - "CLIENT_ID": "DE309E26-942E-11E8-92F5-14109FE0B321" - }) - - result = runner.invoke( - main, - default_args + ["--date-after", "2020-01-01"], + "--created-after", + "2020-01-01" + ] ) print_result_exception(result) assert result.exit_code == 0 From 4afa917ac74480f970c8c8d1247cd76accc11b2e Mon Sep 17 00:00:00 2001 From: Abhishek Sriraman Date: Sun, 4 Feb 2024 16:11:18 +0000 Subject: [PATCH 3/4] Update download photos test to check result dir --- tests/test_download_photos.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_download_photos.py b/tests/test_download_photos.py index 25081324e..d20f759dd 100644 --- a/tests/test_download_photos.py +++ b/tests/test_download_photos.py @@ -2339,6 +2339,11 @@ def test_skip_by_created_before(self): f"DEBUG Skipping IMG_7409.JPG, date is after", self._caplog.text ) + files_in_result = glob.glob(os.path.join( + data_dir, "**/*.*"), recursive=True) + + assert sum(1 for _ in files_in_result) == 0, "No files should have been downloaded." + def test_skip_by_created_after(self): base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3]) cookie_dir = os.path.join(base_dir, "cookie") @@ -2381,4 +2386,6 @@ def test_skip_by_created_after(self): "DEBUG Looking up all photos from album All Photos...", self._caplog.text) self.assertIn( f"DEBUG Skipping IMG_7409.JPG, date is before", self._caplog.text - ) \ No newline at end of file + ) + files_in_result = glob.glob(os.path.join(data_dir, "**/*.*"), recursive=True) + assert sum(1 for _ in files_in_result) == 0, "No files should have been downloaded." From 59e3a06c9d04e61de578f92314cfc1594c99cea7 Mon Sep 17 00:00:00 2001 From: Abhishek Sriraman Date: Sun, 4 Feb 2024 16:30:34 +0000 Subject: [PATCH 4/4] Fix typing errors --- src/icloudpd/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/icloudpd/base.py b/src/icloudpd/base.py index 6f0756b19..8a4f6dcc2 100644 --- a/src/icloudpd/base.py +++ b/src/icloudpd/base.py @@ -418,8 +418,8 @@ def download_builder( set_exif_datetime: bool, skip_live_photos: bool, live_photo_size: str, - created_before: datetime.datetime, - created_after: datetime.datetime, + created_before: datetime.datetime | None, + created_after: datetime.datetime | None, dry_run: bool) -> Callable[[PyiCloudService], Callable[[Counter, PhotoAsset], bool]]: """factory for downloader""" def state_(