From 41be37dc373201157aa497d0baf8b8c35c119fa8 Mon Sep 17 00:00:00 2001 From: Muhammad Lufti Apriliyanto <86290293+pop2pop3@users.noreply.github.com> Date: Wed, 23 Oct 2024 00:25:07 +0800 Subject: [PATCH 1/3] Update base.py fix: New AppleID Auth with SIRP Inspired by the work from @iowk after fixing the authentication of `icloudpd`: https://github.com/icloud-photos-downloader/icloud_photos_downloader/commit/4bcb2ac46a585205cbf3886b3df78179b34b18b1#diff-f2270f557e6afedd3e082f4dd8478d1c96dea051d6ea2e5b1229c0bb58d3d7f4R339 --- pyicloud/base.py | 96 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 81 insertions(+), 15 deletions(-) diff --git a/pyicloud/base.py b/pyicloud/base.py index 6ac8bdbd..b0bb8be0 100644 --- a/pyicloud/base.py +++ b/pyicloud/base.py @@ -9,6 +9,9 @@ from re import match import http.cookiejar as cookielib import getpass +import srp +import base64 +import hashlib from pyicloud.exceptions import ( PyiCloudFailedLoginException, @@ -36,6 +39,8 @@ "X-Apple-ID-Session-Id": "session_id", "X-Apple-Session-Token": "session_token", "X-Apple-TwoSV-Trust-Token": "trust_token", + "X-Apple-I-Rscd": "apple_rscd", + "X-Apple-I-Ercd": "apple_ercd", "scnt": "scnt", } @@ -200,14 +205,11 @@ class PyiCloudService: pyicloud.iphone.location() """ - AUTH_ENDPOINT = "https://idmsa.apple.com/appleauth/auth" - HOME_ENDPOINT = "https://www.icloud.com" - SETUP_ENDPOINT = "https://setup.icloud.com/setup/ws/1" - def __init__( self, apple_id, password=None, + domain="com", cookie_directory=None, verify=True, client_id=None, @@ -225,6 +227,19 @@ def __init__( self.password_filter = PyiCloudPasswordFilter(password) LOGGER.addFilter(self.password_filter) + if (domain == 'com'): + self.AUTH_ENDPOINT = "https://idmsa.apple.com/appleauth/auth" + self.HOME_ENDPOINT = "https://www.icloud.com" + self.SETUP_ENDPOINT = "https://setup.icloud.com/setup/ws/1" + elif (domain == 'cn'): + self.AUTH_ENDPOINT = "https://idmsa.apple.com.cn/appleauth/auth" + self.HOME_ENDPOINT = "https://www.icloud.com.cn" + self.SETUP_ENDPOINT = "https://setup.icloud.com.cn/setup/ws/1" + else: + raise NotImplementedError(f"Domain '{domain}' is not supported yet") + + self.domain = domain + if cookie_directory: self._cookie_directory = path.expanduser(path.normpath(cookie_directory)) if not path.exists(self._cookie_directory): @@ -253,7 +268,9 @@ def __init__( self.session = PyiCloudSession(self) self.session.verify = verify self.session.headers.update( - {"Origin": self.HOME_ENDPOINT, "Referer": "%s/" % self.HOME_ENDPOINT} + {"Origin": self.HOME_ENDPOINT, + "Referer": "%s/" % self.HOME_ENDPOINT, + 'User-Agent': 'Opera/9.52 (X11; Linux i686; U; en)'} ) cookiejar_path = self.cookiejar_path @@ -267,6 +284,15 @@ def __init__( # The cookiejar will get replaced with a valid one after # successful authentication. LOGGER.warning("Failed to read cookiejar %s", cookiejar_path) + + # Unsure if this is still needed + self.params = { + 'clientBuildNumber': '17DHotfix5', + 'clientMasteringNumber': '17DHotfix5', + 'ckjsBuildVersion': '17DProjectDev77', + 'ckjsVersion': '2.0.5', + 'clientId': self.client_id, + } self.authenticate() @@ -306,13 +332,6 @@ def authenticate(self, force_refresh=False, service=None): if not login_successful: LOGGER.debug("Authenticating as %s", self.user["accountName"]) - data = dict(self.user) - - data["rememberMe"] = True - data["trustTokens"] = [] - if self.session_data.get("trust_token"): - data["trustTokens"] = [self.session_data.get("trust_token")] - headers = self._get_auth_headers() if self.session_data.get("scnt"): @@ -320,10 +339,55 @@ def authenticate(self, force_refresh=False, service=None): if self.session_data.get("session_id"): headers["X-Apple-ID-Session-Id"] = self.session_data.get("session_id") - + class SrpPassword(): + def __init__(self, password: str): + self.password = password + def set_encrypt_info(self, salt: bytes, iterations: int, key_length: int): + self.salt = salt + self.iterations = iterations + self.key_length = key_length + def encode(self): + password_hash = hashlib.sha256(self.password.encode('utf-8')).digest() + return hashlib.pbkdf2_hmac('sha256', password_hash, salt, iterations, key_length) + srp_password = SrpPassword(self.user["password"]) + srp.rfc5054_enable() + srp.no_username_in_x() + usr = srp.User(self.user["accountName"], srp_password, hash_alg=srp.SHA256, ng_type=srp.NG_2048) + uname, A = usr.start_authentication() + data = { + 'a': base64.b64encode(A).decode(), + 'accountName': uname, + 'protocols': ['s2k', 's2k_fo'] + } + try: + response = self.session.post("%s/signin/init" % self.AUTH_ENDPOINT, data=json.dumps(data), headers=headers) + response.raise_for_status() + except PyiCloudAPIResponseException as error: + msg = "Failed to initiate srp authentication." + raise PyiCloudFailedLoginException(msg, error) from error + body = response.json() + salt = base64.b64decode(body['salt']) + b = base64.b64decode(body['b']) + c = body['c'] + iterations = body['iteration'] + key_length = 32 + srp_password.set_encrypt_info(salt, iterations, key_length) + m1 = usr.process_challenge( salt, b ) + m2 = usr.H_AMK + data = { + "accountName": uname, + "c": c, + "m1": base64.b64encode(m1).decode(), + "m2": base64.b64encode(m2).decode(), + "rememberMe": True, + "trustTokens": [], + } + if self.session_data.get("trust_token"): + data["trustTokens"] = [self.session_data.get("trust_token")] + try: self.session.post( - "%s/signin" % self.AUTH_ENDPOINT, + "%s/signin/complete" % self.AUTH_ENDPOINT, params={"isRememberMeEnabled": "true"}, data=json.dumps(data), headers=headers, @@ -334,6 +398,8 @@ def authenticate(self, force_refresh=False, service=None): self._authenticate_with_token() + self.params.update({'dsid': self.data['dsInfo']['dsid']}) + self._webservices = self.data["webservices"] LOGGER.debug("Authentication completed successfully") @@ -387,7 +453,7 @@ def _validate_token(self): def _get_auth_headers(self, overrides=None): headers = { - "Accept": "*/*", + "Accept": "application/json, text/javascript", "Content-Type": "application/json", "X-Apple-OAuth-Client-Id": "d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d", "X-Apple-OAuth-Client-Type": "firstPartyAuth", From 5fdbeaff215ac2d3ba87c1d6a268dfc7ea6c3acc Mon Sep 17 00:00:00 2001 From: Eugene Polonsky Date: Mon, 30 Dec 2024 13:56:23 -0800 Subject: [PATCH 2/3] Implemented the ability to add a photo to an album --- pyicloud/services/photos.py | 1311 ++++++++++++++++++----------------- 1 file changed, 673 insertions(+), 638 deletions(-) diff --git a/pyicloud/services/photos.py b/pyicloud/services/photos.py index 06b3dd32..1af1d242 100644 --- a/pyicloud/services/photos.py +++ b/pyicloud/services/photos.py @@ -1,638 +1,673 @@ -"""Photo service.""" -import json -import base64 -from urllib.parse import urlencode - -from datetime import datetime, timezone -from pyicloud.exceptions import PyiCloudServiceNotActivatedException - - -class PhotosService: - """The 'Photos' iCloud service.""" - - SMART_FOLDERS = { - "All Photos": { - "obj_type": "CPLAssetByAddedDate", - "list_type": "CPLAssetAndMasterByAddedDate", - "direction": "ASCENDING", - "query_filter": None, - }, - "Time-lapse": { - "obj_type": "CPLAssetInSmartAlbumByAssetDate:Timelapse", - "list_type": "CPLAssetAndMasterInSmartAlbumByAssetDate", - "direction": "ASCENDING", - "query_filter": [ - { - "fieldName": "smartAlbum", - "comparator": "EQUALS", - "fieldValue": {"type": "STRING", "value": "TIMELAPSE"}, - } - ], - }, - "Videos": { - "obj_type": "CPLAssetInSmartAlbumByAssetDate:Video", - "list_type": "CPLAssetAndMasterInSmartAlbumByAssetDate", - "direction": "ASCENDING", - "query_filter": [ - { - "fieldName": "smartAlbum", - "comparator": "EQUALS", - "fieldValue": {"type": "STRING", "value": "VIDEO"}, - } - ], - }, - "Slo-mo": { - "obj_type": "CPLAssetInSmartAlbumByAssetDate:Slomo", - "list_type": "CPLAssetAndMasterInSmartAlbumByAssetDate", - "direction": "ASCENDING", - "query_filter": [ - { - "fieldName": "smartAlbum", - "comparator": "EQUALS", - "fieldValue": {"type": "STRING", "value": "SLOMO"}, - } - ], - }, - "Bursts": { - "obj_type": "CPLAssetBurstStackAssetByAssetDate", - "list_type": "CPLBurstStackAssetAndMasterByAssetDate", - "direction": "ASCENDING", - "query_filter": None, - }, - "Favorites": { - "obj_type": "CPLAssetInSmartAlbumByAssetDate:Favorite", - "list_type": "CPLAssetAndMasterInSmartAlbumByAssetDate", - "direction": "ASCENDING", - "query_filter": [ - { - "fieldName": "smartAlbum", - "comparator": "EQUALS", - "fieldValue": {"type": "STRING", "value": "FAVORITE"}, - } - ], - }, - "Panoramas": { - "obj_type": "CPLAssetInSmartAlbumByAssetDate:Panorama", - "list_type": "CPLAssetAndMasterInSmartAlbumByAssetDate", - "direction": "ASCENDING", - "query_filter": [ - { - "fieldName": "smartAlbum", - "comparator": "EQUALS", - "fieldValue": {"type": "STRING", "value": "PANORAMA"}, - } - ], - }, - "Screenshots": { - "obj_type": "CPLAssetInSmartAlbumByAssetDate:Screenshot", - "list_type": "CPLAssetAndMasterInSmartAlbumByAssetDate", - "direction": "ASCENDING", - "query_filter": [ - { - "fieldName": "smartAlbum", - "comparator": "EQUALS", - "fieldValue": {"type": "STRING", "value": "SCREENSHOT"}, - } - ], - }, - "Live": { - "obj_type": "CPLAssetInSmartAlbumByAssetDate:Live", - "list_type": "CPLAssetAndMasterInSmartAlbumByAssetDate", - "direction": "ASCENDING", - "query_filter": [ - { - "fieldName": "smartAlbum", - "comparator": "EQUALS", - "fieldValue": {"type": "STRING", "value": "LIVE"}, - } - ], - }, - "Recently Deleted": { - "obj_type": "CPLAssetDeletedByExpungedDate", - "list_type": "CPLAssetAndMasterDeletedByExpungedDate", - "direction": "ASCENDING", - "query_filter": None, - }, - "Hidden": { - "obj_type": "CPLAssetHiddenByAssetDate", - "list_type": "CPLAssetAndMasterHiddenByAssetDate", - "direction": "ASCENDING", - "query_filter": None, - }, - } - - def __init__(self, service_root, session, params): - self.session = session - self.params = dict(params) - self._service_root = service_root - self.service_endpoint = ( - "%s/database/1/com.apple.photos.cloud/production/private" - % self._service_root - ) - - self._albums = None - - self.params.update({"remapEnums": True, "getCurrentSyncToken": True}) - - url = f"{self.service_endpoint}/records/query?{urlencode(self.params)}" - json_data = ( - '{"query":{"recordType":"CheckIndexingState"},' - '"zoneID":{"zoneName":"PrimarySync"}}' - ) - request = self.session.post( - url, data=json_data, headers={"Content-type": "text/plain"} - ) - response = request.json() - indexing_state = response["records"][0]["fields"]["state"]["value"] - if indexing_state != "FINISHED": - raise PyiCloudServiceNotActivatedException( - "iCloud Photo Library not finished indexing. " - "Please try again in a few minutes." - ) - - # TODO: Does syncToken ever change? # pylint: disable=fixme - # self.params.update({ - # 'syncToken': response['syncToken'], - # 'clientInstanceId': self.params.pop('clientId') - # }) - - self._photo_assets = {} - - @property - def albums(self): - """Returns photo albums.""" - if not self._albums: - self._albums = { - name: PhotoAlbum(self, name, **props) - for (name, props) in self.SMART_FOLDERS.items() - } - - for folder in self._fetch_folders(): - - # Skiping albums having null name, that can happen sometime - if "albumNameEnc" not in folder["fields"]: - continue - - # TODO: Handle subfolders # pylint: disable=fixme - if folder["recordName"] == "----Root-Folder----" or ( - folder["fields"].get("isDeleted") - and folder["fields"]["isDeleted"]["value"] - ): - continue - - folder_id = folder["recordName"] - folder_obj_type = ( - "CPLContainerRelationNotDeletedByAssetDate:%s" % folder_id - ) - folder_name = base64.b64decode( - folder["fields"]["albumNameEnc"]["value"] - ).decode("utf-8") - query_filter = [ - { - "fieldName": "parentId", - "comparator": "EQUALS", - "fieldValue": {"type": "STRING", "value": folder_id}, - } - ] - - album = PhotoAlbum( - self, - folder_name, - "CPLContainerRelationLiveByAssetDate", - folder_obj_type, - "ASCENDING", - query_filter, - ) - self._albums[folder_name] = album - - return self._albums - - def _fetch_folders(self): - url = f"{self.service_endpoint}/records/query?{urlencode(self.params)}" - json_data = ( - '{"query":{"recordType":"CPLAlbumByPositionLive"},' - '"zoneID":{"zoneName":"PrimarySync"}}' - ) - - request = self.session.post( - url, data=json_data, headers={"Content-type": "text/plain"} - ) - response = request.json() - - return response["records"] - - @property - def all(self): - """Returns all photos.""" - return self.albums["All Photos"] - - -class PhotoAlbum: - """A photo album.""" - - def __init__( - self, - service, - name, - list_type, - obj_type, - direction, - query_filter=None, - page_size=100, - ): - self.name = name - self.service = service - self.list_type = list_type - self.obj_type = obj_type - self.direction = direction - self.query_filter = query_filter - self.page_size = page_size - - self._len = None - - @property - def title(self): - """Gets the album name.""" - return self.name - - def __iter__(self): - return self.photos - - def __len__(self): - if self._len is None: - url = "{}/internal/records/query/batch?{}".format( - self.service.service_endpoint, - urlencode(self.service.params), - ) - request = self.service.session.post( - url, - data=json.dumps( - { - "batch": [ - { - "resultsLimit": 1, - "query": { - "filterBy": { - "fieldName": "indexCountID", - "fieldValue": { - "type": "STRING_LIST", - "value": [self.obj_type], - }, - "comparator": "IN", - }, - "recordType": "HyperionIndexCountLookup", - }, - "zoneWide": True, - "zoneID": {"zoneName": "PrimarySync"}, - } - ] - } - ), - headers={"Content-type": "text/plain"}, - ) - response = request.json() - - self._len = response["batch"][0]["records"][0]["fields"]["itemCount"][ - "value" - ] - - return self._len - - @property - def photos(self): - """Returns the album photos.""" - if self.direction == "DESCENDING": - offset = len(self) - 1 - else: - offset = 0 - - while True: - url = ("%s/records/query?" % self.service.service_endpoint) + urlencode( - self.service.params - ) - request = self.service.session.post( - url, - data=json.dumps( - self._list_query_gen( - offset, self.list_type, self.direction, self.query_filter - ) - ), - headers={"Content-type": "text/plain"}, - ) - response = request.json() - - asset_records = {} - master_records = [] - for rec in response["records"]: - if rec["recordType"] == "CPLAsset": - master_id = rec["fields"]["masterRef"]["value"]["recordName"] - asset_records[master_id] = rec - elif rec["recordType"] == "CPLMaster": - master_records.append(rec) - - master_records_len = len(master_records) - if master_records_len: - if self.direction == "DESCENDING": - offset = offset - master_records_len - else: - offset = offset + master_records_len - - for master_record in master_records: - record_name = master_record["recordName"] - yield PhotoAsset( - self.service, master_record, asset_records[record_name] - ) - else: - break - - def _list_query_gen(self, offset, list_type, direction, query_filter=None): - query = { - "query": { - "filterBy": [ - { - "fieldName": "startRank", - "fieldValue": {"type": "INT64", "value": offset}, - "comparator": "EQUALS", - }, - { - "fieldName": "direction", - "fieldValue": {"type": "STRING", "value": direction}, - "comparator": "EQUALS", - }, - ], - "recordType": list_type, - }, - "resultsLimit": self.page_size * 2, - "desiredKeys": [ - "resJPEGFullWidth", - "resJPEGFullHeight", - "resJPEGFullFileType", - "resJPEGFullFingerprint", - "resJPEGFullRes", - "resJPEGLargeWidth", - "resJPEGLargeHeight", - "resJPEGLargeFileType", - "resJPEGLargeFingerprint", - "resJPEGLargeRes", - "resJPEGMedWidth", - "resJPEGMedHeight", - "resJPEGMedFileType", - "resJPEGMedFingerprint", - "resJPEGMedRes", - "resJPEGThumbWidth", - "resJPEGThumbHeight", - "resJPEGThumbFileType", - "resJPEGThumbFingerprint", - "resJPEGThumbRes", - "resVidFullWidth", - "resVidFullHeight", - "resVidFullFileType", - "resVidFullFingerprint", - "resVidFullRes", - "resVidMedWidth", - "resVidMedHeight", - "resVidMedFileType", - "resVidMedFingerprint", - "resVidMedRes", - "resVidSmallWidth", - "resVidSmallHeight", - "resVidSmallFileType", - "resVidSmallFingerprint", - "resVidSmallRes", - "resSidecarWidth", - "resSidecarHeight", - "resSidecarFileType", - "resSidecarFingerprint", - "resSidecarRes", - "itemType", - "dataClassType", - "filenameEnc", - "originalOrientation", - "resOriginalWidth", - "resOriginalHeight", - "resOriginalFileType", - "resOriginalFingerprint", - "resOriginalRes", - "resOriginalAltWidth", - "resOriginalAltHeight", - "resOriginalAltFileType", - "resOriginalAltFingerprint", - "resOriginalAltRes", - "resOriginalVidComplWidth", - "resOriginalVidComplHeight", - "resOriginalVidComplFileType", - "resOriginalVidComplFingerprint", - "resOriginalVidComplRes", - "isDeleted", - "isExpunged", - "dateExpunged", - "remappedRef", - "recordName", - "recordType", - "recordChangeTag", - "masterRef", - "adjustmentRenderType", - "assetDate", - "addedDate", - "isFavorite", - "isHidden", - "orientation", - "duration", - "assetSubtype", - "assetSubtypeV2", - "assetHDRType", - "burstFlags", - "burstFlagsExt", - "burstId", - "captionEnc", - "locationEnc", - "locationV2Enc", - "locationLatitude", - "locationLongitude", - "adjustmentType", - "timeZoneOffset", - "vidComplDurValue", - "vidComplDurScale", - "vidComplDispValue", - "vidComplDispScale", - "vidComplVisibilityState", - "customRenderedValue", - "containerId", - "itemId", - "position", - "isKeyAsset", - ], - "zoneID": {"zoneName": "PrimarySync"}, - } - - if query_filter: - query["query"]["filterBy"].extend(query_filter) - - return query - - def __str__(self): - return self.title - - def __repr__(self): - return f"<{type(self).__name__}: '{self}'>" - - -class PhotoAsset: - """A photo.""" - - def __init__(self, service, master_record, asset_record): - self._service = service - self._master_record = master_record - self._asset_record = asset_record - - self._versions = None - - PHOTO_VERSION_LOOKUP = { - "original": "resOriginal", - "medium": "resJPEGMed", - "thumb": "resJPEGThumb", - } - - VIDEO_VERSION_LOOKUP = { - "original": "resOriginal", - "medium": "resVidMed", - "thumb": "resVidSmall", - } - - @property - def id(self): - """Gets the photo id.""" - return self._master_record["recordName"] - - @property - def filename(self): - """Gets the photo file name.""" - return base64.b64decode( - self._master_record["fields"]["filenameEnc"]["value"] - ).decode("utf-8") - - @property - def size(self): - """Gets the photo size.""" - return self._master_record["fields"]["resOriginalRes"]["value"]["size"] - - @property - def created(self): - """Gets the photo created date.""" - return self.asset_date - - @property - def asset_date(self): - """Gets the photo asset date.""" - try: - return datetime.utcfromtimestamp( - self._asset_record["fields"]["assetDate"]["value"] / 1000.0 - ).replace(tzinfo=timezone.utc) - except KeyError: - return datetime.utcfromtimestamp(0).replace(tzinfo=timezone.utc) - - @property - def added_date(self): - """Gets the photo added date.""" - return datetime.utcfromtimestamp( - self._asset_record["fields"]["addedDate"]["value"] / 1000.0 - ).replace(tzinfo=timezone.utc) - - @property - def dimensions(self): - """Gets the photo dimensions.""" - return ( - self._master_record["fields"]["resOriginalWidth"]["value"], - self._master_record["fields"]["resOriginalHeight"]["value"], - ) - - @property - def versions(self): - """Gets the photo versions.""" - if not self._versions: - self._versions = {} - if "resVidSmallRes" in self._master_record["fields"]: - typed_version_lookup = self.VIDEO_VERSION_LOOKUP - else: - typed_version_lookup = self.PHOTO_VERSION_LOOKUP - - for key, prefix in typed_version_lookup.items(): - if "%sRes" % prefix in self._master_record["fields"]: - fields = self._master_record["fields"] - version = {"filename": self.filename} - - width_entry = fields.get("%sWidth" % prefix) - if width_entry: - version["width"] = width_entry["value"] - else: - version["width"] = None - - height_entry = fields.get("%sHeight" % prefix) - if height_entry: - version["height"] = height_entry["value"] - else: - version["height"] = None - - size_entry = fields.get("%sRes" % prefix) - if size_entry: - version["size"] = size_entry["value"]["size"] - version["url"] = size_entry["value"]["downloadURL"] - else: - version["size"] = None - version["url"] = None - - type_entry = fields.get("%sFileType" % prefix) - if type_entry: - version["type"] = type_entry["value"] - else: - version["type"] = None - - self._versions[key] = version - - return self._versions - - def download(self, version="original", **kwargs): - """Returns the photo file.""" - if version not in self.versions: - return None - - return self._service.session.get( - self.versions[version]["url"], stream=True, **kwargs - ) - - def delete(self): - """Deletes the photo.""" - json_data = ( - '{"query":{"recordType":"CheckIndexingState"},' - '"zoneID":{"zoneName":"PrimarySync"}}' - ) - - json_data = ( - '{"operations":[{' - '"operationType":"update",' - '"record":{' - '"recordName":"%s",' - '"recordType":"%s",' - '"recordChangeTag":"%s",' - '"fields":{"isDeleted":{"value":1}' - "}}}]," - '"zoneID":{' - '"zoneName":"PrimarySync"' - '},"atomic":true}' - % ( - self._asset_record["recordName"], - self._asset_record["recordType"], - self._master_record["recordChangeTag"], - ) - ) - - endpoint = self._service.service_endpoint - params = urlencode(self._service.params) - url = f"{endpoint}/records/modify?{params}" - - return self._service.session.post( - url, data=json_data, headers={"Content-type": "text/plain"} - ) - - def __repr__(self): - return f"<{type(self).__name__}: id={self.id}>" +"""Photo service.""" +import json +import base64 +from urllib.parse import urlencode + +from datetime import datetime, timezone +from pyicloud.exceptions import PyiCloudServiceNotActivatedException + + +class PhotosService: + """The 'Photos' iCloud service.""" + + SMART_FOLDERS = { + "All Photos": { + "obj_type": "CPLAssetByAddedDate", + "list_type": "CPLAssetAndMasterByAddedDate", + "direction": "ASCENDING", + "query_filter": None, + }, + "Time-lapse": { + "obj_type": "CPLAssetInSmartAlbumByAssetDate:Timelapse", + "list_type": "CPLAssetAndMasterInSmartAlbumByAssetDate", + "direction": "ASCENDING", + "query_filter": [ + { + "fieldName": "smartAlbum", + "comparator": "EQUALS", + "fieldValue": {"type": "STRING", "value": "TIMELAPSE"}, + } + ], + }, + "Videos": { + "obj_type": "CPLAssetInSmartAlbumByAssetDate:Video", + "list_type": "CPLAssetAndMasterInSmartAlbumByAssetDate", + "direction": "ASCENDING", + "query_filter": [ + { + "fieldName": "smartAlbum", + "comparator": "EQUALS", + "fieldValue": {"type": "STRING", "value": "VIDEO"}, + } + ], + }, + "Slo-mo": { + "obj_type": "CPLAssetInSmartAlbumByAssetDate:Slomo", + "list_type": "CPLAssetAndMasterInSmartAlbumByAssetDate", + "direction": "ASCENDING", + "query_filter": [ + { + "fieldName": "smartAlbum", + "comparator": "EQUALS", + "fieldValue": {"type": "STRING", "value": "SLOMO"}, + } + ], + }, + "Bursts": { + "obj_type": "CPLAssetBurstStackAssetByAssetDate", + "list_type": "CPLBurstStackAssetAndMasterByAssetDate", + "direction": "ASCENDING", + "query_filter": None, + }, + "Favorites": { + "obj_type": "CPLAssetInSmartAlbumByAssetDate:Favorite", + "list_type": "CPLAssetAndMasterInSmartAlbumByAssetDate", + "direction": "ASCENDING", + "query_filter": [ + { + "fieldName": "smartAlbum", + "comparator": "EQUALS", + "fieldValue": {"type": "STRING", "value": "FAVORITE"}, + } + ], + }, + "Panoramas": { + "obj_type": "CPLAssetInSmartAlbumByAssetDate:Panorama", + "list_type": "CPLAssetAndMasterInSmartAlbumByAssetDate", + "direction": "ASCENDING", + "query_filter": [ + { + "fieldName": "smartAlbum", + "comparator": "EQUALS", + "fieldValue": {"type": "STRING", "value": "PANORAMA"}, + } + ], + }, + "Screenshots": { + "obj_type": "CPLAssetInSmartAlbumByAssetDate:Screenshot", + "list_type": "CPLAssetAndMasterInSmartAlbumByAssetDate", + "direction": "ASCENDING", + "query_filter": [ + { + "fieldName": "smartAlbum", + "comparator": "EQUALS", + "fieldValue": {"type": "STRING", "value": "SCREENSHOT"}, + } + ], + }, + "Live": { + "obj_type": "CPLAssetInSmartAlbumByAssetDate:Live", + "list_type": "CPLAssetAndMasterInSmartAlbumByAssetDate", + "direction": "ASCENDING", + "query_filter": [ + { + "fieldName": "smartAlbum", + "comparator": "EQUALS", + "fieldValue": {"type": "STRING", "value": "LIVE"}, + } + ], + }, + "Recently Deleted": { + "obj_type": "CPLAssetDeletedByExpungedDate", + "list_type": "CPLAssetAndMasterDeletedByExpungedDate", + "direction": "ASCENDING", + "query_filter": None, + }, + "Hidden": { + "obj_type": "CPLAssetHiddenByAssetDate", + "list_type": "CPLAssetAndMasterHiddenByAssetDate", + "direction": "ASCENDING", + "query_filter": None, + }, + } + + def __init__(self, service_root, session, params): + self.session = session + self.params = dict(params) + self._service_root = service_root + self.service_endpoint = ( + "%s/database/1/com.apple.photos.cloud/production/private" + % self._service_root + ) + + self._albums = None + + self.params.update({"remapEnums": True, "getCurrentSyncToken": True}) + + url = f"{self.service_endpoint}/records/query?{urlencode(self.params)}" + json_data = ( + '{"query":{"recordType":"CheckIndexingState"},' + '"zoneID":{"zoneName":"PrimarySync"}}' + ) + request = self.session.post( + url, data=json_data, headers={"Content-type": "text/plain"} + ) + response = request.json() + indexing_state = response["records"][0]["fields"]["state"]["value"] + if indexing_state != "FINISHED": + raise PyiCloudServiceNotActivatedException( + "iCloud Photo Library not finished indexing. " + "Please try again in a few minutes." + ) + + # TODO: Does syncToken ever change? # pylint: disable=fixme + # self.params.update({ + # 'syncToken': response['syncToken'], + # 'clientInstanceId': self.params.pop('clientId') + # }) + + self._photo_assets = {} + + @property + def albums(self): + """Returns photo albums.""" + if not self._albums: + self._albums = { + name: PhotoAlbum(self, name, **props) + for (name, props) in self.SMART_FOLDERS.items() + } + + for folder in self._fetch_folders(): + + # Skiping albums having null name, that can happen sometime + if "albumNameEnc" not in folder["fields"]: + continue + + # TODO: Handle subfolders # pylint: disable=fixme + if folder["recordName"] == "----Root-Folder----" or ( + folder["fields"].get("isDeleted") + and folder["fields"]["isDeleted"]["value"] + ): + continue + + folder_id = folder["recordName"] + folder_obj_type = ( + "CPLContainerRelationNotDeletedByAssetDate:%s" % folder_id + ) + folder_name = base64.b64decode( + folder["fields"]["albumNameEnc"]["value"] + ).decode("utf-8") + query_filter = [ + { + "fieldName": "parentId", + "comparator": "EQUALS", + "fieldValue": {"type": "STRING", "value": folder_id}, + } + ] + + album = PhotoAlbum( + self, + folder_name, + "CPLContainerRelationLiveByAssetDate", + folder_obj_type, + "ASCENDING", + query_filter, + ) + self._albums[folder_name] = album + + return self._albums + + def _fetch_folders(self): + url = f"{self.service_endpoint}/records/query?{urlencode(self.params)}" + json_data = ( + '{"query":{"recordType":"CPLAlbumByPositionLive"},' + '"zoneID":{"zoneName":"PrimarySync"}}' + ) + + request = self.session.post( + url, data=json_data, headers={"Content-type": "text/plain"} + ) + response = request.json() + + return response["records"] + + @property + def all(self): + """Returns all photos.""" + return self.albums["All Photos"] + + +class PhotoAlbum: + """A photo album.""" + + def __init__( + self, + service, + name, + list_type, + obj_type, + direction, + query_filter=None, + page_size=100, + ): + self.name = name + self.service = service + self.list_type = list_type + self.obj_type = obj_type + self.direction = direction + self.query_filter = query_filter + self.page_size = page_size + + self._len = None + + @property + def title(self): + """Gets the album name.""" + return self.name + + def __iter__(self): + return self.photos + + def __len__(self): + if self._len is None: + url = "{}/internal/records/query/batch?{}".format( + self.service.service_endpoint, + urlencode(self.service.params), + ) + request = self.service.session.post( + url, + data=json.dumps( + { + "batch": [ + { + "resultsLimit": 1, + "query": { + "filterBy": { + "fieldName": "indexCountID", + "fieldValue": { + "type": "STRING_LIST", + "value": [self.obj_type], + }, + "comparator": "IN", + }, + "recordType": "HyperionIndexCountLookup", + }, + "zoneWide": True, + "zoneID": {"zoneName": "PrimarySync"}, + } + ] + } + ), + headers={"Content-type": "text/plain"}, + ) + response = request.json() + + self._len = response["batch"][0]["records"][0]["fields"]["itemCount"][ + "value" + ] + + return self._len + + @property + def photos(self): + """Returns the album photos.""" + if self.direction == "DESCENDING": + offset = len(self) - 1 + else: + offset = 0 + + while True: + url = ("%s/records/query?" % self.service.service_endpoint) + urlencode( + self.service.params + ) + request = self.service.session.post( + url, + data=json.dumps( + self._list_query_gen( + offset, self.list_type, self.direction, self.query_filter + ) + ), + headers={"Content-type": "text/plain"}, + ) + response = request.json() + + asset_records = {} + master_records = [] + for rec in response["records"]: + if rec["recordType"] == "CPLAsset": + master_id = rec["fields"]["masterRef"]["value"]["recordName"] + asset_records[master_id] = rec + elif rec["recordType"] == "CPLMaster": + master_records.append(rec) + + master_records_len = len(master_records) + if master_records_len: + if self.direction == "DESCENDING": + offset = offset - master_records_len + else: + offset = offset + master_records_len + + for master_record in master_records: + record_name = master_record["recordName"] + yield PhotoAsset( + self.service, master_record, asset_records[record_name] + ) + else: + break + + def _list_query_gen(self, offset, list_type, direction, query_filter=None): + query = { + "query": { + "filterBy": [ + { + "fieldName": "startRank", + "fieldValue": {"type": "INT64", "value": offset}, + "comparator": "EQUALS", + }, + { + "fieldName": "direction", + "fieldValue": {"type": "STRING", "value": direction}, + "comparator": "EQUALS", + }, + ], + "recordType": list_type, + }, + "resultsLimit": self.page_size * 2, + "desiredKeys": [ + "resJPEGFullWidth", + "resJPEGFullHeight", + "resJPEGFullFileType", + "resJPEGFullFingerprint", + "resJPEGFullRes", + "resJPEGLargeWidth", + "resJPEGLargeHeight", + "resJPEGLargeFileType", + "resJPEGLargeFingerprint", + "resJPEGLargeRes", + "resJPEGMedWidth", + "resJPEGMedHeight", + "resJPEGMedFileType", + "resJPEGMedFingerprint", + "resJPEGMedRes", + "resJPEGThumbWidth", + "resJPEGThumbHeight", + "resJPEGThumbFileType", + "resJPEGThumbFingerprint", + "resJPEGThumbRes", + "resVidFullWidth", + "resVidFullHeight", + "resVidFullFileType", + "resVidFullFingerprint", + "resVidFullRes", + "resVidMedWidth", + "resVidMedHeight", + "resVidMedFileType", + "resVidMedFingerprint", + "resVidMedRes", + "resVidSmallWidth", + "resVidSmallHeight", + "resVidSmallFileType", + "resVidSmallFingerprint", + "resVidSmallRes", + "resSidecarWidth", + "resSidecarHeight", + "resSidecarFileType", + "resSidecarFingerprint", + "resSidecarRes", + "itemType", + "dataClassType", + "filenameEnc", + "originalOrientation", + "resOriginalWidth", + "resOriginalHeight", + "resOriginalFileType", + "resOriginalFingerprint", + "resOriginalRes", + "resOriginalAltWidth", + "resOriginalAltHeight", + "resOriginalAltFileType", + "resOriginalAltFingerprint", + "resOriginalAltRes", + "resOriginalVidComplWidth", + "resOriginalVidComplHeight", + "resOriginalVidComplFileType", + "resOriginalVidComplFingerprint", + "resOriginalVidComplRes", + "isDeleted", + "isExpunged", + "dateExpunged", + "remappedRef", + "recordName", + "recordType", + "recordChangeTag", + "masterRef", + "adjustmentRenderType", + "assetDate", + "addedDate", + "isFavorite", + "isHidden", + "orientation", + "duration", + "assetSubtype", + "assetSubtypeV2", + "assetHDRType", + "burstFlags", + "burstFlagsExt", + "burstId", + "captionEnc", + "locationEnc", + "locationV2Enc", + "locationLatitude", + "locationLongitude", + "adjustmentType", + "timeZoneOffset", + "vidComplDurValue", + "vidComplDurScale", + "vidComplDispValue", + "vidComplDispScale", + "vidComplVisibilityState", + "customRenderedValue", + "containerId", + "itemId", + "position", + "isKeyAsset", + ], + "zoneID": {"zoneName": "PrimarySync"}, + } + + if query_filter: + query["query"]["filterBy"].extend(query_filter) + + return query + + def __str__(self): + return self.title + + def __repr__(self): + return f"<{type(self).__name__}: '{self}'>" + + +class PhotoAsset: + """A photo.""" + + def __init__(self, service, master_record, asset_record): + self._service = service + self._master_record = master_record + self._asset_record = asset_record + + self._versions = None + + PHOTO_VERSION_LOOKUP = { + "original": "resOriginal", + "medium": "resJPEGMed", + "thumb": "resJPEGThumb", + } + + VIDEO_VERSION_LOOKUP = { + "original": "resOriginal", + "medium": "resVidMed", + "thumb": "resVidSmall", + } + + @property + def id(self): + """Gets the photo id.""" + return self._master_record["recordName"] + + @property + def filename(self): + """Gets the photo file name.""" + return base64.b64decode( + self._master_record["fields"]["filenameEnc"]["value"] + ).decode("utf-8") + + @property + def size(self): + """Gets the photo size.""" + return self._master_record["fields"]["resOriginalRes"]["value"]["size"] + + @property + def created(self): + """Gets the photo created date.""" + return self.asset_date + + @property + def asset_date(self): + """Gets the photo asset date.""" + try: + return datetime.utcfromtimestamp( + self._asset_record["fields"]["assetDate"]["value"] / 1000.0 + ).replace(tzinfo=timezone.utc) + except KeyError: + return datetime.utcfromtimestamp(0).replace(tzinfo=timezone.utc) + + @property + def added_date(self): + """Gets the photo added date.""" + return datetime.utcfromtimestamp( + self._asset_record["fields"]["addedDate"]["value"] / 1000.0 + ).replace(tzinfo=timezone.utc) + + @property + def dimensions(self): + """Gets the photo dimensions.""" + return ( + self._master_record["fields"]["resOriginalWidth"]["value"], + self._master_record["fields"]["resOriginalHeight"]["value"], + ) + + @property + def versions(self): + """Gets the photo versions.""" + if not self._versions: + self._versions = {} + if "resVidSmallRes" in self._master_record["fields"]: + typed_version_lookup = self.VIDEO_VERSION_LOOKUP + else: + typed_version_lookup = self.PHOTO_VERSION_LOOKUP + + for key, prefix in typed_version_lookup.items(): + if "%sRes" % prefix in self._master_record["fields"]: + fields = self._master_record["fields"] + version = {"filename": self.filename} + + width_entry = fields.get("%sWidth" % prefix) + if width_entry: + version["width"] = width_entry["value"] + else: + version["width"] = None + + height_entry = fields.get("%sHeight" % prefix) + if height_entry: + version["height"] = height_entry["value"] + else: + version["height"] = None + + size_entry = fields.get("%sRes" % prefix) + if size_entry: + version["size"] = size_entry["value"]["size"] + version["url"] = size_entry["value"]["downloadURL"] + else: + version["size"] = None + version["url"] = None + + type_entry = fields.get("%sFileType" % prefix) + if type_entry: + version["type"] = type_entry["value"] + else: + version["type"] = None + + self._versions[key] = version + + return self._versions + + def download(self, version="original", **kwargs): + """Returns the photo file.""" + if version not in self.versions: + return None + + return self._service.session.get( + self.versions[version]["url"], stream=True, **kwargs + ) + + def add_to_album(self, album: PhotoAlbum): + """Moves photo to a new album""" + albumId = album.obj_type.split(':')[1] + recordName = f"{self._asset_record["recordName"]}-IN-{albumId}" + json_data = ( + '{"operations":[{' + '"operationType":"create",' + '"record":{' + '"recordName":"%s",' + '"recordType":"CPLContainerRelation",' + '"fields":{' + '"itemId":{"value":"%s"},' + '"containerId":{"value":"%s"}}}}],' + '"zoneID":{' + '"zoneName":"%s",' + '"ownerRecordName":"%s",' + '"zoneType":"%s"},' + '"atomic":true}' + % ( + recordName, + self._asset_record["recordName"], + albumId, + self._asset_record['zoneID']['zoneName'], + self._asset_record['zoneID']['ownerRecordName'], + self._asset_record['zoneID']['zoneType'] + ) + ) + endpoint = self._service.service_endpoint + params = urlencode(self._service.params) + url = f"{endpoint}/records/modify?{params}" + + return self._service.session.post( + url, data=json_data, headers={"Content-type": "text/plain"} + ) + + def delete(self): + """Deletes the photo.""" + json_data = ( + '{"query":{"recordType":"CheckIndexingState"},' + '"zoneID":{"zoneName":"PrimarySync"}}' + ) + + json_data = ( + '{"operations":[{' + '"operationType":"update",' + '"record":{' + '"recordName":"%s",' + '"recordType":"%s",' + '"recordChangeTag":"%s",' + '"fields":{"isDeleted":{"value":1}' + "}}}]," + '"zoneID":{' + '"zoneName":"PrimarySync"' + '},"atomic":true}' + % ( + self._asset_record["recordName"], + self._asset_record["recordType"], + self._master_record["recordChangeTag"], + ) + ) + + endpoint = self._service.service_endpoint + params = urlencode(self._service.params) + url = f"{endpoint}/records/modify?{params}" + + return self._service.session.post( + url, data=json_data, headers={"Content-type": "text/plain"} + ) + + def __repr__(self): + return f"<{type(self).__name__}: id={self.id}>" From 731c94ec4ef9688f4c2510e785586902c6b3b21b Mon Sep 17 00:00:00 2001 From: Eugene Polonsky Date: Tue, 31 Dec 2024 10:04:05 -0800 Subject: [PATCH 3/3] Add the given photo to an album --- pyicloud/services/photos.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/pyicloud/services/photos.py b/pyicloud/services/photos.py index 06b3dd32..03e74ef7 100644 --- a/pyicloud/services/photos.py +++ b/pyicloud/services/photos.py @@ -600,6 +600,41 @@ def download(self, version="original", **kwargs): self.versions[version]["url"], stream=True, **kwargs ) + def add_to_album(self, album: PhotoAlbum): + """Moves photo to a new album""" + albumId = album.obj_type.split(':')[1] + recordName = f"{self._asset_record["recordName"]}-IN-{albumId}" + json_data = ( + '{"operations":[{' + '"operationType":"create",' + '"record":{' + '"recordName":"%s",' + '"recordType":"CPLContainerRelation",' + '"fields":{' + '"itemId":{"value":"%s"},' + '"containerId":{"value":"%s"}}}}],' + '"zoneID":{' + '"zoneName":"%s",' + '"ownerRecordName":"%s",' + '"zoneType":"%s"},' + '"atomic":true}' + % ( + recordName, + self._asset_record["recordName"], + albumId, + self._asset_record['zoneID']['zoneName'], + self._asset_record['zoneID']['ownerRecordName'], + self._asset_record['zoneID']['zoneType'] + ) + ) + endpoint = self._service.service_endpoint + params = urlencode(self._service.params) + url = f"{endpoint}/records/modify?{params}" + + return self._service.session.post( + url, data=json_data, headers={"Content-type": "text/plain"} + ) + def delete(self): """Deletes the photo.""" json_data = (