From ffcf6f85d052e0eca61451352a1645b5d045f07b Mon Sep 17 00:00:00 2001 From: Dmytro Kutnyakhov Date: Tue, 20 Feb 2024 14:13:57 +0100 Subject: [PATCH 1/6] fixed get metadata function using scicat token after scicat was migrated to new beckend --- sed/loader/flash/metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sed/loader/flash/metadata.py b/sed/loader/flash/metadata.py index cbbfc456..919357a5 100644 --- a/sed/loader/flash/metadata.py +++ b/sed/loader/flash/metadata.py @@ -36,6 +36,7 @@ def __init__(self, metadata_config: Dict, scicat_token: str = None) -> None: "Content-Type": "application/json", "Accept": "application/json", } + self.token = metadata_config["scicat_token"] def get_metadata( self, @@ -87,7 +88,7 @@ def _get_metadata_per_run(self, pid: str) -> Dict: Exception: If the request to retrieve metadata fails. """ headers2 = dict(self.headers) - headers2["Authorization"] = f"Bearer {self.token}" + headers2["Authorization"] = "Bearer {}".format(self.token) try: # Create the dataset URL using the PID dataset_response = requests.get( From 63c7f357a4ce1433fad8991586a829c63dfe3213 Mon Sep 17 00:00:00 2001 From: Dmytro Kutnyakhov Date: Tue, 5 Mar 2024 14:08:10 +0100 Subject: [PATCH 2/6] Corrected get metadata to work with old and new implementation of scicat url --- sed/loader/flash/metadata.py | 41 +++++++++++++++--------------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/sed/loader/flash/metadata.py b/sed/loader/flash/metadata.py index 919357a5..a1856bad 100644 --- a/sed/loader/flash/metadata.py +++ b/sed/loader/flash/metadata.py @@ -89,39 +89,32 @@ def _get_metadata_per_run(self, pid: str) -> Dict: """ headers2 = dict(self.headers) headers2["Authorization"] = "Bearer {}".format(self.token) + try: - # Create the dataset URL using the PID dataset_response = requests.get( - self._create_dataset_url_by_PID(pid), - params={"access_token": self.token}, + self._create_new_dataset_url(pid), headers=headers2, timeout=10, ) - dataset_response.raise_for_status() # Raise HTTPError if request fails + dataset_response.raise_for_status() + # Check if response is an empty object because wrong url for older implementation + if not dataset_response.content: + dataset_response = requests.get(self._create_old_dataset_url(pid), + headers=headers2, timeout=10) # If the dataset request is successful, return the retrieved metadata # as a JSON object - return dataset_response.json() + return dataset_response.json() except requests.exceptions.RequestException as exception: # If the request fails, raise warning - warnings.warn(f"Failed to retrieve metadata for PID {pid}: {str(exception)}") + print(warnings.warn(f"Failed to retrieve metadata for PID {pid}: {str(exception)}")) return {} # Return an empty dictionary for this run + + def _create_old_dataset_url(self, pid: str) -> str: + return "{burl}/{url}/%2F{npid}".format(burl=self.url, url="Datasets", npid=self._reformat_pid(pid)) - def _create_dataset_url_by_PID(self, pid: str) -> str: # pylint: disable=invalid-name - """ - Creates the dataset URL based on the PID. - - Args: - pid (str): The PID of the run. + def _create_new_dataset_url(self, pid: str) -> str: + return "{burl}/{url}/{npid}".format(burl=self.url, url="Datasets", npid=self._reformat_pid(pid)) - Returns: - str: The dataset URL. - - Raises: - Exception: If the token request fails. - """ - npid = pid.replace( - "/", - "%2F", - ) # Replace slashes in the PID with URL-encoded slashes - url = f"{self.url}/Datasets/{npid}" - return url + def _reformat_pid(self, pid: str) -> str: + """ SciCat adds a pid-prefix + "/" but at DESY prefix = "" """ + return (pid).replace("/", "%2F") From 6a535f0b48befe731c31dba0841285dc656ea770 Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Fri, 8 Mar 2024 06:54:41 +0000 Subject: [PATCH 3/6] rebase and fix linting errors --- sed/loader/flash/metadata.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sed/loader/flash/metadata.py b/sed/loader/flash/metadata.py index a1856bad..5598a9ce 100644 --- a/sed/loader/flash/metadata.py +++ b/sed/loader/flash/metadata.py @@ -89,7 +89,7 @@ def _get_metadata_per_run(self, pid: str) -> Dict: """ headers2 = dict(self.headers) headers2["Authorization"] = "Bearer {}".format(self.token) - + try: dataset_response = requests.get( self._create_new_dataset_url(pid), @@ -103,17 +103,19 @@ def _get_metadata_per_run(self, pid: str) -> Dict: headers=headers2, timeout=10) # If the dataset request is successful, return the retrieved metadata # as a JSON object - return dataset_response.json() + return dataset_response.json() except requests.exceptions.RequestException as exception: # If the request fails, raise warning print(warnings.warn(f"Failed to retrieve metadata for PID {pid}: {str(exception)}")) return {} # Return an empty dictionary for this run - + def _create_old_dataset_url(self, pid: str) -> str: - return "{burl}/{url}/%2F{npid}".format(burl=self.url, url="Datasets", npid=self._reformat_pid(pid)) + return "{burl}/{url}/%2F{npid}".format(burl=self.url, url="Datasets", + npid=self._reformat_pid(pid)) def _create_new_dataset_url(self, pid: str) -> str: - return "{burl}/{url}/{npid}".format(burl=self.url, url="Datasets", npid=self._reformat_pid(pid)) + return "{burl}/{url}/{npid}".format(burl=self.url, url="Datasets", + npid=self._reformat_pid(pid)) def _reformat_pid(self, pid: str) -> str: """ SciCat adds a pid-prefix + "/" but at DESY prefix = "" """ From 9ade1f5e2fdeef127641031cc145fdd856609edc Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Fri, 8 Mar 2024 06:59:28 +0000 Subject: [PATCH 4/6] fix more linting errors --- sed/loader/flash/metadata.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/sed/loader/flash/metadata.py b/sed/loader/flash/metadata.py index 5598a9ce..9f23b59a 100644 --- a/sed/loader/flash/metadata.py +++ b/sed/loader/flash/metadata.py @@ -1,7 +1,8 @@ """ The module provides a MetadataRetriever class for retrieving metadata -from a Scicatinstance based on beamtime and run IDs. +from a Scicat Instance based on beamtime and run IDs. """ + import warnings from typing import Dict from typing import Optional @@ -99,8 +100,9 @@ def _get_metadata_per_run(self, pid: str) -> Dict: dataset_response.raise_for_status() # Check if response is an empty object because wrong url for older implementation if not dataset_response.content: - dataset_response = requests.get(self._create_old_dataset_url(pid), - headers=headers2, timeout=10) + dataset_response = requests.get( + self._create_old_dataset_url(pid), headers=headers2, timeout=10 + ) # If the dataset request is successful, return the retrieved metadata # as a JSON object return dataset_response.json() @@ -110,13 +112,15 @@ def _get_metadata_per_run(self, pid: str) -> Dict: return {} # Return an empty dictionary for this run def _create_old_dataset_url(self, pid: str) -> str: - return "{burl}/{url}/%2F{npid}".format(burl=self.url, url="Datasets", - npid=self._reformat_pid(pid)) + return "{burl}/{url}/%2F{npid}".format( + burl=self.url, url="Datasets", npid=self._reformat_pid(pid) + ) def _create_new_dataset_url(self, pid: str) -> str: - return "{burl}/{url}/{npid}".format(burl=self.url, url="Datasets", - npid=self._reformat_pid(pid)) + return "{burl}/{url}/{npid}".format( + burl=self.url, url="Datasets", npid=self._reformat_pid(pid) + ) def _reformat_pid(self, pid: str) -> str: - """ SciCat adds a pid-prefix + "/" but at DESY prefix = "" """ + """SciCat adds a pid-prefix + "/" but at DESY prefix = "" """ return (pid).replace("/", "%2F") From ce5599711ea9db3cabbf73013bad860ca09475a7 Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Fri, 8 Mar 2024 07:13:18 +0000 Subject: [PATCH 5/6] try to fix test --- tests/loader/flash/test_flash_metadata.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/loader/flash/test_flash_metadata.py b/tests/loader/flash/test_flash_metadata.py index 9136d9e9..93f111b2 100644 --- a/tests/loader/flash/test_flash_metadata.py +++ b/tests/loader/flash/test_flash_metadata.py @@ -51,6 +51,10 @@ def test_create_dataset_url_by_PID(): "scicat_token": "fake_token", } retriever = MetadataRetriever(metadata_config) - url = retriever._create_dataset_url_by_PID("11013410/43878") + # Assuming the dataset follows the new format + pid = "11013410/43878" + url = retriever._create_new_dataset_url(pid) + expected_url = "https://example.com/Datasets/11013410/43878" assert isinstance(url, str) - assert url == "https://example.com/Datasets/11013410%2F43878" + assert url == expected_url + From 640b334558cd75b48dd5f0297d01c2185d911ddc Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Sun, 17 Mar 2024 12:00:49 +0100 Subject: [PATCH 6/6] fix test --- tests/loader/flash/test_flash_metadata.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/loader/flash/test_flash_metadata.py b/tests/loader/flash/test_flash_metadata.py index 93f111b2..eee1e9d8 100644 --- a/tests/loader/flash/test_flash_metadata.py +++ b/tests/loader/flash/test_flash_metadata.py @@ -54,7 +54,6 @@ def test_create_dataset_url_by_PID(): # Assuming the dataset follows the new format pid = "11013410/43878" url = retriever._create_new_dataset_url(pid) - expected_url = "https://example.com/Datasets/11013410/43878" + expected_url = "https://example.com/Datasets/11013410%2F43878" assert isinstance(url, str) assert url == expected_url -