Get metadata from scicat using token (#353)

* fixed get metadata function using scicat token after scicat was migrated to new beckend

* Corrected get metadata to work with old and new implementation of scicat url

* rebase and fix linting errors

* fix more linting errors

* try to fix test

* fix test

---------

Co-authored-by: Dmytro Kutnyakhov <[email protected]>
Co-authored-by: Zain Sohail <[email protected]>

sed/loader/flash/metadata.py

@@ -1,7 +1,8 @@
 """
 The module provides a MetadataRetriever class for retrieving metadata
-from a Scicatinstance based on beamtime and run IDs.
+from a Scicat Instance based on beamtime and run IDs.
 """
+
 import warnings
 from typing import Dict
 from typing import Optional
@@ -36,6 +37,7 @@ def __init__(self, metadata_config: Dict, scicat_token: str = None) -> None:
             "Content-Type": "application/json",
             "Accept": "application/json",
         }
+        self.token = metadata_config["scicat_token"]
 
     def get_metadata(
         self,
@@ -87,40 +89,38 @@ def _get_metadata_per_run(self, pid: str) -> Dict:
             Exception: If the request to retrieve metadata fails.
         """
         headers2 = dict(self.headers)
-        headers2["Authorization"] = f"Bearer {self.token}"
+        headers2["Authorization"] = "Bearer {}".format(self.token)
+
         try:
-            # Create the dataset URL using the PID
             dataset_response = requests.get(
-                self._create_dataset_url_by_PID(pid),
-                params={"access_token": self.token},
+                self._create_new_dataset_url(pid),
                 headers=headers2,
                 timeout=10,
             )
-            dataset_response.raise_for_status()  # Raise HTTPError if request fails
+            dataset_response.raise_for_status()
+            # Check if response is an empty object because wrong url for older implementation
+            if not dataset_response.content:
+                dataset_response = requests.get(
+                    self._create_old_dataset_url(pid), headers=headers2, timeout=10
+                )
             # If the dataset request is successful, return the retrieved metadata
             # as a JSON object
             return dataset_response.json()
         except requests.exceptions.RequestException as exception:
             # If the request fails, raise warning
-            warnings.warn(f"Failed to retrieve metadata for PID {pid}: {str(exception)}")
+            print(warnings.warn(f"Failed to retrieve metadata for PID {pid}: {str(exception)}"))
             return {}  # Return an empty dictionary for this run
 
-    def _create_dataset_url_by_PID(self, pid: str) -> str:  # pylint: disable=invalid-name
-        """
-        Creates the dataset URL based on the PID.
+    def _create_old_dataset_url(self, pid: str) -> str:
+        return "{burl}/{url}/%2F{npid}".format(
+            burl=self.url, url="Datasets", npid=self._reformat_pid(pid)
+        )
 
-        Args:
-            pid (str): The PID of the run.
+    def _create_new_dataset_url(self, pid: str) -> str:
+        return "{burl}/{url}/{npid}".format(
+            burl=self.url, url="Datasets", npid=self._reformat_pid(pid)
+        )
 
-        Returns:
-            str: The dataset URL.
-
-        Raises:
-            Exception: If the token request fails.
-        """
-        npid = pid.replace(
-            "/",
-            "%2F",
-        )  # Replace slashes in the PID with URL-encoded slashes
-        url = f"{self.url}/Datasets/{npid}"
-        return url
+    def _reformat_pid(self, pid: str) -> str:
+        """SciCat adds a pid-prefix + "/"  but at DESY prefix = "" """
+        return (pid).replace("/", "%2F")

tests/loader/flash/test_flash_metadata.py

@@ -51,6 +51,9 @@ def test_create_dataset_url_by_PID():
         "scicat_token": "fake_token",
     }
     retriever = MetadataRetriever(metadata_config)
-    url = retriever._create_dataset_url_by_PID("11013410/43878")
+    # Assuming the dataset follows the new format
+    pid = "11013410/43878"
+    url = retriever._create_new_dataset_url(pid)
+    expected_url = "https://example.com/Datasets/11013410%2F43878"
     assert isinstance(url, str)
-    assert url == "https://example.com/Datasets/11013410%2F43878"
+    assert url == expected_url