Add a backend for rucio

strax/__init__.py

@@ -11,6 +11,7 @@
 
 from .storage.common import *
 from .storage.files import *
+from .storage.rucio import *
 from .storage.mongo import *
 from .storage.s3 import *
 from .storage.zipfiles import *

strax/processing/pulse_processing.py

@@ -260,7 +260,7 @@ def _find_hits(records, min_amplitude, min_height_over_noise,
         for i in range(n_samples):
             # We can't use enumerate over r['data'],
             # numba gives errors if we do.
-            # TODO: file issue?
+            # maybe file an issue?
             x = r['data'][i]
 
             satisfy_threshold = x >= threshold

strax/storage/rucio.py

@@ -0,0 +1,48 @@
+import json
+import hashlib
+import os.path as osp
+
+import strax
+from strax.storage.files import dirname_to_prefix
+
+export, __all__ = strax.exporter()
+
+
+@export
+class rucio(strax.StorageBackend):
+    """Get data from a rucio directory
+    """
+    def __init__(self, root_dir, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.root_dir = root_dir
+
+    def get_metadata(self, dirname:str, **kwargs):
+        prefix = dirname_to_prefix(dirname)
+        metadata_json = f'{prefix}-metadata.json'
+        fn = rucio_path(self.root_dir, metadata_json, dirname)
+        folder = osp.join('/', *fn.split('/')[:-1])
+        if not osp.exists(folder):
+            raise strax.DataNotAvailable(f"No folder for matadata at {fn}")
+        if not osp.exists(fn):
+            raise strax.DataCorrupted(f"Folder exists but no matadata at {fn}")
+
+        with open(fn, mode='r') as f:
+            return json.loads(f.read())
+
+    def _read_chunk(self, dirname, chunk_info, dtype, compressor):
+        fn = rucio_path(self.root_dir, chunk_info['filename'], dirname)
+        return strax.load_file(fn, dtype=dtype, compressor=compressor)
+
+    def _saver(self, dirname, metadata):
+        raise NotImplementedError(
+            "Cannot save directly into rucio, upload with admix instead")
+
+
+def rucio_path(root_dir, filename, dirname):
+    """Convert target to path according to rucio convention"""
+    scope = "xnt_"+dirname.split('-')[0]
+    rucio_did = "{0}:{1}".format(scope, filename)
+    rucio_md5 = hashlib.md5(rucio_did.encode('utf-8')).hexdigest()
+    t1 = rucio_md5[0:2]
+    t2 = rucio_md5[2:4]
+    return osp.join(root_dir, scope, t1, t2, filename)

strax/utils.py

@@ -205,7 +205,6 @@ def profile_threaded(filename):
         monitoring_gil = True
     except (RuntimeError, ImportError):
         monitoring_gil = False
-        pass
 
     yappi.start()
     yield