Updates according to DIALS release v1.0.0

.editorconfig

@@ -2,12 +2,19 @@
 # top-most EditorConfig file
 root = true
 
-# Config for all files
 [*]
-insert_final_newline = true
-
-# Config for Python (uncomment what you need)
-[*.py]
 indent_style = space
 indent_size = 4
+charset = utf-8
 trim_trailing_whitespace = true
+insert_final_newline = true
+end_of_line = lf
+
+[*.md]
+trim_trailing_whitespace = false
+
+[Makefile]
+indent_style = tab
+
+[*.{yml,yaml,json}]
+indent_size = 2

.gitignore

@@ -43,6 +43,7 @@ htmlcov/
 .coverage
 .coverage.*
 .cache
+.cache-dev
 nosetests.xml
 coverage.xml
 *.cover

.pre-commit-config.yaml

@@ -1,36 +1,25 @@
-# See https://pre-commit.com for more information
-# See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0
     hooks:
-    - id: trailing-whitespace
-    - id: end-of-file-fixer
-    - id: check-yaml
-    - id: check-added-large-files
-  - repo: https://github.com/psf/black
-    rev: 24.1.1
+      - id: check-added-large-files
+        args: ["--maxkb=500"]
+        exclude: >
+            (?x)^(
+                package-lock\.json
+                poetry\.lock
+            )$
+      - id: fix-byte-order-marker
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: debug-statements
+      - id: detect-private-key
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.2
     hooks:
-    - id: black
-      args: [
-        --line-length=120
-      ]
-  - repo: https://github.com/PyCQA/flake8
-    rev: 7.0.0
-    hooks:
-    - id: flake8
-      additional_dependencies: [flake8-bugbear]
-      args: [
-        --max-line-length=120,
-        --max-complexity=10
-      ]
-  - repo: https://github.com/timothycrosley/isort
-    rev: 5.13.2
-    hooks:
-    - id: isort
-      args: [
-        --atomic,
-        --profile=black,
-        --line-length=120,
-        --skip-gitignore
-      ]
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format

README.md

@@ -53,18 +53,18 @@ from cmsdials import Dials
 from cmsdials.filters import LumisectionHistogram1DFilters
 
 creds = Credentials.from_creds_file()
-dials = Dials(creds, nthreads=2)
+dials = Dials(creds)
 
 # Getting h1d data
-data = dials.h1d.list_all(LumisectionHistogram1DFilters(title="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2"), max_pages=5)
+data = dials.h1d.list_all(LumisectionHistogram1DFilters(me="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2"), max_pages=5)
 ```
 
 ### Workspace
 
 Users are automatically routed to a workspace based on e-groups, but it is possible to overwrite this configuration and inspect data from others workspaces:
 
 ```python
-dials = Dials(creds, workspace="jetmet", nthreads=2)
+dials = Dials(creds, workspace="jetmet")
 ```
 
 ## Available endpoints
@@ -74,11 +74,11 @@ This package interacts with DIALS api endpoints using underlying classes in `Dia
 ### Retrieving a specific object using `get`
 
 ```python
-dials.file_index.get(id=1)
+dials.file_index.get(id=3386119397)
 dials.h1d.get(id=1)
 dials.h2d.get(id=1)
-dials.lumi.get(id=1)
-dials.run.get(id=1)
+dials.lumi.get(dataset_id=14677060, run_number=367094, ls_number=1)
+dials.run.get(dataset_id=14677060, run_number=367094)
 ```
 
 ### Retrieving a list of objects per page using `list`
@@ -95,6 +95,8 @@ dials.run.list()
 
 ### Retrieving all available pages of a list of objects using `list_all`
 
+Note: Keep in mind that running `list_all` without any filter can take too much time, since you will be retrieving all rows in the database.
+
 ```python
 dials.file_index.list_all()
 dials.h1d.list_all()
@@ -122,16 +124,15 @@ from cmsdials.filters import (
     RunFilters
 )
 
+dials.file_index.list(FileIndexFilters(dataset__regex="2024B"))
 
-dials.file_index.list(FileIndexFilters(page="10"))
-
-dials.h1d.list(LumisectionHistogram1DFilters(title="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2", page="15"))
+dials.h1d.list(LumisectionHistogram1DFilters(me="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2"))
 
-dials.h2d.list_all(LumisectionHistogram2DFilters(title_contains="EEOT digi occupancy EE +", min_entries=100, min_run_number=360392, max_run_number=365000), max_pages=5)
+dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls_number=78, entries__gte=100), max_pages=5)
 
 dials.lumi.list_all(LumisectionFilters(run_number=360392), max_pages=5)
 
-dials.run.list_all(RunFilters(min_run_number=360392, max_run_number=365000), max_pages=5)
+dials.run.list_all(RunFilters(run_number__gte=360392, run_number__lte=365000), max_pages=5)
 ```
 
 ### Dials MEs
@@ -157,7 +158,7 @@ DEV_CACHE_DIR = ".cache-dev"
 
 auth = AuthClient(base_url=DEV_URL)
 creds = Credentials.from_creds_file(cache_dir=DEV_CACHE_DIR, client=auth)  # Make sure to specify the auth client with overwritten values, using another cache_dir is recommended
-dials = Dials(creds, nthreads=2, base_url=DEV_URL)
+dials = Dials(creds, base_url=DEV_URL)
 
-dials.h2d.list_all(LumisectionHistogram2DFilters(title_contains="EEOT digi occupancy EE +", min_entries=100, min_run_number=360392, max_run_number=365000), max_pages=5)
+dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="EEOT digi occupancy EE +", entries__gte=100, run_number__gte=360392, run_number__lte=365000), max_pages=5)
 ```

cmsdials/clients/file_index/models.py

@@ -7,11 +7,10 @@
 
 
 class FileIndex(BaseModel):
+    dataset_id: int
+    dataset: str = Field(..., max_length=255)
     file_id: int
     file_size: int
-    era: str = Field(..., max_length=5)
-    campaign: str = Field(..., max_length=15)
-    primary_dataset: str = Field(..., max_length=50)
     creation_date: datetime
     last_modification_date: datetime
     logical_file_name: str
@@ -20,17 +19,17 @@ class FileIndex(BaseModel):
 
 
 class PaginatedFileIndexList(BaseModel):
-    count: int
     next: Optional[AnyUrl]
     previous: Optional[AnyUrl]
     results: List[FileIndex]
 
 
 class FileIndexFilters(OBaseModel):
-    page: Optional[str] = None
-    min_size: Optional[int] = None
-    era: Optional[str] = None
-    campaign: Optional[str] = None
-    primary_dataset: Optional[str] = None
+    next_token: Optional[str] = None
+    dataset_id: Optional[int] = None
     logical_file_name: Optional[str] = None
+    logical_file_name__regex: Optional[str] = None
     status: Optional[str] = None
+    min_size: Optional[int] = None
+    dataset: Optional[str] = None
+    dataset__regex: Optional[str] = None

cmsdials/clients/h1d/client.py

@@ -6,4 +6,4 @@ class LumisectionHistogram1DClient(BaseAuthorizedAPIClient):
     data_model = LumisectionHistogram1D
     pagination_model = PaginatedLumisectionHistogram1DList
     filter_class = LumisectionHistogram1DFilters
-    lookup_url = "lumisection-h1d/"
+    lookup_url = "th1/"

cmsdials/clients/h1d/models.py

@@ -7,10 +7,14 @@
 
 class LumisectionHistogram1D(BaseModel):
     hist_id: int
+    dataset: str = Field(..., max_length=255)
+    me: str = Field(..., max_length=255)
+    dataset_id: int
     file_id: int
     run_number: int
+    ls_number: int
+    me_id: int
     ls_id: int
-    title: str = Field(..., max_length=220)
     x_min: float
     x_max: float
     x_bin: float
@@ -19,25 +23,27 @@ class LumisectionHistogram1D(BaseModel):
 
 
 class PaginatedLumisectionHistogram1DList(BaseModel):
-    count: int
     next: Optional[AnyUrl]
     previous: Optional[AnyUrl]
     results: List[LumisectionHistogram1D]
 
 
 class LumisectionHistogram1DFilters(OBaseModel):
-    page: Optional[str] = None
+    next_token: Optional[str] = None
+    dataset_id: Optional[int] = None
+    file_id: Optional[int] = None
     run_number: Optional[int] = None
+    run_number__lte: Optional[int] = None
+    run_number__gte: Optional[int] = None
     ls_number: Optional[int] = None
+    ls_numbet__lte: Optional[int] = None
+    ls_number__gte: Optional[int] = None
+    me_id: Optional[int] = None
     ls_id: Optional[int] = None
-    title: Optional[str] = None
-    min_run_number: Optional[int] = None
-    max_run_number: Optional[int] = None
-    min_ls_number: Optional[int] = None
-    max_ls_number: Optional[int] = None
-    title_contains: Optional[str] = None
-    min_entries: Optional[int] = None
-    era: Optional[str] = None
-    campaign: Optional[str] = None
-    primary_dataset: Optional[str] = None
-    file_id: Optional[int] = None
+    entries__gte: Optional[int] = None
+    dataset: Optional[str] = None
+    dataset__regex: Optional[str] = None
+    logical_file_name: Optional[str] = None
+    logical_file_name__regex: Optional[str] = None
+    me: Optional[str] = None
+    me__regex: Optional[str] = None

cmsdials/clients/h2d/client.py

@@ -6,4 +6,4 @@ class LumisectionHistogram2DClient(BaseAuthorizedAPIClient):
     data_model = LumisectionHistogram2D
     pagination_model = PaginatedLumisectionHistogram2DList
     filter_class = LumisectionHistogram2DFilters
-    lookup_url = "lumisection-h2d/"
+    lookup_url = "th2/"

cmsdials/clients/h2d/models.py

@@ -7,40 +7,46 @@
 
 class LumisectionHistogram2D(BaseModel):
     hist_id: int
+    dataset: str = Field(..., max_length=255)
+    me: str = Field(..., max_length=255)
+    dataset_id: int
     file_id: int
     run_number: int
+    ls_number: int
+    me_id: int
     ls_id: int
-    title: str = Field(..., max_length=220)
     x_min: float
     x_max: float
     x_bin: float
     y_min: float
     y_max: float
     y_bin: float
     entries: int
-    data: list[float]
+    data: list[list[float]]
 
 
 class PaginatedLumisectionHistogram2DList(BaseModel):
-    count: int
     next: Optional[AnyUrl]
     previous: Optional[AnyUrl]
     results: list[LumisectionHistogram2D]
 
 
 class LumisectionHistogram2DFilters(OBaseModel):
-    page: Optional[str] = None
+    next_token: Optional[str] = None
+    dataset_id: Optional[int] = None
+    file_id: Optional[int] = None
     run_number: Optional[int] = None
+    run_number__lte: Optional[int] = None
+    run_number__gte: Optional[int] = None
     ls_number: Optional[int] = None
+    ls_numbet__lte: Optional[int] = None
+    ls_number__gte: Optional[int] = None
+    me_id: Optional[int] = None
     ls_id: Optional[int] = None
-    title: Optional[str] = None
-    min_run_number: Optional[int] = None
-    max_run_number: Optional[int] = None
-    min_ls_number: Optional[int] = None
-    max_ls_number: Optional[int] = None
-    title_contains: Optional[str] = None
-    min_entries: Optional[int] = None
-    era: Optional[str] = None
-    campaign: Optional[str] = None
-    primary_dataset: Optional[str] = None
-    file_id: Optional[int] = None
+    entries__gte: Optional[int] = None
+    dataset: Optional[str] = None
+    dataset__regex: Optional[str] = None
+    logical_file_name: Optional[str] = None
+    logical_file_name__regex: Optional[str] = None
+    me: Optional[str] = None
+    me__regex: Optional[str] = None

cmsdials/clients/lumisection/client.py

@@ -1,4 +1,8 @@
+import requests
+from requests.exceptions import HTTPError
+
 from ...utils.api_client import BaseAuthorizedAPIClient
+from ...utils.logger import logger
 from .models import Lumisection, LumisectionFilters, PaginatedLumisectionList
 
 
@@ -7,3 +11,17 @@ class LumisectionClient(BaseAuthorizedAPIClient):
     pagination_model = PaginatedLumisectionList
     filter_class = LumisectionFilters
     lookup_url = "lumisection/"
+
+    def get(self, dataset_id: int, run_number: int, ls_number: int):
+        endpoint_url = f"{self.api_url}{self.lookup_url}{dataset_id}/{run_number}/{ls_number}/"
+        headers = self._build_headers()
+        response = requests.get(endpoint_url, headers=headers)
+
+        try:
+            response.raise_for_status()
+        except HTTPError as err:
+            logger.info(f"Api raw response: {response.text}")
+            raise err
+
+        response = response.json()
+        return self.data_model(**response)