Merge pull request #56 from databio/dev

Release 0.6.1

bbconf/_version.py

@@ -1 +1 @@
-__version__ = "0.6.0"
+__version__ = "0.6.1"

bbconf/bbagent.py

@@ -1,6 +1,6 @@
 from functools import cached_property
 from pathlib import Path
-from typing import Union, List
+from typing import List, Union
 
 from sqlalchemy.orm import Session
 from sqlalchemy.sql import distinct, func, select

bbconf/config_parser/models.py

@@ -1,6 +1,6 @@
+import logging
 from pathlib import Path
 from typing import Optional, Union
-import logging
 
 from pydantic import BaseModel, ConfigDict, computed_field, field_validator
 from yacman import load_yaml

bbconf/db_utils.py

@@ -1,18 +1,18 @@
 import datetime
 import logging
 from typing import List, Optional
-import pandas as pd
 
+import pandas as pd
 from sqlalchemy import TIMESTAMP, BigInteger, ForeignKey, Result, Select, event, select
 from sqlalchemy.dialects.postgresql import JSON
 from sqlalchemy.engine import URL, Engine, create_engine
 from sqlalchemy.event import listens_for
-from sqlalchemy.exc import ProgrammingError, IntegrityError
+from sqlalchemy.exc import IntegrityError, ProgrammingError
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, relationship
 from sqlalchemy_schemadisplay import create_schema_graph
 
-from bbconf.const import PKG_NAME, LICENSES_CSV_URL
+from bbconf.const import LICENSES_CSV_URL, PKG_NAME
 
 _LOGGER = logging.getLogger(PKG_NAME)
 
@@ -300,6 +300,48 @@ def delete_bed_universe(mapper, connection, target):
         session.commit()
 
 
+class GeoGseStatus(Base):
+    __tablename__ = "geo_gse_status"
+
+    id: Mapped[int] = mapped_column(primary_key=True, index=True, autoincrement=True)
+    gse: Mapped[str] = mapped_column(nullable=False, comment="GSE number", unique=True)
+    status: Mapped[str] = mapped_column(
+        nullable=False, comment="Status of the GEO project"
+    )
+    submission_date: Mapped[datetime.datetime] = mapped_column(
+        default=deliver_update_date, onupdate=deliver_update_date
+    )
+    number_of_files: Mapped[int] = mapped_column(default=0, comment="Number of files")
+    number_of_success: Mapped[int] = mapped_column(
+        default=0, comment="Number of success"
+    )
+    number_of_skips: Mapped[int] = mapped_column(default=0, comment="Number of skips")
+    number_of_fails: Mapped[int] = mapped_column(default=0, comment="Number of fails")
+
+    gsm_status_mapper: Mapped[List["GeoGsmStatus"]] = relationship(
+        "GeoGsmStatus", back_populates="gse_status_mapper"
+    )
+
+
+class GeoGsmStatus(Base):
+    __tablename__ = "geo_gsm_status"
+
+    id: Mapped[int] = mapped_column(primary_key=True, index=True, autoincrement=True)
+    gse_status_id: Mapped[str] = mapped_column(
+        ForeignKey("geo_gse_status.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    gsm: Mapped[str] = mapped_column(nullable=False, comment="GSM number", unique=False)
+    sample_name: Mapped[str] = mapped_column()
+    status: Mapped[str] = mapped_column(
+        nullable=False, comment="Status of the GEO sample"
+    )
+    error: Mapped[str] = mapped_column(nullable=True, comment="Error message")
+    genome: Mapped[str] = mapped_column(nullable=True, comment="Genome")
+    gse_status_mapper: Mapped["GeoGseStatus"] = relationship(
+        "GeoGseStatus", back_populates="gsm_status_mapper"
+    )
+
+
 class BaseEngine:
     """
     A class with base methods, that are used in several classes.

bbconf/models/bed_models.py

@@ -3,9 +3,10 @@
 
 from pydantic import BaseModel, ConfigDict, Field
 
-from .base_models import FileModel
 from bbconf.const import DEFAULT_LICENSE
 
+from .base_models import FileModel
+
 
 class BedPlots(BaseModel):
     chrombins: FileModel = None

bbconf/modules/bedfiles.py

@@ -3,27 +3,26 @@
 from typing import Dict, Union
 
 import numpy as np
-from tqdm import tqdm
-
 from geniml.bbclient import BBClient
 from geniml.io import RegionSet
 from genimtools.tokenizers import RegionSet as GRegionSet
 from pephubclient.exceptions import ResponseError
 from qdrant_client.models import Distance, PointIdsList, VectorParams
 from sqlalchemy import and_, delete, func, select
 from sqlalchemy.orm import Session
+from tqdm import tqdm
 
 from bbconf.config_parser.bedbaseconfig import BedBaseConfig
-from bbconf.const import PKG_NAME, ZARR_TOKENIZED_FOLDER, DEFAULT_LICENSE
+from bbconf.const import DEFAULT_LICENSE, PKG_NAME, ZARR_TOKENIZED_FOLDER
 from bbconf.db_utils import Bed, BedStats, Files, TokenizedBed, Universes
 from bbconf.exceptions import (
     BedBaseConfError,
     BedFIleExistsError,
     BEDFileNotFoundError,
+    QdrantInstanceNotInitializedError,
     TokenizeFileExistsError,
     TokenizeFileNotExistError,
     UniverseNotFoundError,
-    QdrantInstanceNotInitializedError,
 )
 from bbconf.models.bed_models import (
     BedClassification,
@@ -34,15 +33,15 @@
     BedMetadata,
     BedMetadataBasic,
     BedPEPHub,
+    BedPEPHubRestrict,
     BedPlots,
+    BedSetMinimal,
     BedStatsModel,
     FileModel,
     QdrantSearchResult,
     TokenizedBedResponse,
-    UniverseMetadata,
     TokenizedPathResponse,
-    BedPEPHubRestrict,
-    BedSetMinimal,
+    UniverseMetadata,
 )
 
 _LOGGER = getLogger(PKG_NAME)

bbconf/modules/bedsets.py

@@ -7,7 +7,7 @@
 
 from bbconf.config_parser import BedBaseConfig
 from bbconf.const import PKG_NAME
-from bbconf.db_utils import BedFileBedSetRelation, BedSets, BedStats, Files, Bed
+from bbconf.db_utils import Bed, BedFileBedSetRelation, BedSets, BedStats, Files
 from bbconf.exceptions import BedSetExistsError, BedSetNotFoundError
 from bbconf.models.bed_models import BedStatsModel
 from bbconf.models.bedset_models import (

docs/changelog.md

@@ -2,6 +2,11 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
+# [0.6.1] - 2024-08-21
+## Added 
+
+- DB tables for GEO uploader status
+
 # [0.6.0] - 2024-05-01
 ## Added
 

requirements/requirements-all.txt

@@ -4,7 +4,7 @@ geniml >= 0.4.0
 psycopg >= 3.1.15
 colorlogs
 pydantic >= 2.6.4
-botocore >= 1.34.54
+botocore
 boto3 >=  1.34.54
 pephubclient >= 0.4.1
 sqlalchemy_schemadisplay

tests/test_bedfile.py

@@ -3,9 +3,9 @@
 from sqlalchemy.sql import select
 
 from bbconf.bbagent import BedBaseAgent
+from bbconf.const import DEFAULT_LICENSE
 from bbconf.db_utils import Bed, Files
 from bbconf.exceptions import BedFIleExistsError, BEDFileNotFoundError
-from bbconf.const import DEFAULT_LICENSE
 
 from .conftest import SERVICE_UNAVAILABLE, get_bbagent
 from .utils import BED_TEST_ID, ContextManagerDBTesting

tests/test_common.py

@@ -1,6 +1,7 @@
+import pytest
+
 from bbconf.const import DEFAULT_LICENSE
 
-import pytest
 from .conftest import SERVICE_UNAVAILABLE
 from .utils import ContextManagerDBTesting