From 087837dd1d859bb6b93f1f7bbaaadaa3accfd866 Mon Sep 17 00:00:00 2001 From: Jayaram Kancherla Date: Thu, 16 May 2024 12:26:03 -0700 Subject: [PATCH] Include `asis` in `add` and `update` methods (#10) - Include `asis` as an option, It existed in the earlier versions but at one point there was a regression. - Add tests - Revamp docstrings --- src/pybiocfilecache/BiocFileCache.py | 119 ++++++++++++++++++--------- src/pybiocfilecache/db/Base.py | 10 ++- src/pybiocfilecache/utils.py | 53 ++++++++---- tests/test_cache.py | 8 ++ 4 files changed, 132 insertions(+), 58 deletions(-) diff --git a/src/pybiocfilecache/BiocFileCache.py b/src/pybiocfilecache/BiocFileCache.py index fa4ef6b..d673c44 100644 --- a/src/pybiocfilecache/BiocFileCache.py +++ b/src/pybiocfilecache/BiocFileCache.py @@ -3,17 +3,17 @@ import os from pathlib import Path from time import sleep, time -from typing import List, Optional, Union +from typing import List, Literal, Optional, Union from sqlalchemy import func from sqlalchemy.orm import Session +from ._exceptions import NoFpathError, RnameExistsError, RpathTimeoutError from .db import create_schema from .db.schema import Resource from .utils import copy_or_move, create_tmp_dir, generate_id -from ._exceptions import NoFpathError, RnameExistsError, RpathTimeoutError -__author__ = "jkanche" +__author__ = "Jayaram Kancherla" __copyright__ = "jkanche" __license__ = "MIT" @@ -25,8 +25,10 @@ def __init__(self, cacheDirOrPath: Union[str, Path] = create_tmp_dir()): """Initialize BiocFileCache. Args: - cacheDirOrPath (Union[str, Path], optional): Path to cache. - directory. Defaults to tmp location, `create_tmp_dir()`. + cacheDirOrPath: + Path to cache directory. + + Defaults to tmp location, :py:func:`~.utils.create_tmp_dir`. Raises: Exception: Failed to initialize cache. @@ -51,38 +53,54 @@ def add( self, rname: str, fpath: Union[str, Path], - rtype: str = "local", - action: str = "copy", + rtype: Literal["local", "web", "relative"] = "local", + action: Literal["copy", "move", "asis"] = "copy", ext: bool = False, ) -> Resource: """Add a resource from the provided `fpath` to cache as `rname`. Args: - rname (str): Name of the resource to add to cache. - fpath (Union[str, Path]): Location of the resource. - rtype (str, optional): One of `"local"`, `"web"`, or `"relative"`. - Defaults to `"local"`. - action (str, optional): Either `"copy"`, `"move"` or `"asis"`. - Defaults to `"copy"`. - ext (bool, optional): Use filepath extension when storing in cache. - Defaults to `False`. + rname: + Name of the resource to add to cache. - Returns: - Resource: Database record of the new resource in cache. + fpath: + Location of the resource. + + rtype: + One of ``local``, ``web``, or ``relative``. + Defaults to ``local``. + + action: + Either ``copy``, ``move`` or ``asis``. + Defaults to ``copy``. + + ext: + Whether to use filepath extension when storing in cache. + Defaults to `False`. Raises: - NoFpathError: When the `fpath` does not exist. - RnameExistsError: When the `rname` already exists in the cache. - sqlalchemy exceptions: When something is up with the cache. + NoFpathError: + When the `fpath` does not exist. + + RnameExistsError: + When the `rname` already exists in the cache. + sqlalchemy exceptions: When something is up with the cache. + + Returns: + Database record of the new resource in cache. """ if isinstance(fpath, str): fpath = Path(fpath) if not fpath.exists(): - raise NoFpathError(f"Resource at {fpath} does not exist.") + raise NoFpathError(f"Resource at '{fpath}' does not exist.") rid = generate_id() - rpath = f"{self.cache}/{rid}" + (f".{fpath.suffix}" if ext else "") + rpath = ( + f"{self.cache}/{rid}" + (f".{fpath.suffix}" if ext else "") + if action != "asis" + else str(fpath) + ) # create new record in the database res = Resource( @@ -123,11 +141,15 @@ def query(self, query: str, field: str = "rname") -> List[Resource]: """Search cache for a resource. Args: - query (str): query or keywords to search. - field (str, optional): Field to search. Defaults to "rname". + query: + Query string or keywords to search. + + field: + Field to search. + Defaults to "rname". Returns: - List[Resource]: list of matching resources from cache. + List of matching resources from cache. """ with self.sessionLocal() as session: return ( @@ -140,11 +162,14 @@ def _get(self, session: Session, rname: str) -> Optional[Resource]: """Get a resource with `rname` from given `Session`. Args: - session (Session): The `Session` object to use. - rname (str): The `rname` of the `Resource` to get. + session: + The `Session` object to use. + + rname: + The `rname` of the `Resource` to get. Returns: - (Resource, optional): The `Resource` for the `rname` if any. + The `Resource` for the `rname` if available. """ resource: Optional[Resource] = ( session.query(Resource).filter(Resource.rname == rname).first() @@ -169,10 +194,11 @@ def get(self, rname: str) -> Optional[Resource]: """Get resource by name from cache. Args: - rname (str): Name of the file to search. + rname: + Name of the file to search. Returns: - Optional[Resource]: matched resource from cache if exists. + Matched `Resource` from cache if exists. """ return self._get(self.sessionLocal(), rname) @@ -180,7 +206,8 @@ def remove(self, rname: str) -> None: """Remove a resource from cache by name. Args: - rname (str): Name of the resource to remove. + rname: + Name of the resource to remove. """ with self.sessionLocal() as session: res: Optional[Resource] = self._get(session, rname) @@ -196,18 +223,30 @@ def purge(self): for file in os.scandir(self.cache): os.remove(file.path) + return True + def update( - self, rname: str, fpath: Union[str, Path], action: str = "copy" + self, + rname: str, + fpath: Union[str, Path], + action: Literal["copy", "move", "asis"] = "copy", ) -> Resource: """Update a resource in cache. Args: - rname (str): name of the resource in cache. - fpath (Union[str, Path]): new resource to replace existing file in cache. - action (str, optional): either copy of move. defaults to copy. + rname: + Name of the resource in cache. + + fpath: + New resource to replace existing file in cache. + + action: + Either ``copy``, ``move`` or ``asis``. + + Defaults to ``copy``. Returns: - Resource: Updated resource record in cache. + Updated resource record in cache. """ if isinstance(fpath, str): @@ -220,8 +259,12 @@ def update( res = self._get(session, rname) if res is not None: - # copy the file to cache - copy_or_move(str(fpath), str(res.rpath), rname, action) + if action != "asis": + # copy the file to cache + copy_or_move(str(fpath), str(res.rpath), rname, action) + else: + res.rpath = str(fpath) + res.access_time = res.last_modified_time = func.now() session.merge(res) session.commit() diff --git a/src/pybiocfilecache/db/Base.py b/src/pybiocfilecache/db/Base.py index 0251f90..d898b4a 100644 --- a/src/pybiocfilecache/db/Base.py +++ b/src/pybiocfilecache/db/Base.py @@ -1,9 +1,10 @@ +from typing import Tuple + from sqlalchemy import create_engine from sqlalchemy.engine import Engine # from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker, declarative_base -from typing import Tuple +from sqlalchemy.orm import declarative_base, sessionmaker __author__ = "jkanche" __copyright__ = "jkanche" @@ -16,10 +17,11 @@ def create_schema(cache_dir: str) -> Tuple[Engine, sessionmaker]: """Create the schema in the sqlite database. Args: - cache_dir (str): Location where the cache directory + cache_dir: + Location where the cache directory. Returns: - a tuple of sqlalchemy engine and session maker + A tuple of sqlalchemy engine and session maker. """ try: engine = create_engine( diff --git a/src/pybiocfilecache/utils.py b/src/pybiocfilecache/utils.py index c0b71da..7e5d75f 100644 --- a/src/pybiocfilecache/utils.py +++ b/src/pybiocfilecache/utils.py @@ -1,18 +1,21 @@ +import logging +import sys import tempfile import uuid from pathlib import Path from shutil import copy2, move +from typing import Literal, Union -from typing import Union -import logging -import sys +__author__ = "Jayaram Kancherla" +__copyright__ = "jkanche" +__license__ = "MIT" def create_tmp_dir() -> str: """Create a temporary directory. Returns: - str: path to the directory + Temporary path to the directory. """ return tempfile.mkdtemp() @@ -21,38 +24,56 @@ def generate_id() -> str: """Generate uuid. Returns: - str: unique string for use as id + Unique string for use as id. """ return uuid.uuid4().hex def copy_or_move( - source: Union[str, Path], target: Union[str, Path], rname: str, action: str = "copy" + source: Union[str, Path], + target: Union[str, Path], + rname: str, + action: Literal["copy", "move", "asis"] = "copy", ) -> None: - """Copy or move a resource from `source` to `target` + """Copy or move a resource from ``source`` to ``target``. Args: - source (Union[str, Path]): source location of the resource to copy of move. - target (Union[str, Path]): destination to copy of move to. - rname (str): Name of resource to add to cache - action (str): copy of move file from source. Defaults to copy. + source: + Source location of the resource to copy of move. + + target: + Destination to copy of move to. + + rname: + Name of resource to add to cache. + + action: + Copy of move file from source. + Defaults to copy. Raises: - ValueError: if action is not `copy` or `move`. - Exception: Error storing resource in the cache directory. + ValueError: + If action is not `copy`, `move` or `asis`. + + Exception: + Error storing resource in the cache directory. """ - if action not in ["copy", "move"]: - raise ValueError(f"Action must be either 'move' or 'copy', provided {action}") + if action not in ["copy", "move", "asis"]: + raise ValueError( + f"Action must be either 'move', 'copy' or 'asis', provided {action}." + ) try: if action == "copy": copy2(source, target) elif action == "move": move(str(source), target) + elif action == "asis": + pass except Exception as e: raise Exception( - f"Error storing resource: '{rname}' from: '{source}' in '{target}'", + f"Error storing resource: '{rname}' from: '{source}' in '{target}'.", ) from e diff --git a/tests/test_cache.py b/tests/test_cache.py index bbeebc7..ed732af 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -33,6 +33,14 @@ def test_add_get_operations(): frec2 = open(rec2.rpath, "r").read().strip() assert frec2 == "test2" + bfc.add("test3_asis", os.getcwd() + "/tests/data/test2.txt", action="asis") + rec3 = bfc.get("test3_asis") + assert rec3 is not None + assert rec3.rpath == os.getcwd() + "/tests/data/test2.txt" + + frec3 = open(rec3.rpath, "r").read().strip() + assert frec3 == "test2" + bfc.purge()