From 87dd2b1167d0b34a621c7ec106c3209cabd4e980 Mon Sep 17 00:00:00 2001 From: Danilo Horta Date: Mon, 16 Sep 2024 13:42:57 +0100 Subject: [PATCH] Refactor build process and update H3Result class - Replace CMake with Make for building dependencies - Use GitPython for cloning repositories - Update H3Result class to use file descriptors - Remove unnecessary imports and functions - Update pyproject.toml to include GitPython dependency - Simplify interface.h and remove unused functions - Adjust read_h3result to work with file descriptors --- h3result-py/build_ext.py | 153 +++++--------------------- h3result-py/h3result/h3result.py | 36 ++---- h3result-py/h3result/interface.h | 35 +++--- h3result-py/h3result/read_h3result.py | 13 ++- h3result-py/pyproject.toml | 4 +- 5 files changed, 63 insertions(+), 178 deletions(-) diff --git a/h3result-py/build_ext.py b/h3result-py/build_ext.py index 04cbe93..6dc4b01 100644 --- a/h3result-py/build_ext.py +++ b/h3result-py/build_ext.py @@ -1,154 +1,57 @@ import os -import re import shutil -import sys -import sysconfig -import tarfile -import urllib.request -from dataclasses import dataclass from pathlib import Path from subprocess import check_call -RPATH = "$ORIGIN" if sys.platform.startswith("linux") else "@loader_path" +from cffi import FFI +from git import Repo -PWD = Path(os.path.dirname(os.path.abspath(__file__))) -TMP = PWD / ".build_ext" -PKG = PWD / "h3result" +CWD = Path(".").resolve() +TMP = CWD / ".build_ext" +PKG = CWD / "h3result" INTERFACE = PKG / "interface.h" +LIB = PKG / "lib" +INC = PKG / "include" -BIN = Path(PKG) / "bin" -LIB = Path(PKG) / "lib" -INCL = Path(PKG) / "include" -EXTRA = f"-Wl,-rpath,{RPATH}/lib" -SHARE = Path(PKG) / "share" +SHARE = PKG / "share" -CMAKE_OPTS = [ - "-DCMAKE_BUILD_TYPE=Release", - "-DBUILD_SHARED_LIBS=ON", - f"-DCMAKE_INSTALL_RPATH={RPATH}", - "-DCMAKE_INSTALL_LIBDIR=lib", -] -CPM_OPTS = ["-DCPM_USE_LOCAL_PACKAGES=ON"] +def make(args: list[str] = [], cwd: Path = CWD): + check_call(["make"] + args, cwd=cwd) -@dataclass -class Ext: - github_user: str - github_project: str - git_tag: str - root_dir: str - cmake_opts: list[str] +def build_and_install(root: Path, prefix: str, prj_dir: str, git_url: str): + git_dir = git_url.split("/")[-1].split(".")[-1][:-4] + if not (root / git_dir).exists(): + Repo.clone_from(git_url, root / git_dir, depth=1) + make(cwd=root / prj_dir) + make(["install", f"PREFIX={prefix}"], cwd=root / prj_dir) -EXTS = [ - Ext("EBI-Metagenomics", "lip", "v0.5.4", "./", CMAKE_OPTS), - Ext( - "EBI-Metagenomics", - "hmmer3", - "h3result-v0.2.2", - "./h3result", - CMAKE_OPTS + CPM_OPTS, - ), -] - - -def rm(folder: Path, pattern: str): - for filename in folder.glob(pattern): - filename.unlink() - - -def resolve_bin(bin: str): - paths = [sysconfig.get_path("scripts", x) for x in sysconfig.get_scheme_names()] - paths += ["/usr/local/bin/"] - for x in paths: - y = Path(x) / bin - if y.exists(): - return str(y) - raise RuntimeError(f"Failed to find {bin}.") - - -def build_ext(ext: Ext): - from cmake import CMAKE_BIN_DIR - - url = ( - f"https://github.com/{ext.github_user}/{ext.github_project}" - f"/archive/refs/tags/{ext.git_tag}.tar.gz" - ) - tar_filename = f"{ext.github_project}-{ext.git_tag}.tar.gz" - +if __name__ == "__main__": + shutil.rmtree(TMP, ignore_errors=True) os.makedirs(TMP, exist_ok=True) - with open(TMP / tar_filename, "wb") as lf: - lf.write(urllib.request.urlopen(url).read()) - - with tarfile.open(TMP / tar_filename) as tf: - dir = os.path.commonprefix(tf.getnames()) - tf.extractall(TMP) - prj_dir = TMP / dir / ext.root_dir - bld_dir = prj_dir / "build" - os.makedirs(bld_dir, exist_ok=True) + url = "https://github.com/EBI-Metagenomics/lite-pack.git" + build_and_install(TMP / "lite-pack", str(PKG), ".", url) - cmake = [str(v) for v in Path(CMAKE_BIN_DIR).glob("cmake*")][0] - check_call([cmake, "-S", str(prj_dir), "-B", str(bld_dir)] + ext.cmake_opts) - n = os.cpu_count() - check_call([cmake, "--build", str(bld_dir), "-j", str(n), "--config", "Release"]) + url = "https://github.com/EBI-Metagenomics/lite-pack.git" + build_and_install(TMP / "lite-pack", str(PKG), "ext/", url) - check_call([cmake, "--install", str(bld_dir), "--prefix", str(PKG)]) - - -if __name__ == "__main__": - from cffi import FFI + url = "https://github.com/EBI-Metagenomics/hmmer3.git" + build_and_install(TMP / "hmmer3", str(PKG), "h3result/", url) ffibuilder = FFI() - rm(PKG, "cffi.*") - rm(PKG / "lib", "**/lib*") - shutil.rmtree(TMP, ignore_errors=True) - - if not os.environ.get("H3RESULT_DEVELOP", False): - for ext in EXTS: - build_ext(ext) - - libs = os.environ.get("H3RESULT_LIBRARY_PATH", "").split(";") - incls = os.environ.get("H3RESULT_INCLUDE_PATH", "").split(";") - - libs = [x for x in libs if len(x) > 0] - incls = [x for x in incls if len(x) > 0] - ffibuilder.cdef(open(INTERFACE, "r").read()) ffibuilder.set_source( "h3result.cffi", """ - #include "h3result/h3result.h" + #include "h3r_result.h" """, language="c", - libraries=["h3result"], - library_dirs=libs + [str(LIB)], - include_dirs=incls + [str(INCL)], - extra_link_args=[str(EXTRA)], + libraries=["h3result", "lio", "lite_pack"], + library_dirs=[str(LIB)], + include_dirs=[str(INC)], ) ffibuilder.compile(verbose=True) - - shutil.rmtree(BIN, ignore_errors=True) - shutil.rmtree(INCL, ignore_errors=True) - shutil.rmtree(SHARE, ignore_errors=True) - shutil.rmtree(LIB / "cmake", ignore_errors=True) - - if not os.environ.get("H3RESULT_DEVELOP", False): - if sys.platform == "linux": - patch = [resolve_bin("patchelf"), "--set-rpath", "$ORIGIN"] - for lib in LIB.glob("*.so*"): - check_call(patch + [str(lib)]) - - find = ["/usr/bin/find", str(LIB), "-type", "l"] - exec0 = ["-exec", "/bin/cp", "{}", "{}.tmp", ";"] - exec1 = ["-exec", "/bin/mv", "{}.tmp", "{}", ";"] - check_call(find + exec0 + exec1) - - for x in list(LIB.iterdir()): - linux_pattern = r"lib[^.]*\.so\.[0-9]+" - macos_pattern = r"lib[^.]*\.[0-9]+\.dylib" - pattern = r"^(" + linux_pattern + r"|" + macos_pattern + r")$" - if not re.match(pattern, x.name): - x.unlink() diff --git a/h3result-py/h3result/h3result.py b/h3result-py/h3result/h3result.py index b8bd377..bd7c041 100644 --- a/h3result-py/h3result/h3result.py +++ b/h3result-py/h3result/h3result.py @@ -7,49 +7,37 @@ class H3Result: - def __init__(self, fp): + def __init__(self, fd: int): self._cdata = ffi.NULL - self._cdata = lib.h3result_new() + self._cdata = lib.h3r_new() if self._cdata == ffi.NULL: raise MemoryError() - rc = lib.h3result_unpack(self._cdata, fp) + rc = lib.h3r_unpack(self._cdata, fd) if rc != 0: raise H3ResultError(rc) def print_targets(self, fileno: int): fd = os.dup(fileno) - fp = lib.fdopen(fd, b"w") - if fp == ffi.NULL: - raise RuntimeError() - lib.h3result_print_targets(self._cdata, fp) - lib.fclose(fp) + lib.h3r_print_targets(self._cdata, fd) + os.close(fd) def print_domains(self, fileno: int): fd = os.dup(fileno) - fp = lib.fdopen(fd, b"w") - if fp == ffi.NULL: - raise RuntimeError() - lib.h3result_print_domains(self._cdata, fp) - lib.fclose(fp) + lib.h3r_print_domains(self._cdata, fd) + os.close(fd) def print_targets_table(self, fileno: int): fd = os.dup(fileno) - fp = lib.fdopen(fd, b"w") - if fp == ffi.NULL: - raise RuntimeError() - lib.h3result_print_targets_table(self._cdata, fp) - lib.fclose(fp) + lib.h3r_print_targets_table(self._cdata, fd) + os.close(fd) def print_domains_table(self, fileno: int): fd = os.dup(fileno) - fp = lib.fdopen(fd, b"w") - if fp == ffi.NULL: - raise RuntimeError() - lib.h3result_print_domains_table(self._cdata, fp) - lib.fclose(fp) + lib.h3r_print_domains_table(self._cdata, fd) + os.close(fd) def __del__(self): if getattr(self, "_cdata", ffi.NULL) != ffi.NULL: - lib.h3result_del(self._cdata) + lib.h3r_del(self._cdata) diff --git a/h3result-py/h3result/interface.h b/h3result-py/h3result/interface.h index b1ef2ca..5b1c6ae 100644 --- a/h3result-py/h3result/interface.h +++ b/h3result-py/h3result/interface.h @@ -1,27 +1,20 @@ -struct h3result; +struct h3r; -struct h3result *h3result_new(void); -void h3result_del(struct h3result const *); +struct h3r *h3r_new(void); +void h3r_del(struct h3r const *); -int h3result_pack(struct h3result const *, FILE *); -int h3result_unpack(struct h3result *, FILE *); +int h3r_pack(struct h3r const *, int fd); +int h3r_unpack(struct h3r *, int fd); -int h3result_errnum(struct h3result const *); -char const *h3result_errstr(struct h3result const *); +int h3r_print_targets(struct h3r const *, int fd); +int h3r_print_domains(struct h3r const *, int fd); -void h3result_print_targets(struct h3result const *, FILE *); -void h3result_print_domains(struct h3result const *, FILE *); +int h3r_print_targets_table(struct h3r const *, int fd); +int h3r_print_domains_table(struct h3r const *, int fd); -void h3result_print_targets_table(struct h3result const *, FILE *); -void h3result_print_domains_table(struct h3result const *, FILE *); +unsigned h3r_nhits(struct h3r const *); +char const *h3r_hit_name(struct h3r const *, unsigned idx); +char const *h3r_hit_accession(struct h3r const *, unsigned idx); +double h3r_hit_logevalue(struct h3r const *, unsigned idx); -unsigned h3result_nhits(struct h3result const *); -char const *h3result_hit_name(struct h3result const *, unsigned idx); -char const *h3result_hit_acc(struct h3result const *, unsigned idx); -double h3result_hit_evalue_ln(struct h3result const *, unsigned idx); - -char const *h3result_strerror(int rc); - -FILE *fopen(char const *filename, char const *mode); -FILE *fdopen(int, char const *); -int fclose(FILE *); +char const *h3r_strerror(int rc); diff --git a/h3result-py/h3result/read_h3result.py b/h3result-py/h3result/read_h3result.py index f2ed616..9979055 100644 --- a/h3result-py/h3result/read_h3result.py +++ b/h3result-py/h3result/read_h3result.py @@ -1,7 +1,7 @@ import os from pathlib import Path -from h3result.cffi import ffi, lib +from h3result.cffi import ffi from h3result.h3result import H3Result from h3result.path_like import PathLike @@ -15,16 +15,17 @@ def read_h3result(filename: PathLike | None = None, fileno: int | None = None): assert filename is None if filename: - fp = lib.fopen(bytes(Path(filename)), b"rb") + fd = os.open(bytes(Path(str(filename))), os.O_RDONLY) else: - fp = lib.fdopen(os.dup(fileno), b"rb") + assert fileno is not None + fd = os.dup(fileno) - if fp == ffi.NULL: + if fd == ffi.NULL: raise RuntimeError() try: - x = H3Result(fp) + x = H3Result(fd) finally: - lib.fclose(fp) + os.close(fd) return x diff --git a/h3result-py/pyproject.toml b/h3result-py/pyproject.toml index d35903a..d4d2d66 100644 --- a/h3result-py/pyproject.toml +++ b/h3result-py/pyproject.toml @@ -16,6 +16,7 @@ include = [ [tool.poetry.dependencies] cffi = "^1.16" python = "^3.9" +gitpython = "^3.1.43" [tool.poetry.build] script = "build_ext.py" @@ -28,8 +29,7 @@ pytest = "^8.2" requires = [ "poetry-core", "cffi", - "cmake", - "patchelf;sys_platform=='linux'", + "GitPython", "setuptools;python_version>='3.12'", ] build-backend = "poetry.core.masonry.api"