diff --git a/.gitignore b/.gitignore index 8176471..45ce470 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ /.idea/ *.swp /docs/ +/wheelhouse/ +/dist/ +**.egg-info +**/__pycache__/ diff --git a/docs.md/Creating_the_wheel.md b/docs.md/Creating_the_wheel.md index 68ad156..c6de8ba 100644 --- a/docs.md/Creating_the_wheel.md +++ b/docs.md/Creating_the_wheel.md @@ -7,6 +7,7 @@ older Linux distributions. Indeed, PyPI forbids uploading wheel that do not specify a compatibility tag used to choose the right wheel when running `pip install SQLamarr`. +## Using docker To compile the wheel in compatibility mode, we are using the following recipe, which assumes the working directory is the root of the git repository ```bash @@ -15,7 +16,22 @@ docker run -it -v $PWD:/mylib quay.io/pypa/manylinux2014_x86_64:latest \ python3 -m twine upload wheelhouse/*.whl ``` -Additional information: +## Using Apptainer +In some environments, docker is not available and apptainer should be used instead. + +First convert the docker image to an SIF image (this is only required once). +```bash +apptainer build /tmp/pypa.sif docker://quay.io/pypa/manylinux2014_x86_64:latest +``` + +Then execute the following script (as for the docker case, it is assumed +the working directory is the root of the git repository). + +```build +apptainer exec --bind $PWD:/mylib --writable-tmpfs /tmp/pypa.sif /bin/bash make_wheel.sh +``` + +## Additional information: * Distributing binaries in compatibility mode: https://github.com/pypa/manylinux * Example using travis for CI/CD: https://github.com/pypa/python-manylinux-demo diff --git a/include/SQLamarr/BaseSqlInterface.h b/include/SQLamarr/BaseSqlInterface.h index e3f8b9d..565e839 100644 --- a/include/SQLamarr/BaseSqlInterface.h +++ b/include/SQLamarr/BaseSqlInterface.h @@ -13,7 +13,6 @@ #include #include -#include "SQLamarr/BaseSqlInterface.h" #include "SQLamarr/db_functions.h" namespace SQLamarr @@ -39,12 +38,21 @@ namespace SQLamarr BaseSqlInterface(SQLite3DB& db); virtual ~BaseSqlInterface(); + void sync_database(const std::string& db_uri) + {update_db_connection(m_database, db_uri);} + + /// Invalidate the cache of the queries. + /// Especially useful to allow refreshing the connection when running + /// from Python. + void invalidate_cache(void); + protected: // members SQLite3DB& m_database; ///< Reference to the SQLite database (not owned). private: //members std::unordered_map m_queries; + sqlite3* m_cached_raw_ptr; protected: // methods /// Creates or retrieve from cache a statement @@ -60,7 +68,7 @@ namespace SQLamarr void begin_transaction () { sqlite3_exec(m_database.get(), "BEGIN", 0, 0, 0); } /// End an SQL transaction re-enabling disk updates - void end_transaction () { sqlite3_exec(m_database.get(), "END", 0, 0, 0); } + void end_transaction () { sqlite3_exec(m_database.get(), "COMMIT", 0, 0, 0); } /// Return the index of the last rows inserted in any table int last_insert_row () { return sqlite3_last_insert_rowid(m_database.get()); } diff --git a/include/SQLamarr/UpdateDBConnection.h b/include/SQLamarr/UpdateDBConnection.h new file mode 100644 index 0000000..1b4ea7d --- /dev/null +++ b/include/SQLamarr/UpdateDBConnection.h @@ -0,0 +1,54 @@ +// (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration. +// +// This software is distributed under the terms of the GNU General Public +// Licence version 3 (GPL Version 3), copied verbatim in the file "LICENCE". +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#pragma once + +// STL +#include + +// SQLamarr +#include "SQLamarr/db_functions.h" +#include "SQLamarr/Transformer.h" + +namespace SQLamarr +{ + /** Reset the database connection forcing flushing the db status. + * + * In the interaction with Python or other frameworks it is sometimes + * necessary to ensure db synchronization with disk or shared memory. + * This can be achieved refreshing the connection to the database, + * by closing it and reopening. + * + * WARNING! Executing UpdateDBConnection drops TEMPORARY tables and views. + */ + class UpdateDBConnection: public Transformer + { + public: + /// Constructor + UpdateDBConnection ( + SQLite3DB& db, + ///< Reference to the database + std::string filename, + ///< Filename or URI of the (possibly new) connection to the database + int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI + ///< Flags + ); + + + /// Execute the algorithm, cleaning the database + void execute () override; + + private: // members + SQLite3DB& m_database; ///< Reference to the SQLite database (not owned). + const std::string m_filename; ///< Filename or URI of the database + const int m_flags; ///< SQLite flags to open to database (see sqlite_open_v3) + }; +} + + diff --git a/include/SQLamarr/db_functions.h b/include/SQLamarr/db_functions.h index be6cf51..b0f6845 100644 --- a/include/SQLamarr/db_functions.h +++ b/include/SQLamarr/db_functions.h @@ -36,6 +36,13 @@ namespace SQLamarr /// Ensure a token is alphanumeric void validate_token(const std::string& token); + + /// Force synchronization to disk by closing and opening the connection + void update_db_connection( + SQLite3DB& old_db, + const std::string& db_uri, + int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI + ); } diff --git a/make_wheel.sh b/make_wheel.sh index d5eccda..13ffc33 100644 --- a/make_wheel.sh +++ b/make_wheel.sh @@ -7,7 +7,7 @@ cd /mylib; rm -f wheelhouse/*.whl | echo "Ok"; rm -f dist/*.whl | echo "Ok"; -for PYVERSION in cp311-cp311 cp310-cp310 cp37-cp37m cp38-cp38 cp39-cp39; +for PYVERSION in cp312-cp312 cp311-cp311 cp310-cp310 cp37-cp37m cp38-cp38 cp39-cp39; do export PYBIN=/opt/python/$PYVERSION/bin/python3 echo "Preparing binary distribution with $PYBIN"; diff --git a/pyproject.toml b/pyproject.toml index 3bebc23..e4e2f3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ classifiers = [ ] -version = "0.0c5" +version = "0.0c7" dependencies = ["hepmc3"] diff --git a/python/SQLamarr/HepMC2DataLoader.py b/python/SQLamarr/HepMC2DataLoader.py index a576ff8..0acb42f 100644 --- a/python/SQLamarr/HepMC2DataLoader.py +++ b/python/SQLamarr/HepMC2DataLoader.py @@ -54,12 +54,8 @@ class HepMC2DataLoader: """ def __init__(self, db: SQLite3DB): """Acquires the reference to an open connection to the DB""" - self._self = clib.new_HepMC2DataLoader(db.get()) + self._db = db - def __del__ (self): - """@private: Release the bound class instance""" - clib.del_HepMC2DataLoader(self._self) - def load(self, filename: str, runNumber: int, evtNumber: int): """Loads an ASCII file with [HepMC3::ReaderAsciiHepMC2](http://hepmc.web.cern.ch/hepmc/classHepMC3_1_1ReaderAsciiHepMC2.html). @@ -67,6 +63,8 @@ def load(self, filename: str, runNumber: int, evtNumber: int): if not os.path.exists(filename): raise FileNotFoundError(filename) + _self = clib.new_HepMC2DataLoader(self._db.get()) clib.HepMC2DataLoader_load( - self._self, filename.encode('ascii'), runNumber, evtNumber + _self, filename.encode('ascii'), runNumber, evtNumber, self._db.path.encode('ascii') ) + clib.del_HepMC2DataLoader(_self) diff --git a/python/SQLamarr/UpdateDBConnection.py b/python/SQLamarr/UpdateDBConnection.py new file mode 100644 index 0000000..e97cc56 --- /dev/null +++ b/python/SQLamarr/UpdateDBConnection.py @@ -0,0 +1,43 @@ +# (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration. +# +# This software is distributed under the terms of the GNU General Public +# Licence version 3 (GPL Version 3), copied verbatim in the file "LICENCE". +# +# In applying this licence, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. +import ctypes +from ctypes import POINTER +from SQLamarr import clib, c_TransformerPtr + +from SQLamarr.db_functions import SQLite3DB + +clib.new_UpdateDBConnection.argtypes = (ctypes.c_void_p,) +clib.new_UpdateDBConnection.restype = c_TransformerPtr + +class UpdateDBConnection: + """ + Update the reference to the DB Connection, to ensure synchronization with disk. + + Refer to SQLamarr::UpdateDBConnection for implementation details. + """ + def __init__ (self, db: SQLite3DB): + """ + Configure a Transformer to update the connection to the DB. + + @param db: An open database connection. + """ + self._self = clib.new_UpdateDBConnection(db.get(), db.path.encode('ascii')) + + def __del__(self): + """@private: Release the bound class instance""" + clib.del_Transformer(self._self) + + @property + def raw_pointer(self): + """@private: Return the raw pointer to the algorithm.""" + return self._self + + + + diff --git a/python/SQLamarr/__init__.py b/python/SQLamarr/__init__.py index c0fbc91..5e3aa72 100644 --- a/python/SQLamarr/__init__.py +++ b/python/SQLamarr/__init__.py @@ -40,6 +40,7 @@ class c_TransformerPtr (ctypes.Structure): from SQLamarr.TemporaryTable import TemporaryTable from SQLamarr.CleanEventStore import CleanEventStore from SQLamarr.EditEventStore import EditEventStore +from SQLamarr.UpdateDBConnection import UpdateDBConnection ## Python Transfomer from SQLamarr.PyTransformer import PyTransformer diff --git a/python/SQLamarr/db_functions.py b/python/SQLamarr/db_functions.py index ff13a55..f71c0f1 100644 --- a/python/SQLamarr/db_functions.py +++ b/python/SQLamarr/db_functions.py @@ -61,6 +61,10 @@ def __del__(self): """@private: Return the raw pointer to the algorithm.""" clib.del_database(self._pointer) + @property + def path(self): + return self._path + def get(self): """@private: Return the raw pointer to the database.""" return self._pointer diff --git a/src/BaseSqlInterface.cpp b/src/BaseSqlInterface.cpp index 9fbb616..7a79b54 100644 --- a/src/BaseSqlInterface.cpp +++ b/src/BaseSqlInterface.cpp @@ -21,6 +21,8 @@ namespace SQLamarr //========================================================================== BaseSqlInterface::BaseSqlInterface(SQLite3DB& db) : m_database (db) + , m_queries () + , m_cached_raw_ptr (nullptr) { sqlamarr_create_sql_functions(db.get()); } @@ -29,9 +31,19 @@ namespace SQLamarr // Destructor //========================================================================== BaseSqlInterface::~BaseSqlInterface() + { + invalidate_cache(); + } + + //========================================================================== + // invalidate_cache + //========================================================================== + void BaseSqlInterface::invalidate_cache(void) { for (auto q = m_queries.begin(); q != m_queries.end(); ++q) sqlite3_finalize(q->second); + + m_queries.clear(); // Cache invalidation } //========================================================================== @@ -42,6 +54,12 @@ namespace SQLamarr const std::string& query ) { + if (m_database.get() != m_cached_raw_ptr) + { + m_cached_raw_ptr = m_database.get(); + invalidate_cache(); + } + if (m_queries.find(name) == m_queries.end()) m_queries[name] = prepare_statement(m_database, query); diff --git a/src/PVReconstruction.cpp b/src/PVReconstruction.cpp index 95aec9a..e5c0902 100644 --- a/src/PVReconstruction.cpp +++ b/src/PVReconstruction.cpp @@ -63,9 +63,7 @@ namespace SQLamarr ) : BaseSqlInterface(db) , m_parametrization (parametrization) - { - using_sql_function( "rnd_ggg", 6, &_sql_rnd_ggg ); - } + {} //============================================================================ // SQLite3 extension: rnd_ggg @@ -160,6 +158,8 @@ namespace SQLamarr //============================================================================ void PVReconstruction::execute () { + using_sql_function( "rnd_ggg", 6, &_sql_rnd_ggg ); + sqlite3_stmt* reco_pv = get_statement("reco_pv", R"( INSERT INTO Vertices ( mcvertex_id, genevent_id, diff --git a/src/UpdateDBConnection.cpp b/src/UpdateDBConnection.cpp new file mode 100644 index 0000000..f74ebbe --- /dev/null +++ b/src/UpdateDBConnection.cpp @@ -0,0 +1,47 @@ +// (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration. +// +// This software is distributed under the terms of the GNU General Public +// Licence version 3 (GPL Version 3), copied verbatim in the file "LICENCE". +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + + +// STL +#include +#include + +// SQLite3 +#include "sqlite3.h" + +// SQLamarr +#include "SQLamarr/UpdateDBConnection.h" +#include "SQLamarr/GlobalPRNG.h" +#include "SQLamarr/db_functions.h" + +namespace SQLamarr +{ + //============================================================================ + // Constructor + //============================================================================ + UpdateDBConnection::UpdateDBConnection( + SQLite3DB& db, + std::string filename, + int flags + ) + : m_database(db) + , m_filename(filename) + , m_flags(flags) + {} + + //============================================================================ + // execute + //============================================================================ + void UpdateDBConnection::execute() + { + update_db_connection(m_database, m_filename, m_flags); + } +} + + diff --git a/src/db_functions.cpp b/src/db_functions.cpp index 72e7ef1..52e397c 100644 --- a/src/db_functions.cpp +++ b/src/db_functions.cpp @@ -17,6 +17,7 @@ #include "SQLamarr/db_functions.h" #include "SQLamarr/GlobalPRNG.h" #include "SQLamarr/SQLiteError.h" +#include "SQLamarr/custom_sql_functions.h" #include "schema.sql" namespace SQLamarr @@ -110,9 +111,14 @@ namespace SQLamarr db, [](sqlite3* ptr) { SQLamarr::GlobalPRNG::release(ptr); - sqlite3_close(ptr); + int ret = sqlite3_close(ptr); + if (ret != SQLITE_OK) + { + std::cerr << "sqlite3_close returned errorcode: " << ret << std::endl; + throw (SQLiteError("Failed closing the connection")); } - ); + } + ); } //========================================================================== @@ -237,4 +243,27 @@ namespace SQLamarr } } + + //========================================================================== + // update_db_connection + //========================================================================== + void update_db_connection(SQLite3DB& old_db, const std::string& db_uri, int flags) + { + // Create the new connection to the database + SQLite3DB new_database = make_database(db_uri, flags); + + // Generate a seed for the new database, using the chain of random numbers + // of the previous one. + auto old_generator = GlobalPRNG::get_or_create(old_db.get()); + std::uniform_int_distribution distribution(0, 0xFFFFFFFFL); + uint64_t new_seed = distribution(*old_generator); + GlobalPRNG::get_or_create(new_database.get(), new_seed); + + // Define the SQLamarr-custom SQLite functions for the new connection + sqlamarr_create_sql_functions(new_database.get()); + + // Replace the old db connection with the new one, and then destroy it + old_db.swap(new_database); + } + } diff --git a/src/python_bindings.cpp b/src/python_bindings.cpp index 3019bb3..024c0f5 100644 --- a/src/python_bindings.cpp +++ b/src/python_bindings.cpp @@ -23,6 +23,7 @@ #include "SQLamarr/TemporaryTable.h" #include "SQLamarr/CleanEventStore.h" #include "SQLamarr/EditEventStore.h" +#include "SQLamarr/UpdateDBConnection.h" #include "SQLamarr/SQLiteError.h" constexpr int SQL_ERRORSHIFT = 10000; @@ -39,6 +40,7 @@ typedef enum { , TemporaryTable , CleanEventStore , EditEventStore + , UpdateDBConnection } TransformerType; struct TransformerPtr { @@ -47,6 +49,7 @@ struct TransformerPtr { }; SQLamarr::Transformer* resolve_polymorphic_transformer(TransformerPtr); +void clear_cache_if_any(TransformerPtr); std::vector tokenize (const char*); //============================================================================== @@ -101,11 +104,15 @@ void HepMC2DataLoader_load ( void *self, const char* file_path, size_t runNumber, - size_t evtNumber + size_t evtNumber, + const char *db_uri ) { - reinterpret_cast(self) - ->load(file_path, runNumber, evtNumber); + auto loader = reinterpret_cast(self); + loader->sync_database(db_uri); + loader->load(file_path, runNumber, evtNumber); + loader->invalidate_cache(); + loader->sync_database(db_uri); } @@ -254,6 +261,19 @@ TransformerPtr new_EditEventStore ( )}; } +//============================================================================== +// UpdateDBConnection +//============================================================================== +extern "C" +TransformerPtr new_UpdateDBConnection ( + void *db, + const char* path + ) +{ + SQLite3DB *udb = reinterpret_cast(db); + return {UpdateDBConnection, new SQLamarr::UpdateDBConnection(*udb, path)}; +} + //============================================================================== // Delete Transformer //============================================================================== @@ -286,6 +306,9 @@ void del_Transformer (TransformerPtr self) case EditEventStore: delete reinterpret_cast (self.p); break; + case UpdateDBConnection: + delete reinterpret_cast (self.p); + break; default: throw std::bad_cast(); } @@ -315,11 +338,49 @@ SQLamarr::Transformer* resolve_polymorphic_transformer(TransformerPtr self) return reinterpret_cast (self.p); case EditEventStore: return reinterpret_cast (self.p); + case UpdateDBConnection: + return reinterpret_cast (self.p); } throw std::bad_cast(); } +//============================================================================== +// clear_cache_if_any +//============================================================================== +void clear_cache_if_any(TransformerPtr self) +{ + switch (self.dtype) + { + case PVFinder: + reinterpret_cast (self.p)->invalidate_cache(); + break; + case MCParticleSelector: + reinterpret_cast (self.p)->invalidate_cache(); + break; + case PVReconstruction: + reinterpret_cast (self.p)->invalidate_cache(); + break; + case Plugin: + reinterpret_cast (self.p)->invalidate_cache(); + break; + case GenerativePlugin: + reinterpret_cast (self.p)->invalidate_cache(); + break; + case TemporaryTable: + reinterpret_cast (self.p)->invalidate_cache(); + break; + case CleanEventStore: + reinterpret_cast (self.p)->invalidate_cache(); + break; + case EditEventStore: + reinterpret_cast (self.p)->invalidate_cache(); + break; + case UpdateDBConnection: + break; + } +} + //============================================================================== // Execute Pipeline //============================================================================== @@ -334,6 +395,7 @@ int execute_pipeline(int algc, TransformerPtr* algv) try { t->execute(); + clear_cache_if_any(algv[iAlg]); } catch (const SQLamarr::SQLiteError& e) { diff --git a/src/version b/src/version index 3c31be0..67e892f 100644 --- a/src/version +++ b/src/version @@ -1 +1 @@ -"0.0c5" +"0.0c7"