Added UpdateDBConnection to refresh DB connection for python clients

* First attempt of adding UpdateDBConnection

* Fixed several bugs due to caching of sqlite3 objects

---------

Co-authored-by: Lucio Anderlini <[email protected]>

.gitignore

@@ -4,3 +4,7 @@
 /.idea/
 *.swp
 /docs/
+/wheelhouse/
+/dist/
+**.egg-info
+**/__pycache__/

docs.md/Creating_the_wheel.md

@@ -7,6 +7,7 @@ older Linux distributions.
 Indeed, PyPI forbids uploading wheel that do not specify a compatibility tag
 used to choose the right wheel when running `pip install SQLamarr`.
 
+## Using docker 
 To compile the wheel in compatibility mode, we are using the following recipe,
 which assumes the working directory is the root of the git repository
 ```bash
@@ -15,7 +16,22 @@ docker run -it -v $PWD:/mylib quay.io/pypa/manylinux2014_x86_64:latest \
 python3 -m twine upload wheelhouse/*.whl
 ```
 
-Additional information:
+## Using Apptainer
+In some environments, docker is not available and apptainer should be used instead.
+
+First convert the docker image to an SIF image (this is only required once). 
+```bash
+apptainer build /tmp/pypa.sif docker://quay.io/pypa/manylinux2014_x86_64:latest
+```
+
+Then execute the following script (as for the docker case, it is assumed 
+the working directory is the root of the git repository).
+
+```build
+apptainer exec --bind $PWD:/mylib --writable-tmpfs /tmp/pypa.sif /bin/bash make_wheel.sh
+```
+
+## Additional information:
  * Distributing binaries in compatibility mode: https://github.com/pypa/manylinux
  * Example using travis for CI/CD: https://github.com/pypa/python-manylinux-demo
 

include/SQLamarr/BaseSqlInterface.h

@@ -13,7 +13,6 @@
 #include <string>
 #include <unordered_map>
 
-#include "SQLamarr/BaseSqlInterface.h"
 #include "SQLamarr/db_functions.h"
 
 namespace SQLamarr
@@ -39,12 +38,21 @@ namespace SQLamarr
       BaseSqlInterface(SQLite3DB& db);
       virtual ~BaseSqlInterface();
 
+      void sync_database(const std::string& db_uri)
+      {update_db_connection(m_database, db_uri);}
+
+      /// Invalidate the cache of the queries. 
+      /// Especially useful to allow refreshing the connection when running 
+      /// from Python.
+      void invalidate_cache(void);
+
     protected: // members
       SQLite3DB& m_database; ///< Reference to the SQLite database (not owned).
 
 
     private: //members
       std::unordered_map<std::string, sqlite3_stmt*> m_queries;
+      sqlite3* m_cached_raw_ptr;
 
     protected: // methods
       /// Creates or retrieve from cache a statement
@@ -60,7 +68,7 @@ namespace SQLamarr
       void begin_transaction () { sqlite3_exec(m_database.get(), "BEGIN", 0, 0, 0); }
 
       /// End an SQL transaction re-enabling disk updates
-      void end_transaction () { sqlite3_exec(m_database.get(), "END", 0, 0, 0); }
+      void end_transaction () { sqlite3_exec(m_database.get(), "COMMIT", 0, 0, 0); }
 
       /// Return the index of the last rows inserted in any table
       int last_insert_row () { return sqlite3_last_insert_rowid(m_database.get()); }

include/SQLamarr/UpdateDBConnection.h

@@ -0,0 +1,54 @@
+// (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration.
+//
+// This software is distributed under the terms of the GNU General Public
+// Licence version 3 (GPL Version 3), copied verbatim in the file "LICENCE".
+//
+// In applying this licence, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#pragma once
+
+// STL
+#include <string>
+
+// SQLamarr
+#include "SQLamarr/db_functions.h"
+#include "SQLamarr/Transformer.h"
+
+namespace SQLamarr
+{
+  /** Reset the database connection forcing flushing the db status.
+   *
+   * In the interaction with Python or other frameworks it is sometimes
+   * necessary to ensure db synchronization with disk or shared memory.
+   * This can be achieved refreshing the connection to the database, 
+   * by closing it and reopening. 
+   *
+   * WARNING! Executing UpdateDBConnection drops TEMPORARY tables and views. 
+   */
+  class UpdateDBConnection: public Transformer
+  {
+    public:
+      /// Constructor
+      UpdateDBConnection (
+          SQLite3DB& db, 
+            ///< Reference to the database
+          std::string filename,
+            ///< Filename or URI of the (possibly new) connection to the database
+          int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI
+            ///< Flags
+          );
+
+
+      /// Execute the algorithm, cleaning the database 
+      void execute () override;
+
+    private: // members
+      SQLite3DB& m_database; ///< Reference to the SQLite database (not owned).
+      const std::string m_filename; ///< Filename or URI of the database
+      const int m_flags; ///< SQLite flags to open to database (see sqlite_open_v3)
+  };
+}
+
+

include/SQLamarr/db_functions.h

@@ -36,6 +36,13 @@ namespace SQLamarr
 
   /// Ensure a token is alphanumeric
   void validate_token(const std::string& token);
+
+  /// Force synchronization to disk by closing and opening the connection
+  void update_db_connection(
+      SQLite3DB& old_db, 
+      const std::string& db_uri, 
+      int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI
+      );
 }
 
 

make_wheel.sh

@@ -7,7 +7,7 @@ cd /mylib;
 rm -f wheelhouse/*.whl | echo "Ok";
 rm -f dist/*.whl | echo "Ok";
 
-for PYVERSION in cp311-cp311 cp310-cp310 cp37-cp37m cp38-cp38 cp39-cp39;
+for PYVERSION in cp312-cp312 cp311-cp311 cp310-cp310 cp37-cp37m cp38-cp38 cp39-cp39;
 do
   export PYBIN=/opt/python/$PYVERSION/bin/python3
   echo "Preparing binary distribution with $PYBIN";

pyproject.toml

@@ -44,7 +44,7 @@ classifiers = [
   ]
 
 
-version = "0.0c5"
+version = "0.0c7"
 
 dependencies = ["hepmc3"]
 

python/SQLamarr/HepMC2DataLoader.py

@@ -54,19 +54,17 @@ class HepMC2DataLoader:
   """
   def __init__(self, db: SQLite3DB):
     """Acquires the reference to an open connection to the DB"""
-    self._self = clib.new_HepMC2DataLoader(db.get())
+    self._db = db
 
-  def __del__ (self):
-    """@private: Release the bound class instance"""
-    clib.del_HepMC2DataLoader(self._self)
-
   def load(self, filename: str, runNumber: int, evtNumber: int):
     """Loads an ASCII file with
     [HepMC3::ReaderAsciiHepMC2](http://hepmc.web.cern.ch/hepmc/classHepMC3_1_1ReaderAsciiHepMC2.html).
     """
     if not os.path.exists(filename):
       raise FileNotFoundError(filename)
 
+    _self = clib.new_HepMC2DataLoader(self._db.get())
     clib.HepMC2DataLoader_load(
-        self._self, filename.encode('ascii'), runNumber, evtNumber
+        _self, filename.encode('ascii'), runNumber, evtNumber, self._db.path.encode('ascii')
         )
+    clib.del_HepMC2DataLoader(_self)

python/SQLamarr/UpdateDBConnection.py

@@ -0,0 +1,43 @@
+# (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration. 
+#                                                                             
+# This software is distributed under the terms of the GNU General Public
+# Licence version 3 (GPL Version 3), copied verbatim in the file "LICENCE".
+#                                                                             
+# In applying this licence, CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as an Intergovernmental Organization  
+# or submit itself to any jurisdiction.
+import ctypes
+from ctypes import POINTER 
+from SQLamarr import clib, c_TransformerPtr
+
+from SQLamarr.db_functions import SQLite3DB
+
+clib.new_UpdateDBConnection.argtypes = (ctypes.c_void_p,)
+clib.new_UpdateDBConnection.restype = c_TransformerPtr
+
+class UpdateDBConnection:
+  """
+  Update the reference to the DB Connection, to ensure synchronization with disk.
+
+  Refer to SQLamarr::UpdateDBConnection for implementation details.
+  """
+  def __init__ (self, db: SQLite3DB):
+    """
+    Configure a Transformer to update the connection to the DB.
+
+    @param db: An open database connection.
+    """
+    self._self = clib.new_UpdateDBConnection(db.get(), db.path.encode('ascii'))
+
+  def __del__(self):
+    """@private: Release the bound class instance"""
+    clib.del_Transformer(self._self)
+
+  @property
+  def raw_pointer(self):
+    """@private: Return the raw pointer to the algorithm."""
+    return self._self
+
+
+
+

python/SQLamarr/__init__.py

@@ -40,6 +40,7 @@ class c_TransformerPtr (ctypes.Structure):
 from SQLamarr.TemporaryTable import TemporaryTable
 from SQLamarr.CleanEventStore import CleanEventStore
 from SQLamarr.EditEventStore import EditEventStore
+from SQLamarr.UpdateDBConnection import UpdateDBConnection
 
 ## Python Transfomer
 from SQLamarr.PyTransformer import PyTransformer

python/SQLamarr/db_functions.py

@@ -61,6 +61,10 @@ def __del__(self):
         """@private: Return the raw pointer to the algorithm."""
         clib.del_database(self._pointer)
 
+    @property
+    def path(self):
+      return self._path
+
     def get(self):
         """@private: Return the raw pointer to the database."""
         return self._pointer

src/BaseSqlInterface.cpp

@@ -21,6 +21,8 @@ namespace SQLamarr
   //==========================================================================
   BaseSqlInterface::BaseSqlInterface(SQLite3DB& db)
   : m_database (db)
+  , m_queries ()
+  , m_cached_raw_ptr (nullptr)
   {
     sqlamarr_create_sql_functions(db.get());
   }
@@ -29,9 +31,19 @@ namespace SQLamarr
   // Destructor
   //==========================================================================
   BaseSqlInterface::~BaseSqlInterface()
+  {
+    invalidate_cache();
+  }
+
+  //==========================================================================
+  // invalidate_cache
+  //==========================================================================
+  void BaseSqlInterface::invalidate_cache(void)
   {
     for (auto q = m_queries.begin(); q != m_queries.end(); ++q)
       sqlite3_finalize(q->second);
+
+    m_queries.clear();  // Cache invalidation
   }
 
   //==========================================================================
@@ -42,6 +54,12 @@ namespace SQLamarr
       const std::string& query
       )
   {
+    if (m_database.get() != m_cached_raw_ptr)
+    {
+      m_cached_raw_ptr = m_database.get();
+      invalidate_cache();
+    }
+
     if (m_queries.find(name) == m_queries.end())
       m_queries[name] = prepare_statement(m_database, query);
 

src/PVReconstruction.cpp

@@ -63,9 +63,7 @@ namespace SQLamarr
       )
     : BaseSqlInterface(db)
     , m_parametrization (parametrization)
-  {
-    using_sql_function( "rnd_ggg", 6, &_sql_rnd_ggg );
-  }
+  {}
 
   //============================================================================
   // SQLite3 extension: rnd_ggg
@@ -160,6 +158,8 @@ namespace SQLamarr
   //============================================================================
   void PVReconstruction::execute ()
   {
+    using_sql_function( "rnd_ggg", 6, &_sql_rnd_ggg );
+
     sqlite3_stmt* reco_pv = get_statement("reco_pv", R"(
       INSERT INTO Vertices (
         mcvertex_id, genevent_id, 

src/UpdateDBConnection.cpp

@@ -0,0 +1,47 @@
+// (c) Copyright 2022 CERN for the benefit of the LHCb Collaboration.
+//
+// This software is distributed under the terms of the GNU General Public
+// Licence version 3 (GPL Version 3), copied verbatim in the file "LICENCE".
+//
+// In applying this licence, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+
+// STL
+#include <memory>
+#include <random>
+
+// SQLite3
+#include "sqlite3.h"
+
+// SQLamarr
+#include "SQLamarr/UpdateDBConnection.h"
+#include "SQLamarr/GlobalPRNG.h"
+#include "SQLamarr/db_functions.h"
+
+namespace SQLamarr
+{
+  //============================================================================
+  // Constructor
+  //============================================================================
+  UpdateDBConnection::UpdateDBConnection(
+      SQLite3DB& db,
+      std::string filename,
+      int flags
+      )
+    : m_database(db)
+    , m_filename(filename)
+    , m_flags(flags)
+  {}
+
+  //============================================================================
+  // execute
+  //============================================================================
+  void UpdateDBConnection::execute()
+  {
+    update_db_connection(m_database, m_filename, m_flags);
+  }
+}
+
+