Transactional api integration

.circleci/config.yml

@@ -270,6 +270,7 @@ workflows:
         only:
          - develop
          - final-exam
+         - transactional
          - wire-message-receipts
    - docker-build-trycp-server:
       requires:
@@ -284,6 +285,7 @@ workflows:
         only:
          - develop
          - final-exam
+         - transactional
          - wire-message-receipts
    - docker-build-circle-build:
       requires:

crates/cli/Cargo.toml

@@ -24,8 +24,8 @@ in_stream = { version = "=0.0.43-alpha3", path = "../in_stream" }
 url2 = "=0.0.4"
 lib3h_sodium = "=0.0.38"
 holochain_json_api = "=0.0.23"
-holochain_persistence_api = "=0.0.17"
-holochain_persistence_file = "=0.0.17"
+holochain_persistence_api = { git = "https://github.com/holochain/holochain-persistence", branch = "transactional" }
+holochain_persistence_file = { git = "https://github.com/holochain/holochain-persistence", branch = "transactional" }
 holochain_wasm_utils = { path = "../wasm_utils" }
 crossbeam-channel = "=0.3.8"
 structopt = "=0.3.3"
@@ -45,7 +45,7 @@ rustyline = "=5.0.0"
 json-patch = "=0.2.2"
 reqwest = "=0.9.11"
 tempfile = "=3.0.7"
-lib3h_protocol = "=0.0.38"
+lib3h_protocol = { git = "https://github.com/holochain/lib3h", branch = "transactional" }
 tar = "=0.4.26"
 flate2 = "=1.0.12"
 log = "=0.4.8"

crates/cli/src/cli/chain_log.rs

@@ -8,9 +8,7 @@ use holochain_core::{
     content_store::GetContent,
 };
 use holochain_core_types::{chain_header::ChainHeader, entry::Entry};
-use holochain_locksmith::RwLock;
 use holochain_persistence_api::cas::content::Address;
-use holochain_persistence_file::cas::file::FilesystemStorage;
 use std::{fs, path::PathBuf};
 
 // TODO: use system-agnostic default path
@@ -21,10 +19,13 @@ pub fn chain_log(storage_path: Option<PathBuf>, instance_id: String) -> DefaultR
     let storage_path = storage_path.ok_or_else(|| {
         format_err!("Please specify the path to CAS storage with the --path option.")
     })?;
-    let cas_path = storage_path.join(instance_id).join("cas");
-    let chain_store = ChainStore::new(std::sync::Arc::new(RwLock::new(
-        FilesystemStorage::new(cas_path.clone()).expect("Could not create chain store"),
-    )));
+    let cas_path = storage_path.join(instance_id.clone()).join("cas");
+    let eav_path = storage_path.join(instance_id).join("eav");
+
+    let chain_store = ChainStore::new(std::sync::Arc::new(
+        holochain_persistence_file::txn::new_manager(cas_path.clone(), eav_path)
+            .expect("Could not create chain store"),
+    ));
 
     let agent = chain_store
         .get_raw(&Address::from("AgentState"))?

crates/cli/src/main.rs

@@ -5,10 +5,8 @@ extern crate holochain_conductor_lib;
 extern crate holochain_core;
 extern crate holochain_core_types;
 extern crate holochain_json_api;
-extern crate holochain_locksmith;
 extern crate holochain_net;
 extern crate holochain_persistence_api;
-extern crate holochain_persistence_file;
 extern crate json_patch;
 extern crate lib3h_crypto_api;
 extern crate lib3h_protocol;

crates/conductor_lib/Cargo.toml

@@ -14,18 +14,18 @@ holochain_core_types = { version = "=0.0.43-alpha3", path = "../core_types" }
 holochain_locksmith = { version = "=0.0.43-alpha3", path = "../locksmith" }
 holochain_json_derive = "=0.0.23"
 holochain_json_api = "=0.0.23"
-holochain_persistence_api = "=0.0.17"
-holochain_persistence_mem = "=0.0.17"
-holochain_persistence_file = "=0.0.17"
-holochain_persistence_pickle = "=0.0.17"
-holochain_persistence_lmdb = "=0.0.17"
+holochain_persistence_api = { git = "https://github.com/holochain/holochain-persistence", branch = "transactional" }
+holochain_persistence_mem = { git = "https://github.com/holochain/holochain-persistence", branch = "transactional" }
+holochain_persistence_file = { git = "https://github.com/holochain/holochain-persistence", branch = "transactional" }
+holochain_persistence_pickle = { git = "https://github.com/holochain/holochain-persistence", branch = "transactional" }
+holochain_persistence_lmdb = { git = "https://github.com/holochain/holochain-persistence", branch = "transactional" }
 holochain_common = { version = "=0.0.43-alpha3", path = "../common" }
 holochain_dpki = { version = "=0.0.43-alpha3", path = "../dpki" }
 holochain_net = { version = "=0.0.43-alpha3", path = "../net" }
 holochain_tracing = "=0.0.19"
 holochain_tracing_macros = "=0.0.19"
-lib3h = "=0.0.38"
-lib3h_sodium = "=0.0.38"
+lib3h = { git = "https://github.com/holochain/lib3h", branch = "transactional" }
+lib3h_sodium = { git = "https://github.com/holochain/lib3h", branch = "transactional" }
 holochain_metrics = { version = "=0.0.43-alpha3", path = "../metrics" }
 chrono = "=0.4.6"
 serde = "=1.0.104"
@@ -54,8 +54,8 @@ log = "=0.4.8"
 holochain_logging = "=0.0.7"
 nickel = "=0.11.0"
 url = { version = "=2.1.0", features = ["serde"] }
-snowflake = "=1.3.0"
 newrelic="0.2"
+
 [dev-dependencies]
 test_utils = { version = "=0.0.43-alpha3", path = "../../test_utils" }
 tempfile = "=3.0.7"

crates/conductor_lib/src/conductor/admin.rs

@@ -219,6 +219,7 @@ impl ConductorAdmin for Conductor {
             None | Some("lmdb") => StorageConfiguration::Lmdb {
                 path: storage_path,
                 initial_mmap_bytes: None,
+                staging_path_prefix: None,
             },
             Some(s) => {
                 return Err(HolochainError::ConfigError(format!(

crates/conductor_lib/src/conductor/base.rs

@@ -866,10 +866,10 @@ impl Conductor {
                                     format!("Error creating context: {}", hc_err.to_string())
                                 })?
                     }
-                    StorageConfiguration::Lmdb { path, initial_mmap_bytes } => {
+                    StorageConfiguration::Lmdb { path, initial_mmap_bytes, staging_path_prefix } => {
                         context_builder =
                             context_builder
-                                .with_lmdb_storage(path, initial_mmap_bytes)
+                                .with_lmdb_storage(path, staging_path_prefix, initial_mmap_bytes)
                                 .map_err(|hc_err| {
                                     format!("Error creating context: {}", hc_err.to_string())
                                 })?

crates/conductor_lib/src/config.rs

@@ -741,6 +741,7 @@ pub enum StorageConfiguration {
     Lmdb {
         path: String,
         initial_mmap_bytes: Option<usize>,
+        staging_path_prefix: Option<String>,
     },
 }
 

crates/conductor_lib/src/context_builder.rs

@@ -2,13 +2,7 @@ use holochain_core::{context::Context, persister::SimplePersister, signal::Signa
 use holochain_core_types::{agent::AgentId, eav::Attribute, error::HolochainError};
 use holochain_locksmith::RwLock;
 use holochain_net::p2p_config::P2pConfig;
-use holochain_persistence_api::{
-    cas::storage::ContentAddressableStorage, eav::EntityAttributeValueStorage,
-};
-use holochain_persistence_file::{cas::file::FilesystemStorage, eav::file::EavFileStorage};
-use holochain_persistence_lmdb::{cas::lmdb::LmdbStorage, eav::lmdb::EavLmdbStorage};
-use holochain_persistence_mem::{cas::memory::MemoryStorage, eav::memory::EavMemoryStorage};
-use holochain_persistence_pickle::{cas::pickle::PickleStorage, eav::pickle::EavPickleStorage};
+use holochain_persistence_api::txn::PersistenceManagerDyn;
 use holochain_tracing;
 
 use jsonrpc_core::IoHandler;
@@ -34,9 +28,7 @@ pub struct ContextBuilder {
     // Persister is currently set to a reasonable default in spawn().
     // TODO: add with_persister() function to ContextBuilder.
     //persister: Option<Arc<Mutex<Persister>>>,
-    chain_storage: Option<Arc<RwLock<dyn ContentAddressableStorage>>>,
-    dht_storage: Option<Arc<RwLock<dyn ContentAddressableStorage>>>,
-    eav_storage: Option<Arc<RwLock<dyn EntityAttributeValueStorage<Attribute>>>>,
+    persistence_manager: Option<Arc<dyn PersistenceManagerDyn<Attribute>>>,
     p2p_config: Option<P2pConfig>,
     conductor_api: Option<Arc<RwLock<IoHandler>>>,
     signal_tx: Option<SignalSender>,
@@ -50,9 +42,7 @@ impl ContextBuilder {
         ContextBuilder {
             instance_name: None,
             agent_id: None,
-            chain_storage: None,
-            dht_storage: None,
-            eav_storage: None,
+            persistence_manager: None,
             p2p_config: None,
             conductor_api: None,
             signal_tx: None,
@@ -71,12 +61,8 @@ impl ContextBuilder {
     /// Sets all three storages, chain, DHT and EAV storage, to transient memory implementations.
     /// Chain and DHT storages get set to the same memory CAS.
     pub fn with_memory_storage(mut self) -> Self {
-        let cas = Arc::new(RwLock::new(MemoryStorage::new()));
-        let eav = //Arc<RwLock<holochain_persistence_api::eav::EntityAttributeValueStorage<Attribute>>> =
-            Arc::new(RwLock::new(EavMemoryStorage::new()));
-        self.chain_storage = Some(cas.clone());
-        self.dht_storage = Some(cas);
-        self.eav_storage = Some(eav);
+        let persistence_manager = Arc::new(holochain_persistence_mem::txn::new_manager());
+        self.persistence_manager = Some(persistence_manager);
         self
     }
 
@@ -90,12 +76,10 @@ impl ContextBuilder {
         fs::create_dir_all(&cas_path)?;
         fs::create_dir_all(&eav_path)?;
 
-        let file_storage = Arc::new(RwLock::new(FilesystemStorage::new(&cas_path)?));
-        let eav_storage: Arc<RwLock<dyn EntityAttributeValueStorage<Attribute>>> =
-            Arc::new(RwLock::new(EavFileStorage::new(eav_path)?));
-        self.chain_storage = Some(file_storage.clone());
-        self.dht_storage = Some(file_storage);
-        self.eav_storage = Some(eav_storage);
+        let persistence_manager: Arc<dyn PersistenceManagerDyn<Attribute>> =
+            Arc::new(holochain_persistence_file::txn::new_manager(cas_path, eav_path).unwrap());
+
+        self.persistence_manager = Some(persistence_manager);
         Ok(self)
     }
 
@@ -108,37 +92,34 @@ impl ContextBuilder {
         let eav_path = base_path.join("eav");
         fs::create_dir_all(&cas_path)?;
         fs::create_dir_all(&eav_path)?;
-
-        let file_storage = Arc::new(RwLock::new(PickleStorage::new(&cas_path)));
-        let eav_storage = Arc::new(RwLock::new(EavPickleStorage::new(eav_path)));
-        self.chain_storage = Some(file_storage.clone());
-        self.dht_storage = Some(file_storage);
-        self.eav_storage = Some(eav_storage);
+        let persistence_manager: Arc<dyn PersistenceManagerDyn<Attribute>> = Arc::new(
+            holochain_persistence_pickle::txn::new_manager(cas_path, eav_path),
+        );
+        self.persistence_manager = Some(persistence_manager);
         Ok(self)
     }
 
     /// Sets all three storages, chain, DHT and EAV storage, to persistent lmdb based implementations.
     /// Chain and DHT storages get set to the same pikcle CAS.
     /// Returns an error if no lmdb storage could be spawned on the given path.
-    pub fn with_lmdb_storage<P: AsRef<Path>>(
+    pub fn with_lmdb_storage<P: AsRef<Path>, P2: AsRef<Path> + Clone>(
         mut self,
         path: P,
+        staging_path_prefix: Option<P2>,
         initial_mmap_bytes: Option<usize>,
     ) -> Result<Self, HolochainError> {
-        let base_path: PathBuf = path.as_ref().into();
-        let cas_path = base_path.join("cas");
-        let eav_path = base_path.join("eav");
-        fs::create_dir_all(&cas_path)?;
-        fs::create_dir_all(&eav_path)?;
+        let env_path: PathBuf = path.as_ref().into();
 
-        let cas_storage = Arc::new(RwLock::new(LmdbStorage::new(&cas_path, initial_mmap_bytes)));
-        let eav_storage = Arc::new(RwLock::new(EavLmdbStorage::new(
-            eav_path,
-            initial_mmap_bytes,
-        )));
-        self.chain_storage = Some(cas_storage.clone());
-        self.dht_storage = Some(cas_storage);
-        self.eav_storage = Some(eav_storage);
+        let persistence_manager: Arc<dyn PersistenceManagerDyn<Attribute>> =
+            Arc::new(holochain_persistence_lmdb::txn::new_manager(
+                env_path,
+                staging_path_prefix,
+                initial_mmap_bytes,
+                None,
+                None,
+                None,
+            ));
+        self.persistence_manager = Some(persistence_manager);
         Ok(self)
     }
 
@@ -199,15 +180,9 @@ impl ContextBuilder {
     /// Defaults to memory storages, an in-memory network config and a fake agent called "alice".
     /// The persister gets set to SimplePersister based on the chain storage.
     pub fn spawn(self) -> Context {
-        let chain_storage = self
-            .chain_storage
-            .unwrap_or_else(|| Arc::new(RwLock::new(MemoryStorage::new())));
-        let dht_storage = self
-            .dht_storage
-            .unwrap_or_else(|| Arc::new(RwLock::new(MemoryStorage::new())));
-        let eav_storage = self
-            .eav_storage
-            .unwrap_or_else(|| Arc::new(RwLock::new(EavMemoryStorage::new())));
+        let persistence_manager = self
+            .persistence_manager
+            .unwrap_or_else(|| Arc::new(holochain_persistence_mem::txn::new_manager()));
         let metric_publisher = self
             .metric_publisher
             .unwrap_or_else(|| Arc::new(RwLock::new(DefaultMetricPublisher::default())));
@@ -218,10 +193,10 @@ impl ContextBuilder {
                 .unwrap_or_else(|| "Anonymous-instance".to_string()),
             self.agent_id
                 .unwrap_or_else(|| AgentId::generate_fake("alice")),
-            Arc::new(RwLock::new(SimplePersister::new(chain_storage.clone()))),
-            chain_storage,
-            dht_storage,
-            eav_storage,
+            Arc::new(RwLock::new(SimplePersister::new(
+                persistence_manager.clone(),
+            ))),
+            persistence_manager,
             // TODO BLOCKER pass a peer list here?
             self.p2p_config
                 .unwrap_or_else(P2pConfig::new_with_unique_memory_backend),

crates/conductor_lib/src/holochain.rs

@@ -179,7 +179,7 @@ impl Holochain {
     }
 
     pub fn load(context: Arc<Context>) -> Result<Self, HolochainError> {
-        let persister = SimplePersister::new(context.dht_storage.clone());
+        let persister = SimplePersister::new(context.persistence_manager.clone());
         let loaded_state = persister.load(context.clone())?.ok_or_else(|| {
             HolochainError::ErrorGeneric("State could not be loaded due to NoneError".to_string())
         })?;

crates/conductor_lib/test-bridge-caller/Cargo.toml

@@ -18,4 +18,4 @@ serde_json = { version = "=1.0.47", features = ["preserve_order"] }
 hdk = { path = "../../hdk" }
 serde_derive = "=1.0.104"
 holochain_json_derive = "=0.0.23"
-holochain_persistence_api = "=0.0.17"
+holochain_persistence_api = { git = "https://github.com/holochain/holochain-persistence", branch = "transactional" }

crates/core/Cargo.toml

@@ -25,17 +25,17 @@ num-traits = "=0.2.6"
 num-derive = "=0.2.4"
 toml = "=0.5.0"
 futures = { version = "=0.3.2", features = [ "thread-pool" ] }
+lib3h_protocol = { git = "https://github.com/holochain/lib3h", branch = "transactional" }
+lib3h_sodium = { git = "https://github.com/holochain/lib3h", branch = "transactional" }
 holochain_net = { version = "=0.0.43-alpha3", path = "../net" }
 holochain_wasm_utils = { version = "=0.0.43-alpha3", path = "../wasm_utils" }
 holochain_common = { version = "=0.0.43-alpha3", path = "../common" }
 holochain_conductor_lib_api = { version = "=0.0.43-alpha3", path = "../conductor_api" }
-lib3h_protocol = "=0.0.38"
-lib3h_sodium = "=0.0.38"
 holochain_json_derive = "=0.0.23"
 holochain_json_api = "=0.0.23"
-holochain_persistence_api = "=0.0.17"
-holochain_persistence_file = "=0.0.17"
-holochain_persistence_mem = "=0.0.17"
+holochain_persistence_api = { git = "https://github.com/holochain/holochain-persistence", branch = "transactional" }
+holochain_persistence_file = { git = "https://github.com/holochain/holochain-persistence", branch = "transactional" }
+holochain_persistence_mem = { git = "https://github.com/holochain/holochain-persistence", branch = "transactional" }
 holochain_core_types = { version = "=0.0.43-alpha3", path = "../core_types" }
 holochain_dpki = { version = "=0.0.43-alpha3", path = "../dpki" }
 holochain_locksmith = { version = "=0.0.43-alpha3", path = "../locksmith" }
@@ -65,4 +65,4 @@ newrelic="0.2"
 wabt = "=0.7.4"
 test_utils = { version = "=0.0.43-alpha3", path = "../../test_utils" }
 tempfile = "=3.0.7"
-holochain_persistence_lmdb = "=0.0.17"
+holochain_persistence_lmdb = { git = "https://github.com/holochain/holochain-persistence", branch = "transactional" }