Fix pending diffs & p-diff-sync, and enable headless executor to do capability request

.circleci/config.yml

@@ -52,6 +52,9 @@ jobs:
       - run:
           name: Install dependencies
           command: pnpm install --no-frozen-lockfile
+      - run:
+          name: Build dapp
+          command: pnpm build-dapp
       - run:
           name: Install core dependencies
           command: cd ./core && pnpm install --no-frozen-lockfile

CHANGELOG

@@ -11,6 +11,7 @@ This project _loosely_ adheres to [Semantic Versioning](https://semver.org/spec/
  - Hard-wired Hosted ad4m client `AgentInfo` in the executor. [PR#453](https://github.com/coasys/ad4m/pull/453)
  - Added ability to handle multiple agents in launcher. [PR#459](https://github.com/coasys/ad4m/pull/459)
  - Added a way to show & add new `AgentInfo` in launcher. [PR#463](https://github.com/coasys/ad4m/pull/463)
+ - `ad4m-executor` binary prints capability request challange to stdout to enable app hand-shake [PR#471](https://github.com/coasys/ad4m/pull/471)
 
 ### Changed
  - Much improved ADAM Launcher setup flow [PR#440](https://github.com/coasys/ad4m/pull/440) and [PR#444](https://github.com/coasys/ad4m/pull/444):
@@ -43,6 +44,7 @@ This project _loosely_ adheres to [Semantic Versioning](https://semver.org/spec/
   - Fixed value returns as undefined if the property was boolean and set to false in `SubjectEntity.
   - Fixed links in docs.
 - Fixed flaky integration tests [PR#462](https://github.com/coasys/ad4m/pull/462)
+- Fixed `p-diff-sync`'s Deno incompatibilities [PR#471](https://github.com/coasys/ad4m/pull/471)
 
 ## [0.8.0] - 12/12/2023
 

bootstrap-languages/p-diff-sync/linksAdapter.ts

@@ -3,10 +3,11 @@ import { LinkSyncAdapter, PerspectiveDiffObserver, HolochainLanguageDelegate, La
 import type { SyncStateChangeObserver } from "https://esm.sh/@perspect3vism/[email protected]";
 import { Mutex, withTimeout } from "https://esm.sh/[email protected]";
 import { DNA_NICK, ZOME_NAME } from "./build/dna.js";
+import { encodeBase64 } from "https://deno.land/[email protected]/encoding/base64.ts";
 
 class PeerInfo {
   //@ts-ignore
-  currentRevision: Buffer;
+  currentRevision: Uint8Array;
   //@ts-ignore
   lastSeen: Date;
 };
@@ -19,7 +20,7 @@ export class LinkAdapter implements LinkSyncAdapter {
   generalMutex: Mutex = withTimeout(new Mutex(), 10000, new Error('PerspectiveDiffSync: generalMutex timeout'));
   me: DID
   gossipLogCount: number = 0;
-  myCurrentRevision: Buffer | null = null;
+  myCurrentRevision: Uint8Array | null = null;
 
   constructor(context: LanguageContext) {
     //@ts-ignore
@@ -56,8 +57,7 @@ export class LinkAdapter implements LinkSyncAdapter {
     try {
       //@ts-ignore
       let current_revision = await this.hcDna.call(DNA_NICK, ZOME_NAME, "sync", null);
-      console.log("PerspectiveDiffSync.sync(); current_revision", current_revision);
-      if (current_revision && Buffer.isBuffer(current_revision)) {
+      if (current_revision && current_revision instanceof Uint8Array) {
         this.myCurrentRevision = current_revision; 
       }
     } catch (e) {
@@ -90,13 +90,13 @@ export class LinkAdapter implements LinkSyncAdapter {
       peers.push(this.me);
 
       // Lexically sort the peers
-      peers.sort();
+      peers = peers.sort();
 
       // If we are the first peer, we are the scribe
-      let is_scribe = peers[0] == this.me;
+      let is_scribe = (peers[0] == this.me);
 
       // Get a deduped set of all peer's current revisions
-      let revisions = new Set<Buffer>();
+      let revisions = new Set<Uint8Array>();
       for(const peerInfo of this.peers.values()) {
         if (peerInfo.currentRevision) revisions.add(peerInfo.currentRevision);
       }
@@ -107,15 +107,15 @@ export class LinkAdapter implements LinkSyncAdapter {
       //Get a copied array of revisions that are different than mine
       let differentRevisions;
 
-      function generateRevisionStates(myCurrentRevision: Buffer) {
+      function generateRevisionStates(myCurrentRevision: Uint8Array) {
         sameRevisions = revisions.size == 0 ? [] : Array.from(revisions).filter( (revision) => {
-          return myCurrentRevision && revision.equals(myCurrentRevision);
+          return myCurrentRevision && (encodeBase64(revision) == encodeBase64(myCurrentRevision));
         });
         if (myCurrentRevision) {
           sameRevisions.push(myCurrentRevision);
         };
         differentRevisions = revisions.size == 0 ? [] : Array.from(revisions).filter( (revision) => {
-          return myCurrentRevision && !revision.equals(myCurrentRevision);
+          return myCurrentRevision && !(encodeBase64(revision) == encodeBase64(myCurrentRevision));
         });
       }
 
@@ -137,11 +137,13 @@ export class LinkAdapter implements LinkSyncAdapter {
 
       for (const hash of Array.from(revisions)) {
         if(!hash) continue
-        if (this.myCurrentRevision && hash.equals(this.myCurrentRevision)) continue
+        if (this.myCurrentRevision && (encodeBase64(hash) == encodeBase64(this.myCurrentRevision))) continue;
+
         let pullResult = await this.hcDna.call(DNA_NICK, ZOME_NAME, "pull", { 
           hash,
           is_scribe 
         });
+
         if (pullResult) {
           if (pullResult.current_revision && Buffer.isBuffer(pullResult.current_revision)) {
             let myRevision = pullResult.current_revision;
@@ -169,12 +171,12 @@ export class LinkAdapter implements LinkSyncAdapter {
         --
         ${Array.from(this.peers.entries()).map( ([peer, peerInfo]) => {
           //@ts-ignore
-          return `${peer}: ${peerInfo.currentRevision.toString('base64')} ${peerInfo.lastSeen.toISOString()}\n`
+          return `${peer}: ${encodeBase64(peerInfo.currentRevision)} ${peerInfo.lastSeen.toISOString()}\n`
         })}
         --
         revisions: ${Array.from(revisions).map( (hash) => {
           //@ts-ignore
-          return hash.toString('base64')
+          return encodeBase64(hash)
         })}
         `);
         this.gossipLogCount = 0;

cli/mainnet_seed.json

@@ -4,7 +4,7 @@
     "did:key:z6MkvPpWxwXAnLtMcoc9sX7GEoJ96oNnQ3VcQJRLspNJfpE7"
   ],
   "knownLinkLanguages": [
-    "QmzSYwdiSAPyaXzUN7Zgq6EMPEu8JQG3ck5UgH7REhAicWbVAo2",
+    "QmzSYwdj7LmLiY7p5vEzBPuQpW3CmAB41vEJe9hkVB9w6ndhcE5",
     "QmzSYwdnyTVrzufV8HfUfFRwDSiZZjRoBimrm95qjh6KCG9Z6YW",
     "QmzSYwdnHrRH8MmuPWKKrDvFoVyW5CophNpT1ipQUCcenPVTQnd"
   ],

cli/src/ad4m_executor.rs

@@ -181,7 +181,8 @@ async fn main() -> Result<()> {
                 hc_proxy_url,
                 hc_bootstrap_url,
                 connect_holochain,
-                admin_credential
+                admin_credential,
+                auto_permit_cap_requests: Some(true)
             }).await;
         }).await;
 

cli/src/dev.rs

@@ -48,6 +48,7 @@ pub async fn run(command: DevFunctions) -> Result<()> {
                     admin_credential: Some(String::from("*")),
                     hc_proxy_url: None,
                     hc_bootstrap_url: None,
+                    auto_permit_cap_requests: Some(true),
                 })
                 .await
                 .join()
@@ -178,6 +179,7 @@ pub async fn run(command: DevFunctions) -> Result<()> {
                     admin_credential: None,
                     hc_proxy_url: None,
                     hc_bootstrap_url: None,
+                    auto_permit_cap_requests: Some(true),
                 })
                 .await
                 .join()

rust-executor/src/config.rs

@@ -22,6 +22,7 @@ pub struct Ad4mConfig {
     pub hc_bootstrap_url: Option<String>,
     pub connect_holochain: Option<bool>,
     pub admin_credential: Option<String>,
+    pub auto_permit_cap_requests: Option<bool>,
 }
 
 impl Ad4mConfig {
@@ -96,6 +97,7 @@ impl Default for Ad4mConfig {
             hc_bootstrap_url: None,
             connect_holochain: None,
             admin_credential: None,
+            auto_permit_cap_requests: None,
         };
         config.prepare();
         config

rust-executor/src/dapp_server.rs

@@ -1,27 +1,27 @@
 use std::net::Ipv4Addr;
 use std::path::Path;
 
-use rocket::fs::{FileServer, relative};
-use rocket::{Config, Route, State};
+use rocket::fs::FileServer;
+use rocket::Config;
 use include_dir::{include_dir, Dir};
 
 const DAPP: Dir = include_dir!("dapp/dist");
 
-pub(crate) async fn serve_dapp(port: u16) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+pub(crate) async fn serve_dapp(port: u16, app_dir: String) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let config = Config {
         port,
         address: Ipv4Addr::new(127, 0, 0, 1).into(),
         ..Config::debug_default()
     };
 
-    let dir = relative!("dapp/dist");
-    if !Path::new(dir).exists() {
-        DAPP.extract("dapp/dist")?;
+    let dir = Path::new(&app_dir).join("dapp");
+    if !dir.exists() {
+        DAPP.extract(dir.clone())?;
     }
 
     rocket::build()
         .configure(&config)
-        .mount("/", FileServer::from(dir))
+        .mount("/", FileServer::from(dir.to_str().expect("Failed to convert path to string")))
         .launch()
         .await?;
 

rust-executor/src/graphql/graphql_types.rs

@@ -13,6 +13,7 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize};
 pub struct RequestContext {
     pub capabilities: Result<Vec<Capability>, String>,
     pub js_handle: JsCoreHandle,
+    pub auto_permit_cap_requests: bool,
 }
 
 #[derive(GraphQLObject, Default, Debug, Deserialize, Serialize, Clone)]

rust-executor/src/graphql/mod.rs

@@ -67,6 +67,7 @@ pub async fn start_server(js_core_handle: JsCoreHandle, config: Ad4mConfig) -> R
             RequestContext {
                 capabilities,
                 js_handle: js_core_handle_cloned1.clone(),
+                auto_permit_cap_requests: config.auto_permit_cap_requests.clone().unwrap_or(false),
             }
         });
     let qm_graphql_filter = coasys_juniper_warp::make_graphql_filter(qm_schema, qm_state.boxed());
@@ -80,11 +81,12 @@ pub async fn start_server(js_core_handle: JsCoreHandle, config: Ad4mConfig) -> R
             let root_node = root_node.clone();
             let js_core_handle = js_core_handle.clone();
             let admin_credential_arc = admin_credential_arc.clone();
+            let auto_permit_cap_requests = config.auto_permit_cap_requests.clone().unwrap_or(false);
             ws.on_upgrade(move |websocket| async move {
                 serve_graphql_transport_ws(
                     websocket,
                     root_node,
-                    |val: HashMap<String, InputValue>| async move {
+                    move |val: HashMap<String, InputValue>| async move {
                         let mut auth_header = String::from("");
 
                         if let Some(headers) = val.get("headers") {
@@ -102,6 +104,7 @@ pub async fn start_server(js_core_handle: JsCoreHandle, config: Ad4mConfig) -> R
                         let context = RequestContext {
                             capabilities,
                             js_handle: js_core_handle.clone(),
+                            auto_permit_cap_requests: auto_permit_cap_requests
                         };
                         Ok(ConnectionConfig::new(context))
                             as Result<ConnectionConfig<_>, Infallible>

rust-executor/src/graphql/mutation_resolvers.rs

@@ -154,7 +154,21 @@ impl Mutation {
         auth_info: AuthInfoInput,
     ) -> FieldResult<String> {
         check_capability(&context.capabilities, &AGENT_AUTH_CAPABILITY)?;
-        Ok(agent::capabilities::request_capability(auth_info.into()).await)
+        let auth_info: AuthInfo = auth_info.into();
+        let request_id = agent::capabilities::request_capability(auth_info.clone()).await;
+        if true == context.auto_permit_cap_requests {
+            println!("======================================");
+            println!("Got capability request: \n{:?}", auth_info);
+            let random_number_challenge = agent::capabilities::permit_capability(AuthInfoExtended {
+                request_id: request_id.clone(), 
+                auth: auth_info
+            })?;
+            println!("--------------------------------------");
+            println!("Random number challenge: {}", random_number_challenge);
+            println!("======================================");
+        } 
+
+        Ok(request_id)
     }
 
     //NOTE: all the functions from here on out have not been tested by calling the cli <-> rust graphql server

rust-executor/src/holochain_service/mod.rs

@@ -387,9 +387,9 @@ impl HolochainService {
             .build()
             .await;
 
-        if conductor.is_err() {
-            info!("Could not start holochain conductor: {:#?}", conductor.err());
-            panic!("Could not start holochain conductor");
+        if let Err(e) = conductor {
+            info!("Could not start holochain conductor: {:#?}", e);
+            panic!("Could not start holochain conductor: {:#?}", e);
         }
 
         info!("Started holochain conductor");

rust-executor/src/languages/language.rs

@@ -12,7 +12,7 @@ fn parse_revision(js_result: String) -> Result<Option<String>, AnyError> {
     if let Ok(maybe_revision) = serde_json::from_str::<Option<ByteArray>>(&js_result) {
         Ok(maybe_revision.map(|revision| {
             let vec: Vec<u8> = revision.into();
-            String::from_utf8(vec).unwrap()
+            base64::encode(&vec)
         }))
     } else {
         Ok(serde_json::from_str::<Option<String>>(&js_result)?)

rust-executor/src/lib.rs

@@ -26,7 +26,7 @@ mod test_utils;
 
 use tokio;
 use std::{env, thread::JoinHandle};
-use log::{info, warn};
+use log::{info, warn, error};
 
 use js_core::JsCore;
 
@@ -79,14 +79,21 @@ pub async fn run(mut config: Ad4mConfig) -> JoinHandle<()> {
 
     info!("Starting GraphQL...");
 
+    let app_dir = config.app_data_path
+        .as_ref()
+        .expect("App data path not set in Ad4mConfig")
+        .clone();
+
     if let Some(true) = config.run_dapp_server {
         std::thread::spawn(|| {
             let runtime = tokio::runtime::Builder::new_multi_thread()
                 .thread_name(String::from("dapp_server"))
                 .enable_all()
                 .build()
                 .unwrap();
-            let _ = runtime.block_on(serve_dapp(8080));
+            if let Err(e) = runtime.block_on(serve_dapp(8080, app_dir)) {
+                error!("Failed to start dapp server: {:?}", e);
+            }
         });
     };
 

rust-executor/src/mainnet_seed.json

@@ -4,7 +4,7 @@
     "did:key:z6MkvPpWxwXAnLtMcoc9sX7GEoJ96oNnQ3VcQJRLspNJfpE7"
   ],
   "knownLinkLanguages": [
-    "QmzSYwdiSAPyaXzUN7Zgq6EMPEu8JQG3ck5UgH7REhAicWbVAo2",
+    "QmzSYwdj7LmLiY7p5vEzBPuQpW3CmAB41vEJe9hkVB9w6ndhcE5",
     "QmzSYwdnyTVrzufV8HfUfFRwDSiZZjRoBimrm95qjh6KCG9Z6YW",
     "QmzSYwdnHrRH8MmuPWKKrDvFoVyW5CophNpT1ipQUCcenPVTQnd"
   ],

rust-executor/src/perspectives/perspective_instance.rs

@@ -144,7 +144,13 @@ impl PerspectiveInstance {
                     if link_language.current_revision().await.map_err(|e| anyhow!("current_revision error: {}",e))?.is_some() {
                         // Ok, we are synced and have a revision. Let's commit our pending diffs.
                         let pending_diffs = Ad4mDb::with_global_instance(|db| db.get_pending_diffs(&uuid)).map_err(|e| anyhow!("get_pending_diffs error: {}",e))?;
+
+                        if pending_diffs.additions.is_empty() && pending_diffs.removals.is_empty() {
+                            return Ok(());
+                        }
+                        log::info!("Found pending diffs: {:?}\n Committing...", pending_diffs);
                         let commit_result = link_language.commit(pending_diffs).await;
+                        log::info!("Pending diffs commit result: {:?}", commit_result);
                         return match commit_result {
                             Ok(Some(_)) => {
                                 Ad4mDb::with_global_instance(|db| db.clear_pending_diffs(&uuid)).map_err(|e| anyhow!("clear_pending_diffs error: {}",e))?;

turbo.json

@@ -6,7 +6,7 @@
       "outputs": ["dist/**", "lib/**", "build/**"]
     },
     "build-libs": {
-      "dependsOn": ["@coasys/ad4m#build", "@coasys/ad4m-connect#build", "@coasys/ad4m-executor#build", "@coasys/ad4m-cli#build", "@coasys/dapp#build"],
+      "dependsOn": ["@coasys/dapp#build", "@coasys/ad4m#build", "@coasys/ad4m-connect#build", "@coasys/ad4m-executor#build", "@coasys/ad4m-cli#build"],
       "outputs": ["dist/**", "lib/**", "build/**"]
     },
     "build-core-executor": {
@@ -28,6 +28,16 @@
       "outputs": ["dist/**", "lib/**", "build/**"]
     },
 
+    "rust-ad4m-executor#build": {
+      "dependsOn": ["@coasys/dapp#build"],
+      "outputs": ["dist/**", "lib/**", "build/**"]
+    },
+
+    "ad4m-cli#build": {
+      "dependsOn": ["@coasys/dapp#build"],
+      "outputs": ["dist/**", "lib/**", "build/**"]
+    },
+
     "ad4m-launcher#package-ad4m": {
       "dependsOn": ["build-libs"],
       "outputs": ["dist/**"]