- Added a getter for the filesystem in SmartBlocks to streamline acce…

…ss to smart sources.

- Implemented a process_load_queue method in AjsonMultiFileBlocksDataAdapter to log skipped loading actions.
- Enhanced the process_load_queue method in AjsonMultiFileCollectionDataAdapter to check for and create the data directory if it doesn't exist.
- Updated error handling in AjsonMultiFileItemDataAdapter to improve logging of loading errors.
- Modified the SmartSource import method to comment out the load_item_if_updated call, potentially altering import behavior.
- Ensured the SmartSources class processes the save queue for block collections, improving data management.

smart-blocks/adapters/data/ajson_multi_file.js

@@ -47,6 +47,10 @@ export class AjsonMultiFileBlocksDataAdapter extends AjsonMultiFileCollectionDat
     console.log(`Saved ${this.collection.collection_key} in ${Date.now() - time_start}ms`);
     this.collection.notices?.remove('saving');
   }
+  process_load_queue(){
+    // handled in sources
+    console.log(`Skipping loading ${this.collection.collection_key}...`);
+  }
 }
 
 

smart-blocks/smart_blocks.js

@@ -72,6 +72,7 @@ export class SmartBlocks extends SmartEntities {
 
 
 
+  get fs() { return this.env.smart_sources.fs; }
 
   /**
    * Retrieves the embedding model associated with the SmartSources collection.

smart-collections/adapters/ajson_multi_file.js

@@ -66,6 +66,12 @@ export class AjsonMultiFileCollectionDataAdapter extends FileCollectionDataAdapt
    */
   async process_load_queue() {
     this.collection.notices?.show('loading', `Loading ${this.collection.collection_key}...`, { timeout: 0 });
+
+    // check if directory exists
+    if(!(await this.fs.exists(this.collection.data_dir))){
+      // create directory
+      await this.fs.mkdir(this.collection.data_dir);
+    }
 
     const load_queue = Object.values(this.collection.items).filter(item => item._queue_load);
     if (!load_queue.length) {
@@ -194,7 +200,7 @@ export class AjsonMultiFileItemDataAdapter extends FileItemDataAdapter {
         else await this.fs.remove(this.data_path);
       }
     } catch (e) {
-      // console.warn("Error loading item (queueing import)", this.item.key, this.data_path, e);
+      console.warn("Error loading item (queueing import)", this.item.key, this.data_path, e);
       this.item.queue_import();
     }
   }

smart-entities/adapters/default.js

@@ -148,6 +148,7 @@ export class DefaultEntitiesVectorAdapter extends EntitiesVectorAdapter {
       // Update hash and stats
       batch.forEach(item => {
         item.embed_hash = item.read_hash;
+        item._queue_save = true;
       });
       this.embedded_total += batch.length;
       this.total_tokens += batch.reduce((acc, item) => acc + (item.tokens || 0), 0);
@@ -160,14 +161,15 @@ export class DefaultEntitiesVectorAdapter extends EntitiesVectorAdapter {
         this.last_save_total = this.embedded_total;
         await this.collection.process_save_queue();
         if(this.collection.block_collection) {
+          console.log(`Saving ${this.collection.block_collection.collection_key} block collection`);
           await this.collection.block_collection.process_save_queue();
         }
       }
     }
 
     // Show completion notice
     this._show_embed_completion_notice(embed_queue.length);
-    this.collection.process_save_queue();
+    await this.collection.process_save_queue();
     if(this.collection.block_collection) {
       await this.collection.block_collection.process_save_queue();
     }

smart-sources/smart_source.js

@@ -52,7 +52,7 @@ export class SmartSource extends SmartEntity {
   async import(){
     this._queue_import = false;
     try{
-      await this.data_adapter.load_item_if_updated(this);
+      // await this.data_adapter.load_item_if_updated(this);
       await this.source_adapter.import();
     }catch(err){
       if(err.code === "ENOENT"){

smart-sources/smart_sources.js

@@ -291,6 +291,7 @@ export class SmartSources extends SmartEntities {
     this.build_links_map();
     await this.process_embed_queue();
     await this.process_save_queue();
+    await this.block_collection?.process_save_queue();
   }
 
   /**