diff --git a/smart-blocks/adapters/data/ajson_multi_file.js b/smart-blocks/adapters/data/ajson_multi_file.js index abae4789..972a5e58 100644 --- a/smart-blocks/adapters/data/ajson_multi_file.js +++ b/smart-blocks/adapters/data/ajson_multi_file.js @@ -47,6 +47,10 @@ export class AjsonMultiFileBlocksDataAdapter extends AjsonMultiFileCollectionDat console.log(`Saved ${this.collection.collection_key} in ${Date.now() - time_start}ms`); this.collection.notices?.remove('saving'); } + process_load_queue(){ + // handled in sources + console.log(`Skipping loading ${this.collection.collection_key}...`); + } } diff --git a/smart-blocks/smart_blocks.js b/smart-blocks/smart_blocks.js index 0718f2fd..1e20517c 100644 --- a/smart-blocks/smart_blocks.js +++ b/smart-blocks/smart_blocks.js @@ -72,6 +72,7 @@ export class SmartBlocks extends SmartEntities { + get fs() { return this.env.smart_sources.fs; } /** * Retrieves the embedding model associated with the SmartSources collection. diff --git a/smart-collections/adapters/ajson_multi_file.js b/smart-collections/adapters/ajson_multi_file.js index a2dd1f73..2ed38624 100644 --- a/smart-collections/adapters/ajson_multi_file.js +++ b/smart-collections/adapters/ajson_multi_file.js @@ -66,6 +66,12 @@ export class AjsonMultiFileCollectionDataAdapter extends FileCollectionDataAdapt */ async process_load_queue() { this.collection.notices?.show('loading', `Loading ${this.collection.collection_key}...`, { timeout: 0 }); + + // check if directory exists + if(!(await this.fs.exists(this.collection.data_dir))){ + // create directory + await this.fs.mkdir(this.collection.data_dir); + } const load_queue = Object.values(this.collection.items).filter(item => item._queue_load); if (!load_queue.length) { @@ -194,7 +200,7 @@ export class AjsonMultiFileItemDataAdapter extends FileItemDataAdapter { else await this.fs.remove(this.data_path); } } catch (e) { - // console.warn("Error loading item (queueing import)", this.item.key, this.data_path, e); + console.warn("Error loading item (queueing import)", this.item.key, this.data_path, e); this.item.queue_import(); } } diff --git a/smart-entities/adapters/default.js b/smart-entities/adapters/default.js index af1e772d..81468ac3 100644 --- a/smart-entities/adapters/default.js +++ b/smart-entities/adapters/default.js @@ -148,6 +148,7 @@ export class DefaultEntitiesVectorAdapter extends EntitiesVectorAdapter { // Update hash and stats batch.forEach(item => { item.embed_hash = item.read_hash; + item._queue_save = true; }); this.embedded_total += batch.length; this.total_tokens += batch.reduce((acc, item) => acc + (item.tokens || 0), 0); @@ -160,6 +161,7 @@ export class DefaultEntitiesVectorAdapter extends EntitiesVectorAdapter { this.last_save_total = this.embedded_total; await this.collection.process_save_queue(); if(this.collection.block_collection) { + console.log(`Saving ${this.collection.block_collection.collection_key} block collection`); await this.collection.block_collection.process_save_queue(); } } @@ -167,7 +169,7 @@ export class DefaultEntitiesVectorAdapter extends EntitiesVectorAdapter { // Show completion notice this._show_embed_completion_notice(embed_queue.length); - this.collection.process_save_queue(); + await this.collection.process_save_queue(); if(this.collection.block_collection) { await this.collection.block_collection.process_save_queue(); } diff --git a/smart-sources/smart_source.js b/smart-sources/smart_source.js index 411f342e..221686dc 100644 --- a/smart-sources/smart_source.js +++ b/smart-sources/smart_source.js @@ -52,7 +52,7 @@ export class SmartSource extends SmartEntity { async import(){ this._queue_import = false; try{ - await this.data_adapter.load_item_if_updated(this); + // await this.data_adapter.load_item_if_updated(this); await this.source_adapter.import(); }catch(err){ if(err.code === "ENOENT"){ diff --git a/smart-sources/smart_sources.js b/smart-sources/smart_sources.js index f863d8c6..a6c56099 100644 --- a/smart-sources/smart_sources.js +++ b/smart-sources/smart_sources.js @@ -291,6 +291,7 @@ export class SmartSources extends SmartEntities { this.build_links_map(); await this.process_embed_queue(); await this.process_save_queue(); + await this.block_collection?.process_save_queue(); } /**