From 54906a5297a5590786d077672da11b57d9f6ac5f Mon Sep 17 00:00:00 2001 From: WFH Brian Date: Thu, 3 Oct 2024 11:44:57 -0400 Subject: [PATCH] work on test cases --- smart-collections/adapters/_adapter.js | 36 +- smart-collections/adapters/_test.js | 102 ++++-- smart-collections/adapters/multi_file.js | 168 ++++----- smart-collections/test/_env.js | 8 +- smart-embed-model-v1/smart_embed_model.js | 2 +- smart-embed-model/smart_embed_model.js | 2 +- smart-entities/test/_env.js | 8 +- smart-entities/test/env.test.js | 4 +- smart-environment/smart_env.js | 8 + smart-fs/adapters/_test.js | 146 +++++--- smart-fs/smart_fs.test.js | 75 ++++- smart-settings/smart_settings.test.js | 4 +- smart-sources-v1/adapters/_test.js | 2 +- smart-sources-v1/test/_env.js | 12 +- smart-sources-v1/test/env.test.js | 4 +- smart-sources/adapters/_test.js | 328 +++++++++++++++++- smart-sources/adapters/markdown.js | 39 +-- smart-sources/blocks/markdown.test.js | 393 ++++++++++++++++++++++ smart-sources/test/_env.js | 49 ++- smart-sources/test/env.test.js | 4 +- smart-sources/utils/get_line_range.js | 4 + smart-sources/utils/get_markdown_links.js | 27 ++ 22 files changed, 1155 insertions(+), 270 deletions(-) create mode 100644 smart-sources/blocks/markdown.test.js create mode 100644 smart-sources/utils/get_line_range.js create mode 100644 smart-sources/utils/get_markdown_links.js diff --git a/smart-collections/adapters/_adapter.js b/smart-collections/adapters/_adapter.js index 01d26fc9..083f2f14 100644 --- a/smart-collections/adapters/_adapter.js +++ b/smart-collections/adapters/_adapter.js @@ -1,20 +1,3 @@ -export class SmartCollectionItemDataAdapter{ - constructor(item) { - this.item = item; - } - // REQUIRED METHODS IN SUBCLASSES - async load() { throw new Error("SmartCollectionItemAdapter: load() not implemented"); } - async save() { throw new Error("SmartCollectionItemAdapter: save() not implemented"); } - // END REQUIRED METHODS IN SUBCLASSES - - get env() { return this.item.env; } - get data_path() { return 'collection_item.json'; } - get key() { return this.item.key; } - get collection_key() { return this.item.collection_key; } - get collection() { return this.item.collection; } - -} - export class SmartCollectionDataAdapter{ constructor(collection) { this.collection = collection; @@ -28,4 +11,21 @@ export class SmartCollectionDataAdapter{ get data_path() { return 'collection.json'; } get collection_key() { return this.collection.collection_key; } -} \ No newline at end of file +} + +// export class SmartCollectionItemDataAdapter{ +// constructor(item) { +// this.item = item; +// } +// // REQUIRED METHODS IN SUBCLASSES +// async load() { throw new Error("SmartCollectionItemAdapter: load() not implemented"); } +// async save() { throw new Error("SmartCollectionItemAdapter: save() not implemented"); } +// // END REQUIRED METHODS IN SUBCLASSES + +// get env() { return this.item.env; } +// get data_path() { return 'collection_item.json'; } +// get key() { return this.item.key; } +// get collection_key() { return this.item.collection_key; } +// get collection() { return this.item.collection; } + +// } \ No newline at end of file diff --git a/smart-collections/adapters/_test.js b/smart-collections/adapters/_test.js index 128c29da..c40c5be6 100644 --- a/smart-collections/adapters/_test.js +++ b/smart-collections/adapters/_test.js @@ -1,30 +1,92 @@ import fs from 'fs'; -import { SmartCollectionItemDataAdapter } from './_adapter.js'; -export class TestSmartCollectionDataAdapter extends SmartCollectionItemDataAdapter{ +import { SmartCollectionDataAdapter } from './_adapter.js'; +import { ajson_merge } from '../utils/ajson_merge.js'; + +const class_to_collection_key = { + 'SmartSource': 'smart_sources', + 'SmartNote': 'smart_sources', // DEPRECATED: added for backward compatibility + 'SmartBlock': 'smart_blocks', + 'SmartDirectory': 'smart_directories', +}; + +export class SmartCollectionTestDataAdapter extends SmartCollectionDataAdapter { constructor(collection) { super(collection); this.test_data = JSON.parse(fs.readFileSync(this.data_path, 'utf8')); } + get data_path() { return "./test/_data.json"; } - async load() { - // console.log("Loading test collection..."); - Object.entries(this.test_data).forEach(([ajson_key, value]) => { - const [class_name, key] = ajson_key.split(":"); - const entity = new (this.env.item_types[class_name])(this.env, value); - // if value has content, this.collection.fs.write() - if(value.content) this.collection.fs.write(key, value.content); - this.add_to_collection(entity); - }); - // console.log("Loaded test collection"); + + async load(item) { + try { + const ajson_key = `${item.constructor.name}:${item.key}`; + const data = this.test_data[ajson_key]; + + if (!data) { + console.log(`Data not found for: ${ajson_key}`); + return item.queue_import(); + } + + const parsed_data = {}; + data.split('\n').forEach(line => { + try { + const parsed = JSON.parse(`{${line}}`); + if (Object.values(parsed)[0] === null) { + if (parsed_data[Object.keys(parsed)[0]]) delete parsed_data[Object.keys(parsed)[0]]; + } else { + Object.assign(parsed_data, parsed); + } + } catch (err) { + console.warn("Error parsing line: ", line); + console.warn(err.stack); + } + }); + + Object.entries(parsed_data).forEach(([parsed_ajson_key, value]) => { + if (!value) return; // handle null values (deleted) + const [class_name, ...key_parts] = parsed_ajson_key.split(":"); + const entity_key = key_parts.join(":"); + if (entity_key === item.key) { + item.data = value; + } else { + if (!this.env[class_to_collection_key[class_name]]) { + return console.warn(`Collection class not found: ${class_name}`); + } + this.env[class_to_collection_key[class_name]].items[entity_key] = new this.env.item_types[class_name](this.env, value); + } + }); + + item._queue_load = false; + item.loaded_at = Date.now(); + } catch (err) { + console.log(`Error loading collection item: ${item.key}`); + console.warn(err.stack); + item.queue_load(); + } } - async save_item(key) { - delete this._save_queue[key]; - const item = this.collection.get(key); - if(!item) return console.warn("Item not found: " + key); - if(!item.deleted) this.test_data[key] = item.ajson; - else { - delete this.test_data[key]; - this.collection.delete_item(key); + + async save(item, ajson = null) { + if (!ajson) ajson = item.ajson; + const ajson_key = `${item.constructor.name}:${item.key}`; + + try { + if (item.deleted) { + this.collection.delete_item(item.key); + delete this.test_data[ajson_key]; + } else { + this.test_data[ajson_key] += '\n' + ajson; + } + + // Simulate writing to file + fs.writeFileSync(this.data_path, JSON.stringify(this.test_data, null, 2)); + + item._queue_save = false; + return true; + } catch (err) { + console.warn(`Error saving collection item: ${item.key}`); + console.warn(err.stack); + item.queue_save(); + return false; } } } \ No newline at end of file diff --git a/smart-collections/adapters/multi_file.js b/smart-collections/adapters/multi_file.js index 1dcf4e71..110abfeb 100644 --- a/smart-collections/adapters/multi_file.js +++ b/smart-collections/adapters/multi_file.js @@ -1,5 +1,5 @@ import { ajson_merge } from '../utils/ajson_merge.js'; -import { SmartCollectionItemDataAdapter, SmartCollectionDataAdapter } from './_adapter.js'; +import { SmartCollectionDataAdapter } from './_adapter.js'; // DO: replace this better way in future @@ -99,91 +99,91 @@ export class SmartCollectionMultiFileDataAdapter extends SmartCollectionDataAdap -export class MultiFileSmartCollectionItemDataAdapter extends SmartCollectionItemDataAdapter { - get fs() { return this.collection.data_fs || this.env.data_fs; } - /** - * @returns {string} The data folder that contains .ajson files. - */ - get data_folder() { return 'multi'; } - /** - * @returns {string} The data path for .ajson file. - */ - get data_path() { return this.data_folder + "/" + this.item.multi_ajson_file_name + '.ajson'; } +// export class MultiFileSmartCollectionItemDataAdapter extends SmartCollectionItemDataAdapter { +// get fs() { return this.collection.data_fs || this.env.data_fs; } +// /** +// * @returns {string} The data folder that contains .ajson files. +// */ +// get data_folder() { return 'multi'; } +// /** +// * @returns {string} The data path for .ajson file. +// */ +// get data_path() { return this.data_folder + "/" + this.item.multi_ajson_file_name + '.ajson'; } - /** - * Asynchronously loads collection item data from .ajson file specified by data_path. - */ - async load() { - try{ - const data_ajson = (await this.fs.read(this.data_path)).trim(); - if(!data_ajson){ - return this.item.queue_import(); // queue import and return early if data file missing or empty - } - const ajson_lines = data_ajson.split('\n'); - const parsed_data = ajson_lines - .reduce((acc, line) => { - try{ - const parsed = JSON.parse(`{${line}}`); - if(Object.values(parsed)[0] === null){ - if(acc[Object.keys(parsed)[0]]) delete acc[Object.keys(parsed)[0]]; - return acc; - } - return ajson_merge(acc, parsed); - }catch(err){ - console.warn("Error parsing line: ", line); - console.warn(err.stack); - return acc; - } - }, {}) - ; - // array with same length as parsed_data - const rebuilt_ajson = []; - Object.entries(parsed_data) - .forEach(([ajson_key, value], index) => { - if(!value) return; // handle null values (deleted) - rebuilt_ajson.push(`${JSON.stringify(ajson_key)}: ${JSON.stringify(value)}`); - const [class_name, ...key_parts] = ajson_key.split(":"); - const entity_key = key_parts.join(":"); // key is file path - if(entity_key === this.key) this.item.data = value; - else { - if(!this.env[class_to_collection_key[class_name]]) return console.warn(`Collection class not found: ${class_name}`); - this.env[class_to_collection_key[class_name]].items[entity_key] = new this.env.item_types[class_name](this.env, value); - } - }) - ; - this.item._queue_load = false; - if(ajson_lines.length !== Object.keys(parsed_data).length) this.fs.write(this.data_path, rebuilt_ajson.join('\n')); - }catch(err){ - // if file not found, queue import - if(err.message.includes("ENOENT")) return this.item.queue_import(); - console.log("Error loading collection item: " + this.key); - console.warn(err.stack); - this.item.queue_load(); - return; - } - } +// /** +// * Asynchronously loads collection item data from .ajson file specified by data_path. +// */ +// async load() { +// try{ +// const data_ajson = (await this.fs.read(this.data_path)).trim(); +// if(!data_ajson){ +// return this.item.queue_import(); // queue import and return early if data file missing or empty +// } +// const ajson_lines = data_ajson.split('\n'); +// const parsed_data = ajson_lines +// .reduce((acc, line) => { +// try{ +// const parsed = JSON.parse(`{${line}}`); +// if(Object.values(parsed)[0] === null){ +// if(acc[Object.keys(parsed)[0]]) delete acc[Object.keys(parsed)[0]]; +// return acc; +// } +// return ajson_merge(acc, parsed); +// }catch(err){ +// console.warn("Error parsing line: ", line); +// console.warn(err.stack); +// return acc; +// } +// }, {}) +// ; +// // array with same length as parsed_data +// const rebuilt_ajson = []; +// Object.entries(parsed_data) +// .forEach(([ajson_key, value], index) => { +// if(!value) return; // handle null values (deleted) +// rebuilt_ajson.push(`${JSON.stringify(ajson_key)}: ${JSON.stringify(value)}`); +// const [class_name, ...key_parts] = ajson_key.split(":"); +// const entity_key = key_parts.join(":"); // key is file path +// if(entity_key === this.key) this.item.data = value; +// else { +// if(!this.env[class_to_collection_key[class_name]]) return console.warn(`Collection class not found: ${class_name}`); +// this.env[class_to_collection_key[class_name]].items[entity_key] = new this.env.item_types[class_name](this.env, value); +// } +// }) +// ; +// this.item._queue_load = false; +// if(ajson_lines.length !== Object.keys(parsed_data).length) this.fs.write(this.data_path, rebuilt_ajson.join('\n')); +// }catch(err){ +// // if file not found, queue import +// if(err.message.includes("ENOENT")) return this.item.queue_import(); +// console.log("Error loading collection item: " + this.key); +// console.warn(err.stack); +// this.item.queue_load(); +// return; +// } +// } - async save() { - if(!(await this.fs.exists(this.data_folder))) await this.fs.mkdir(this.data_folder); - try { - if(this.item.deleted){ - this.collection.delete_item(this.key); - if((await this.fs.exists(this.data_path))) await this.fs.remove(this.data_path); - } else { - // await this.fs.write(this.data_path, this.item.ajson); - await this.fs.append(this.data_path, '\n' + this.item.ajson); // prevent overwriting the file - } - this.item._queue_save = false; - return true; - } catch (err) { - if(err.message.includes("ENOENT")) return; // already deleted - console.warn("Error saving collection item: ", this.key); - console.warn(err.stack); - this.item.queue_save(); - return false; - } - } -} +// async save() { +// if(!(await this.fs.exists(this.data_folder))) await this.fs.mkdir(this.data_folder); +// try { +// if(this.item.deleted){ +// this.collection.delete_item(this.key); +// if((await this.fs.exists(this.data_path))) await this.fs.remove(this.data_path); +// } else { +// // await this.fs.write(this.data_path, this.item.ajson); +// await this.fs.append(this.data_path, '\n' + this.item.ajson); // prevent overwriting the file +// } +// this.item._queue_save = false; +// return true; +// } catch (err) { +// if(err.message.includes("ENOENT")) return; // already deleted +// console.warn("Error saving collection item: ", this.key); +// console.warn(err.stack); +// this.item.queue_save(); +// return false; +// } +// } +// } /** diff --git a/smart-collections/test/_env.js b/smart-collections/test/_env.js index fee427ed..d39ada4c 100644 --- a/smart-collections/test/_env.js +++ b/smart-collections/test/_env.js @@ -1,11 +1,11 @@ import { CollectionItem } from '../main.js'; import { Collection } from '../main.js'; -import { TestSmartCollectionAdapter } from '../adapters/_test.js'; +import { SmartCollectionTestDataAdapter } from '../adapters/_test.js'; import { SmartEnv } from '../../smart-environment/smart_env.js'; import { SmartChunks } from '../../smart-chunks/smart_chunks.js'; import { SmartEmbedModel } from '../../smart-embed-model/smart_embed_model.js'; import { SmartFs } from '../../smart-fs/smart_fs.js'; -import { TestSmartFsAdapter } from '../../smart-fs/adapters/_test.js'; +import { SmartFsTestAdapter } from '../../smart-fs/adapters/_test.js'; const __dirname = new URL('.', import.meta.url).pathname; @@ -22,13 +22,13 @@ class TestMain { smart_embed_model: SmartEmbedModel, smart_fs: { class: SmartFs, - adapter: TestSmartFsAdapter, + adapter: SmartFsTestAdapter, } }, collections: { collection: { class: Collection, - data_adapter: TestSmartCollectionAdapter, + data_adapter: SmartCollectionTestDataAdapter, }, }, item_types: { diff --git a/smart-embed-model-v1/smart_embed_model.js b/smart-embed-model-v1/smart_embed_model.js index f1e56c12..e6c6ed00 100644 --- a/smart-embed-model-v1/smart_embed_model.js +++ b/smart-embed-model-v1/smart_embed_model.js @@ -44,7 +44,7 @@ export class SmartEmbedModel extends SmartModel { if(!this.opts.adapter) return console.warn('SmartEmbedModel adapter not set'); if(!this.opts.adapters[this.opts.adapter]) return console.warn(`SmartEmbedModel adapter ${this.opts.adapter} not found`); // prepare opts for GPU (likely better handled in future) - this.opts.use_gpu = !!navigator.gpu && this.opts.gpu_batch_size !== 0; + this.opts.use_gpu = typeof navigator !== 'undefined' && !!navigator?.gpu && this.opts.gpu_batch_size !== 0; if(this.opts.adapter === 'transformers' && this.opts.use_gpu) this.opts.batch_size = this.opts.gpu_batch_size || 10; } get adapters() { return this.opts.adapters || this.env.opts.modules.smart_embed_model.adapters; } diff --git a/smart-embed-model/smart_embed_model.js b/smart-embed-model/smart_embed_model.js index cd1641d6..9f70b80d 100644 --- a/smart-embed-model/smart_embed_model.js +++ b/smart-embed-model/smart_embed_model.js @@ -38,7 +38,7 @@ export class SmartEmbedModel extends SmartModel { if (!this.opts.adapter) return console.warn('SmartEmbedModel adapter not set'); if (!this.opts.adapters[this.opts.adapter]) return console.warn(`SmartEmbedModel adapter ${this.opts.adapter} not found`); // prepare opts for GPU (likely better handled in future) - this.opts.use_gpu = !!navigator.gpu && this.opts.gpu_batch_size !== 0; + this.opts.use_gpu = !!navigator?.gpu && this.opts.gpu_batch_size !== 0; if (this.opts.adapter === 'transformers' && this.opts.use_gpu) this.opts.batch_size = this.opts.gpu_batch_size || 10; } async load() { diff --git a/smart-entities/test/_env.js b/smart-entities/test/_env.js index 6e15489f..40611544 100644 --- a/smart-entities/test/_env.js +++ b/smart-entities/test/_env.js @@ -1,5 +1,5 @@ -import { TestSmartCollectionAdapter } from '../../smart-collections/adapters/_test.js'; -import { TestSmartFsAdapter } from '../../smart-fs/adapters/_test.js'; +import { SmartCollectionTestDataAdapter } from '../../smart-collections/adapters/_test.js'; +import { SmartFsTestAdapter } from '../../smart-fs/adapters/_test.js'; import { SmartEntity } from '../smart_entity.js'; import { SmartEntities } from '../smart_entities.js'; import { SmartEnv } from '../../smart-environment/smart_env.js'; @@ -42,13 +42,13 @@ class TestMain { smart_embed_model: SmartEmbedModel, smart_fs: { class: SmartFs, - adapter: TestSmartFsAdapter, + adapter: SmartFsTestAdapter, } }, collections: { smart_entities: { class: SmartEntities, - data_adapter: TestSmartCollectionAdapter, + data_adapter: SmartCollectionTestDataAdapter, }, }, item_types: { diff --git a/smart-entities/test/env.test.js b/smart-entities/test/env.test.js index 3a4a243e..74e13646 100644 --- a/smart-entities/test/env.test.js +++ b/smart-entities/test/env.test.js @@ -1,7 +1,7 @@ import test from 'ava'; import { load_test_env } from './_env.js'; import { TestSmartCollectionAdapter } from '../../smart-collections/adapters/_test.js'; -import { TestSmartFsAdapter } from '../../smart-fs/adapters/_test.js'; +import { SmartFsTestAdapter } from '../../smart-fs/adapters/_test.js'; import { SmartEntity } from '../smart_entity.js'; import { SmartEntities } from '../smart_entities.js'; import { SmartEnv } from '../../smart-environment/smart_env.js'; @@ -35,7 +35,7 @@ test('SmartEnv is initialized with correct options', t => { t.is(env_opts.modules.smart_chunks, SmartChunks); t.is(env_opts.modules.smart_embed_model, SmartEmbedModel); t.is(env_opts.modules.smart_fs.class, SmartFs); - t.is(env_opts.modules.smart_fs.adapter, TestSmartFsAdapter); + t.is(env_opts.modules.smart_fs.adapter, SmartFsTestAdapter); t.is(env_opts.collections.smart_entities.data_adapter, TestSmartCollectionAdapter); t.is(env_opts.collections.smart_entities, SmartEntities); t.is(env_opts.item_types.SmartEntity, SmartEntity); diff --git a/smart-environment/smart_env.js b/smart-environment/smart_env.js index e66e2164..7cac01e8 100644 --- a/smart-environment/smart_env.js +++ b/smart-environment/smart_env.js @@ -354,6 +354,14 @@ function normalize_opts(opts) { delete opts.collections[key]; } }); + Object.entries(opts.modules).forEach(([key, value]) => { + if (typeof value === 'function') opts.modules[key] = { class: value }; + // if key is CamelCase, convert to snake_case + if (key[0] === key[0].toUpperCase()) { + opts.modules[camel_case_to_snake_case(key)] = { ...opts.modules[key] }; + delete opts.modules[key]; + } + }); return opts; } diff --git a/smart-fs/adapters/_test.js b/smart-fs/adapters/_test.js index e6ff3088..ae95fec0 100644 --- a/smart-fs/adapters/_test.js +++ b/smart-fs/adapters/_test.js @@ -1,5 +1,5 @@ /** - * TestSmartFsAdapter class + * SmartFsTestAdapter class * * This class provides a mock file system adapter for testing purposes. * It simulates file system operations in memory, making it ideal for unit tests. @@ -7,13 +7,34 @@ * @class * @classdesc Mock file system adapter for SmartFs testing */ -export class TestSmartFsAdapter { +import path from 'path'; +export class SmartFsTestAdapter { constructor(smart_fs) { this.smart_fs = smart_fs; this.files = {}; this.sep = '/'; } + get_file(file_path) { + const file = {}; + file.path = file_path.replace(/^\//, ''); // remove leading slash + file.type = 'file'; + file.extension = file.path.split('.').pop().toLowerCase(); + file.name = file.path.split('/').pop(); + file.basename = file.name.split('.').shift(); + Object.defineProperty(file, 'stat', { + get: () => { + const stat = this.statSync(file_path); + return { + ctime: stat.ctime.getTime(), + mtime: stat.mtime.getTime(), + size: stat.size, + }; + } + }); + return file; + } + async append(rel_path, content) { this.files[rel_path] = (this.files[rel_path] || '') + content; } @@ -35,57 +56,83 @@ export class TestSmartFsAdapter { return rel_path in this.files; } - async list(rel_path, opts = {}) { - const items = Object.keys(this.files) - .filter(key => key.startsWith(rel_path) && key !== rel_path) - .map(key => { - const name = key.slice(rel_path.length + 1).split(this.sep)[0]; - const full_path = rel_path + this.sep + name; - const is_file = this.files[full_path] !== '[DIRECTORY]'; - return { - basename: name.split('.')[0], - extension: is_file ? name.slice(name.lastIndexOf('.') + 1) : '', - name: name, - path: full_path, - type: is_file ? 'file' : 'folder', - }; - }); - + + async list(rel_path = '', opts = {}) { + if(rel_path === '/') rel_path = ''; + const items = {}; + for (const key of Object.keys(this.files)) { + if (key === rel_path) continue; + if (rel_path && !key.startsWith(rel_path)) continue; + + // Remove the rel_path from key and remove any leading slashes + let remaining_path = key.slice(rel_path.length); + if (remaining_path.startsWith(this.sep)) remaining_path = remaining_path.slice(1); + + const parts = remaining_path.split(this.sep); + const name = parts[0]; + const full_path = rel_path ? path.join(rel_path, name) : name; + + // Skip if already added + if (items[full_path]) continue; + + const is_file = this.files[full_path] !== '[DIRECTORY]'; + const file = this.get_file(full_path); + file.type = is_file ? 'file' : 'folder'; + + if (!is_file) { + delete file.basename; + delete file.extension; + Object.defineProperty(file, 'children', { + get: () => { + return Object.keys(this.files) + .filter(k => k.startsWith(full_path) && k !== full_path) + .map(k => this.get_file(k)); + } + }); + } + items[full_path] = file; + } + + let result = Object.values(items); + if (opts.type === 'file') { - return items.filter(item => item.type === 'file'); + return result.filter(item => item.type === 'file'); } else if (opts.type === 'folder') { - return items.filter(item => item.type === 'folder'); + return result.filter(item => item.type === 'folder'); } - return items; + return result; + } + + async list_recursive(rel_path = '', opts = {}) { + const all_items = []; + const process_items = async (current_path) => { + const items = await this.list(current_path); + for (const item of items) { + all_items.push(item); + if (item.type === 'folder') { + await process_items(item.path); + } + } + }; + await process_items(rel_path); + return all_items; } - async list_recursive(rel_path, opts = {}) { - return this.list(rel_path, { ...opts, recursive: true }); - } async list_files(rel_path, opts = {}) { - return this.list(rel_path, { ...opts, type: 'file' }); + return await this.list(rel_path, { ...opts, type: 'file' }); } async list_files_recursive(rel_path, opts = {}) { - return this.list_recursive(rel_path, { ...opts, type: 'file' }); + return await this.list_recursive(rel_path, { ...opts, type: 'file' }); } async list_folders(rel_path, opts = {}) { - return this.list(rel_path, { ...opts, type: 'folder' }); + return await this.list(rel_path, { ...opts, type: 'folder' }); } async list_folders_recursive(rel_path = '', opts = {}) { - const all_paths = Object.keys(this.files) - .filter(key => key.startsWith(rel_path) && this.files[key] === '[DIRECTORY]' && key !== rel_path) - .map(key => ({ - basename: key.split(this.sep).pop(), - name: key.split(this.sep).pop(), - path: key, - type: 'folder' - })); - - return all_paths; + return await this.list_recursive(rel_path, { ...opts, type: 'folder' }); } async read(rel_path, encoding = 'utf-8') { @@ -102,12 +149,19 @@ export class TestSmartFsAdapter { delete this.files[rel_path]; } - async remove_dir(rel_path) { - Object.keys(this.files).forEach(key => { - if (key === rel_path || key.startsWith(rel_path + this.sep)) { - delete this.files[key]; + async remove_dir(rel_path, recursive = false) { + if (recursive) { + Object.keys(this.files).forEach(key => { + if (key === rel_path || key.startsWith(rel_path + this.sep)) { + delete this.files[key]; + } + }); + } else { + if (Object.keys(this.files).some(key => key.startsWith(rel_path + this.sep))) { + throw new Error(`Directory not empty: ${rel_path}`); } - }); + delete this.files[rel_path]; + } } async rename(rel_path, new_rel_path) { @@ -132,6 +186,14 @@ export class TestSmartFsAdapter { }; } + statSync(rel_path) { + return { + ctime: new Date(), + mtime: new Date(), + size: this.files[rel_path].length, + }; + } + async write(rel_path, content) { const dir_path = rel_path.split(this.sep).slice(0, -1).join(this.sep); if (dir_path) { diff --git a/smart-fs/smart_fs.test.js b/smart-fs/smart_fs.test.js index e873e83c..fc424308 100644 --- a/smart-fs/smart_fs.test.js +++ b/smart-fs/smart_fs.test.js @@ -1,33 +1,31 @@ import test from 'ava'; import { SmartFs } from './smart_fs.js'; import { glob_to_regex } from './utils/match_glob.js'; -// Mock adapter for testing -class MockAdapter { - constructor() { - this.calls = []; - } - - async mockMethod(...args) { - this.calls.push({ method: 'mockMethod', args }); - return args; - } -} +import { SmartFsTestAdapter } from './adapters/_test.js'; test('SmartFs.use_adapter calls adapter method and applies pre/post processing', async t => { const env = { config: { fs_path: '/test/path' } }; - const smart_fs = new SmartFs(env, { adapter: MockAdapter }); + const smart_fs = new SmartFs(env, { adapter: SmartFsTestAdapter }); // Mock excluded patterns smart_fs.excluded_patterns = [glob_to_regex('*.excluded')]; - await smart_fs.use_adapter('mockMethod', ['file.txt'], 'arg1', 'arg2'); + await smart_fs.use_adapter('write', ['file.txt'], 'some_text'); + await smart_fs.load_files(); - t.deepEqual(smart_fs.adapter.calls, [{ method: 'mockMethod', args: ['file.txt', 'arg1', 'arg2'] }]); + t.deepEqual(smart_fs.files, { 'file.txt': { + path: 'file.txt', + type: 'file', + extension: 'txt', + name: 'file.txt', + basename: 'file', + } }); + t.truthy(smart_fs.files['file.txt'].stat); }); test('SmartFs.pre_process throws error for excluded paths', async t => { const env = { config: { fs_path: '/test/path' } }; - const smart_fs = new SmartFs(env, { adapter: MockAdapter }); + const smart_fs = new SmartFs(env, { adapter: SmartFsTestAdapter }); smart_fs.excluded_patterns = [glob_to_regex('*.excluded')]; @@ -39,7 +37,7 @@ test('SmartFs.pre_process throws error for excluded paths', async t => { test('SmartFs.post_process filters out excluded paths', t => { const env = { config: { fs_path: '/test/path' } }; - const smart_fs = new SmartFs(env, { adapter: MockAdapter }); + const smart_fs = new SmartFs(env, { adapter: SmartFsTestAdapter }); smart_fs.excluded_patterns = [glob_to_regex('*.excluded')]; @@ -57,10 +55,53 @@ test('SmartFs constructor throws error when adapter is not set', async t => { test('SmartFs.use_adapter throws error when method is not found in adapter', async t => { const env = { config: { fs_path: '/test/path' } }; - const smart_fs = new SmartFs(env, { adapter: MockAdapter }); + const smart_fs = new SmartFs(env, { adapter: SmartFsTestAdapter }); await t.throwsAsync( async () => smart_fs.use_adapter('nonExistentMethod', ['file.txt']), { message: 'Method nonExistentMethod not found in adapter' } ); }); + +test('SmartFs.load_files builds files and file_paths correctly', async t => { + const env = { config: { fs_path: '/test/path' } }; + const smart_fs = new SmartFs(env, { adapter: SmartFsTestAdapter }); + + // Set up mock files in the test adapter + await smart_fs.adapter.write('file1.txt', 'Content 1'); + await smart_fs.adapter.write('folder/file2.txt', 'Content 2'); + await smart_fs.adapter.write('folder/subfolder/file3.txt', 'Content 3'); + + // Call load_files + await smart_fs.load_files(); + + // Check files object + t.truthy(smart_fs.files['file1.txt']); + t.truthy(smart_fs.files['folder/file2.txt']); + t.truthy(smart_fs.files['folder/subfolder/file3.txt']); + + t.is(smart_fs.files['file1.txt'].type, 'file'); + t.is(smart_fs.files['file1.txt'].extension, 'txt'); + t.is(smart_fs.files['file1.txt'].name, 'file1.txt'); + t.is(smart_fs.files['file1.txt'].basename, 'file1'); + + // Check file_paths array + t.deepEqual(smart_fs.file_paths.sort(), [ + 'file1.txt', + 'folder/file2.txt', + 'folder/subfolder/file3.txt' + ].sort()); + + // Check folders object + t.truthy(smart_fs.folders['folder']); + t.truthy(smart_fs.folders['folder/subfolder']); + + t.is(smart_fs.folders['folder'].type, 'folder'); + t.is(smart_fs.folders['folder'].name, 'folder'); + + // Check folder_paths array + t.deepEqual(smart_fs.folder_paths.sort(), [ + 'folder', + 'folder/subfolder' + ].sort()); +}); diff --git a/smart-settings/smart_settings.test.js b/smart-settings/smart_settings.test.js index 19713606..b733a0df 100644 --- a/smart-settings/smart_settings.test.js +++ b/smart-settings/smart_settings.test.js @@ -1,6 +1,6 @@ import test from "ava"; import { SmartFs } from "../smart-fs/smart_fs.js"; -import { TestSmartFsAdapter } from "../smart-fs/adapters/_test.js"; +import { SmartFsTestAdapter } from "../smart-fs/adapters/_test.js"; import { SmartEnvSettings } from "./smart_env_settings.js"; class MockMain { @@ -23,7 +23,7 @@ class MockEnv { this.mains = ["mock_main"]; this.opts.modules.smart_fs = { class: SmartFs, - adapter: TestSmartFsAdapter, + adapter: SmartFsTestAdapter, }; } } diff --git a/smart-sources-v1/adapters/_test.js b/smart-sources-v1/adapters/_test.js index 8040f2d3..c29a4a7e 100644 --- a/smart-sources-v1/adapters/_test.js +++ b/smart-sources-v1/adapters/_test.js @@ -1,6 +1,6 @@ import { SourceAdapter } from "./_adapter.js"; -export class TestSourceAdapter extends SourceAdapter { +export class SourceTestAdapter extends SourceAdapter { constructor(smart_source) { super(smart_source); this.content = this.smart_source.collection.adapter.test_data[this.smart_source.constructor.name+":"+this.smart_source.data.path].content; diff --git a/smart-sources-v1/test/_env.js b/smart-sources-v1/test/_env.js index ba9b41c0..2ec52144 100644 --- a/smart-sources-v1/test/_env.js +++ b/smart-sources-v1/test/_env.js @@ -1,6 +1,6 @@ -import { TestSmartCollectionAdapter } from '../../smart-collections/adapters/_test.js'; -import { TestSmartFsAdapter } from '../../smart-fs/adapters/_test.js'; -import { TestSourceAdapter } from '../adapters/_test.js'; +import { SmartCollectionTestDataAdapter } from '../../smart-collections/adapters/_test.js'; +import { SmartFsTestAdapter } from '../../smart-fs/adapters/_test.js'; +import { SourceTestAdapter } from '../adapters/_test.js'; import { MarkdownSourceAdapter } from '../adapters/markdown.js'; import { SmartSource } from '../smart_source.js'; import { SmartSources } from '../smart_sources.js'; @@ -27,13 +27,13 @@ class TestMain { smart_embed_model: SmartEmbedModel, smart_fs: { class: SmartFs, - adapter: TestSmartFsAdapter, + adapter: SmartFsTestAdapter, } }, collections: { smart_sources: { class: SmartSources, - data_adapter: TestSmartCollectionAdapter, + data_adapter: SmartCollectionTestDataAdapter, }, smart_blocks: SmartBlocks, smart_directories: SmartDirectories, @@ -44,7 +44,7 @@ class TestMain { SmartDirectory, }, source_adapters: { - test: TestSourceAdapter, + test: SourceTestAdapter, }, }; } diff --git a/smart-sources-v1/test/env.test.js b/smart-sources-v1/test/env.test.js index 75fe876a..140144cf 100644 --- a/smart-sources-v1/test/env.test.js +++ b/smart-sources-v1/test/env.test.js @@ -1,7 +1,7 @@ import test from 'ava'; import { load_test_env } from './_env.js'; import { TestSmartCollectionAdapter } from '../../smart-collections/adapters/_test.js'; -import { TestSmartFsAdapter } from '../../smart-fs/adapters/_test.js'; +import { SmartFsTestAdapter } from '../../smart-fs/adapters/_test.js'; import { SmartSource } from '../smart_source.js'; import { SmartSources } from '../smart_sources.js'; import { SmartBlock } from '../smart_block.js'; @@ -36,7 +36,7 @@ test('SmartEnv is initialized with correct options', t => { t.is(env_opts.modules.smart_chunks, SmartChunks); t.is(env_opts.modules.smart_embed_model, SmartEmbedModel); t.is(env_opts.modules.smart_fs.class, SmartFs); - t.is(env_opts.modules.smart_fs.adapter, TestSmartFsAdapter); + t.is(env_opts.modules.smart_fs.adapter, SmartFsTestAdapter); t.is(env_opts.collections.smart_sources, SmartSources); t.is(env_opts.collections.smart_sources.adapter_class, TestSmartCollectionAdapter); t.is(env_opts.collections.smart_blocks, SmartBlocks); diff --git a/smart-sources/adapters/_test.js b/smart-sources/adapters/_test.js index 8040f2d3..a910af65 100644 --- a/smart-sources/adapters/_test.js +++ b/smart-sources/adapters/_test.js @@ -1,51 +1,353 @@ import { SourceAdapter } from "./_adapter.js"; +import { markdown_to_blocks } from "../blocks/markdown_to_blocks.js"; +import { create_hash } from "../utils/create_hash.js"; +import { get_markdown_links } from "../utils/get_markdown_links.js"; +import { get_line_range } from "../utils/get_line_range.js"; +import { increase_heading_depth } from "../utils/increase_heading_depth.js"; -export class TestSourceAdapter extends SourceAdapter { - constructor(smart_source) { - super(smart_source); - this.content = this.smart_source.collection.adapter.test_data[this.smart_source.constructor.name+":"+this.smart_source.data.path].content; +export class SourceTestAdapter extends SourceAdapter { + constructor(item) { + super(item); + this.content = this.item.collection.adapter.test_data[this.item.constructor.name+":"+this.item.data.path].content; } + get fs() { return this.source_collection.fs; } + get data() { return this.item.data; } + get file_path() { return this.item.data.path; } + get source() { return this.item.source ? this.item.source : this.item; } + async append(content) { + if (this.smart_change) { + content = this.smart_change.wrap("content", { + before: "", + after: content, + adapter: this.item.smart_change_adapter + }); + } this.content += "\n" + content; } async update(full_content, opts = {}) { - this.content = full_content; + const { mode = "replace_all" } = opts; + if (mode === "replace_all") { + await this.merge(full_content, { mode: "replace_all" }); + } else if (mode === "merge_replace") { + await this.merge(full_content, { mode: "replace_blocks" }); + } else if (mode === "merge_append") { + await this.merge(full_content, { mode: "append_blocks" }); + } } + async _update(content) { this.content = content; } async read(opts = {}) { - return this.content; + let content = this.content; + this.source.data.last_read_hash = await create_hash(content); + if(this.source.last_read_hash !== this.source.hash){ + this.source.loaded_at = null; + await this.source.import(); + } + if (opts.no_changes) { + const unwrapped = this.smart_change.unwrap(content, {file_type: this.item.file_type}); + content = unwrapped[opts.no_changes === 'after' ? 'after' : 'before']; + } + if (opts.add_depth) { + content = increase_heading_depth(content, opts.add_depth); + } + return content; } + async _read() { return this.content; } async remove() { this.content = ""; - this.smart_source.delete(); + this.item.delete(); } async move_to(entity_ref) { - // For testing purposes, we'll just update the path const new_path = typeof entity_ref === "string" ? entity_ref : entity_ref.key; if (!new_path) { throw new Error("Invalid entity reference for move_to operation"); } - this.smart_source.data.path = new_path; + + const current_content = await this.read(); + const target_source_key = new_path.split("#")[0]; + const target_source = this.env.smart_sources.get(target_source_key); + + if (new_path.includes("#")) { + const headings = new_path.split("#").slice(1); + const new_headings_content = headings.map((heading, i) => `${"#".repeat(i + 1)} ${heading}`).join("\n"); + const new_content = new_headings_content + "\n" + current_content; + await this._update(new_content); + } + + if (target_source) { + await target_source.merge(current_content, { mode: 'append_blocks' }); + } else { + this.item.data.path = target_source_key; + const new_source = await this.item.collection.create_or_update({ path: target_source_key, content: current_content }); + await new_source.import(); + } + + if (this.item.key !== target_source_key) await this.remove(); } async merge(content, opts = {}) { const { mode = 'append_blocks' } = opts; - if (mode === 'replace_all') { - this.content = content; + const { blocks } = await this.item.smart_chunks.parse({ + content, + file_path: this.file_path, + }); + + if (!Array.isArray(blocks)) throw new Error("merge error: parse returned blocks that were not an array", blocks); + + blocks.forEach(block => { + block.content = content + .split("\n") + .slice(block.lines[0], block.lines[1] + 1) + .join("\n"); + + const match = this.item.blocks.find(b => b.key === block.path); + if (match) { + block.matched = true; + match.matched = true; + } + }); + + if (mode === "replace_all") { + if (this.smart_change) { + let all = ""; + const new_blocks = blocks.sort((a, b) => a.lines[0] - b.lines[0]); + if (new_blocks[0].lines[0] > 0) { + all += content.split("\n").slice(0, new_blocks[0].lines[0]).join("\n"); + } + for (let i = 0; i < new_blocks.length; i++) { + const block = new_blocks[i]; + if (all.length) all += "\n"; + if (block.matched) { + const og = this.env.smart_blocks.get(block.path); + all += this.smart_change.wrap("content", { + before: await og.read({ no_changes: "before", headings: "last" }), + after: block.content, + adapter: this.item.smart_change_adapter + }); + } else { + all += this.smart_change.wrap("content", { + before: "", + after: block.content, + adapter: this.item.smart_change_adapter + }); + } + } + const unmatched_old = this.item.blocks.filter(b => !b.matched); + for (let i = 0; i < unmatched_old.length; i++) { + const block = unmatched_old[i]; + all += (all.length ? "\n" : "") + this.smart_change.wrap("content", { + before: await block.read({ no_changes: "before", headings: "last" }), + after: "", + adapter: this.item.smart_change_adapter + }); + } + await this._update(all); + } else { + await this._update(content); + } } else { - this.content += "\n" + content; + for (let i = 0; i < blocks.length; i++) { + const block = blocks[i]; + if (block.matched) { + const to_block = this.env.smart_blocks.get(block.path); + if (mode === "append_blocks") { + await to_block.append(block.content); + } else { + await to_block.update(block.content); + } + } + } + const unmatched_content = blocks + .filter(block => !block.matched) + .map(block => block.content) + .join("\n"); + if (unmatched_content.length) { + await this.append(unmatched_content); + } } - await this.smart_source.parse_content(); } + async import() { + if (!this.content) { + console.warn("No content to import for " + this.file_path); + return; + } + + const hash = await create_hash(this.content); + if (this.data.hash === hash) { + console.log("File stats changed, but content is the same. Skipping import."); + return; + } + + this.data.hash = hash; + this.data.last_read_hash = hash; + + const blocks = markdown_to_blocks(this.content); + this.data.blocks = blocks; + + const outlinks = get_markdown_links(this.content); + this.data.outlinks = outlinks; + + for (const [sub_key, value] of Object.entries(blocks)) { + const block_key = this.item.key + sub_key; + const block_content = this.content.split("\n").slice(value[0] - 1, value[1]).join("\n"); + const block_outlinks = get_markdown_links(block_content); + + const block = await this.item.block_collection.create_or_update({ + key: block_key, + outlinks: block_outlinks, + size: block_content.length, + }); + + block._embed_input = block.breadcrumbs + "\n" + block_content; + block.data.hash = await create_hash(block._embed_input); + } + } + + async block_read(opts = {}) { + if(!this.item.line_start) return "BLOCK NOT FOUND (no line_start)"; + const source_content = await this.read(); + const block_content = get_line_range(source_content, this.item.line_start, this.item.line_end); + const breadcrumbs = this.item.breadcrumbs; + const embed_input = breadcrumbs + "\n" + block_content; + const hash = await create_hash(embed_input); + if(hash !== this.item.hash){ + const blocks = markdown_to_blocks(source_content); + this.item.source?.queue_import(); + const block_range = Object.entries(blocks).find(([k,v]) => k === this.item.sub_key)?.[1]; + if(!block_range) return "BLOCK NOT FOUND (not in updated parsed blocks)"; + return get_line_range(source_content, block_range[0], block_range[1]); + } + return block_content; + } + + prepend_headings(content, mode) { + const headings = this.file_path.split('#').slice(1); + let prepend_content = ''; + + if (mode === 'all') { + prepend_content = headings.map((h, i) => '#'.repeat(i + 1) + ' ' + h).join('\n'); + } else if (mode === 'last') { + prepend_content = '#'.repeat(headings.length) + ' ' + headings[headings.length - 1]; + } + + return prepend_content + (prepend_content ? '\n' : '') + content; + } + + async block_append(append_content) { + let all_lines = (await this.read()).split("\n"); + if(all_lines[this.item.line_start] === append_content.split("\n")[0]){ + append_content = append_content.split("\n").slice(1).join("\n"); + } + if(this.smart_change) append_content = this.smart_change.wrap("content", { before: "", after: append_content, adapter: this.item.smart_change_adapter }); + await this._block_append(append_content); + } + + async _block_append(append_content) { + let all_lines = (await this.read()).split("\n"); + const content_before = all_lines.slice(0, this.item.line_end + 1); + const content_after = all_lines.slice(this.item.line_end + 1); + const new_content = [ + ...content_before, + "", + append_content, + ...content_after, + ].join("\n").trim(); + await this.item.source._update(new_content); + await this.item.source.parse_content(); + } + + async block_update(new_block_content, opts = {}) { + if(this.smart_change) new_block_content = this.smart_change.wrap("content", { + before: await this.block_read({ no_changes: "before", headings: "last" }), + after: new_block_content, + adapter: this.item.smart_change_adapter + }); + await this._block_update(new_block_content); + } + + async _block_update(new_block_content) { + const full_content = await this.read(); + const all_lines = full_content.split("\n"); + const new_content = [ + ...all_lines.slice(0, this.item.line_start), + new_block_content, + ...all_lines.slice(this.item.line_end + 1), + ].join("\n"); + await this.item.source._update(new_content); + await this.item.source.parse_content(); + } + + async block_remove() { + if(this.item.sub_blocks.length){ + await this._block_update((await this.block_read({ no_changes: "before", headings: "last" })).split("\n")[0]); + }else{ + await this._block_update(""); + } + this.item.delete(); + } + + async block_move_to(to_key) { + const to_collection_key = to_key.includes("#") ? "smart_blocks" : "smart_sources"; + const to_entity = this.env[to_collection_key].get(to_key); + let content = await this.block_read({ no_changes: "before", headings: "last" }); + try { + if(this.smart_change){ + const smart_change = this.smart_change.wrap('location', { + to_key: to_key, + before: await this.block_read({headings: 'last', no_change: 'before'}), + adapter: this.item.smart_change_adapter + }); + this._block_update(smart_change); + }else{ + this.block_remove(); + } + } catch (e) { + console.warn("error removing block: ", e); + } + try { + if(to_entity) { + if(this.smart_change){ + content = this.smart_change.wrap("location", { from_key: this.item.source.key, after: content, adapter: this.item.smart_change_adapter }); + await to_entity._append(content); + }else{ + await to_entity.append(content); + } + } else { + const target_source_key = to_key.split("#")[0]; + const target_source = this.env.smart_sources.get(target_source_key); + if (to_key.includes("#")) { + const headings = to_key.split("#").slice(1); + const new_headings_content = headings.map((heading, i) => `${"#".repeat(i + 1)} ${heading}`).join("\n"); + let new_content = [ + new_headings_content, + ...content.split("\n").slice(1) + ].join("\n").trim(); + if(this.smart_change) new_content = this.smart_change.wrap("location", { from_key: this.item.source.key, after: new_content, adapter: this.item.smart_change_adapter }); + if(target_source) await target_source._append(new_content); + else await this.env.smart_sources.create(target_source_key, new_content); + } else { + if(this.smart_change) content = this.smart_change.wrap("location", { from_key: this.item.source.key, after: content, adapter: this.item.smart_change_adapter }); + if(target_source) await target_source._append(content); + else await this.env.smart_sources.create(target_source_key, content); + } + } + } catch (e) { + console.warn("error moving block: ", e); + this.item.deleted = false; + await this.block_update(content); + } + await this.item.source.parse_content(); + } } \ No newline at end of file diff --git a/smart-sources/adapters/markdown.js b/smart-sources/adapters/markdown.js index 2ed1bcb3..89758438 100644 --- a/smart-sources/adapters/markdown.js +++ b/smart-sources/adapters/markdown.js @@ -2,6 +2,8 @@ import { increase_heading_depth } from "../utils/increase_heading_depth.js"; import { TextSourceAdapter } from "./text.js"; import { markdown_to_blocks } from "../blocks/markdown_to_blocks.js"; import { create_hash } from "../utils/create_hash.js"; +import { get_markdown_links } from "../utils/get_markdown_links.js"; +import { get_line_range } from "../utils/get_line_range.js"; export class MarkdownSourceAdapter extends TextSourceAdapter { get fs() { return this.source_collection.fs; } @@ -267,7 +269,7 @@ export class MarkdownSourceAdapter extends TextSourceAdapter { ...content_after, ].join("\n").trim(); await this.item.source._update(new_content); - await this.item.source.parse_content(); + await this.item.source.import(); } async block_update(new_block_content, opts = {}) { @@ -288,7 +290,7 @@ export class MarkdownSourceAdapter extends TextSourceAdapter { ...all_lines.slice(this.item.line_end + 1), ].join("\n"); await this.item.source._update(new_content); - await this.item.source.parse_content(); + await this.item.source.import(); } async block_remove() { @@ -352,39 +354,8 @@ export class MarkdownSourceAdapter extends TextSourceAdapter { this.item.deleted = false; await this.block_update(content); } - await this.item.source.parse_content(); + await this.item.source.import(); } } -/** - * Extracts links from markdown content. - * @param {string} content - * @returns {Array<{title: string, target: string, line: number}>} - */ -function get_markdown_links(content) { - const markdown_link_pattern = /\[([^\]]+)\]\(([^)]+)\)/g; - const wikilink_pattern = /\[\[([^\|\]]+)(?:\|([^\]]+))?\]\]/g; - const result = []; - - const extract_links_from_pattern = (pattern, type) => { - let match; - while ((match = pattern.exec(content)) !== null) { - const title = type === 'markdown' ? match[1] : (match[2] || match[1]); - const target = type === 'markdown' ? match[2] : match[1]; - const line = content.substring(0, match.index).split('\n').length; - result.push({ title, target, line }); - } - }; - - extract_links_from_pattern(markdown_link_pattern, 'markdown'); - extract_links_from_pattern(wikilink_pattern, 'wikilink'); - - result.sort((a, b) => a.line - b.line || a.target.localeCompare(b.target)); - - return result; -} -function get_line_range(content, start_line, end_line) { - const lines = content.split("\n"); - return lines.slice(start_line - 1, end_line).join("\n"); -} diff --git a/smart-sources/blocks/markdown.test.js b/smart-sources/blocks/markdown.test.js new file mode 100644 index 00000000..ed2513eb --- /dev/null +++ b/smart-sources/blocks/markdown.test.js @@ -0,0 +1,393 @@ +import test from 'ava'; +import { load_test_env } from '../test/_env.js'; + +test.beforeEach(async t => { + await load_test_env(t); +}); + +test.serial('MarkdownSourceAdapter import method', async t => { + const env = t.context.env; + const fs = t.context.fs; + + // Create a markdown file + const content = `# Heading 1 +Some content under heading 1. + +## Heading 2 +Some content under heading 2. + +[Link to other file](other.md) +`; + + await fs.write('test.md', content); + await fs.load_files(); + + // Create or update the SmartSource + const source = await env.smart_sources.create_or_update({ path: 'test.md' }); + + // Import the content + await source.import(); + + // Check that the blocks are correctly parsed + t.truthy(source.data.blocks); + t.deepEqual(Object.keys(source.data.blocks), ['#Heading 1', '#Heading 1#Heading 2']); + + // Check that outlinks are correctly set + t.truthy(source.data.outlinks); + t.is(source.data.outlinks.length, 1); + t.is(source.data.outlinks[0].target, 'other.md'); + + // Check that blocks are created in smart_blocks collection + const block1 = env.smart_blocks.get('test.md#Heading 1'); + t.truthy(block1); + const block2 = env.smart_blocks.get('test.md#Heading 1#Heading 2'); + t.truthy(block2); +}); + +test.serial('MarkdownSourceAdapter append method', async t => { + const env = t.context.env; + const fs = t.context.fs; + + // Initial content + const initial_content = '# Initial Heading\nInitial content.'; + await fs.write('append_test.md', initial_content); + await fs.load_files(); + + const source = await env.smart_sources.create_or_update({ path: 'append_test.md' }); + + // Append content + const append_content = '\n# Appended Heading\nAppended content.'; + await source.append(append_content); + + // Read the content + const content = await source.read(); + + // Check that content is appended + const expected_content = initial_content + append_content; + t.is(content, expected_content, 'Content should be appended correctly.'); +}); + +test.serial('MarkdownSourceAdapter update method - replace_all mode', async t => { + const env = t.context.env; + const fs = t.context.fs; + + const initial_content = '# Heading\nInitial content.'; + await fs.write('update_test.md', initial_content); + await fs.load_files(); + + const source = await env.smart_sources.create_or_update({ path: 'update_test.md' }); + + const new_content = '# New Heading\nNew content.'; + await source.update(new_content, { mode: 'replace_all' }); + + const content = await source.read(); + t.is(content, new_content, 'Content should be replaced entirely.'); +}); + +test.serial('MarkdownSourceAdapter update method - merge_replace mode', async t => { + const env = t.context.env; + const fs = t.context.fs; + + const initial_content = '# Heading\nInitial content.\n## Subheading\nSubcontent.'; + await fs.write('update_test.md', initial_content); + await fs.load_files(); + + const source = await env.smart_sources.create_or_update({ path: 'update_test.md' }); + + const new_content = '# Heading\nUpdated content.\n## Subheading\nUpdated subcontent.'; + await source.update(new_content, { mode: 'merge_replace' }); + + const content = await source.read(); + + // Since 'merge_replace' should replace matching blocks + const expected_content = new_content; + t.is(content, expected_content, 'Content should have matching blocks replaced.'); +}); + +test.serial('MarkdownSourceAdapter update method - merge_append mode', async t => { + const env = t.context.env; + const fs = t.context.fs; + + const initial_content = '# Heading\nInitial content.'; + await fs.write('update_test.md', initial_content); + await fs.load_files(); + + const source = await env.smart_sources.create_or_update({ path: 'update_test.md' }); + + const new_content = '# Heading\nAppended content.\n## New Subheading\nNew subcontent.'; + await source.update(new_content, { mode: 'merge_append' }); + + const content = await source.read(); + + // Since 'merge_append' should append to existing blocks + const expected_content = '# Heading\nInitial content.\n\nAppended content.\n## New Subheading\nNew subcontent.'; + t.is(content, expected_content, 'Content should have new blocks appended.'); +}); + +test.serial('MarkdownSourceAdapter read method with no_changes option', async t => { + const env = t.context.env; + const fs = t.context.fs; + + const content_with_changes = `<<<<<<< HEAD +# Original Heading +======= +# New Heading +>>>>>>> +Some content`; + + await fs.write('changes_test.md', content_with_changes); + await fs.load_files(); + const source = await env.smart_sources.create_or_update({ path: 'changes_test.md' }); + + const content_before = await source.read({ no_changes: 'before' }); + t.is(content_before, '# Original Heading\nSome content', 'Should return content before changes'); + + const content_after = await source.read({ no_changes: 'after' }); + t.is(content_after, '# New Heading\nSome content', 'Should return content after changes'); + + const content_with_changes_result = await source.read({ no_changes: false }); + t.is(content_with_changes_result, content_with_changes, 'Should return content with change syntax'); +}); + +test.serial('MarkdownSourceAdapter read method with add_depth option', async t => { + const env = t.context.env; + const fs = t.context.fs; + + const content = `# Heading 1 +## Heading 2 +Some content`; + + await fs.write('depth_test.md', content); + await fs.load_files(); + const source = await env.smart_sources.create_or_update({ path: 'depth_test.md' }); + + const content_depth_1 = await source.read({ add_depth: 1 }); + const expected_depth_1 = `## Heading 1 +### Heading 2 +Some content`; + t.is(content_depth_1, expected_depth_1, 'Should increase heading depth by 1'); +}); + +test.serial('MarkdownSourceAdapter remove method', async t => { + const env = t.context.env; + const fs = t.context.fs; + + const content = 'Some content to remove.'; + await fs.write('remove_test.md', content); + await fs.load_files(); + + const source = await env.smart_sources.create_or_update({ path: 'remove_test.md' }); + + await source.remove(); + + const exists = await fs.exists('remove_test.md'); + t.false(exists, 'File should be removed'); + + // Ensure that source is deleted from smart_sources collection + const source_in_collection = env.smart_sources.get('remove_test.md'); + t.falsy(source_in_collection, 'Source should be removed from collection'); +}); + +test.serial('MarkdownSourceAdapter move_to method - move to new path', async t => { + const env = t.context.env; + const fs = t.context.fs; + + const content = 'Content to move.'; + await fs.write('move_from.md', content); + await fs.load_files(); + + const source = await env.smart_sources.create_or_update({ path: 'move_from.md' }); + + await source.move_to('move_to.md'); + + const exists_old = await fs.exists('move_from.md'); + t.false(exists_old, 'Old file should not exist'); + + const exists_new = await fs.exists('move_to.md'); + t.true(exists_new, 'New file should exist'); + + const new_content = await fs.read('move_to.md'); + t.is(new_content, content, 'Content should be moved correctly'); + + // Ensure that source is updated in smart_sources collection + const source_new = env.smart_sources.get('move_to.md'); + t.truthy(source_new, 'Source should be updated in collection'); +}); + +test.serial('MarkdownSourceAdapter move_to method - move to existing source (merge)', async t => { + const env = t.context.env; + const fs = t.context.fs; + + const content_from = 'Content from source.'; + const content_to = 'Content in destination source.'; + + await fs.write('move_from.md', content_from); + await fs.write('move_to.md', content_to); + await fs.load_files(); + + const source_from = await env.smart_sources.create_or_update({ path: 'move_from.md' }); + const source_to = await env.smart_sources.create_or_update({ path: 'move_to.md' }); + + await source_from.move_to('move_to.md'); + + const exists_old = await fs.exists('move_from.md'); + t.false(exists_old, 'Old file should not exist'); + + const new_content = await fs.read('move_to.md'); + const expected_content = content_to + '\n\n' + content_from; + t.is(new_content, expected_content, 'Content should be merged in destination'); + + // Ensure that source_from is removed from smart_sources collection + const source_from_in_collection = env.smart_sources.get('move_from.md'); + t.falsy(source_from_in_collection, 'Source from should be removed from collection'); +}); + +test.serial('MarkdownSourceAdapter block_read method', async t => { + const env = t.context.env; + const fs = t.context.fs; + + const content = `# Heading 1 +Content under heading 1. +## Heading 2 +Content under heading 2.`; + + await fs.write('block_read_test.md', content); + await fs.load_files(); + + const source = await env.smart_sources.create_or_update({ path: 'block_read_test.md' }); + await source.import(); + + const block = env.smart_blocks.get('block_read_test.md#Heading 1'); + + const block_content = await block.read(); + + const expected_block_content = `# Heading 1 +Content under heading 1.`; + + t.is(block_content, expected_block_content, 'Block content should be read correctly.'); +}); + +test.serial('MarkdownSourceAdapter block_append method', async t => { + const env = t.context.env; + const fs = t.context.fs; + + const content = `# Heading 1 +Content under heading 1.`; + + await fs.write('block_append_test.md', content); + await fs.load_files(); + + const source = await env.smart_sources.create_or_update({ path: 'block_append_test.md' }); + await source.import(); + + const block = env.smart_blocks.get('block_append_test.md#Heading 1'); + + const append_content = 'Additional content for heading 1.'; + await block.append(append_content); + + const block_content = await block.read(); + + const expected_block_content = `# Heading 1 +Content under heading 1. + +Additional content for heading 1.`; + + t.is(block_content, expected_block_content, 'Content should be appended to the block.'); +}); + +test.serial('MarkdownSourceAdapter block_update method', async t => { + const env = t.context.env; + const fs = t.context.fs; + + const content = `# Heading 1 +Content under heading 1.`; + + await fs.write('block_update_test.md', content); + await fs.load_files(); + + const source = await env.smart_sources.create_or_update({ path: 'block_update_test.md' }); + await source.import(); + + const block = env.smart_blocks.get('block_update_test.md#Heading 1'); + + const new_block_content = `# Heading 1 +Updated content under heading 1.`; + + await block.update(new_block_content); + + const block_content = await block.read(); + + t.is(block_content, new_block_content, 'Block content should be updated.'); +}); + +test.serial('MarkdownSourceAdapter block_remove method', async t => { + const env = t.context.env; + const fs = t.context.fs; + + const content = `# Heading 1 +Content under heading 1. +## Heading 2 +Content under heading 2.`; + + await fs.write('block_remove_test.md', content); + await fs.load_files(); + + const source = await env.smart_sources.create_or_update({ path: 'block_remove_test.md' }); + await source.import(); + + const block = env.smart_blocks.get('block_remove_test.md#Heading 1'); + + await block.remove(); + + const source_content = await source.read(); + + const expected_content = `## Heading 2 +Content under heading 2.`; + + t.is(source_content, expected_content, 'Block should be removed from source.'); + + const block_in_collection = env.smart_blocks.get('block_remove_test.md#Heading 1'); + t.falsy(block_in_collection, 'Block should be removed from collection.'); +}); + +test.serial('MarkdownSourceAdapter block_move_to method', async t => { + const env = t.context.env; + const fs = t.context.fs; + + const content_source = `# Heading 1 +Content under heading 1. +## Subheading +Subcontent.`; + + await fs.write('block_move_from.md', content_source); + + const content_target = `# Existing Heading +Existing content.`; + + await fs.write('block_move_to.md', content_target); + await fs.load_files(); + + const source = await env.smart_sources.create_or_update({ path: 'block_move_from.md' }); + await source.import(); + + const target = await env.smart_sources.create_or_update({ path: 'block_move_to.md' }); + + const block = env.smart_blocks.get('block_move_from.md#Heading 1'); + + await block.move_to('block_move_to.md'); + + // Check that the block is removed from source + const source_content = await source.read(); + const expected_source_content = `## Subheading +Subcontent.`; + t.is(source_content, expected_source_content, 'Block should be removed from source.'); + + // Check that the block content is appended to target + const target_content = await target.read(); + const expected_target_content = content_target + '\n\n# Heading 1\nContent under heading 1.'; + t.is(target_content, expected_target_content, 'Block content should be appended to target.'); + + // Ensure block is updated in collection + const block_in_collection = env.smart_blocks.get('block_move_from.md#Heading 1'); + t.truthy(block_in_collection.deleted, 'Block should be marked as deleted in collection.'); +}); diff --git a/smart-sources/test/_env.js b/smart-sources/test/_env.js index ba9b41c0..12329fbe 100644 --- a/smart-sources/test/_env.js +++ b/smart-sources/test/_env.js @@ -1,17 +1,20 @@ -import { TestSmartCollectionAdapter } from '../../smart-collections/adapters/_test.js'; -import { TestSmartFsAdapter } from '../../smart-fs/adapters/_test.js'; -import { TestSourceAdapter } from '../adapters/_test.js'; +import { SmartCollectionTestDataAdapter } from '../../smart-collections/adapters/_test.js'; +import { SmartFsTestAdapter } from '../../smart-fs/adapters/_test.js'; +import { SourceTestAdapter } from '../adapters/_test.js'; import { MarkdownSourceAdapter } from '../adapters/markdown.js'; import { SmartSource } from '../smart_source.js'; import { SmartSources } from '../smart_sources.js'; import { SmartBlock } from '../smart_block.js'; import { SmartBlocks } from '../smart_blocks.js'; -import { SmartDirectory } from '../smart_directory.js'; -import { SmartDirectories } from '../smart_directories.js'; +// import { SmartDirectory } from '../smart_directory.js'; +// import { SmartDirectories } from '../smart_directories.js'; import { SmartEnv } from '../../smart-environment/smart_env.js'; import { SmartChunks } from '../../smart-chunks/smart_chunks.js'; -import { SmartEmbedModel } from '../../smart-embed-model/smart_embed_model.js'; +import { SmartEmbedModel } from '../../smart-embed-model-v1/smart_embed_model.js'; +import { SmartEmbedTransformersAdapter } from '../../smart-embed-model-v1/adapters/transformers.js'; +import { SmartEmbedOpenAIAdapter } from '../../smart-embed-model-v1/adapters/openai.js'; import { SmartFs } from '../../smart-fs/smart_fs.js'; +import { SmartSettings } from '../../smart-settings/smart_settings.js'; const __dirname = new URL('.', import.meta.url).pathname; class TestMain { @@ -24,27 +27,38 @@ class TestMain { env_data_dir: 'test', modules: { smart_chunks: SmartChunks, - smart_embed_model: SmartEmbedModel, + smart_embed_model: { + class: SmartEmbedModel, + adapters: { + transformers: SmartEmbedTransformersAdapter, + openai: SmartEmbedOpenAIAdapter, + }, + }, smart_fs: { class: SmartFs, - adapter: TestSmartFsAdapter, - } + adapter: SmartFsTestAdapter, + }, + smart_settings: { + class: SmartSettings, + }, }, collections: { smart_sources: { class: SmartSources, - data_adapter: TestSmartCollectionAdapter, + data_adapter: SmartCollectionTestDataAdapter, + source_adapters: { + test: SourceTestAdapter, + md: MarkdownSourceAdapter, + default: MarkdownSourceAdapter + }, }, smart_blocks: SmartBlocks, - smart_directories: SmartDirectories, + // smart_directories: SmartDirectories, }, item_types: { SmartSource, SmartBlock, - SmartDirectory, - }, - source_adapters: { - test: TestSourceAdapter, + // SmartDirectory, }, }; } @@ -52,8 +66,9 @@ class TestMain { export async function load_test_env(t) { const main = new TestMain(); - const env = new SmartEnv(main, main.smart_env_config); - await env.init(); + const env = await SmartEnv.create(main, main.smart_env_config); + env.smart_sources.settings.smart_change = {}; + env.smart_sources.settings.smart_change.active = false; t.context.env = env; t.context.fs = env.smart_sources.fs; } \ No newline at end of file diff --git a/smart-sources/test/env.test.js b/smart-sources/test/env.test.js index 75fe876a..140144cf 100644 --- a/smart-sources/test/env.test.js +++ b/smart-sources/test/env.test.js @@ -1,7 +1,7 @@ import test from 'ava'; import { load_test_env } from './_env.js'; import { TestSmartCollectionAdapter } from '../../smart-collections/adapters/_test.js'; -import { TestSmartFsAdapter } from '../../smart-fs/adapters/_test.js'; +import { SmartFsTestAdapter } from '../../smart-fs/adapters/_test.js'; import { SmartSource } from '../smart_source.js'; import { SmartSources } from '../smart_sources.js'; import { SmartBlock } from '../smart_block.js'; @@ -36,7 +36,7 @@ test('SmartEnv is initialized with correct options', t => { t.is(env_opts.modules.smart_chunks, SmartChunks); t.is(env_opts.modules.smart_embed_model, SmartEmbedModel); t.is(env_opts.modules.smart_fs.class, SmartFs); - t.is(env_opts.modules.smart_fs.adapter, TestSmartFsAdapter); + t.is(env_opts.modules.smart_fs.adapter, SmartFsTestAdapter); t.is(env_opts.collections.smart_sources, SmartSources); t.is(env_opts.collections.smart_sources.adapter_class, TestSmartCollectionAdapter); t.is(env_opts.collections.smart_blocks, SmartBlocks); diff --git a/smart-sources/utils/get_line_range.js b/smart-sources/utils/get_line_range.js new file mode 100644 index 00000000..4abfba1e --- /dev/null +++ b/smart-sources/utils/get_line_range.js @@ -0,0 +1,4 @@ +export function get_line_range(content, start_line, end_line) { + const lines = content.split("\n"); + return lines.slice(start_line - 1, end_line).join("\n"); +} diff --git a/smart-sources/utils/get_markdown_links.js b/smart-sources/utils/get_markdown_links.js new file mode 100644 index 00000000..710fb9f4 --- /dev/null +++ b/smart-sources/utils/get_markdown_links.js @@ -0,0 +1,27 @@ +/** + * Extracts links from markdown content. + * @param {string} content + * @returns {Array<{title: string, target: string, line: number}>} + */ +export function get_markdown_links(content) { + const markdown_link_pattern = /\[([^\]]+)\]\(([^)]+)\)/g; + const wikilink_pattern = /\[\[([^\|\]]+)(?:\|([^\]]+))?\]\]/g; + const result = []; + + const extract_links_from_pattern = (pattern, type) => { + let match; + while ((match = pattern.exec(content)) !== null) { + const title = type === 'markdown' ? match[1] : (match[2] || match[1]); + const target = type === 'markdown' ? match[2] : match[1]; + const line = content.substring(0, match.index).split('\n').length; + result.push({ title, target, line }); + } + }; + + extract_links_from_pattern(markdown_link_pattern, 'markdown'); + extract_links_from_pattern(wikilink_pattern, 'wikilink'); + + result.sort((a, b) => a.line - b.line || a.target.localeCompare(b.target)); + + return result; +}