From ff1ea05eebf4db74474495bfb0f90d7a0a273b38 Mon Sep 17 00:00:00 2001 From: Brian Joseph Petro Date: Fri, 20 Dec 2024 16:22:04 -0500 Subject: [PATCH] Refactor SmartDirectories to integrate SmartGroups and enhance directory management - Introduced `SmartGroups` and `SmartGroup` classes, replacing `SmartEntities` in `SmartDirectories` and `SmartDirectory` for improved group handling. - Added `DirectoryGroupAdapter` and `SourceDirectoryGroupsAdapter` to manage directory group creation and structure. - Updated `package.json` to reflect changes in main entry point and dependencies. - Removed obsolete components related to directory rendering, streamlining the codebase. - Added a test script to create a directory structure for integration testing of SmartDirectories. These changes enhance the modularity and functionality of the SmartDirectories system, facilitating better directory management and integration with smart groups. --- smart-directories/adapters/_adapter.js | 10 ++ smart-directories/adapters/sources.js | 47 +++++++ smart-directories/components/directories.js | 67 --------- smart-directories/components/directory.js | 76 ---------- smart-directories/{main.js => index.js} | 2 + smart-directories/package.json | 4 +- smart-directories/smart_directories.js | 17 +-- smart-directories/smart_directory.js | 130 ++---------------- smart-directories/test/test_content.js | 46 +++++++ smart-entities/smart_entity.js | 7 + smart-groups/adapters/_adapter.js | 39 ++++++ .../adapters/data/ajson_multi_file.js | 40 ++++++ .../adapters/vector/median_members.js | 79 +++++++++++ smart-groups/index.js | 11 ++ smart-groups/package.json | 35 +++++ smart-groups/smart_group.js | 127 +++++++++++++++++ smart-groups/smart_groups.js | 16 +++ 17 files changed, 479 insertions(+), 274 deletions(-) create mode 100644 smart-directories/adapters/_adapter.js create mode 100644 smart-directories/adapters/sources.js delete mode 100644 smart-directories/components/directories.js delete mode 100644 smart-directories/components/directory.js rename smart-directories/{main.js => index.js} (62%) create mode 100644 smart-directories/test/test_content.js create mode 100644 smart-groups/adapters/_adapter.js create mode 100644 smart-groups/adapters/data/ajson_multi_file.js create mode 100644 smart-groups/adapters/vector/median_members.js create mode 100644 smart-groups/index.js create mode 100644 smart-groups/package.json create mode 100644 smart-groups/smart_group.js create mode 100644 smart-groups/smart_groups.js diff --git a/smart-directories/adapters/_adapter.js b/smart-directories/adapters/_adapter.js new file mode 100644 index 00000000..32840639 --- /dev/null +++ b/smart-directories/adapters/_adapter.js @@ -0,0 +1,10 @@ +import { GroupAdapter } from '../../smart-groups/adapters/_adapter.js'; + +/** + * @class DirectoryGroupAdapter + * @extends GroupAdapter + * @description + * A base class for group-level adapters that build groups from directories. + */ +export class DirectoryGroupAdapter extends GroupAdapter { +} \ No newline at end of file diff --git a/smart-directories/adapters/sources.js b/smart-directories/adapters/sources.js new file mode 100644 index 00000000..aa49e10d --- /dev/null +++ b/smart-directories/adapters/sources.js @@ -0,0 +1,47 @@ +/** + * @file SourceDirectoryGroupsAdapter.js + * @description Adapts directory groups by scanning SmartSources to build directory items. + */ + +import { GroupCollectionAdapter, GroupItemAdapter } from '../../smart-groups/adapters/_adapter.js'; + +export class SourceDirectoryGroupsAdapter extends GroupCollectionAdapter { + /** + * Build groups by scanning the `smart_sources` collection. + * For each source, derive its directory path and ensure a SmartDirectory group item exists. + */ + async build_groups() { + const source_paths = Object.keys(this.collection.env.smart_sources.items); + const created_dirs = new Set(); + + for (const path of source_paths) { + const dir_path = path.split('/').slice(0, -1).join('/') + '/'; + await this.ensure_parent_directories(dir_path, created_dirs); + } + } + + async ensure_parent_directories(dir_path, created_dirs) { + const parts = dir_path.split('/').filter(p => p); + let current_path = ''; + + for (const part of parts) { + current_path += part + '/'; + if (!created_dirs.has(current_path)) { + const existing = this.collection.get(current_path); + if (!existing) { + const item = this.collection.create_or_update({ path: current_path }); + // item.init() if needed + } + created_dirs.add(current_path); + } + } + } +} + +export class SourceDirectoryGroupAdapter extends GroupItemAdapter { +} + +export default { + collection: SourceDirectoryGroupsAdapter, + item: SourceDirectoryGroupAdapter +}; diff --git a/smart-directories/components/directories.js b/smart-directories/components/directories.js deleted file mode 100644 index 35d00466..00000000 --- a/smart-directories/components/directories.js +++ /dev/null @@ -1,67 +0,0 @@ -import { render as render_directory } from "./directory.js"; - -export async function build_html(directories, opts = {}) { - const html = `
-
-
- - - - -
-
-
-
-
`; - - return html; -} - -export async function render(directories, opts = {}) { - const html = await build_html.call(this, directories, opts); - const frag = this.create_doc_fragment(html); - - // Render each directory - const sg_list = frag.querySelector('.sg-list'); - const directory_frags = await Promise.all( - Object.values(directories.items) - .filter(dir => directories.settings.show_subdirectories ? true : !dir.data.path.slice(0, -1).includes('/')) - .sort((a, b) => a.data.path.localeCompare(b.data.path)) - .map(directory => - render_directory.call(this, directory, opts) - ) - ); - directory_frags.forEach(dir_frag => sg_list.appendChild(dir_frag)); - - return await post_process.call(this, directories, frag, opts); -} - -export async function post_process(directories, frag, opts = {}) { - // Refresh view - const refresh_button = frag.querySelector(".sg-refresh"); - refresh_button.addEventListener("click", () => { - opts.refresh_view(); - }); - - // Help documentation - const help_button = frag.querySelector(".sg-help"); - help_button.addEventListener("click", () => { - window.open("https://docs.smartconnections.app/directories", "_blank"); - }); - - // Sort directories - const sort_button = frag.querySelector(".sg-sort"); - sort_button.addEventListener("click", () => { - directories.settings.sort_nearest = !directories.settings.sort_nearest; - opts.refresh_view(); - }); - - // Show/hide subdirectories - const subdirectories_button = frag.querySelector(".sg-subdirectories"); - subdirectories_button.addEventListener("click", () => { - directories.settings.show_subdirectories = !directories.settings.show_subdirectories; - opts.refresh_view(); - }); - - return frag; -} diff --git a/smart-directories/components/directory.js b/smart-directories/components/directory.js deleted file mode 100644 index 818f055c..00000000 --- a/smart-directories/components/directory.js +++ /dev/null @@ -1,76 +0,0 @@ -import { render as render_results } from "../../smart-entities/components/results.js"; - -export async function build_html(directory, opts = {}) { - const expanded_view = opts.expanded_view || directory.env.settings.expanded_view; - const sources = directory.direct_sources; - const subdirs = directory.direct_subdirectories; - - return `
-
- ${this.get_icon_html('right-triangle')} - - ${directory.data.path.slice(0, -1)} - - - ${sources.length} files${subdirs.length ? `, ${subdirs.length} subdirs` : ''} - -
-
-
-
-
-
`; -} - -export async function render(directory, opts = {}) { - const html = await build_html.call(this, directory, opts); - const frag = this.create_doc_fragment(html); - return await post_process.call(this, directory, frag, opts); -} - -export async function post_process(directory, frag, opts = {}) { - const dir_item = frag.querySelector('.sg-directory-item'); - const sources_container = dir_item.querySelector('.sg-directory-sources'); - const subdirs_container = dir_item.querySelector('.sg-subdirectories'); - - // Toggle expand/collapse with enhanced event handling - const header = dir_item.querySelector('.sg-directory-header'); - - // Handle header click (including icon) - toggle expand/collapse - header.addEventListener('click', async (e) => { - e.stopPropagation(); - - const was_collapsed = dir_item.classList.contains('sg-collapsed'); - dir_item.classList.toggle('sg-collapsed'); - - // Only load content when expanding and content not already loaded - if (was_collapsed && !sources_container.innerHTML.trim()) { - await render_content.call(this, directory, sources_container, subdirs_container, opts); - } - }); - - // Initialize based on expanded state - const start_expanded = opts.expanded_view || directory.env.settings.expanded_view; - if (start_expanded) { - dir_item.classList.remove('sg-collapsed'); - await render_content.call(this, directory, sources_container, subdirs_container, opts); - } - - return dir_item; -} - -async function render_content(directory, sources_container, subdirs_container, opts) { - // Only render if not already rendered - if (!sources_container.innerHTML.trim()) { - sources_container.innerHTML = ''; - subdirs_container.innerHTML = ''; - - const results = directory.settings.sort_nearest - ? await directory.get_nearest_sources_results() - : await directory.get_furthest_sources_results(); - const result_frags = await render_results.call(this, results, opts); - sources_container.appendChild(result_frags); - } -} diff --git a/smart-directories/main.js b/smart-directories/index.js similarity index 62% rename from smart-directories/main.js rename to smart-directories/index.js index 9089c097..ccb9bd22 100644 --- a/smart-directories/main.js +++ b/smart-directories/index.js @@ -1,8 +1,10 @@ import { SmartDirectory } from "./smart_directory.js"; import { SmartDirectories } from "./smart_directories.js"; +import source_directory_group_adapter from "./adapters/sources.js"; export { SmartDirectory, SmartDirectories, + source_directory_group_adapter }; diff --git a/smart-directories/package.json b/smart-directories/package.json index 7a2fc59f..566a3148 100644 --- a/smart-directories/package.json +++ b/smart-directories/package.json @@ -5,7 +5,7 @@ "version": "0.0.1", "type": "module", "description": "Easy to manage embedded directories.", - "main": "main.js", + "main": "index.js", "scripts": { "test": "npx ava --verbose" }, @@ -25,7 +25,7 @@ "ava": "^6.0.1" }, "dependencies": { - "smart-entities": "file:../smart-entities" + "smart-groups": "file:../smart-groups" }, "ava": { "files": [ diff --git a/smart-directories/smart_directories.js b/smart-directories/smart_directories.js index 917190bc..b4ffd3a7 100644 --- a/smart-directories/smart_directories.js +++ b/smart-directories/smart_directories.js @@ -1,11 +1,8 @@ -import { SmartEntities } from "smart-entities"; -import { SmartDirectory } from "./smart_directory.js"; -import { render as render_directories_component } from "./components/directories.js"; +import { SmartGroups } from "smart-groups"; -export class SmartDirectories extends SmartEntities { +export class SmartDirectories extends SmartGroups { static get defaults() { return { - item_type: SmartDirectory, collection_key: 'smart_directories', }; } @@ -46,13 +43,9 @@ export class SmartDirectories extends SmartEntities { */ async init() { await super.init(); - - // Create directories for all source paths - const source_paths = Object.keys(this.env.smart_sources.items); - for (const path of source_paths) { - const dir_path = path.split('/').slice(0, -1).join('/') + '/'; - await this.ensure_parent_directories(dir_path); - } + // Build groups from sources + await this.group_adapter.build_groups(); + await this.process_save_queue(); } /** diff --git a/smart-directories/smart_directory.js b/smart-directories/smart_directory.js index 5e2357f4..bb2cc977 100644 --- a/smart-directories/smart_directory.js +++ b/smart-directories/smart_directory.js @@ -1,8 +1,6 @@ -import { SmartEntity } from "smart-entities"; -import { render as directory_component } from "./components/directory.js"; -import { sort_by_score_ascending, sort_by_score_descending } from "smart-entities/utils/sort_by_score.js"; +import { SmartGroup } from "smart-groups"; -export class SmartDirectory extends SmartEntity { +export class SmartDirectory extends SmartGroup { static get defaults() { return { data: { @@ -25,14 +23,12 @@ export class SmartDirectory extends SmartEntity { async init() { this.data.path = this.data.path.replace(/\\/g, "/"); - // await this.create(this.data.path); this.queue_save(); } get fs() { return this.env.smart_sources.fs; } get file_type() { return 'directory'; } - get smart_embed() { return false; } async read() { @@ -45,7 +41,6 @@ export class SmartDirectory extends SmartEntity { async move_to(new_path) { const old_path = this.data.path; - if (!(await this.fs.exists(old_path))) { throw new Error(`Directory not found: ${old_path}`); } @@ -95,27 +90,17 @@ export class SmartDirectory extends SmartEntity { ); } + /** + * @deprecated use get_nearest_members() instead + */ async get_nearest_sources_results() { - if(!this.median_vec) { - console.log(`no median vec for directory: ${this.data.path}`); - return []; - } - const filter = { - key_starts_with: this.data.path - } - const results = await this.env.smart_sources.nearest(this.median_vec, filter); - return results.sort(sort_by_score_descending); + return this.vector_adapter.nearest_members(); } + /** + * @deprecated use get_furthest_members() instead + */ async get_furthest_sources_results() { - if(!this.median_vec) { - console.log(`no median vec for directory: ${this.data.path}`); - return []; - } - const filter = { - key_starts_with: this.data.path - } - const results = await this.env.smart_sources.furthest(this.median_vec, filter); - return results.sort(sort_by_score_ascending); + return this.vector_adapter.furthest_members(); } /** @@ -151,28 +136,7 @@ export class SmartDirectory extends SmartEntity { * Gets the median vector of all contained sources */ get median_vec() { - if (this.data.median_vec) return this.data.median_vec; - - const source_vecs = this.sources - .map(source => source.vec) - .filter(vec => vec); - - if (!source_vecs.length) return null; - - const vec_length = source_vecs[0].length; - const median_vec = new Array(vec_length); - const mid = Math.floor(source_vecs.length / 2); - - for (let i = 0; i < vec_length; i++) { - const values = source_vecs.map(vec => vec[i]).sort((a, b) => a - b); - median_vec[i] = source_vecs.length % 2 !== 0 - ? values[mid] - : (values[mid - 1] + values[mid]) / 2; - } - - this.data.median_vec = median_vec; - // this.queue_save(); - return median_vec; + return this.vector_adapter.median_vec; } get vec() { return this.median_vec; } @@ -181,74 +145,7 @@ export class SmartDirectory extends SmartEntity { * Gets the median vector of all contained blocks */ get median_block_vec() { - if (this.data.median_block_vec) return this.data.median_block_vec; - - const block_vecs = this.sources - .flatMap(source => source.blocks) - .map(block => block.vec) - .filter(vec => vec); - - if (!block_vecs.length) return null; - - const vec_length = block_vecs[0].length; - const median_vec = new Array(vec_length); - const mid = Math.floor(block_vecs.length / 2); - - for (let i = 0; i < vec_length; i++) { - const values = block_vecs.map(vec => vec[i]).sort((a, b) => a - b); - median_vec[i] = block_vecs.length % 2 !== 0 - ? values[mid] - : (values[mid - 1] + values[mid]) / 2; - } - - this.data.median_block_vec = median_vec; - // this.queue_save(); - return median_vec; - } - - /** - * Performs a lookup within this directory's sources - */ - async lookup(opts = {}) { - return await this.env.smart_sources.lookup({ - ...opts, - filter: { - ...(opts.filter || {}), - key_starts_with: this.data.path - } - }); - } - - // Add method to update directory statistics - async update_stats() { - const sources = this.sources; - this.data.metadata.stats = { - total_files: sources.length, - total_size: sources.reduce((sum, src) => sum + (src.size || 0), 0), - last_scan: Date.now() - }; - this.queue_save(); - } - - // Add method to manage directory labels - async update_label(label, q_score, block_key = null) { - if (!this.data.metadata.labels[label]) { - this.data.metadata.labels[label] = { - q_score: 0, - supporting_blocks: {} - }; - } - - if (block_key) { - this.data.metadata.labels[label].supporting_blocks[block_key] = q_score; - } - - // Recalculate overall q-score for label - const scores = Object.values(this.data.metadata.labels[label].supporting_blocks); - this.data.metadata.labels[label].q_score = - scores.reduce((sum, score) => sum + score, 0) / scores.length; - - this.queue_save(); + return this.vector_adapter.median_block_vec; } // Track directory changes @@ -260,5 +157,4 @@ export class SmartDirectory extends SmartEntity { this.queue_save(); } - get component() { return directory_component; } -} \ No newline at end of file +} diff --git a/smart-directories/test/test_content.js b/smart-directories/test/test_content.js new file mode 100644 index 00000000..8034cdea --- /dev/null +++ b/smart-directories/test/test_content.js @@ -0,0 +1,46 @@ +/** + * @file test_content.js + * Creates test directories and markdown files for integration tests of SmartDirectories. + */ + +import fs from "fs"; +import path from "path"; + +const baseDir = path.join(process.cwd(), "test", "test-directories-content"); +if (!fs.existsSync(baseDir)) { + fs.mkdirSync(baseDir, { recursive: true }); +} + +// Create a nested directory structure +const dirs = [ + 'project/', + 'project/docs/', + 'project/docs/guides/', + 'notes/', + 'notes/2024/', + 'notes/archive/', +]; + +dirs.forEach(d => { + const fullPath = path.join(baseDir, d); + if (!fs.existsSync(fullPath)) { + fs.mkdirSync(fullPath, { recursive: true }); + } +}); + +// Create some markdown files in these directories +const files = { + 'project/README.md': '# Project Readme\nSome basic info.', + 'project/docs/guide.md': '# Guide\nDetailed instructions.', + 'project/docs/guides/advanced.md': '# Advanced Guide\nMore details.', + 'notes/2024/january.md': '# January Notes\n- Task 1\n- Task 2', + 'notes/2024/february.md': '# February Notes\n- Task A\n- Task B', + 'notes/archive/old.md': '# Old Notes\nThis is outdated content.', +}; + +Object.entries(files).forEach(([relPath, content]) => { + const fullPath = path.join(baseDir, relPath); + fs.writeFileSync(fullPath, content, 'utf8'); +}); + +console.log("Test directories and markdown files created at:", baseDir); \ No newline at end of file diff --git a/smart-entities/smart_entity.js b/smart-entities/smart_entity.js index f4febd26..1f00d9e0 100644 --- a/smart-entities/smart_entity.js +++ b/smart-entities/smart_entity.js @@ -24,6 +24,7 @@ export class SmartEntity extends CollectionItem { super(env, opts); /** * @type {DefaultEntityVectorAdapter} + * @deprecated use vector_adapter instead * @description Adapter for this entity's vector operations. */ this.entity_adapter = new DefaultEntityVectorAdapter(this); @@ -46,6 +47,12 @@ export class SmartEntity extends CollectionItem { }, }; } + get vector_adapter() { + if(!this._vector_adapter) { + this._vector_adapter = new this.collection.opts.vector_adapter.item(this); + } + return this._vector_adapter; + } /** * Initializes the SmartEntity instance. diff --git a/smart-groups/adapters/_adapter.js b/smart-groups/adapters/_adapter.js new file mode 100644 index 00000000..c91c80b7 --- /dev/null +++ b/smart-groups/adapters/_adapter.js @@ -0,0 +1,39 @@ +/** + * @file _adapter.js + * @description Base adapter classes for Groups (clusters, directories). + * + * Similar to how SourceContentAdapter works for sources, we have a GroupAdapter interface + * and concrete implementations that handle building and maintaining directory groups. + */ + +/** + * @interface GroupAdapter + * @description + * Provides an interface for building and maintaining groups from underlying data sources. + * Groups are collections of items (e.g., directories) derived from another primary collection (e.g., sources). + */ +export class GroupCollectionAdapter { + /** + * @constructor + * @param {Object} collection - The group collection instance. + */ + constructor(collection) { + this.collection = collection; + } + /** + * Build groups by scanning the primary source collection. + * This should identify group keys (e.g., directory paths) and create/update items. + * @async + * @returns {Promise} + */ + async build_groups() { throw new Error("Not implemented"); } +} + +export class GroupItemAdapter { +} + +export default { + collection: GroupCollectionAdapter, + item: GroupItemAdapter +}; + diff --git a/smart-groups/adapters/data/ajson_multi_file.js b/smart-groups/adapters/data/ajson_multi_file.js new file mode 100644 index 00000000..f4f03b9a --- /dev/null +++ b/smart-groups/adapters/data/ajson_multi_file.js @@ -0,0 +1,40 @@ +import { AjsonMultiFileCollectionDataAdapter, AjsonMultiFileItemDataAdapter } from 'smart-collections/adapters/ajson_multi_file.js'; + +/** + * @class AjsonMultiFileGroupsDataAdapter + * @extends AjsonMultiFileCollectionDataAdapter + * @description + * Similar to the sources adapter, but for groups (directories). + * Handles load/save/delete operations for directory groups. + */ +export class AjsonMultiFileGroupsDataAdapter extends AjsonMultiFileCollectionDataAdapter { + ItemDataAdapter = AjsonMultiFileGroupDataAdapter; +} + +/** + * @class AjsonMultiFileGroupDataAdapter + * @extends AjsonMultiFileItemDataAdapter + * @description + * Handles individual directory group items stored in append-only AJSON files. + */ +export class AjsonMultiFileGroupDataAdapter extends AjsonMultiFileItemDataAdapter { + get_data_path() { + const dir = this.collection_adapter.collection.data_dir || 'groups'; + const sep = this.fs?.sep || '/'; + const file_name = this._get_data_file_name(this.item.key); + return dir + sep + file_name + '.ajson'; + } + + _get_data_file_name(key) { + return key + .replace(/[\s\/\.]/g, '_') + .replace(/^_+/, '') + .replace(/_+$/, '') + ; + } +} + +export default { + collection: AjsonMultiFileGroupsDataAdapter, + item: AjsonMultiFileGroupDataAdapter +}; diff --git a/smart-groups/adapters/vector/median_members.js b/smart-groups/adapters/vector/median_members.js new file mode 100644 index 00000000..c72d295e --- /dev/null +++ b/smart-groups/adapters/vector/median_members.js @@ -0,0 +1,79 @@ +import { DefaultEntitiesVectorAdapter, DefaultEntityVectorAdapter } from "smart-entities/adapters/default.js"; +import { sort_by_score_ascending, sort_by_score_descending } from "smart-entities/utils/sort_by_score.js"; + +export class MedianMemberVectorsAdapter extends DefaultEntitiesVectorAdapter { +} + +export class MedianMemberVectorAdapter extends DefaultEntityVectorAdapter { + get members() { + return this.item.members; + } + get member_collection() { + return this.item.member_collection; + } + async nearest_members(filter = {}) { + filter.key_starts_with = this.item.data.path; + const results = await this.member_collection.nearest(this.median_vec, filter); + return results.sort(sort_by_score_descending); + } + async furthest_members(filter = {}) { + filter.key_starts_with = this.item.data.path; + const results = await this.member_collection.furthest(this.median_vec, filter); + return results.sort(sort_by_score_ascending); + } + get median_vec() { + if (!this._median_vec) { + this._median_vec = this.calculate_median_vec(); + } + return this._median_vec; + } + calculate_median_vec() { + const member_vecs = this.members + .map(member => member.vec) + .filter(vec => vec); + + if (!member_vecs.length) return null; + const vec_length = member_vecs[0].length; + const median_vec = new Array(vec_length); + const mid = Math.floor(member_vecs.length / 2); + for (let i = 0; i < vec_length; i++) { + const values = member_vecs.map(vec => vec[i]).sort((a, b) => a - b); + median_vec[i] = member_vecs.length % 2 !== 0 + ? values[mid] + : (values[mid - 1] + values[mid]) / 2; + } + return median_vec; + } + get median_block_vec() { + if (!this._median_block_vec) { + if(!this.item.members[0].blocks.length){ + throw new Error("No blocks found for median block vector calculation"); + } + this._median_block_vec = this.calculate_median_block_vec(); + } + return this._median_block_vec; + } + calculate_median_block_vec() { + const block_vecs = this.members + .flatMap(member => member.blocks) + .map(block => block.vec) + .filter(vec => vec); + + if (!block_vecs.length) return null; + + const vec_length = block_vecs[0].length; + const median_vec = new Array(vec_length); + const mid = Math.floor(block_vecs.length / 2); + for (let i = 0; i < vec_length; i++) { + const values = block_vecs.map(vec => vec[i]).sort((a, b) => a - b); + median_vec[i] = block_vecs.length % 2 !== 0 + ? values[mid] + : (values[mid - 1] + values[mid]) / 2; + } + return median_vec; + } +} +export default { + collection: MedianMemberVectorsAdapter, + item: MedianMemberVectorAdapter +} \ No newline at end of file diff --git a/smart-groups/index.js b/smart-groups/index.js new file mode 100644 index 00000000..4e52ea43 --- /dev/null +++ b/smart-groups/index.js @@ -0,0 +1,11 @@ +import { SmartGroup } from "./smart_group.js"; +import { SmartGroups } from "./smart_groups.js"; +import ajson_multi_file_groups_data_adapter from "./adapters/data/ajson_multi_file.js"; +import median_members_vector_adapter from "./adapters/vector/median_members.js"; + +export { + SmartGroup, + SmartGroups, + ajson_multi_file_groups_data_adapter, + median_members_vector_adapter +}; \ No newline at end of file diff --git a/smart-groups/package.json b/smart-groups/package.json new file mode 100644 index 00000000..a4a797e0 --- /dev/null +++ b/smart-groups/package.json @@ -0,0 +1,35 @@ +{ + "name": "smart-directories", + "author": "Brian Joseph Petro (🌴 Brian)", + "license": "MIT", + "version": "0.0.1", + "type": "module", + "description": "Easy to manage embedded directories.", + "main": "index.js", + "scripts": { + "test": "npx ava --verbose" + }, + "keywords": [ + "embeddings", + "directories" + ], + "repository": { + "type": "git", + "url": "brianpetro/jsbrains" + }, + "bugs": { + "url": "https://github.com/brianpetro/jsbrains/issues" + }, + "homepage": "https://jsbrains.org", + "devDependencies": { + "ava": "^6.0.1" + }, + "dependencies": { + "smart-entities": "file:../smart-entities" + }, + "ava": { + "files": [ + "test/**/*.test.js" + ] + } +} diff --git a/smart-groups/smart_group.js b/smart-groups/smart_group.js new file mode 100644 index 00000000..e3575d12 --- /dev/null +++ b/smart-groups/smart_group.js @@ -0,0 +1,127 @@ +import { SmartEntity } from "smart-entities"; + +export class SmartGroup extends SmartEntity { + static get defaults() { + return { + data: { + path: '', + median_vec: null, + median_block_vec: null, + member_collection: null, + members: [], // Cache of contained member keys + metadata: { + labels: {}, // Store directory labels/tags with q-scores + last_modified: 0, // Track directory changes + stats: { // Directory statistics + total_files: 0, + total_size: 0, + last_scan: 0 + } + } + }, + }; + } + get group_adapter() { + if(!this._group_adapter) { + this._group_adapter = new this.collection.opts.group_adapter.item(this); + } + return this._group_adapter; + } + + /** + * Gets all SmartSources contained in this directory + * @returns {SmartSource[]} Array of SmartSource instances + */ + get members() { + return this.member_collection.filter(source => + source.path.startsWith(this.data.path) + ); + } + get member_collection() { + return this.env[this.member_collection_key]; + } + get member_collection_key() { + return this.data.member_collection || 'smart_sources'; + } + + async get_nearest_members() { + if(!this.median_vec) { + console.log(`no median vec for directory: ${this.data.path}`); + return []; + } + return this.vector_adapter.nearest_members(); + } + async get_furthest_members() { + if(!this.median_vec) { + console.log(`no median vec for directory: ${this.data.path}`); + return []; + } + return this.vector_adapter.furthest_members(); + } + + /** + * Gets the median vector of all contained sources + */ + get median_vec() { + return this.entity_adapter.median_vec; + } + get vec() { return this.median_vec; } + + /** + * Gets the median vector of all contained blocks + */ + get median_block_vec() { + if (this.data.median_block_vec) return this.data.median_block_vec; + + const block_vecs = this.sources + .flatMap(source => source.blocks) + .map(block => block.vec) + .filter(vec => vec); + + if (!block_vecs.length) return null; + + const vec_length = block_vecs[0].length; + const median_vec = new Array(vec_length); + const mid = Math.floor(block_vecs.length / 2); + for (let i = 0; i < vec_length; i++) { + const values = block_vecs.map(vec => vec[i]).sort((a, b) => a - b); + median_vec[i] = block_vecs.length % 2 !== 0 + ? values[mid] + : (values[mid - 1] + values[mid]) / 2; + } + this.data.median_block_vec = median_vec; + return median_vec; + } + + // Add method to update directory statistics + async update_stats() { + const sources = this.sources; + this.data.metadata.stats = { + total_files: sources.length, + total_size: sources.reduce((sum, src) => sum + (src.size || 0), 0), + last_scan: Date.now() + }; + this.queue_save(); + } + + // Add method to manage directory labels + async update_label(label, q_score, block_key = null) { + if (!this.data.metadata.labels[label]) { + this.data.metadata.labels[label] = { + q_score: 0, + supporting_blocks: {} + }; + } + if (block_key) { + this.data.metadata.labels[label].supporting_blocks[block_key] = q_score; + } + + // Recalculate overall q-score for label + const scores = Object.values(this.data.metadata.labels[label].supporting_blocks); + this.data.metadata.labels[label].q_score = + scores.reduce((sum, score) => sum + score, 0) / scores.length; + + this.queue_save(); + } + +} \ No newline at end of file diff --git a/smart-groups/smart_groups.js b/smart-groups/smart_groups.js new file mode 100644 index 00000000..4e12b3ae --- /dev/null +++ b/smart-groups/smart_groups.js @@ -0,0 +1,16 @@ +import { SmartEntities } from "smart-entities"; + +export class SmartGroups extends SmartEntities { + get data_dir() { return 'groups'; } + + /** + * Introduce a group adapter to build directory structure from sources + */ + get group_adapter() { + if (!this._group_adapter) { + this._group_adapter = new this.opts.group_adapter.collection(this); + } + return this._group_adapter; + } + +}