Skip to content

Commit

Permalink
work on test cases
Browse files Browse the repository at this point in the history
  • Loading branch information
brianpetro committed Oct 3, 2024
1 parent 3067bc9 commit 54906a5
Show file tree
Hide file tree
Showing 22 changed files with 1,155 additions and 270 deletions.
36 changes: 18 additions & 18 deletions smart-collections/adapters/_adapter.js
Original file line number Diff line number Diff line change
@@ -1,20 +1,3 @@
export class SmartCollectionItemDataAdapter{
constructor(item) {
this.item = item;
}
// REQUIRED METHODS IN SUBCLASSES
async load() { throw new Error("SmartCollectionItemAdapter: load() not implemented"); }
async save() { throw new Error("SmartCollectionItemAdapter: save() not implemented"); }
// END REQUIRED METHODS IN SUBCLASSES

get env() { return this.item.env; }
get data_path() { return 'collection_item.json'; }
get key() { return this.item.key; }
get collection_key() { return this.item.collection_key; }
get collection() { return this.item.collection; }

}

export class SmartCollectionDataAdapter{
constructor(collection) {
this.collection = collection;
Expand All @@ -28,4 +11,21 @@ export class SmartCollectionDataAdapter{
get data_path() { return 'collection.json'; }
get collection_key() { return this.collection.collection_key; }

}
}

// export class SmartCollectionItemDataAdapter{
// constructor(item) {
// this.item = item;
// }
// // REQUIRED METHODS IN SUBCLASSES
// async load() { throw new Error("SmartCollectionItemAdapter: load() not implemented"); }
// async save() { throw new Error("SmartCollectionItemAdapter: save() not implemented"); }
// // END REQUIRED METHODS IN SUBCLASSES

// get env() { return this.item.env; }
// get data_path() { return 'collection_item.json'; }
// get key() { return this.item.key; }
// get collection_key() { return this.item.collection_key; }
// get collection() { return this.item.collection; }

// }
102 changes: 82 additions & 20 deletions smart-collections/adapters/_test.js
Original file line number Diff line number Diff line change
@@ -1,30 +1,92 @@
import fs from 'fs';
import { SmartCollectionItemDataAdapter } from './_adapter.js';
export class TestSmartCollectionDataAdapter extends SmartCollectionItemDataAdapter{
import { SmartCollectionDataAdapter } from './_adapter.js';
import { ajson_merge } from '../utils/ajson_merge.js';

const class_to_collection_key = {
'SmartSource': 'smart_sources',
'SmartNote': 'smart_sources', // DEPRECATED: added for backward compatibility
'SmartBlock': 'smart_blocks',
'SmartDirectory': 'smart_directories',
};

export class SmartCollectionTestDataAdapter extends SmartCollectionDataAdapter {
constructor(collection) {
super(collection);
this.test_data = JSON.parse(fs.readFileSync(this.data_path, 'utf8'));
}

get data_path() { return "./test/_data.json"; }
async load() {
// console.log("Loading test collection...");
Object.entries(this.test_data).forEach(([ajson_key, value]) => {
const [class_name, key] = ajson_key.split(":");
const entity = new (this.env.item_types[class_name])(this.env, value);
// if value has content, this.collection.fs.write()
if(value.content) this.collection.fs.write(key, value.content);
this.add_to_collection(entity);
});
// console.log("Loaded test collection");

async load(item) {
try {
const ajson_key = `${item.constructor.name}:${item.key}`;
const data = this.test_data[ajson_key];

if (!data) {
console.log(`Data not found for: ${ajson_key}`);
return item.queue_import();
}

const parsed_data = {};
data.split('\n').forEach(line => {
try {
const parsed = JSON.parse(`{${line}}`);
if (Object.values(parsed)[0] === null) {
if (parsed_data[Object.keys(parsed)[0]]) delete parsed_data[Object.keys(parsed)[0]];
} else {
Object.assign(parsed_data, parsed);
}
} catch (err) {
console.warn("Error parsing line: ", line);
console.warn(err.stack);
}
});

Object.entries(parsed_data).forEach(([parsed_ajson_key, value]) => {
if (!value) return; // handle null values (deleted)
const [class_name, ...key_parts] = parsed_ajson_key.split(":");
const entity_key = key_parts.join(":");
if (entity_key === item.key) {
item.data = value;
} else {
if (!this.env[class_to_collection_key[class_name]]) {
return console.warn(`Collection class not found: ${class_name}`);
}
this.env[class_to_collection_key[class_name]].items[entity_key] = new this.env.item_types[class_name](this.env, value);
}
});

item._queue_load = false;
item.loaded_at = Date.now();
} catch (err) {
console.log(`Error loading collection item: ${item.key}`);
console.warn(err.stack);
item.queue_load();
}
}
async save_item(key) {
delete this._save_queue[key];
const item = this.collection.get(key);
if(!item) return console.warn("Item not found: " + key);
if(!item.deleted) this.test_data[key] = item.ajson;
else {
delete this.test_data[key];
this.collection.delete_item(key);

async save(item, ajson = null) {
if (!ajson) ajson = item.ajson;
const ajson_key = `${item.constructor.name}:${item.key}`;

try {
if (item.deleted) {
this.collection.delete_item(item.key);
delete this.test_data[ajson_key];
} else {
this.test_data[ajson_key] += '\n' + ajson;
}

// Simulate writing to file
fs.writeFileSync(this.data_path, JSON.stringify(this.test_data, null, 2));

item._queue_save = false;
return true;
} catch (err) {
console.warn(`Error saving collection item: ${item.key}`);
console.warn(err.stack);
item.queue_save();
return false;
}
}
}
168 changes: 84 additions & 84 deletions smart-collections/adapters/multi_file.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ajson_merge } from '../utils/ajson_merge.js';
import { SmartCollectionItemDataAdapter, SmartCollectionDataAdapter } from './_adapter.js';
import { SmartCollectionDataAdapter } from './_adapter.js';


// DO: replace this better way in future
Expand Down Expand Up @@ -99,91 +99,91 @@ export class SmartCollectionMultiFileDataAdapter extends SmartCollectionDataAdap



export class MultiFileSmartCollectionItemDataAdapter extends SmartCollectionItemDataAdapter {
get fs() { return this.collection.data_fs || this.env.data_fs; }
/**
* @returns {string} The data folder that contains .ajson files.
*/
get data_folder() { return 'multi'; }
/**
* @returns {string} The data path for .ajson file.
*/
get data_path() { return this.data_folder + "/" + this.item.multi_ajson_file_name + '.ajson'; }
// export class MultiFileSmartCollectionItemDataAdapter extends SmartCollectionItemDataAdapter {
// get fs() { return this.collection.data_fs || this.env.data_fs; }
// /**
// * @returns {string} The data folder that contains .ajson files.
// */
// get data_folder() { return 'multi'; }
// /**
// * @returns {string} The data path for .ajson file.
// */
// get data_path() { return this.data_folder + "/" + this.item.multi_ajson_file_name + '.ajson'; }

/**
* Asynchronously loads collection item data from .ajson file specified by data_path.
*/
async load() {
try{
const data_ajson = (await this.fs.read(this.data_path)).trim();
if(!data_ajson){
return this.item.queue_import(); // queue import and return early if data file missing or empty
}
const ajson_lines = data_ajson.split('\n');
const parsed_data = ajson_lines
.reduce((acc, line) => {
try{
const parsed = JSON.parse(`{${line}}`);
if(Object.values(parsed)[0] === null){
if(acc[Object.keys(parsed)[0]]) delete acc[Object.keys(parsed)[0]];
return acc;
}
return ajson_merge(acc, parsed);
}catch(err){
console.warn("Error parsing line: ", line);
console.warn(err.stack);
return acc;
}
}, {})
;
// array with same length as parsed_data
const rebuilt_ajson = [];
Object.entries(parsed_data)
.forEach(([ajson_key, value], index) => {
if(!value) return; // handle null values (deleted)
rebuilt_ajson.push(`${JSON.stringify(ajson_key)}: ${JSON.stringify(value)}`);
const [class_name, ...key_parts] = ajson_key.split(":");
const entity_key = key_parts.join(":"); // key is file path
if(entity_key === this.key) this.item.data = value;
else {
if(!this.env[class_to_collection_key[class_name]]) return console.warn(`Collection class not found: ${class_name}`);
this.env[class_to_collection_key[class_name]].items[entity_key] = new this.env.item_types[class_name](this.env, value);
}
})
;
this.item._queue_load = false;
if(ajson_lines.length !== Object.keys(parsed_data).length) this.fs.write(this.data_path, rebuilt_ajson.join('\n'));
}catch(err){
// if file not found, queue import
if(err.message.includes("ENOENT")) return this.item.queue_import();
console.log("Error loading collection item: " + this.key);
console.warn(err.stack);
this.item.queue_load();
return;
}
}
// /**
// * Asynchronously loads collection item data from .ajson file specified by data_path.
// */
// async load() {
// try{
// const data_ajson = (await this.fs.read(this.data_path)).trim();
// if(!data_ajson){
// return this.item.queue_import(); // queue import and return early if data file missing or empty
// }
// const ajson_lines = data_ajson.split('\n');
// const parsed_data = ajson_lines
// .reduce((acc, line) => {
// try{
// const parsed = JSON.parse(`{${line}}`);
// if(Object.values(parsed)[0] === null){
// if(acc[Object.keys(parsed)[0]]) delete acc[Object.keys(parsed)[0]];
// return acc;
// }
// return ajson_merge(acc, parsed);
// }catch(err){
// console.warn("Error parsing line: ", line);
// console.warn(err.stack);
// return acc;
// }
// }, {})
// ;
// // array with same length as parsed_data
// const rebuilt_ajson = [];
// Object.entries(parsed_data)
// .forEach(([ajson_key, value], index) => {
// if(!value) return; // handle null values (deleted)
// rebuilt_ajson.push(`${JSON.stringify(ajson_key)}: ${JSON.stringify(value)}`);
// const [class_name, ...key_parts] = ajson_key.split(":");
// const entity_key = key_parts.join(":"); // key is file path
// if(entity_key === this.key) this.item.data = value;
// else {
// if(!this.env[class_to_collection_key[class_name]]) return console.warn(`Collection class not found: ${class_name}`);
// this.env[class_to_collection_key[class_name]].items[entity_key] = new this.env.item_types[class_name](this.env, value);
// }
// })
// ;
// this.item._queue_load = false;
// if(ajson_lines.length !== Object.keys(parsed_data).length) this.fs.write(this.data_path, rebuilt_ajson.join('\n'));
// }catch(err){
// // if file not found, queue import
// if(err.message.includes("ENOENT")) return this.item.queue_import();
// console.log("Error loading collection item: " + this.key);
// console.warn(err.stack);
// this.item.queue_load();
// return;
// }
// }

async save() {
if(!(await this.fs.exists(this.data_folder))) await this.fs.mkdir(this.data_folder);
try {
if(this.item.deleted){
this.collection.delete_item(this.key);
if((await this.fs.exists(this.data_path))) await this.fs.remove(this.data_path);
} else {
// await this.fs.write(this.data_path, this.item.ajson);
await this.fs.append(this.data_path, '\n' + this.item.ajson); // prevent overwriting the file
}
this.item._queue_save = false;
return true;
} catch (err) {
if(err.message.includes("ENOENT")) return; // already deleted
console.warn("Error saving collection item: ", this.key);
console.warn(err.stack);
this.item.queue_save();
return false;
}
}
}
// async save() {
// if(!(await this.fs.exists(this.data_folder))) await this.fs.mkdir(this.data_folder);
// try {
// if(this.item.deleted){
// this.collection.delete_item(this.key);
// if((await this.fs.exists(this.data_path))) await this.fs.remove(this.data_path);
// } else {
// // await this.fs.write(this.data_path, this.item.ajson);
// await this.fs.append(this.data_path, '\n' + this.item.ajson); // prevent overwriting the file
// }
// this.item._queue_save = false;
// return true;
// } catch (err) {
// if(err.message.includes("ENOENT")) return; // already deleted
// console.warn("Error saving collection item: ", this.key);
// console.warn(err.stack);
// this.item.queue_save();
// return false;
// }
// }
// }


/**
Expand Down
8 changes: 4 additions & 4 deletions smart-collections/test/_env.js
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import { CollectionItem } from '../main.js';
import { Collection } from '../main.js';
import { TestSmartCollectionAdapter } from '../adapters/_test.js';
import { SmartCollectionTestDataAdapter } from '../adapters/_test.js';
import { SmartEnv } from '../../smart-environment/smart_env.js';
import { SmartChunks } from '../../smart-chunks/smart_chunks.js';
import { SmartEmbedModel } from '../../smart-embed-model/smart_embed_model.js';
import { SmartFs } from '../../smart-fs/smart_fs.js';
import { TestSmartFsAdapter } from '../../smart-fs/adapters/_test.js';
import { SmartFsTestAdapter } from '../../smart-fs/adapters/_test.js';

const __dirname = new URL('.', import.meta.url).pathname;

Expand All @@ -22,13 +22,13 @@ class TestMain {
smart_embed_model: SmartEmbedModel,
smart_fs: {
class: SmartFs,
adapter: TestSmartFsAdapter,
adapter: SmartFsTestAdapter,
}
},
collections: {
collection: {
class: Collection,
data_adapter: TestSmartCollectionAdapter,
data_adapter: SmartCollectionTestDataAdapter,
},
},
item_types: {
Expand Down
2 changes: 1 addition & 1 deletion smart-embed-model-v1/smart_embed_model.js
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ export class SmartEmbedModel extends SmartModel {
if(!this.opts.adapter) return console.warn('SmartEmbedModel adapter not set');
if(!this.opts.adapters[this.opts.adapter]) return console.warn(`SmartEmbedModel adapter ${this.opts.adapter} not found`);
// prepare opts for GPU (likely better handled in future)
this.opts.use_gpu = !!navigator.gpu && this.opts.gpu_batch_size !== 0;
this.opts.use_gpu = typeof navigator !== 'undefined' && !!navigator?.gpu && this.opts.gpu_batch_size !== 0;
if(this.opts.adapter === 'transformers' && this.opts.use_gpu) this.opts.batch_size = this.opts.gpu_batch_size || 10;
}
get adapters() { return this.opts.adapters || this.env.opts.modules.smart_embed_model.adapters; }
Expand Down
2 changes: 1 addition & 1 deletion smart-embed-model/smart_embed_model.js
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ export class SmartEmbedModel extends SmartModel {
if (!this.opts.adapter) return console.warn('SmartEmbedModel adapter not set');
if (!this.opts.adapters[this.opts.adapter]) return console.warn(`SmartEmbedModel adapter ${this.opts.adapter} not found`);
// prepare opts for GPU (likely better handled in future)
this.opts.use_gpu = !!navigator.gpu && this.opts.gpu_batch_size !== 0;
this.opts.use_gpu = !!navigator?.gpu && this.opts.gpu_batch_size !== 0;
if (this.opts.adapter === 'transformers' && this.opts.use_gpu) this.opts.batch_size = this.opts.gpu_batch_size || 10;
}
async load() {
Expand Down
Loading

0 comments on commit 54906a5

Please sign in to comment.