Skip to content

Commit

Permalink
Renamed 'overlord' to 'core' as it is a more obvious name.
Browse files Browse the repository at this point in the history
  • Loading branch information
LTLA committed Dec 16, 2023
1 parent 2186e03 commit 228ea76
Show file tree
Hide file tree
Showing 9 changed files with 27 additions and 27 deletions.
10 changes: 5 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ Each JSON schema is used to generate its a separate SQLite file that contains on

### File contents

Each SQLite file contains a `primary` table.
Each SQLite file contains a `core` table.
Each row corresponds to an indexed object in **gypsum**.
The table will have at least the following fields, all of which have the `TEXT` type:

Expand All @@ -37,17 +37,17 @@ The table contains at least the `_key` column (same as above), with one or more
If the items are booleans, integers, strings or numbers, the table contains exactly one `item` column of the corresponding type;
if the items are objects, the table contains columns corresponding to the properties of the object.

For `primary` and `multi_<FIELD>` tables, we generate an index for each column.
For `core` and `multi_<FIELD>` tables, we generate an index for each column.
Clients can perform most complex queries efficiently with the relevant inner joins.

### Table generation rules

Each JSON schema is used to generate a SQLite table according to some simple rules:

- The schema must be a top-level `"type": "object"`.
- An `integer` property is converted to an `INTEGER` field on the `primary` table.
- A `boolean` property is converted to an `INTEGER` field on the `primary` table.
- A `number` property is converted to a `REAL` field on the `primary` table.
- An `integer` property is converted to an `INTEGER` field on the `core` table.
- A `boolean` property is converted to an `INTEGER` field on the `core` table.
- A `number` property is converted to a `REAL` field on the `core` table.
- A `string` property is usually converted to a `TEXT` field.
However, if its `_attributes` contain `"free_text"`, it will instead be converted into a column of `free_text`.
- An `array` property is converted to a separate `multi_<FIELD>` table, where `<FIELD>` is the name of the property.
Expand Down
10 changes: 5 additions & 5 deletions src/converterFromSchema.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
export function converterFromSchema(schema) {
let overlord_inserts = [];
let core_inserts = [];
let free_text_inserts = [];
let array_simple = {};
let array_complex = {};
Expand All @@ -9,7 +9,7 @@ export function converterFromSchema(schema) {
if ("_attributes" in x && x._attributes.indexOf("free_text") >= 0) {
free_text_inserts.push(n);
} else {
overlord_inserts.push(n);
core_inserts.push(n);
}

} else if (x.type == "array") {
Expand All @@ -21,7 +21,7 @@ export function converterFromSchema(schema) {
}

} else {
overlord_inserts.push(n);
core_inserts.push(n);
}
}

Expand All @@ -32,14 +32,14 @@ export function converterFromSchema(schema) {
{
let current_values = [key, project, asset, version, path, object];
let current_columns = ["_key", "_project", "_asset", "_version", "_path", "_object"];
for (const y of overlord_inserts) {
for (const y of core_inserts) {
if (y in metadata) {
current_columns.push(y);
current_values.push(metadata[y]);
}
}
statements.push({
statement: `INSERT INTO overlord (${current_columns}) VALUES(${Array(current_columns.length).fill('?')});`,
statement: `INSERT INTO core (${current_columns}) VALUES(${Array(current_columns.length).fill('?')});`,
parameters: current_values
});
}
Expand Down
6 changes: 3 additions & 3 deletions src/initializeFromSchema.js
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,10 @@ export function initializeFromSchema(schema) {
}
}

commands.push("DROP TABLE IF EXISTS overlord;");
commands.push(`CREATE TABLE overlord (_key TEXT PRIMARY_KEY, ${main_table.map(y => y[0] + " " + y[1])});`);
commands.push("DROP TABLE IF EXISTS core;");
commands.push(`CREATE TABLE core (_key TEXT PRIMARY_KEY, ${main_table.map(y => y[0] + " " + y[1])});`);
for (const y of main_table) {
commands.push(`CREATE INDEX index_overlord_${y[0]} ON overlord(${y[0]});`);
commands.push(`CREATE INDEX index_core_${y[0]} ON core(${y[0]});`);
}

commands.push("DROP TABLE IF EXISTS free_text;");
Expand Down
4 changes: 2 additions & 2 deletions src/wiperFromSchema.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
export function wiperFromSchema(schema) {
const tables = new Set(["overlord"]);
const tables = new Set(["core"]);

for (const [n, x] of Object.entries(schema.properties)) {
if (x.type == "string") {
Expand All @@ -12,7 +12,7 @@ export function wiperFromSchema(schema) {
}

return (project, asset, version) => {
let selector = "CREATE TEMP TABLE tmp_deleted AS SELECT _key FROM overlord WHERE _project = ?";
let selector = "CREATE TEMP TABLE tmp_deleted AS SELECT _key FROM core WHERE _project = ?";
let params = [ project ];

if (asset !== null) {
Expand Down
2 changes: 1 addition & 1 deletion tests/converterFromSchema.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ test("converterFromSchema works as expected", () => {
res = db.prepare("SELECT COUNT(*) FROM multi_taxonomy_id").all();
expect(res[0]["COUNT(*)"]).toBe(2);

res = db.prepare("SELECT COUNT(*) FROM overlord").all();
res = db.prepare("SELECT COUNT(*) FROM core").all();
expect(res[0]["COUNT(*)"]).toBe(1);

res = db.prepare("SELECT COUNT(*) FROM free_text").all();
Expand Down
2 changes: 1 addition & 1 deletion tests/initializeFromSchema.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ test("initializerFromSchema works as expected", () => {

let listing = db.pragma("table_list");
let available = listing.filter(y => y.type == "table" || y.type == "virtual").map(y => y.name);
expect(available.indexOf("overlord") >= 0).toBe(true);
expect(available.indexOf("core") >= 0).toBe(true);
expect(available.indexOf("free_text") >= 0).toBe(true);
expect(available.indexOf("multi_sources") >= 0).toBe(true);
expect(available.indexOf("multi_genome") >= 0).toBe(true);
Expand Down
12 changes: 6 additions & 6 deletions tests/scripts/handleAction.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ const asset = "zeisel-brain-2015";
const version = "2023-12-14";

function extract_count(handle) {
let res = handle.prepare("SELECT COUNT(*) FROM overlord").all();
let res = handle.prepare("SELECT COUNT(*) FROM core").all();
return res[0]["COUNT(*)"];
}

Expand Down Expand Up @@ -74,7 +74,7 @@ test("handleAction works correctly when deleting a version", async () => {
// Injecting a bunch of crap.
for (const db of Object.values(handles)) {
for (const key of ["foo", "bar"]) {
db.prepare("INSERT INTO overlord(_key, _project, _asset) VALUES(?,?,?)").run(key, project, asset);
db.prepare("INSERT INTO core(_key, _project, _asset) VALUES(?,?,?)").run(key, project, asset);
}
}
expect(extract_count(handles.bioconductor)).toEqual(2);
Expand All @@ -98,8 +98,8 @@ test("handleAction works correctly when deleting an asset", async () => {

// Injecting a bunch of crap.
for (const db of Object.values(handles)) {
db.prepare("INSERT INTO overlord(_key, _project, _asset) VALUES(?,?,?)").run("YAY", project, asset);
db.prepare("INSERT INTO overlord(_key, _project, _asset) VALUES(?,?,?)").run("YAY2", project, asset + "2");
db.prepare("INSERT INTO core(_key, _project, _asset) VALUES(?,?,?)").run("YAY", project, asset);
db.prepare("INSERT INTO core(_key, _project, _asset) VALUES(?,?,?)").run("YAY2", project, asset + "2");
}
expect(extract_count(handles.bioconductor)).toEqual(2);
expect(extract_count(handles.scRNAseq)).toEqual(2);
Expand All @@ -119,8 +119,8 @@ test("handleAction works correctly when deleting a project", async () => {

// Injecting a bunch of crap.
for (const db of Object.values(handles)) {
db.prepare("INSERT INTO overlord(_key, _project, _asset) VALUES(?,?,?)").run("YAY", project, asset);
db.prepare("INSERT INTO overlord(_key, _project, _asset) VALUES(?,?,?)").run("YAY2", project, asset + "2");
db.prepare("INSERT INTO core(_key, _project, _asset) VALUES(?,?,?)").run("YAY", project, asset);
db.prepare("INSERT INTO core(_key, _project, _asset) VALUES(?,?,?)").run("YAY2", project, asset + "2");
}
expect(extract_count(handles.bioconductor)).toEqual(2);
expect(extract_count(handles.scRNAseq)).toEqual(2);
Expand Down
4 changes: 2 additions & 2 deletions tests/scripts/indexVersion.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ test("indexAction works correctly", async () => {
let statements = await indexVersion(project, asset, version, cv, cc);

expect(statements.bioconductor.length).toBeGreaterThan(1);
expect(statements.bioconductor[0].statement).toMatch("overlord");
expect(statements.bioconductor[0].statement).toMatch("core");
expect(statements.bioconductor[1].statement).toMatch("free_text");

expect(statements.scRNAseq.length).toBeGreaterThan(0);
expect(statements.bioconductor[0].statement).toMatch("overlord");
expect(statements.bioconductor[0].statement).toMatch("core");
})
4 changes: 2 additions & 2 deletions tests/wiperFromSchema.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ test("wiperFromSchema works as expected", () => {
res = db.prepare("SELECT COUNT(*) FROM multi_taxonomy_id").all();
expect(res[0]["COUNT(*)"]).toBe(3);

res = db.prepare("SELECT COUNT(*) FROM overlord").all();
res = db.prepare("SELECT COUNT(*) FROM core").all();
expect(res[0]["COUNT(*)"]).toBe(2);

res = db.prepare("SELECT COUNT(*) FROM free_text").all();
Expand All @@ -43,7 +43,7 @@ test("wiperFromSchema works as expected", () => {
res = db.prepare("SELECT COUNT(*) FROM multi_taxonomy_id").all();
expect(res[0]["COUNT(*)"]).toBe(1);

res = db.prepare("SELECT COUNT(*) FROM overlord").all();
res = db.prepare("SELECT COUNT(*) FROM core").all();
expect(res[0]["COUNT(*)"]).toBe(1);

res = db.prepare("SELECT COUNT(*) FROM free_text").all();
Expand Down

0 comments on commit 228ea76

Please sign in to comment.