From 54827e85ee332ecef57b0a4fcf0edb9f7c28a8bd Mon Sep 17 00:00:00 2001
From: Brian Joseph Petro <brian@smartconnections.app>
Date: Thu, 19 Dec 2024 20:47:16 -0500
Subject: [PATCH] - Added a new test script (`test_content.sh`) to set up
 various Markdown files for comprehensive integration testing of the
 SmartSources Markdown adapter and Block parser. - Introduced a new test file
 (`markdown_source.test.js`) containing integration-level tests for verifying
 the functionality of the SmartSources Markdown adapter, including block
 parsing, CRUD operations, and handling of edge cases. - Updated
 `package.json` to include new test scripts and dependencies for improved
 testing capabilities. - Refactored `SmartSource` to utilize updated data
 loading methods, enhancing data integrity during imports.

These changes aim to improve the testing framework and ensure robust functionality of the SmartSources system.
---
 smart-sources/package.json                 |  10 +-
 smart-sources/smart_source.js              |   2 +-
 smart-sources/test/markdown_source.test.js | 318 +++++++++++++++++++++
 smart-sources/test/test_content.sh         | 284 ++++++++++++++++++
 4 files changed, 612 insertions(+), 2 deletions(-)
 create mode 100644 smart-sources/test/markdown_source.test.js
 create mode 100644 smart-sources/test/test_content.sh

diff --git a/smart-sources/package.json b/smart-sources/package.json
index af217751..9d6a18d1 100644
--- a/smart-sources/package.json
+++ b/smart-sources/package.json
@@ -7,8 +7,12 @@
   "description": "Easy to manage collections mapped to local or remote sources.",
   "main": "main.js",
   "scripts": {
+    "test_content": "test/test_content.sh",
     "test": "npx ava --verbose"
   },
+  "bin": {
+    "test_content": "./test/test_content.sh"
+  },
   "keywords": [
     "embeddings",
     "collections",
@@ -23,7 +27,11 @@
   },
   "homepage": "https://jsbrains.org",
   "devDependencies": {
-    "ava": "^6.0.1"
+    "ava": "^6.0.1",
+    "smart-blocks": "file:../smart-blocks",
+    "smart-environment": "file:../smart-environment",
+    "smart-fs": "file:../smart-fs",
+    "smart-settings": "file:../smart-settings"
   },
   "dependencies": {
     "smart-collections": "file:../smart-collections",
diff --git a/smart-sources/smart_source.js b/smart-sources/smart_source.js
index 9642aa2a..5f9d412a 100644
--- a/smart-sources/smart_source.js
+++ b/smart-sources/smart_source.js
@@ -52,7 +52,7 @@ export class SmartSource extends SmartEntity {
   async import(){
     this._queue_import = false;
     try{
-      await this.data_adapter.load_if_updated(this);
+      await this.data_adapter.load_item_if_updated(this);
       await this.source_adapter.import();
     }catch(err){
       if(err.code === "ENOENT"){
diff --git a/smart-sources/test/markdown_source.test.js b/smart-sources/test/markdown_source.test.js
new file mode 100644
index 00000000..573d4b4c
--- /dev/null
+++ b/smart-sources/test/markdown_source.test.js
@@ -0,0 +1,318 @@
+/**
+ * @file markdown_source.test.js
+ * @description Integration-level tests for the SmartSources Markdown adapter and Block parser.
+ *
+ * These tests assume the note variations created by the bash script are present in `test/test-content/variations/`.
+ * We run through a series of integration tests:
+ * - Importing each note
+ * - Confirming block parsing correctness
+ * - Performing CRUD operations (read, update, destroy) on frontmatter, headings, subheadings, lists, code blocks
+ * - Checking repeated headings, special characters, large files, and empty files
+ * - Ensuring stable AJSON state after round-trip load/save
+ * - Verifying that no errors occur for various edge cases
+ *
+ * Setup:
+ *  - Ensure that `test/test-content/variations/` is created by running the provided bash script before these tests.
+ *  - The environment should have `SmartEnv`, `SmartSources`, `SmartBlocks`, and other required classes loaded.
+ *  - These tests assume `SmartEnv` and its dependencies are globally accessible or imported from prior setup code.
+ *
+ * Each test scenario comments reference the note created by the bash script. After each note test scenario, we verify:
+ * - import success
+ * - block count correctness
+ * - sample block CRUD operations and confirm results in `.ajson` files
+ * - search or embedding tests if applicable
+ *
+ * Note: This file outlines test cases and expectations. Actual test runner code (like AVA or Jest) may need to be integrated.
+ */
+
+import test from 'ava';
+import { SmartEnv } from 'smart-environment/smart_env.js'; // Adjust path as needed
+import { SmartFs } from 'smart-fs/smart_fs.js';
+import { NodeFsSmartFsAdapter } from 'smart-fs/adapters/node_fs.js';
+import { SmartSources } from 'smart-sources/smart_sources.js';
+import { SmartSource } from 'smart-sources/smart_source.js';
+import { SmartBlocks } from 'smart-blocks/smart_blocks.js';
+import { SmartBlock } from 'smart-blocks/smart_block.js';
+import { SmartSettings } from 'smart-settings/smart_settings.js';
+import { MarkdownSourceContentAdapter } from 'smart-sources/adapters/markdown_source.js';
+import ajson_data_adapter from 'smart-sources/adapters/data/ajson_multi_file.js';
+
+const VARIATIONS_DIR = 'test/test-content/variations';
+
+async function create_test_env() {
+  const env = await SmartEnv.create({
+    // Mock main or configuration object
+    load_settings(){ return {}; },
+    save_settings(){},
+    get settings(){ return {}; }
+  }, {
+    env_path: 'test/test-content',
+    modules: {
+      smart_fs: { class: SmartFs, adapter: NodeFsSmartFsAdapter },
+      smart_settings: { class: SmartSettings },
+    },
+
+    collections: {
+      smart_sources: {
+        class: SmartSources,
+        data_adapter: ajson_data_adapter,
+        source_adapters: {
+          default: MarkdownSourceContentAdapter,
+          md: MarkdownSourceContentAdapter,
+        }
+      },
+      smart_blocks: {
+        class: SmartBlocks,
+        data_adapter: ajson_data_adapter,
+      },
+    },
+    item_types: {
+      SmartSource,
+      SmartBlock,
+    },
+    default_settings: {
+      smart_sources: {
+        data_dir: 'multi'
+      }
+    }
+  });
+
+  await env.smart_sources.init_items();
+  return env;
+}
+
+test.before(async t => {
+  t.context.env = await create_test_env();
+  // Process initial import and save queues to ensure a stable state
+  await t.context.env.smart_sources.process_source_import_queue();
+  await t.context.env.smart_sources.process_save_queue();
+});
+
+test.serial('frontmatter_note.md: import and verify blocks', async t => {
+  const { env } = t.context;
+  const source = env.smart_sources.get('variations/frontmatter_note.md');
+  t.truthy(source, 'frontmatter_note.md should be imported');
+
+  // Verify frontmatter block
+  const frontmatter_block = env.smart_blocks.get('variations/frontmatter_note.md#---frontmatter---');
+  t.truthy(frontmatter_block, 'Frontmatter block should exist');
+
+  // Update frontmatter block
+  const new_frontmatter = `---
+title: "Updated Title"
+date: 2024-01-02
+tags: [test, updated]
+---`;
+  await frontmatter_block.update(new_frontmatter);
+  await env.smart_sources.process_save_queue();
+
+  // Re-import and confirm changes
+  await source.import();
+  t.true(source.data.last_read?.hash !== null, 'Source re-imported successfully');
+
+  // Destroy frontmatter block
+  await frontmatter_block.remove();
+  await env.smart_sources.process_save_queue();
+
+  // Confirm frontmatter removed
+  await source.import();
+  t.falsy(env.smart_blocks.get('variations/frontmatter_note.md#---frontmatter---'), 'Frontmatter block should be removed after destroy');
+});
+
+test.serial('nested_headings.md: verify deep heading structure', async t => {
+  const { env } = t.context;
+  const source = env.smart_sources.get('variations/nested_headings.md');
+  t.truthy(source, 'nested_headings.md should be imported');
+
+  // Check up to level 6 headings
+  const level6_block = env.smart_blocks.get('variations/nested_headings.md#Level 1#Level 2#Level 3#Level 4#Level 5#Level 6');
+  t.truthy(level6_block, 'Level 6 heading block should exist');
+
+  // Append content to level 3 block and verify
+  const level3_block = env.smart_blocks.get('variations/nested_headings.md#Level 1#Level 2#Level 3');
+  await level3_block.append("Appended content line");
+  await env.smart_sources.process_save_queue();
+  
+  await source.import();
+  const updated_content = await level3_block.read();
+  t.regex(updated_content, /Appended content line/, 'Appended line should appear in level 3 block content');
+});
+
+test.serial('code_blocks.md: ensure code fences handled correctly', async t => {
+  const { env } = t.context;
+  const source = env.smart_sources.get('variations/code_blocks.md');
+  t.truthy(source, 'code_blocks.md should be imported');
+
+  // Confirm code blocks do not break parsing
+  const main_heading = env.smart_blocks.get('variations/code_blocks.md#Code Blocks Test');
+  t.truthy(main_heading, 'Main heading block should be present');
+
+  // Update code block content via block update
+  // The code blocks are part of the main heading block content in this scenario
+  const content_before = await main_heading.read();
+  const replaced_content = content_before.replace('console.log("Hello, world!");', 'console.log("Hello, integration test!");');
+  await main_heading.update(replaced_content);
+  await env.smart_sources.process_save_queue();
+  
+  await source.import();
+  const content_after = await main_heading.read();
+  t.regex(content_after, /Hello, integration test!/, 'Code block updated correctly');
+});
+
+test.serial('no_headings.md: single root block scenario', async t => {
+  const { env } = t.context;
+  const source = env.smart_sources.get('variations/no_headings.md');
+  t.truthy(source, 'no_headings.md should be imported');
+
+  const root_block = env.smart_blocks.get('variations/no_headings.md#');
+  t.truthy(root_block, 'Root block should exist when no headings present');
+
+  // Update entire file content
+  await root_block.update("Updated entire file content with no headings.");
+  await env.smart_sources.process_save_queue();
+
+  await source.import();
+  const updated = await root_block.read();
+  t.is(updated.trim(), "Updated entire file content with no headings.", 'Entire content updated successfully');
+});
+
+test.serial('only_lists.md: lists as blocks', async t => {
+  const { env } = t.context;
+  const source = env.smart_sources.get('variations/only_lists.md');
+  t.truthy(source, 'only_lists.md should be imported');
+
+  // Confirm top-level list items become sub-blocks
+  const first_item_block = env.smart_blocks.get('variations/only_lists.md#{1}');
+  t.truthy(first_item_block, 'First list item block should exist');
+
+  // Remove a nested list block if any (not fully implemented, but we can remove lines)
+  const before_remove = await first_item_block.read();
+  const updated_content = before_remove.replace('Nested item', '');
+  await first_item_block.update(updated_content);
+  await env.smart_sources.process_save_queue();
+
+  await source.import();
+  const after_remove = await first_item_block.read();
+  t.false(after_remove.includes('Nested item'), 'Nested list line removed successfully');
+});
+
+test.serial('repeated_headings.md: verify block keys for repeated headings', async t => {
+  const { env } = t.context;
+  const source = env.smart_sources.get('variations/repeated_headings.md');
+  t.truthy(source, 'repeated_headings.md should be imported');
+
+  // Check repeated top-level headings
+  const first_repeated = env.smart_blocks.get('variations/repeated_headings.md#Repeated');
+  const second_repeated = env.smart_blocks.get('variations/repeated_headings.md#Repeated[2]');
+  t.truthy(first_repeated, 'First occurrence of repeated heading exists');
+  t.truthy(second_repeated, 'Second occurrence of repeated heading exists');
+
+  // Remove the second repeated heading block
+  await second_repeated.remove();
+  await env.smart_sources.process_save_queue();
+  
+  await source.import();
+  t.falsy(env.smart_blocks.get('variations/repeated_headings.md#Repeated[2]'), 'Second repeated heading block removed successfully');
+});
+
+test.serial('mixed_content.md: complex scenario with frontmatter, lists, code', async t => {
+  const { env } = t.context;
+  const source = env.smart_sources.get('variations/mixed_content.md');
+  t.truthy(source, 'mixed_content.md should be imported');
+
+  const frontmatter_block = env.smart_blocks.get('variations/mixed_content.md#---frontmatter---');
+  t.truthy(frontmatter_block, 'Frontmatter block present in mixed_content.md');
+
+  const main_heading = env.smart_blocks.get('variations/mixed_content.md#Main Heading');
+  const subheading = env.smart_blocks.get('variations/mixed_content.md#Main Heading#Subheading');
+  const code_inside = await main_heading.read();
+  t.regex(code_inside, /```python/, 'Code block found inside main heading content');
+
+  // Update subheading content
+  await subheading.update("Replaced subheading content entirely.");
+  await env.smart_sources.process_save_queue();
+
+  await source.import();
+  const subheading_updated = await subheading.read();
+  t.is(subheading_updated.trim(), "Replaced subheading content entirely.", 'Subheading content updated');
+});
+
+test.serial('large_note.md: performance and stability checks', async t => {
+  const { env } = t.context;
+  const source = env.smart_sources.get('variations/large_note.md');
+  t.truthy(source, 'large_note.md should be imported');
+
+  // Ensure we have large blocks
+  const main_block = env.smart_blocks.get('variations/large_note.md#Large Note');
+  const another_heading_block = env.smart_blocks.get('variations/large_note.md#Large Note#Another Heading');
+  t.truthy(main_block, 'Main block in large note exists');
+  t.truthy(another_heading_block, 'Another heading block in large note exists');
+
+  // Just read and ensure no timeout or error occurs
+  const content_main = await main_block.read();
+  t.true(content_main.length > 1000, 'Main block content is large as expected');
+
+  // Update a large portion of text
+  await another_heading_block.update("Shortened content for the second heading block.");
+  await env.smart_sources.process_save_queue();
+
+  await source.import();
+  const updated_content = await another_heading_block.read();
+  t.is(updated_content.trim(), "Shortened content for the second heading block.", 'Large note block updated successfully');
+});
+
+test.serial('empty_note.md: no content scenario', async t => {
+  const { env } = t.context;
+  const source = env.smart_sources.get('variations/empty_note.md');
+  t.truthy(source, 'empty_note.md imported even if empty');
+
+  // Check if a root block is created
+  const root_block = env.smart_blocks.get('variations/empty_note.md#');
+  t.truthy(root_block, 'Root block should exist even if empty file');
+  
+  const content = await root_block.read();
+  t.is(content.trim(), '', 'Empty file block content is empty as expected');
+});
+
+test.serial('special_chars_headings.md: special chars in headings', async t => {
+  const { env } = t.context;
+  const source = env.smart_sources.get('variations/special_chars_headings.md');
+  t.truthy(source, 'special_chars_headings.md should be imported');
+
+  const special_heading = env.smart_blocks.get('variations/special_chars_headings.md#Heading with $Special_Char');
+  t.truthy(special_heading, 'Block with special chars in heading found');
+
+  await special_heading.append("Appended content with $ and _ characters.");
+  await env.smart_sources.process_save_queue();
+
+  await source.import();
+  const updated = await special_heading.read();
+  t.regex(updated, /\$ and _ characters/, 'Special chars preserved after update');
+});
+
+test.serial('frontmatter_complex.md: nested frontmatter structures', async t => {
+  const { env } = t.context;
+  const source = env.smart_sources.get('variations/frontmatter_complex.md');
+  t.truthy(source, 'frontmatter_complex.md should be imported');
+
+  const fm_block = env.smart_blocks.get('variations/frontmatter_complex.md#---frontmatter---');
+  t.truthy(fm_block, 'Nested frontmatter block found');
+
+  // Update frontmatter
+  const new_fm = `---
+title: Complex Frontmatter Updated
+authors:
+  - name: Charlie
+    role: Maintainer
+metadata:
+  tags: [complex, updated]
+  version: 3
+---`;
+  await fm_block.update(new_fm);
+  await env.smart_sources.process_save_queue();
+  
+  await source.import();
+  const fm_content = await fm_block.read();
+  t.regex(fm_content, /Charlie/, 'Frontmatter updated with nested data successfully');
+});
+
diff --git a/smart-sources/test/test_content.sh b/smart-sources/test/test_content.sh
new file mode 100644
index 00000000..44a8881a
--- /dev/null
+++ b/smart-sources/test/test_content.sh
@@ -0,0 +1,284 @@
+#!/usr/bin/env bash
+
+# This script sets up a variety of Markdown note files to test the SmartSources Markdown adapter and Block parser.
+# It creates a structured folder with multiple Markdown files, each containing different patterns and quirks to ensure
+# comprehensive integration testing of parsing, importing, block extraction, and CRUD operations.
+
+# Directory structure:
+# test/test-content/variations/
+# ├── frontmatter_note.md
+# ├── nested_headings.md
+# ├── code_blocks.md
+# ├── no_headings.md
+# ├── only_lists.md
+# ├── repeated_headings.md
+# ├── mixed_content.md
+# ├── large_note.md
+# ├── empty_note.md
+# ├── special_chars_headings.md
+# └── frontmatter_complex.md
+
+set -e
+
+BASE_DIR="test/test-content/variations"
+
+# Create base directory
+mkdir -p "$BASE_DIR"
+
+#############################################
+# 1. Note with frontmatter and headings
+#############################################
+cat > "$BASE_DIR/frontmatter_note.md" <<EOF
+---
+title: "Test Frontmatter"
+date: 2024-01-01
+tags: [test, markdown, frontmatter]
+---
+
+# Heading 1
+
+Some introductory content.
+
+## Heading 2
+
+More content under heading 2.
+
+### Heading 3
+
+Deeper level heading content.
+EOF
+
+# Integration test notes:
+# - Test that frontmatter is parsed as a block (#---frontmatter---) and can be updated/destroyed.
+# - Verify that headings and subheadings are correctly extracted as blocks.
+# - Check that outlinks (if any) are parsed.
+# - Confirm block CRUD operations (block_read, block_update, block_destroy) against frontmatter or headings.
+# - Ensure that removing or rewriting frontmatter updates AJSON correctly.
+
+#############################################
+# 2. Note with deeply nested headings
+#############################################
+cat > "$BASE_DIR/nested_headings.md" <<EOF
+# Level 1
+
+## Level 2
+
+### Level 3
+
+#### Level 4
+
+##### Level 5
+
+###### Level 6
+
+Content at the deepest heading level.
+EOF
+
+# Integration test notes:
+# - Verify that up to six levels of headings are correctly identified as blocks.
+# - Check how block removal or updates propagate when dealing with deeply nested headings.
+# - Test that markdown_to_blocks does not break with extreme heading nesting.
+# - Validate that rewriting minimal files after import preserves heading structure.
+
+#############################################
+# 3. Note with code blocks
+#############################################
+cat > "$BASE_DIR/code_blocks.md" <<EOF
+# Code Blocks Test
+
+Here is some code:
+
+\`\`\`javascript
+function hello() {
+  console.log("Hello, world!");
+}
+\`\`\`
+
+## Another Heading
+
+\`\`\`bash
+echo "test"
+\`\`\`
+
+EOF
+
+# Integration test notes:
+# - Confirm that code blocks are treated as normal content lines and not misinterpreted as headings.
+# - Verify block extraction does not fail due to code fences.
+# - Test block updates within code fenced areas (append, update content inside code blocks).
+# - Ensure code blocks do not cause JSON parse errors in AJSON.
+
+#############################################
+# 4. Note with no headings at all
+#############################################
+cat > "$BASE_DIR/no_headings.md" <<EOF
+This file has no headings.
+
+Just plain text.
+
+Even multiple lines of it.
+
+EOF
+
+# Integration test notes:
+# - Ensure that when no headings are present, a single root-level block (#) is created.
+# - Test CRUD on the root block content.
+# - Verify searching and embedding still works with a no-heading scenario.
+# - Confirm rewriting minimal file still works with no headings.
+
+#############################################
+# 5. Note with only lists
+#############################################
+cat > "$BASE_DIR/only_lists.md" <<EOF
+- First item
+  - Nested item
+- Second item
+
+- Third item
+  - Another nested level
+    - Deeply nested
+EOF
+
+# Integration test notes:
+# - Ensure lists are converted into blocks as content blocks under root.
+# - Check that each top-level list item creates a separate sub-block.
+# - Validate block_destroy removes only that item’s lines.
+# - Confirm that updates to list items reflect in AJSON.
+
+#############################################
+# 6. Note with repeated headings
+#############################################
+cat > "$BASE_DIR/repeated_headings.md" <<EOF
+# Repeated
+Content under the first occurrence.
+
+# Repeated
+Second occurrence of the same heading text.
+
+## Repeated
+Subheading also repeated multiple times.
+
+## Repeated
+Another subheading with the same name.
+EOF
+
+# Integration test notes:
+# - Verify that repeated headings get suffixed with [2], [3], etc. for top-level duplicates.
+# - Check that block references to repeated headings are stable.
+# - Ensure import/export is correct and stable for repeated heading names.
+# - Test block removal of one repeated heading does not affect others incorrectly.
+
+#############################################
+# 7. Note with mixed content (frontmatter, code, lists, headings)
+#############################################
+cat > "$BASE_DIR/mixed_content.md" <<EOF
+---
+title: Mixed Content Test
+---
+
+# Main Heading
+
+Some intro text.
+
+- A list item
+- Another list item
+
+\`\`\`python
+print("Hello")
+\`\`\`
+
+## Subheading
+
+More text under subheading.
+EOF
+
+# Integration test notes:
+# - Test complex scenario: frontmatter + headings + lists + code blocks all in one file.
+# - Confirm block parsing remains correct with multiple content types.
+# - Test partial updates: removing list block, updating code block content, rewriting frontmatter.
+# - Validate stable AJSON after multiple round-trip import/export cycles.
+
+#############################################
+# 8. Large note
+#############################################
+cat > "$BASE_DIR/large_note.md" <<EOF
+# Large Note
+
+EOF
+
+# Append a large amount of dummy content
+for i in {1..200}; do
+    echo "Paragraph line $i with some content to increase note size." >> "$BASE_DIR/large_note.md"
+done
+
+echo "## Another Heading" >> "$BASE_DIR/large_note.md"
+for i in {1..200}; do
+    echo "Another big paragraph block line $i." >> "$BASE_DIR/large_note.md"
+done
+
+# Integration test notes:
+# - Tests performance and correctness with large file sizes.
+# - Validate that line references remain stable after large content changes.
+# - Check block updates and deletions still function at scale.
+# - Ensure no memory or JSON parse issues with large content.
+
+#############################################
+# 9. Empty note
+#############################################
+cat > "$BASE_DIR/empty_note.md" <<EOF
+
+EOF
+
+# Integration test notes:
+# - Confirm that an empty file still imports without errors.
+# - Verify that no blocks are created (or a root block) and that CRUD ops handle empty gracefully.
+# - Check that saving after load doesn't create malformed AJSON.
+
+#############################################
+# 10. Headings with special characters
+#############################################
+cat > "$BASE_DIR/special_chars_headings.md" <<EOF
+# Heading with $Special_Char
+
+## Heading With Spaces and (Parentheses)
+
+### Heading_with_underscores_and_$variables
+
+Content under tricky headings.
+EOF
+
+# Integration test notes:
+# - Ensure special characters in headings do not break block keys.
+# - Validate that AJSON keys are properly JSON-stringified and remain stable across imports.
+# - Check CRUD operations on these special heading blocks.
+
+#############################################
+# 11. Complex frontmatter with lists, nested data
+#############################################
+cat > "$BASE_DIR/frontmatter_complex.md" <<EOF
+---
+title: Complex Frontmatter
+authors:
+  - name: Alice
+    role: Editor
+  - name: Bob
+    role: Reviewer
+metadata:
+  tags: [complex, nested, frontmatter]
+  version: 2
+---
+
+# Main Content
+
+This note tests complex nested frontmatter.
+EOF
+
+# Integration test notes:
+# - Test parsing and rewriting of complex, nested frontmatter data structures.
+# - Confirm block_update on frontmatter accurately replaces nested YAML.
+# - Validate block_destroy can remove frontmatter entirely and rewrite file.
+# - Ensure stable keys and JSON parse logic in AJSON after complex frontmatter import.
+
+#############################################
+echo "All test notes have been created in $BASE_DIR."
+echo "Use these notes to run integration tests on SmartSources Markdown adapter and block parser."