Managed test case examples (#158)

Added example for how to use managed test cases with the testing SDK

JavaScript/testing-sdk/README.md

@@ -60,6 +60,26 @@ You should see something like:
 You can click on the links next to each test name to dig into more details.
 You can also find all of your tests on the testing homepage in the [Autoblocks application](https://app.autoblocks.ai/testing/local).
 
+### Managed test cases
+
+After you have ran your first test following the instructions above, you will see a test suite created called **flashcard-generator-managed** in the UI. Visit the [Test Cases Page](https://app.autoblocks.ai/test-cases) and click on this test suite.
+
+Create a new test case with the following JSON body. [Click here to learn how to manage test cases](https://docs.autoblocks.ai/testing/test-case-management).
+
+```json
+{
+  "notes": "The Industrial Revolution, which began in Britain in the late 18th century, brought about significant changes in society, economy, and technology, leading to the transition from agrarian to industrial economies."
+}
+```
+
+Once updated, run the testing command again.
+
+```bash
+npx autoblocks testing exec -m "my second run" -- npm run start
+```
+
+You will now see test results for **flashcard-generator-managed** with the test cases you just setup in the UI.
+
 ## GitHub Actions setup
 
 A starter workflow was added in [`.github/workflows/autoblocks-testing.yml`](./.github/workflows/autoblocks-testing.yml).

JavaScript/testing-sdk/package.json

@@ -12,7 +12,7 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@autoblocks/client": "^0.0.41",
+    "@autoblocks/client": "^0.0.42",
     "dotenv-cli": "^7.3.0",
     "openai": "^4.6.0",
     "tsx": "^4.1.3",

JavaScript/testing-sdk/src/run.ts

@@ -1,6 +1,11 @@
 import * as flashcardGenerator from './test-suites/flashcard-generator';
 import * as studyGuideOutline from './test-suites/study-guide-outline';
+import * as flashcardGeneratorManaged from './test-suites/flashcard-generator-managed';
 
 (async () => {
-  await Promise.all([flashcardGenerator.run(), studyGuideOutline.run()]);
+  await Promise.all([
+    flashcardGenerator.run(),
+    studyGuideOutline.run(),
+    flashcardGeneratorManaged.run(),
+  ]);
 })();

JavaScript/testing-sdk/src/test-suites/flashcard-generator-managed/index.ts

@@ -0,0 +1,39 @@
+import { runTestSuite } from '@autoblocks/client/testing';
+import { type TestCase } from '../flashcard-generator/test-cases';
+import {
+  IsProfessionalTone,
+  IsSupportedByNotes,
+} from '../flashcard-generator/evaluators';
+import {
+  genFlashcardsFromNotes,
+  type Flashcard,
+} from '../../tasks/flashcard-generator';
+import { AutoblocksAPIClient } from '@autoblocks/client';
+
+const TEST_SUITE_ID = 'flashcard-generator-managed';
+
+export async function run() {
+  const inCodeTestCases = [{ notes: 'Initial test case' }];
+  let managedTestCases: TestCase[] = [];
+
+  try {
+    const client = new AutoblocksAPIClient();
+
+    const response = await client.getTestCases<TestCase>({
+      testSuiteId: TEST_SUITE_ID,
+    });
+    managedTestCases = response.testCases.map((testCase) => testCase.body);
+  } catch {
+    console.warn('Test suite does not exist yet,');
+  }
+
+  await runTestSuite<TestCase, Flashcard[]>({
+    id: TEST_SUITE_ID,
+    testCases: [...managedTestCases, ...inCodeTestCases],
+    testCaseHash: ['notes'],
+    evaluators: [new IsProfessionalTone(), new IsSupportedByNotes()],
+    fn: (args: { testCase: TestCase }) =>
+      genFlashcardsFromNotes(args.testCase.notes),
+    maxTestCaseConcurrency: 5,
+  });
+}

Python/testing-sdk/README.md

@@ -66,6 +66,26 @@ You should see something like:
 You can click on the links next to each test name to dig into more details.
 You can also find all of your tests on the testing homepage in the [Autoblocks application](https://app.autoblocks.ai/testing/local).
 
+### Managed test cases
+
+After you have ran your first test following the instructions above, you will see a test suite created called **flashcard-generator-managed** in the UI. Visit the [Test Cases Page](https://app.autoblocks.ai/test-cases) and click on this test suite.
+
+Create a new test case with the following JSON body. [Click here to learn how to manage test cases](https://docs.autoblocks.ai/testing/test-case-management).
+
+```json
+{
+  "notes": "The Industrial Revolution, which began in Britain in the late 18th century, brought about significant changes in society, economy, and technology, leading to the transition from agrarian to industrial economies."
+}
+```
+
+Once updated, run the testing command again.
+
+```bash
+npx autoblocks testing exec -m "my second run" -- poetry run start
+```
+
+You will now see test results for **flashcard-generator-managed** with the test cases you just setup in the UI.
+
 ## GitHub Actions setup
 
 A starter workflow was added in [`.github/workflows/autoblocks-testing.yml`](./.github/workflows/autoblocks-testing.yml).

Python/testing-sdk/my_project/run.py

@@ -1,5 +1,6 @@
 from my_project.test_suites import flashcard_generator
 from my_project.test_suites import study_guide_outline
+from my_project.test_suites import flashcard_generator_managed
 
 
 def run() -> None:
@@ -9,3 +10,4 @@ def run() -> None:
     # run() function and Autoblocks will handle the rest.
     flashcard_generator.run()
     study_guide_outline.run()
+    flashcard_generator_managed.run()

Python/testing-sdk/my_project/test_suites/flashcard_generator_managed/__init__.py

@@ -0,0 +1,42 @@
+from typing import List
+
+from autoblocks.testing.run import run_test_suite
+from autoblocks.api.client import AutoblocksAPIClient
+
+from my_project.tasks.flashcard_generator import Flashcard
+from my_project.tasks.flashcard_generator import gen_flashcards_from_notes
+
+from my_project.test_suites.flashcard_generator.evaluators import IsProfessionalTone
+from my_project.test_suites.flashcard_generator.evaluators import IsSupportedByNotes
+from my_project.test_suites.flashcard_generator.test_cases import TestCase
+
+TEST_SUITE_ID = "flashcard-generator-managed"
+
+
+def test_fn(test_case: TestCase) -> List[Flashcard]:
+    return gen_flashcards_from_notes(test_case.notes)
+
+
+def run() -> None:
+    in_code_test_cases = [TestCase(notes="Initial test case")]
+    managed_test_cases = []
+
+    try:
+        client = AutoblocksAPIClient()
+        test_cases_response = client.get_test_cases(test_suite_id=TEST_SUITE_ID)
+        managed_test_cases = [
+            TestCase(**test_case.body) for test_case in test_cases_response.test_cases
+        ]
+    except:
+        print("Test suite does not exist yet.")
+
+    run_test_suite(
+        id=TEST_SUITE_ID,
+        test_cases=managed_test_cases + in_code_test_cases,
+        evaluators=[
+            IsSupportedByNotes(),
+            IsProfessionalTone(),
+        ],
+        fn=test_fn,
+        max_test_case_concurrency=5,
+    )