feat(js): Vitest/Jest evals

js/.gitignore

@@ -59,10 +59,26 @@ Chinook_Sqlite.sql
 /langchain.js
 /langchain.d.ts
 /langchain.d.cts
+/jest.cjs
+/jest.js
+/jest.d.ts
+/jest.d.cts
+/jest/reporter.cjs
+/jest/reporter.js
+/jest/reporter.d.ts
+/jest/reporter.d.cts
 /vercel.cjs
 /vercel.js
 /vercel.d.ts
 /vercel.d.cts
+/vitest.cjs
+/vitest.js
+/vitest.d.ts
+/vitest.d.cts
+/vitest/reporter.cjs
+/vitest/reporter.js
+/vitest/reporter.d.ts
+/vitest/reporter.d.cts
 /wrappers.cjs
 /wrappers.js
 /wrappers.d.ts

js/ls.vitest.config.ts

@@ -0,0 +1,9 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    include: ["**/*.vitesteval.?(c|m)[jt]s"],
+    reporters: ["./src/vitest/reporter.ts"],
+    setupFiles: ["dotenv/config"],
+  },
+});

js/package.json

@@ -33,10 +33,26 @@
     "langchain.js",
     "langchain.d.ts",
     "langchain.d.cts",
+    "jest.cjs",
+    "jest.js",
+    "jest.d.ts",
+    "jest.d.cts",
+    "jest/reporter.cjs",
+    "jest/reporter.js",
+    "jest/reporter.d.ts",
+    "jest/reporter.d.cts",
     "vercel.cjs",
     "vercel.js",
     "vercel.d.ts",
     "vercel.d.cts",
+    "vitest.cjs",
+    "vitest.js",
+    "vitest.d.ts",
+    "vitest.d.cts",
+    "vitest/reporter.cjs",
+    "vitest/reporter.js",
+    "vitest/reporter.d.ts",
+    "vitest/reporter.d.cts",
     "wrappers.cjs",
     "wrappers.js",
     "wrappers.d.ts",
@@ -103,7 +119,8 @@
   "homepage": "https://github.com/langchain-ai/langsmith-sdk#readme",
   "dependencies": {
     "@types/uuid": "^10.0.0",
-    "commander": "^10.0.1",
+    "chalk": "^4.1.2",
+    "console-table-printer": "^2.12.1",
     "p-queue": "^6.6.2",
     "p-retry": "4",
     "semver": "^7.6.3",
@@ -114,6 +131,7 @@
     "@babel/preset-env": "^7.22.4",
     "@faker-js/faker": "^8.4.1",
     "@jest/globals": "^29.5.0",
+    "@jest/reporters": "^29.7.0",
     "@langchain/core": "^0.3.14",
     "@langchain/langgraph": "^0.2.20",
     "@langchain/openai": "^0.3.11",
@@ -143,6 +161,7 @@
     "typedoc": "^0.27.6",
     "typedoc-plugin-expand-object-like-types": "^0.1.2",
     "typescript": "^5.4.5",
+    "vitest": "^2.1.8",
     "zod": "^3.23.8"
   },
   "peerDependencies": {
@@ -232,6 +251,24 @@
       "import": "./langchain.js",
       "require": "./langchain.cjs"
     },
+    "./jest": {
+      "types": {
+        "import": "./jest.d.ts",
+        "require": "./jest.d.cts",
+        "default": "./jest.d.ts"
+      },
+      "import": "./jest.js",
+      "require": "./jest.cjs"
+    },
+    "./jest/reporter": {
+      "types": {
+        "import": "./jest/reporter.d.ts",
+        "require": "./jest/reporter.d.cts",
+        "default": "./jest/reporter.d.ts"
+      },
+      "import": "./jest/reporter.js",
+      "require": "./jest/reporter.cjs"
+    },
     "./vercel": {
       "types": {
         "import": "./vercel.d.ts",
@@ -241,6 +278,24 @@
       "import": "./vercel.js",
       "require": "./vercel.cjs"
     },
+    "./vitest": {
+      "types": {
+        "import": "./vitest.d.ts",
+        "require": "./vitest.d.cts",
+        "default": "./vitest.d.ts"
+      },
+      "import": "./vitest.js",
+      "require": "./vitest.cjs"
+    },
+    "./vitest/reporter": {
+      "types": {
+        "import": "./vitest/reporter.d.ts",
+        "require": "./vitest/reporter.d.cts",
+        "default": "./vitest/reporter.d.ts"
+      },
+      "import": "./vitest/reporter.js",
+      "require": "./vitest/reporter.cjs"
+    },
     "./wrappers": {
       "types": {
         "import": "./wrappers.d.ts",

js/scripts/create-entrypoints.js

@@ -14,14 +14,22 @@ const entrypoints = {
   "evaluation/langchain": "evaluation/langchain",
   schemas: "schemas",
   langchain: "langchain",
+  jest: "jest/index",
+  "jest/reporter": "jest/reporter",
   vercel: "vercel",
+  vitest: "vitest/index",
+  "vitest/reporter": "vitest/reporter",
   wrappers: "wrappers/index",
   anonymizer: "anonymizer/index",
   "wrappers/openai": "wrappers/openai",
   "wrappers/vercel": "wrappers/vercel",
   "singletons/traceable": "singletons/traceable",
 };
 
+const defaultEntrypoints = [
+  "vitest/reporter"
+];
+
 const updateJsonFile = (relativePath, updateFunction) => {
   const contents = fs.readFileSync(relativePath).toString();
   const res = updateFunction(JSON.parse(contents));
@@ -34,6 +42,17 @@ const generateFiles = () => {
       const nrOfDots = key.split("/").length - 1;
       const relativePath = "../".repeat(nrOfDots) || "./";
       const compiledPath = `${relativePath}dist/${value}.js`;
+      if (defaultEntrypoints.includes(key)) {
+        return [
+          [
+            `${key}.cjs`,
+            `module.exports = require('${relativePath}dist/${value}.cjs').default;`,
+          ],
+          [`${key}.js`, `export { default } from '${compiledPath}'`],
+          [`${key}.d.ts`, `export { default } from '${compiledPath}'`],
+          [`${key}.d.cts`, `export { default } from '${compiledPath}'`],
+        ];
+      }
       return [
         [
           `${key}.cjs`,

js/src/client.ts

@@ -42,6 +42,7 @@
   RawExample,
   AttachmentInfo,
   AttachmentData,
+  DatasetVersion,
 } from "./schemas.js";
 import {
   convertLangChainMessageToExample,
@@ -284,6 +285,15 @@
   sourceRunId?: string;
 };
 
+export type CreateProjectParams = {
+  projectName: string;
+  description?: string | null;
+  metadata?: RecordStringAny | null;
+  upsert?: boolean;
+  projectExtra?: RecordStringAny | null;
+  referenceDatasetId?: string | null;
+};
+
 type AutoBatchQueueItem = {
   action: "create" | "update";
   item: RunCreate | RunUpdate;
@@ -429,7 +439,7 @@
     // If there is an item on the queue we were unable to pop,
     // just return it as a single batch.
     if (popped.length === 0 && this.items.length > 0) {
       const item = this.items.shift()!;
       popped.push(item);
       poppedSizeBytes += item.size;
       this.sizeBytes -= item.size;
@@ -862,7 +872,7 @@
         if (this._serverInfo === undefined) {
           try {
             this._serverInfo = await this._getServerInfo();
           } catch (e) {
             console.warn(
               `[WARNING]: LangSmith failed to fetch info on supported operations. Falling back to batch operations and default limits.`
             );
@@ -1597,7 +1607,7 @@
     treeFilter?: string;
     isRoot?: boolean;
     dataSourceType?: string;
   }): Promise<any> {
     let projectIds_ = projectIds || [];
     if (projectNames) {
       projectIds_ = [
@@ -1885,7 +1895,7 @@
         `Failed to list shared examples: ${response.status} ${response.statusText}`
       );
     }
     return result.map((example: any) => ({
       ...example,
       _hostUrl: this.getHostUrl(),
     }));
@@ -1898,14 +1908,7 @@
     upsert = false,
     projectExtra = null,
     referenceDatasetId = null,
-  }: {
-    projectName: string;
-    description?: string | null;
-    metadata?: RecordStringAny | null;
-    upsert?: boolean;
-    projectExtra?: RecordStringAny | null;
-    referenceDatasetId?: string | null;
-  }): Promise<TracerSession> {
+  }: CreateProjectParams): Promise<TracerSession> {
     const upsert_ = upsert ? `?upsert=true` : "";
     const endpoint = `${this.apiUrl}/sessions${upsert_}`;
     const extra: RecordStringAny = projectExtra || {};
@@ -2022,7 +2025,7 @@
       }
       // projectId querying
       return true;
     } catch (e) {
       return false;
     }
   }
@@ -2480,6 +2483,53 @@
     return (await response.json()) as Dataset;
   }
 
+  /**
+   * Updates a tag on a dataset.
+   *
+   * If the tag is already assigned to a different version of this dataset,
+   * the tag will be moved to the new version. The as_of parameter is used to
+   * determine which version of the dataset to apply the new tags to.
+   *
+   * It must be an exact version of the dataset to succeed. You can
+   * use the "readDatasetVersion" method to find the exact version
+   * to apply the tags to.
+   * @param params.datasetId The ID of the dataset to update. Must be provided if "datasetName" is not provided.
+   * @param params.datasetName The name of the dataset to update. Must be provided if "datasetId" is not provided.
+   * @param params.asOf The timestamp of the dataset to apply the new tags to.
+   * @param params.tag The new tag to apply to the dataset.
+   */
+  public async updateDatasetTag(props: {
+    datasetId?: string;
+    datasetName?: string;
+    asOf: string | Date;
+    tag: string;
+  }): Promise<void> {
+    const { datasetId, datasetName, asOf, tag } = props;
+
+    if (!datasetId && !datasetName) {
+      throw new Error("Must provide either datasetName or datasetId");
+    }
+    const _datasetId =
+      datasetId ?? (await this.readDataset({ datasetName })).id;
+    assertUuid(_datasetId);
+
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${this.apiUrl}/datasets/${_datasetId}/tags`,
+      {
+        method: "PUT",
+        headers: { ...this.headers, "Content-Type": "application/json" },
+        body: JSON.stringify({
+          as_of: typeof asOf === "string" ? asOf : asOf.toISOString(),
+          tag,
+        }),
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+    await raiseForStatus(response, "update dataset tags");
+  }
+
   public async deleteDataset({
     datasetId,
     datasetName,
@@ -2939,6 +2989,72 @@
     return result;
   }
 
+  /**
+   * Get dataset version by closest date or exact tag.
+   *
+   * Use this to resolve the nearest version to a given timestamp or for a given tag.
+   *
+   * @param options The options for getting the dataset version
+   * @param options.datasetId The ID of the dataset
+   * @param options.datasetName The name of the dataset
+   * @param options.asOf The timestamp of the dataset to retrieve
+   * @param options.tag The tag of the dataset to retrieve
+   * @returns The dataset version
+   */
+  public async readDatasetVersion({
+    datasetId,
+    datasetName,
+    asOf,
+    tag,
+  }: {
+    datasetId?: string;
+    datasetName?: string;
+    asOf?: string | Date;
+    tag?: string;
+  }): Promise<DatasetVersion> {
+    let resolvedDatasetId: string;
+    if (!datasetId) {
+      const dataset = await this.readDataset({ datasetName });
+      resolvedDatasetId = dataset.id;
+    } else {
+      resolvedDatasetId = datasetId;
+    }
+
+    assertUuid(resolvedDatasetId);
+
+    if ((asOf && tag) || (!asOf && !tag)) {
+      throw new Error("Exactly one of asOf and tag must be specified.");
+    }
+
+    const params = new URLSearchParams();
+    if (asOf !== undefined) {
+      params.append(
+        "as_of",
+        typeof asOf === "string" ? asOf : asOf.toISOString()
+      );
+    }
+    if (tag !== undefined) {
+      params.append("tag", tag);
+    }
+
+    const response = await this.caller.call(
+      _getFetchImplementation(),
+      `${
+        this.apiUrl
+      }/datasets/${resolvedDatasetId}/version?${params.toString()}`,
+      {
+        method: "GET",
+        headers: { ...this.headers },
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+
+    await raiseForStatus(response, "read dataset version");
+
+    return await response.json();
+  }
+
   public async listDatasetSplits({
     datasetId,
     datasetName,
@@ -3399,7 +3515,7 @@
   async _logEvaluationFeedback(
     evaluatorResponse: EvaluationResult | EvaluationResults,
     run?: Run,
     sourceInfo?: { [key: string]: any }
   ): Promise<[results: EvaluationResult[], feedbacks: Feedback[]]> {
     const evalResults: Array<EvaluationResult> =
       this._selectEvalResults(evaluatorResponse);
@@ -3438,7 +3554,7 @@
   public async logEvaluationFeedback(
     evaluatorResponse: EvaluationResult | EvaluationResults,
     run?: Run,
     sourceInfo?: { [key: string]: any }
   ): Promise<EvaluationResult[]> {
     const [results] = await this._logEvaluationFeedback(
       evaluatorResponse,
@@ -3934,7 +4050,7 @@
 
   public async createCommit(
     promptIdentifier: string,
     object: any,
     options?: {
       parentCommitHash?: string;
     }
@@ -4166,7 +4282,7 @@
       isPublic?: boolean;
       isArchived?: boolean;
     }
   ): Promise<Record<string, any>> {
     if (!(await this.promptExists(promptIdentifier))) {
       throw new Error("Prompt does not exist, you must create it first.");
     }
@@ -4177,7 +4293,7 @@
       throw await this._ownerConflictError("update a prompt", owner);
     }
 
     const payload: Record<string, any> = {};
 
     if (options?.description !== undefined)
       payload.description = options.description;