EPD-945 Add human review URL (#90)

## Local

![Screenshot 2024-06-05 at 11 06
23 AM](https://github.com/autoblocksai/cli/assets/7498009/28f073e0-1975-4636-ba85-e422cf365996)

## CI

<img width="920" alt="Screenshot 2024-06-05 at 11 43 07 AM"
src="https://github.com/autoblocksai/cli/assets/7498009/f0acd5be-c443-4305-a70b-03eee98fd97e">

.github/workflows/ci.yml

@@ -11,7 +11,6 @@ concurrency:
 env:
   AUTOBLOCKS_API_KEY: ${{ secrets.AUTOBLOCKS_API_KEY }}
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  TSUP_PUBLIC_AUTOBLOCKS_INGESTION_KEY: test
 
 jobs:
   ci:

.github/workflows/e2e.yml

@@ -14,9 +14,6 @@ on:
   schedule:
     - cron: '17 15 * * *'
 
-env:
-  TSUP_PUBLIC_AUTOBLOCKS_INGESTION_KEY: test
-
 jobs:
   py:
     name: python-e2e-${{ github.event_name }}

src/handlers/testing/exec/components/progress/index.tsx

@@ -1,12 +1,14 @@
-import { Box, Spacer, Static, Text, render } from 'ink';
+import { Box, Static, Text, render } from 'ink';
 import Spinner from 'ink-spinner';
 import { useEffect, useState } from 'react';
 import { EventName, emitter, type EventSchemas } from '../../util/emitter';
 import { makeTestRunStatusFromEvaluations } from '../../util/evals';
 import { EvaluationPassed, TestRunStatus } from '../../util/models';
 import {
-  makeAutoblocksCIBuildHtmlUrl,
-  makeAutoblocksLocalTestHtmlUrl,
+  makeAutoblocksCIBuildSummaryHtmlUrl,
+  makeAutoblocksLocalTestResultsHtmlUrl,
+  makeAutoblocksCITestResultsHtmlUrl,
+  makeAutoblocksHumanReviewHtmlUrl,
 } from '../../util/url';
 
 type ConsoleLog = EventSchemas[EventName.CONSOLE_LOG];
@@ -21,6 +23,12 @@ interface AppProps {
   ciBuildId: string | undefined;
 }
 
+interface RunMeta {
+  id: string;
+  ended: boolean;
+  shareUrl: string | undefined;
+}
+
 const MAX_TEST_CASES = 100;
 
 /**
@@ -73,9 +81,9 @@ function makeColorFromLogLevel(
 }
 
 function TestRow(props: {
-  runIsOver: boolean;
-  shareUrl?: string;
   testExternalId: string;
+  ciBranchId: string | undefined;
+  runMeta: RunMeta | undefined;
   evals: Evaluation[];
   errors: UncaughtError[];
 }) {
@@ -96,35 +104,33 @@ function TestRow(props: {
   const { color: testStatusColor, icon: testStatusIcon } =
     statusToColorAndIcon[testStatus];
 
+  const resultsUrl: string | null = (() => {
+    if (!process.env.CI) {
+      return makeAutoblocksLocalTestResultsHtmlUrl({
+        testExternalId: props.testExternalId,
+      });
+    }
+    if (props.ciBranchId) {
+      return makeAutoblocksCITestResultsHtmlUrl({
+        testExternalId: props.testExternalId,
+        branchId: props.ciBranchId,
+      });
+    }
+    return null;
+  })();
+
   return (
     <Box flexDirection="column">
       <Box columnGap={1}>
-        {props.runIsOver ? (
+        {props.runMeta?.ended ? (
           <Text color={testStatusColor}>{testStatusIcon}</Text>
         ) : (
           <Spinner type="dots" />
         )}
         <Text bold={true}>{props.testExternalId}</Text>
-        {/* 
-          GitHub Actions does not respect width=100%, so URLs here typically end up wrapping and breaking the box's border.
-          For CI builds we show a link to the summary page above the box, which should be sufficient for navigating to
-          the test results.
-         */}
-        {!process.env.CI && (
-          <>
-            <Spacer />
-            <Box>
-              <Text color="cyan" dimColor={true}>
-                {makeAutoblocksLocalTestHtmlUrl({
-                  testExternalId: props.testExternalId,
-                })}
-              </Text>
-            </Box>
-          </>
-        )}
       </Box>
       <Box paddingLeft={2} columnGap={2}>
-        {props.runIsOver && testStatus === TestRunStatus.NO_RESULTS && (
+        {props.runMeta?.ended && testStatus === TestRunStatus.NO_RESULTS && (
           <Text color="gray">No results.</Text>
         )}
         <Box flexDirection="column">
@@ -175,14 +181,37 @@ function TestRow(props: {
           })}
         </Box>
       </Box>
-      {props.shareUrl && (
-        <Box paddingLeft={2} paddingTop={1}>
-          <Text>Shareable Url: </Text>
-          <Text color="cyan" dimColor={true}>
-            {props.shareUrl}
-          </Text>
-        </Box>
-      )}
+      {/* Links */}
+      <Box flexDirection="column" paddingTop={1} paddingLeft={2}>
+        {resultsUrl && <ExternalLink name="Results" url={resultsUrl} />}
+        {props.runMeta && (
+          <ExternalLink
+            name="Human Review"
+            url={makeAutoblocksHumanReviewHtmlUrl({
+              testExternalId: props.testExternalId,
+              runId: props.runMeta.id,
+            })}
+          />
+        )}
+      </Box>
+    </Box>
+  );
+}
+
+function ExternalLink(props: { name: string; url: string }) {
+  const prefix = '>>';
+  const columnGap = 1;
+  // Hardcode the width so that it doesn't wrap in CI terminals
+  const width = [prefix, props.name, props.url].join(
+    ' '.repeat(columnGap),
+  ).length;
+  return (
+    <Box columnGap={columnGap} width={width}>
+      <Text>{prefix}</Text>
+      <Text>{props.name}</Text>
+      <Text color="cyan" dimColor={true}>
+        {props.url}
+      </Text>
     </Box>
   );
 }
@@ -192,14 +221,8 @@ const App = (props: AppProps) => {
   const [consoleLogs, setConsoleLogs] = useState<ConsoleLog[]>([]);
   const [uncaughtErrors, setUncaughtErrors] = useState<UncaughtError[]>([]);
   const [evals, setEvals] = useState<Evaluation[]>([]);
-  const [testIdToRunIsOver, setTestIdToRunIsOver] = useState<
-    Record<
-      string,
-      {
-        ended: boolean;
-        shareUrl: string | undefined;
-      }
-    >
+  const [testIdToRunMeta, setTestIdToRunMeta] = useState<
+    Record<string, RunMeta>
   >({});
 
   useEffect(() => {
@@ -231,10 +254,11 @@ const App = (props: AppProps) => {
     };
 
     const runEndListener = (runEnded: RunEnded) => {
-      setTestIdToRunIsOver((prevRunIsOver) => {
+      setTestIdToRunMeta((prev) => {
         return {
-          ...prevRunIsOver,
+          ...prev,
           [runEnded.testExternalId]: {
+            id: runEnded.id,
             ended: true,
             shareUrl: runEnded.shareUrl,
           },
@@ -259,14 +283,6 @@ const App = (props: AppProps) => {
     };
   }, []);
 
-  const autoblocksCIBuildHtmlUrl =
-    props.ciBranchId && props.ciBuildId
-      ? makeAutoblocksCIBuildHtmlUrl({
-          branchId: props.ciBranchId,
-          buildId: props.ciBuildId,
-        })
-      : undefined;
-
   return (
     <>
       <Static items={consoleLogs}>
@@ -296,24 +312,7 @@ const App = (props: AppProps) => {
         width="100%"
         flexDirection="column"
       >
-        {autoblocksCIBuildHtmlUrl && (
-          // Hardcode the width so that the URL doesn't wrap.
-          <Box width={`View results at ${autoblocksCIBuildHtmlUrl}`.length}>
-            <Text color="gray">View results at</Text>
-            <Space />
-            <Text color="cyan" dimColor={true}>
-              {autoblocksCIBuildHtmlUrl}
-            </Text>
-          </Box>
-        )}
-        <Box
-          paddingX={1}
-          flexDirection="column"
-          borderStyle="round"
-          borderColor="gray"
-          minHeight={12}
-          rowGap={1}
-        >
+        <Box paddingX={1} flexDirection="column" minHeight={12} rowGap={1}>
           {testExternalIds.length === 0 && (
             <Box>
               <Text color="gray">
@@ -323,6 +322,7 @@ const App = (props: AppProps) => {
             </Box>
           )}
           {testExternalIds.map((testExternalId) => {
+            const runMeta = testIdToRunMeta[testExternalId];
             const testEvals = evals.filter(
               (e) => e.testExternalId === testExternalId,
             );
@@ -332,16 +332,28 @@ const App = (props: AppProps) => {
             return (
               <TestRow
                 key={testExternalId}
-                runIsOver={Boolean(testIdToRunIsOver[testExternalId]?.ended)}
-                shareUrl={testIdToRunIsOver[testExternalId]?.shareUrl}
                 testExternalId={testExternalId}
+                ciBranchId={props.ciBranchId}
+                runMeta={runMeta}
                 evals={testEvals}
                 errors={testErrors}
               />
             );
           })}
         </Box>
       </Box>
+      {/* Links */}
+      <Box flexDirection="column" paddingTop={1}>
+        {props.ciBranchId && props.ciBuildId && (
+          <ExternalLink
+            name="Build Summary"
+            url={makeAutoblocksCIBuildSummaryHtmlUrl({
+              branchId: props.ciBranchId,
+              buildId: props.ciBuildId,
+            })}
+          />
+        )}
+      </Box>
     </>
   );
 };

src/handlers/testing/exec/util/comments.ts

@@ -1,7 +1,7 @@
 import type { CIContext } from './ci';
 import { makeTestRunStatusFromEvaluations } from './evals';
 import { type Evaluation, EvaluationPassed, TestRunStatus } from './models';
-import { makeAutoblocksCIBuildHtmlUrl } from './url';
+import { makeAutoblocksCIBuildSummaryHtmlUrl } from './url';
 import github from '@actions/github';
 
 // Commit messages are truncated if they're longer than this
@@ -214,7 +214,7 @@ function makeGitHubComment(args: {
   runDurationMs: number;
   evaluations: Evaluation[];
 }): string {
-  const autoblocksUrl = makeAutoblocksCIBuildHtmlUrl({
+  const autoblocksUrl = makeAutoblocksCIBuildSummaryHtmlUrl({
     branchId: args.branchId,
     buildId: args.buildId,
   });
@@ -298,7 +298,7 @@ function makeSlackMessageBlocks(args: {
             text: 'View in Autoblocks',
             emoji: true,
           },
-          url: makeAutoblocksCIBuildHtmlUrl({
+          url: makeAutoblocksCIBuildSummaryHtmlUrl({
             branchId: args.branchId,
             buildId: args.buildId,
           }),

src/handlers/testing/exec/util/emitter.ts

@@ -37,6 +37,7 @@ const zRunStartedSchema = z.object({
 });
 
 const zRunEndedSchema = z.object({
+  id: z.string(),
   testExternalId: z.string(),
   shareUrl: z.string().optional(),
 });

src/handlers/testing/exec/util/run-manager.ts

@@ -190,6 +190,7 @@ export class RunManager {
     });
 
     emitter.emit(EventName.RUN_ENDED, {
+      id: this.currentRunId({ testExternalId: args.testExternalId }),
       testExternalId: args.testExternalId,
       shareUrl,
     });

src/handlers/testing/exec/util/url.ts

@@ -1,14 +1,38 @@
 import { AUTOBLOCKS_WEBAPP_BASE_URL } from '../../../../util/constants';
 
-export function makeAutoblocksCIBuildHtmlUrl(args: {
+/**
+ * Links to the summary dashboard for all tests for a given branch + build.
+ */
+export function makeAutoblocksCIBuildSummaryHtmlUrl(args: {
   branchId: string;
   buildId: string;
 }): string {
   return `${AUTOBLOCKS_WEBAPP_BASE_URL}/testing/ci?branchId=${args.branchId}&buildId=${args.buildId}`;
 }
 
-export function makeAutoblocksLocalTestHtmlUrl(args: {
+export function makeAutoblocksLocalTestResultsHtmlUrl(args: {
   testExternalId: string;
 }): string {
   return `${AUTOBLOCKS_WEBAPP_BASE_URL}/testing/local/test/${encodeURIComponent(args.testExternalId)}`;
 }
+
+export function makeAutoblocksCITestResultsHtmlUrl(args: {
+  testExternalId: string;
+  branchId: string;
+}): string {
+  const testId = encodeURIComponent(args.testExternalId);
+  const branchId = encodeURIComponent(args.branchId);
+  return `${AUTOBLOCKS_WEBAPP_BASE_URL}/testing/ci/test/${testId}/branch/${branchId}`;
+}
+
+/**
+ * Links to the human review UI for a particular test run (local or CI).
+ */
+export function makeAutoblocksHumanReviewHtmlUrl(args: {
+  testExternalId: string;
+  runId: string;
+}): string {
+  const testId = encodeURIComponent(args.testExternalId);
+  const runId = encodeURIComponent(args.runId);
+  return `${AUTOBLOCKS_WEBAPP_BASE_URL}/testing/test/${testId}/run/${runId}/human-review`;
+}

tsup.config.js

@@ -6,6 +6,6 @@ export default defineConfig({
   clean: true,
   env: {
     TSUP_PUBLIC_AUTOBLOCKS_INGESTION_KEY:
-      process.env.TSUP_PUBLIC_AUTOBLOCKS_INGESTION_KEY,
+      process.env.TSUP_PUBLIC_AUTOBLOCKS_INGESTION_KEY || '',
   },
 });