feat: download artifacts from current workflow run, improve API usage

package.json

@@ -10,13 +10,13 @@
     "release": "release-it --only-version"
   },
   "devDependencies": {
-    "@actions/artifact": "^0.6.1",
+    "@actions/artifact": "^1.1.0",
     "@actions/core": "^1.6.0",
     "@tsconfig/node12": "^1.0.9",
     "@types/fs-extra": "^9.0.13",
     "@types/node": "^17.0.16",
     "@vercel/ncc": "^0.33.1",
-    "axios": "^0.25.0",
+    "axios": "^0.27.2",
     "express": "^4.17.2",
     "express-async-handler": "^1.2.0",
     "fs-extra": "^10.0.0",

src/starter.ts

@@ -1,21 +1,34 @@
-import { info, saveState } from '@actions/core';
+import { info, saveState, setFailed } from '@actions/core';
 import { spawn } from 'child_process';
 import fs from 'fs-extra';
 import path from 'path';
 import { cacheDir, States } from './utils/constants';
+import { downloadSameWorkflowArtifacts } from './utils/downloadArtifact';
 
-fs.ensureDirSync(cacheDir);
+async function main() {
+  await fs.ensureDir(cacheDir);
 
-const out = fs.openSync(path.join(cacheDir, 'out.log'), 'a');
-const err = fs.openSync(path.join(cacheDir, 'out.log'), 'a');
+  await downloadSameWorkflowArtifacts();
 
-const subprocess = spawn('node', [path.resolve(__dirname, '../turboServer/index.js')], {
-  detached: true,
-  stdio: ['ignore', out, err],
-  env: process.env,
-});
+  const out = fs.openSync(path.join(cacheDir, 'out.log'), 'a');
+  const err = fs.openSync(path.join(cacheDir, 'out.log'), 'a');
+
+  const subprocess = spawn(
+    'node',
+    [path.resolve(__dirname, '../turboServer/index.js')],
+    {
+      detached: true,
+      stdio: ['ignore', out, err],
+      env: process.env,
+    }
+  );
 
-subprocess.unref();
+  subprocess.unref();
 
-info(`${States.TURBO_LOCAL_SERVER_PID}: ${subprocess.pid}`);
-saveState(States.TURBO_LOCAL_SERVER_PID, subprocess.pid);
+  info(`${States.TURBO_LOCAL_SERVER_PID}: ${subprocess.pid}`);
+  saveState(States.TURBO_LOCAL_SERVER_PID, subprocess.pid);
+}
+
+main().catch((error) => {
+  setFailed(error);
+});

src/turboServer.ts

@@ -3,7 +3,7 @@ import express from 'express';
 import asyncHandler from 'express-async-handler';
 import fs from 'fs-extra';
 import path from 'path';
-import { artifactApi } from './utils/artifactApi';
+import { artifactApi, IArtifactListResponse } from './utils/artifactApi';
 import { cacheDir, Inputs } from './utils/constants';
 import { downloadArtifact } from './utils/downloadArtifact';
 
@@ -18,6 +18,8 @@ async function startServer() {
     trimWhitespace: true,
   });
 
+  let artifactList: IArtifactListResponse | undefined;
+
   app.all('*', (req, res, next) => {
     console.info(`Got a ${req.method} request`, req.path);
     const { authorization = '' } = req.headers;
@@ -34,22 +36,33 @@ async function startServer() {
     '/v8/artifacts/:artifactId',
     asyncHandler(async (req: any, res: any) => {
       const { artifactId } = req.params;
-      const list = await artifactApi.listArtifacts();
 
-      const existingArtifact = list.artifacts?.find(
-        (artifact) => artifact.name === artifactId
-      );
+      const filepath = path.join(cacheDir, `${artifactId}.gz`);
+
+      if (!fs.pathExistsSync(filepath)) {
+        // Requested artifact does not belong to the current workflow, so it wasn't downloaded before
+        // Check if it exists for another one
+        if (!artifactList) {
+          // Cache the response for the runtime of the server
+          // The server is typically short-lived, so it is very unlikely that we're going to miss an artifact this way
+          artifactList = await artifactApi.listArtifacts();
+        }
+
+        const existingArtifact = artifactList.artifacts?.find(
+          (artifact) => artifact.name === artifactId
+        );
 
-      if (existingArtifact) {
-        console.log(`Artifact ${artifactId} found.`);
-        await downloadArtifact(existingArtifact, cacheDir);
+        if (existingArtifact) {
+          console.log(`Artifact ${artifactId} found.`);
+          await downloadArtifact(existingArtifact, cacheDir);
+        }
         console.log(
-          `Artifact ${artifactId} downloaded successfully to ${cacheDir}/${artifactId}.gz.`
+          `Artifact ${artifactId} downloaded successfully to ${filepath}.`
         );
+      } else {
+        console.log(`Artifact ${artifactId} already exists.`);
       }
 
-      const filepath = path.join(cacheDir, `${artifactId}.gz`);
-
       if (!fs.pathExistsSync(filepath)) {
         console.log(`Artifact ${artifactId} not found.`);
         return res.status(404).send('Not found');
@@ -85,6 +98,11 @@ async function startServer() {
     });
   });
 
+  app.post('/v8/artifacts/events', (req, res) => {
+    // Ignore
+    res.status(200).send();
+  });
+
   app.disable('etag').listen(port, () => {
     console.log(`Cache dir: ${cacheDir}`);
     console.log(`Local Turbo server is listening at http://127.0.0.1:${port}`);

src/utils/artifactApi.ts

@@ -2,7 +2,7 @@ import { getInput } from '@actions/core';
 import { Axios } from 'axios';
 import { Inputs } from './constants';
 
-interface IArtifactListResponse {
+export interface IArtifactListResponse {
   total_count: number;
   artifacts?: Array<{
     id: number;

src/utils/downloadArtifact.ts

@@ -3,6 +3,8 @@ import StreamZip from 'node-stream-zip';
 import path from 'path';
 import { artifactApi } from './artifactApi';
 import os from 'os';
+import { create } from '@actions/artifact';
+import { cacheDir } from './constants';
 
 const tempArchiveFolder = path.join(
   process.env.RUNNER_TEMP || os.tmpdir(),
@@ -33,3 +35,36 @@ export async function downloadArtifact(artifact, destFolder) {
   await zip.extract(null, destFolder);
   await zip.close();
 }
+
+export async function downloadSameWorkflowArtifacts() {
+  const client = create();
+  // Try to download all artifacts from the current workflow, but do not fail the build if this fails
+  const artifacts = await client.downloadAllArtifacts(cacheDir).catch((e) => {
+    console.error(`Failed to download workflow artifacts: ${e.message}`);
+    return [];
+  });
+
+  // downloadAllArtifacts creates folder with the artifact name and puts the artifact in there
+  // We need to move the artifact to the destFolder, so the server can find them
+  for (const artifact of artifacts) {
+    const artifactFileName = path.join(
+      artifact.downloadPath,
+      `${artifact.artifactName}.gz`
+    );
+    try {
+      await fs.move(
+        artifactFileName,
+        path.join(cacheDir, `${artifact.artifactName}.gz`)
+      );
+      // Remember that this artifact was downloaded from the current workflow
+      await fs.createFile(
+        path.join(cacheDir, `${artifact.artifactName}.local`)
+      );
+      await fs.remove(artifact.downloadPath);
+    } catch (e: any) {
+      console.error(
+        `Failed to download artifact ${artifact.artifactName}: ${e.message}`
+      );
+    }
+  }
+}

src/utils/uploadArtifacts.ts

@@ -2,22 +2,28 @@ import { create } from '@actions/artifact';
 import { debug, info } from '@actions/core';
 import fs from 'fs-extra';
 import path from 'path';
-import { artifactApi } from './artifactApi';
 import { cacheDir } from './constants';
 
 export async function uploadArtifacts() {
-  const list = await artifactApi.listArtifacts();
-  const existingArtifacts = (list.artifacts || []).map(
-    (artifact) => artifact.name
-  );
-
+  // Upload all artifacts that were not downloaded from the current workflow
+  // This avoids a list request for every cache miss in subsequent jobs of the same workflow
   const client = create();
 
   const files = fs.readdirSync(cacheDir);
 
   const artifactFiles = files.filter((filename) => filename.endsWith('.gz'));
+  const currentWorkflowArtifacts = files
+    .filter((filename) => filename.endsWith('.local'))
+    .map((file) => path.basename(file, '.local'));
 
   debug(`artifact files: ${JSON.stringify(artifactFiles, null, 2)}`);
+  debug(
+    `artifacts downloaded from the current workflow: ${JSON.stringify(
+      currentWorkflowArtifacts,
+      null,
+      2
+    )}`
+  );
 
   const artifactsToUpload = artifactFiles
     .map((artifactFilename) => {
@@ -28,7 +34,7 @@ export async function uploadArtifacts() {
 
       return { artifactFilename, artifactId };
     })
-    .filter(({ artifactId }) => !existingArtifacts.includes(artifactId));
+    .filter(({ artifactId }) => !currentWorkflowArtifacts.includes(artifactId));
 
   if (artifactsToUpload.length) {
     info(`Gonna upload ${artifactsToUpload.length} artifacts:`);