Showcase how to integrate with Dagger (stateful#1414)

Work in progress. Not finished yet.

.eslintignore

@@ -1,4 +1,5 @@
 out
+dagger
 node_modules
 __generated__
 __generated-platform__

dagger/Pipeline.md

@@ -0,0 +1,92 @@
+---
+cwd: ..
+runme:
+  id: 01HTNVRGFMWZERW6S2CZZ9E990
+  version: v3
+---
+
+# Express Dagger Pipelines in Notebooks
+
+One function's output is being used as another function's input. This is a common pattern to compose pipelines.
+
+In this notebook, we will explore how to express such pipelines using Dagger inside a literal notebook environment. As an example, we build and bundle this VS Code extension.
+
+> 💡 This demo is using a "slightly" modified version of the dagger binary based on https://github.com/dagger/dagger/pull/7479.
+
+### Build the Kernel Binary
+
+Let's use [github.com/purpleclay/daggerverse](https://daggerverse.dev/mod/github.com/purpleclay/daggerverse/golang) to build the kernel binary.
+
+```sh {"id":"01J04HR247XE1TK2MVB9SR4W51","name":"KERNEL_BINARY"}
+dagger call --progress=$PROGRESS \
+  -m golang \
+  --src ../runme \
+  build \
+  file \
+    --path runme
+```
+
+### Grab the Presetup Script
+
+We don't want to maintain a build container. Let's grab the script used to provision it on the fly.
+
+```sh {"id":"01J04N5MHBFHPQQZ0HDGVQVC70","name":"PRESETUP"}
+dagger call --progress=$PROGRESS \
+  get-repo-file \
+    --repo "https://github.com/stateful/vscode-runme#seb/dagger-integr" \
+    --path dagger/scripts/presetup.sh
+```
+
+### Build the Extension
+
+Putting everything together, we build the extension. While we're issuing a query here, we really want to do this with the `dagger` CLI via `dagger call`.
+
+This call will currently fail because passing IDs from a module appears to bypass caching. However, we hear this is on Dagger's roadmap.
+
+```sh {"id":"01J04KG1K4S8ZND9RYXKFVP4GK"}
+dagger --progress=$PROGRESS query <<EOF
+{
+  vscodeRunme {
+    withRemote(remote: "github.com/stateful/vscode-runme", ref: "main") {
+      withContainer(
+        binary: "$DAGGER_ID_KERNEL_BINARY"
+        presetup: "$DAGGER_ID_PRESETUP"
+      ) {
+        buildExtension(githubToken: "$GITHUB_TOKEN") {
+          export(path: "dagger/extension.xsix")
+        }
+      }
+    }
+  }
+}
+EOF
+```
+
+This is the error we're seeing:
+
+<pre>🚨 17: ! failed to load ref for blob snapshot: missing descriptor handlers for lazy blobs [sha256:74eed75e10a8e6dceef2b446cf20daed65774c11cc827ec80b41c7f476c819af]</pre>
+
+## What it could look like...
+
+> 💡 This won't actually run.... yet.
+
+Essentially, above's query would run this module, however, using the CLI without relying on the host filesystem.
+
+```sh {"excludeFromRunAll":"true","id":"01J04HR247XE1TK2MVBBPV7ZM7","name":"EXTENSION_VSIX"}
+dagger call --progress=$PROGRESS \
+  with-remote \
+    --remote "github.com/stateful/vscode-runme" \
+    --ref "seb/dagger-integr" \
+  with-container \
+    --binary $DAGGER_ID_KERNEL_BINARY \
+    --presetup $DAGGER_ID_PRESETUP \
+  with-github-token \
+    --token $DAGGER_ID_TOKEN \
+  build-extension
+```
+
+Even above's module's output could become another function/pipeline's input.
+
+```sh {"id":"01J094XAEPWN0JM13BFT9VVVB3","interactive":"false"}
+curl -s "https://runme.dev/img/thankyou.png"
+```

dagger/README.md

@@ -10,11 +10,11 @@ export PROGRESS_OUTPUT="plain"
 echo "Using $PROGRESS_OUTPUT for progress logging"
 ```
 
-```sh {"id":"01HTNVRK3AJ2AT8M24TA996RCJ","terminalRows":"15"}
+```sh {"excludeFromRunAll":"true","id":"01HTNVRK3AJ2AT8M24TA996RCJ","terminalRows":"15"}
 dagger -m vscode-runme functions
 ```
 
-```sh {"id":"01HTQBSZTS5M1HP3GGP4T99PT0","name":"KERNEL_BINARY","terminalRows":"28"}
+```sh {"id":"01HTQBSZTS5M1HP3GGP4T99PT0","name":"KERNEL_BINARY"}
 dagger call --progress $PROGRESS_OUTPUT \
   -m golang \
   --src ../runme \
@@ -24,15 +24,14 @@ dagger call --progress $PROGRESS_OUTPUT \
     --path runme
 ```
 
-```sh {"id":"01HTNZBARHB97RPQPCVQZ7PNRN","name":"EXTENSION_VSIX","terminalRows":"25"}
-dagger --progress $PROGRESS_OUTPUT \
-  call \
+```sh {"id":"01HTNZBARHB97RPQPCVQZ7PNRN","name":"EXTENSION_VSIX"}
+dagger call --progress $PROGRESS_OUTPUT \
   with-remote \
     --remote "github.com/stateful/vscode-runme" \
     --ref "main" \
   with-container \
     --binary /tmp/runme/runme \
     --presetup dagger/scripts/presetup.sh \
   build-extension \
-    --gh-token $(gh auth token)
+    --gh-token cmd:"gh auth token"
 ```

dagger/src/index.ts

@@ -5,7 +5,7 @@
  */
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-import { dag, Container, File, Directory, object, func, field } from '@dagger.io/dagger'
+import { dag, Container, File, Directory, object, func, field, Secret } from '@dagger.io/dagger'
 
 @object()
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -14,10 +14,10 @@ class VscodeRunme {
    * The working repository directory for the VscodeRunme instance.
    */
   @field()
-  dir: Directory
+  directory: Directory
 
   /**
-   * The base container being used for building the extension
+   * The base container being used for building the extension.
    */
   @field()
   container: Container
@@ -30,7 +30,7 @@ class VscodeRunme {
    */
   @func()
   withRemote(remote: string, ref: string): VscodeRunme {
-    this.dir = dag
+    this.directory = dag
     .git(`https://${remote}.git`)
     .ref(ref)
     .tree()
@@ -45,31 +45,68 @@ class VscodeRunme {
    * @returns The modified VscodeRunme instance.
    */
   @func()
-  withContainer(binary?: File, presetup?: File): VscodeRunme {
+  withContainer(binary: File, presetup: File): VscodeRunme {
     this.container = dag
       .container()
       .from('node:18')
       .withEnvVariable('EXTENSION_NAME', 'runme')
       .withFile('/usr/local/bin/runme', binary)
       .withFile('/usr/local/bin/presetup', presetup)
       .withEntrypoint([])
-      .withMountedDirectory('/mnt/vscode-runme', this.dir)
+      .withMountedDirectory('/mnt/vscode-runme', this.directory)
       .withWorkdir('/mnt/vscode-runme')
       .withExec('bash /usr/local/bin/presetup'.split(' '))
 
       return this
   }
 
+  /**
+   * Sets up the container for the VscodeRunme instance.
+   * @param path - Path to file inside the container.
+   * @returns The file or error
+   */
+  @func()
+  async getFile(path: string): Promise<File> {
+    return this.container.file(path)
+  }
+
+  /**
+   * Sets up the container for the VscodeRunme instance.
+   * @param path - Path to file inside the container.
+   * @returns The file or error
+   */
+  @func()
+  async getRepoFile(repo: string, path: string): Promise<File> {
+    return dag
+    .git(repo)
+    .head()
+    .tree()
+    .file(path)
+  }
+
+  /**
+   * Sets up the container for the VscodeRunme instance.
+   * @param name - Name of the secret.
+   * @param plain - Plaintext.
+   * @returns The Secret or error
+   */
+  @func()
+  async getSecret(name: string, value: Secret): Promise<Secret> {
+    const plain = await value.plaintext()
+    return dag.setSecret(name, plain)
+  }
+
   /**
    * Sets up the container for the VscodeRunme instance.
    * @param ghToken - Valid GitHub access token for API access.
-   * @returns The modified VscodeRunme instance.
+   * @returns The packaged VSIX extension file.
    */
   @func()
-  async buildExtension(ghToken: string): Promise<string> {
+  async buildExtension(githubToken: string): Promise<File> {
     return this.container
-      .withEnvVariable('GITHUB_TOKEN', ghToken)
-      .withExec('runme run setup build'.split(' ')).
-      stdout()
+      .withEnvVariable('GITHUB_TOKEN', githubToken)
+      // .withExec('runme run setup'.split(' '))
+      .withExec('runme run setup build bundle'.split(' '))
+      .file('runme-extension.vsix')
   }
 }

examples/dagger/Notebook.md

@@ -0,0 +1,33 @@
+# Runme ▶️ for Dagger
+
+```sh {"id":"01J097BHJHQS28M29YR0WCZ3B8","interactive":"false"}
+curl -s "https://framerusercontent.com/images/tpJEZ337KKxXU4q1SSUXDx4FG4.png?scale-down-to=512"
+```
+
+Showcase the notebook experience to **author, debug, express, and run** Dagger pipelines.
+
+### Let's use a Dagger function to build the Runme binary
+
+```sh {"id":"01HZSMYF33TFKMEVRX5P64BNTB","interactive":"true","name":"RUNME_BINARY"}
+dagger --progress=$PROGRESS \
+    call \
+        -m github.com/purpleclay/daggerverse/golang@7e83bccc350fa981e975ac0c8619f92a1b729958 \
+        --src "https://github.com/stateful/runme#main" \
+    build \
+        --arch $(go env GOARCH) \
+        --os $(go env GOOS) \
+    file \
+        --path runme
+```
+
+### What does the 🐮 cow say?
+
+```sh {"id":"01J022WD7Z6TM1QQ075X09BTK4","interactive":"true","name":"COWSAY"}
+dagger --progress=plain \
+    call \
+        -m github.com/shykes/daggerverse/[email protected] \
+    container \
+        --packages=cowsay
+```
+
+👉 Let's continue over here with the [pipeline demo](../../dagger/Pipeline.md).

package.json

@@ -519,7 +519,8 @@
           "stateful.runme/error",
           "stateful.runme/github-stdout",
           "stateful.runme/gcp",
-          "stateful.runme/aws"
+          "stateful.runme/aws",
+          "stateful.runme/dagger"
         ],
         "requiresMessaging": "optional"
       }
@@ -1138,6 +1139,7 @@
     "data-guardian": "^1.1.2",
     "dotenv": "^16.3.1",
     "filenamify": "^6.0.0",
+    "filesize": "^10.1.2",
     "get-port": "^7.0.0",
     "got": "^11.8.2",
     "graphql": "^16.8.0",