implement [data_blobs]

This implements support for `[data_blobs]` for service workers, which lets you bind some data from a file as an ArrayBuffer in service-worker format workers. It also adds a `--data-blob` cli argument for the same.
cloudflare · Apr 1, 2022 · 4d71297 · 4d71297
1 parent 57dd23d
commit 4d71297
Show file tree

Hide file tree

Showing 4 changed files with 85 additions and 14 deletions.
diff --git a/packages/core/src/plugins/bindings.ts b/packages/core/src/plugins/bindings.ts
@@ -46,6 +46,7 @@ export interface BindingsOptions {
   globals?: Record<string, any>;
   wasmBindings?: Record<string, string>;
   textBlobBindings?: Record<string, string>;
+  dataBlobBindings?: Record<string, string>;
   serviceBindings?: ServiceBindingsOptions;
 }
 
@@ -175,6 +176,16 @@ export class BindingsPlugin
   })
   textBlobBindings?: Record<string, string>;
 
+  @Option({
+    type: OptionType.OBJECT,
+    typeFormat: "NAME=PATH",
+    name: "data-blob",
+    description: "Data blob to bind",
+    logName: "Data Blob Bindings",
+    fromWrangler: ({ data_blobs }) => data_blobs,
+  })
+  dataBlobBindings?: Record<string, string>;
+
   @Option({
     type: OptionType.OBJECT,
     typeFormat: "NAME=MOUNT[@ENV]",
@@ -258,8 +269,9 @@ export class BindingsPlugin
     // 2) .env Variables
     // 3) WASM Module Bindings
     // 4) Text blob Bindings
-    // 5) Service Bindings
-    // 6) Custom Bindings
+    // 5) Data blob Bindings
+    // 6) Service Bindings
+    // 7) Custom Bindings
 
     const bindings: Context = {};
     const watch: string[] = [];
@@ -303,12 +315,22 @@ export class BindingsPlugin
       }
     }
 
-    // 5) Load service bindings
+    // 5) Load data blobs from files
+    if (this.dataBlobBindings) {
+      // eslint-disable-next-line prefer-const
+      for (let [name, dataPath] of Object.entries(this.dataBlobBindings)) {
+        dataPath = path.resolve(this.ctx.rootPath, dataPath);
+        bindings[name] = await fs.readFile(dataPath, "utf-8");
+        watch.push(dataPath);
+      }
+    }
+
+    // 6) Load service bindings
     for (const { name, service } of this.#processedServiceBindings) {
       bindings[name] = new Fetcher(service, this.#getServiceFetch);
     }
 
-    // 6) Copy user's arbitrary bindings
+    // 7) Copy user's arbitrary bindings
     Object.assign(bindings, this.bindings);
 
     return { globals: this.globals, bindings, watch };

diff --git a/packages/core/test/fixtures/some-binary-data.bin b/packages/core/test/fixtures/some-binary-data.bin
@@ -0,0 +1,7 @@
+Yar Pirate Ipsum
+
+Prow scuttle parrel provost Sail ho shrouds spirits boom mizzenmast yardarm. Pinnace holystone mizzenmast quarter crow's nest nipperkin grog yardarm hempen halter furl. Swab barque interloper chantey doubloon starboard grog black jack gangway rutters.
+
+Deadlights jack lad schooner scallywag dance the hempen jig carouser broadside cable strike colors. Bring a spring upon her cable holystone blow the man down spanker Shiver me timbers to go on account lookout wherry doubloon chase. Belay yo-ho-ho keelhaul squiffy black spot yardarm spyglass sheet transom heave to.
+
+Trysail Sail ho Corsair red ensign hulk smartly boom jib rum gangway. Case shot Shiver me timbers gangplank crack Jennys tea cup ballast Blimey lee snow crow's nest rutters. Fluke jib scourge of the seven seas boatswain schooner gaff booty Jack Tar transom spirits.
diff --git a/packages/core/test/plugins/bindings.spec.ts b/packages/core/test/plugins/bindings.spec.ts
@@ -42,6 +42,10 @@ const addModulePath = path.join(fixturesPath, "add.wasm");
 const loremIpsumPath = path.join(fixturesPath, "lorem-ipsum.txt");
 const loremIpsum = readFileSync(loremIpsumPath, "utf-8");
 
+// some-binary-data.bin is also some paragraphs of nonsense text
+const someBinaryDataPath = path.join(fixturesPath, "some-binary-data.bin");
+const someBinaryData = readFileSync(someBinaryDataPath, "binary");
+
 test("BindingsPlugin: parses options from argv", (t) => {
   let options = parsePluginArgv(BindingsPlugin, [
     "--env",
@@ -62,6 +66,10 @@ test("BindingsPlugin: parses options from argv", (t) => {
     "TEXT1=text-blob-1.txt",
     "--text-blob",
     "TEXT2=text-blob-2.txt",
+    "--data-blob",
+    "DATA1=data-blob-1.bin",
+    "--data-blob",
+    "DATA2=data-blob-2.bin",
     "--service",
     "SERVICE1=service1",
     "--service",
@@ -73,6 +81,7 @@ test("BindingsPlugin: parses options from argv", (t) => {
     globals: { KEY3: "value3", KEY4: "value4" },
     wasmBindings: { MODULE1: "module1.wasm", MODULE2: "module2.wasm" },
     textBlobBindings: { TEXT1: "text-blob-1.txt", TEXT2: "text-blob-2.txt" },
+    dataBlobBindings: { DATA1: "data-blob-1.bin", DATA2: "data-blob-2.bin" },
     serviceBindings: {
       SERVICE1: "service1",
       SERVICE2: { service: "service2", environment: "development" },
@@ -109,6 +118,10 @@ test("BindingsPlugin: parses options from wrangler config", async (t) => {
       TEXT1: "text-blob-1.txt",
       TEXT2: "text-blob-2.txt",
     },
+    data_blobs: {
+      DATA1: "data-blob-1.bin",
+      DATA2: "data-blob-2.bin",
+    },
     experimental_services: [
       { name: "SERVICE1", service: "service1", environment: "development" },
       { name: "SERVICE2", service: "service2", environment: "production" },
@@ -123,6 +136,7 @@ test("BindingsPlugin: parses options from wrangler config", async (t) => {
     globals: { KEY5: "value5", KEY6: false, KEY7: 10 },
     wasmBindings: { MODULE1: "module1.wasm", MODULE2: "module2.wasm" },
     textBlobBindings: { TEXT1: "text-blob-1.txt", TEXT2: "text-blob-2.txt" },
+    dataBlobBindings: { DATA1: "data-blob-1.bin", DATA2: "data-blob-2.bin" },
     serviceBindings: {
       SERVICE1: { service: "service1", environment: "development" },
       SERVICE2: { service: "service2", environment: "production" },
@@ -156,6 +170,7 @@ test("BindingsPlugin: logs options", (t) => {
     globals: { KEY5: "value5", KEY6: "value6" },
     wasmBindings: { MODULE1: "module1.wasm", MODULE2: "module2.wasm" },
     textBlobBindings: { TEXT1: "text-blob-1.txt", TEXT2: "text-blob-2.txt" },
+    dataBlobBindings: { DATA1: "data-blob-1.bin", DATA2: "data-blob-2.bin" },
     serviceBindings: {
       SERVICE1: "service1",
       SERVICE2: { service: "service2", environment: "development" },
@@ -168,6 +183,7 @@ test("BindingsPlugin: logs options", (t) => {
     "Custom Globals: KEY5, KEY6",
     "WASM Bindings: MODULE1, MODULE2",
     "Text Blob Bindings: TEXT1, TEXT2",
+    "Data Blob Bindings: DATA1, DATA2",
     "Service Bindings: SERVICE1, SERVICE2",
   ]);
   logs = logPluginOptions(BindingsPlugin, { envPath: true });
@@ -292,23 +308,40 @@ test("BindingsPlugin: setup: loads text blob bindings", async (t) => {
   t.is(result.bindings?.LOREM_IPSUM, loremIpsum);
 });
 
+test("BindingsPlugin: setup: loads data blob bindings", async (t) => {
+  let plugin = new BindingsPlugin(ctx, {
+    dataBlobBindings: { BINARY_DATA: someBinaryDataPath },
+  });
+  let result = await plugin.setup();
+  t.is(result.bindings?.BINARY_DATA, someBinaryData);
+
+  // Check resolves text blob bindings path relative to rootPath
+  plugin = new BindingsPlugin(
+    { log, compat, rootPath: path.dirname(someBinaryDataPath) },
+    { dataBlobBindings: { BINARY_DATA: "some-binary-data.bin" } }
+  );
+  result = await plugin.setup();
+  t.is(result.bindings?.BINARY_DATA, someBinaryData);
+});
+
 test("BindingsPlugin: setup: loads bindings from all sources", async (t) => {
   // Bindings should be loaded in this order, from lowest to highest priority:
   // 1) Wrangler [vars]
   // 2) .env Variables
   // 3) WASM Module Bindings
   // 4) Text Blob Bindings
-  // 5) Service Bindings
-  // 6) Custom Bindings
+  // 5) Data Blob Bindings
+  // 6) Service Bindings
+  // 7) Custom Bindings
 
   // wranglerOptions should contain [kWranglerBindings]
   const wranglerOptions = parsePluginWranglerConfig(BindingsPlugin, {
-    vars: { A: "w", B: "w", C: "w", D: "w", E: "w", F: "w" },
+    vars: { A: "w", B: "w", C: "w", D: "w", E: "w", F: "w", G: "w" },
   });
 
   const tmp = await useTmp(t);
   const envPath = path.join(tmp, ".env");
-  await fs.writeFile(envPath, "A=env\nB=env\nC=env\nD=env\nE=env");
+  await fs.writeFile(envPath, "A=env\nB=env\nC=env\nD=env\nE=env\nF=env");
 
   const obj = { ping: "pong" };
   const throws = () => {
@@ -321,11 +354,18 @@ test("BindingsPlugin: setup: loads bindings from all sources", async (t) => {
       B: addModulePath,
       C: addModulePath,
       D: addModulePath,
+      E: addModulePath,
     },
     textBlobBindings: {
       A: loremIpsumPath,
       B: loremIpsumPath,
       C: loremIpsumPath,
+      D: loremIpsumPath,
+    },
+    dataBlobBindings: {
+      A: someBinaryDataPath,
+      B: someBinaryDataPath,
+      C: someBinaryDataPath,
     },
     serviceBindings: { A: throws, B: throws },
     bindings: { A: obj },
@@ -334,10 +374,11 @@ test("BindingsPlugin: setup: loads bindings from all sources", async (t) => {
   const result = await plugin.setup();
   assert(result.bindings);
 
-  t.is(result.bindings.F, "w");
-  t.is(result.bindings.E, "env");
-  t.true(result.bindings.D instanceof WebAssembly.Module);
-  t.is(result.bindings.C, loremIpsum);
+  t.is(result.bindings.G, "w");
+  t.is(result.bindings.F, "env");
+  t.true(result.bindings.E instanceof WebAssembly.Module);
+  t.is(result.bindings.D, loremIpsum);
+  t.is(result.bindings.C, someBinaryData);
   t.true(result.bindings.B instanceof Fetcher);
   t.is(result.bindings.A, obj);
 });

diff --git a/packages/shared/src/wrangler.ts b/packages/shared/src/wrangler.ts
@@ -34,8 +34,9 @@ export interface WranglerEnvironmentConfig {
     crons?: string[];
   }; // inherited
   usage_model?: "bundled" | "unbound"; // inherited
-  wasm_modules?: Record<string, string>; // (probably) inherited
-  text_blobs?: Record<string, string>;
+  wasm_modules?: Record<string, string>; // inherited
+  text_blobs?: Record<string, string>; // inherited
+  data_blobs?: Record<string, string>; // inherited
   experimental_services?: {
     name: string;
     service: string;