From 4d712979f63900760fa1278ac3b54fcd1db515cc Mon Sep 17 00:00:00 2001 From: Sunil Pai Date: Fri, 1 Apr 2022 11:10:01 +0530 Subject: [PATCH] implement `[data_blobs]` This implements support for `[data_blobs]` for service workers, which lets you bind some data from a file as an ArrayBuffer in service-worker format workers. It also adds a `--data-blob` cli argument for the same. --- packages/core/src/plugins/bindings.ts | 30 ++++++++-- .../core/test/fixtures/some-binary-data.bin | 7 +++ packages/core/test/plugins/bindings.spec.ts | 57 ++++++++++++++++--- packages/shared/src/wrangler.ts | 5 +- 4 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 packages/core/test/fixtures/some-binary-data.bin diff --git a/packages/core/src/plugins/bindings.ts b/packages/core/src/plugins/bindings.ts index 9b135088c..80578db92 100644 --- a/packages/core/src/plugins/bindings.ts +++ b/packages/core/src/plugins/bindings.ts @@ -46,6 +46,7 @@ export interface BindingsOptions { globals?: Record; wasmBindings?: Record; textBlobBindings?: Record; + dataBlobBindings?: Record; serviceBindings?: ServiceBindingsOptions; } @@ -175,6 +176,16 @@ export class BindingsPlugin }) textBlobBindings?: Record; + @Option({ + type: OptionType.OBJECT, + typeFormat: "NAME=PATH", + name: "data-blob", + description: "Data blob to bind", + logName: "Data Blob Bindings", + fromWrangler: ({ data_blobs }) => data_blobs, + }) + dataBlobBindings?: Record; + @Option({ type: OptionType.OBJECT, typeFormat: "NAME=MOUNT[@ENV]", @@ -258,8 +269,9 @@ export class BindingsPlugin // 2) .env Variables // 3) WASM Module Bindings // 4) Text blob Bindings - // 5) Service Bindings - // 6) Custom Bindings + // 5) Data blob Bindings + // 6) Service Bindings + // 7) Custom Bindings const bindings: Context = {}; const watch: string[] = []; @@ -303,12 +315,22 @@ export class BindingsPlugin } } - // 5) Load service bindings + // 5) Load data blobs from files + if (this.dataBlobBindings) { + // eslint-disable-next-line prefer-const + for (let [name, dataPath] of Object.entries(this.dataBlobBindings)) { + dataPath = path.resolve(this.ctx.rootPath, dataPath); + bindings[name] = await fs.readFile(dataPath, "utf-8"); + watch.push(dataPath); + } + } + + // 6) Load service bindings for (const { name, service } of this.#processedServiceBindings) { bindings[name] = new Fetcher(service, this.#getServiceFetch); } - // 6) Copy user's arbitrary bindings + // 7) Copy user's arbitrary bindings Object.assign(bindings, this.bindings); return { globals: this.globals, bindings, watch }; diff --git a/packages/core/test/fixtures/some-binary-data.bin b/packages/core/test/fixtures/some-binary-data.bin new file mode 100644 index 000000000..440f778c6 --- /dev/null +++ b/packages/core/test/fixtures/some-binary-data.bin @@ -0,0 +1,7 @@ +Yar Pirate Ipsum + +Prow scuttle parrel provost Sail ho shrouds spirits boom mizzenmast yardarm. Pinnace holystone mizzenmast quarter crow's nest nipperkin grog yardarm hempen halter furl. Swab barque interloper chantey doubloon starboard grog black jack gangway rutters. + +Deadlights jack lad schooner scallywag dance the hempen jig carouser broadside cable strike colors. Bring a spring upon her cable holystone blow the man down spanker Shiver me timbers to go on account lookout wherry doubloon chase. Belay yo-ho-ho keelhaul squiffy black spot yardarm spyglass sheet transom heave to. + +Trysail Sail ho Corsair red ensign hulk smartly boom jib rum gangway. Case shot Shiver me timbers gangplank crack Jennys tea cup ballast Blimey lee snow crow's nest rutters. Fluke jib scourge of the seven seas boatswain schooner gaff booty Jack Tar transom spirits. \ No newline at end of file diff --git a/packages/core/test/plugins/bindings.spec.ts b/packages/core/test/plugins/bindings.spec.ts index 3b8ba0ee6..d7b6165ec 100644 --- a/packages/core/test/plugins/bindings.spec.ts +++ b/packages/core/test/plugins/bindings.spec.ts @@ -42,6 +42,10 @@ const addModulePath = path.join(fixturesPath, "add.wasm"); const loremIpsumPath = path.join(fixturesPath, "lorem-ipsum.txt"); const loremIpsum = readFileSync(loremIpsumPath, "utf-8"); +// some-binary-data.bin is also some paragraphs of nonsense text +const someBinaryDataPath = path.join(fixturesPath, "some-binary-data.bin"); +const someBinaryData = readFileSync(someBinaryDataPath, "binary"); + test("BindingsPlugin: parses options from argv", (t) => { let options = parsePluginArgv(BindingsPlugin, [ "--env", @@ -62,6 +66,10 @@ test("BindingsPlugin: parses options from argv", (t) => { "TEXT1=text-blob-1.txt", "--text-blob", "TEXT2=text-blob-2.txt", + "--data-blob", + "DATA1=data-blob-1.bin", + "--data-blob", + "DATA2=data-blob-2.bin", "--service", "SERVICE1=service1", "--service", @@ -73,6 +81,7 @@ test("BindingsPlugin: parses options from argv", (t) => { globals: { KEY3: "value3", KEY4: "value4" }, wasmBindings: { MODULE1: "module1.wasm", MODULE2: "module2.wasm" }, textBlobBindings: { TEXT1: "text-blob-1.txt", TEXT2: "text-blob-2.txt" }, + dataBlobBindings: { DATA1: "data-blob-1.bin", DATA2: "data-blob-2.bin" }, serviceBindings: { SERVICE1: "service1", SERVICE2: { service: "service2", environment: "development" }, @@ -109,6 +118,10 @@ test("BindingsPlugin: parses options from wrangler config", async (t) => { TEXT1: "text-blob-1.txt", TEXT2: "text-blob-2.txt", }, + data_blobs: { + DATA1: "data-blob-1.bin", + DATA2: "data-blob-2.bin", + }, experimental_services: [ { name: "SERVICE1", service: "service1", environment: "development" }, { name: "SERVICE2", service: "service2", environment: "production" }, @@ -123,6 +136,7 @@ test("BindingsPlugin: parses options from wrangler config", async (t) => { globals: { KEY5: "value5", KEY6: false, KEY7: 10 }, wasmBindings: { MODULE1: "module1.wasm", MODULE2: "module2.wasm" }, textBlobBindings: { TEXT1: "text-blob-1.txt", TEXT2: "text-blob-2.txt" }, + dataBlobBindings: { DATA1: "data-blob-1.bin", DATA2: "data-blob-2.bin" }, serviceBindings: { SERVICE1: { service: "service1", environment: "development" }, SERVICE2: { service: "service2", environment: "production" }, @@ -156,6 +170,7 @@ test("BindingsPlugin: logs options", (t) => { globals: { KEY5: "value5", KEY6: "value6" }, wasmBindings: { MODULE1: "module1.wasm", MODULE2: "module2.wasm" }, textBlobBindings: { TEXT1: "text-blob-1.txt", TEXT2: "text-blob-2.txt" }, + dataBlobBindings: { DATA1: "data-blob-1.bin", DATA2: "data-blob-2.bin" }, serviceBindings: { SERVICE1: "service1", SERVICE2: { service: "service2", environment: "development" }, @@ -168,6 +183,7 @@ test("BindingsPlugin: logs options", (t) => { "Custom Globals: KEY5, KEY6", "WASM Bindings: MODULE1, MODULE2", "Text Blob Bindings: TEXT1, TEXT2", + "Data Blob Bindings: DATA1, DATA2", "Service Bindings: SERVICE1, SERVICE2", ]); logs = logPluginOptions(BindingsPlugin, { envPath: true }); @@ -292,23 +308,40 @@ test("BindingsPlugin: setup: loads text blob bindings", async (t) => { t.is(result.bindings?.LOREM_IPSUM, loremIpsum); }); +test("BindingsPlugin: setup: loads data blob bindings", async (t) => { + let plugin = new BindingsPlugin(ctx, { + dataBlobBindings: { BINARY_DATA: someBinaryDataPath }, + }); + let result = await plugin.setup(); + t.is(result.bindings?.BINARY_DATA, someBinaryData); + + // Check resolves text blob bindings path relative to rootPath + plugin = new BindingsPlugin( + { log, compat, rootPath: path.dirname(someBinaryDataPath) }, + { dataBlobBindings: { BINARY_DATA: "some-binary-data.bin" } } + ); + result = await plugin.setup(); + t.is(result.bindings?.BINARY_DATA, someBinaryData); +}); + test("BindingsPlugin: setup: loads bindings from all sources", async (t) => { // Bindings should be loaded in this order, from lowest to highest priority: // 1) Wrangler [vars] // 2) .env Variables // 3) WASM Module Bindings // 4) Text Blob Bindings - // 5) Service Bindings - // 6) Custom Bindings + // 5) Data Blob Bindings + // 6) Service Bindings + // 7) Custom Bindings // wranglerOptions should contain [kWranglerBindings] const wranglerOptions = parsePluginWranglerConfig(BindingsPlugin, { - vars: { A: "w", B: "w", C: "w", D: "w", E: "w", F: "w" }, + vars: { A: "w", B: "w", C: "w", D: "w", E: "w", F: "w", G: "w" }, }); const tmp = await useTmp(t); const envPath = path.join(tmp, ".env"); - await fs.writeFile(envPath, "A=env\nB=env\nC=env\nD=env\nE=env"); + await fs.writeFile(envPath, "A=env\nB=env\nC=env\nD=env\nE=env\nF=env"); const obj = { ping: "pong" }; const throws = () => { @@ -321,11 +354,18 @@ test("BindingsPlugin: setup: loads bindings from all sources", async (t) => { B: addModulePath, C: addModulePath, D: addModulePath, + E: addModulePath, }, textBlobBindings: { A: loremIpsumPath, B: loremIpsumPath, C: loremIpsumPath, + D: loremIpsumPath, + }, + dataBlobBindings: { + A: someBinaryDataPath, + B: someBinaryDataPath, + C: someBinaryDataPath, }, serviceBindings: { A: throws, B: throws }, bindings: { A: obj }, @@ -334,10 +374,11 @@ test("BindingsPlugin: setup: loads bindings from all sources", async (t) => { const result = await plugin.setup(); assert(result.bindings); - t.is(result.bindings.F, "w"); - t.is(result.bindings.E, "env"); - t.true(result.bindings.D instanceof WebAssembly.Module); - t.is(result.bindings.C, loremIpsum); + t.is(result.bindings.G, "w"); + t.is(result.bindings.F, "env"); + t.true(result.bindings.E instanceof WebAssembly.Module); + t.is(result.bindings.D, loremIpsum); + t.is(result.bindings.C, someBinaryData); t.true(result.bindings.B instanceof Fetcher); t.is(result.bindings.A, obj); }); diff --git a/packages/shared/src/wrangler.ts b/packages/shared/src/wrangler.ts index 8f2a97800..2f59a6b03 100644 --- a/packages/shared/src/wrangler.ts +++ b/packages/shared/src/wrangler.ts @@ -34,8 +34,9 @@ export interface WranglerEnvironmentConfig { crons?: string[]; }; // inherited usage_model?: "bundled" | "unbound"; // inherited - wasm_modules?: Record; // (probably) inherited - text_blobs?: Record; + wasm_modules?: Record; // inherited + text_blobs?: Record; // inherited + data_blobs?: Record; // inherited experimental_services?: { name: string; service: string;