feat: implement [data_blobs]

This implements `[data_blobs]` support for service-worker workers, as well as enabling Data module support for service-worker workers. `data_blob` is a supported binding type, but we never implemented support for it in v1. This implements support, and utilises it for supporting Data modules in service worker format. Implementation wise, it's incredibly similar to how we implemented `text_blobs`, with relevant changes. Partial fix for #740, pending local mode support.
cloudflare · Mar 31, 2022 · 0ed22cd · 0ed22cd
1 parent c38ae3d
commit 0ed22cd
Show file tree

Hide file tree

Showing 11 changed files with 270 additions and 24 deletions.
diff --git a/packages/wrangler/src/__tests__/configuration.test.ts b/packages/wrangler/src/__tests__/configuration.test.ts
@@ -56,6 +56,7 @@ describe("normalizeAndValidateConfig()", () => {
       usage_model: undefined,
       vars: {},
       wasm_modules: undefined,
+      data_blobs: undefined,
       workers_dev: undefined,
       zone_id: undefined,
     });
@@ -434,6 +435,59 @@ describe("normalizeAndValidateConfig()", () => {
       `);
     });
 
+    it("should map `data_blobs` paths from relative to the config path to relative to the cwd", () => {
+      const expectedConfig: RawConfig = {
+        data_blobs: {
+          BLOB_1: "path/to/data1.bin",
+          BLOB_2: "data2.bin",
+        },
+      };
+
+      const { config, diagnostics } = normalizeAndValidateConfig(
+        expectedConfig,
+        "project/wrangler.toml",
+        { env: undefined }
+      );
+
+      expect(config).toEqual(
+        expect.objectContaining({
+          data_blobs: {
+            BLOB_1: path.normalize("project/path/to/data1.bin"),
+            BLOB_2: path.normalize("project/data2.bin"),
+          },
+        })
+      );
+      expect(diagnostics.hasErrors()).toBe(false);
+      expect(diagnostics.hasWarnings()).toBe(false);
+    });
+
+    it("should error on invalid `data_blob` paths", () => {
+      const expectedConfig = {
+        data_blobs: {
+          MODULE_1: 111,
+          MODULE_2: 222,
+        },
+      };
+
+      const { config, diagnostics } = normalizeAndValidateConfig(
+        expectedConfig as unknown as RawConfig,
+        "project/wrangler.toml",
+        { env: undefined }
+      );
+
+      expect(config).toEqual(
+        expect.objectContaining({
+          data_blobs: {},
+        })
+      );
+      expect(diagnostics.hasWarnings()).toBe(false);
+      expect(normalizePath(diagnostics.renderErrors())).toMatchInlineSnapshot(`
+        "Processing project/wrangler.toml configuration:
+          - Expected \\"data_blobs['MODULE_1']\\" to be of type string but got 111.
+          - Expected \\"data_blobs['MODULE_2']\\" to be of type string but got 222."
+      `);
+    });
+
     it("should resolve tsconfig relative to wrangler.toml", async () => {
       const expectedConfig: RawEnvironment = {
         tsconfig: "path/to/some-tsconfig.json",

diff --git a/packages/wrangler/src/__tests__/publish.test.ts b/packages/wrangler/src/__tests__/publish.test.ts
@@ -2114,6 +2114,10 @@ export default{
           WASM_MODULE_ONE: "./some_wasm.wasm",
           WASM_MODULE_TWO: "./more_wasm.wasm",
         },
+        data_blobs: {
+          DATA_BLOB_ONE: "./some-data-blob.bin",
+          DATA_BLOB_TWO: "./more-data-blob.bin",
+        },
       });
 
       writeWorkerSource({ type: "sw" });
@@ -2125,6 +2129,9 @@ export default{
       fs.writeFileSync("./some_wasm.wasm", "some wasm");
       fs.writeFileSync("./more_wasm.wasm", "more wasm");
 
+      fs.writeFileSync("./some-data-blob.bin", "some data");
+      fs.writeFileSync("./more-data-blob.bin", "more data");
+
       mockUploadWorkerRequest({
         expectedType: "sw",
         expectedBindings: [
@@ -2178,6 +2185,8 @@ export default{
           },
           { name: "TEXT_BLOB_ONE", part: "TEXT_BLOB_ONE", type: "text_blob" },
           { name: "TEXT_BLOB_TWO", part: "TEXT_BLOB_TWO", type: "text_blob" },
+          { name: "DATA_BLOB_ONE", part: "DATA_BLOB_ONE", type: "data_blob" },
+          { name: "DATA_BLOB_TWO", part: "DATA_BLOB_TWO", type: "data_blob" },
           {
             data: { some: { unsafe: "thing" } },
             name: "UNSAFE_BINDING_ONE",
@@ -2261,6 +2270,10 @@ export default{
           WASM_MODULE_ONE: "./some_wasm.wasm",
           CONFLICTING_NAME_THREE: "./more_wasm.wasm",
         },
+        data_blobs: {
+          DATA_BLOB_ONE: "./some_data.bin",
+          CONFLICTING_NAME_THREE: "./more_data.bin",
+        },
       });
 
       writeWorkerSource({ type: "sw" });
@@ -2277,7 +2290,7 @@ export default{
               [Error: Processing wrangler.toml configuration:
                 - CONFLICTING_NAME_ONE assigned to Durable Object, KV Namespace, and R2 Bucket bindings.
                 - CONFLICTING_NAME_TWO assigned to Durable Object and KV Namespace bindings.
-                - CONFLICTING_NAME_THREE assigned to R2 Bucket, Text Blob, Unsafe, Environment Variable, and WASM Module bindings.
+                - CONFLICTING_NAME_THREE assigned to R2 Bucket, Text Blob, Unsafe, Environment Variable, WASM Module, and Data Blob bindings.
                 - CONFLICTING_NAME_FOUR assigned to Text Blob and Unsafe bindings.
                 - Bindings must have unique names, so that they can all be referenced in the worker.
                   Please change your bindings to have unique names.]
@@ -2287,7 +2300,7 @@ export default{
         "Processing wrangler.toml configuration:
           - CONFLICTING_NAME_ONE assigned to Durable Object, KV Namespace, and R2 Bucket bindings.
           - CONFLICTING_NAME_TWO assigned to Durable Object and KV Namespace bindings.
-          - CONFLICTING_NAME_THREE assigned to R2 Bucket, Text Blob, Unsafe, Environment Variable, and WASM Module bindings.
+          - CONFLICTING_NAME_THREE assigned to R2 Bucket, Text Blob, Unsafe, Environment Variable, WASM Module, and Data Blob bindings.
           - CONFLICTING_NAME_FOUR assigned to Text Blob and Unsafe bindings.
           - Bindings must have unique names, so that they can all be referenced in the worker.
             Please change your bindings to have unique names.
@@ -2344,7 +2357,7 @@ export default{
             },
           ],
         },
-        // text_blobs, vars, and wasm_modules are fine because they're object literals,
+        // text_blobs, vars, wasm_modules and data_blobs are fine because they're object literals,
         // and by definition cannot have two keys of the same name
         //
         // text_blobs: {
@@ -2473,6 +2486,10 @@ export default{
           WASM_MODULE_ONE: "./some_wasm.wasm",
           CONFLICTING_NAME_THREE: "./more_wasm.wasm",
         },
+        data_blobs: {
+          DATA_BLOB_ONE: "./some_data.bin",
+          CONFLICTING_NAME_THREE: "./more_data.bin",
+        },
       });
 
       writeWorkerSource({ type: "sw" });
@@ -2490,7 +2507,7 @@ export default{
                 - CONFLICTING_DURABLE_OBJECT_NAME assigned to multiple Durable Object bindings.
                 - CONFLICTING_KV_NAMESPACE_NAME assigned to multiple KV Namespace bindings.
                 - CONFLICTING_R2_BUCKET_NAME assigned to multiple R2 Bucket bindings.
-                - CONFLICTING_NAME_THREE assigned to R2 Bucket, Text Blob, Unsafe, Environment Variable, and WASM Module bindings.
+                - CONFLICTING_NAME_THREE assigned to R2 Bucket, Text Blob, Unsafe, Environment Variable, WASM Module, and Data Blob bindings.
                 - CONFLICTING_NAME_FOUR assigned to R2 Bucket, Text Blob, and Unsafe bindings.
                 - CONFLICTING_UNSAFE_NAME assigned to multiple Unsafe bindings.
                 - Bindings must have unique names, so that they can all be referenced in the worker.
@@ -2502,7 +2519,7 @@ export default{
           - CONFLICTING_DURABLE_OBJECT_NAME assigned to multiple Durable Object bindings.
           - CONFLICTING_KV_NAMESPACE_NAME assigned to multiple KV Namespace bindings.
           - CONFLICTING_R2_BUCKET_NAME assigned to multiple R2 Bucket bindings.
-          - CONFLICTING_NAME_THREE assigned to R2 Bucket, Text Blob, Unsafe, Environment Variable, and WASM Module bindings.
+          - CONFLICTING_NAME_THREE assigned to R2 Bucket, Text Blob, Unsafe, Environment Variable, WASM Module, and Data Blob bindings.
           - CONFLICTING_NAME_FOUR assigned to R2 Bucket, Text Blob, and Unsafe bindings.
           - CONFLICTING_UNSAFE_NAME assigned to multiple Unsafe bindings.
           - Bindings must have unique names, so that they can all be referenced in the worker.
@@ -2685,14 +2702,14 @@ export default{
         await expect(
           runWrangler("publish index.js")
         ).rejects.toThrowErrorMatchingInlineSnapshot(
-          `"You cannot configure [text_blobs] with an ES module worker. Instead, import the file directly in your code, and optionally configure \`[build.upload.rules]\` in your wrangler.toml"`
+          `"You cannot configure [text_blobs] with an ES module worker. Instead, import the file directly in your code, and optionally configure \`[rules]\` in your wrangler.toml"`
         );
         expect(std.out).toMatchInlineSnapshot(`""`);
         expect(std.err).toMatchInlineSnapshot(`
-                  "You cannot configure [text_blobs] with an ES module worker. Instead, import the file directly in your code, and optionally configure \`[build.upload.rules]\` in your wrangler.toml
+          "You cannot configure [text_blobs] with an ES module worker. Instead, import the file directly in your code, and optionally configure \`[rules]\` in your wrangler.toml
 
-                  [32m%s[0m If you think this is a bug then please create an issue at https://github.com/cloudflare/wrangler2/issues/new."
-              `);
+          [32m%s[0m If you think this is a bug then please create an issue at https://github.com/cloudflare/wrangler2/issues/new."
+        `);
         expect(std.warn).toMatchInlineSnapshot(`""`);
       });
 
@@ -2740,6 +2757,106 @@ export default{
       });
     });
 
+    describe("[data_blobs]", () => {
+      it("should be able to define data blobs for service-worker format workers", async () => {
+        writeWranglerToml({
+          data_blobs: {
+            TESTDATABLOBNAME: "./path/to/data.bin",
+          },
+        });
+        writeWorkerSource({ type: "sw" });
+        fs.mkdirSync("./path/to", { recursive: true });
+        fs.writeFileSync("./path/to/data.bin", "SOME DATA CONTENT");
+        mockUploadWorkerRequest({
+          expectedType: "sw",
+          expectedModules: { TESTDATABLOBNAME: "SOME DATA CONTENT" },
+          expectedBindings: [
+            {
+              name: "TESTDATABLOBNAME",
+              part: "TESTDATABLOBNAME",
+              type: "data_blob",
+            },
+          ],
+        });
+        mockSubDomainRequest();
+        await runWrangler("publish index.js");
+        expect(std.out).toMatchInlineSnapshot(`
+                  "Uploaded test-name (TIMINGS)
+                  Published test-name (TIMINGS)
+                    test-name.test-sub-domain.workers.dev"
+              `);
+        expect(std.err).toMatchInlineSnapshot(`""`);
+        expect(std.warn).toMatchInlineSnapshot(`""`);
+      });
+
+      it("should error when defining data blobs for modules format workers", async () => {
+        writeWranglerToml({
+          data_blobs: {
+            TESTDATABLOBNAME: "./path/to/data.bin",
+          },
+        });
+        writeWorkerSource({ type: "esm" });
+        fs.mkdirSync("./path/to", { recursive: true });
+        fs.writeFileSync("./path/to/data.bin", "SOME DATA CONTENT");
+
+        await expect(
+          runWrangler("publish index.js")
+        ).rejects.toThrowErrorMatchingInlineSnapshot(
+          `"You cannot configure [data_blobs] with an ES module worker. Instead, import the file directly in your code, and optionally configure \`[rules]\` in your wrangler.toml"`
+        );
+        expect(std.out).toMatchInlineSnapshot(`""`);
+        expect(std.err).toMatchInlineSnapshot(`
+                  "You cannot configure [data_blobs] with an ES module worker. Instead, import the file directly in your code, and optionally configure \`[rules]\` in your wrangler.toml
+
+                  [32m%s[0m If you think this is a bug then please create an issue at https://github.com/cloudflare/wrangler2/issues/new."
+              `);
+        expect(std.warn).toMatchInlineSnapshot(`""`);
+      });
+
+      it("should resolve data blobs relative to the wrangler.toml file", async () => {
+        fs.mkdirSync("./path/to/and/the/path/to/", { recursive: true });
+        fs.writeFileSync(
+          "./path/to/wrangler.toml",
+          TOML.stringify({
+            compatibility_date: "2022-01-12",
+            name: "test-name",
+            data_blobs: {
+              TESTDATABLOBNAME: "./and/the/path/to/data.bin",
+            },
+          }),
+
+          "utf-8"
+        );
+
+        writeWorkerSource({ type: "sw" });
+        fs.writeFileSync(
+          "./path/to/and/the/path/to/data.bin",
+          "SOME DATA CONTENT"
+        );
+        mockUploadWorkerRequest({
+          expectedType: "sw",
+          expectedModules: { TESTDATABLOBNAME: "SOME DATA CONTENT" },
+          expectedBindings: [
+            {
+              name: "TESTDATABLOBNAME",
+              part: "TESTDATABLOBNAME",
+              type: "data_blob",
+            },
+          ],
+          expectedCompatibilityDate: "2022-01-12",
+        });
+        mockSubDomainRequest();
+        await runWrangler("publish index.js --config ./path/to/wrangler.toml");
+        expect(std.out).toMatchInlineSnapshot(`
+                  "Uploaded test-name (TIMINGS)
+                  Published test-name (TIMINGS)
+                    test-name.test-sub-domain.workers.dev"
+              `);
+        expect(std.err).toMatchInlineSnapshot(`""`);
+        expect(std.warn).toMatchInlineSnapshot(`""`);
+      });
+    });
+
     describe("[vars]", () => {
       it("should support json bindings", async () => {
         writeWranglerToml({

diff --git a/packages/wrangler/src/config/config.ts b/packages/wrangler/src/config/config.ts
@@ -135,6 +135,17 @@ export interface ConfigFields<Dev extends RawDevConfig> {
         [key: string]: string;
       }
     | undefined;
+
+  /**
+   * A list of data files that your worker should be bound to. This is
+   * the "legacy" way of binding to a data file. ES module workers should
+   * do proper module imports.
+   */
+  data_blobs:
+    | {
+        [key: string]: string;
+      }
+    | undefined;
 }
 
 export interface DevConfig {

diff --git a/packages/wrangler/src/config/validation.ts b/packages/wrangler/src/config/validation.ts
@@ -182,6 +182,12 @@ export function normalizeAndValidateConfig(
       "text_blobs",
       rawConfig.text_blobs
     ),
+    data_blobs: normalizeAndValidateModulePaths(
+      diagnostics,
+      configPath,
+      "data_blobs",
+      rawConfig.data_blobs
+    ),
   };
 
   validateBindingsHaveUniqueNames(diagnostics, config);
@@ -427,12 +433,12 @@ function normalizeAndValidateSite(
 }
 
 /**
- * Map the paths of the `wasm_modules` or `text_blobs` configuration to be relative to the current working directory.
+ * Map the paths of the `wasm_modules`, `text_blobs` or `data_blobs` configuration to be relative to the current working directory.
  */
 function normalizeAndValidateModulePaths(
   diagnostics: Diagnostics,
   configPath: string | undefined,
-  field: "wasm_modules" | "text_blobs",
+  field: "wasm_modules" | "text_blobs" | "data_blobs",
   rawMapping: Record<string, string> | undefined
 ): Record<string, string> | undefined {
   if (rawMapping === undefined) {
@@ -1153,6 +1159,7 @@ const validateBindingsHaveUniqueNames = (
     unsafe,
     vars,
     wasm_modules,
+    data_blobs,
   }: Partial<Config>
 ): boolean => {
   let hasDuplicates = false;
@@ -1165,6 +1172,7 @@ const validateBindingsHaveUniqueNames = (
     Unsafe: getBindingNames(unsafe),
     "Environment Variable": getBindingNames(vars),
     "WASM Module": getBindingNames(wasm_modules),
+    "Data Blob": getBindingNames(data_blobs),
   } as Record<string, string[]>;
 
   const bindingsGroupedByName: Record<string, string[]> = {};