feat: configurable custom wasm sections (#2679)

See https://dfinity.atlassian.net/browse/SDK-660, https://dfinity.atlassian.net/browse/SDK-347

CHANGELOG.md

@@ -85,6 +85,29 @@ If you want to disable this behavior, you can config it in dfx.json:
         }
     }
 
+### feat: configurable custom wasm sections
+
+It's now possible to define custom wasm metadata sections and their visibility in dfx.json.
+
+At present, dfx can only add wasm metadata sections to canisters that are in wasm format.  It cannot add metadata sections to compressed canisters.  Since the frontend canister is now compressed, this means that at present it is not possible to add custom metadata sections to the frontend canister.
+
+dfx no longer adds `candid:service` metadata to canisters of type `"custom"` by default.  If you want dfx to add your canister's candid definition to your custom canister, you can do so like this:
+
+```
+    "my_canister_name": {
+      "type": "custom",
+      "candid": "main.did",
+      "wasm": "main.wasm",
+      "metadata": [
+        {
+          "name": "candid:service"
+        }
+      ]
+    },
+```
+
+This changelog entry doesn't go into all of the details of the possible configuration.  For that, please see [concepts/canister-metadata](docs/concepts/canister-metadata.md) and the docs in the JSON schema.
+
 ### fix: Valid canister-based env vars
 
 Hyphens are not valid in shell environment variables, but do occur in canister names such as `smiley-dapp`. This poses a problem for vars with names such as `CANISTER_ID_${CANISTER_NAME}`.  With this change, hyphens are replaced with underscores in environment variables.  The canister id of `smiley-dapp` will be available as `CANISTER_ID_smiley_dapp`.  Other environment variables are unaffected.

docs/concepts/canister-metadata.md

@@ -0,0 +1,121 @@
+# Canister Metadata
+
+## Overview
+
+Canisters can store custom metadata, which is available from the state tree at `/canister/<canister id>/metadata/<name>`.
+
+You can configure this metadata in dfx.json, per canister, in the `metadata` array.
+
+Here is a simple example:
+
+```json
+{
+  "canisters": {
+    "app_backend": {
+      "main": "src/app_backend/main.mo",
+      "type": "motoko"
+    },
+    "app_frontend": {
+      "dependencies": [
+        "app_backend"
+      ],
+      "frontend": {
+        "entrypoint": "src/app_frontend/src/index.html"
+      },
+      "source": [
+        "src/app_frontend/assets",
+        "dist/app_frontend/"
+      ],
+      "type": "assets",
+      "metadata": [
+        {
+          "name": "alternative-domains",
+          "visibility": "public",
+          "path": "src/app_frontend/metadata/alternative-domains.cbor"
+        }
+      ]
+    }
+  },
+  "version": 1
+}
+```
+## Fields
+
+The JSON schema also documents these fields.
+
+### name
+
+A string containing the name of the wasm section.
+
+### visibility
+
+A string containing either `private` or `public` (the default).
+
+Anyone can read the public metadata of a canister.
+
+Only a controller of the canister can read its private metadata.
+
+It is not possible to define metadata with the same name with both `private` and `public` visibility, unless they are for different networks.
+
+### networks
+
+An array of strings containing the names of the networks that this metadata applies to.
+
+If this field is absent, it applies to all networks.
+
+If this field is present as an empty array, it does not apply to any networks.
+
+If dfx.json contains more than one metadata entry with a given name, dfx will use the first entry that matches the current network and ignore any that follow.
+
+### path
+
+A string containing the path of a file containing the wasm section contents.
+
+## The candid:service metadata
+
+Dfx automatically adds `candid:service` metadata, with public visibility, for Rust and Motoko canisters.
+
+You can, however, override this behavior by defining a metadata entry with `"name": "candid:service"`.  You can change the visibility or the contents.
+
+For Motoko canisters, if you specify a `path` for candid:service metadata (replacing the candid:service definition generated by `moc`), dfx will verify that the candid:service definition you provide is a valid subtype of the definition that `moc` generated.
+
+## A more complex example
+
+In this example, we change the visibility of the `candid:service` metadata on the ic and staging networks to private, but leave it public for the local network.
+
+```json
+{
+  "canisters": {
+    "app_backend": {
+      "main": "src/app_backend/main.mo",
+      "type": "motoko",
+      "metadata": [
+        {
+          "name": "candid:service",
+          "networks": [ "ic", "staging" ],
+          "visibility": "private"
+        },
+        {
+          "name": "candid:service",
+          "networks": [ "local" ],
+          "visibility": "public"
+        }
+      ]
+    },
+    "app_frontend": {
+      "dependencies": [
+        "app_backend"
+      ],
+      "frontend": {
+        "entrypoint": "src/app_frontend/src/index.html"
+      },
+      "source": [
+        "src/app_frontend/assets",
+        "dist/app_frontend/"
+      ],
+      "type": "assets"
+    }
+  },
+  "version": 1
+}
+```

docs/concepts/index.md

@@ -0,0 +1,3 @@
+# DFX Concepts
+
+- [Canister metadata](./canister-metadata.md)

docs/dfx-json-schema.json

@@ -115,6 +115,50 @@
         }
       }
     },
+    "CanisterMetadataSection": {
+      "title": "Canister Metadata Configuration",
+      "description": "Configures a custom metadata section for the canister wasm. dfx uses the first definition of a given name matching the current network, ignoring any of the same name that follow.",
+      "type": "object",
+      "required": [
+        "name"
+      ],
+      "properties": {
+        "name": {
+          "title": "Name",
+          "description": "The name of the wasm section",
+          "type": "string"
+        },
+        "networks": {
+          "title": "Networks",
+          "description": "Networks this section applies to. If this field is absent, then it applies to all networks. An empty array means this element will not apply to any network.",
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "type": "string"
+          },
+          "uniqueItems": true
+        },
+        "path": {
+          "title": "Path",
+          "description": "Path to file containing section contents. For sections with name=`candid:service`, this field is optional, and if not specified, dfx will use the canister's candid definition. If specified for a Motoko canister, the service defined in the specified path must be a valid subtype of the canister's actual candid service definition.",
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "visibility": {
+          "title": "Visibility",
+          "default": "public",
+          "allOf": [
+            {
+              "$ref": "#/definitions/MetadataVisibility"
+            }
+          ]
+        }
+      }
+    },
     "ConfigCanistersCanister": {
       "title": "Canister Configuration",
       "description": "Configurations for a single canister.",
@@ -290,6 +334,15 @@
             "null"
           ]
         },
+        "metadata": {
+          "title": "Metadata",
+          "description": "Defines metadata sections to set in the canister .wasm",
+          "default": [],
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/CanisterMetadataSection"
+          }
+        },
         "post_install": {
           "title": "Post-Install Commands",
           "description": "One or more commands to run post canister installation.",
@@ -689,6 +742,13 @@
         }
       }
     },
+    "MetadataVisibility": {
+      "type": "string",
+      "enum": [
+        "public",
+        "private"
+      ]
+    },
     "NetworkType": {
       "title": "Network Type",
       "description": "Type 'ephemeral' is used for networks that are regularly reset. Type 'persistent' is used for networks that last for a long time and where it is preferred that canister IDs get stored in source control.",

e2e/assets/metadata/custom/custom_with_default_metadata.did

@@ -0,0 +1,5 @@
+service : {
+  // custom_with_default_metadata
+  getCanisterId: () -> (principal) query;
+  amInitializer: () -> (bool) query;
+}

e2e/assets/metadata/custom/custom_with_private_candid_service_metadata.did

@@ -0,0 +1,5 @@
+service : {
+  // custom_with_private_candid_service_metadata
+  getCanisterId: () -> (principal) query;
+  amInitializer: () -> (bool) query;
+}

e2e/assets/metadata/custom/custom_with_standard_candid_service_metadata.did

@@ -0,0 +1,5 @@
+service : {
+  // custom_with_standard_candid_service_metadata
+  getCanisterId: () -> (principal) query;
+  amInitializer: () -> (bool) query;
+}

e2e/assets/metadata/custom/dfx.json

@@ -0,0 +1,32 @@
+{
+  "version": 1,
+  "canisters": {
+    "custom_with_default_metadata": {
+      "type": "custom",
+      "candid": "custom_with_default_metadata.did",
+      "wasm": "main.wasm",
+      "build": "echo anything"
+    },
+    "custom_with_standard_candid_service_metadata": {
+      "type": "custom",
+      "candid": "custom_with_standard_candid_service_metadata.did",
+      "wasm": "main.wasm",
+      "metadata": [
+        {
+          "name": "candid:service"
+        }
+      ]
+    },
+    "custom_with_private_candid_service_metadata": {
+      "type": "custom",
+      "candid": "custom_with_private_candid_service_metadata.did",
+      "wasm": "main.wasm",
+      "metadata": [
+        {
+          "name": "candid:service",
+          "visibility": "private"
+        }
+      ]
+    }
+  }
+}

e2e/assets/metadata/motoko/main.mo

@@ -0,0 +1,17 @@
+actor {
+  public query func greet(name : Text) : async Text {
+    return "Hello, " # name # "!";
+  };
+
+  stable var a : Nat = 0;
+  public func inc_a() : async Nat {
+    a += 1;
+    return a;
+  };
+
+  stable var b : Int = 0;
+  public func inc_b() : async Int {
+    b += 1;
+    return b;
+  };
+};

e2e/assets/metadata/motoko/not_subtype_numbertype.did

@@ -0,0 +1,4 @@
+service : {
+  greet: (text) -> (text) query;
+  inc_b: () -> (nat);
+}

e2e/assets/metadata/motoko/not_subtype_rename.did

@@ -0,0 +1,3 @@
+service : {
+  new_method: (text) -> (text) query;
+}

e2e/assets/metadata/motoko/patch.bash

@@ -0,0 +1 @@
+jq '.canisters.e2e_project_backend.main="main.mo"' dfx.json | sponge dfx.json

e2e/assets/metadata/motoko/valid_subtype.did

@@ -0,0 +1,4 @@
+service : {
+  greet: (text) -> (text) query;
+  inc_a: () -> (int);
+}

e2e/assets/prebuilt_custom_canister/dfx.json

@@ -23,6 +23,12 @@
       "candid": "prebuilt_custom_blank_build.did",
       "wasm": "main.wasm",
       "build": []
+    },
+    "prebuilt_no_metadata_defined": {
+      "type": "custom",
+      "candid": "custom_with_build_step.did",
+      "wasm": "main.wasm",
+      "build": "echo just a build step"
     }
   },
   "networks": {

e2e/tests-dfx/build.bash

@@ -257,27 +257,3 @@ teardown() {
   assert_command ls .dfx/actuallylocal/canisters/e2e_project_backend/
   assert_command ls .dfx/actuallylocal/canisters/e2e_project_backend/e2e_project_backend.wasm
 }
-
-@test "does not add candid:service metadata for a custom canister if there are no build steps" {
-  install_asset prebuilt_custom_canister
-  install_asset wasm/identity
-
-  dfx_start
-  dfx deploy
-
-  # this canister has a build step, so dfx sets the candid metadata
-  dfx canister metadata custom_with_build_step candid:service >from_canister.txt
-  diff custom_with_build_step.did from_canister.txt
-
-  # this canister doesn't have a build step, so dfx leaves the candid metadata as-is
-  dfx canister metadata prebuilt_custom_no_build candid:service >from_canister.txt
-  diff main.did from_canister.txt
-
-  # this canister has a build step, but it is an empty string, so dfx leaves the candid:service metadata as-is
-  dfx canister metadata prebuilt_custom_blank_build candid:service >from_canister.txt
-  diff main.did from_canister.txt
-
-  # this canister has a build step, but it is an empty array, so dfx leaves the candid:service metadata as-is
-  dfx canister metadata prebuilt_custom_empty_build candid:service >from_canister.txt
-  diff main.did from_canister.txt
-}