Add Definition::Enum::tag_width field

The field allows specifying how many bytes the discriminant byte takes.
This in turn allows for better support of custom encoding formats with
more than 256 variants and untagged unions.

Issue: near#181

borsh-derive/src/internals/schema/enums/mod.rs

@@ -53,8 +53,10 @@ pub fn process(input: &ItemEnum, cratename: Path) -> syn::Result<TokenStream2> {
         fn add_definitions_recursively(definitions: &mut #cratename::__private::maybestd::collections::BTreeMap<#cratename::schema::Declaration, #cratename::schema::Definition>) {
             #inner_defs
             #add_recursive_defs
-            let variants = #cratename::__private::maybestd::vec![#(#variants_defs),*];
-            let definition = #cratename::schema::Definition::Enum{variants};
+            let definition = #cratename::schema::Definition::Enum {
+                tag_width: 1,
+                variants: #cratename::__private::maybestd::vec![#(#variants_defs),*],
+            };
             #cratename::schema::add_definition(Self::declaration(), definition, definitions);
         }
     };

borsh-derive/src/internals/schema/enums/snapshots/complex_enum.snap

@@ -35,13 +35,13 @@ impl borsh::BorshSchema for A {
         <AEggs as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASalad as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASausage as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
-            AEggs > ::declaration()), ("Salad".to_string(), < ASalad > ::declaration()),
-            ("Sausage".to_string(), < ASausage > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
+                AEggs > ::declaration()), ("Salad".to_string(), < ASalad >
+                ::declaration()), ("Sausage".to_string(), < ASausage > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

borsh-derive/src/internals/schema/enums/snapshots/complex_enum_generics.snap

@@ -42,13 +42,14 @@ where
         <AEggs as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASalad<C> as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASausage<W> as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
-            AEggs > ::declaration()), ("Salad".to_string(), < ASalad < C > >
-            ::declaration()), ("Sausage".to_string(), < ASausage < W > > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
+                AEggs > ::declaration()), ("Salad".to_string(), < ASalad < C > >
+                ::declaration()), ("Sausage".to_string(), < ASausage < W > >
+                ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

borsh-derive/src/internals/schema/enums/snapshots/complex_enum_generics_borsh_skip_named_field.snap

@@ -44,14 +44,14 @@ where
         <AEggs as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASalad<C> as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASausage<W, U> as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
-            AEggs > ::declaration()), ("Salad".to_string(), < ASalad < C > >
-            ::declaration()), ("Sausage".to_string(), < ASausage < W, U > >
-            ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
+                AEggs > ::declaration()), ("Salad".to_string(), < ASalad < C > >
+                ::declaration()), ("Sausage".to_string(), < ASausage < W, U > >
+                ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

borsh-derive/src/internals/schema/enums/snapshots/complex_enum_generics_borsh_skip_tuple_field.snap

@@ -43,13 +43,14 @@ where
         <AEggs as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASalad<C> as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ASausage<W> as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
-            AEggs > ::declaration()), ("Salad".to_string(), < ASalad < C > >
-            ::declaration()), ("Sausage".to_string(), < ASausage < W > > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
+                AEggs > ::declaration()), ("Salad".to_string(), < ASalad < C > >
+                ::declaration()), ("Sausage".to_string(), < ASausage < W > >
+                ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

borsh-derive/src/internals/schema/enums/snapshots/filter_foreign_attrs.snap

@@ -29,12 +29,12 @@ impl borsh::BorshSchema for A {
         }
         <AB as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <ANegative as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("B".to_string(), < AB > ::declaration()), ("Negative".to_string(), <
-            ANegative > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("B".to_string(), < AB > ::declaration()), ("Negative".to_string(), <
+                ANegative > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

borsh-derive/src/internals/schema/enums/snapshots/generic_associated_type.snap

@@ -55,12 +55,12 @@ where
         <EnumParametrizedC<
             T,
         > as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("B".to_string(), < EnumParametrizedB < K, V > > ::declaration()), ("C"
-            .to_string(), < EnumParametrizedC < T > > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("B".to_string(), < EnumParametrizedB < K, V > > ::declaration()), ("C"
+                .to_string(), < EnumParametrizedC < T > > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

borsh-derive/src/internals/schema/enums/snapshots/generic_associated_type_param_override.snap

@@ -56,12 +56,12 @@ where
         <EnumParametrizedC<
             T,
         > as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("B".to_string(), < EnumParametrizedB < K, V > > ::declaration()), ("C"
-            .to_string(), < EnumParametrizedC < T > > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("B".to_string(), < EnumParametrizedB < K, V > > ::declaration()), ("C"
+                .to_string(), < EnumParametrizedC < T > > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

borsh-derive/src/internals/schema/enums/snapshots/recursive_enum.snap

@@ -36,12 +36,12 @@ where
         struct AC<K: Key>(K, Vec<A>);
         <AB<K, V> as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <AC<K> as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("B".to_string(), < AB < K, V > > ::declaration()), ("C".to_string(), < AC <
-            K > > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("B".to_string(), < AB < K, V > > ::declaration()), ("C".to_string(), <
+                AC < K > > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

borsh-derive/src/internals/schema/enums/snapshots/simple_enum.snap

@@ -22,12 +22,12 @@ impl borsh::BorshSchema for A {
         struct AEggs;
         <ABacon as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <AEggs as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
-            AEggs > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
+                AEggs > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

borsh-derive/src/internals/schema/enums/snapshots/simple_enum_with_custom_crate.snap

@@ -26,12 +26,12 @@ impl reexporter::borsh::BorshSchema for A {
         <AEggs as reexporter::borsh::BorshSchema>::add_definitions_recursively(
             definitions,
         );
-        let variants = reexporter::borsh::__private::maybestd::vec![
-            ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
-            AEggs > ::declaration())
-        ];
         let definition = reexporter::borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: reexporter::borsh::__private::maybestd::vec![
+                ("Bacon".to_string(), < ABacon > ::declaration()), ("Eggs".to_string(), <
+                AEggs > ::declaration())
+            ],
         };
         reexporter::borsh::schema::add_definition(
             Self::declaration(),

borsh-derive/src/internals/schema/enums/snapshots/single_field_enum.snap

@@ -17,11 +17,11 @@ impl borsh::BorshSchema for A {
         #[borsh(crate = "borsh")]
         struct ABacon;
         <ABacon as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("Bacon".to_string(), < ABacon > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("Bacon".to_string(), < ABacon > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

borsh-derive/src/internals/schema/enums/snapshots/trailing_comma_generics.snap

@@ -39,12 +39,12 @@ where
             B: Display + Debug;
         <SideLeft<A> as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <SideRight<B> as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("Left".to_string(), < SideLeft < A > > ::declaration()), ("Right"
-            .to_string(), < SideRight < B > > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("Left".to_string(), < SideLeft < A > > ::declaration()), ("Right"
+                .to_string(), < SideRight < B > > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

borsh-derive/src/internals/schema/enums/snapshots/with_funcs_attr.snap

@@ -40,12 +40,12 @@ where
         );
         <CC3 as borsh::BorshSchema>::add_definitions_recursively(definitions);
         <CC4<K, V> as borsh::BorshSchema>::add_definitions_recursively(definitions);
-        let variants = borsh::__private::maybestd::vec![
-            ("C3".to_string(), < CC3 > ::declaration()), ("C4".to_string(), < CC4 < K, V
-            > > ::declaration())
-        ];
         let definition = borsh::schema::Definition::Enum {
-            variants,
+            tag_width: 1,
+            variants: borsh::__private::maybestd::vec![
+                ("C3".to_string(), < CC3 > ::declaration()), ("C4".to_string(), < CC4 <
+                K, V > > ::declaration())
+            ],
         };
         borsh::schema::add_definition(Self::declaration(), definition, definitions);
     }

borsh/src/lib.rs

@@ -62,7 +62,6 @@
 
 */
 
-#[cfg(not(feature = "std"))]
 extern crate alloc;
 
 /// Derive macro available if borsh is built with `features = ["derive", "schema"]`.

borsh/src/schema.rs

@@ -35,6 +35,16 @@ pub type VariantName = String;
 pub type FieldName = String;
 /// The type that we use to represent the definition of the Borsh type.
 
+/// Description of data encoding on the wire.
+///
+/// Note: Since at the end of the day users can define arbitrary serialisation,
+/// it’s not always possible to express using definitions how a type is encoded.
+/// For example, let’s say programmer uses [varint] encoding for their data.
+/// Such type cannot be fully expressed using `BorshSchema` (or at least not
+/// easily).  As a consequence, a tool which validates whether binary data
+/// matches a schema wouldn’t be able to validate data including such types.
+///
+/// [varint]: https://en.wikipedia.org/wiki/Variable-length_quantity#Variants
 #[derive(Clone, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize, BorshSchemaMacro)]
 pub enum Definition {
     /// A fixed-size array with the length known at the compile time and the same-type elements.
@@ -44,10 +54,31 @@ pub enum Definition {
     /// A fixed-size tuple with the length known at the compile time and the elements of different
     /// types.
     Tuple { elements: Vec<Declaration> },
-    /// A tagged union, a.k.a enum. Tagged-unions have variants with associated structures.
+
+    /// A possibly tagged union, a.k.a enum.
+    ///
+    /// Tagged unions are prefixed by a tag identifying encoded variant followed
+    /// by encoding of that variant.
+    ///
+    /// Untagged unions don’t have a separate tag which means that knowledge of
+    /// the type is necessary to fully analyse the binary.  Variants may still
+    /// be used to list possible values or determine the longest possible
+    /// encoding.
     Enum {
+        /// Width in bytes of the discriminant tag.
+        ///
+        /// Zero indicates this is an untagged union.  In standard borsh
+        /// encoding this is one however custom encoding formats may use larger
+        /// width if they need to encode more than 256 variants.
+        ///
+        /// Note: This definition must not be used if the tag is not encoded
+        /// using little-endian format.
+        tag_width: u8,
+
+        /// Possible variants of the enumeration.
         variants: Vec<(VariantName, Declaration)>,
     },
+
     /// A structure, structurally similar to a tuple.
     Struct { fields: Fields },
 }
@@ -281,6 +312,7 @@ where
 {
     fn add_definitions_recursively(definitions: &mut BTreeMap<Declaration, Definition>) {
         let definition = Definition::Enum {
+            tag_width: 1,
             variants: vec![
                 ("None".to_string(), <()>::declaration()),
                 ("Some".to_string(), T::declaration()),
@@ -302,6 +334,7 @@ where
 {
     fn add_definitions_recursively(definitions: &mut BTreeMap<Declaration, Definition>) {
         let definition = Definition::Enum {
+            tag_width: 1,
             variants: vec![
                 ("Ok".to_string(), T::declaration()),
                 ("Err".to_string(), E::declaration()),
@@ -521,11 +554,14 @@ mod tests {
         Option::<u64>::add_definitions_recursively(&mut actual_defs);
         assert_eq!("Option<u64>", actual_name);
         assert_eq!(
-            map! {"Option<u64>" =>
-            Definition::Enum{ variants: vec![
-                ("None".to_string(), "nil".to_string()),
-                ("Some".to_string(), "u64".to_string()),
-            ]}
+            map! {
+                "Option<u64>" => Definition::Enum {
+                    tag_width: 1,
+                    variants: vec![
+                        ("None".to_string(), "nil".to_string()),
+                        ("Some".to_string(), "u64".to_string()),
+                    ]
+                }
             },
             actual_defs
         );
@@ -539,16 +575,20 @@ mod tests {
         assert_eq!("Option<Option<u64>>", actual_name);
         assert_eq!(
             map! {
-            "Option<u64>" =>
-                Definition::Enum {variants: vec![
-                ("None".to_string(), "nil".to_string()),
-                ("Some".to_string(), "u64".to_string()),
-                ]},
-            "Option<Option<u64>>" =>
-                Definition::Enum {variants: vec![
-                ("None".to_string(), "nil".to_string()),
-                ("Some".to_string(), "Option<u64>".to_string()),
-                ]}
+                "Option<u64>" => Definition::Enum {
+                    tag_width: 1,
+                    variants: vec![
+                        ("None".to_string(), "nil".to_string()),
+                        ("Some".to_string(), "u64".to_string()),
+                    ]
+                },
+                "Option<Option<u64>>" => Definition::Enum {
+                    tag_width: 1,
+                    variants: vec![
+                        ("None".to_string(), "nil".to_string()),
+                        ("Some".to_string(), "Option<u64>".to_string()),
+                    ]
+                }
             },
             actual_defs
         );