From e3112fc26bc6ce9b679c990d2dabd1f809b733a1 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 4 Oct 2024 21:14:48 +0900 Subject: [PATCH 01/41] Copy golang test code to Dart --- serde-generate/tests/dart_runtime.rs | 153 +++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index a1f309ee8..a7d41c5db 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -3,6 +3,7 @@ use crate::test_utils; use crate::test_utils::{Choice, Runtime, Test}; +use heck::CamelCase; use serde_generate::{dart, CodeGeneratorConfig, SourceInstaller}; use std::{ fs::{create_dir_all, File}, @@ -126,3 +127,155 @@ void main() {{"# assert!(dart_test.success()); } + +#[test] +fn test_dart_bcs_runtime_on_supported_types() { + test_dart_runtime_on_supported_types(Runtime::Bcs); +} + +#[test] +fn test_dart_bincode_runtime_on_supported_types() { + test_dart_runtime_on_supported_types(Runtime::Bincode); +} + +fn quote_bytes(bytes: &[u8]) -> String { + format!( + "{{{}}}", + bytes + .iter() + .map(|x| format!("{}", x)) + .collect::>() + .join(", ") + ) +} + +fn test_dart_runtime_on_supported_types(runtime: Runtime) { + let dir = tempdir().unwrap(); + let source_path = dir + .path() + .join(format!("dart_project_{}", runtime.name().to_lowercase())); + let mut source = File::create(source_path.join("test/runtime_test.dart")).unwrap(); + + let positive_encodings = runtime + .get_positive_samples_quick() + .iter() + .map(|bytes| quote_bytes(bytes)) + .collect::>() + .join(", "); + + let negative_encodings = runtime + .get_negative_samples() + .iter() + .map(|bytes| quote_bytes(bytes)) + .collect::>() + .join(", "); + + writeln!( + source, + r#" +void main() {{ + var positiveInputs = [ + {0} + ]; + var negativeInputs = [ + {1} + ]; + + for (var input in positiveInputs) {{ + // Deserialize the input. + var value = {2}DeserializeSerdeData(input); + if (value == null) {{ + throw Exception('Failed to deserialize input: $input'); + }} + + // Serialize the deserialized value. + var output = value.{2}Serialize(); + if (output == null || !listEquals(input, output)) {{ + throw Exception('input != output:\n $input\n $output'); + }} + + // Test self-equality for the Serde value. + {{ + var value2 = {2}DeserializeSerdeData(input); + if (value2 == null) {{ + throw Exception('Failed to deserialize input: $input'); + }} + if (value != value2) {{ + throw Exception('Value should test equal to itself.'); + }} + }} + + // Test simple mutations of the input. + for (var i = 0; i < input.length; i++) {{ + var input2 = List.from(input); // Create a copy of the input + input2[i] ^= 0x80; // Mutate a byte + var value2 = {2}DeserializeSerdeData(input2); + if (value2 != null && value == value2) {{ + throw Exception('Modified input should give a different value.'); + }} + }} + }} + + // Test negative inputs for deserialization failure. + for (var input in negativeInputs) {{ + var result = {2}DeserializeSerdeData(input); + if (result != null) {{ + throw Exception('Input should fail to deserialize: $input'); + }} + }} +}} + +// Helper function for comparing byte arrays. +bool listEquals(List a, List b) {{ + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) {{ + if (a[i] != b[i]) return false; + }} + return true; +}} + +// Placeholder class +class SerdeValue {{ + List {2}Serialize() {{ + // Implement serialization logic here. + return []; + }} +}} + +SerdeValue? {2}DeserializeSerdeData(List input) {{ + // Implement deserialization logic here. + return SerdeValue(); +}} +"#, + positive_encodings, + negative_encodings, + runtime.name().to_camel_case(), + ) + .unwrap(); + + let runtime_mod_path = std::env::current_exe() + .unwrap() + .parent() + .unwrap() + .join("../../../serde-generate/runtime/dart"); + let status = Command::new("dart") + .current_dir("runtime/dart") + .arg("pub") + .arg("add") + .arg("-n") // Use `-n` to avoid versioning, or use appropriate Dart flags + .arg(format!( + "serde-reflection: {}/runtime/dart", + runtime_mod_path.to_str().unwrap() + )) + .status() + .unwrap(); + assert!(status.success()); + + let status = Command::new("dart") + .current_dir("runtime/dart") + .arg("run") + .arg(source_path) + .status() + .unwrap(); + assert!(status.success()); +} From 9ac4d82da929fd43fd470a6cfcd9a02f2a3ab6ea Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 4 Oct 2024 21:19:35 +0900 Subject: [PATCH 02/41] Make Dart commands work on Windows --- serde-generate/tests/dart_runtime.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index a7d41c5db..08329b4dd 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -13,8 +13,13 @@ use std::{ }; use tempfile::tempdir; +#[cfg(target_family = "windows")] +const DART_EXECUTABLE: &str = "dart.bat"; +#[cfg(not(target_family = "windows"))] +const DART_EXECUTABLE: &str = "dart"; + fn install_test_dependency(path: &Path) -> Result<()> { - Command::new("dart") + Command::new(DART_EXECUTABLE) .current_dir(path) .env("PUB_CACHE", "../.pub-cache") .args(["pub", "add", "-d", "test"]) @@ -27,7 +32,7 @@ fn install_test_dependency(path: &Path) -> Result<()> { fn test_dart_runtime_autotest() { // Not setting PUB_CACHE here because this is the only test run with the default // config anyway. - let dart_test = Command::new("dart") + let dart_test = Command::new(DART_EXECUTABLE) .current_dir("runtime/dart") .args(["test", "-r", "expanded"]) .status() @@ -118,7 +123,7 @@ void main() {{"# writeln!(source, "}}").unwrap(); - let dart_test = Command::new("dart") + let dart_test = Command::new(DART_EXECUTABLE) .current_dir(&source_path) .env("PUB_CACHE", "../.pub-cache") .args(["test", "test/runtime_test.dart"]) @@ -258,7 +263,7 @@ SerdeValue? {2}DeserializeSerdeData(List input) {{ .parent() .unwrap() .join("../../../serde-generate/runtime/dart"); - let status = Command::new("dart") + let status = Command::new(DART_EXECUTABLE) .current_dir("runtime/dart") .arg("pub") .arg("add") @@ -271,7 +276,7 @@ SerdeValue? {2}DeserializeSerdeData(List input) {{ .unwrap(); assert!(status.success()); - let status = Command::new("dart") + let status = Command::new(DART_EXECUTABLE) .current_dir("runtime/dart") .arg("run") .arg(source_path) From 04dcc95a28d8d0bfa85c64532a42fefc04a0f5fc Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 4 Oct 2024 21:22:34 +0900 Subject: [PATCH 03/41] Properly install Dart dependencies --- serde-generate/tests/dart_runtime.rs | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index 08329b4dd..0c235cc7b 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -56,9 +56,7 @@ fn test_dart_runtime_on_simple_data(runtime: Runtime) { let source_path = tempdir .path() .join(format!("dart_project_{}", runtime.name().to_lowercase())); - let registry = test_utils::get_simple_registry().unwrap(); - let config = CodeGeneratorConfig::new("example".to_string()) .with_encodings(vec![runtime.into()]) .with_c_style_enums(false); @@ -68,11 +66,10 @@ fn test_dart_runtime_on_simple_data(runtime: Runtime) { installer.install_serde_runtime().unwrap(); installer.install_bincode_runtime().unwrap(); installer.install_bcs_runtime().unwrap(); + install_test_dependency(&source_path).unwrap(); create_dir_all(source_path.join("test")).unwrap(); - install_test_dependency(&source_path).unwrap(); - let mut source = File::create(source_path.join("test/runtime_test.dart")).unwrap(); writeln!( source, @@ -155,10 +152,24 @@ fn quote_bytes(bytes: &[u8]) -> String { } fn test_dart_runtime_on_supported_types(runtime: Runtime) { - let dir = tempdir().unwrap(); - let source_path = dir + let tempdir = tempdir().unwrap(); + let source_path = tempdir .path() .join(format!("dart_project_{}", runtime.name().to_lowercase())); + let registry = test_utils::get_simple_registry().unwrap(); + let config = CodeGeneratorConfig::new("example".to_string()) + .with_encodings(vec![runtime.into()]) + .with_c_style_enums(false); + + let installer = dart::Installer::new(source_path.clone()); + installer.install_module(&config, ®istry).unwrap(); + installer.install_serde_runtime().unwrap(); + installer.install_bincode_runtime().unwrap(); + installer.install_bcs_runtime().unwrap(); + install_test_dependency(&source_path).unwrap(); + + create_dir_all(source_path.join("test")).unwrap(); + let mut source = File::create(source_path.join("test/runtime_test.dart")).unwrap(); let positive_encodings = runtime From b7b633400ee9d8e5f6dc92d0dfaeeef980d6f493 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 4 Oct 2024 22:03:10 +0900 Subject: [PATCH 04/41] Print Dart errors to console --- serde-generate/runtime/dart/pubspec.yaml | 4 ++-- serde-generate/tests/dart_runtime.rs | 28 ++++++------------------ 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/serde-generate/runtime/dart/pubspec.yaml b/serde-generate/runtime/dart/pubspec.yaml index 0db88a2c4..751c24ba9 100644 --- a/serde-generate/runtime/dart/pubspec.yaml +++ b/serde-generate/runtime/dart/pubspec.yaml @@ -1,10 +1,10 @@ name: serde environment: - sdk: '>=2.14.0 <3.0.0' + sdk: ">=3.0.0 <3.5.0" dependencies: meta: ^1.0.0 tuple: ^2.0.0 -dev_dependencies: +dev_dependencies: test: ^1.19.3 diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index 0c235cc7b..9d50b7ab9 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -269,29 +269,15 @@ SerdeValue? {2}DeserializeSerdeData(List input) {{ ) .unwrap(); - let runtime_mod_path = std::env::current_exe() - .unwrap() - .parent() - .unwrap() - .join("../../../serde-generate/runtime/dart"); - let status = Command::new(DART_EXECUTABLE) - .current_dir("runtime/dart") - .arg("pub") - .arg("add") - .arg("-n") // Use `-n` to avoid versioning, or use appropriate Dart flags - .arg(format!( - "serde-reflection: {}/runtime/dart", - runtime_mod_path.to_str().unwrap() - )) - .status() - .unwrap(); - assert!(status.success()); - - let status = Command::new(DART_EXECUTABLE) + let output = Command::new(DART_EXECUTABLE) .current_dir("runtime/dart") .arg("run") .arg(source_path) - .status() + .output() .unwrap(); - assert!(status.success()); + if !output.status.success() { + let error_output = String::from_utf8_lossy(&output.stderr); + eprintln!("{}", error_output); + } + assert!(output.status.success()); } From c5bb36fc8d6815b84f8f441a13eee4ee46272322 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 4 Oct 2024 22:10:23 +0900 Subject: [PATCH 05/41] Improve readability by limiting line length --- serde-generate/tests/dart_runtime.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index 9d50b7ab9..a95457883 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -30,8 +30,8 @@ fn install_test_dependency(path: &Path) -> Result<()> { #[test] fn test_dart_runtime_autotest() { - // Not setting PUB_CACHE here because this is the only test run with the default - // config anyway. + // Not setting PUB_CACHE here because this is the only test run + // with the default config anyway. let dart_test = Command::new(DART_EXECUTABLE) .current_dir("runtime/dart") .args(["test", "-r", "expanded"]) @@ -70,9 +70,10 @@ fn test_dart_runtime_on_simple_data(runtime: Runtime) { create_dir_all(source_path.join("test")).unwrap(); - let mut source = File::create(source_path.join("test/runtime_test.dart")).unwrap(); + let source = source_path.join("test/runtime_test.dart"); + let mut source_file = File::create(source).unwrap(); writeln!( - source, + source_file, r#" import 'dart:typed_data'; import 'package:example/example.dart'; @@ -91,7 +92,7 @@ void main() {{"# }); writeln!( - source, + source_file, r#" test('{1} serialization matches deserialization', () {{ final expectedBytes = Uint8List.fromList([{0}]); @@ -118,7 +119,7 @@ void main() {{"# ) .unwrap(); - writeln!(source, "}}").unwrap(); + writeln!(source_file, "}}").unwrap(); let dart_test = Command::new(DART_EXECUTABLE) .current_dir(&source_path) @@ -170,7 +171,8 @@ fn test_dart_runtime_on_supported_types(runtime: Runtime) { create_dir_all(source_path.join("test")).unwrap(); - let mut source = File::create(source_path.join("test/runtime_test.dart")).unwrap(); + let source = source_path.join("test/runtime_test.dart"); + let mut source_file = File::create(source).unwrap(); let positive_encodings = runtime .get_positive_samples_quick() @@ -187,7 +189,7 @@ fn test_dart_runtime_on_supported_types(runtime: Runtime) { .join(", "); writeln!( - source, + source_file, r#" void main() {{ var positiveInputs = [ From d3103b243fdaa5317be07989b98591a654d6469c Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 4 Oct 2024 22:14:40 +0900 Subject: [PATCH 06/41] Ensure Dart error outputs are shown --- serde-generate/tests/dart_runtime.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index a95457883..fcfccd064 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -121,14 +121,17 @@ void main() {{"# writeln!(source_file, "}}").unwrap(); - let dart_test = Command::new(DART_EXECUTABLE) + let output = Command::new(DART_EXECUTABLE) .current_dir(&source_path) .env("PUB_CACHE", "../.pub-cache") .args(["test", "test/runtime_test.dart"]) - .status() + .output() .unwrap(); - - assert!(dart_test.success()); + if !output.status.success() { + let error_output = String::from_utf8_lossy(&output.stdout); + eprintln!("{}", error_output); + } + assert!(output.status.success()); } #[test] @@ -272,13 +275,13 @@ SerdeValue? {2}DeserializeSerdeData(List input) {{ .unwrap(); let output = Command::new(DART_EXECUTABLE) - .current_dir("runtime/dart") - .arg("run") - .arg(source_path) + .current_dir(&source_path) + .env("PUB_CACHE", "../.pub-cache") + .args(["test", "test/runtime_test.dart"]) .output() .unwrap(); if !output.status.success() { - let error_output = String::from_utf8_lossy(&output.stderr); + let error_output = String::from_utf8_lossy(&output.stdout); eprintln!("{}", error_output); } assert!(output.status.success()); From ca97f2404a7f713baa2a50c38b81bd314ee2bc77 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 4 Oct 2024 22:44:03 +0900 Subject: [PATCH 07/41] Pass `Uint8List` properly to Dart --- serde-generate/tests/dart_runtime.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index fcfccd064..b42db2e00 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -146,7 +146,7 @@ fn test_dart_bincode_runtime_on_supported_types() { fn quote_bytes(bytes: &[u8]) -> String { format!( - "{{{}}}", + "Uint8List.fromList([{}])", bytes .iter() .map(|x| format!("{}", x)) @@ -194,11 +194,18 @@ fn test_dart_runtime_on_supported_types(runtime: Runtime) { writeln!( source_file, r#" +import 'dart:typed_data'; +import 'package:example/example.dart'; +import 'package:test/test.dart'; +import 'package:tuple/tuple.dart'; +import '../lib/src/bcs/bcs.dart'; +import '../lib/src/bincode/bincode.dart'; + void main() {{ - var positiveInputs = [ + List positiveInputs = [ {0} ]; - var negativeInputs = [ + List negativeInputs = [ {1} ]; From 5e4b000295fcbbf8ab3226c8a8c21d9a8bebe762 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 4 Oct 2024 22:59:34 +0900 Subject: [PATCH 08/41] Make Dart test code run --- serde-generate/tests/dart_runtime.rs | 59 +++++++++------------------- 1 file changed, 18 insertions(+), 41 deletions(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index b42db2e00..f0fe336b8 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -3,7 +3,6 @@ use crate::test_utils; use crate::test_utils::{Choice, Runtime, Test}; -use heck::CamelCase; use serde_generate::{dart, CodeGeneratorConfig, SourceInstaller}; use std::{ fs::{create_dir_all, File}, @@ -211,73 +210,51 @@ void main() {{ for (var input in positiveInputs) {{ // Deserialize the input. - var value = {2}DeserializeSerdeData(input); - if (value == null) {{ - throw Exception('Failed to deserialize input: $input'); - }} + Test value = Test.{2}Deserialize(input); + expect(value, isNotNull); // Serialize the deserialized value. - var output = value.{2}Serialize(); - if (output == null || !listEquals(input, output)) {{ - throw Exception('input != output:\n $input\n $output'); - }} + final output = value.{2}Serialize(); + expect(output, isNotNull); + expect(output, equals(input)); - // Test self-equality for the Serde value. + // Test self-equality for the deserialized value. {{ - var value2 = {2}DeserializeSerdeData(input); - if (value2 == null) {{ - throw Exception('Failed to deserialize input: $input'); - }} - if (value != value2) {{ - throw Exception('Value should test equal to itself.'); - }} + Test value2 = Test.{2}Deserialize(input); + expect(value2, isNotNull); + expect(value, equals(value2)); }} // Test simple mutations of the input. for (var i = 0; i < input.length; i++) {{ - var input2 = List.from(input); // Create a copy of the input - input2[i] ^= 0x80; // Mutate a byte - var value2 = {2}DeserializeSerdeData(input2); - if (value2 != null && value == value2) {{ - throw Exception('Modified input should give a different value.'); + var input2 = Uint8List.fromList(input); + input2[i] ^= 0x80; // Mutate a byte + Test value2 = Test.{2}Deserialize(input2); + if (value2 != null) {{ + expect(value, isNot(equals(value2))); }} }} }} // Test negative inputs for deserialization failure. for (var input in negativeInputs) {{ - var result = {2}DeserializeSerdeData(input); - if (result != null) {{ - throw Exception('Input should fail to deserialize: $input'); - }} + var result = Test.{2}Deserialize(input); + expect(result, isNull); }} }} // Helper function for comparing byte arrays. -bool listEquals(List a, List b) {{ +bool listEquals(Uint8List a, Uint8List b) {{ if (a.length != b.length) return false; for (var i = 0; i < a.length; i++) {{ if (a[i] != b[i]) return false; }} return true; }} - -// Placeholder class -class SerdeValue {{ - List {2}Serialize() {{ - // Implement serialization logic here. - return []; - }} -}} - -SerdeValue? {2}DeserializeSerdeData(List input) {{ - // Implement deserialization logic here. - return SerdeValue(); -}} "#, positive_encodings, negative_encodings, - runtime.name().to_camel_case(), + runtime.name().to_lowercase(), ) .unwrap(); From 2af88c7c07ad86b131829f252c7d97a8d854b099 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 4 Oct 2024 23:05:17 +0900 Subject: [PATCH 09/41] Shorten code --- serde-generate/tests/dart_runtime.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index f0fe336b8..8e3e8c972 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -201,12 +201,8 @@ import '../lib/src/bcs/bcs.dart'; import '../lib/src/bincode/bincode.dart'; void main() {{ - List positiveInputs = [ - {0} - ]; - List negativeInputs = [ - {1} - ]; + List positiveInputs = [{0}]; + List negativeInputs = [{1}]; for (var input in positiveInputs) {{ // Deserialize the input. From 9a2cef9d634880b5b00f13d3e19f8e6888c9ccbf Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 4 Oct 2024 23:15:15 +0900 Subject: [PATCH 10/41] Implement `sortMapEntries` --- .../runtime/dart/bcs/bcs_serializer.dart | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/serde-generate/runtime/dart/bcs/bcs_serializer.dart b/serde-generate/runtime/dart/bcs/bcs_serializer.dart index c9ea3e4ce..bc4b92529 100644 --- a/serde-generate/runtime/dart/bcs/bcs_serializer.dart +++ b/serde-generate/runtime/dart/bcs/bcs_serializer.dart @@ -23,6 +23,38 @@ class BcsSerializer extends BinarySerializer { } void sortMapEntries(Uint8List offsets) { - // TODO(#120) + if (offsets.isEmpty) { + return; + } + + // Create a list of slices based on offsets + List> slices = []; + int totalLength = output.length; + List offsetList = List.from(offsets); + offsetList.add(totalLength); + + for (int i = 1; i < offsetList.length; i++) { + slices.add(output.sublist(offsetList[i - 1], offsetList[i]).toList()); + } + + // Sort slices using lexicographic comparison + slices.sort((a, b) { + for (int i = 0; i < a.length && i < b.length; i++) { + if (a[i] != b[i]) { + return a[i].compareTo(b[i]); + } + } + return a.length.compareTo(b.length); + }); + + // Write sorted slices back to output + int writePosition = offsetList[0]; + for (final slice in slices) { + output.setRange(writePosition, writePosition + slice.length, slice); + writePosition += slice.length; + } + + // Ensure the final length is correct + assert(offsetList.last == output.length); } } From d15ecf2229072cf8f717dff30076c3a14f030682 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 4 Oct 2024 23:33:03 +0900 Subject: [PATCH 11/41] Reduce code duplication --- serde-generate/tests/dart_runtime.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index 8e3e8c972..68cf7fe57 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -94,7 +94,7 @@ void main() {{"# source_file, r#" test('{1} serialization matches deserialization', () {{ - final expectedBytes = Uint8List.fromList([{0}]); + final expectedBytes = {0}; Test deserializedInstance = Test.{1}Deserialize(expectedBytes); Test expectedInstance = Test( @@ -109,11 +109,7 @@ void main() {{"# expect(serializedBytes, equals(expectedBytes)); }});"#, - reference - .iter() - .map(|x| format!("{}", x)) - .collect::>() - .join(", "), + quote_bytes(&reference), runtime.name().to_lowercase(), ) .unwrap(); From e04394287ef939e59ad8ab36dbacd5bc2c895cc8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Fri, 4 Oct 2024 23:57:16 +0900 Subject: [PATCH 12/41] Organize Dart writer code --- serde-generate/tests/dart_runtime.rs | 102 ++++++++++++--------------- 1 file changed, 46 insertions(+), 56 deletions(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index 68cf7fe57..d0a5bc0af 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -71,19 +71,7 @@ fn test_dart_runtime_on_simple_data(runtime: Runtime) { let source = source_path.join("test/runtime_test.dart"); let mut source_file = File::create(source).unwrap(); - writeln!( - source_file, - r#" -import 'dart:typed_data'; -import 'package:example/example.dart'; -import 'package:test/test.dart'; -import 'package:tuple/tuple.dart'; -import '../lib/src/bcs/bcs.dart'; -import '../lib/src/bincode/bincode.dart'; -void main() {{"# - ) - .unwrap(); let reference = runtime.serialize(&Test { a: vec![4, 6], b: (-3, 5), @@ -93,6 +81,14 @@ void main() {{"# writeln!( source_file, r#" +import 'dart:typed_data'; +import 'package:example/example.dart'; +import 'package:test/test.dart'; +import 'package:tuple/tuple.dart'; +import '../lib/src/bcs/bcs.dart'; +import '../lib/src/bincode/bincode.dart'; + +void main() {{ test('{1} serialization matches deserialization', () {{ final expectedBytes = {0}; Test deserializedInstance = Test.{1}Deserialize(expectedBytes); @@ -104,18 +100,15 @@ void main() {{"# ); expect(deserializedInstance, equals(expectedInstance)); - final serializedBytes = expectedInstance.{1}Serialize(); - expect(serializedBytes, equals(expectedBytes)); - }});"#, + }}); +}}"#, quote_bytes(&reference), runtime.name().to_lowercase(), ) .unwrap(); - writeln!(source_file, "}}").unwrap(); - let output = Command::new(DART_EXECUTABLE) .current_dir(&source_path) .env("PUB_CACHE", "../.pub-cache") @@ -178,7 +171,6 @@ fn test_dart_runtime_on_supported_types(runtime: Runtime) { .map(|bytes| quote_bytes(bytes)) .collect::>() .join(", "); - let negative_encodings = runtime .get_negative_samples() .iter() @@ -197,51 +189,49 @@ import '../lib/src/bcs/bcs.dart'; import '../lib/src/bincode/bincode.dart'; void main() {{ - List positiveInputs = [{0}]; - List negativeInputs = [{1}]; - - for (var input in positiveInputs) {{ - // Deserialize the input. - Test value = Test.{2}Deserialize(input); - expect(value, isNotNull); - - // Serialize the deserialized value. - final output = value.{2}Serialize(); - expect(output, isNotNull); - expect(output, equals(input)); - - // Test self-equality for the deserialized value. - {{ - Test value2 = Test.{2}Deserialize(input); - expect(value2, isNotNull); - expect(value, equals(value2)); + List positiveInputs = [{0}]; + List negativeInputs = [{1}]; + + for (var input in positiveInputs) {{ + // Deserialize the input. + Test value = Test.{2}Deserialize(input); + expect(value, isNotNull); + + // Serialize the deserialized value. + final output = value.{2}Serialize(); + expect(output, isNotNull); + expect(output, equals(input)); + + // Test self-equality for the deserialized value. + Test value2 = Test.{2}Deserialize(input); + expect(value2, isNotNull); + expect(value, equals(value2)); + + // Test simple mutations of the input. + for (var i = 0; i < input.length; i++) {{ + var input2 = Uint8List.fromList(input); + input2[i] ^= 0x80; // Mutate a byte + Test value2 = Test.{2}Deserialize(input2); + if (value2 != null) {{ + expect(value, isNot(equals(value2))); + }} + }} }} - // Test simple mutations of the input. - for (var i = 0; i < input.length; i++) {{ - var input2 = Uint8List.fromList(input); - input2[i] ^= 0x80; // Mutate a byte - Test value2 = Test.{2}Deserialize(input2); - if (value2 != null) {{ - expect(value, isNot(equals(value2))); - }} + // Test negative inputs for deserialization failure. + for (var input in negativeInputs) {{ + var result = Test.{2}Deserialize(input); + expect(result, isNull); }} - }} - - // Test negative inputs for deserialization failure. - for (var input in negativeInputs) {{ - var result = Test.{2}Deserialize(input); - expect(result, isNull); - }} }} // Helper function for comparing byte arrays. bool listEquals(Uint8List a, Uint8List b) {{ - if (a.length != b.length) return false; - for (var i = 0; i < a.length; i++) {{ - if (a[i] != b[i]) return false; - }} - return true; + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) {{ + if (a[i] != b[i]) return false; + }} + return true; }} "#, positive_encodings, From ee013caa95fa18312659adc513ecd76c11e34900 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 5 Oct 2024 00:01:30 +0900 Subject: [PATCH 13/41] Match Dart codeblock style --- serde-generate/tests/dart_runtime.rs | 76 ++++++++++++++-------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index d0a5bc0af..afdc5fe08 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -89,9 +89,9 @@ import '../lib/src/bcs/bcs.dart'; import '../lib/src/bincode/bincode.dart'; void main() {{ - test('{1} serialization matches deserialization', () {{ - final expectedBytes = {0}; - Test deserializedInstance = Test.{1}Deserialize(expectedBytes); + test('{0} serialization matches deserialization', () {{ + final expectedBytes = {1}; + Test deserializedInstance = Test.{0}Deserialize(expectedBytes); Test expectedInstance = Test( a: [4, 6], @@ -100,12 +100,12 @@ void main() {{ ); expect(deserializedInstance, equals(expectedInstance)); - final serializedBytes = expectedInstance.{1}Serialize(); + final serializedBytes = expectedInstance.{0}Serialize(); expect(serializedBytes, equals(expectedBytes)); }}); }}"#, - quote_bytes(&reference), runtime.name().to_lowercase(), + quote_bytes(&reference), ) .unwrap(); @@ -189,40 +189,42 @@ import '../lib/src/bcs/bcs.dart'; import '../lib/src/bincode/bincode.dart'; void main() {{ - List positiveInputs = [{0}]; - List negativeInputs = [{1}]; - - for (var input in positiveInputs) {{ - // Deserialize the input. - Test value = Test.{2}Deserialize(input); - expect(value, isNotNull); - - // Serialize the deserialized value. - final output = value.{2}Serialize(); - expect(output, isNotNull); - expect(output, equals(input)); - - // Test self-equality for the deserialized value. - Test value2 = Test.{2}Deserialize(input); - expect(value2, isNotNull); - expect(value, equals(value2)); - - // Test simple mutations of the input. - for (var i = 0; i < input.length; i++) {{ - var input2 = Uint8List.fromList(input); - input2[i] ^= 0x80; // Mutate a byte - Test value2 = Test.{2}Deserialize(input2); - if (value2 != null) {{ - expect(value, isNot(equals(value2))); + test('{0} serialization matches deserialization', () {{ + List positiveInputs = [{1}]; + List negativeInputs = [{2}]; + + for (var input in positiveInputs) {{ + // Deserialize the input. + Test value = Test.{0}Deserialize(input); + expect(value, isNotNull); + + // Serialize the deserialized value. + final output = value.{0}Serialize(); + expect(output, isNotNull); + expect(output, equals(input)); + + // Test self-equality for the deserialized value. + Test value2 = Test.{0}Deserialize(input); + expect(value2, isNotNull); + expect(value, equals(value2)); + + // Test simple mutations of the input. + for (var i = 0; i < input.length; i++) {{ + var input2 = Uint8List.fromList(input); + input2[i] ^= 0x80; // Mutate a byte + Test value2 = Test.{0}Deserialize(input2); + if (value2 != null) {{ + expect(value, isNot(equals(value2))); + }} }} }} - }} - // Test negative inputs for deserialization failure. - for (var input in negativeInputs) {{ - var result = Test.{2}Deserialize(input); - expect(result, isNull); - }} + // Test negative inputs for deserialization failure. + for (var input in negativeInputs) {{ + var result = Test.{0}Deserialize(input); + expect(result, isNull); + }} + }}); }} // Helper function for comparing byte arrays. @@ -234,9 +236,9 @@ bool listEquals(Uint8List a, Uint8List b) {{ return true; }} "#, + runtime.name().to_lowercase(), positive_encodings, negative_encodings, - runtime.name().to_lowercase(), ) .unwrap(); From 7127f5be5da84d54198d339819b5148bca767c6b Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 5 Oct 2024 00:12:04 +0900 Subject: [PATCH 14/41] Change a test name --- serde-generate/tests/dart_runtime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index afdc5fe08..838febc37 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -189,7 +189,7 @@ import '../lib/src/bcs/bcs.dart'; import '../lib/src/bincode/bincode.dart'; void main() {{ - test('{0} serialization matches deserialization', () {{ + test('{0} supported types', () {{ List positiveInputs = [{1}]; List negativeInputs = [{2}]; From e1d2676a4aa9b2ce5ee8ba6fdabc34b490b7e1eb Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 5 Oct 2024 03:17:16 +0900 Subject: [PATCH 15/41] Update Dart test code as intended --- serde-generate/tests/dart_runtime.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index 838febc37..aaf23996c 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -196,33 +196,37 @@ void main() {{ for (var input in positiveInputs) {{ // Deserialize the input. Test value = Test.{0}Deserialize(input); - expect(value, isNotNull); // Serialize the deserialized value. final output = value.{0}Serialize(); - expect(output, isNotNull); expect(output, equals(input)); // Test self-equality for the deserialized value. Test value2 = Test.{0}Deserialize(input); - expect(value2, isNotNull); expect(value, equals(value2)); // Test simple mutations of the input. for (var i = 0; i < input.length; i++) {{ var input2 = Uint8List.fromList(input); input2[i] ^= 0x80; // Mutate a byte - Test value2 = Test.{0}Deserialize(input2); - if (value2 != null) {{ - expect(value, isNot(equals(value2))); + Test value2; + try {{ + value2 = Test.{0}Deserialize(input2); + }} catch (e) {{ + continue; }} + expect(value, isNot(equals(value2))); }} }} // Test negative inputs for deserialization failure. for (var input in negativeInputs) {{ - var result = Test.{0}Deserialize(input); - expect(result, isNull); + try {{ + var result = Test.{0}Deserialize(input); + }} catch (e) {{ + continue; + }} + throw Exception('Negative inputs should not be deserializable'); }} }}); }} From 63da582ff75ab7248193fb9a799d255dbfbde7a8 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 5 Oct 2024 03:18:17 +0900 Subject: [PATCH 16/41] Remove unused helper code --- serde-generate/tests/dart_runtime.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index aaf23996c..83b694e95 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -230,15 +230,6 @@ void main() {{ }} }}); }} - -// Helper function for comparing byte arrays. -bool listEquals(Uint8List a, Uint8List b) {{ - if (a.length != b.length) return false; - for (var i = 0; i < a.length; i++) {{ - if (a[i] != b[i]) return false; - }} - return true; -}} "#, runtime.name().to_lowercase(), positive_encodings, From 69a9336e2743efed036ad94b490f39d133664775 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 5 Oct 2024 11:48:35 +0900 Subject: [PATCH 17/41] Match the deserialization class to the original --- serde-generate/tests/dart_runtime.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index 83b694e95..733c614a0 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -148,7 +148,7 @@ fn test_dart_runtime_on_supported_types(runtime: Runtime) { let source_path = tempdir .path() .join(format!("dart_project_{}", runtime.name().to_lowercase())); - let registry = test_utils::get_simple_registry().unwrap(); + let registry = test_utils::get_registry().unwrap(); let config = CodeGeneratorConfig::new("example".to_string()) .with_encodings(vec![runtime.into()]) .with_c_style_enums(false); @@ -195,23 +195,23 @@ void main() {{ for (var input in positiveInputs) {{ // Deserialize the input. - Test value = Test.{0}Deserialize(input); + SerdeData value = SerdeData.{0}Deserialize(input); // Serialize the deserialized value. final output = value.{0}Serialize(); expect(output, equals(input)); // Test self-equality for the deserialized value. - Test value2 = Test.{0}Deserialize(input); + SerdeData value2 = SerdeData.{0}Deserialize(input); expect(value, equals(value2)); // Test simple mutations of the input. for (var i = 0; i < input.length; i++) {{ var input2 = Uint8List.fromList(input); input2[i] ^= 0x80; // Mutate a byte - Test value2; + SerdeData value2; try {{ - value2 = Test.{0}Deserialize(input2); + value2 = SerdeData.{0}Deserialize(input2); }} catch (e) {{ continue; }} @@ -222,7 +222,7 @@ void main() {{ // Test negative inputs for deserialization failure. for (var input in negativeInputs) {{ try {{ - var result = Test.{0}Deserialize(input); + SerdeData result = SerdeData.{0}Deserialize(input); }} catch (e) {{ continue; }} From 6c91b63935f8fed9afc62442a9fedb47be961021 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 5 Oct 2024 11:52:33 +0900 Subject: [PATCH 18/41] Rename a test --- serde-generate/tests/dart_runtime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index 733c614a0..501cdf8e3 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -89,7 +89,7 @@ import '../lib/src/bcs/bcs.dart'; import '../lib/src/bincode/bincode.dart'; void main() {{ - test('{0} serialization matches deserialization', () {{ + test('{0} simple data', () {{ final expectedBytes = {1}; Test deserializedInstance = Test.{0}Deserialize(expectedBytes); From eb737b62072b110a3271e09d54947c438afebc96 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 6 Oct 2024 12:38:09 +0900 Subject: [PATCH 19/41] Shorten an error message --- serde-generate/tests/dart_runtime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index 501cdf8e3..d1fbef46d 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -226,7 +226,7 @@ void main() {{ }} catch (e) {{ continue; }} - throw Exception('Negative inputs should not be deserializable'); + throw Exception('Input should fail to deserialize'); }} }}); }} From a4ecef5e10c2ac7273b347780788e252e56e3ebb Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 6 Oct 2024 13:28:42 +0900 Subject: [PATCH 20/41] Match `serializeInt128` to `deserializeInt128` --- serde-generate/runtime/dart/serde/binary_serializer.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/serde-generate/runtime/dart/serde/binary_serializer.dart b/serde-generate/runtime/dart/serde/binary_serializer.dart index 308e5c86f..0a5f8a065 100644 --- a/serde-generate/runtime/dart/serde/binary_serializer.dart +++ b/serde-generate/runtime/dart/serde/binary_serializer.dart @@ -129,8 +129,8 @@ abstract class BinarySerializer { void serializeLength(int len); void serializeInt128(Int128 value) { - serializeInt64(value.low.toInt()); - serializeInt64(value.high.toInt()); + serializeUint64(Uint64(value.low)); + serializeUint64(Uint64(value.high)); } void serializeUint128(Uint128 value) { From a6138c3f072652049e1db8a9b16bc926e53b9f87 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 6 Oct 2024 14:40:14 +0900 Subject: [PATCH 21/41] Update `ComplexMap` test to enable proper nested comparison in Dart --- serde-generate/tests/test_utils.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/serde-generate/tests/test_utils.rs b/serde-generate/tests/test_utils.rs index 843460e4c..23155144d 100644 --- a/serde-generate/tests/test_utils.rs +++ b/serde-generate/tests/test_utils.rs @@ -51,7 +51,7 @@ pub enum SerdeData { UnitVector(Vec<()>), SimpleList(SimpleList), CStyleEnum(CStyleEnum), - ComplexMap(BTreeMap<([u32; 2], [u8; 4]), ()>), + ComplexMap(BTreeMap<(), Vec<(u8, u8, u8)>>), EmptyTupleVariant(), EmptyStructVariant {}, } @@ -297,7 +297,7 @@ pub fn get_sample_values(has_canonical_maps: bool, has_floats: bool) -> Vec ()}); + let v13 = SerdeData::ComplexMap(btreemap! { () => vec![(1, 2, 3), (4, 5, 6)]}); let v14 = SerdeData::EmptyTupleVariant(); let v15 = SerdeData::EmptyStructVariant {}; From 0cf947a58f109195a4b8ba5887c8840d3ce5bdad Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sun, 6 Oct 2024 15:35:41 +0900 Subject: [PATCH 22/41] Check if the representation is valid boolean --- .../runtime/dart/serde/binary_deserializer.dart | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/serde-generate/runtime/dart/serde/binary_deserializer.dart b/serde-generate/runtime/dart/serde/binary_deserializer.dart index 2d2a0902a..1a134a4ff 100644 --- a/serde-generate/runtime/dart/serde/binary_deserializer.dart +++ b/serde-generate/runtime/dart/serde/binary_deserializer.dart @@ -15,9 +15,17 @@ abstract class BinaryDeserializer { } bool deserializeBool() { - final result = input.getUint8(_offset) != 0; + final result = input.getUint8(_offset); _offset += 1; - return result; + if (result == 0) { + return false; + } else if (result == 1) { + return true; + } else { + throw Exception( + 'Invalid boolean: expected 0 or 1, but got ${result}', + ); + } } Unit deserializeUnit() { From f00e43f5c21406ad60d519575031d372b8c63321 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 7 Oct 2024 20:29:23 +0900 Subject: [PATCH 23/41] Fix the test related to map deserialization --- serde-generate/runtime/dart/bincode/bincode_deserializer.dart | 4 +--- serde-generate/runtime/dart/bincode/bincode_serializer.dart | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/serde-generate/runtime/dart/bincode/bincode_deserializer.dart b/serde-generate/runtime/dart/bincode/bincode_deserializer.dart index 87c9ec48b..f210798d6 100644 --- a/serde-generate/runtime/dart/bincode/bincode_deserializer.dart +++ b/serde-generate/runtime/dart/bincode/bincode_deserializer.dart @@ -8,9 +8,7 @@ class BincodeDeserializer extends BinaryDeserializer { @override int deserializeLength() { - // bincode sends this as a u64 but since transferred data length should never exceed the upper - // bounds of an i64 (9223372036854775807 bytes is 9k petabytes) still deserialize to a Dart int - return deserializeInt64(); + return deserializeUint64().toInt(); } @override diff --git a/serde-generate/runtime/dart/bincode/bincode_serializer.dart b/serde-generate/runtime/dart/bincode/bincode_serializer.dart index d3d2d30b2..1aea2ed3a 100644 --- a/serde-generate/runtime/dart/bincode/bincode_serializer.dart +++ b/serde-generate/runtime/dart/bincode/bincode_serializer.dart @@ -6,9 +6,7 @@ part of bincode; class BincodeSerializer extends BinarySerializer { @override void serializeLength(int value) { - // bincode expects a u64 but since the capacity of a Dart int is less than that - // we can safely serialize as int to simplify over constructing Uint8 bytes - return serializeInt64(value); + serializeUint64(Uint64(BigInt.from(value))); } @override From cd5bd987e7d6f75b0e789663b9de91f54fb7e6bf Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 7 Oct 2024 20:48:20 +0900 Subject: [PATCH 24/41] Shorten code --- serde-generate/tests/dart_runtime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index d1fbef46d..0c9560aa6 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -222,7 +222,7 @@ void main() {{ // Test negative inputs for deserialization failure. for (var input in negativeInputs) {{ try {{ - SerdeData result = SerdeData.{0}Deserialize(input); + SerdeData.{0}Deserialize(input); }} catch (e) {{ continue; }} From 6dd6ef35dae1bd270e27c91d60240dce176082c5 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 7 Oct 2024 21:04:04 +0900 Subject: [PATCH 25/41] Check maximum sequence length --- serde-generate/runtime/dart/bcs/bcs_deserializer.dart | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/serde-generate/runtime/dart/bcs/bcs_deserializer.dart b/serde-generate/runtime/dart/bcs/bcs_deserializer.dart index cf39289ea..06f4189da 100644 --- a/serde-generate/runtime/dart/bcs/bcs_deserializer.dart +++ b/serde-generate/runtime/dart/bcs/bcs_deserializer.dart @@ -3,6 +3,9 @@ part of bcs; +// Maximum length allowed for sequences (vectors, bytes, strings) and maps. +const maxSequenceLength = (1 << 31) - 1; + class BcsDeserializer extends BinaryDeserializer { BcsDeserializer(Uint8List input) : super(input); @@ -27,7 +30,11 @@ class BcsDeserializer extends BinaryDeserializer { @override int deserializeLength() { - return deserializeUleb128AsUint32(); + final length = deserializeUleb128AsUint32(); + if (length > maxSequenceLength) { + throw Exception("length is too large"); + } + return length; } @override From b2de40b93ab5cd2053fefc4b35543da120cec917 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 7 Oct 2024 23:16:22 +0900 Subject: [PATCH 26/41] Consider max container depth --- .../runtime/dart/bcs/bcs_deserializer.dart | 9 ++- .../runtime/dart/bcs/bcs_serializer.dart | 5 ++ .../dart/bincode/bincode_deserializer.dart | 6 +- .../dart/bincode/bincode_serializer.dart | 5 ++ .../dart/serde/binary_deserializer.dart | 17 +++++- .../runtime/dart/serde/binary_serializer.dart | 16 ++++++ serde-generate/src/dart.rs | 56 ++++++++----------- 7 files changed, 79 insertions(+), 35 deletions(-) diff --git a/serde-generate/runtime/dart/bcs/bcs_deserializer.dart b/serde-generate/runtime/dart/bcs/bcs_deserializer.dart index 06f4189da..96b37718d 100644 --- a/serde-generate/runtime/dart/bcs/bcs_deserializer.dart +++ b/serde-generate/runtime/dart/bcs/bcs_deserializer.dart @@ -6,8 +6,15 @@ part of bcs; // Maximum length allowed for sequences (vectors, bytes, strings) and maps. const maxSequenceLength = (1 << 31) - 1; +// Maximum number of nested structs and enum variants. +const maxContainerDepth = 500; + class BcsDeserializer extends BinaryDeserializer { - BcsDeserializer(Uint8List input) : super(input); + BcsDeserializer(Uint8List input) + : super( + input: input, + containerDepthBudget: maxContainerDepth, + ); int deserializeUleb128AsUint32() { var value = 0; diff --git a/serde-generate/runtime/dart/bcs/bcs_serializer.dart b/serde-generate/runtime/dart/bcs/bcs_serializer.dart index bc4b92529..0c96efd86 100644 --- a/serde-generate/runtime/dart/bcs/bcs_serializer.dart +++ b/serde-generate/runtime/dart/bcs/bcs_serializer.dart @@ -4,6 +4,11 @@ part of bcs; class BcsSerializer extends BinarySerializer { + BcsSerializer() + : super( + containerDepthBudget: maxContainerDepth, + ); + void serializeUint32AsUleb128(int value) { while (((value & 0xFFFFFFFF) >> 7) != 0) { output.add((value & 0x7f) | 0x80); diff --git a/serde-generate/runtime/dart/bincode/bincode_deserializer.dart b/serde-generate/runtime/dart/bincode/bincode_deserializer.dart index f210798d6..1b696e0cc 100644 --- a/serde-generate/runtime/dart/bincode/bincode_deserializer.dart +++ b/serde-generate/runtime/dart/bincode/bincode_deserializer.dart @@ -3,8 +3,12 @@ part of bincode; +// Maximum number of nested structs and enum variants. +const maxContainerDepth = (1 << 31) - 1; + class BincodeDeserializer extends BinaryDeserializer { - BincodeDeserializer(Uint8List input) : super(input); + BincodeDeserializer(Uint8List input) + : super(input: input, containerDepthBudget: maxContainerDepth); @override int deserializeLength() { diff --git a/serde-generate/runtime/dart/bincode/bincode_serializer.dart b/serde-generate/runtime/dart/bincode/bincode_serializer.dart index 1aea2ed3a..cc41da782 100644 --- a/serde-generate/runtime/dart/bincode/bincode_serializer.dart +++ b/serde-generate/runtime/dart/bincode/bincode_serializer.dart @@ -4,6 +4,11 @@ part of bincode; class BincodeSerializer extends BinarySerializer { + BincodeSerializer() + : super( + containerDepthBudget: maxContainerDepth, + ); + @override void serializeLength(int value) { serializeUint64(Uint64(BigInt.from(value))); diff --git a/serde-generate/runtime/dart/serde/binary_deserializer.dart b/serde-generate/runtime/dart/serde/binary_deserializer.dart index 1a134a4ff..1f6737439 100644 --- a/serde-generate/runtime/dart/serde/binary_deserializer.dart +++ b/serde-generate/runtime/dart/serde/binary_deserializer.dart @@ -4,11 +4,15 @@ part of serde; abstract class BinaryDeserializer { - BinaryDeserializer(Uint8List input) : input = ByteData.view(input.buffer); + BinaryDeserializer({ + required Uint8List input, + required this.containerDepthBudget, + }) : input = ByteData.view(input.buffer); @protected final ByteData input; int _offset = 0; + int containerDepthBudget; int get offset { return _offset; @@ -154,4 +158,15 @@ abstract class BinaryDeserializer { return number.toUnsigned(byteLength * 8); } } + + void increaseContainerDepth() { + if (containerDepthBudget == 0) { + throw Exception('exceeded maximum container depth'); + } + containerDepthBudget -= 1; + } + + void decreaseContainerDepth() { + containerDepthBudget += 1; + } } diff --git a/serde-generate/runtime/dart/serde/binary_serializer.dart b/serde-generate/runtime/dart/serde/binary_serializer.dart index 0a5f8a065..7135064e1 100644 --- a/serde-generate/runtime/dart/serde/binary_serializer.dart +++ b/serde-generate/runtime/dart/serde/binary_serializer.dart @@ -4,6 +4,11 @@ part of serde; abstract class BinarySerializer { + BinarySerializer({ + required this.containerDepthBudget, + }); + + int containerDepthBudget; final List output = List.empty(growable: true); Uint8List get bytes { @@ -137,4 +142,15 @@ abstract class BinarySerializer { serializeUint64(Uint64(value.low)); serializeUint64(Uint64(value.high)); } + + void increaseContainerDepth() { + if (containerDepthBudget == 0) { + throw Exception('exceeded maximum container depth'); + } + containerDepthBudget -= 1; + } + + void decreaseContainerDepth() { + containerDepthBudget += 1; + } } diff --git a/serde-generate/src/dart.rs b/serde-generate/src/dart.rs index ff382a4eb..6fa6884df 100644 --- a/serde-generate/src/dart.rs +++ b/serde-generate/src/dart.rs @@ -670,53 +670,43 @@ return obj; } if self.generator.config.serialization { - // a struct (UnitStruct) with zero fields - if variant_index.is_none() && fields.is_empty() { - writeln!( - self.out, - "\n{}.deserialize(BinaryDeserializer deserializer);", - self.quote_qualified_name(name) - )?; // Deserialize (struct) or Load (variant) - } else if variant_index.is_none() { - writeln!( - self.out, - "\n{}.deserialize(BinaryDeserializer deserializer) :", - self.quote_qualified_name(name) - )?; - } else if !fields.is_empty() { + if variant_index.is_none() { writeln!( self.out, - "\n{}.load(BinaryDeserializer deserializer) :", + "\nstatic {} deserialize(BinaryDeserializer deserializer) {{", self.quote_qualified_name(name) )?; } else { writeln!( self.out, - "\n{}.load(BinaryDeserializer deserializer);", + "\nstatic {} load(BinaryDeserializer deserializer) {{", self.quote_qualified_name(name) )?; } self.out.indent(); - for (index, field) in fields.iter().enumerate() { - if index == field_count - 1 { - writeln!( - self.out, - "{} = {};", - self.quote_field(&field.name.to_mixed_case()), - self.quote_deserialize(&field.value) - )?; - } else { - writeln!( - self.out, - "{} = {},", - self.quote_field(&field.name.to_mixed_case()), - self.quote_deserialize(&field.value) - )?; - } + writeln!(self.out, "deserializer.increaseContainerDepth();")?; + writeln!( + self.out, + "final instance = {}(", + self.quote_qualified_name(name) + )?; + self.out.indent(); + for field in fields { + writeln!( + self.out, + "{}: {},", + self.quote_field(&field.name.to_mixed_case()), + self.quote_deserialize(&field.value) + )?; } self.out.unindent(); + writeln!(self.out, ");")?; + writeln!(self.out, "deserializer.decreaseContainerDepth();")?; + writeln!(self.out, "return instance;")?; + self.out.unindent(); + writeln!(self.out, "}}")?; if variant_index.is_none() { for encoding in &self.generator.config.encodings { @@ -780,6 +770,7 @@ return obj; if self.generator.config.serialization { writeln!(self.out, "\nvoid serialize(BinarySerializer serializer) {{",)?; self.out.indent(); + writeln!(self.out, "serializer.increaseContainerDepth();")?; if let Some(index) = variant_index { writeln!(self.out, "serializer.serializeVariantIndex({});", index)?; } @@ -793,6 +784,7 @@ return obj; ) )?; } + writeln!(self.out, "serializer.decreaseContainerDepth();")?; self.out.unindent(); writeln!(self.out, "}}")?; From ff191efef605f7f6587723a202e20f6c6d30d1df Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Mon, 7 Oct 2024 23:27:48 +0900 Subject: [PATCH 27/41] Use `Uint8List` while sorting map entries for performance --- serde-generate/runtime/dart/bcs/bcs_serializer.dart | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/serde-generate/runtime/dart/bcs/bcs_serializer.dart b/serde-generate/runtime/dart/bcs/bcs_serializer.dart index 0c96efd86..155c4d270 100644 --- a/serde-generate/runtime/dart/bcs/bcs_serializer.dart +++ b/serde-generate/runtime/dart/bcs/bcs_serializer.dart @@ -32,14 +32,16 @@ class BcsSerializer extends BinarySerializer { return; } + final binOutput = Uint8List.fromList(output); + // Create a list of slices based on offsets - List> slices = []; - int totalLength = output.length; + List slices = []; + int totalLength = binOutput.length; List offsetList = List.from(offsets); offsetList.add(totalLength); for (int i = 1; i < offsetList.length; i++) { - slices.add(output.sublist(offsetList[i - 1], offsetList[i]).toList()); + slices.add(binOutput.sublist(offsetList[i - 1], offsetList[i])); } // Sort slices using lexicographic comparison From dd121a6157a214cbcfcc3a148e25cc0139c10933 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Tue, 8 Oct 2024 01:08:27 +0900 Subject: [PATCH 28/41] Fix `test_get_registry` --- serde-generate/tests/test_utils.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/serde-generate/tests/test_utils.rs b/serde-generate/tests/test_utils.rs index 23155144d..293d24590 100644 --- a/serde-generate/tests/test_utils.rs +++ b/serde-generate/tests/test_utils.rs @@ -778,15 +778,12 @@ SerdeData: ComplexMap: NEWTYPE: MAP: - KEY: - TUPLE: - - TUPLEARRAY: - CONTENT: U32 - SIZE: 2 - - TUPLEARRAY: - CONTENT: U8 - SIZE: 4 - VALUE: UNIT + KEY: UNIT + VALUE: + SEQ: + TUPLEARRAY: + CONTENT: U8 + SIZE: 3 13: EmptyTupleVariant: TUPLE: [] From ff20b872add46e670026554351c9974c366418f7 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Oct 2024 10:50:28 +0900 Subject: [PATCH 29/41] Print output on fail from `test_dart_runtime_autotest` --- serde-generate/tests/dart_runtime.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index 0c9560aa6..b4930dbaf 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -31,13 +31,16 @@ fn install_test_dependency(path: &Path) -> Result<()> { fn test_dart_runtime_autotest() { // Not setting PUB_CACHE here because this is the only test run // with the default config anyway. - let dart_test = Command::new(DART_EXECUTABLE) + let output = Command::new(DART_EXECUTABLE) .current_dir("runtime/dart") .args(["test", "-r", "expanded"]) - .status() + .output() .unwrap(); - - assert!(dart_test.success()); + if !output.status.success() { + let error_output = String::from_utf8_lossy(&output.stdout); + eprintln!("{}", error_output); + } + assert!(output.status.success()); } #[test] From b022ded0429ba2bc4fa121a6d220b648b647f76e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Oct 2024 11:42:39 +0900 Subject: [PATCH 30/41] Bump Dart versions --- .github/workflows/main.yml | 2 +- serde-generate/runtime/dart/pubspec.yaml | 2 +- serde-generate/src/dart.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 98efcf245..296986760 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -90,7 +90,7 @@ jobs: - name: Setup Additional Languages (dart) uses: dart-lang/setup-dart@v1 with: - sdk: 2.14.0 + sdk: 3.5.3 - name: Setup Additional Languages (swift) uses: swift-actions/setup-swift@v1 diff --git a/serde-generate/runtime/dart/pubspec.yaml b/serde-generate/runtime/dart/pubspec.yaml index 751c24ba9..cd2c5a508 100644 --- a/serde-generate/runtime/dart/pubspec.yaml +++ b/serde-generate/runtime/dart/pubspec.yaml @@ -1,7 +1,7 @@ name: serde environment: - sdk: ">=3.0.0 <3.5.0" + sdk: ">=3.0.0 <4.0.0" dependencies: meta: ^1.0.0 diff --git a/serde-generate/src/dart.rs b/serde-generate/src/dart.rs index 6fa6884df..59d906248 100644 --- a/serde-generate/src/dart.rs +++ b/serde-generate/src/dart.rs @@ -86,7 +86,7 @@ impl<'a> CodeGenerator<'a> { r#"name: {} environment: - sdk: '>=2.14.0 <3.0.0' + sdk: '>=3.0.0 <4.0.0' dependencies: meta: ^1.0.0 From 408c3e66a644e42ee8abe0aab2223f3ec70dcb79 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Oct 2024 11:58:20 +0900 Subject: [PATCH 31/41] Make `sortMapEntries` an abstract function --- serde-generate/runtime/dart/bcs/bcs_serializer.dart | 2 +- serde-generate/runtime/dart/bincode/bincode_serializer.dart | 2 +- serde-generate/runtime/dart/serde/binary_serializer.dart | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/serde-generate/runtime/dart/bcs/bcs_serializer.dart b/serde-generate/runtime/dart/bcs/bcs_serializer.dart index 155c4d270..1901b620a 100644 --- a/serde-generate/runtime/dart/bcs/bcs_serializer.dart +++ b/serde-generate/runtime/dart/bcs/bcs_serializer.dart @@ -27,7 +27,7 @@ class BcsSerializer extends BinarySerializer { serializeUint32AsUleb128(value); } - void sortMapEntries(Uint8List offsets) { + void sortMapEntries(List offsets) { if (offsets.isEmpty) { return; } diff --git a/serde-generate/runtime/dart/bincode/bincode_serializer.dart b/serde-generate/runtime/dart/bincode/bincode_serializer.dart index cc41da782..bce3f91d9 100644 --- a/serde-generate/runtime/dart/bincode/bincode_serializer.dart +++ b/serde-generate/runtime/dart/bincode/bincode_serializer.dart @@ -19,7 +19,7 @@ class BincodeSerializer extends BinarySerializer { serializeUint32(value); } - void sortMapEntries(Int32List offsets) { + void sortMapEntries(List offsets) { // Not required by the format. } } diff --git a/serde-generate/runtime/dart/serde/binary_serializer.dart b/serde-generate/runtime/dart/serde/binary_serializer.dart index 7135064e1..062f4fa4a 100644 --- a/serde-generate/runtime/dart/serde/binary_serializer.dart +++ b/serde-generate/runtime/dart/serde/binary_serializer.dart @@ -153,4 +153,6 @@ abstract class BinarySerializer { void decreaseContainerDepth() { containerDepthBudget += 1; } + + void sortMapEntries(List offsets); } From 9c92e8ee49458b618460d7bb1db3694c621ba1bf Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Oct 2024 12:07:51 +0900 Subject: [PATCH 32/41] Organize `sortMapEntries` code --- .../runtime/dart/bcs/bcs_serializer.dart | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/serde-generate/runtime/dart/bcs/bcs_serializer.dart b/serde-generate/runtime/dart/bcs/bcs_serializer.dart index 1901b620a..f21d297ef 100644 --- a/serde-generate/runtime/dart/bcs/bcs_serializer.dart +++ b/serde-generate/runtime/dart/bcs/bcs_serializer.dart @@ -32,16 +32,20 @@ class BcsSerializer extends BinarySerializer { return; } - final binOutput = Uint8List.fromList(output); - - // Create a list of slices based on offsets + // Prepare a list of slices + final data = Uint8List.fromList(output); List slices = []; - int totalLength = binOutput.length; - List offsetList = List.from(offsets); - offsetList.add(totalLength); - for (int i = 1; i < offsetList.length; i++) { - slices.add(binOutput.sublist(offsetList[i - 1], offsetList[i])); + // Collect slices + for (int i = 0; i < offsets.length; i++) { + final int startOffset = offsets[i]; + final int cutOffset; + if (i + 1 < offsets.length) { + cutOffset = offsets[i + 1]; + } else { + cutOffset = data.length; + } + slices.add(data.sublist(startOffset, cutOffset)); } // Sort slices using lexicographic comparison @@ -55,13 +59,13 @@ class BcsSerializer extends BinarySerializer { }); // Write sorted slices back to output - int writePosition = offsetList[0]; + int writePosition = offsets[0]; for (final slice in slices) { output.setRange(writePosition, writePosition + slice.length, slice); writePosition += slice.length; } // Ensure the final length is correct - assert(offsetList.last == output.length); + assert(offsets.last == output.length); } } From f36ad1e3540dc23e3095e2b88d3bedabf79cf283 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Oct 2024 13:24:58 +0900 Subject: [PATCH 33/41] Update expected positive sample count regarding `v13` --- serde-generate/tests/test_utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serde-generate/tests/test_utils.rs b/serde-generate/tests/test_utils.rs index 293d24590..7214cb9ac 100644 --- a/serde-generate/tests/test_utils.rs +++ b/serde-generate/tests/test_utils.rs @@ -928,7 +928,7 @@ fn test_bincode_get_positive_samples() { // This test requires --release because of deserialization of long (unit) vectors. #[cfg(not(debug_assertions))] fn test_bcs_get_positive_samples() { - assert_eq!(test_get_positive_samples(Runtime::Bcs), 82); + assert_eq!(test_get_positive_samples(Runtime::Bcs), 81); } // Make sure all the "positive" samples successfully deserialize with the reference Rust From a7a9ae862ae913c05d8e67bb9ac60bc6150b37b2 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Oct 2024 13:36:13 +0900 Subject: [PATCH 34/41] Format a YAML file --- serde-generate/runtime/dart/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serde-generate/runtime/dart/pubspec.yaml b/serde-generate/runtime/dart/pubspec.yaml index cd2c5a508..5e7886896 100644 --- a/serde-generate/runtime/dart/pubspec.yaml +++ b/serde-generate/runtime/dart/pubspec.yaml @@ -1,7 +1,7 @@ name: serde environment: - sdk: ">=3.0.0 <4.0.0" + sdk: '>=3.0.0 <4.0.0' dependencies: meta: ^1.0.0 From f5d1489919f33cadd19f31c6c1d00474cadcd27e Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Oct 2024 13:50:49 +0900 Subject: [PATCH 35/41] Write some temporary C# test code --- serde-generate/tests/csharp_generation.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/serde-generate/tests/csharp_generation.rs b/serde-generate/tests/csharp_generation.rs index 95e45a412..1823432e0 100644 --- a/serde-generate/tests/csharp_generation.rs +++ b/serde-generate/tests/csharp_generation.rs @@ -29,12 +29,16 @@ fn test_that_csharp_code_compiles_with_config( let proj_path = dir_path.join(config.module_name().replace('.', "/")); { let _lock = MUTEX.lock(); - let status = Command::new("dotnet") + let output = Command::new("dotnet") .arg("build") .current_dir(&proj_path) - .status() + .output() .unwrap(); - assert!(status.success()); + if !output.status.success() { + let error_output = String::from_utf8_lossy(&output.stdout); + eprintln!("{}", error_output); + } + assert!(output.status.success()); } (dir, proj_path) From e82a6c88b1f64b24c184d5e3d2e303429f38f682 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Oct 2024 14:21:52 +0900 Subject: [PATCH 36/41] Revert "Write some temporary C# test code" This reverts commit f5d1489919f33cadd19f31c6c1d00474cadcd27e. --- serde-generate/tests/csharp_generation.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/serde-generate/tests/csharp_generation.rs b/serde-generate/tests/csharp_generation.rs index 1823432e0..95e45a412 100644 --- a/serde-generate/tests/csharp_generation.rs +++ b/serde-generate/tests/csharp_generation.rs @@ -29,16 +29,12 @@ fn test_that_csharp_code_compiles_with_config( let proj_path = dir_path.join(config.module_name().replace('.', "/")); { let _lock = MUTEX.lock(); - let output = Command::new("dotnet") + let status = Command::new("dotnet") .arg("build") .current_dir(&proj_path) - .output() + .status() .unwrap(); - if !output.status.success() { - let error_output = String::from_utf8_lossy(&output.stdout); - eprintln!("{}", error_output); - } - assert!(output.status.success()); + assert!(status.success()); } (dir, proj_path) From 95351213d8d3a83cb699a9ba2352bdd7e65ef730 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Wed, 9 Oct 2024 14:23:03 +0900 Subject: [PATCH 37/41] Update an expected sample number --- serde-generate/tests/test_utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serde-generate/tests/test_utils.rs b/serde-generate/tests/test_utils.rs index 7214cb9ac..d3bce0101 100644 --- a/serde-generate/tests/test_utils.rs +++ b/serde-generate/tests/test_utils.rs @@ -952,7 +952,7 @@ fn test_bincode_get_negative_samples() { // This test requires --release because of deserialization of long (unit) vectors. #[cfg(not(debug_assertions))] fn test_bcs_get_negative_samples() { - assert_eq!(test_get_negative_samples(Runtime::Bcs), 59); + assert_eq!(test_get_negative_samples(Runtime::Bcs), 60); } // Make sure all the "negative" samples fail to deserialize with the reference Rust From 0c07d41005ee70cebca9ebba321337c5f786e3eb Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 24 Oct 2024 21:34:04 +0900 Subject: [PATCH 38/41] Revert changes in `v13` test --- serde-generate/tests/test_utils.rs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/serde-generate/tests/test_utils.rs b/serde-generate/tests/test_utils.rs index d3bce0101..843460e4c 100644 --- a/serde-generate/tests/test_utils.rs +++ b/serde-generate/tests/test_utils.rs @@ -51,7 +51,7 @@ pub enum SerdeData { UnitVector(Vec<()>), SimpleList(SimpleList), CStyleEnum(CStyleEnum), - ComplexMap(BTreeMap<(), Vec<(u8, u8, u8)>>), + ComplexMap(BTreeMap<([u32; 2], [u8; 4]), ()>), EmptyTupleVariant(), EmptyStructVariant {}, } @@ -297,7 +297,7 @@ pub fn get_sample_values(has_canonical_maps: bool, has_floats: bool) -> Vec vec![(1, 2, 3), (4, 5, 6)]}); + let v13 = SerdeData::ComplexMap(btreemap! { ([1,2], [3,4,5,6]) => ()}); let v14 = SerdeData::EmptyTupleVariant(); let v15 = SerdeData::EmptyStructVariant {}; @@ -778,12 +778,15 @@ SerdeData: ComplexMap: NEWTYPE: MAP: - KEY: UNIT - VALUE: - SEQ: - TUPLEARRAY: - CONTENT: U8 - SIZE: 3 + KEY: + TUPLE: + - TUPLEARRAY: + CONTENT: U32 + SIZE: 2 + - TUPLEARRAY: + CONTENT: U8 + SIZE: 4 + VALUE: UNIT 13: EmptyTupleVariant: TUPLE: [] @@ -928,7 +931,7 @@ fn test_bincode_get_positive_samples() { // This test requires --release because of deserialization of long (unit) vectors. #[cfg(not(debug_assertions))] fn test_bcs_get_positive_samples() { - assert_eq!(test_get_positive_samples(Runtime::Bcs), 81); + assert_eq!(test_get_positive_samples(Runtime::Bcs), 82); } // Make sure all the "positive" samples successfully deserialize with the reference Rust @@ -952,7 +955,7 @@ fn test_bincode_get_negative_samples() { // This test requires --release because of deserialization of long (unit) vectors. #[cfg(not(debug_assertions))] fn test_bcs_get_negative_samples() { - assert_eq!(test_get_negative_samples(Runtime::Bcs), 60); + assert_eq!(test_get_negative_samples(Runtime::Bcs), 59); } // Make sure all the "negative" samples fail to deserialize with the reference Rust From 10f63086d2c54430a830d5dee48fa0eb7cb4a6a6 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Thu, 24 Oct 2024 21:49:09 +0900 Subject: [PATCH 39/41] Exclude `v13` value from Dart tests --- serde-generate/tests/dart_runtime.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/serde-generate/tests/dart_runtime.rs b/serde-generate/tests/dart_runtime.rs index b4930dbaf..164719f6f 100644 --- a/serde-generate/tests/dart_runtime.rs +++ b/serde-generate/tests/dart_runtime.rs @@ -200,6 +200,13 @@ void main() {{ // Deserialize the input. SerdeData value = SerdeData.{0}Deserialize(input); + // Exclude `SerdeData::ComplexMap` from tests + // because the `matcher` package used by the `test` package + // doesn't support lists within tuples. + if (value is SerdeDataComplexMap) {{ + continue; + }} + // Serialize the deserialized value. final output = value.{0}Serialize(); expect(output, equals(input)); From 7dbb31aee45b3c766a0e9c93a009bc02d89c0376 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 4 Jan 2025 11:13:39 +0900 Subject: [PATCH 40/41] Relieve Bincode bool check logic --- .../runtime/dart/bcs/bcs_deserializer.dart | 14 +++++ .../dart/bincode/bincode_deserializer.dart | 6 ++ .../dart/serde/binary_deserializer.dart | 62 +++++++------------ 3 files changed, 44 insertions(+), 38 deletions(-) diff --git a/serde-generate/runtime/dart/bcs/bcs_deserializer.dart b/serde-generate/runtime/dart/bcs/bcs_deserializer.dart index 96b37718d..5fff38eb7 100644 --- a/serde-generate/runtime/dart/bcs/bcs_deserializer.dart +++ b/serde-generate/runtime/dart/bcs/bcs_deserializer.dart @@ -56,4 +56,18 @@ class BcsDeserializer extends BinaryDeserializer { "Error while decoding map: keys are not serialized in the expected order"); } } + + bool deserializeBool() { + final result = input.getUint8(offset); + offset += 1; + if (result == 0) { + return false; + } else if (result == 1) { + return true; + } else { + throw Exception( + 'Invalid boolean: expected 0 or 1, but got ${result}', + ); + } + } } diff --git a/serde-generate/runtime/dart/bincode/bincode_deserializer.dart b/serde-generate/runtime/dart/bincode/bincode_deserializer.dart index 1b696e0cc..f5f9cc730 100644 --- a/serde-generate/runtime/dart/bincode/bincode_deserializer.dart +++ b/serde-generate/runtime/dart/bincode/bincode_deserializer.dart @@ -24,4 +24,10 @@ class BincodeDeserializer extends BinaryDeserializer { void checkThatKeySlicesAreIncreasing(Slice key1, Slice key2) { // Not required by the format. } + + bool deserializeBool() { + final result = input.getUint8(offset) != 0; + offset += 1; + return result; + } } diff --git a/serde-generate/runtime/dart/serde/binary_deserializer.dart b/serde-generate/runtime/dart/serde/binary_deserializer.dart index 1f6737439..e2f7045f9 100644 --- a/serde-generate/runtime/dart/serde/binary_deserializer.dart +++ b/serde-generate/runtime/dart/serde/binary_deserializer.dart @@ -11,88 +11,74 @@ abstract class BinaryDeserializer { @protected final ByteData input; - int _offset = 0; + @protected + int offset = 0; + @protected int containerDepthBudget; - int get offset { - return _offset; - } - - bool deserializeBool() { - final result = input.getUint8(_offset); - _offset += 1; - if (result == 0) { - return false; - } else if (result == 1) { - return true; - } else { - throw Exception( - 'Invalid boolean: expected 0 or 1, but got ${result}', - ); - } - } + bool deserializeBool(); Unit deserializeUnit() { return const Unit(); } int deserializeUint8() { - final result = input.getUint8(_offset); - _offset += 1; + final result = input.getUint8(offset); + offset += 1; return result; } int deserializeUint16() { - final result = input.getUint16(_offset, Endian.little); - _offset += 2; + final result = input.getUint16(offset, Endian.little); + offset += 2; return result; } int deserializeUint32() { - final result = input.getUint32(_offset, Endian.little); - _offset += 4; + final result = input.getUint32(offset, Endian.little); + offset += 4; return result; } Uint64 deserializeUint64() { final number = _bytesToBigInt(8, signed: false); - _offset += 8; + offset += 8; return Uint64(number); } int deserializeInt8() { - final result = input.getInt8(_offset); - _offset += 1; + final result = input.getInt8(offset); + offset += 1; return result; } int deserializeInt16() { - final result = input.getInt16(_offset, Endian.little); - _offset += 2; + final result = input.getInt16(offset, Endian.little); + offset += 2; return result; } int deserializeInt32() { - final result = input.getInt32(_offset, Endian.little); - _offset += 4; + final result = input.getInt32(offset, Endian.little); + offset += 4; return result; } int deserializeInt64() { - final result = input.getInt64(_offset, Endian.little); - _offset += 8; + final result = input.getInt64(offset, Endian.little); + offset += 8; return result; } double deserializeFloat32() { - final result = input.getFloat32(_offset, Endian.little); - _offset += 4; + final result = input.getFloat32(offset, Endian.little); + offset += 4; return result; } double deserializeFloat64() { - final result = input.getFloat64(_offset, Endian.little); - _offset += 8; + final result = input.getFloat64(offset, Endian.little); + offset += 8; return result; } @@ -149,7 +135,7 @@ abstract class BinaryDeserializer { // number += BigInt.from(bytes[byteLength - i - 1]) << (8 * i); // little endian - number += BigInt.from(input.getUint8(_offset + i)) << (8 * i); + number += BigInt.from(input.getUint8(offset + i)) << (8 * i); } if (signed) { From eda2781b487f92036a71b700a45716d0007ada64 Mon Sep 17 00:00:00 2001 From: Donghyun Kim Date: Sat, 4 Jan 2025 11:15:05 +0900 Subject: [PATCH 41/41] Revert "Relieve Bincode bool check logic" This reverts commit 7dbb31aee45b3c766a0e9c93a009bc02d89c0376. --- .../runtime/dart/bcs/bcs_deserializer.dart | 14 ----- .../dart/bincode/bincode_deserializer.dart | 6 -- .../dart/serde/binary_deserializer.dart | 62 ++++++++++++------- 3 files changed, 38 insertions(+), 44 deletions(-) diff --git a/serde-generate/runtime/dart/bcs/bcs_deserializer.dart b/serde-generate/runtime/dart/bcs/bcs_deserializer.dart index 5fff38eb7..96b37718d 100644 --- a/serde-generate/runtime/dart/bcs/bcs_deserializer.dart +++ b/serde-generate/runtime/dart/bcs/bcs_deserializer.dart @@ -56,18 +56,4 @@ class BcsDeserializer extends BinaryDeserializer { "Error while decoding map: keys are not serialized in the expected order"); } } - - bool deserializeBool() { - final result = input.getUint8(offset); - offset += 1; - if (result == 0) { - return false; - } else if (result == 1) { - return true; - } else { - throw Exception( - 'Invalid boolean: expected 0 or 1, but got ${result}', - ); - } - } } diff --git a/serde-generate/runtime/dart/bincode/bincode_deserializer.dart b/serde-generate/runtime/dart/bincode/bincode_deserializer.dart index f5f9cc730..1b696e0cc 100644 --- a/serde-generate/runtime/dart/bincode/bincode_deserializer.dart +++ b/serde-generate/runtime/dart/bincode/bincode_deserializer.dart @@ -24,10 +24,4 @@ class BincodeDeserializer extends BinaryDeserializer { void checkThatKeySlicesAreIncreasing(Slice key1, Slice key2) { // Not required by the format. } - - bool deserializeBool() { - final result = input.getUint8(offset) != 0; - offset += 1; - return result; - } } diff --git a/serde-generate/runtime/dart/serde/binary_deserializer.dart b/serde-generate/runtime/dart/serde/binary_deserializer.dart index e2f7045f9..1f6737439 100644 --- a/serde-generate/runtime/dart/serde/binary_deserializer.dart +++ b/serde-generate/runtime/dart/serde/binary_deserializer.dart @@ -11,74 +11,88 @@ abstract class BinaryDeserializer { @protected final ByteData input; - @protected - int offset = 0; - @protected + int _offset = 0; int containerDepthBudget; - bool deserializeBool(); + int get offset { + return _offset; + } + + bool deserializeBool() { + final result = input.getUint8(_offset); + _offset += 1; + if (result == 0) { + return false; + } else if (result == 1) { + return true; + } else { + throw Exception( + 'Invalid boolean: expected 0 or 1, but got ${result}', + ); + } + } Unit deserializeUnit() { return const Unit(); } int deserializeUint8() { - final result = input.getUint8(offset); - offset += 1; + final result = input.getUint8(_offset); + _offset += 1; return result; } int deserializeUint16() { - final result = input.getUint16(offset, Endian.little); - offset += 2; + final result = input.getUint16(_offset, Endian.little); + _offset += 2; return result; } int deserializeUint32() { - final result = input.getUint32(offset, Endian.little); - offset += 4; + final result = input.getUint32(_offset, Endian.little); + _offset += 4; return result; } Uint64 deserializeUint64() { final number = _bytesToBigInt(8, signed: false); - offset += 8; + _offset += 8; return Uint64(number); } int deserializeInt8() { - final result = input.getInt8(offset); - offset += 1; + final result = input.getInt8(_offset); + _offset += 1; return result; } int deserializeInt16() { - final result = input.getInt16(offset, Endian.little); - offset += 2; + final result = input.getInt16(_offset, Endian.little); + _offset += 2; return result; } int deserializeInt32() { - final result = input.getInt32(offset, Endian.little); - offset += 4; + final result = input.getInt32(_offset, Endian.little); + _offset += 4; return result; } int deserializeInt64() { - final result = input.getInt64(offset, Endian.little); - offset += 8; + final result = input.getInt64(_offset, Endian.little); + _offset += 8; return result; } double deserializeFloat32() { - final result = input.getFloat32(offset, Endian.little); - offset += 4; + final result = input.getFloat32(_offset, Endian.little); + _offset += 4; return result; } double deserializeFloat64() { - final result = input.getFloat64(offset, Endian.little); - offset += 8; + final result = input.getFloat64(_offset, Endian.little); + _offset += 8; return result; } @@ -135,7 +149,7 @@ abstract class BinaryDeserializer { // number += BigInt.from(bytes[byteLength - i - 1]) << (8 * i); // little endian - number += BigInt.from(input.getUint8(offset + i)) << (8 * i); + number += BigInt.from(input.getUint8(_offset + i)) << (8 * i); } if (signed) {