From 6be8b54c99afbc649e1ec245f97bd999e7832352 Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Wed, 9 Aug 2023 13:03:55 +0200 Subject: [PATCH 01/25] fix that always 'TABLES - 1' where generated in test files --- lib/tests/files/generate_big.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tests/files/generate_big.py b/lib/tests/files/generate_big.py index c1ffeca..641436c 100644 --- a/lib/tests/files/generate_big.py +++ b/lib/tests/files/generate_big.py @@ -45,7 +45,7 @@ def write_table(file: IO[bytes], fields: int, number: int) -> None: tables = int(args["tables"]) fields = int(args["fields"]) - for i in range(1, tables): + for i in range(tables): write_table(file, fields, i); print(f"Created file: '{file_name}'") From afa8908c1024b6c8d5cc79b2ffd3a4a34d8d5481 Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Wed, 9 Aug 2023 13:05:30 +0200 Subject: [PATCH 02/25] generate test files and then bench with them, needs python3 in the path --- Cargo.lock | 602 ++++++++++++++++++++++++++++++++++ lib/Cargo.toml | 7 + lib/benches/parsing_tables.rs | 72 ++++ lib/tests/files/.gitignore | 3 +- 4 files changed, 683 insertions(+), 1 deletion(-) create mode 100644 lib/benches/parsing_tables.rs diff --git a/Cargo.lock b/Cargo.lock index c7d0a08..86ac6b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,18 +2,295 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aho-corasick" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anyhow" version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "ciborium" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" + +[[package]] +name = "ciborium-ll" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "bitflags", + "clap_lex", + "indexmap", + "textwrap", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "criterion" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +dependencies = [ + "anes", + "atty", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "js-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "log" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" + [[package]] name = "memchr" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -30,23 +307,237 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-traits" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.2", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + +[[package]] +name = "os_str_bytes" +version = "6.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" + [[package]] name = "pico-args" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315" +[[package]] +name = "plotters" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" + +[[package]] +name = "plotters-svg" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "proc-macro2" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "regex" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "syn" +version = "2.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tsql" version = "0.1.0" dependencies = [ "anyhow", + "criterion", "nom", "static_assertions", ] @@ -58,3 +549,114 @@ dependencies = [ "pico-args", "tsql", ] + +[[package]] +name = "unicode-ident" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" + +[[package]] +name = "walkdir" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + +[[package]] +name = "web-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 0471fa5..c710bfe 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -12,3 +12,10 @@ repository.workspace = true anyhow = "1.0.75" nom = "7.1.3" static_assertions = "1.1.0" + +[dev-dependencies] +criterion = { version = "0.4", features = ["html_reports"] } + +[[bench]] +name = "parsing_tables" +harness = false diff --git a/lib/benches/parsing_tables.rs b/lib/benches/parsing_tables.rs new file mode 100644 index 0000000..98f1442 --- /dev/null +++ b/lib/benches/parsing_tables.rs @@ -0,0 +1,72 @@ +use std::fmt::Display; +use std::path::PathBuf; +use std::process::Command; +use std::{env, fs}; + +use anyhow::Result; +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use tsql::parse_str; + +struct Arguments { + tables: usize, + fields: usize, +} + +impl Display for Arguments { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!( + "(tables: {}, fields: {})", + self.tables, self.fields + )) + } +} + +fn generate_test_file(tables: usize, fields: usize) -> Result<(Arguments, PathBuf)> { + let current_dir = env::current_dir()?; + let working_dir = current_dir.as_path().join("tests").join("files"); + + let file_name = format!("{}_{}.tsql", tables, fields); + + // TODO test for if the `python` in the path is compatible with `generate_big.py` + // TODO rewrite `generate_big.py` as a rust binary under `/cli` + let output = Command::new("python3") + .current_dir(working_dir.clone()) + .arg("generate_big.py") + .args(["--name", &file_name]) + .args(["--tables", &tables.to_string()]) + .args(["--fields", &fields.to_string()]) + .status()?; + + assert!(output.success()); + + Ok(( + Arguments { tables, fields }, + working_dir.join(file_name).to_path_buf(), + )) +} + +fn generate_test_files() -> Result> { + Ok(vec![ + generate_test_file(1, 256)?, + generate_test_file(16, 256)?, + generate_test_file(64, 256)?, + generate_test_file(1, 1024)?, + generate_test_file(16, 1024)?, + generate_test_file(64, 1024)?, + ]) +} + +fn criterion_benchmark(c: &mut Criterion) { + let file_paths = generate_test_files().unwrap(); + + for (args, path) in file_paths { + let content = fs::read_to_string(path).unwrap().replace('\n', ""); + + c.bench_function(&format!("{}", args), |b| { + b.iter(|| black_box(parse_str(&content))) + }); + } +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/lib/tests/files/.gitignore b/lib/tests/files/.gitignore index 8456ecd..c21699d 100644 --- a/lib/tests/files/.gitignore +++ b/lib/tests/files/.gitignore @@ -1 +1,2 @@ -big*.tsql \ No newline at end of file +big*.tsql +*_*.tsql \ No newline at end of file From c6cbb70394374a00cb53c26251363dc6ad7ed834 Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Wed, 9 Aug 2023 13:16:11 +0200 Subject: [PATCH 03/25] use custom config --- lib/benches/parsing_tables.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/benches/parsing_tables.rs b/lib/benches/parsing_tables.rs index 98f1442..51f367a 100644 --- a/lib/benches/parsing_tables.rs +++ b/lib/benches/parsing_tables.rs @@ -1,6 +1,7 @@ use std::fmt::Display; use std::path::PathBuf; use std::process::Command; +use std::time::Duration; use std::{env, fs}; use anyhow::Result; @@ -68,5 +69,9 @@ fn criterion_benchmark(c: &mut Criterion) { } } -criterion_group!(benches, criterion_benchmark); +criterion_group!( + name = benches; + config = Criterion::default().sample_size(250).measurement_time(Duration::from_secs(20)); + targets = criterion_benchmark +); criterion_main!(benches); From fbe13cd61964e770b42d07bd70a90d0bf316fd6e Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Wed, 9 Aug 2023 13:48:45 +0200 Subject: [PATCH 04/25] implement helper function to write a sql comment --- cli/src/helper.rs | 10 ++++++++++ cli/src/main.rs | 24 ++++++++++++------------ 2 files changed, 22 insertions(+), 12 deletions(-) create mode 100644 cli/src/helper.rs diff --git a/cli/src/helper.rs b/cli/src/helper.rs new file mode 100644 index 0000000..3c89a92 --- /dev/null +++ b/cli/src/helper.rs @@ -0,0 +1,10 @@ +use std::io::Write; + +pub const COMMENT_LINE: &'static str = "================================"; + +pub fn writeln_sql_comment>( + writer: &mut W, + content: S, +) -> std::io::Result<()> { + writeln!(writer, "-- {}", content.into()) +} diff --git a/cli/src/main.rs b/cli/src/main.rs index 1f192e1..765a9db 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -3,8 +3,11 @@ use std::io::{BufWriter, Write}; use std::path::PathBuf; use std::process::exit; +use helper::{writeln_sql_comment, COMMENT_LINE}; use tsql::{parse_file, TransformSQL}; +mod helper; + const HELP: &str = "\ tsql @@ -34,20 +37,17 @@ fn main() { let mut file = BufWriter::new(File::create(&args.out_path).unwrap()); - let out = (0..32).map(|_| "=").collect::>().join(""); - writeln!(file, "-- {}", &out).unwrap(); - writeln!( - file, - "-- Warning! This file has been generated with tsql. Keep in mind that manuel changes will be overridden." - ) - .unwrap(); - writeln!( - file, - "-- Executable tsql build with git commit {:?}", - env!("GIT_HASH") + writeln_sql_comment(&mut file, COMMENT_LINE).unwrap(); + writeln_sql_comment(&mut file, "Warning! This file has been generated with tsql. Keep in mind that manuel changes will be overridden.").unwrap(); + writeln_sql_comment( + &mut file, + format!( + "Executable tsql build with git commit {:?}", + env!("GIT_HASH") + ), ) .unwrap(); - writeln!(file, "-- {}", &out).unwrap(); + writeln_sql_comment(&mut file, COMMENT_LINE).unwrap(); for (_, table) in tables { table.transform(&mut file).unwrap(); From b20e6414f67dcb414402746f61d3e4a38a768660 Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Fri, 11 Aug 2023 11:40:37 +0200 Subject: [PATCH 05/25] replace python script 'generate_big.py' with rust binary 'generate_tsql'. Wanring: before calling 'cargo bench' you have to build 'generate_tsql' first --- Cargo.lock | 7 ++ cli/Cargo.toml | 5 ++ cli/bin/generate_tsql.rs | 129 ++++++++++++++++++++++++++++++++ lib/benches/parsing_tables.rs | 25 +++++-- lib/src/lib.rs | 5 ++ lib/src/parser/types.rs | 26 ++++++- lib/src/types.rs | 85 +++++++++++++++++++-- lib/tests/files/generate_big.py | 51 ------------- 8 files changed, 267 insertions(+), 66 deletions(-) create mode 100644 cli/bin/generate_tsql.rs delete mode 100644 lib/tests/files/generate_big.py diff --git a/Cargo.lock b/Cargo.lock index 86ac6b5..8fb3a54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -224,6 +224,12 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +[[package]] +name = "hmac-sha256" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3688e69b38018fec1557254f64c8dc2cc8ec502890182f395dbb0aa997aa5735" + [[package]] name = "indexmap" version = "1.9.3" @@ -546,6 +552,7 @@ dependencies = [ name = "tsql_cli" version = "0.1.0" dependencies = [ + "hmac-sha256", "pico-args", "tsql", ] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 7ecf42a..64b51c0 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -8,6 +8,11 @@ repository.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[[bin]] +name = "generate_tsql" +path = "./bin/generate_tsql.rs" + [dependencies] tsql = { version = "0.1.0", path = "../lib" } pico-args = "0.5.0" +hmac-sha256 = "1.1.7" diff --git a/cli/bin/generate_tsql.rs b/cli/bin/generate_tsql.rs new file mode 100644 index 0000000..fef2c6c --- /dev/null +++ b/cli/bin/generate_tsql.rs @@ -0,0 +1,129 @@ +use std::collections::HashMap; +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::process::exit; + +use hmac_sha256::HMAC; +use tsql::types::{DataType, Field, Table, TableExtra}; +use tsql::TransformTSQL; + +#[derive(Debug)] +struct AppArgs { + file_name: String, + tables: usize, + fields_per_table: usize, +} + +const HELP: &str = "\ +generate_tsql + +USAGE: + generate_tsql --name [FILE_NAME] --tables [TABLE_COUNT] --fields [FIELDS_COUNT] + +FLAGS: + -h, --help Prints help information + + --name Sets the name of the output file + + --tables How many tables should be generated + + --fields How many fields/rows per table should be generated. + Be aware that: + |rows| = TABLE_COUNT * FIELDS_COUNT +"; + +fn main() { + let args = match parse_args() { + Ok(v) => v, + Err(e) => { + eprintln!("Error: {}.", e); + exit(1); + } + }; + + let mut file = BufWriter::new(File::create(&args.file_name).unwrap()); + + for i in 0..args.tables { + let table = generate_table(i, args.fields_per_table); + + table.transform_tsql(&mut file).unwrap(); + } + + file.flush().unwrap(); +} + +fn parse_args() -> Result { + let mut pargs = pico_args::Arguments::from_env(); + + if pargs.contains(["-h", "--help"]) { + print!("{}", HELP); + exit(0); + } + + // TODO refactor/remove this "hack" + let args = AppArgs { + file_name: { + let _: String = pargs.free_from_str()?; + pargs.free_from_str()? + }, + tables: { + let _: String = pargs.free_from_str()?; + pargs.free_from_str()? + }, + fields_per_table: { + let _: String = pargs.free_from_str()?; + pargs.free_from_str()? + }, + }; + + let remaining = pargs.finish(); + if !remaining.is_empty() { + eprintln!("Warning: unused arguments left: {:?}.", remaining); + } + + Ok(args) +} + +fn number_to_string(number: usize) -> String { + let bytes = number.to_le_bytes(); + let h = HMAC::new(bytes); + let hash = h.finalize(); + + hash.to_vec() + .iter() + // TODO check if this maps all u8 values into ascii lowercase values + .map(|item| (item % 24 + 65) as char) + .collect::() +} + +fn generate_table(counter: usize, fields_per_table: usize) -> Table { + const DATATYPES: &[DataType] = &[ + DataType::Int, + DataType::Double, + DataType::VarChar(100), + DataType::Char(6), + DataType::Uuid, + ]; + + let name = number_to_string(counter); + + let mut fields = HashMap::new(); + + for i in 0..fields_per_table { + let field_name = number_to_string(i + 100 + counter); + let datatype = DATATYPES[i % DATATYPES.len()]; + + let field = Field::new(&field_name, datatype); + + fields.insert(field_name, field); + } + + // TODO check if `fields_per_table > 0` + let first_field_for_pk = fields.keys().next().unwrap().clone(); + + Table::new( + name, + fields, + TableExtra::new_with_pk(vec![first_field_for_pk]), + ) +} diff --git a/lib/benches/parsing_tables.rs b/lib/benches/parsing_tables.rs index 51f367a..719215c 100644 --- a/lib/benches/parsing_tables.rs +++ b/lib/benches/parsing_tables.rs @@ -24,25 +24,36 @@ impl Display for Arguments { fn generate_test_file(tables: usize, fields: usize) -> Result<(Arguments, PathBuf)> { let current_dir = env::current_dir()?; - let working_dir = current_dir.as_path().join("tests").join("files"); + // TODO remove this "hacky" way to get the target directory. + // Because I think this will fail with the user sets a custom directory for the `target` dir. + // Maybe some env variable has the absolute path to the dir. Or figure it out with a build script. + let target_dir = current_dir + .as_path() + .parent() + .unwrap() + .join("target") + .join("debug"); let file_name = format!("{}_{}.tsql", tables, fields); - // TODO test for if the `python` in the path is compatible with `generate_big.py` - // TODO rewrite `generate_big.py` as a rust binary under `/cli` - let output = Command::new("python3") - .current_dir(working_dir.clone()) - .arg("generate_big.py") + // TODO don't call the executable, call the functions directly + // TODO don't write to the file system, maybe it's possible that the child process writes to stdout and we capture stdout into a buffer. + // But with this approach I think we should restructure the benches a little bit. + // Now: generates files on fs -> loop: read file for bench + bench with the file -> jump back to loop + // Future: loop: generate dummy content + bench with content -> jump back to loop + let output = Command::new(&target_dir.join("generate_tsql")) + .current_dir(&target_dir) .args(["--name", &file_name]) .args(["--tables", &tables.to_string()]) .args(["--fields", &fields.to_string()]) .status()?; + // TODO throw custom error assert!(output.success()); Ok(( Arguments { tables, fields }, - working_dir.join(file_name).to_path_buf(), + target_dir.join(file_name).to_path_buf(), )) } diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 21cde83..f4e0b59 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -45,3 +45,8 @@ pub fn parse_file>(path: P) -> Result { pub trait TransformSQL { fn transform(&self, buffer: &mut W) -> Result<()>; } + +// TODO add docs +pub trait TransformTSQL { + fn transform_tsql(&self, buffer: &mut W) -> Result<()>; +} diff --git a/lib/src/parser/types.rs b/lib/src/parser/types.rs index ef62453..ba5e824 100644 --- a/lib/src/parser/types.rs +++ b/lib/src/parser/types.rs @@ -1,4 +1,9 @@ use std::collections::HashMap; +use std::io::Write; + +use anyhow::Result; + +use crate::TransformTSQL; #[derive(Debug)] pub struct RawTable { @@ -108,7 +113,26 @@ impl RawDataType { #[derive(Debug, Default)] pub struct TableExtra { - pub primary_key: Vec, + pub(crate) primary_key: Vec, +} + +impl TableExtra { + pub fn new_with_pk(primary_key: Vec) -> Self { + TableExtra { + primary_key, + ..Self::default() + } + } +} + +impl TransformTSQL for TableExtra { + fn transform_tsql(&self, buffer: &mut W) -> Result<()> { + if !self.primary_key.is_empty() { + writeln!(buffer, "@primary_key({})", self.primary_key.join(", "))?; + } + + Ok(()) + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/lib/src/types.rs b/lib/src/types.rs index 18dd3c2..bea3fd0 100644 --- a/lib/src/types.rs +++ b/lib/src/types.rs @@ -6,8 +6,11 @@ use std::rc::Rc; use anyhow::{bail, Result}; use static_assertions::const_assert_eq; -use crate::parser::types::{FieldExtra, FieldType, RawDataType, RawField, RawTable, TableExtra}; -use crate::TransformSQL; +use crate::parser::types::{FieldExtra, FieldType, RawDataType, RawField, RawTable}; +use crate::{TransformSQL, TransformTSQL}; + +// TODO move `TableExtra` from `crate::parser::types` to here +pub type TableExtra = crate::parser::types::TableExtra; pub type GenericCollection = BTreeMap; pub type TableCollection = GenericCollection; @@ -20,6 +23,7 @@ fn get_first_element(collection: &BTreeMap) -> Option<(&K, &V)> Some((key, item)) } +// TODO remove `pub(crate)` #[derive(Debug, Default)] pub struct Table { pub(crate) extra: TableExtra, @@ -30,6 +34,18 @@ pub struct Table { } impl Table { + pub fn new>( + name: S, + fields: HashMap, + extra: TableExtra, + ) -> Self { + Table { + extra, + name: name.into(), + fields, + } + } + pub fn get_field(&self, key: &str) -> Option<&Field> { self.fields.get(key) } @@ -210,6 +226,22 @@ impl TransformSQL for Table { } } +impl TransformTSQL for Table { + fn transform_tsql(&self, buffer: &mut W) -> Result<()> { + self.extra.transform_tsql(buffer)?; + + writeln!(buffer, "table {} {{", self.name)?; + + for field in self.fields.values() { + field.transform_tsql(buffer)?; + } + + writeln!(buffer, "}};")?; + + Ok(()) + } +} + #[derive(Debug, Clone)] pub struct Field { pub(crate) name: String, @@ -219,6 +251,22 @@ pub struct Field { } impl Field { + pub fn new>(name: S, datatype: DataType) -> Field { + Self::new_with_fk(name, datatype, None) + } + + pub fn new_with_fk>( + name: S, + datatype: DataType, + foreign_key_reference: Option<(String, Rc)>, + ) -> Field { + Field { + name: name.into(), + datatype, + foreign_key_reference, + } + } + fn parse(raw: &RawField) -> Result { Ok(Field { name: raw.name.to_string(), @@ -241,6 +289,16 @@ impl TransformSQL for Field { } } +impl TransformTSQL for Field { + fn transform_tsql(&self, buffer: &mut W) -> Result<()> { + write!(buffer, "\t")?; + self.datatype.transform_tsql(buffer)?; + writeln!(buffer, " {},", self.name)?; + + Ok(()) + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum DataType { Int, @@ -290,11 +348,10 @@ impl DataType { } } } -} -impl TransformSQL for DataType { - fn transform(&self, buffer: &mut W) -> Result<()> { - let formatted = match self { + // TODO maybe replace with `&'a str` if possible? + fn format(&self) -> String { + match self { DataType::Int => "int".to_string(), DataType::Bool => "boolean".to_string(), DataType::BigInt => "bigint".to_string(), @@ -310,10 +367,24 @@ impl TransformSQL for DataType { DataType::Text(args) => format!("text({})", args), DataType::Decimal(precision, scale) => format!("decimal({}, {})", precision, scale), - }; + } + } +} + +impl TransformSQL for DataType { + fn transform(&self, buffer: &mut W) -> Result<()> { + let formatted = self.format(); writeln!(buffer, "{},", formatted)?; Ok(()) } } + +impl TransformTSQL for DataType { + fn transform_tsql(&self, buffer: &mut W) -> Result<()> { + write!(buffer, "{}", self.format())?; + + Ok(()) + } +} diff --git a/lib/tests/files/generate_big.py b/lib/tests/files/generate_big.py deleted file mode 100644 index 641436c..0000000 --- a/lib/tests/files/generate_big.py +++ /dev/null @@ -1,51 +0,0 @@ -import hashlib -import sys -from typing import IO - -def number_to_string(number: int) -> str: - byte_representation = number.to_bytes((number.bit_length() + 7) // 8, 'big') - - hash_object = hashlib.sha256(byte_representation) - hash_value = hash_object.digest() - - letters = [chr((byte % 26) + ord('A')) for byte in hash_value] - hash_string = ''.join(letters) - - return hash_string - -def write_table(file: IO[bytes], fields: int, number: int) -> None: - datatypes = ["int", "double", "varchar(100)", "char(6)", "uuid"] - - fields = [f"\t{datatypes[i % len(datatypes)]} {number_to_string(i + 100)},\n" for i in range(fields)] - - lines = ["@primary_key(id)\n", f"table {number_to_string(number)} " + "{\n", "\tint id,\n" ] - lines.extend(fields) - lines.extend(["};\n"]) - - file.writelines(lines) - -if __name__ == "__main__": - args = sys.argv[1:] - - if len(args) != 6: - HELP = """ - python3 generate_big.py - - USAGE: - python3 generate_big.py --name FILE_NAME --tables TABLE_COUNT --fields FIELDS_COUNT - """ - print(HELP) - exit(1) - - args = {args[i].replace("--", ""): args[i + 1] for i in range(0, len(args), 2)} - - file_name = args["name"] - - with open(file_name, "w") as file: - tables = int(args["tables"]) - fields = int(args["fields"]) - - for i in range(tables): - write_table(file, fields, i); - - print(f"Created file: '{file_name}'") From d83d93eb813d06556827c148a67dee7ef1c2fa9b Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Fri, 11 Aug 2023 14:30:20 +0200 Subject: [PATCH 06/25] refactor logic to parse cli arguments --- cli/bin/generate_tsql.rs | 67 ++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/cli/bin/generate_tsql.rs b/cli/bin/generate_tsql.rs index fef2c6c..3cfc5e6 100644 --- a/cli/bin/generate_tsql.rs +++ b/cli/bin/generate_tsql.rs @@ -7,7 +7,7 @@ use hmac_sha256::HMAC; use tsql::types::{DataType, Field, Table, TableExtra}; use tsql::TransformTSQL; -#[derive(Debug)] +#[derive(Debug, Default)] struct AppArgs { file_name: String, tables: usize, @@ -55,32 +55,61 @@ fn main() { fn parse_args() -> Result { let mut pargs = pico_args::Arguments::from_env(); + // TODO add this logic into the `loop { ... }` if pargs.contains(["-h", "--help"]) { print!("{}", HELP); exit(0); } - // TODO refactor/remove this "hack" - let args = AppArgs { - file_name: { - let _: String = pargs.free_from_str()?; - pargs.free_from_str()? - }, - tables: { - let _: String = pargs.free_from_str()?; - pargs.free_from_str()? - }, - fields_per_table: { - let _: String = pargs.free_from_str()?; - pargs.free_from_str()? - }, - }; + // TODO change to builder pattern + let mut args = AppArgs::default(); + + loop { + let argument = pargs.free_from_str(); + + if let Err(err) = argument { + match err { + // The error `pico_args::Error::MissingArgument` should be thrown when we + // don't have any arguments left to parse. + // So it's save to ignore the error and break out of the loop. + pico_args::Error::MissingArgument => break, + // Some error occurred while parsing, so we just pass the error up. + _ => return Err(err), + }; + } + + // An argument like '--name' or '--fields' always has the prefix '--'. + // So we check if the argument starts with '--'. + // If not we _currently_ panic. In the future this if clause should return a custom error. + if argument + .as_ref() + .map(|item: &String| !item.starts_with("--")) + .unwrap_or(false) + { + // TODO return custom error + panic!("Arguments have to start with '--' but got {:?}", argument); + } - let remaining = pargs.finish(); - if !remaining.is_empty() { - eprintln!("Warning: unused arguments left: {:?}.", remaining); + // At this time in the program we know that argument is `Ok(_)` so we can unwrap save + // and we can replace the prefix with nothing. + // Makes it cleaner in the next match statement. + let argument: String = argument.unwrap().replace("--", ""); + + // The arguments `generate_tsql` accepts and sets the corresponding value into the struct AppArgs. + // TODO replace `args.[FIELD] = sth` with a builder pattern that can also check if we supplied + // it with enough arguments, e.g.: file_name + tables + fields + match argument.as_str() { + "name" => args.file_name = pargs.free_from_str()?, + "tables" => args.tables = pargs.free_from_str()?, + "fields" => args.fields_per_table = pargs.free_from_str()?, + // TODO implement custom error + _ => panic!("Unknown argument: {}", argument), + } } + // TODO implement custom error, but `pargs.finish()` _should_ always return a empty vec... in theory + assert!(pargs.finish().is_empty()); + Ok(args) } From 55536d4738b101087f335542356c4b3a099ec22b Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Fri, 11 Aug 2023 16:46:57 +0200 Subject: [PATCH 07/25] add argument '--stdout' to write output to stdout instead saving to a file --- cli/bin/generate_tsql.rs | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/cli/bin/generate_tsql.rs b/cli/bin/generate_tsql.rs index 3cfc5e6..e30b8c3 100644 --- a/cli/bin/generate_tsql.rs +++ b/cli/bin/generate_tsql.rs @@ -7,9 +7,16 @@ use hmac_sha256::HMAC; use tsql::types::{DataType, Field, Table, TableExtra}; use tsql::TransformTSQL; +#[derive(Debug, Default)] +enum Output { + File(String), + #[default] + Stdout, +} + #[derive(Debug, Default)] struct AppArgs { - file_name: String, + output: Output, tables: usize, fields_per_table: usize, } @@ -23,7 +30,9 @@ USAGE: FLAGS: -h, --help Prints help information - --name Sets the name of the output file + --name Saves the generated content into the given file + + --stdout Print the output to the stdout --tables How many tables should be generated @@ -41,15 +50,18 @@ fn main() { } }; - let mut file = BufWriter::new(File::create(&args.file_name).unwrap()); + let mut buffer_to_write = match &args.output { + Output::File(path) => Box::new(File::create(path).unwrap()) as Box, + Output::Stdout => Box::new(std::io::stdout()) as Box, + }; for i in 0..args.tables { let table = generate_table(i, args.fields_per_table); - table.transform_tsql(&mut file).unwrap(); + table.transform_tsql(&mut buffer_to_write).unwrap(); } - file.flush().unwrap(); + buffer_to_write.flush().unwrap(); } fn parse_args() -> Result { @@ -99,7 +111,8 @@ fn parse_args() -> Result { // TODO replace `args.[FIELD] = sth` with a builder pattern that can also check if we supplied // it with enough arguments, e.g.: file_name + tables + fields match argument.as_str() { - "name" => args.file_name = pargs.free_from_str()?, + "name" => args.output = Output::File(pargs.free_from_str()?), + "stdout" => args.output = Output::Stdout, "tables" => args.tables = pargs.free_from_str()?, "fields" => args.fields_per_table = pargs.free_from_str()?, // TODO implement custom error From 8d7fe800c82d6ac8ead1a8f8a4f39c820bdd8b78 Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Fri, 11 Aug 2023 17:02:42 +0200 Subject: [PATCH 08/25] use stdout option from generate_tsql binary --- lib/benches/parsing_tables.rs | 62 +++++++++++++++-------------------- 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/lib/benches/parsing_tables.rs b/lib/benches/parsing_tables.rs index 719215c..09dee46 100644 --- a/lib/benches/parsing_tables.rs +++ b/lib/benches/parsing_tables.rs @@ -1,8 +1,7 @@ +use std::env; use std::fmt::Display; -use std::path::PathBuf; -use std::process::Command; +use std::process::{Command, Stdio}; use std::time::Duration; -use std::{env, fs}; use anyhow::Result; use criterion::{black_box, criterion_group, criterion_main, Criterion}; @@ -13,6 +12,12 @@ struct Arguments { fields: usize, } +impl Arguments { + fn new(tables: usize, fields: usize) -> Self { + Arguments { tables, fields } + } +} + impl Display for Arguments { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_fmt(format_args!( @@ -22,7 +27,7 @@ impl Display for Arguments { } } -fn generate_test_file(tables: usize, fields: usize) -> Result<(Arguments, PathBuf)> { +fn generate_test_file(arguments: &Arguments) -> Result { let current_dir = env::current_dir()?; // TODO remove this "hacky" way to get the target directory. // Because I think this will fail with the user sets a custom directory for the `target` dir. @@ -34,47 +39,32 @@ fn generate_test_file(tables: usize, fields: usize) -> Result<(Arguments, PathBu .join("target") .join("debug"); - let file_name = format!("{}_{}.tsql", tables, fields); - // TODO don't call the executable, call the functions directly - // TODO don't write to the file system, maybe it's possible that the child process writes to stdout and we capture stdout into a buffer. - // But with this approach I think we should restructure the benches a little bit. - // Now: generates files on fs -> loop: read file for bench + bench with the file -> jump back to loop - // Future: loop: generate dummy content + bench with content -> jump back to loop let output = Command::new(&target_dir.join("generate_tsql")) .current_dir(&target_dir) - .args(["--name", &file_name]) - .args(["--tables", &tables.to_string()]) - .args(["--fields", &fields.to_string()]) - .status()?; - - // TODO throw custom error - assert!(output.success()); - - Ok(( - Arguments { tables, fields }, - target_dir.join(file_name).to_path_buf(), - )) -} + .arg("--stdout") + .args(["--tables", &arguments.tables.to_string()]) + .args(["--fields", &arguments.fields.to_string()]) + .stdout(Stdio::piped()) + .output()?; -fn generate_test_files() -> Result> { - Ok(vec![ - generate_test_file(1, 256)?, - generate_test_file(16, 256)?, - generate_test_file(64, 256)?, - generate_test_file(1, 1024)?, - generate_test_file(16, 1024)?, - generate_test_file(64, 1024)?, - ]) + Ok(String::from_utf8(output.stdout)?) } fn criterion_benchmark(c: &mut Criterion) { - let file_paths = generate_test_files().unwrap(); + let arguments = [ + Arguments::new(1, 256), + Arguments::new(16, 256), + Arguments::new(64, 256), + Arguments::new(1, 1024), + Arguments::new(16, 1024), + Arguments::new(64, 1024), + ]; - for (args, path) in file_paths { - let content = fs::read_to_string(path).unwrap().replace('\n', ""); + for argument in arguments { + let content = generate_test_file(&argument).unwrap(); - c.bench_function(&format!("{}", args), |b| { + c.bench_function(&format!("{}", argument), |b| { b.iter(|| black_box(parse_str(&content))) }); } From 14b0a02649bb85a7fc53fa1effa22aaaa5943878 Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Fri, 11 Aug 2023 23:01:40 +0200 Subject: [PATCH 09/25] use builder pattern to parse all the args --- cli/bin/generate_tsql.rs | 78 +++++++++++++++++++++++++++++++++------- 1 file changed, 66 insertions(+), 12 deletions(-) diff --git a/cli/bin/generate_tsql.rs b/cli/bin/generate_tsql.rs index e30b8c3..6b2577b 100644 --- a/cli/bin/generate_tsql.rs +++ b/cli/bin/generate_tsql.rs @@ -1,26 +1,80 @@ use std::collections::HashMap; use std::fs::File; -use std::io::{BufWriter, Write}; +use std::io::Write; use std::process::exit; use hmac_sha256::HMAC; use tsql::types::{DataType, Field, Table, TableExtra}; use tsql::TransformTSQL; -#[derive(Debug, Default)] +#[derive(Debug)] enum Output { - File(String), - #[default] Stdout, + File(String), } -#[derive(Debug, Default)] +#[derive(Debug)] struct AppArgs { output: Output, tables: usize, fields_per_table: usize, } +#[derive(Debug)] +struct AppArgsBuilder { + finish: usize, + + data_output: Output, + data_tables: usize, + data_fields_per_table: usize, +} + +impl AppArgsBuilder { + const DATA_OUTPUT: usize = 0x01 << 0; + const DATA_TABLES: usize = 0x01 << 1; + const DATA_FIELDS_PER_TABLE: usize = 0x01 << 2; + + fn new() -> Self { + AppArgsBuilder { + finish: 0, + data_output: Output::Stdout, + data_tables: Default::default(), + data_fields_per_table: Default::default(), + } + } + + fn build(self) -> Option { + if !self.is_finish() { + return None; + } + + Some(AppArgs { + output: self.data_output, + tables: self.data_tables, + fields_per_table: self.data_fields_per_table, + }) + } + + fn is_finish(&self) -> bool { + self.finish == (Self::DATA_OUTPUT | Self::DATA_TABLES | Self::DATA_FIELDS_PER_TABLE) + } + + fn set_output(&mut self, output: Output) { + self.data_output = output; + self.finish |= Self::DATA_OUTPUT; + } + + fn set_tables(&mut self, tables: usize) { + self.data_tables = tables; + self.finish |= Self::DATA_TABLES; + } + + fn set_fields_per_table(&mut self, fields_per_table: usize) { + self.data_fields_per_table = fields_per_table; + self.finish |= Self::DATA_FIELDS_PER_TABLE; + } +} + const HELP: &str = "\ generate_tsql @@ -73,8 +127,7 @@ fn parse_args() -> Result { exit(0); } - // TODO change to builder pattern - let mut args = AppArgs::default(); + let mut args = AppArgsBuilder::new(); loop { let argument = pargs.free_from_str(); @@ -111,10 +164,10 @@ fn parse_args() -> Result { // TODO replace `args.[FIELD] = sth` with a builder pattern that can also check if we supplied // it with enough arguments, e.g.: file_name + tables + fields match argument.as_str() { - "name" => args.output = Output::File(pargs.free_from_str()?), - "stdout" => args.output = Output::Stdout, - "tables" => args.tables = pargs.free_from_str()?, - "fields" => args.fields_per_table = pargs.free_from_str()?, + "name" => args.set_output(Output::File(pargs.free_from_str()?)), + "stdout" => args.set_output(Output::Stdout), + "tables" => args.set_tables(pargs.free_from_str()?), + "fields" => args.set_fields_per_table(pargs.free_from_str()?), // TODO implement custom error _ => panic!("Unknown argument: {}", argument), } @@ -123,7 +176,8 @@ fn parse_args() -> Result { // TODO implement custom error, but `pargs.finish()` _should_ always return a empty vec... in theory assert!(pargs.finish().is_empty()); - Ok(args) + // TODO implement custom error + Ok(args.build().unwrap()) } fn number_to_string(number: usize) -> String { From 4b62fd7134fe3db259e971624d7c84c4c47121b4 Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Fri, 11 Aug 2023 23:08:36 +0200 Subject: [PATCH 10/25] replace cryptic with 'write!' macro --- lib/benches/parsing_tables.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/benches/parsing_tables.rs b/lib/benches/parsing_tables.rs index 09dee46..7c5e800 100644 --- a/lib/benches/parsing_tables.rs +++ b/lib/benches/parsing_tables.rs @@ -20,10 +20,7 @@ impl Arguments { impl Display for Arguments { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_fmt(format_args!( - "(tables: {}, fields: {})", - self.tables, self.fields - )) + write!(f, "(tables: {}, fields: {})", self.tables, self.fields) } } From b733123d04ab7d9df34ebc7f0e65010ba31f4029 Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Fri, 11 Aug 2023 23:10:27 +0200 Subject: [PATCH 11/25] rename 'TransformTSQL::transform_tsql' with 'TransformTSQL::transform_into_tsql' --- cli/bin/generate_tsql.rs | 2 +- lib/src/lib.rs | 2 +- lib/src/parser/types.rs | 2 +- lib/src/types.rs | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cli/bin/generate_tsql.rs b/cli/bin/generate_tsql.rs index 6b2577b..4f1e247 100644 --- a/cli/bin/generate_tsql.rs +++ b/cli/bin/generate_tsql.rs @@ -112,7 +112,7 @@ fn main() { for i in 0..args.tables { let table = generate_table(i, args.fields_per_table); - table.transform_tsql(&mut buffer_to_write).unwrap(); + table.transform_into_tsql(&mut buffer_to_write).unwrap(); } buffer_to_write.flush().unwrap(); diff --git a/lib/src/lib.rs b/lib/src/lib.rs index f4e0b59..838f85e 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -48,5 +48,5 @@ pub trait TransformSQL { // TODO add docs pub trait TransformTSQL { - fn transform_tsql(&self, buffer: &mut W) -> Result<()>; + fn transform_into_tsql(&self, buffer: &mut W) -> Result<()>; } diff --git a/lib/src/parser/types.rs b/lib/src/parser/types.rs index ba5e824..50d6de1 100644 --- a/lib/src/parser/types.rs +++ b/lib/src/parser/types.rs @@ -126,7 +126,7 @@ impl TableExtra { } impl TransformTSQL for TableExtra { - fn transform_tsql(&self, buffer: &mut W) -> Result<()> { + fn transform_into_tsql(&self, buffer: &mut W) -> Result<()> { if !self.primary_key.is_empty() { writeln!(buffer, "@primary_key({})", self.primary_key.join(", "))?; } diff --git a/lib/src/types.rs b/lib/src/types.rs index bea3fd0..e473a65 100644 --- a/lib/src/types.rs +++ b/lib/src/types.rs @@ -227,13 +227,13 @@ impl TransformSQL for Table { } impl TransformTSQL for Table { - fn transform_tsql(&self, buffer: &mut W) -> Result<()> { - self.extra.transform_tsql(buffer)?; + fn transform_into_tsql(&self, buffer: &mut W) -> Result<()> { + self.extra.transform_into_tsql(buffer)?; writeln!(buffer, "table {} {{", self.name)?; for field in self.fields.values() { - field.transform_tsql(buffer)?; + field.transform_into_tsql(buffer)?; } writeln!(buffer, "}};")?; @@ -290,9 +290,9 @@ impl TransformSQL for Field { } impl TransformTSQL for Field { - fn transform_tsql(&self, buffer: &mut W) -> Result<()> { + fn transform_into_tsql(&self, buffer: &mut W) -> Result<()> { write!(buffer, "\t")?; - self.datatype.transform_tsql(buffer)?; + self.datatype.transform_into_tsql(buffer)?; writeln!(buffer, " {},", self.name)?; Ok(()) @@ -382,7 +382,7 @@ impl TransformSQL for DataType { } impl TransformTSQL for DataType { - fn transform_tsql(&self, buffer: &mut W) -> Result<()> { + fn transform_into_tsql(&self, buffer: &mut W) -> Result<()> { write!(buffer, "{}", self.format())?; Ok(()) From a471c3ce5d9982d7a5744194d13f0a533547b3ba Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Fri, 11 Aug 2023 23:12:36 +0200 Subject: [PATCH 12/25] rename 'TransformSQL::transform' with 'TransformSQL::transform_into_sql' --- cli/src/main.rs | 2 +- lib/src/lib.rs | 3 ++- lib/src/types.rs | 10 +++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 765a9db..021b9a0 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -50,7 +50,7 @@ fn main() { writeln_sql_comment(&mut file, COMMENT_LINE).unwrap(); for (_, table) in tables { - table.transform(&mut file).unwrap(); + table.transform_into_sql(&mut file).unwrap(); } file.flush().unwrap(); } diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 838f85e..356fa39 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -42,8 +42,9 @@ pub fn parse_file>(path: P) -> Result { parse_str(&content) } +// TODO add docs pub trait TransformSQL { - fn transform(&self, buffer: &mut W) -> Result<()>; + fn transform_into_sql(&self, buffer: &mut W) -> Result<()>; } // TODO add docs diff --git a/lib/src/types.rs b/lib/src/types.rs index e473a65..e0b848f 100644 --- a/lib/src/types.rs +++ b/lib/src/types.rs @@ -175,13 +175,13 @@ impl Table { } impl TransformSQL for Table { - fn transform(&self, buffer: &mut W) -> Result<()> { + fn transform_into_sql(&self, buffer: &mut W) -> Result<()> { writeln!(buffer, "CREATE TABLE {} (", self.name)?; let mut foreign_keys_table_fields: HashMap> = HashMap::new(); for field in self.fields.values() { - field.transform(buffer)?; + field.transform_into_sql(buffer)?; if field.foreign_key_reference.is_some() { let table = &field.foreign_key_reference.as_ref().unwrap().0; @@ -281,9 +281,9 @@ impl Field { } impl TransformSQL for Field { - fn transform(&self, buffer: &mut W) -> Result<()> { + fn transform_into_sql(&self, buffer: &mut W) -> Result<()> { write!(buffer, "{} ", self.name)?; - self.datatype.transform(buffer)?; + self.datatype.transform_into_sql(buffer)?; Ok(()) } @@ -372,7 +372,7 @@ impl DataType { } impl TransformSQL for DataType { - fn transform(&self, buffer: &mut W) -> Result<()> { + fn transform_into_sql(&self, buffer: &mut W) -> Result<()> { let formatted = self.format(); writeln!(buffer, "{},", formatted)?; From dbbdffa25ddedef8c0662d60ab98340e08327dfe Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Fri, 11 Aug 2023 23:29:12 +0200 Subject: [PATCH 13/25] move 'tsql::parser::types::TableExtra' to 'tsql::types::TableExtra' --- lib/src/lib.rs | 19 ++++++++++++++----- lib/src/parser/mod.rs | 5 +++-- lib/src/parser/types.rs | 29 +---------------------------- lib/src/types.rs | 39 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 54 insertions(+), 38 deletions(-) diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 356fa39..f25fe26 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -1,22 +1,22 @@ #![feature(variant_count)] use std::cell::RefCell; -use std::collections::BTreeMap; use std::fs::read_to_string; use std::io::Write; use std::path::Path; use std::rc::Rc; use anyhow::{bail, Result}; -use types::{Table, TableCollection}; +use types::{RawTableCollection, Table, TableCollection}; use crate::parser::parse; mod parser; pub mod types; +/// Starts the parsing process with the given `&str`. pub fn parse_str(mut content: &str) -> Result { - let mut raw_tables = BTreeMap::new(); + let mut raw_tables = RawTableCollection::new(); while !content.is_empty() { let out = parse(content); @@ -35,6 +35,9 @@ pub fn parse_str(mut content: &str) -> Result { Table::parse_raw_tables(raw_tables) } +/// Starts the parsing process with a path as argument by reading the whole file into memory. +/// +/// For more info see: [`parse_str`] pub fn parse_file>(path: P) -> Result { // TODO check if the `.replace(...)` is necessary let content = read_to_string(path)?.replace('\n', ""); @@ -42,12 +45,18 @@ pub fn parse_file>(path: P) -> Result { parse_str(&content) } -// TODO add docs +/// Trait with methods to transform a struct into `sql`-code. pub trait TransformSQL { + /// Transforms the struct into `sql`-code. + /// + /// Writes the output to a generic buffer, which implements [`Write`]. fn transform_into_sql(&self, buffer: &mut W) -> Result<()>; } -// TODO add docs +/// Trait with methods to transform a struct into `tsql`-code. pub trait TransformTSQL { + /// Transforms the struct into `tsql`-code. + /// + /// Writes the output to a generic buffer, which implements [`Write`]. fn transform_into_tsql(&self, buffer: &mut W) -> Result<()>; } diff --git a/lib/src/parser/mod.rs b/lib/src/parser/mod.rs index c43301e..089f2c2 100644 --- a/lib/src/parser/mod.rs +++ b/lib/src/parser/mod.rs @@ -9,7 +9,8 @@ pub mod types; use crate::parser::helper::preceded_space_get_word; use crate::parser::parser::{parse_table_body, parse_table_extra, parse_table_fields}; -use crate::parser::types::*; +use crate::parser::types::{FieldType, RawDataType, RawField, RawTable, TagHelper}; +use crate::types::TableExtra; pub fn parse(input: &str) -> IResult<&str, RawTable> { let (input, extra) = table_extra(input)?; @@ -42,7 +43,7 @@ fn table_extra(input: &str) -> IResult<&str, TableExtra> { let mut table_extra = TableExtra::default(); match item { - Some((TagHelper::PrimaryKey, values)) => table_extra.primary_key.append( + Some((TagHelper::PrimaryKey, values)) => table_extra.primary_key_mut().append( &mut values .iter() .map(|item| item.to_string()) diff --git a/lib/src/parser/types.rs b/lib/src/parser/types.rs index 50d6de1..49d0c42 100644 --- a/lib/src/parser/types.rs +++ b/lib/src/parser/types.rs @@ -1,9 +1,6 @@ use std::collections::HashMap; -use std::io::Write; -use anyhow::Result; - -use crate::TransformTSQL; +use crate::types::TableExtra; #[derive(Debug)] pub struct RawTable { @@ -111,30 +108,6 @@ impl RawDataType { } } -#[derive(Debug, Default)] -pub struct TableExtra { - pub(crate) primary_key: Vec, -} - -impl TableExtra { - pub fn new_with_pk(primary_key: Vec) -> Self { - TableExtra { - primary_key, - ..Self::default() - } - } -} - -impl TransformTSQL for TableExtra { - fn transform_into_tsql(&self, buffer: &mut W) -> Result<()> { - if !self.primary_key.is_empty() { - writeln!(buffer, "@primary_key({})", self.primary_key.join(", "))?; - } - - Ok(()) - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FieldExtra { ForeignKey, diff --git a/lib/src/types.rs b/lib/src/types.rs index e0b848f..5b756d9 100644 --- a/lib/src/types.rs +++ b/lib/src/types.rs @@ -9,11 +9,11 @@ use static_assertions::const_assert_eq; use crate::parser::types::{FieldExtra, FieldType, RawDataType, RawField, RawTable}; use crate::{TransformSQL, TransformTSQL}; -// TODO move `TableExtra` from `crate::parser::types` to here -pub type TableExtra = crate::parser::types::TableExtra; - +/// A `BTreeMap` where the key is always of type `String` and the value type is generic `T` pub type GenericCollection = BTreeMap; +/// A `BTreeMap` with the type of a key is a `String` and the type of the value is a [`Table`] pub type TableCollection = GenericCollection
; +/// A `BTreeMap` with the type of a key is a `String` and the type of the value is a [`Rc>`] pub(crate) type RawTableCollection = GenericCollection>>; fn get_first_element(collection: &BTreeMap) -> Option<(&K, &V)> { @@ -388,3 +388,36 @@ impl TransformTSQL for DataType { Ok(()) } } + +/// Holds metadata for a [`Table`] +#[derive(Debug, Default)] +pub struct TableExtra { + primary_key: Vec, +} + +impl TableExtra { + pub fn new_with_pk(primary_key: Vec) -> Self { + TableExtra { + primary_key, + ..Self::default() + } + } + + pub fn primary_key(&self) -> &Vec { + &self.primary_key + } + + pub fn primary_key_mut(&mut self) -> &mut Vec { + &mut self.primary_key + } +} + +impl TransformTSQL for TableExtra { + fn transform_into_tsql(&self, buffer: &mut W) -> Result<()> { + if !self.primary_key.is_empty() { + writeln!(buffer, "@primary_key({})", self.primary_key.join(", "))?; + } + + Ok(()) + } +} From 979e0d98635556a4a51780307f4c91eba41f69fc Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Mon, 14 Aug 2023 17:43:38 +0200 Subject: [PATCH 14/25] remove outdated TODO --- cli/bin/generate_tsql.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/cli/bin/generate_tsql.rs b/cli/bin/generate_tsql.rs index 4f1e247..ac31bc7 100644 --- a/cli/bin/generate_tsql.rs +++ b/cli/bin/generate_tsql.rs @@ -161,8 +161,6 @@ fn parse_args() -> Result { let argument: String = argument.unwrap().replace("--", ""); // The arguments `generate_tsql` accepts and sets the corresponding value into the struct AppArgs. - // TODO replace `args.[FIELD] = sth` with a builder pattern that can also check if we supplied - // it with enough arguments, e.g.: file_name + tables + fields match argument.as_str() { "name" => args.set_output(Output::File(pargs.free_from_str()?)), "stdout" => args.set_output(Output::Stdout), From 18163af15ab8f889d5cf9b2a568484227bc82a15 Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Mon, 14 Aug 2023 18:06:23 +0200 Subject: [PATCH 15/25] use 'wrapping_{add, mul}' just in case --- cli/bin/generate_tsql.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/bin/generate_tsql.rs b/cli/bin/generate_tsql.rs index ac31bc7..c2297df 100644 --- a/cli/bin/generate_tsql.rs +++ b/cli/bin/generate_tsql.rs @@ -204,8 +204,8 @@ fn generate_table(counter: usize, fields_per_table: usize) -> Table { let mut fields = HashMap::new(); for i in 0..fields_per_table { - let field_name = number_to_string(i + 100 + counter); - let datatype = DATATYPES[i % DATATYPES.len()]; + let field_name = number_to_string(i.wrapping_add(counter.wrapping_mul(100))); + let datatype = DATATYPES[i.wrapping_add(counter) % DATATYPES.len()]; let field = Field::new(&field_name, datatype); From e1aac5cabeee48920f5af6574f155a3b5822cad6 Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Mon, 14 Aug 2023 18:51:45 +0200 Subject: [PATCH 16/25] move dummy generation of tables into tsql lib --- Cargo.lock | 2 +- cli/Cargo.toml | 3 +-- cli/bin/generate_tsql.rs | 48 +--------------------------------------- lib/Cargo.toml | 5 +++++ lib/src/generate.rs | 44 ++++++++++++++++++++++++++++++++++++ lib/src/lib.rs | 5 +++++ lib/src/types.rs | 28 +++++++++++++++++++++++ 7 files changed, 85 insertions(+), 50 deletions(-) create mode 100644 lib/src/generate.rs diff --git a/Cargo.lock b/Cargo.lock index 8fb3a54..1917ebf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -544,6 +544,7 @@ version = "0.1.0" dependencies = [ "anyhow", "criterion", + "hmac-sha256", "nom", "static_assertions", ] @@ -552,7 +553,6 @@ dependencies = [ name = "tsql_cli" version = "0.1.0" dependencies = [ - "hmac-sha256", "pico-args", "tsql", ] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 64b51c0..1451b84 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -13,6 +13,5 @@ name = "generate_tsql" path = "./bin/generate_tsql.rs" [dependencies] -tsql = { version = "0.1.0", path = "../lib" } +tsql = { version = "0.1.0", path = "../lib", features = ["generate"] } pico-args = "0.5.0" -hmac-sha256 = "1.1.7" diff --git a/cli/bin/generate_tsql.rs b/cli/bin/generate_tsql.rs index c2297df..eb4c8ac 100644 --- a/cli/bin/generate_tsql.rs +++ b/cli/bin/generate_tsql.rs @@ -1,10 +1,8 @@ -use std::collections::HashMap; use std::fs::File; use std::io::Write; use std::process::exit; -use hmac_sha256::HMAC; -use tsql::types::{DataType, Field, Table, TableExtra}; +use tsql::generate::generate_table; use tsql::TransformTSQL; #[derive(Debug)] @@ -177,47 +175,3 @@ fn parse_args() -> Result { // TODO implement custom error Ok(args.build().unwrap()) } - -fn number_to_string(number: usize) -> String { - let bytes = number.to_le_bytes(); - let h = HMAC::new(bytes); - let hash = h.finalize(); - - hash.to_vec() - .iter() - // TODO check if this maps all u8 values into ascii lowercase values - .map(|item| (item % 24 + 65) as char) - .collect::() -} - -fn generate_table(counter: usize, fields_per_table: usize) -> Table { - const DATATYPES: &[DataType] = &[ - DataType::Int, - DataType::Double, - DataType::VarChar(100), - DataType::Char(6), - DataType::Uuid, - ]; - - let name = number_to_string(counter); - - let mut fields = HashMap::new(); - - for i in 0..fields_per_table { - let field_name = number_to_string(i.wrapping_add(counter.wrapping_mul(100))); - let datatype = DATATYPES[i.wrapping_add(counter) % DATATYPES.len()]; - - let field = Field::new(&field_name, datatype); - - fields.insert(field_name, field); - } - - // TODO check if `fields_per_table > 0` - let first_field_for_pk = fields.keys().next().unwrap().clone(); - - Table::new( - name, - fields, - TableExtra::new_with_pk(vec![first_field_for_pk]), - ) -} diff --git a/lib/Cargo.toml b/lib/Cargo.toml index c710bfe..7a06547 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -12,10 +12,15 @@ repository.workspace = true anyhow = "1.0.75" nom = "7.1.3" static_assertions = "1.1.0" +hmac-sha256 = { version = "1.1.7", optional = true } [dev-dependencies] criterion = { version = "0.4", features = ["html_reports"] } +[features] +default = ["generate"] +generate = ["dep:hmac-sha256"] + [[bench]] name = "parsing_tables" harness = false diff --git a/lib/src/generate.rs b/lib/src/generate.rs new file mode 100644 index 0000000..b0fd003 --- /dev/null +++ b/lib/src/generate.rs @@ -0,0 +1,44 @@ +use std::collections::HashMap; + +use hmac_sha256::HMAC; + +use crate::types::{Field, Table, TableExtra}; + +pub trait GenerateDummy { + fn generate_dummy(number: usize) -> Self; +} + +pub(crate) fn hash_number(input: usize) -> [u8; 32] { + let bytes = input.to_le_bytes(); + let h = HMAC::new(bytes); + + h.finalize() +} + +pub(crate) fn u8s_to_string(input: &[u8]) -> String { + input.iter().map(|item| (item % 24 + 65) as char).collect() +} + +pub(crate) fn hash_number_and_stringify(input: usize) -> String { + u8s_to_string(&hash_number(input)) +} + +pub fn generate_table(counter: usize, fields_per_table: usize) -> Table { + let name = hash_number_and_stringify(counter); + + let mut fields = HashMap::new(); + + for i in 0..fields_per_table { + let field = Field::generate_dummy(i.wrapping_add(counter.wrapping_mul(100))); + + fields.insert(field.name.clone(), field); + } + + let first_field_for_pk = fields.keys().next().unwrap().clone(); + + Table::new( + name, + fields, + TableExtra::new_with_pk(vec![first_field_for_pk]), + ) +} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index f25fe26..719ea25 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -11,6 +11,11 @@ use types::{RawTableCollection, Table, TableCollection}; use crate::parser::parse; +#[cfg(feature = "generate")] +pub mod generate; +#[cfg(not(feature = "generate"))] +mod generate; + mod parser; pub mod types; diff --git a/lib/src/types.rs b/lib/src/types.rs index 5b756d9..14cf30d 100644 --- a/lib/src/types.rs +++ b/lib/src/types.rs @@ -6,6 +6,9 @@ use std::rc::Rc; use anyhow::{bail, Result}; use static_assertions::const_assert_eq; +use crate::generate::hash_number_and_stringify; +#[cfg(feature = "generate")] +use crate::generate::GenerateDummy; use crate::parser::types::{FieldExtra, FieldType, RawDataType, RawField, RawTable}; use crate::{TransformSQL, TransformTSQL}; @@ -299,6 +302,15 @@ impl TransformTSQL for Field { } } +impl GenerateDummy for Field { + fn generate_dummy(number: usize) -> Self { + let name = hash_number_and_stringify(number); + let datatype = DataType::generate_dummy(number); + + Field::new(name, datatype) + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum DataType { Int, @@ -389,6 +401,22 @@ impl TransformTSQL for DataType { } } +#[cfg(feature = "generate")] +impl GenerateDummy for DataType { + fn generate_dummy(number: usize) -> Self { + // TODO add more variants + const DATATYPES: &[DataType] = &[ + DataType::Int, + DataType::Double, + DataType::VarChar(100), + DataType::Char(6), + DataType::Uuid, + ]; + + DATATYPES[number % DATATYPES.len()] + } +} + /// Holds metadata for a [`Table`] #[derive(Debug, Default)] pub struct TableExtra { From 426423564d0cbe5ece268c84e4670535fa5439ee Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Mon, 14 Aug 2023 19:02:19 +0200 Subject: [PATCH 17/25] remove cli dependency to binary 'generate_tsql' from library 'tsql' benches --- lib/Cargo.toml | 2 +- lib/benches/parsing_tables.rs | 44 ++++++++++++++++------------------- lib/src/lib.rs | 2 -- lib/src/types.rs | 4 ++-- 4 files changed, 23 insertions(+), 29 deletions(-) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 7a06547..ed08b39 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -18,7 +18,7 @@ hmac-sha256 = { version = "1.1.7", optional = true } criterion = { version = "0.4", features = ["html_reports"] } [features] -default = ["generate"] +default = [] generate = ["dep:hmac-sha256"] [[bench]] diff --git a/lib/benches/parsing_tables.rs b/lib/benches/parsing_tables.rs index 7c5e800..77707c4 100644 --- a/lib/benches/parsing_tables.rs +++ b/lib/benches/parsing_tables.rs @@ -1,11 +1,12 @@ -use std::env; use std::fmt::Display; -use std::process::{Command, Stdio}; use std::time::Duration; use anyhow::Result; use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use tsql::parse_str; +#[cfg(feature = "generate")] +use tsql::generate::generate_table; +use tsql::types::Table; +use tsql::{parse_str, TransformTSQL}; struct Arguments { tables: usize, @@ -24,28 +25,23 @@ impl Display for Arguments { } } -fn generate_test_file(arguments: &Arguments) -> Result { - let current_dir = env::current_dir()?; - // TODO remove this "hacky" way to get the target directory. - // Because I think this will fail with the user sets a custom directory for the `target` dir. - // Maybe some env variable has the absolute path to the dir. Or figure it out with a build script. - let target_dir = current_dir - .as_path() - .parent() - .unwrap() - .join("target") - .join("debug"); +fn generate_test_content(arguments: &Arguments) -> Result { + let mut buffer = Vec::new(); - // TODO don't call the executable, call the functions directly - let output = Command::new(&target_dir.join("generate_tsql")) - .current_dir(&target_dir) - .arg("--stdout") - .args(["--tables", &arguments.tables.to_string()]) - .args(["--fields", &arguments.fields.to_string()]) - .stdout(Stdio::piped()) - .output()?; + for i in 0..arguments.tables { + #[cfg(feature = "generate")] + let table: Table = generate_table(i, arguments.fields); + #[cfg(not(feature = "generate"))] + let table: Table = { + // TODO make this nicer and maybe into a compile time warning and not a runtime + panic!("The feature `generate` has to be enabled to run this benches."); + todo!() + }; - Ok(String::from_utf8(output.stdout)?) + table.transform_into_tsql(&mut buffer)?; + } + + Ok(String::from_utf8(buffer)?) } fn criterion_benchmark(c: &mut Criterion) { @@ -59,7 +55,7 @@ fn criterion_benchmark(c: &mut Criterion) { ]; for argument in arguments { - let content = generate_test_file(&argument).unwrap(); + let content = generate_test_content(&argument).unwrap(); c.bench_function(&format!("{}", argument), |b| { b.iter(|| black_box(parse_str(&content))) diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 719ea25..33d64ed 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -13,8 +13,6 @@ use crate::parser::parse; #[cfg(feature = "generate")] pub mod generate; -#[cfg(not(feature = "generate"))] -mod generate; mod parser; pub mod types; diff --git a/lib/src/types.rs b/lib/src/types.rs index 14cf30d..563620e 100644 --- a/lib/src/types.rs +++ b/lib/src/types.rs @@ -6,9 +6,8 @@ use std::rc::Rc; use anyhow::{bail, Result}; use static_assertions::const_assert_eq; -use crate::generate::hash_number_and_stringify; #[cfg(feature = "generate")] -use crate::generate::GenerateDummy; +use crate::generate::{hash_number_and_stringify, GenerateDummy}; use crate::parser::types::{FieldExtra, FieldType, RawDataType, RawField, RawTable}; use crate::{TransformSQL, TransformTSQL}; @@ -302,6 +301,7 @@ impl TransformTSQL for Field { } } +#[cfg(feature = "generate")] impl GenerateDummy for Field { fn generate_dummy(number: usize) -> Self { let name = hash_number_and_stringify(number); From 593da45d1dae1389fb7c45ee2271f20fa841fe96 Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Sat, 19 Aug 2023 13:56:49 +0200 Subject: [PATCH 18/25] add 'cargo-nextest' to step 'taiki-e/install-action' --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6e2600..e77ab20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: - uses: taiki-e/install-action@v2 with: - tool: cargo-binstall,cargo-llvm-cov + tool: cargo-llvm-cov,cargo-nextest - uses: actions/cache@v3 with: From 3a43ee2f345ffcc9730071c7892633637f09fe83 Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Sat, 19 Aug 2023 14:00:19 +0200 Subject: [PATCH 19/25] use 'generate_tsql' to generate test files in ci --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e77ab20..cb343f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,11 +51,10 @@ jobs: run: cargo build --verbose --all-targets --all --all-features - name: Create test files - working-directory: ./lib/tests/files continue-on-error: true run: | - python generate_big.py --name "big_smaller.tsql" --tables 1000 --fields 500 - python generate_big.py --name "big_170mb.tsql" --tables 4000 --fields 1000 + ./target/debug/generate_tsql --name ./lib/tests/filesbig_smaller.tsql --tables 1000 --fields 500 + ./target/debug/generate_tsql --name ./lib/tests/filesbig_170mb.tsql --tables 4000 --fields 1000 - name: Test run: cargo nextest run --all-features --all From f9c569f5f949067166bde412105907d0a23b161c Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Sat, 19 Aug 2023 16:44:38 +0200 Subject: [PATCH 20/25] added some tests in 'lib/src/parser/types.rs' --- Cargo.lock | 14 ++- lib/.gitignore | 1 + lib/Cargo.toml | 1 + lib/src/parser/types.rs | 209 +++++++++++++++++++++++++++++++++++++++- 4 files changed, 222 insertions(+), 3 deletions(-) create mode 100644 lib/.gitignore diff --git a/Cargo.lock b/Cargo.lock index 1917ebf..b5ae4dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,7 +124,7 @@ dependencies = [ "ciborium", "clap", "criterion-plot", - "itertools", + "itertools 0.10.5", "lazy_static", "num-traits", "oorandom", @@ -145,7 +145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools", + "itertools 0.10.5", ] [[package]] @@ -249,6 +249,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -545,6 +554,7 @@ dependencies = [ "anyhow", "criterion", "hmac-sha256", + "itertools 0.11.0", "nom", "static_assertions", ] diff --git a/lib/.gitignore b/lib/.gitignore new file mode 100644 index 0000000..9a22ac0 --- /dev/null +++ b/lib/.gitignore @@ -0,0 +1 @@ +ttests/ \ No newline at end of file diff --git a/lib/Cargo.toml b/lib/Cargo.toml index ed08b39..0d18573 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -16,6 +16,7 @@ hmac-sha256 = { version = "1.1.7", optional = true } [dev-dependencies] criterion = { version = "0.4", features = ["html_reports"] } +itertools = "0.11.0" [features] default = [] diff --git a/lib/src/parser/types.rs b/lib/src/parser/types.rs index 49d0c42..f582da4 100644 --- a/lib/src/parser/types.rs +++ b/lib/src/parser/types.rs @@ -19,6 +19,8 @@ impl RawTable { pub fn fk_tables(&self) -> Vec { let mut table_names = Vec::with_capacity(self.fields.len()); + dbg!(&self); + for field_type in self.fields.values() { match field_type { FieldType::Real(_) => (), @@ -26,7 +28,7 @@ impl RawTable { RawDataType::ForeignKeyTable(table_name) => { table_names.push(table_name.clone()) } - _ => (), + _ => panic!("Error with field '{:?}'.\nThe field type is `FieldType::Virtual` but the datatype is not `RawDataType::ForeignKeyTable`.", field), }, }; } @@ -117,3 +119,208 @@ pub enum FieldExtra { pub enum TagHelper { PrimaryKey, } + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::RawTable; + use crate::parser::types::{FieldExtra, FieldType, RawDataType, RawField}; + use crate::types::TableExtra; + + #[test] + fn raw_table() { + let table = RawTable { + extra: TableExtra::default(), + name: "".to_string(), + fields: HashMap::new(), + }; + + assert_eq!(table.has_fk(), false); + assert_eq!(table.fk_tables(), Vec::::new()); + + let table = RawTable { + extra: TableExtra::default(), + name: "".to_string(), + fields: { + let mut map = HashMap::new(); + + // TODO I think this fk value is bullshit, so I have to check this with the real parsing program + map.insert( + "human".to_string(), + FieldType::Virtual(( + RawField { + name: "name".to_string(), + datatype: RawDataType::ForeignKeyTable("human".to_string()), + }, + FieldExtra::ForeignKey, + )), + ); + + map.insert( + "bday".to_string(), + FieldType::Real(RawField { + name: "bday".to_string(), + datatype: RawDataType::DateTime, + }), + ); + + map + }, + }; + + assert_eq!(table.has_fk(), true); + assert_eq!(table.fk_tables(), vec!["human".to_string()]); + } + + mod raw_data_type { + use itertools::Itertools; + + use crate::parser::types::RawDataType; + + #[test] + fn arguments_len_0() { + let fields = [ + ("int", RawDataType::Int), + ("bool", RawDataType::Bool), + ("bigint", RawDataType::BigInt), + ("date", RawDataType::Date), + ("datetime", RawDataType::DateTime), + ("time", RawDataType::Time), + ("double", RawDataType::Double), + ("float", RawDataType::Float), + ("uuid", RawDataType::Uuid), + ("_", RawDataType::Unknown), + ( + "custom_data_type", + RawDataType::ForeignKeyTable("custom_data_type".to_string()), + ), + ]; + + for (raw, result) in fields { + assert_eq!(RawDataType::parse(raw, Vec::new()), Some(result)); + + assert_eq!(RawDataType::parse(raw, vec![Default::default(); 4]), None); + } + } + + #[test] + fn arguments_number_parsing() { + enum DT { + U8, + U16, + } + + impl DT { + fn max_value(&self) -> String { + match self { + DT::U8 => u8::MAX.to_string(), + DT::U16 => u16::MAX.to_string(), + } + } + + fn min_value(&self) -> String { + match self { + DT::U8 => u8::MIN.to_string(), + DT::U16 => u16::MIN.to_string(), + } + } + + fn wrong_values(&self) -> Vec { + let mut specific_values = match self { + DT::U8 => vec![u32::MAX.to_string()], + DT::U16 => vec![u32::MAX.to_string()], + }; + + let mut values = vec!["a".to_string(), "abc".to_string()]; + values.append(&mut specific_values); + values + } + } + + // TODO that's some really weird witch magic and I _think_ it would be good if we can refactor this (a lot) + // TODO a good first think to todo would be to remove the `unwrap()`s and return a `Result` or an `Option` + // The `Box RawDataType>` is so that we can create the type with the value in the test + let fields: &[(&str, Box RawDataType>, DT)] = &[ + ( + "varchar", + Box::new(|value: &str| RawDataType::VarChar(value.parse().unwrap())) as _, + DT::U16, + ), + ( + "char", + Box::new(|value: &str| RawDataType::Char(value.parse().unwrap())) as _, + DT::U8, + ), + ( + "text", + Box::new(|value: &str| RawDataType::Text(value.parse().unwrap())) as _, + DT::U16, + ), + ]; + + for (raw, result_type, arguments) in fields { + let out = RawDataType::parse(raw, vec![&arguments.min_value()]); + assert!(out.is_some()); + assert_eq!(out, Some(result_type(&arguments.min_value()))); + + let out = RawDataType::parse(raw, vec![&arguments.max_value()]); + assert!(out.is_some()); + assert_eq!(out, Some(result_type(&arguments.max_value()))); + + for wrong_value in arguments.wrong_values() { + assert_eq!(RawDataType::parse(raw, vec![&wrong_value]), None); + } + } + + // ! same as in the comment before + let fields: &[(&str, Box RawDataType>, (DT, DT))] = &[( + "decimal", + Box::new(|(v1, v2): (&str, &str)| { + RawDataType::Decimal(v1.parse().unwrap(), v2.parse().unwrap()) + }) as _, + (DT::U8, DT::U8), + )]; + + for (raw, result_type, arguments) in fields { + let out = RawDataType::parse( + raw, + vec![&arguments.0.min_value(), &arguments.1.min_value()], + ); + assert!(out.is_some()); + assert_eq!( + out, + Some(result_type(( + &arguments.0.min_value(), + &arguments.1.min_value() + ))) + ); + + let out = RawDataType::parse( + raw, + vec![&arguments.0.max_value(), &arguments.1.max_value()], + ); + assert!(out.is_some()); + assert_eq!( + out, + Some(result_type(( + &arguments.0.max_value(), + &arguments.1.max_value() + ))) + ); + + for wrong_value in arguments + .0 + .wrong_values() + .iter() + .cartesian_product(arguments.1.wrong_values()) + { + assert_eq!( + RawDataType::parse(raw, vec![wrong_value.0, &wrong_value.1]), + None + ); + } + } + } + } +} From e94d134ef56ca398a0c7e5ab416c9738798bbdca Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Sat, 19 Aug 2023 16:50:40 +0200 Subject: [PATCH 21/25] build 'generate_tsql' in release mode because this step takes over 2m in debug mode, CI' --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb343f3..37940f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,8 +53,9 @@ jobs: - name: Create test files continue-on-error: true run: | - ./target/debug/generate_tsql --name ./lib/tests/filesbig_smaller.tsql --tables 1000 --fields 500 - ./target/debug/generate_tsql --name ./lib/tests/filesbig_170mb.tsql --tables 4000 --fields 1000 + cargo build --release --bin generate_tsql + ./target/release/generate_tsql --name ./lib/tests/filesbig_smaller.tsql --tables 1000 --fields 500 + ./target/release/generate_tsql --name ./lib/tests/filesbig_170mb.tsql --tables 4000 --fields 1000 - name: Test run: cargo nextest run --all-features --all From 25ea39fc67fa195638943086855c37ea643ba922 Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Sat, 19 Aug 2023 19:51:57 +0200 Subject: [PATCH 22/25] added some tests in 'lib/src/parser/mod.rs' --- lib/src/parser/mod.rs | 33 +++++++++++++++++++++++++++++++++ lib/src/types.rs | 10 +++++++--- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/lib/src/parser/mod.rs b/lib/src/parser/mod.rs index 089f2c2..5f4356c 100644 --- a/lib/src/parser/mod.rs +++ b/lib/src/parser/mod.rs @@ -96,3 +96,36 @@ fn table_body(input: &str) -> IResult<&str, HashMap> { Ok((input, fields)) } + +#[cfg(test)] +mod tests { + + mod table_extra { + use super::super::table_extra; + use crate::types::TableExtra; + + #[test] + fn just_works() { + assert_eq!( + table_extra("@primary_key(id) something else for another parser"), + Ok(( + " something else for another parser", + TableExtra::new_with_pk(vec!["id"]) + )) + ); + + assert_eq!( + table_extra("@primary_key(id, another) something else for another parser"), + Ok(( + " something else for another parser", + TableExtra::new_with_pk(vec!["id", "another"]) + )) + ); + + assert_eq!( + table_extra("something else for another parser"), + Ok(("something else for another parser", TableExtra::default())) + ); + } + } +} diff --git a/lib/src/types.rs b/lib/src/types.rs index 563620e..9a888d3 100644 --- a/lib/src/types.rs +++ b/lib/src/types.rs @@ -418,15 +418,19 @@ impl GenerateDummy for DataType { } /// Holds metadata for a [`Table`] -#[derive(Debug, Default)] +#[derive(Debug, Default, PartialEq, Eq)] pub struct TableExtra { primary_key: Vec, } impl TableExtra { - pub fn new_with_pk(primary_key: Vec) -> Self { + pub fn new_with_pk + Clone>(primary_key: Vec) -> Self { TableExtra { - primary_key, + primary_key: primary_key + .iter() + // TODO remove call to `.clone()` + .map(|item| item.clone().into()) + .collect::>(), ..Self::default() } } From 5d9780ceabc3462ac0e886257c21463aa937ad92 Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Sat, 19 Aug 2023 20:02:54 +0200 Subject: [PATCH 23/25] add some tests and documentation for 'get_first_element' --- lib/src/helper.rs | 33 +++++++++++++++++++++++++++++++++ lib/src/lib.rs | 1 + lib/src/types.rs | 8 +------- 3 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 lib/src/helper.rs diff --git a/lib/src/helper.rs b/lib/src/helper.rs new file mode 100644 index 0000000..18389e2 --- /dev/null +++ b/lib/src/helper.rs @@ -0,0 +1,33 @@ +use std::collections::BTreeMap; + +/// Get, if possible, the first key-value tuple from a [`BTreeMap`] as reference. +/// +/// This function is generic over the key `K: Ord` and value `V`. +pub fn get_first_element(collection: &BTreeMap) -> Option<(&K, &V)> { + // `BTreeMap.keys()` returns the keys in an sorted iterator and with `Iterator.next()` we get the first value as an Option. + // If the function returns `None`, we can return early and return with `None`. + let key = collection.keys().next()?; + + // this get _should_ theoretically always return a value because if not how can we get a key out of the `BTreeMap`? + let value = collection.get(key)?; + + Some((key, value)) +} + +#[cfg(test)] +mod tests { + mod get_first_element { + use std::collections::BTreeMap; + + use crate::helper::get_first_element; + + #[test] + fn just_works() { + let mut map = BTreeMap::new(); + assert_eq!(get_first_element(&map), None); + + map.insert("value", 1); + assert_eq!(get_first_element(&map), Some((&"value", &1))); + } + } +} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 33d64ed..cc3bd68 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -14,6 +14,7 @@ use crate::parser::parse; #[cfg(feature = "generate")] pub mod generate; +mod helper; mod parser; pub mod types; diff --git a/lib/src/types.rs b/lib/src/types.rs index 9a888d3..efc0c42 100644 --- a/lib/src/types.rs +++ b/lib/src/types.rs @@ -8,6 +8,7 @@ use static_assertions::const_assert_eq; #[cfg(feature = "generate")] use crate::generate::{hash_number_and_stringify, GenerateDummy}; +use crate::helper::get_first_element; use crate::parser::types::{FieldExtra, FieldType, RawDataType, RawField, RawTable}; use crate::{TransformSQL, TransformTSQL}; @@ -18,13 +19,6 @@ pub type TableCollection = GenericCollection
; /// A `BTreeMap` with the type of a key is a `String` and the type of the value is a [`Rc>`] pub(crate) type RawTableCollection = GenericCollection>>; -fn get_first_element(collection: &BTreeMap) -> Option<(&K, &V)> { - let key = collection.keys().next()?; - let item = collection.get(key)?; - - Some((key, item)) -} - // TODO remove `pub(crate)` #[derive(Debug, Default)] pub struct Table { From 32a0e8cc1af2969a0b565876aa1a37606fda2e5a Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Sun, 20 Aug 2023 02:50:45 +0200 Subject: [PATCH 24/25] added some tests in 'lib/src/generate.rs' --- lib/src/generate.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/src/generate.rs b/lib/src/generate.rs index b0fd003..9596121 100644 --- a/lib/src/generate.rs +++ b/lib/src/generate.rs @@ -42,3 +42,26 @@ pub fn generate_table(counter: usize, fields_per_table: usize) -> Table { TableExtra::new_with_pk(vec![first_field_for_pk]), ) } + +#[cfg(test)] +mod tests { + use crate::generate::{hash_number, hash_number_and_stringify, u8s_to_string}; + + #[test] + fn hash_number_just_works() { + assert_eq!(hash_number(1), hash_number(1)); + assert_ne!(hash_number(1), hash_number(2)); + } + + #[test] + fn u8s_to_string_just_works() { + assert_eq!(u8s_to_string(&[0, 1, 2, 3, 4]), "ABCDE".to_string()); + assert_eq!(u8s_to_string(&[24, 25, 26, 27, 28]), "ABCDE".to_string()); + } + + #[test] + fn hash_number_and_stringify_just_works() { + assert_eq!(hash_number_and_stringify(1), hash_number_and_stringify(1)); + assert_ne!(hash_number_and_stringify(1), hash_number_and_stringify(2)); + } +} From f2ef7bf39c60290ed1a02b118f9376da7d3ec891 Mon Sep 17 00:00:00 2001 From: LetsMelon Date: Sun, 20 Aug 2023 03:25:00 +0200 Subject: [PATCH 25/25] remove generated files from e2e tests. 'cargo bench' already tests with generated '.tsql' files --- .github/workflows/ci.yml | 7 ------- lib/src/parser/types.rs | 2 -- lib/tests/e2e.rs | 23 ++++++++++++++--------- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37940f4..71d1f87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,13 +50,6 @@ jobs: - name: Build run: cargo build --verbose --all-targets --all --all-features - - name: Create test files - continue-on-error: true - run: | - cargo build --release --bin generate_tsql - ./target/release/generate_tsql --name ./lib/tests/filesbig_smaller.tsql --tables 1000 --fields 500 - ./target/release/generate_tsql --name ./lib/tests/filesbig_170mb.tsql --tables 4000 --fields 1000 - - name: Test run: cargo nextest run --all-features --all diff --git a/lib/src/parser/types.rs b/lib/src/parser/types.rs index f582da4..dc65c2b 100644 --- a/lib/src/parser/types.rs +++ b/lib/src/parser/types.rs @@ -19,8 +19,6 @@ impl RawTable { pub fn fk_tables(&self) -> Vec { let mut table_names = Vec::with_capacity(self.fields.len()); - dbg!(&self); - for field_type in self.fields.values() { match field_type { FieldType::Real(_) => (), diff --git a/lib/tests/e2e.rs b/lib/tests/e2e.rs index ad43704..043b552 100644 --- a/lib/tests/e2e.rs +++ b/lib/tests/e2e.rs @@ -11,19 +11,24 @@ fn e2e_parse_all_files() { let paths = path .read_dir() .unwrap() - .filter(|item| item.is_ok()) - .map(|item| item.unwrap()) - .filter(|item| match item.path().extension() { - Some(ending) if ending == OsStr::new("tsql") => true, - _ => false, + .filter_map(|item| { + if item.is_err() { + return None; + } + + let item = item.unwrap(); + + if let Some(ending) = item.path().extension() { + if ending == OsStr::new("tsql") { + return Some(item.path()); + } + } + + return None; }) - .map(|item| item.path()) .collect::>(); - println!("{paths:?}"); - for path in paths { - println!("path: {:?}", path); let out = parse_file(path); assert!(out.is_ok()); }