[CN-Test-Gen] Various tweaks and nitpicks (#877)

* [CN-Test-Gen] UBSan info in `TESTING.md`

* [CN-Test-Gen] Use `enum`s for CLI arguments

* [CN-Test-Gen] Support strings in C testing CLI

* [CN-Test-Gen] Set default sizing strategy to QuickCheck's

backend/cn/TESTING.md

@@ -12,6 +12,9 @@ This script compiles the C files and runs the tests.
 By default, running `cn test` will automatically run `run_tests.sh`, which produces a test executable `tests.out`.
 This can be disabled by using the `--no-run` flag.
 
+The default behavior of testing is to rely on Fulminate for checking, which does not detect undefined behavior.
+If you would like to also check for undefined behavior, you can use a sanitizer via `--sanitize=undefined`.
+
 The output directory for these files can be set by using `--output-dir=<DIR>`.
 If the directory does not already exist, it is created.
 

backend/cn/bin/main.ml

@@ -469,6 +469,7 @@ let run_tests
   max_unfolds
   max_array_length
   with_static_hack
+  build_tool
   sanitizers
   input_timeout
   null_in_every
@@ -518,6 +519,7 @@ let run_tests
           max_unfolds;
           max_array_length;
           with_static_hack;
+          build_tool;
           sanitizers;
           input_timeout;
           null_in_every;
@@ -980,6 +982,14 @@ module Testing_flags = struct
     Arg.(value & flag & info [ "with-static-hack" ] ~doc)
 
 
+  let build_tool =
+    let doc = "Set which build tool to use." in
+    Arg.(
+      value
+      & opt (enum TestGeneration.Options.build_tool) TestGeneration.default_cfg.build_tool
+      & info [ "build-tool" ] ~doc)
+
+
   let sanitize =
     let doc = "Forwarded to the '-fsanitize' argument of the C compiler" in
     Arg.(
@@ -1021,26 +1031,29 @@ module Testing_flags = struct
     let doc = "Set the logging level for failing inputs from tests" in
     Arg.(
       value
-      & opt (some int) TestGeneration.default_cfg.logging_level
+      & opt
+          (some (enum TestGeneration.Options.logging_level))
+          TestGeneration.default_cfg.logging_level
       & info [ "logging-level" ] ~doc)
 
 
   let trace_granularity =
     let doc = "Set the trace granularity for failing inputs from tests" in
     Arg.(
       value
-      & opt (some int) TestGeneration.default_cfg.trace_granularity
+      & opt
+          (some (enum TestGeneration.Options.trace_granularity))
+          TestGeneration.default_cfg.trace_granularity
       & info [ "trace-granularity" ] ~doc)
 
 
   let progress_level =
-    let doc =
-      "Set the level of detail for progress updates (0 = Quiet, 1 = Per function, 2 = \
-       Per test case)"
-    in
+    let doc = "Set the frequency of progress updates." in
     Arg.(
       value
-      & opt (some int) TestGeneration.default_cfg.progress_level
+      & opt
+          (some (enum TestGeneration.Options.progress_level))
+          TestGeneration.default_cfg.progress_level
       & info [ "progress-level" ] ~doc)
 
 
@@ -1084,12 +1097,12 @@ module Testing_flags = struct
 
 
   let sizing_strategy =
-    let doc =
-      "Strategy for deciding test case size (0 = Uniform, 1 = Quartile, 2 = QuickCheck)"
-    in
+    let doc = "Strategy for deciding test case size." in
     Arg.(
       value
-      & opt (some int) TestGeneration.default_cfg.sizing_strategy
+      & opt
+          (some (enum TestGeneration.Options.sizing_strategy))
+          TestGeneration.default_cfg.sizing_strategy
       & info [ "sizing-strategy" ] ~doc)
 
 
@@ -1168,6 +1181,7 @@ let testing_cmd =
     $ Testing_flags.gen_max_unfolds
     $ Testing_flags.max_array_length
     $ Testing_flags.with_static_hack
+    $ Testing_flags.build_tool
     $ Term.product Testing_flags.sanitize Testing_flags.no_sanitize
     $ Testing_flags.input_timeout
     $ Testing_flags.null_in_every

backend/cn/lib/testGeneration/buildScript.ml

@@ -143,17 +143,16 @@ let run () =
           |> Option.map (fun seed -> [ "--seed"; seed ])
           |> Option.to_list
           |> List.flatten)
-       @ (Config.has_logging_level ()
-          |> Option.map (fun level -> [ "--logging-level"; string_of_int level ])
+       @ (Config.has_logging_level_str ()
+          |> Option.map (fun level -> [ "--logging-level"; level ])
           |> Option.to_list
           |> List.flatten)
-       @ (Config.has_trace_granularity ()
-          |> Option.map (fun granularity ->
-            [ "--trace-granularity"; string_of_int granularity ])
+       @ (Config.has_trace_granularity_str ()
+          |> Option.map (fun granularity -> [ "--trace-granularity"; granularity ])
           |> Option.to_list
           |> List.flatten)
-       @ (Config.has_progress_level ()
-          |> Option.map (fun level -> [ "--progress-level"; string_of_int level ])
+       @ (Config.has_progress_level_str ()
+          |> Option.map (fun level -> [ "--progress-level"; level ])
           |> Option.to_list
           |> List.flatten)
        @ (match Config.is_until_timeout () with
@@ -173,9 +172,8 @@ let run () =
             [ "--max-generator-size"; string_of_int max_generator_size ])
           |> Option.to_list
           |> List.flatten)
-       @ (Config.has_sizing_strategy ()
-          |> Option.map (fun sizing_strategy ->
-            [ "--sizing-strategy"; string_of_int sizing_strategy ])
+       @ (Config.has_sizing_strategy_str ()
+          |> Option.map (fun sizing_strategy -> [ "--sizing-strategy"; sizing_strategy ])
           |> Option.to_list
           |> List.flatten)
        @ (if Config.is_sized_null () then

backend/cn/lib/testGeneration/testGenConfig.ml

@@ -1,24 +1,47 @@
+type build_tool = Bash
+
+type logging_level =
+  | None
+  | Error
+  | Info
+
+type trace_granularity =
+  | None
+  | End
+  | All
+
+type progress_level =
+  | Silent
+  | PerFunc
+  | PerTestCase
+
+type sizing_strategy =
+  | Uniform
+  | Quartile
+  | QuickCheck
+
 type t =
   { (* Compile time *)
     num_samples : int;
     max_backtracks : int;
     max_unfolds : int option;
     max_array_length : int;
     with_static_hack : bool;
+    build_tool : build_tool;
     sanitizers : string option * string option;
     (* Run time *)
     input_timeout : int option;
     null_in_every : int option;
     seed : string option;
-    logging_level : int option;
-    trace_granularity : int option;
-    progress_level : int option;
+    logging_level : logging_level option;
+    trace_granularity : trace_granularity option;
+    progress_level : progress_level option;
     until_timeout : int option;
     exit_fast : bool;
     max_stack_depth : int option;
     allowed_depth_failures : int option;
     max_generator_size : int option;
-    sizing_strategy : int option;
+    sizing_strategy : sizing_strategy option;
     random_size_splits : bool;
     allowed_size_split_backtracks : int option;
     sized_null : bool;
@@ -33,6 +56,7 @@ let default =
     max_unfolds = None;
     max_array_length = 50;
     with_static_hack = false;
+    build_tool = Bash;
     sanitizers = (None, None);
     input_timeout = None;
     null_in_every = None;
@@ -55,6 +79,51 @@ let default =
   }
 
 
+let string_of_logging_level (logging_level : logging_level) =
+  match logging_level with None -> "none" | Error -> "error" | Info -> "info"
+
+
+let string_of_trace_granularity (trace_granularity : trace_granularity) =
+  match trace_granularity with None -> "none" | End -> "end" | All -> "all"
+
+
+let string_of_progress_level (progress_level : progress_level) =
+  match progress_level with
+  | Silent -> "silent"
+  | PerFunc -> "function"
+  | PerTestCase -> "testcase"
+
+
+let string_of_sizing_strategy (sizing_strategy : sizing_strategy) =
+  match sizing_strategy with
+  | Uniform -> "uniform"
+  | Quartile -> "quartile"
+  | QuickCheck -> "quickcheck"
+
+
+module Options = struct
+  let build_tool = [ ("bash", Bash) ]
+
+  let logging_level : (string * logging_level) list =
+    List.map (fun lvl -> (string_of_logging_level lvl, lvl)) [ None; Error; Info ]
+
+
+  let trace_granularity : (string * trace_granularity) list =
+    List.map (fun gran -> (string_of_trace_granularity gran, gran)) [ None; End; All ]
+
+
+  let progress_level : (string * progress_level) list =
+    List.map
+      (fun lvl -> (string_of_progress_level lvl, lvl))
+      [ Silent; PerFunc; PerTestCase ]
+
+
+  let sizing_strategy : (string * sizing_strategy) list =
+    List.map
+      (fun strat -> (string_of_sizing_strategy strat, strat))
+      [ Uniform; Quartile; QuickCheck ]
+end
+
 let instance = ref default
 
 let initialize (cfg : t) = instance := cfg
@@ -69,6 +138,8 @@ let get_max_array_length () = !instance.max_array_length
 
 let with_static_hack () = !instance.with_static_hack
 
+let get_build_tool () = !instance.build_tool
+
 let has_sanitizers () = !instance.sanitizers
 
 let has_input_timeout () = !instance.input_timeout
@@ -79,10 +150,20 @@ let has_seed () = !instance.seed
 
 let has_logging_level () = !instance.logging_level
 
+let has_logging_level_str () = Option.map string_of_logging_level !instance.logging_level
+
 let has_trace_granularity () = !instance.trace_granularity
 
+let has_trace_granularity_str () =
+  Option.map string_of_trace_granularity !instance.trace_granularity
+
+
 let has_progress_level () = !instance.progress_level
 
+let has_progress_level_str () =
+  Option.map string_of_progress_level !instance.progress_level
+
+
 let is_until_timeout () = !instance.until_timeout
 
 let is_exit_fast () = !instance.exit_fast
@@ -95,6 +176,10 @@ let has_max_generator_size () = !instance.max_generator_size
 
 let has_sizing_strategy () = !instance.sizing_strategy
 
+let has_sizing_strategy_str () =
+  Option.map string_of_sizing_strategy !instance.sizing_strategy
+
+
 let is_random_size_splits () = !instance.random_size_splits
 
 let has_allowed_size_split_backtracks () = !instance.allowed_size_split_backtracks