Merge branch 'main' into sandbox/debug-perf-parse_comp_units

* main: (56 commits)
  feature: add terminal ui backend based on NoTTY (ocaml#6996)
  doc(coq): update documentation about coqdep
  fix(rules): don't descend into automatic subdirs infinitely (ocaml#7208)
  benchmark: add warm run (ocaml#7198)
  test: vendored and public libs (ocaml#7197)
  test: use sh in concurrent test (ocaml#7205)
  fix: custom log file path (ocaml#7200)
  test(melange): add test exercising ocaml#7104 (ocaml#7204)
  test(melange): add a test that introduces rules in the target dir (ocaml#7196)
  test: duplicate packages in vendor dir (ocaml#7194)
  melange: interpret `melc --where` as a list of `:`-separated paths (ocaml#7176)
  perf: add synthetic benchmark (ocaml#7189)
  Test case for bug report (ocaml#6725)
  Add test illustrating ocaml#6575 (ocaml#6576)
  chore: add rule streaming proposal (ocaml#7195)
  test(stdlib): merge wrapped/unwrapped tests
  test: move all stdlib tests
  fix: allow unwrapped libraries with `(stdlib ..)`
  test: demonstrate crash in modules.ml when `(stdlib .. )` used with `(wrapped false)`
  fix(install): respect display options (ocaml#7116)
  ...

.github/workflows/melange-bench.yml → .github/workflows/bench.yml

@@ -1,10 +1,11 @@
-name: Melange demo app build time benchmark
+name: Build time benchmarks
 
 # Do not run this workflow on pull request since this workflow has permission to modify contents.
 on:
   push:
     branches:
       - main
+      - bench/add-warm-run
 
 permissions:
   # deployments permission to deploy GitHub pages website
@@ -65,7 +66,7 @@ jobs:
 
       - name: Run pupilfirst benchmark
         working-directory: pupilfirst
-        run: ../bench/gen-melange-benchmark.sh 'opam exec -- ../_boot/dune.exe build --root=. @main' 'opam exec -- ../_boot/dune.exe clean --root=.' 'pupilfirst build time (${{ runner.os }})' > melange-benchmark-result.json
+        run: ../bench/gen-benchmark.sh 'opam exec -- ../_boot/dune.exe build --root=. @main' 'opam exec -- ../_boot/dune.exe clean --root=.' 'pupilfirst build time (${{ runner.os }})' > melange-benchmark-result.json
 
       - name: Print pupilfirst benchmark results
         working-directory: pupilfirst
@@ -82,7 +83,65 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           auto-push: true
           # Ratio indicating how worse the current benchmark result is.
-          # 150% means if last build took 40s and current takes 50s, it will trigger an alert
+          # 150% means if last build took 40s and current takes >60s, it will trigger an alert
+          alert-threshold: "150%"
+          fail-on-alert: true
+          # Enable alert commit comment
+          comment-on-alert: true
+          # Mention @jchavarri in the commit comment
+          alert-comment-cc-users: '@jchavarri'
+
+      - name: Create synthetic benchmark
+        working-directory: bench
+        run: opam exec -- ../_boot/dune.exe exec ./gen_synthetic.exe -- -n 2000 synthetic
+
+      - name: Run cold synthetic benchmark
+        working-directory: bench/synthetic
+        run: ../gen-benchmark.sh 'opam exec -- ../../_boot/dune.exe build @all' 'opam exec -- ../../_boot/dune.exe clean' 'synthetic build time (cold, ${{ runner.os }})' > synthetic-benchmark-result.json
+
+      - name: Print cold synthetic benchmark results
+        working-directory: bench/synthetic
+        run: |
+          cat bench.json
+          cat synthetic-benchmark-result.json
+
+      - name: Store cold synthetic benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Synthetic Benchmark
+          tool: "customSmallerIsBetter"
+          output-file-path: bench/synthetic/synthetic-benchmark-result.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          # Ratio indicating how worse the current benchmark result is.
+          # 150% means if last build took 40s and current takes >60s, it will trigger an alert
+          alert-threshold: "150%"
+          fail-on-alert: true
+          # Enable alert commit comment
+          comment-on-alert: true
+          # Mention @jchavarri in the commit comment
+          alert-comment-cc-users: '@jchavarri'
+
+      - name: Run warm synthetic benchmark
+        working-directory: bench/synthetic
+        run: ../gen-benchmark.sh 'opam exec -- ../../_boot/dune.exe build @all' 'true' 'synthetic build time (warm, ${{ runner.os }})' > synthetic-benchmark-result.json
+
+      - name: Print warm synthetic benchmark results
+        working-directory: bench/synthetic
+        run: |
+          cat bench.json
+          cat synthetic-benchmark-result.json
+
+      - name: Store warm synthetic benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Synthetic Benchmark
+          tool: "customSmallerIsBetter"
+          output-file-path: bench/synthetic/synthetic-benchmark-result.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          # Ratio indicating how worse the current benchmark result is.
+          # 150% means if last build took 40s and current takes >60s, it will trigger an alert
           alert-threshold: "150%"
           fail-on-alert: true
           # Enable alert commit comment

.github/workflows/workflow.yml

@@ -78,7 +78,7 @@ jobs:
       - run: opam install ./dune.opam --deps-only --with-test
 
       - name: Install system deps on macOS
-        run: brew install coreutils
+        run: brew install coreutils pkg-config file
         if: ${{ matrix.os == 'macos-latest' }}
 
       # dune doesn't have any additional dependencies so we can build it right

.gitignore

@@ -3,6 +3,7 @@ _build
 _boot
 _perf
 _coverage
+__pycache__
 *.install
 
 # vim swap files

CHANGES.md

@@ -1,18 +1,42 @@
 Unreleased
 ----------
 
+- Do not re-render UI on every frame if the UI doesn't change (#7186, fix
+  #7184, @rgrinberg)
+
 - Fix preludes not being recorded as dependencies in the `(mdx)` stanza (#7109,
   fixes #7077, @emillon).
 
-- Invoke preprocessor commands from directory of dune file containing the
-  commands rather than from the workspace root (#7057, fixes #7043, @gridbugs)
+- Make coq_db creation in scope lazy (@ejgallego, #7133)
+
+- Non-user proccesses such as version control or config checking are now run
+  silently. (#6994, fixes #4066, @Alizter)
 
 - Add the `--display-separate-messages` flag to separate the error messages
   produced by commands with a blank line. (#6823, fixes #6158, @esope)
 
 - Accept the Ordered Set Language for the `modes` field in `library` stanzas
   (#6611, @anmonteiro).
 
+- dune install now respects --display quiet mode (#7116, fixes #4573, fixes
+  #7106, @Alizter)
+
+- Stub shared libraries (dllXXX_stubs.so) in Dune-installed libraries could not
+  be used as dependencies of libraries in the workspace (eg when compiling to
+  bytecode and/or Javascript).  This is now fixed. (#7151, @nojb)
+
+- Allow the main module of a library with `(stdlib ...)` to depend on other
+  libraries (#7154, @anmonteiro).
+
+- Bytecode executables built for JSOO are linked with `-noautolink` and no
+  longer depend on the shared stubs of their dependent libraries (#7156, @nojb)
+
+- Added a new user action `(concurrent )` which is like `(progn )` but runs the
+  actions concurrently. (#6933, @Alizter)
+
+- Allow `(stdlib ...)` to be used with `(wrapped false)` in library stanzas
+  (#7139, @anmonteiro).
+
 3.7.0 (2023-02-17)
 ------------------
 

bench/bench.ml

@@ -76,9 +76,8 @@ module Package = struct
     let stdout_to = Process.Io.make_stdout Swallow in
     let stderr_to = Process.Io.make_stderr Swallow in
     let stdin_from = Process.Io.(null In) in
-    Process.run Strict
-      ~display:!Dune_engine.Clflags.display
-      ~stdout_to ~stderr_to ~stdin_from (Lazy.force git)
+    Process.run Strict ~display:Quiet ~stdout_to ~stderr_to ~stdin_from
+      (Lazy.force git)
       [ "clone"; uri t ]
 end
 
@@ -98,9 +97,7 @@ let dune_build () =
   let stderr_to = Process.Io.make_stderr Swallow in
   let open Fiber.O in
   let+ times =
-    Process.run_with_times dune
-      ~display:!Dune_engine.Clflags.display
-      ~stdin_from ~stdout_to ~stderr_to
+    Process.run_with_times dune ~display:Quiet ~stdin_from ~stdout_to ~stderr_to
       [ "build"; "@install"; "--release" ]
   in
   times.elapsed_time
@@ -128,7 +125,7 @@ let () =
   Path.Build.set_build_dir (Path.Outside_build_dir.of_string "_build");
   let module Scheduler = Dune_engine.Scheduler in
   let config =
-    Dune_engine.Clflags.display := Dune_engine.Display.Quiet;
+    Dune_engine.Clflags.display := Quiet;
     { Scheduler.Config.concurrency = 10
     ; stats = None
     ; insignificant_changes = `React

bench/dune

@@ -1,8 +1,14 @@
 (executable
  (name bench)
+ (modules bench)
  (libraries dune_stats chrome_trace stdune fiber dune_engine dune_util))
 
 (rule
  (alias bench)
  (action
   (run ./bench.exe %{bin:dune})))
+
+(executable
+ (modules gen_synthetic)
+ (libraries unix)
+ (name gen_synthetic))

bench/gen_synthetic.ml

@@ -0,0 +1,33 @@
+open Printf
+
+let write_modules basedir num_modules =
+  for current_mod = 1 to num_modules do
+    let modname = sprintf "%s/m_%d" basedir current_mod in
+    let f = open_out (sprintf "%s.ml" modname) in
+    close_out f
+  done
+
+let dune = {|
+(library
+ (name test))
+|}
+
+let write basedir =
+  let () = Unix.mkdir basedir 0o777 in
+  let f = open_out (Filename.concat basedir "dune") in
+  output_string f dune;
+  let () = close_out f in
+  write_modules basedir
+
+let () =
+  let basedir = ref "." in
+  let num_modules = ref 0 in
+  Arg.parse
+    [ ( "-n"
+      , Arg.Int (fun n -> num_modules := n)
+      , "<n>  number of modules to include in the synthetic library" )
+    ]
+    (fun d -> basedir := d)
+    (sprintf "usage: %s [basedir]" (Filename.basename Sys.argv.(0)));
+
+  write !basedir !num_modules

bench/micro/dune_bench/scheduler_bench.ml

@@ -19,8 +19,7 @@ let setup =
 
 let prog = Option.value_exn (Bin.which ~path:(Env_path.path Env.initial) "true")
 
-let run () =
-  Process.run ~display:!Clflags.display ~env:Env.initial Strict prog []
+let run () = Process.run ~display:Quiet ~env:Env.initial Strict prog []
 
 let go ~jobs fiber =
   Scheduler.Run.go

bin/coq/coqtop.ml

@@ -97,7 +97,7 @@ let term =
           let stanza =
             Dune_rules.Coq_sources.lookup_module coq_src coq_module
           in
-          let args, use_stdlib, wrapper_name =
+          let args, use_stdlib, coq_lang_version, wrapper_name =
             match stanza with
             | None ->
               User_error.raise
@@ -106,17 +106,19 @@ let term =
               ( Dune_rules.Coq_rules.coqtop_args_theory ~sctx ~dir
                   ~dir_contents:dc theory coq_module
               , theory.buildable.use_stdlib
+              , theory.buildable.coq_lang_version
               , Dune_rules.Coq_lib_name.wrapper (snd theory.name) )
             | Some (`Extraction extr) ->
               ( Dune_rules.Coq_rules.coqtop_args_extraction ~sctx ~dir extr
                   coq_module
               , extr.buildable.use_stdlib
+              , extr.buildable.coq_lang_version
               , "DuneExtraction" )
           in
           let* (_ : unit * Dep.Fact.t Dep.Map.t) =
             let deps_of =
               Dune_rules.Coq_rules.deps_of ~dir ~use_stdlib ~wrapper_name
-                coq_module
+                ~coq_lang_version coq_module
             in
             Action_builder.(run deps_of) Eager
           in

bin/diagnostics.ml

@@ -31,8 +31,9 @@ let format_diagnostic (err : Dune_rpc_private.Diagnostic.t) : User_message.t =
 let exec () =
   let open Fiber.O in
   let where = Rpc.active_server () in
+  let module Client = Dune_rpc_client.Client in
   let+ errors =
-    let* connect = Dune_rpc_impl.Client.Connection.connect_exn where in
+    let* connect = Client.Connection.connect_exn where in
     Dune_rpc_impl.Client.client connect
       (Dune_rpc_private.Initialize.Request.create
          ~id:(Dune_rpc_private.Id.make (Sexp.Atom "diagnostics_cmd")))

bin/dune

@@ -37,6 +37,7 @@
   csexp_rpc
   dune_rpc_impl
   dune_rpc_private
+  dune_rpc_client
   spawn)
  (bootstrap_info bootstrap-info))
 

bin/install_uninstall.ml

@@ -17,8 +17,13 @@ let synopsis =
        present"
   ]
 
-let print_line fmt =
-  Printf.ksprintf (fun s -> Console.print [ Pp.verbatim s ]) fmt
+let print_line ~(verbosity : Dune_engine.Display.t) fmt =
+  Printf.ksprintf
+    (fun s ->
+      match verbosity with
+      | Quiet -> ()
+      | _ -> Console.print [ Pp.verbatim s ])
+    fmt
 
 let interpret_destdir ~destdir path =
   match destdir with
@@ -136,7 +141,13 @@ module type File_operations = sig
   val remove_dir_if_exists : if_non_empty:rmdir_mode -> Path.t -> unit
 end
 
-module File_ops_dry_run : File_operations = struct
+module File_ops_dry_run (Verbosity : sig
+  val verbosity : Dune_engine.Display.t
+end) : File_operations = struct
+  open Verbosity
+
+  let print_line fmt = print_line ~verbosity fmt
+
   let copy_file ~src ~dst ~executable ~special_file:_ ~package:_ ~conf:_ =
     print_line "Copying %s to %s (executable: %b)"
       (Path.to_string_maybe_quoted src)
@@ -159,10 +170,14 @@ module File_ops_dry_run : File_operations = struct
 end
 
 module File_ops_real (W : sig
+  val verbosity : Dune_engine.Display.t
+
   val workspace : Workspace.t
 end) : File_operations = struct
   open W
 
+  let print_line = print_line ~verbosity
+
   let get_vcs p = Dune_engine.Source_tree.nearest_vcs p
 
   type load_special_file_result =
@@ -390,11 +405,16 @@ module Sections = struct
     | Only set -> Section.Set.mem set section
 end
 
-let file_operations ~dry_run ~workspace : (module File_operations) =
-  if dry_run then (module File_ops_dry_run)
+let file_operations ~verbosity ~dry_run ~workspace : (module File_operations) =
+  if dry_run then
+    (module File_ops_dry_run (struct
+      let verbosity = verbosity
+    end))
   else
     (module File_ops_real (struct
       let workspace = workspace
+
+      let verbosity = verbosity
     end))
 
 let package_is_vendored (pkg : Dune_engine.Package.t) =
@@ -648,9 +668,13 @@ let install_uninstall ~what =
                 [ Pp.text "Option --prefix is needed with --relocation" ]
           else None
         in
-
+        let verbosity =
+          match config.display with
+          | Simple display -> display.verbosity
+          | Tui -> Quiet
+        in
         let open Fiber.O in
-        let (module Ops) = file_operations ~dry_run ~workspace in
+        let (module Ops) = file_operations ~verbosity ~dry_run ~workspace in
         let files_deleted_in = ref Path.Set.empty in
         let from_command_line =
           let open Install.Section.Paths.Roots in
@@ -714,7 +738,7 @@ let install_uninstall ~what =
                               | true ->
                                 Ops.remove_dir_if_exists ~if_non_empty:Fail dst
                               | false -> Ops.remove_file_if_exists dst);
-                              print_line "%s %s" msg
+                              print_line ~verbosity "%s %s" msg
                                 (Path.to_string_maybe_quoted dst);
                               Ops.mkdir_p dir;
                               let executable =