Allow libraries without archives

CHANGES.md

@@ -120,6 +120,9 @@ Unreleased
   most cases and not correct in other cases in particular on arm32.
   (#4085, fixes #4069, fixes #2527, @emillon)
 
+- Generate archive rules compatible with 4.12. Dune longer attempt to generate
+  an archive file if it's unnecessary (#3973, fixes #3766, @rgrinberg)
+
 2.7.1 (2/09/2020)
 -----------------
 

bin/describe.ml

@@ -60,11 +60,10 @@ module Crawl = struct
         :: acc)
 
   let executables sctx ~project ~dir exes =
-    let first_exe = snd (List.hd exes.Dune_file.Executables.names) in
-    let obj_dir = Dune_file.Executables.obj_dir exes ~dir in
-    let modules_ =
+    let modules_, obj_dir =
+      let first_exe = snd (List.hd exes.Dune_file.Executables.names) in
       Dir_contents.get sctx ~dir |> Dir_contents.ocaml
-      |> Ml_sources.modules_of_executables ~first_exe ~obj_dir
+      |> Ml_sources.modules_and_obj_dir ~for_:(Exe { first_exe })
     in
     let obj_dir = Obj_dir.of_local obj_dir in
     let modules_ = modules ~obj_dir modules_ in
@@ -103,7 +102,7 @@ module Crawl = struct
         if Lib.is_local lib then
           Dir_contents.get sctx ~dir:(Path.as_in_build_dir_exn src_dir)
           |> Dir_contents.ocaml
-          |> Ml_sources.modules_of_library ~name
+          |> Ml_sources.modules ~for_:(Library name)
           |> modules ~obj_dir
         else
           []

src/dune_engine/build_system.ml

@@ -1187,7 +1187,7 @@ end = struct
       | Non_memoized : 'a Build.t -> 'a t
       | Memoized : Rule.t -> Action.t t
 
-    let build (type a) (t : a t) =
+    let build (type a) (t : a t) : a Build.t =
       match t with
       | Non_memoized build -> build
       | Memoized rule -> rule.action.build
@@ -1208,7 +1208,8 @@ end = struct
         (fun rule ->
           evaluate_and_discover_dynamic_deps_unmemoized (Memoized rule))
 
-    let evaluate_and_discover_dynamic_deps (type a) (t : a t) =
+    let evaluate_and_discover_dynamic_deps (type a) (t : a t) :
+        (a * Dep.Set.t) Fiber.t =
       match t with
       | Non_memoized _ -> evaluate_and_discover_dynamic_deps_unmemoized t
       | Memoized rule -> Memo.exec memo rule

src/dune_rules/compilation_context.ml

@@ -3,8 +3,6 @@ open! Stdune
 open Import
 module SC = Super_context
 
-let modules_of_lib = Fdecl.create Dyn.Encoder.opaque
-
 module Includes = struct
   type t = Command.Args.dynamic Command.Args.t Cm_kind.Dict.t
 
@@ -229,5 +227,4 @@ let without_bin_annot t = { t with bin_annot = false }
 let root_module_entries t : Module_name.t list Or_exn.t =
   let open Result.O in
   let* requires = t.requires_compile in
-  let local_lib = Fdecl.get modules_of_lib t.super_context in
-  Result.List.concat_map requires ~f:(Lib.entry_module_names ~local_lib)
+  Result.List.concat_map requires ~f:Lib.entry_module_names

src/dune_rules/compilation_context.mli

@@ -21,10 +21,6 @@ type opaque =
   | Inherit_from_settings
       (** Determined from the version of OCaml and the profile *)
 
-val modules_of_lib :
-  (* to avoid a cycle with [Dir_contents] *)
-  (Super_context.t -> dir:Path.Build.t -> name:Lib_name.t -> Modules.t) Fdecl.t
-
 (** Create a compilation context. *)
 val create :
      super_context:Super_context.t

src/dune_rules/dir_contents.ml

@@ -339,9 +339,9 @@ end = struct
     let f sctx ~dir ~name =
       let t = get sctx ~dir in
       let ml_sources = ocaml t in
-      Ml_sources.modules_of_library ml_sources ~name
+      Ml_sources.modules ml_sources ~for_:(Library name)
     in
-    Fdecl.set Compilation_context.modules_of_lib f
+    Fdecl.set Super_context.modules_of_lib f
 
   let gen_rules sctx ~dir =
     match Memo.exec memo0 (sctx, dir) with

src/dune_rules/dune_file.ml

@@ -771,8 +771,9 @@ module Library = struct
     List.map (foreign_archives t) ~f:(fun archive ->
         Foreign.Archive.dll_file ~archive ~dir ~ext_dll)
 
-  let archive t ~dir ~ext =
-    Path.Build.relative dir (Lib_name.Local.to_string (snd t.name) ^ ext)
+  let archive_basename t ~ext = Lib_name.Local.to_string (snd t.name) ^ ext
+
+  let archive t ~dir ~ext = Path.Build.relative dir (archive_basename t ~ext)
 
   let best_name t =
     match t.visibility with
@@ -834,10 +835,18 @@ module Library = struct
     let virtual_library = is_virtual conf in
     let foreign_archives = foreign_lib_files conf ~dir ~ext_lib in
     let native_archives =
-      if modes.native then
-        [ archive ext_lib ]
+      let archive = archive ext_lib in
+      if virtual_library || not modes.native then
+        Lib_info.Files []
+      else if
+        Option.is_some conf.implements
+        || Lib_config.linker_can_create_empty_archives lib_config
+           && Ocaml_version.ocamlopt_always_calls_library_linker
+                lib_config.ocaml_version
+      then
+        Lib_info.Files [ archive ]
       else
-        []
+        Lib_info.Needs_module_info archive
     in
     let foreign_dll_files = foreign_dll_files conf ~dir ~ext_dll in
     let exit_module = Option.bind conf.stdlib ~f:(fun x -> x.exit_module) in
@@ -916,12 +925,12 @@ module Library = struct
     let special_builtin_support = conf.special_builtin_support in
     let instrumentation_backend = conf.instrumentation_backend in
     let entry_modules = Lib_info.Source.Local in
-    Lib_info.create ~loc ~name ~kind ~status ~src_dir ~orig_src_dir ~obj_dir
-      ~version ~synopsis ~main_module_name ~sub_systems ~requires
-      ~foreign_objects ~plugins ~archives ~ppx_runtime_deps ~foreign_archives
-      ~native_archives ~foreign_dll_files ~jsoo_runtime ~jsoo_archive
-      ~preprocess ~enabled ~virtual_deps ~dune_version ~virtual_ ~entry_modules
-      ~implements ~default_implementation ~modes ~wrapped
+    Lib_info.create ~loc ~path_kind:Local ~name ~kind ~status ~src_dir
+      ~orig_src_dir ~obj_dir ~version ~synopsis ~main_module_name ~sub_systems
+      ~requires ~foreign_objects ~plugins ~archives ~ppx_runtime_deps
+      ~foreign_archives ~native_archives ~foreign_dll_files ~jsoo_runtime
+      ~jsoo_archive ~preprocess ~enabled ~virtual_deps ~dune_version ~virtual_
+      ~entry_modules ~implements ~default_implementation ~modes ~wrapped
       ~special_builtin_support ~exit_module ~instrumentation_backend
 end
 

src/dune_rules/dune_package.ml

@@ -26,6 +26,11 @@ module Lib = struct
     let info = Lib_info.map_path info ~f:map_path in
     { info; main_module_name; modules }
 
+  let of_dune_lib ~info ~main_module_name ~modules =
+    make ~info ~main_module_name ~modules:(Some modules)
+
+  let of_findlib info = make ~info ~main_module_name:None ~modules:None
+
   let dir_of_name name =
     let _, components = Lib_name.split name in
     Path.Local.L.relative Path.Local.root components
@@ -61,6 +66,12 @@ module Lib = struct
     let jsoo_runtime = Lib_info.jsoo_runtime info in
     let virtual_ = Option.is_some (Lib_info.virtual_ info) in
     let instrumentation_backend = Lib_info.instrumentation_backend info in
+    let native_archives =
+      match Lib_info.native_archives info with
+      | Lib_info.Files f -> f
+      | Needs_module_info _ ->
+        Code_error.raise "caller must set native archives to known value" []
+    in
     record_fields
     @@ [ field "name" Lib_name.encode name
        ; field "kind" Lib_kind.encode kind
@@ -71,7 +82,7 @@ module Lib = struct
        ; mode_paths "plugins" plugins
        ; paths "foreign_objects" foreign_objects
        ; paths "foreign_archives" (Lib_info.foreign_archives info)
-       ; paths "native_archives" (Lib_info.native_archives info)
+       ; paths "native_archives" native_archives
        ; paths "jsoo_runtime" jsoo_runtime
        ; Lib_dep.L.field_encode requires ~name:"requires"
        ; libs "ppx_runtime_deps" ppx_runtime_deps
@@ -181,10 +192,11 @@ module Lib = struct
            Some (Lib_info.Inherited.This (Modules.wrapped modules))
          in
          let entry_modules = Lib_info.Source.External (Ok entry_modules) in
-         Lib_info.create ~loc ~name ~kind ~status ~src_dir ~orig_src_dir
-           ~obj_dir ~version ~synopsis ~main_module_name ~sub_systems ~requires
-           ~foreign_objects ~plugins ~archives ~ppx_runtime_deps
-           ~foreign_archives ~native_archives ~foreign_dll_files:[]
+         Lib_info.create ~path_kind:External ~loc ~name ~kind ~status ~src_dir
+           ~orig_src_dir ~obj_dir ~version ~synopsis ~main_module_name
+           ~sub_systems ~requires ~foreign_objects ~plugins ~archives
+           ~ppx_runtime_deps ~foreign_archives
+           ~native_archives:(Files native_archives) ~foreign_dll_files:[]
            ~jsoo_runtime ~jsoo_archive ~preprocess ~enabled ~virtual_deps
            ~dune_version ~virtual_ ~entry_modules ~implements
            ~default_implementation ~modes ~wrapped ~special_builtin_support

src/dune_rules/dune_package.mli

@@ -19,10 +19,12 @@ module Lib : sig
 
   val info : t -> Path.t Lib_info.t
 
-  val make :
+  val of_findlib : Path.t Lib_info.t -> t
+
+  val of_dune_lib :
        info:Path.t Lib_info.t
     -> main_module_name:Module_name.t option
-    -> modules:Modules.t option
+    -> modules:Modules.t
     -> t
 
   val to_dyn : t Dyn.Encoder.t
@@ -76,8 +78,6 @@ module Or_meta : sig
     | Use_meta
     | Dune_package of t
 
-  val encode : dune_version:Dune_lang.Syntax.Version.t -> t -> Dune_lang.t list
-
   val pp :
     dune_version:Dune_lang.Syntax.Version.t -> Format.formatter -> t -> unit
 

src/dune_rules/exe_rules.ml

@@ -92,13 +92,12 @@ let executables_rules ~sctx ~dir ~expander ~dir_contents ~scope ~compile_info
     ~embed_in_plugin_libraries (exes : Dune_file.Executables.t) =
   (* Use "eobjs" rather than "objs" to avoid a potential conflict with a library
      of the same name *)
-  let obj_dir = Dune_file.Executables.obj_dir exes ~dir in
-  Check_rules.add_obj_dir sctx ~obj_dir;
-  let modules =
+  let modules, obj_dir =
     let first_exe = first_exe exes in
-    let ml_sources = Dir_contents.ocaml dir_contents in
-    Ml_sources.modules_of_executables ml_sources ~first_exe ~obj_dir
+    Dir_contents.ocaml dir_contents
+    |> Ml_sources.modules_and_obj_dir ~for_:(Exe { first_exe })
   in
+  Check_rules.add_obj_dir sctx ~obj_dir;
   let ctx = Super_context.context sctx in
   let pp =
     let preprocess =

src/dune_rules/findlib/findlib.ml

@@ -410,16 +410,17 @@ end = struct
                         | Ok s -> Ok (Some s)
                         | Error e -> Error (User_error.E e) )) ) )
         in
-        Lib_info.create ~loc ~name:t.name ~kind ~status ~src_dir ~orig_src_dir
-          ~obj_dir ~version ~synopsis ~main_module_name ~sub_systems ~requires
-          ~foreign_objects ~plugins ~archives ~ppx_runtime_deps
-          ~foreign_archives ~native_archives ~foreign_dll_files:[] ~jsoo_runtime
-          ~jsoo_archive ~preprocess ~enabled ~virtual_deps ~dune_version
-          ~virtual_ ~implements ~default_implementation ~modes ~wrapped
-          ~special_builtin_support ~exit_module:None
+        Lib_info.create ~path_kind:External ~loc ~name:t.name ~kind ~status
+          ~src_dir ~orig_src_dir ~obj_dir ~version ~synopsis ~main_module_name
+          ~sub_systems ~requires ~foreign_objects ~plugins ~archives
+          ~ppx_runtime_deps ~foreign_archives
+          ~native_archives:(Files native_archives) ~foreign_dll_files:[]
+          ~jsoo_runtime ~jsoo_archive ~preprocess ~enabled ~virtual_deps
+          ~dune_version ~virtual_ ~implements ~default_implementation ~modes
+          ~wrapped ~special_builtin_support ~exit_module:None
           ~instrumentation_backend:None ~entry_modules
       in
-      Dune_package.Lib.make ~info ~modules:None ~main_module_name:None
+      Dune_package.Lib.of_findlib info
   end
 
   (* Parse all the packages defined in a META file *)

src/dune_rules/install_rules.ml

@@ -53,11 +53,10 @@ end = struct
     else
       []
 
-  let lib_files ~modes ~dir_contents ~dir ~lib_config lib =
+  let lib_files ~dir_contents ~dir ~lib_config lib =
     let virtual_library = Option.is_some (Lib_info.virtual_ lib) in
     let { Lib_config.ext_obj; _ } = lib_config in
     let archives = Lib_info.archives lib in
-    let { Mode.Dict.byte = _; native } = modes in
     List.concat
       [ archives.byte
       ; archives.native
@@ -68,18 +67,12 @@ end = struct
           Foreign.Sources.object_files files ~dir ~ext_obj
         else
           Lib_info.foreign_archives lib )
-      ; if_
-          (native && not virtual_library)
-          ((* TODO remove the if check once Lib_info.native_archives always
-              returns the correct value for libs without modules *)
-           let modules =
-             Dir_contents.ocaml dir_contents
-             |> Ml_sources.modules_of_library ~name:(Lib_info.name lib)
-           in
-           if Lib_info.has_native_archive lib_config modules then
-             Lib_info.native_archives lib
-           else
-             [])
+      ; (let modules =
+           Dir_contents.ocaml dir_contents
+           |> Ml_sources.modules ~for_:(Library (Lib_info.name lib))
+           |> Option.some
+         in
+         Lib_info.eval_native_archives_exn lib ~modules)
       ; Lib_info.jsoo_runtime lib
       ; (Lib_info.plugins lib).native
       ]
@@ -118,7 +111,7 @@ end = struct
     in
     let installable_modules =
       Dir_contents.ocaml dir_contents
-      |> Ml_sources.modules_of_library ~name:(Library.best_name lib)
+      |> Ml_sources.modules ~for_:(Library (Library.best_name lib))
       |> Modules.fold_no_vlib ~init:[] ~f:(fun m acc -> m :: acc)
     in
     let sources =
@@ -183,7 +176,7 @@ end = struct
           other_cm_files)
     in
     let lib_files, dll_files =
-      let lib_files = lib_files ~modes ~dir ~dir_contents ~lib_config info in
+      let lib_files = lib_files ~dir ~dir_contents ~lib_config info in
       let dll_files = dll_files ~modes ~dynlink:lib.dynlink ~ctx info in
       (lib_files, dll_files)
     in
@@ -450,7 +443,7 @@ end = struct
             in
             let modules =
               Dir_contents.ocaml dir_contents
-              |> Ml_sources.modules_of_library ~name
+              |> Ml_sources.modules ~for_:(Library name)
             in
             Lib_name.Map.add_exn acc name
               (Library