feat: per-module narrowing for Melange consumers

doc/dev/per-module-narrowing.md

@@ -45,8 +45,7 @@ the source-language kind (`Ml_kind.t` — `Impl` or `Intf`), and the mode
 1. If a small set of preconditions fails (`can_filter = false`), fall
    back to the cctx-wide glob — the same dep set every compile rule
    would have had prior to this work. This avoids needing soundness
-   arguments for module kinds that the narrowing was not designed for,
-   and lets `Melange` paths pass through unchanged.
+   arguments for module kinds that the narrowing was not designed for.
 
 2. Otherwise, if the consumer cctx already carries a virtual library
    in `requires` (`has_virtual_impl = true`), fall back to the
@@ -95,8 +94,6 @@ The entry point for the narrowing is
 (`src/dune_rules/module_compilation.ml`). Before any narrowing runs,
 the function computes a `can_filter` boolean conjunction:
 
-- The compile is `Ocaml _` (Melange has its own cm-kind story and is
-  not narrowed).
 - The supplied `dep_graph` is the real one for this cctx (its `dir`
   equals the cctx's `obj_dir`), not a `Dep_graph.dummy` — synthesised
   / link-time-generated / alias / root modules have no usable
@@ -566,9 +563,18 @@ Three categories of fallback to the glob:
    inherited into the intermediate's `.mli` via the open) would
    silently drop the leaf library's `.cmi` from its compile rule.
 
-`Melange` paths bypass the narrowing entirely at the `can_filter`
-check; the cm_kind machinery there is different and L9 leaves it
-unchanged.
+`Melange` consumer compiles run the same narrowing pipeline as `Ocaml`.
+The `can_filter` check is mode-agnostic. Two code paths inside the
+pipeline match on `cm_kind`:
+
+1. `Lib_file_deps.deps_of_entry_modules` emits per-module `.cmj` in
+   addition to `.cmi` when `cm_kind = Melange Cmj`, mirroring the
+   broad-dep path's `groups_for_cm_kind`.
+2. `need_impl_deps_of` (in `lib_deps_for_module`) reads a trans-dep's
+   `.ml`-side ocamldep only for `cm_kind = Ocaml Cmx` (cross-module
+   inlining input). `Ocaml (Cmi | Cmo)` and `Melange _` are handled
+   symmetrically — neither reads impl deps. The distinction is
+   Cmx-vs-rest, not OCaml-vs-Melange.
 
 ## Cost characteristics
 

src/dune_rules/dep_rules.ml

@@ -506,10 +506,11 @@ let for_module ~obj_dir ~modules ~sandbox ~impl ~dir ~sctx ~for_ module_ =
 
 let rules ~obj_dir ~modules ~sandbox ~impl ~sctx ~dir ~for_ ~has_library_deps =
   match Modules.With_vlib.as_singleton modules with
-  | Some m when (not has_library_deps) || Compilation_mode.equal for_ Melange ->
-    (* Single-module stanzas have no intra-stanza deps; the dep graph is only
-       consumed by the per-module filter in [lib_deps_for_module]. That filter
-       doesn't activate for Melange (see its [can_filter]) — skip ocamldep. *)
+  | Some m when not has_library_deps ->
+    (* Single-module stanzas with no library deps have nothing to filter — the
+       dep graph is only consumed by the per-module filter in
+       [lib_deps_for_module], and that filter has no libs to narrow. Skip
+       ocamldep. *)
     Memo.return (Dep_graph.Ml_kind.dummy m)
   | Some _ | None ->
     let transitive_deps, imported_vlib_deps =

src/dune_rules/lib_file_deps.ml

@@ -85,6 +85,11 @@ let deps_of_entry_modules ~opaque ~(cm_kind : Lib_mode.Cm_kind.t) lib modules =
     | Ocaml Cmx -> not (opaque && Lib.is_local lib)
     | _ -> false
   in
+  let want_cmj =
+    match cm_kind with
+    | Melange Cmj -> true
+    | _ -> false
+  in
   List.fold_left modules ~init:Dep.Set.empty ~f:(fun acc m ->
     let acc =
       match Obj_dir.Module.cm_public_file obj_dir m ~kind:cmi_kind with
@@ -97,16 +102,30 @@ let deps_of_entry_modules ~opaque ~(cm_kind : Lib_mode.Cm_kind.t) lib modules =
            in tight_modules"
           [ "module", Module.to_dyn m; "lib", Lib.to_dyn lib ]
     in
-    if want_cmx && Module.has m ~ml_kind:Impl
+    let acc =
+      if want_cmx && Module.has m ~ml_kind:Impl
+      then (
+        match Obj_dir.Module.cm_public_file obj_dir m ~kind:(Ocaml Cmx) with
+        | Some path -> Dep.Set.add acc (Dep.file path)
+        | None ->
+          (* Unreachable. [cm_public_file ~kind:(Ocaml Cmx)] returns [None] only
+             when [not has_impl]. The enclosing [if] guarantees
+             [Module.has m ~ml_kind:Impl] (= [has_impl]). *)
+          Code_error.raise
+            "deps_of_entry_modules: [cm_public_file] returned [None] for cmx despite \
+             [Module.has m ~ml_kind:Impl] holding"
+            [ "module", Module.to_dyn m ])
+      else acc
+    in
+    if want_cmj && Module.has m ~ml_kind:Impl
     then (
-      match Obj_dir.Module.cm_public_file obj_dir m ~kind:(Ocaml Cmx) with
+      match Obj_dir.Module.cm_public_file obj_dir m ~kind:(Melange Cmj) with
       | Some path -> Dep.Set.add acc (Dep.file path)
       | None ->
-        (* Unreachable. [cm_public_file ~kind:(Ocaml Cmx)] returns [None] only
-           when [not has_impl]. The enclosing [if] guarantees
-           [Module.has m ~ml_kind:Impl] (= [has_impl]). *)
+        (* Unreachable. Symmetric with the [Ocaml Cmx] arm above: [cm_public_file
+           ~kind:(Melange Cmj)] returns [None] only when [not has_impl]. *)
         Code_error.raise
-          "deps_of_entry_modules: [cm_public_file] returned [None] for cmx despite \
+          "deps_of_entry_modules: [cm_public_file] returned [None] for cmj despite \
            [Module.has m ~ml_kind:Impl] holding"
           [ "module", Module.to_dyn m ])
     else acc)

src/dune_rules/lib_file_deps.mli

@@ -22,12 +22,10 @@ val deps_of_entries : opaque:bool -> cm_kind:Lib_mode.Cm_kind.t -> Lib.t list ->
 (** Specific-file deps on the [modules] of [lib]. Only valid for local libraries
     (where [Module.t] values are available).
 
-    Currently produces complete per-module deps only for [cm_kind = Ocaml _]
-    (cmi + cmx); for [Melange _] only the cmi is emitted — there is no
-    per-module cmj arm, asymmetric with [deps_of_entries]. The sole caller
-    ([Module_compilation.lib_deps_for_module]) gates Melange out before
-    reaching this function, so this asymmetry is not observable today. If a
-    future Melange caller is added, extend with a [want_cmj] arm. *)
+    Per [cm_kind]: [Ocaml Cmx] emits cmi + cmx (cmx omitted when [opaque] and
+    [Lib.is_local lib]); [Ocaml (Cmi | Cmo)] emits cmi only; [Melange Cmj]
+    emits cmi + cmj; [Melange Cmi] emits cmi only. The cmi/cmj selection
+    mirrors [groups_for_cm_kind] used by the broad-dep path. *)
 val deps_of_entry_modules
   :  opaque:bool
   -> cm_kind:Lib_mode.Cm_kind.t

src/dune_rules/melange/melange_rules.ml

@@ -542,6 +542,11 @@ let setup_emit_cmj_rules
       Modules.With_vlib.modules modules, pp
     in
     let requires_link = Lib.Compile.requires_link compile_info ~for_ in
+    let pps_runtime_libs =
+      let open Resolve.Memo.O in
+      let* pps = Lib.Compile.pps compile_info ~for_ in
+      Resolve.Memo.List.concat_map pps ~f:(Lib.ppx_runtime_deps ~for_)
+    in
     let* flags = melange_compile_flags ~sctx ~dir mel in
     let* cctx =
       let direct_requires = Lib.Compile.direct_requires compile_info ~for_ in
@@ -555,7 +560,7 @@ let setup_emit_cmj_rules
         ~flags
         ~requires_link
         ~requires_compile:direct_requires
-        ~pps_runtime_libs:(Resolve.Memo.return [])
+        ~pps_runtime_libs
         ~preprocessing:pp
         ~js_of_ocaml:(Js_of_ocaml.Mode.Pair.make None)
         ~opaque:Inherit_from_settings

src/dune_rules/module_compilation.ml

@@ -90,13 +90,10 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin
     Lib_mode.Cm_kind.Map.get (Compilation_context.includes cctx) cm_kind
   in
   let can_filter =
-    (match Lib_mode.of_cm_kind cm_kind with
-     | Melange -> false
-     | Ocaml _ -> true)
     (* Skip when [dep_graph] is the dummy ([Dep_graph.dummy] with
        [dir = Path.Build.root]); used for singleton-module stanzas and
        link-time-synthesised modules where no transitive deps are available. *)
-    && Path.Build.equal (Dep_graph.dir dep_graph) (Obj_dir.dir obj_dir)
+    Path.Build.equal (Dep_graph.dir dep_graph) (Obj_dir.dir obj_dir)
     (* Modules synthesised outside the stanza, handed to [ocamlc_i]. *)
     && Dep_graph.mem dep_graph m
     && module_kind_is_filterable m
@@ -183,7 +180,22 @@ let lib_deps_for_module ~cctx ~obj_dir ~for_ ~dep_graph ~opaque ~cm_kind ~ml_kin
       let all_raw = Module_name.Set.union m_raw trans_raw in
       let* flags = Ocaml_flags.get (Compilation_context.flags cctx) mode in
       let open_modules = Ocaml_flags.extract_open_module_names flags in
-      let referenced = Module_name.Set.union all_raw open_modules in
+      (* [Stdlib] is implicitly opened by every compile and so is referenced by
+         every consumer even when ocamldep doesn't list it. For OCaml-mode
+         compiles the compiler's built-in stdlib path supplies it (no dune lib
+         to keep); for Melange-mode compiles the [melange]/[stdlib] lib supplies
+         it as a regular dune dependency, and would otherwise be dropped from
+         [kept_libs] for consumers that don't syntactically name a [Stdlib.X]
+         member. Force it onto the frontier so the [Lib_index] lookup adds it
+         when present. *)
+      let referenced =
+        let implicit_stdlib =
+          match Module_name.of_string_opt "Stdlib" with
+          | Some n -> Module_name.Set.singleton n
+          | None -> Module_name.Set.empty
+        in
+        Module_name.Set.union (Module_name.Set.union all_raw open_modules) implicit_stdlib
+      in
       let { Lib_file_deps.Lib_index.tight; non_tight } =
         Lib_file_deps.Lib_index.filter_libs_with_modules
           lib_index

test/blackbox-tests/test-cases/melange/depend-on-installed.t

@@ -57,6 +57,7 @@ Test dependency on installed package
       ocamldep (internal)
       ocamldep (internal)
           melc .b.objs/melange/b.{cmi,cmj,cmt}
+      ocamldep (internal)
           melc .b.objs/melange/b__Bar.{cmi,cmj,cmt}
           melc .b.objs/melange/b__Foo.{cmi,cmj,cmt}
   Leaving directory 'b'

test/blackbox-tests/test-cases/melange/emit-installed-two-modes.t

@@ -59,6 +59,7 @@ Test dependency on installed package
   Entering directory 'b'
           melc dist/node_modules/a/a.js
           melc dist/node_modules/a/foo.js
+      ocamldep (internal)
           melc dist/bar.js
   Leaving directory 'b'
 

test/blackbox-tests/test-cases/melange/emit-installed.t

@@ -62,6 +62,7 @@ Test dependency on installed package
           melc dist/node_modules/a/a.js
           melc dist/node_modules/a/foo.js
           melc dist/node_modules/a/sub/sub.js
+      ocamldep (internal)
           melc dist/bar.js
   Leaving directory 'b'
 

test/blackbox-tests/test-cases/melange/emit-private.t

@@ -47,6 +47,7 @@ Test dependency on a private library in the same package as melange.emit
   > EOF
 
   $ OCAMLPATH=$PWD/prefix/lib/:$OCAMLPATH dune build @dist --display=short 2>&1 | grep -v melange
+      ocamldep (internal)
           melc b/dist/node_modules/pkg.__private__.a/foo.js
           melc b/dist/b/bar.js
 

test/blackbox-tests/test-cases/melange/melange-conditional-modules.t

@@ -34,8 +34,9 @@ Show `(melange.modules ..)` subset is preferred when compiling in Melange mode
           melc lib/.a.objs/melange/a.{cmi,cmj,cmt}
       ocamldep (internal)
       ocamldep (internal)
-          melc lib/.a.objs/melange/a__Foo.{cmi,cmj,cmt}
+      ocamldep (internal)
         ocamlc lib/.a.objs/byte/a__Helper.{cmi,cmo,cmt}
+          melc lib/.a.objs/melange/a__Foo.{cmi,cmj,cmt}
         ocamlc lib/.a.objs/byte/a__Foo.{cmi,cmo,cmt}
         ocamlc lib/a.cma
   Leaving directory 'a'

test/blackbox-tests/test-cases/melange/missing-melc.t

@@ -88,4 +88,13 @@ But trying to build any melange artifacts will fail
   Error: Program melc not found in the tree or in PATH
    (context: default)
   Hint: opam install melange
+  File "dune", lines 10-14, characters 0-94:
+  10 | (melange.emit
+  11 |  (target output)
+  12 |  (emit_stdlib false)
+  13 |  (modules main_melange)
+  14 |  (libraries lib1))
+  Error: Program melc not found in the tree or in PATH
+   (context: default)
+  Hint: opam install melange
   [1]