Ninja: use header dependencies for better rebuild support

configure.py

@@ -1249,6 +1249,9 @@ def __init__(self, infofile):
                 'ar_output_to': '',
                 'werror_flags': '',
                 'supports_gcc_inline_asm': 'no',
+                'ninja_header_deps_style': '',
+                'header_deps_flag': '',
+                'header_deps_out': '',
             })
 
         self.add_framework_option = lex.add_framework_option
@@ -1293,6 +1296,9 @@ def __init__(self, infofile):
         self.warning_flags = lex.warning_flags
         self.werror_flags = lex.werror_flags
         self.minimum_supported_version = lex.minimum_supported_version
+        self.ninja_header_deps_style = lex.ninja_header_deps_style
+        self.header_deps_flag = lex.header_deps_flag
+        self.header_deps_out = lex.header_deps_out
 
     def cross_check(self, os_info, arch_info, all_isas):
 
@@ -1728,23 +1734,34 @@ class SimpleTemplate:
 
         def __init__(self, vals):
             self.vals = vals
-            self.value_pattern = re.compile(r'%{([a-z][a-z_0-9\|]+)}')
+            self.value_pattern = re.compile(r'%{([a-z][a-z_0-9\|]+)(?::([^}]+))?}')
             self.cond_pattern = re.compile('%{(if|unless) ([a-z][a-z_0-9]+)}')
             self.for_pattern = re.compile('(.*)%{for ([a-z][a-z_0-9]+)}')
             self.omitlast_pattern = re.compile('(.*)%{omitlast ([^}]*)}(.*)', re.DOTALL)
             self.join_pattern = re.compile('%{join ([a-z][a-z_0-9]+)}')
 
         def substitute(self, template):
+            def get_replacement(k):
+                if k not in self.vals:
+                    raise KeyError(k)
+                return str(self.vals.get(k))
+
             def insert_value(match):
-                v = match.group(1)
-                if v in self.vals:
-                    return str(self.vals.get(v))
-                if v.endswith('|upper'):
-                    v = v.replace('|upper', '')
-                    if v in self.vals:
-                        return str(self.vals.get(v)).upper()
+                k = match.group(1)
+                if k.endswith('|upper'):
+                    k = k.replace('|upper', '')
+                    v = get_replacement(k).upper()
+                elif k.endswith('|concat'):
+                    k = k.replace('|concat', '')
+                    if not match.group(2):
+                        raise InternalError("|concat must be of the form '%{val|concat:<some static value>}'")
+                    v = get_replacement(k)
+                    if v:
+                        v = f"{v}{match.group(2)}"
+                else:
+                    v = get_replacement(k)
 
-                raise KeyError(v)
+                return v
 
             def insert_join(match):
                 var = match.group(1)
@@ -2221,6 +2238,9 @@ def test_exe_extra_ldflags():
         'cc_warning_flags': cc.cc_warning_flags(options),
         'output_to_exe': cc.output_to_exe,
         'cc_macro': cc.macro_name,
+        'ninja_header_deps_style': cc.ninja_header_deps_style,
+        'header_deps_flag': cc.header_deps_flag,
+        'header_deps_out': cc.header_deps_out,
 
         'visibility_attribute': cc.gen_visibility_attribute(options),
 

doc/dev_ref/configure.rst

@@ -77,6 +77,12 @@ to the output unmodified. The template elements are:
    If a variable reference ends with ``|upper``, the value is uppercased before
    being inserted into the template output.
 
+   Using ``|concat:<some string>`` as a suffix, it is possible to conditionally
+   concatenate the variable value with a static string defined in the template.
+   This is useful for compiler switches that require a template-defined
+   parameter value. If the substitution value is not set (i.e. "empty"), also
+   the static concatenation value is omitted.
+
  * Iteration, ``%{for variable} block %{endfor}``. This iterates over a list and
    repeats the block as many times as it is included. Variables within the block
    are expanded. The two template elements ``%{for ...}`` and ``%{endfor}`` must
@@ -382,6 +388,11 @@ Variables:
     when generation shared libraries.
   * ``visibility_attribute`` gives the attribute to use in the ``BOTAN_DLL`` macro
     to specify visibility when generation shared libraries.
+  * ``ninja_header_deps_style`` style of include dependency tracking for Ninja,
+    see also https://ninja-build.org/manual.html#ref_headers.
+  * ``header_deps_flag`` flag to write out dependency information in the style
+    required by ``ninja_header_deps_style``.
+  * ``header_deps_out`` flag to specify name of the dependency output file.
   * ``ar_command`` gives the command to build static libraries
   * ``ar_options`` gives the options to pass to ``ar_command``, if not set here
     takes this from the OS specific information.

src/build-data/cc/clang.txt

@@ -37,6 +37,12 @@ stack_protector_flags "-fstack-protector"
 visibility_build_flags "-fvisibility=hidden"
 visibility_attribute '__attribute__((visibility("default")))'
 
+# Include dependency tracking for Ninja
+# See: https://ninja-build.org/manual.html#ref_headers
+ninja_header_deps_style 'gcc'
+header_deps_flag '-MD'
+header_deps_out '-MF'
+
 <so_link_commands>
 macos         -> "{cxx} -dynamiclib -fPIC -install_name {libdir}/{soname_abi} -current_version {macos_so_current_ver} -compatibility_version {macos_so_compat_ver}"
 

src/build-data/cc/gcc.txt

@@ -39,6 +39,12 @@ undefined -> "-fsanitize=undefined -fno-sanitize-recover=undefined"
 visibility_build_flags "-fvisibility=hidden"
 visibility_attribute '__attribute__((visibility("default")))'
 
+# Include dependency tracking for Ninja
+# See: https://ninja-build.org/manual.html#ref_headers
+ninja_header_deps_style 'gcc'
+header_deps_flag '-MD'
+header_deps_out '-MF'
+
 <so_link_commands>
 # The default works for GNU ld and several other Unix linkers
 default       -> "{cxx} -shared -fPIC -Wl,-soname,{soname_abi}"

src/build-data/cc/msvc.txt

@@ -39,6 +39,11 @@ werror_flags "/WX"
 visibility_build_flags "/DBOTAN_DLL=__declspec(dllexport)"
 visibility_attribute "__declspec(dllimport)"
 
+# Include dependency tracking for Ninja
+# See: https://ninja-build.org/manual.html#ref_headers
+ninja_header_deps_style 'msvc'
+header_deps_flag '/showIncludes'
+
 ar_command lib
 ar_options "/nologo"
 ar_output_to "/OUT:"

src/build-data/ninja.in

@@ -22,13 +22,31 @@ SCRIPTS_DIR    = %{scripts_dir}
 INSTALLED_LIB_DIR = %{libdir}
 
 rule compile_lib
-  command = %{cxx} %{lib_flags} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} ${isa_flags} %{include_paths} %{dash_c} $in %{dash_o}$out
+%{if header_deps_out}
+  depfile = $out.d
+%{endif}
+%{if ninja_header_deps_style}
+  deps = %{ninja_header_deps_style}
+%{endif}
+  command = %{cxx} %{lib_flags} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} ${isa_flags} %{include_paths} %{header_deps_flag} %{header_deps_out|concat: $out.d} %{dash_c} $in %{dash_o}$out
 
 rule compile_exe
-  command = %{cxx} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} ${isa_flags} %{include_paths} %{dash_c} $in %{dash_o}$out
+%{if header_deps_out}
+  depfile = $out.d
+%{endif}
+%{if ninja_header_deps_style}
+  deps = %{ninja_header_deps_style}
+%{endif}
+  command = %{cxx} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} ${isa_flags} %{include_paths} %{header_deps_flag} %{header_deps_out|concat: $out.d} %{dash_c} $in %{dash_o}$out
 
 rule compile_example_exe
-  command = %{cxx} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} ${WARN_FLAGS} ${isa_flags} %{include_paths} %{dash_c} $in %{dash_o}$out
+%{if header_deps_out}
+  depfile = $out.d
+%{endif}
+%{if ninja_header_deps_style}
+  deps = %{ninja_header_deps_style}
+%{endif}
+  command = %{cxx} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} ${WARN_FLAGS} ${isa_flags} %{include_paths} %{header_deps_flag} %{header_deps_out|concat: $out.d} %{dash_c} $in %{dash_o}$out
 
 # The primary target
 build all: phony %{all_targets}

src/editors/vscode/tasks.json

@@ -6,7 +6,7 @@
             "detail": "Default ./configure.py invocation for gcc. Run your own if you want.",
             "group": "build",
             "type": "shell",
-            "command": "python3 ./configure.py --cc gcc --compiler-cache=ccache --without-documentation --debug-mode --build-targets=\"static,tests,bogo_shim\"",
+            "command": "python3 ./configure.py --cc gcc --compiler-cache=ccache --build-tool=ninja --without-documentation --debug-mode --build-targets=\"static,tests,bogo_shim\"",
             "presentation": {
                 "reveal": "always",
                 "panel": "shared",
@@ -22,7 +22,7 @@
                 "args": [
                 ]
             },
-            "command": "python ./configure.py --cc msvc --compiler-cache=sccache.exe --without-documentation --debug-mode --build-targets=\"static,tests\"",
+            "command": "python ./configure.py --cc msvc --compiler-cache=sccache.exe --build-tool=ninja --link-method=hardlink --without-documentation --debug-mode --build-targets=\"static,tests\"",
             "presentation": {
                 "reveal": "always",
                 "panel": "shared",