Hermetic pkg config (#979)

.bazelci/config.yaml

@@ -131,8 +131,6 @@ tasks:
       - "-//cmake_with_data/..."
     build_targets: *windows_targets
     test_targets: *windows_targets
-    test_flags:
-      - "--enable_runfiles"
   rbe_ubuntu1604_flags:
     name: Flags
     platform: rbe_ubuntu1604
@@ -193,8 +191,14 @@ tasks:
     platform: ubuntu1804
     build_targets:
       - "//..."
+    # experimental_enable_aggregating_middleman=False is required in bazel 4 otherwise ctx.resolve_command fails.
+    # This was resolved in https://github.com/bazelbuild/bazel/commit/fb1c369530bd26f9a13560c5979929a999e585e2
+    build_flags:
+      - "--experimental_enable_aggregating_middleman=False"
     test_targets:
       - "//..."
+    test_flags:
+      - "--experimental_enable_aggregating_middleman=False"
   min_supported_version_examples:
     name: "Minimum Supported Version Examples"
     bazel: "4.0.0"
@@ -203,7 +207,12 @@ tasks:
     min_supported_targets: &min_supported_targets
       - "//..."
     build_targets: *min_supported_targets
+    # See comment above regarding --experimental_enable_aggregating_middleman=False
+    build_flags:
+      - "--experimental_enable_aggregating_middleman=False"
     test_targets: *min_supported_targets
+    test_flags:
+      - "--experimental_enable_aggregating_middleman=False"
 
 buildifier:
   version: "4.2.5" 

README.md

@@ -13,7 +13,11 @@ Documentation for all rules and providers are available at: https://bazelbuild.g
 
 ## Bazel versions compatibility
 
-Works with Bazel after 4.0.0 without any flags.
+Works with Bazel after 4.0.0. 
+
+The following flags are required in Bazel 4.x but not Bazel 5.x or newer:
+
+- `--experimental_enable_aggregating_middleman=False`
 
 Note that the rules may be compatible with older versions of Bazel but support may break
 in future changes as these older versions are not tested.

WORKSPACE.bazel

@@ -3,7 +3,7 @@ workspace(name = "rules_foreign_cc")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
 
-rules_foreign_cc_dependencies()
+rules_foreign_cc_dependencies(register_built_pkgconfig_toolchain = True)
 
 local_repository(
     name = "rules_foreign_cc_examples",

examples/.bazelrc

@@ -0,0 +1,8 @@
+
+# Required for Windows
+build --enable_runfiles
+
+# These are required otherwise paths are too long
+startup --windows_enable_symlinks
+build --action_env=MSYS=winsymlinks:nativestrict
+test --action_env=MSYS=winsymlinks:nativestrict

examples/WORKSPACE.bazel

@@ -7,8 +7,7 @@ local_repository(
 
 load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
 
-# Don't use preinstalled tools to ensure builds are as hermetic as possible
-rules_foreign_cc_dependencies(register_preinstalled_tools = False)
+rules_foreign_cc_dependencies(register_built_pkgconfig_toolchain = True)
 
 load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
 

examples/third_party/WORKSPACE.bazel

@@ -7,7 +7,7 @@ local_repository(
 
 load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
 
-rules_foreign_cc_dependencies(register_preinstalled_tools = False)
+rules_foreign_cc_dependencies(register_built_pkgconfig_toolchain = True)
 
 local_repository(
     name = "rules_foreign_cc_examples",

examples/third_party/openssl/BUILD.openssl.bazel

@@ -125,10 +125,7 @@ filegroup(
 
 runnable_binary(
     name = "runnable_openssl",
-    binary = select({
-        "@platforms//os:windows": "openssl.exe",
-        "//conditions:default": "openssl",
-    }),
+    binary = "openssl",
     foreign_cc_target = "@openssl//:openssl",
     visibility = ["//visibility:public"],
 )

foreign_cc/built_tools/pkgconfig_build.bzl

@@ -0,0 +1,111 @@
+""" Rule for building pkg-config from source. """
+
+load("//foreign_cc:defs.bzl", "make_variant", "runnable_binary")
+load(
+    "//foreign_cc/built_tools/private:built_tools_framework.bzl",
+    "FOREIGN_CC_BUILT_TOOLS_ATTRS",
+    "FOREIGN_CC_BUILT_TOOLS_FRAGMENTS",
+    "FOREIGN_CC_BUILT_TOOLS_HOST_FRAGMENTS",
+    "built_tool_rule_impl",
+)
+load("//toolchains/native_tools:tool_access.bzl", "get_make_data")
+
+def _pkgconfig_tool_impl(ctx):
+    make_data = get_make_data(ctx)
+    script = [
+        "./configure  --with-internal-glib --prefix=$$INSTALLDIR$$",
+        "%s" % make_data.path,
+        "%s install" % make_data.path,
+    ]
+
+    additional_tools = depset(transitive = [dep.files for dep in make_data.deps])
+
+    return built_tool_rule_impl(
+        ctx,
+        script,
+        ctx.actions.declare_directory("pkgconfig"),
+        "BootstrapPkgConfig",
+        additional_tools,
+    )
+
+pkgconfig_tool_unix = rule(
+    doc = "Rule for building pkgconfig on Unix operating systems",
+    attrs = FOREIGN_CC_BUILT_TOOLS_ATTRS,
+    host_fragments = FOREIGN_CC_BUILT_TOOLS_HOST_FRAGMENTS,
+    fragments = FOREIGN_CC_BUILT_TOOLS_FRAGMENTS,
+    output_to_genfiles = True,
+    implementation = _pkgconfig_tool_impl,
+    toolchains = [
+        "@rules_foreign_cc//foreign_cc/private/framework:shell_toolchain",
+        "@rules_foreign_cc//toolchains:make_toolchain",
+        "@bazel_tools//tools/cpp:toolchain_type",
+    ],
+)
+
+def pkgconfig_tool(name, srcs, **kwargs):
+    """
+    Macro that provides targets for building pkg-config from source
+
+    Args:
+        name: The target name
+        srcs: The pkg-config source files
+        **kwargs: Remaining keyword arguments
+    """
+    tags = ["manual"] + kwargs.pop("tags", [])
+
+    native.config_setting(
+        name = "msvc_compiler",
+        flag_values = {
+            "@bazel_tools//tools/cpp:compiler": "msvc-cl",
+        },
+    )
+
+    native.alias(
+        name = name,
+        actual = select({
+            ":msvc_compiler": "{}_msvc".format(name),
+            "//conditions:default": "{}_default".format(name),
+        }),
+    )
+
+    pkgconfig_tool_unix(
+        name = "{}_default".format(name),
+        srcs = srcs,
+        tags = tags,
+        **kwargs
+    )
+
+    make_variant(
+        name = "{}_msvc_build".format(name),
+        lib_source = srcs,
+        args = [
+            "-f Makefile.vc",
+            "CFG=release",
+            "GLIB_PREFIX=\"$$EXT_BUILD_ROOT/external/glib_dev\"",
+        ],
+        out_binaries = ["pkg-config.exe"],
+        env = {"INCLUDE": "$$EXT_BUILD_ROOT/external/glib_src"},
+        out_static_libs = [],
+        out_shared_libs = [],
+        deps = [
+            "@glib_dev",
+            "@glib_src//:msvc_hdr",
+            "@gettext_runtime",
+        ],
+        postfix_script = select({
+            "@platforms//os:windows": "cp release/x64/pkg-config.exe $$INSTALLDIR$$/bin",
+            "//conditions:default": "",
+        }),
+        toolchain = "@rules_foreign_cc//toolchains:preinstalled_nmake_toolchain",
+        tags = tags,
+        **kwargs
+    )
+
+    runnable_binary(
+        name = "{}_msvc".format(name),
+        binary = "pkg-config",
+        foreign_cc_target = "{}_msvc_build".format(name),
+        # Tools like CMake and Meson search for "pkg-config" on the PATH
+        match_binary_name = True,
+        tags = tags,
+    )

foreign_cc/configure.bzl

@@ -20,12 +20,13 @@ load(
 )
 load("//foreign_cc/private:transitions.bzl", "make_variant")
 load("//foreign_cc/private/framework:platform.bzl", "os_name")
-load("//toolchains/native_tools:tool_access.bzl", "get_make_data")
+load("//toolchains/native_tools:tool_access.bzl", "get_make_data", "get_pkgconfig_data")
 
 def _configure_make(ctx):
     make_data = get_make_data(ctx)
+    pkg_config_data = get_pkgconfig_data(ctx)
 
-    tools_deps = ctx.attr.tools_deps + make_data.deps
+    tools_deps = ctx.attr.tools_deps + make_data.deps + pkg_config_data.deps
 
     if ctx.attr.autogen and not ctx.attr.configure_in_place:
         fail("`autogen` requires `configure_in_place = True`. Please update {}".format(

foreign_cc/private/framework.bzl

@@ -274,6 +274,9 @@ dependencies.""",
     ),
 )
 
+def _is_msvc_var(var):
+    return var == "INCLUDE" or var == "LIB"
+
 def get_env_prelude(ctx, lib_name, data_dependencies, target_root):
     """Generate a bash snippet containing environment variable definitions
 
@@ -321,8 +324,10 @@ def get_env_prelude(ctx, lib_name, data_dependencies, target_root):
 
     # If user has defined a PATH variable (e.g. PATH, LD_LIBRARY_PATH, CPATH) prepend it to the existing variable
     for user_var in user_vars:
-        if "PATH" in user_var and cc_env.get(user_var):
-            env.update({user_var: user_vars.get(user_var) + ":" + cc_env.get(user_var)})
+        is_existing_var = "PATH" in user_var or _is_msvc_var(user_var)
+        list_delimiter = ";" if _is_msvc_var(user_var) else ":"
+        if is_existing_var and cc_env.get(user_var):
+            env.update({user_var: user_vars.get(user_var) + list_delimiter + cc_env.get(user_var)})
 
     cc_toolchain = find_cpp_toolchain(ctx)
     if cc_toolchain.compiler == "msvc-cl":
@@ -404,7 +409,7 @@ def cc_external_rule_impl(ctx, attrs):
     installdir_copy = copy_directory(ctx.actions, "$$INSTALLDIR$$", "copy_{}/{}".format(lib_name, lib_name))
     target_root = paths.dirname(installdir_copy.file.dirname)
 
-    data_dependencies = ctx.attr.data + ctx.attr.build_data + ctx.attr.toolchains
+    data_dependencies = ctx.attr.data + ctx.attr.build_data + ctx.attr.toolchains + attrs.tools_deps
 
     # Also add legacy dependencies while they're still available
     data_dependencies += ctx.attr.tools_deps + ctx.attr.additional_tools
@@ -496,7 +501,15 @@ def cc_external_rule_impl(ctx, attrs):
 
     # Gather runfiles transitively as per the documentation in:
     # https://docs.bazel.build/versions/master/skylark/rules.html#runfiles
-    runfiles = ctx.runfiles(files = ctx.files.data + outputs.libraries.shared_libraries)
+
+    # Include shared libraries of transitive dependencies in runfiles, facilitating the "runnable_binary" macro
+    transitive_shared_libraries = []
+    for linker_input in out_cc_info.linking_context.linker_inputs.to_list():
+        for lib in linker_input.libraries:
+            if lib.dynamic_library:
+                transitive_shared_libraries.append(lib.dynamic_library)
+
+    runfiles = ctx.runfiles(files = ctx.files.data + transitive_shared_libraries)
     for target in [ctx.attr.lib_source] + ctx.attr.deps + ctx.attr.data:
         runfiles = runfiles.merge(target[DefaultInfo].default_runfiles)
 

foreign_cc/private/framework/toolchains/freebsd_commands.bzl

@@ -229,7 +229,7 @@ if [ -d {dir_} ]; then
   for tool in $tools;
   do
     if  [[ -d \"$tool\" ]] || [[ -L \"$tool\" ]]; then
-      export PATH=$PATH:$tool
+      export PATH=$tool:$PATH
     fi
   done
 fi""".format(dir_ = dir_)

foreign_cc/private/framework/toolchains/linux_commands.bzl

@@ -211,7 +211,7 @@ if [ -d {dir_} ]; then
   for tool in $tools;
   do
     if  [[ -d \"$tool\" ]] || [[ -L \"$tool\" ]]; then
-      export PATH=$PATH:$tool
+      export PATH=$tool:$PATH
     fi
   done
 fi""".format(dir_ = dir_)

foreign_cc/private/framework/toolchains/macos_commands.bzl

@@ -220,7 +220,7 @@ if [ -d {dir_} ]; then
   for tool in $tools;
   do
     if  [[ -d \"$tool\" ]] || [[ -L \"$tool\" ]]; then
-      export PATH=$PATH:$tool
+      export PATH=$tool:$PATH
     fi
   done
 fi""".format(dir_ = dir_)

foreign_cc/private/framework/toolchains/windows_commands.bzl

@@ -225,7 +225,7 @@ if [ -d {dir_} ]; then
   for tool in $tools;
   do
     if  [[ -d \"$tool\" ]] || [[ -L \"$tool\" ]]; then
-      export PATH=$PATH:$tool
+      export PATH=$tool:$PATH
     fi
   done
 fi""".format(dir_ = dir_)

foreign_cc/private/runnable_binary_wrapper.sh

@@ -40,7 +40,7 @@ IFS=" " read -r -a SHARED_LIBS_DIRS_ARRAY <<< "$(tr ' ' '\n' <<< "${SHARED_LIBS_
 # Allow unbound variable here, in case LD_LIBRARY_PATH or similar is not already set
 set +u
 for dir in "${SHARED_LIBS_DIRS_ARRAY[@]}"; do
-    export ${LIB_PATH_VAR}="${!LIB_PATH_VAR}":"$dir"
+    export ${LIB_PATH_VAR}="$dir":"${!LIB_PATH_VAR}"
 done
 set -u
 

foreign_cc/repositories.bzl

@@ -12,9 +12,11 @@ def rules_foreign_cc_dependencies(
         cmake_version = "3.23.2",
         make_version = "4.4",
         ninja_version = "1.11.0",
+        pkgconfig_version = "0.29.2",
         register_preinstalled_tools = True,
         register_built_tools = True,
-        register_toolchains = True):
+        register_toolchains = True,
+        register_built_pkgconfig_toolchain = False):
     """Call this function from the WORKSPACE file to initialize rules_foreign_cc \
     dependencies and let neccesary code generation happen \
     (Code generation is needed to support different variants of the C++ Starlark API.).
@@ -38,26 +40,29 @@ def rules_foreign_cc_dependencies(
         ninja_version: The target version of the ninja toolchain if `register_default_tools`
             or `register_built_tools` is set to `True`.
 
+        pkgconfig_version: The target version of the pkg_config toolchain if `register_built_tools` is set to `True`.
+
         register_preinstalled_tools: If true, toolchains will be registered for the native built tools
             installed on the exec host
 
         register_built_tools: If true, toolchains that build the tools from source are registered
 
         register_toolchains: If true, registers the toolchains via native.register_toolchains. Used by bzlmod
+
+        register_built_pkgconfig_toolchain: If true, the built pkgconfig toolchain will be registered. On Windows it may be preferrable to set this to False, as
+            this requires the --enable_runfiles bazel option. Also note that building pkgconfig from source under bazel results in paths that are more
+            than 256 characters long, which will not work on Windows unless the following options are added to the .bazelrc and symlinks are enabled in Windows.
+
+            startup --windows_enable_symlinks -> This is required to enable symlinking to avoid long runfile paths
+            build --action_env=MSYS=winsymlinks:nativestrict -> This is required to enable symlinking to avoid long runfile paths
+            startup --output_user_root=C:/b  -> This is required to keep paths as short as possible
     """
 
     register_framework_toolchains(register_toolchains = register_toolchains)
 
     if register_toolchains:
         native.register_toolchains(*native_tools_toolchains)
 
-        native.register_toolchains(
-            "@rules_foreign_cc//toolchains:preinstalled_autoconf_toolchain",
-            "@rules_foreign_cc//toolchains:preinstalled_automake_toolchain",
-            "@rules_foreign_cc//toolchains:preinstalled_m4_toolchain",
-            "@rules_foreign_cc//toolchains:preinstalled_pkgconfig_toolchain",
-        )
-
     if register_default_tools:
         prebuilt_toolchains(cmake_version, ninja_version, register_toolchains)
 
@@ -66,7 +71,9 @@ def rules_foreign_cc_dependencies(
             cmake_version = cmake_version,
             make_version = make_version,
             ninja_version = ninja_version,
+            pkgconfig_version = pkgconfig_version,
             register_toolchains = register_toolchains,
+            register_built_pkgconfig_toolchain = register_built_pkgconfig_toolchain,
         )
 
     if register_preinstalled_tools: