Index service implementations as well as packages

MODULE.bazel

@@ -272,6 +272,14 @@ dev_maven.install(
         "https://repo.spring.io/plugins-release/",
     ],
 )
+dev_maven.install(
+    name = "service_indexing_testing",
+    artifacts = [
+        "org.projectlombok:lombok:1.18.22",
+    ],
+    lock_file = "//tests/custom_maven_install:service_indexing_testing.json",
+)
+
 dev_maven.install(
     name = "jvm_import_test",
     artifacts = [
@@ -675,6 +683,10 @@ use_repo(
     "json_artifacts_testing",
     "unpinned_json_artifacts_testing",
 
+    # Pinned repo
+    "service_indexing_testing",
+    "unpinned_service_indexing_testing",
+
     # Unpinned repo
     "jvm_import_test",
     "manifest_stamp_testing",
@@ -791,3 +803,12 @@ http_file(
         "https://repo1.maven.org/maven2/com/google/apis/google-api-services-compute/v1-rev235-1.25.0/google-api-services-compute-v1-rev235-1.25.0-javadoc.jar",
     ],
 )
+
+http_file(
+    name = "lombok_for_test",
+    downloaded_file_path = "lombok-1.18.22.jar",
+    sha256 = "ecef1581411d7a82cc04281667ee0bac5d7c0a5aae74cfc38430396c91c31831",
+    urls = [
+        "https://repo1.maven.org/maven2/org/projectlombok/lombok/1.18.22/lombok-1.18.22.jar",
+    ],
+)

WORKSPACE

@@ -440,6 +440,14 @@ maven_install(
     ],
 )
 
+maven_install(
+    name = "service_indexing_testing",
+    artifacts = [
+        "org.projectlombok.lombok:1.18.22",
+    ],
+    maven_install_json = "//tests/custom_maven_install:service_indexing_testing.json"
+)
+
 maven_install(
     name = "jvm_import_test",
     artifacts = [
@@ -695,6 +703,15 @@ http_file(
     ],
 )
 
+http_file(
+    name = "lombok_for_test",
+    downloaded_file_path = "lombok-1.18.22.jar",
+    sha256 = "ecef1581411d7a82cc04281667ee0bac5d7c0a5aae74cfc38430396c91c31831",
+    urls = [
+        "https://repo1.maven.org/maven2/org/projectlombok/lombok/1.18.22/lombok-1.18.22.jar",
+    ],
+)
+
 # End test dependencies
 
 http_archive(

coursier.bzl

@@ -1125,20 +1125,20 @@ def _coursier_fetch_impl(repository_ctx):
         path = str(repository_ctx.path(parts[1]))
         shas[path] = parts[0]
 
-    list_packages_stdout = _execute_with_argsfile(
+    index_jars_stdout = _execute_with_argsfile(
         repository_ctx,
-        repository_ctx.attr._list_packages,
-        "package_lister",
-        "Indexing jar packages",
-        "indexing jar packages",
+        repository_ctx.attr._index_jar,
+        "jar_indexer",
+        "Indexing jars",
+        "indexing jars",
         files_to_inspect,
     )
 
-    jars_to_packages = json.decode(list_packages_stdout)
-    for jar, packages in jars_to_packages.items():
+    jars_to_index_results = json.decode(index_jars_stdout)
+    for jar, index_results in jars_to_index_results.items():
         path = str(repository_ctx.path(jar))
         if path != jar:
-            jars_to_packages[path] = jars_to_packages.pop(jar)
+            jars_to_index_results[path] = jars_to_index_results.pop(jar)
 
     for artifact in dep_tree["dependencies"]:
         file = artifact["file"]
@@ -1159,7 +1159,10 @@ def _coursier_fetch_impl(repository_ctx):
                 artifact.pop("url")
             continue
         artifact.update({"sha256": shas[path]})
-        artifact.update({"packages": jars_to_packages[path]})
+        artifact.update({"packages": jars_to_index_results[path]["packages"]})
+        service_implementations = jars_to_index_results[path].get("serviceImplementations", {})
+        if service_implementations:
+            artifact.update({"services": service_implementations})
 
     # Keep the original output from coursier for debugging
     repository_ctx.file(
@@ -1375,7 +1378,7 @@ pinned_coursier_fetch = repository_rule(
 coursier_fetch = repository_rule(
     attrs = {
         "_sha256_hasher": attr.label(default = "//private/tools/prebuilt:hasher_deploy.jar"),
-        "_list_packages": attr.label(default = "//private/tools/prebuilt:list_packages_deploy.jar"),
+        "_index_jar": attr.label(default = "//private/tools/prebuilt:index_jar_deploy.jar"),
         "_lock_file_converter": attr.label(default = "//private/tools/prebuilt:lock_file_converter_deploy.jar"),
         "_pin": attr.label(default = "//private:pin.sh"),
         "_compat_repository": attr.label(default = "//private:compat_repository.bzl"),

maven_install.json

@@ -430,6 +430,7 @@
       "org.objectweb.asm.signature"
     ]
   },
+  "services": {},
   "repositories": {
     "https://repo1.maven.org/maven2/": [
       "com.google.auto.value:auto-value-annotations",

private/rules/v2_lock_file.bzl

@@ -165,6 +165,7 @@ def _render_lock_file(lock_file_contents, input_hash):
     if lock_file_contents.get("skipped"):
         contents.append("  \"skipped\": %s," % json.encode_indent(lock_file_contents["skipped"], prefix = "  ", indent = "  "))
     contents.append("  \"packages\": %s," % json.encode_indent(lock_file_contents["packages"], prefix = "  ", indent = "  "))
+    contents.append("  \"services\": %s," % json.encode_indent(lock_file_contents["services"], prefix = "  ", indent = "  "))
     if lock_file_contents.get("m2local"):
         contents.append("  \"m2local\": true,")
     contents.append("  \"repositories\": {")

private/tools/java/com/github/bazelbuild/rules_jvm_external/coursier/LockFileConverter.java

@@ -38,6 +38,9 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.stream.Collectors;
 
@@ -178,14 +181,21 @@ public Set<DependencyInfo> getDependencies() {
         packages = new TreeSet<>(depPackages);
       }
 
+      SortedMap<String, SortedSet<String>> services = new TreeMap<>();
+      Object rawServices = coursierDep.get("services");
+      if (rawServices != null) {
+        services = new TreeMap<>((Map<String, SortedSet<String>>) rawServices);
+      }
+
       toReturn.add(
           new DependencyInfo(
               coords,
               repos,
               Optional.ofNullable(file).map(Paths::get),
               Optional.ofNullable((String) coursierDep.get("sha256")),
               directDeps,
-              packages));
+              packages,
+              services));
     }
 
     return toReturn;

private/tools/java/com/github/bazelbuild/rules_jvm_external/coursier/NebulaFormat.java

@@ -24,6 +24,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.stream.Collectors;
@@ -41,6 +42,7 @@ public Map<String, Object> render(
     Map<String, Map<String, Object>> artifacts = new TreeMap<>();
     Map<String, Set<String>> deps = new TreeMap<>();
     Map<String, Set<String>> packages = new TreeMap<>();
+    Map<String, Map<String, SortedSet<String>>> services = new TreeMap<>();
     Map<String, Set<String>> repos = new LinkedHashMap<>();
     repositories.forEach(r -> repos.put(stripAuthenticationInformation(r), new TreeSet<>()));
 
@@ -99,6 +101,7 @@ public Map<String, Object> render(
                   .map(Object::toString)
                   .collect(Collectors.toCollection(TreeSet::new)));
           packages.put(key, info.getPackages());
+          services.put(key, info.getServices());
 
           if (info.getPath().isPresent()) {
             // Regularise paths to UNIX format
@@ -110,6 +113,7 @@ public Map<String, Object> render(
     lock.put("artifacts", ensureArtifactsAllHaveAtLeastOneShaSum(artifacts));
     lock.put("dependencies", removeEmptyItems(deps));
     lock.put("packages", removeEmptyItems(packages));
+    lock.put("services", removeEmptyItemsMap(services));
     if (isUsingM2Local) {
       lock.put("m2local", true);
     }
@@ -153,6 +157,20 @@ private <K, V extends Collection> Map<K, V> removeEmptyItems(Map<K, V> input) {
                 TreeMap::new));
   }
 
+  private <K, VK, VV> Map<K, Map<VK, VV>> removeEmptyItemsMap(Map<K, Map<VK, VV>> input) {
+    return input.entrySet().stream()
+        .filter(e -> !e.getValue().isEmpty())
+        .collect(
+            Collectors.toMap(
+                Map.Entry::getKey,
+                Map.Entry::getValue,
+                (l, r) -> {
+                  l.putAll(r);
+                  return l;
+                },
+                TreeMap::new));
+}
+
   private String stripAuthenticationInformation(String possibleUri) {
     try {
       URI uri = new URI(possibleUri);

private/tools/java/com/github/bazelbuild/rules_jvm_external/jar/BUILD

@@ -17,8 +17,11 @@ java_binary(
 )
 
 java_library(
-    name = "ListPackages-lib",
-    srcs = ["ListPackages.java"],
+    name = "IndexJar-lib",
+    srcs = [
+        "IndexJar.java",
+        "PerJarIndexResults.java",
+    ],
     visibility = [
         "//private/tools/java/com/github/bazelbuild/rules_jvm_external/resolver/cmd:__pkg__",
         "//tests/com/github/bazelbuild/rules_jvm_external/jar:__pkg__",
@@ -32,11 +35,11 @@ java_library(
 )
 
 java_binary(
-    name = "ListPackages",
-    main_class = "com.github.bazelbuild.rules_jvm_external.jar.ListPackages",
+    name = "IndexJar",
+    main_class = "com.github.bazelbuild.rules_jvm_external.jar.IndexJar",
     visibility = ["//visibility:public"],
     runtime_deps = [
-        ":ListPackages-lib",
+        ":IndexJar-lib",
     ],
 )
 

private/tools/java/com/github/bazelbuild/rules_jvm_external/jar/ListPackages.java → private/tools/java/com/github/bazelbuild/rules_jvm_external/jar/IndexJar.java

@@ -15,16 +15,21 @@
 package com.github.bazelbuild.rules_jvm_external.jar;
 
 import com.google.gson.Gson;
+
 import java.io.BufferedInputStream;
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.AbstractMap;
 import java.util.Arrays;
 import java.util.Map;
+import java.util.Enumeration;
+import java.util.SortedMap;
 import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
@@ -34,33 +39,36 @@
 import java.util.stream.Stream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipException;
+import java.util.zip.ZipFile;
 import java.util.zip.ZipInputStream;
 
-public class ListPackages {
+public class IndexJar {
 
   private static final Predicate<String> IS_NUMERIC_VERSION =
       Pattern.compile("[1-9][0-9]*").asPredicate();
 
+  private static final String SERVICES_DIRECTORY_PREFIX = "META-INF/services/";
+
   public static void main(String[] args) throws IOException {
     if (args.length != 2 || !"--argsfile".equals(args[0])) {
       System.err.printf("Required args: --argsfile /path/to/argsfile%n");
       System.exit(1);
     }
 
     Path argsFile = Paths.get(args[1]);
-    Map<String, SortedSet<String>> index = new ListPackages().getPackages(Files.lines(argsFile));
+    Map<String, PerJarIndexResults> index = new IndexJar().index(Files.lines(argsFile));
     System.out.println(new Gson().toJson(index));
   }
 
-  public Map<String, SortedSet<String>> getPackages(Stream<String> source) {
-    TreeMap<String, SortedSet<String>> index =
+  public Map<String, PerJarIndexResults> index(Stream<String> source) {
+    TreeMap<String, PerJarIndexResults> index =
         source
             .parallel()
             .map(
                 path -> {
                   try {
-                    SortedSet<String> packages = getPackages(Paths.get(path));
-                    return new AbstractMap.SimpleEntry<>(path, packages);
+                    PerJarIndexResults results = index(Paths.get(path));
+                    return new AbstractMap.SimpleEntry<>(path, results);
                   } catch (IOException e) {
                     throw new UncheckedIOException(e);
                   }
@@ -77,27 +85,41 @@ public Map<String, SortedSet<String>> getPackages(Stream<String> source) {
     return index;
   }
 
-  public SortedSet<String> getPackages(Path path) throws IOException {
+  public PerJarIndexResults index(Path path) throws IOException {
     SortedSet<String> packages = new TreeSet<>();
-    try (InputStream fis = new BufferedInputStream(Files.newInputStream(path));
-        ZipInputStream zis = new ZipInputStream(fis)) {
+    SortedMap<String, SortedSet<String>> serviceImplementations = new TreeMap<>();
+    try (InputStream fis = new BufferedInputStream(Files.newInputStream(path)); ZipInputStream zis = new ZipInputStream(fis)) {
       try {
         ZipEntry entry;
         while ((entry = zis.getNextEntry()) != null) {
+          if (entry.getName().startsWith(SERVICES_DIRECTORY_PREFIX) && !SERVICES_DIRECTORY_PREFIX.equals(entry.getName())) {
+            String serviceInterface = entry.getName().substring(SERVICES_DIRECTORY_PREFIX.length());
+            SortedSet<String> implementingClasses = new TreeSet<>();
+            // We can't close zis here or it will also prevent us from reading subsequent entries.
+            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(zis));
+            String implementingClass = bufferedReader.readLine();
+            while (implementingClass != null) {
+              if (!implementingClass.isEmpty() && !implementingClass.startsWith("#")) {
+                implementingClasses.add(implementingClass);
+              }
+              implementingClass = bufferedReader.readLine();
+            }
+            serviceImplementations.put(serviceInterface, implementingClasses);
+          }
           if (!entry.getName().endsWith(".class")) {
             continue;
           }
           if ("module-info.class".equals(entry.getName())
-              || entry.getName().endsWith("/module-info.class")) {
+                  || entry.getName().endsWith("/module-info.class")) {
             continue;
           }
           packages.add(extractPackageName(entry.getName()));
         }
       } catch (ZipException e) {
         System.err.printf("Caught ZipException: %s%n", e);
       }
+      return new PerJarIndexResults(packages, serviceImplementations);
     }
-    return packages;
   }
 
   private String extractPackageName(String zipEntryName) {

private/tools/java/com/github/bazelbuild/rules_jvm_external/jar/PerJarIndexResults.java

@@ -0,0 +1,22 @@
+package com.github.bazelbuild.rules_jvm_external.jar;
+
+import java.util.SortedMap;
+import java.util.SortedSet;
+
+public class PerJarIndexResults {
+    private final SortedSet<String> packages;
+    private final SortedMap<String, SortedSet<String>> serviceImplementations;
+
+    public PerJarIndexResults(SortedSet<String> packages, SortedMap<String, SortedSet<String>> serviceImplementations) {
+        this.packages = packages;
+        this.serviceImplementations = serviceImplementations;
+    }
+
+    public SortedSet<String> getPackages() {
+        return this.packages;
+    }
+
+    public SortedMap<String, SortedSet<String>> getServiceImplementations() {
+        return this.serviceImplementations;
+    }
+}