Feature Request: Allow to automatically use https://github.com/oracle/graalvm-reachability-metadata

[guizmaii]

As described in the README of https://github.com/oracle/graalvm-reachability-metadata, the Maven and Gradle plugins are able to automatically use the metadata contained in this repo to facilitate the build of native images.

That'd be awesome if sbt-native-image could have the same feature.

Gradle plugin related doc: https://graalvm.github.io/native-build-tools/latest/gradle-plugin.html#metadata-support
Maven plugin related doc: https://graalvm.github.io/native-build-tools/latest/maven-plugin.html#metadata-support

Related question: oracle/graalvm-reachability-metadata#343

[dnestoro]

@melix do we have a common method to access Reachability Metadata that is used by both Gradle and Maven plugins?

[melix]

You just have to take a look at what the Maven and Gradle plugins do. They both make use of the common reachability metadata module (published on Maven Central).

[alexandru]

For anyone looking for a temporary solution, I'm currently doing this manually (sbt task configuration, without the available metadata library mentioned above). This may be error prone, but works for me 🤷‍♂️

In project/plugins.sbt add these dependencies:

libraryDependencies ++= Seq(
    "io.circe" %% "circe-parser" % "0.14.1",
    "org.eclipse.jgit" % "org.eclipse.jgit" % "6.9.0.202403050737-r",
)

Create a new file in project/NativeUtils.scala and add this:

import cats.syntax.all._
import io.circe._
import io.circe.parser._
import io.circe.syntax._
import java.io.File
import org.eclipse.jgit.api.Git
import org.apache.ivy.plugins.repository.Resource
import sbt.std.TaskStreams
import sbt.util.Logger

final class NativeImageGenerateMetadataFiles(
  targetDirectory: File,
)(implicit logger: Logger) {
  import NativeImageGenerateMetadataFiles._

  lazy val localRepoMetadata: File = {
    val remoteUrl = "https://github.com/oracle/graalvm-reachability-metadata.git"
    val localPath = new File(targetDirectory, "graalvm-reachability-metadata")
    if (!localPath.exists()) {
      logger.info(s"[native-image-utils] Cloning to $localPath")
      Git.cloneRepository()
        .setURI(remoteUrl)
        .setDirectory(localPath)
        .call()
    } else {
      logger.info(s"[native-image-utils] Pulling $localPath")
      Git.open(localPath).pull().call()
    }
    new File(localPath, "metadata")
  }

  lazy val localRepoMetadataIndex = {
    val path = new File(localRepoMetadata, "index.json")
    val jsonTxt = scala.io.Source.fromFile(path).mkString
    decode[List[ModuleIndexEntry]](jsonTxt).valueOr(throw _)
      .map(v => (v.key, v))
      .foldLeft(Map.empty[Module, ModuleIndexEntry]) {
        case (acc, (module, entry)) =>
          if (acc.contains(module))
            throw new IllegalArgumentException(s"Duplicate module: $module")
          acc.updated(module, entry)
      }
  }

  def readModuleVersions(module: Module): Option[ModuleVersions] = {
    for {
      metaEntry <- localRepoMetadataIndex.get(module)
      directory <- metaEntry.directory
      directoryPath = new File(localRepoMetadata, directory)
      indexJsonPath = new File(directoryPath, "index.json")
    } yield {
      val jsonTxt = scala.io.Source.fromFile(indexJsonPath).mkString
      val list = decode[List[ModuleVersionsIndexEntry]](jsonTxt).valueOr(throw _)
      ModuleVersions(metaEntry, directoryPath, list)
    }
  }

  def findArtefactVersion(module: Module, version: Option[String]): Option[ArtefactMeta] = {
    val ret = readModuleVersions(module).flatMap { versions =>
      versions.entries.find { entry =>
        version match {
          case None =>
            entry.latest.getOrElse(false)
          case Some(info) =>
            entry.defaultFor.exists(regex => info.matches(regex))
        }
      }.map(ArtefactMeta(versions, _))
    }
    if (ret.isEmpty && version.isDefined)
      findArtefactVersion(module, None)
    else
      ret
  }

  def findArtefact(module: Module, version: Option[String]): Option[(ArtefactMeta, ArtefactFiles)] = {
    findArtefactVersion(module, version).map { meta =>
      val dirPath = new File(meta.allVersions.directory, meta.version.metadataVersion)
      val index = readAndDecodeFile[List[String]](new File(dirPath, "index.json"))

      val reflectConfig = index
        .find(_.endsWith("reflect-config.json"))
        .map { name =>
          val path = new File(dirPath, name)
          readAndDecodeFile[List[JsonObject]](path)
        }
      val resourcesConfig = index
        .find(_.endsWith("resource-config.json"))
        .map { name =>
          val path = new File(dirPath, name)
          readAndDecodeFile[ResourcesJson](path)
        }

      meta -> ArtefactFiles(resourcesConfig, reflectConfig)
    }
  }

  def findAndBuildFiles(artefacts: List[Artefact]): ArtefactFiles =
    artefacts
      .flatMap { artefact =>
        findArtefact(artefact.module, artefact.version).map(_._2)
      }.foldLeft(ArtefactFiles(None, None)) {
        (acc, res) => acc ++ res
      }

  def buildResourcesOfArtefactsIds(artefacts: List[String]): ArtefactFiles =
    findAndBuildFiles(artefacts.map(Artefact.apply))

  def buildResources(items: List[ResourceType]): ArtefactFiles =
    items.foldLeft(ArtefactFiles(None, None)) {
      case (acc, artefact: Artefact) =>
        acc ++ findAndBuildFiles(List(artefact))
      case (acc, ProjectResourceConfigFile(name)) =>
        acc ++ ArtefactFiles(Some(readAndDecodeResource[ResourcesJson](name)), None)
      case (acc, ProjectReflectConfigFile(name)) =>
        acc ++ ArtefactFiles(None, Some(readAndDecodeResource[List[JsonObject]](name)))
    }

  def generateResourceFiles(root: File, items: List[ResourceType]): List[File] =
    buildResources(items.toList).writeFilesContent(root)

  private def readAndDecodeFile[T: Decoder](file: File): T = {
    val jsonTxt = scala.io.Source.fromFile(file).mkString
    decode[T](jsonTxt).valueOr(throw _)
  }

  private def readAndDecodeResource[T: Decoder](name: String): T = {
    val res = getClass().getClassLoader().getResourceAsStream(name)
    val jsonTxt = scala.io.Source.fromInputStream(res).mkString
    decode[T](jsonTxt).valueOr(throw _)
  }
}

object NativeImageGenerateMetadataFiles {
  def generateResourceFiles(
    targetDirectory: File,
    generatedFilesDirectory: File,
    items: List[ResourceType]
  )(implicit logger: Logger): List[File] =
    new NativeImageGenerateMetadataFiles(targetDirectory)
      .generateResourceFiles(generatedFilesDirectory, items)

  case class Module(
    groupId: String,
    artifactId: String,
  ) {
    override def toString = s"$groupId:$artifactId"
  }

  object Module {
    def apply(raw: String): Module =
      raw.split(":").toList match {
        case g :: a :: rest if rest.length <= 1 => Module(g, a)
        case _ => throw new IllegalArgumentException(s"Invalid package: $raw")
      }
  }

  sealed trait ResourceType

  case class Artefact(
    module: Module,
    version: Option[String],
  ) extends ResourceType {
    override def toString = s"$module:$version"
  }

  object Artefact {
    def apply(raw: String): Artefact =
      raw.split(":").toList match {
        case g :: a :: v :: Nil => Artefact(Module(g, a), Some(v))
        case g :: a :: Nil => Artefact(Module(g, a), None)
        case _ => throw new IllegalArgumentException(s"Invalid package: $raw")
      }
  }

  case class ProjectResourceConfigFile(
    resourceName: String,
  ) extends ResourceType {
    override def toString = s"resource:$resourceName"
  }

  case class ProjectReflectConfigFile(
    resourceName: String,
  ) extends ResourceType {
    override def toString = s"resource:$resourceName"
  }

  final case class ModuleIndexEntry(
    allowedPackages: List[String],
    directory: Option[String],
    module: String,
    requires: Option[List[String]]
  ) {
    val key: Module = Module(module)
  }

  object ModuleIndexEntry {
    implicit val json: Codec[ModuleIndexEntry] =
      Codec.forProduct4(
        "allowed-packages",
        "directory",
        "module",
        "requires"
      )(ModuleIndexEntry.apply)(m =>
        (m.allowedPackages, m.directory, m.module, m.requires)
      )
  }

  final case class ModuleVersions(
    metadata: ModuleIndexEntry,
    directory: File,
    entries: List[ModuleVersionsIndexEntry]
  )

  final case class ModuleVersionsIndexEntry(
    latest: Option[Boolean],
    metadataVersion: String,
    module: String,
    defaultFor: Option[String],
    testedVersions: Option[List[String]],
  )

  object ModuleVersionsIndexEntry {
    implicit val json: Codec[ModuleVersionsIndexEntry] =
      Codec.forProduct5(
        "latest",
        "metadata-version",
        "module",
        "default-for",
        "tested-versions"
      )(ModuleVersionsIndexEntry.apply)(m =>
        (m.latest, m.metadataVersion, m.module, m.defaultFor, m.testedVersions)
      )
  }

  final case class ArtefactMeta(
    allVersions: ModuleVersions,
    version: ModuleVersionsIndexEntry,
  )

  final case class ResourcesJsonPatterns(
    includes: Option[List[JsonObject]],
    excludes: Option[List[JsonObject]],
  ) {
    def nonEmpty = includes.nonEmpty || excludes.nonEmpty
  }

  object ResourcesJsonPatterns {
    implicit val json: Codec[ResourcesJsonPatterns] =
      Codec.forProduct2(
        "includes",
        "excludes"
      )(ResourcesJsonPatterns.apply)(m =>
        (m.includes, m.excludes)
      )
  }

  final case class ResourcesJson(
    resources: Option[ResourcesJsonPatterns],
    bundles: Option[List[JsonObject]],
  )

  object ResourcesJson {
    implicit val json: Codec[ResourcesJson] =
      Codec.forProduct2(
        "resources",
        "bundles"
      )(ResourcesJson.apply)(m =>
        (m.resources, m.bundles)
      )
  }

  case class ArtefactFiles(
    resourcesConfig: Option[ResourcesJson],
    reflectConfig: Option[List[JsonObject]],
  ) {
    def toFilesContent: Map[String, String] =
      Map(
        "reflect-config.json" -> reflectConfig.fold("")(_.asJson.spaces2),
        "resource-config.json" -> resourcesConfig.fold("")(_.asJson.spaces2),
      ).filter(_._2.nonEmpty)

    def writeFilesContent(root: File)(implicit logger: Logger): List[File] =
        toFilesContent.map {
            case (name, content) =>
            val file = new File(root, name)
            sbt.IO.write(file, content)
            logger.info(s"[native-image-utils] Generated: $file")
            file
        }.toList

    def ++(other: ArtefactFiles): ArtefactFiles = {
      val newRes = (resourcesConfig, other.resourcesConfig) match {
        case (Some(acc), Some(res)) =>
          val newInc = Some(
            acc.resources.flatMap(_.includes).getOrElse(Nil) ++
            res.resources.flatMap(_.includes).getOrElse(Nil)
          ).filter(_.nonEmpty)
          val newExc = Some(
            acc.resources.flatMap(_.excludes).getOrElse(Nil) ++
            res.resources.flatMap(_.excludes).getOrElse(Nil)
          ).filter(_.nonEmpty)
          val newBundles = Some(
            acc.bundles.getOrElse(Nil) ++ res.bundles.getOrElse(Nil)
          ).filter(_.nonEmpty)
          Some(ResourcesJson(
            Some(ResourcesJsonPatterns(newInc, newExc)).filter(_.nonEmpty),
            newBundles
          ))
        case _ =>
          resourcesConfig.orElse(other.resourcesConfig)
      }
      val newReflect = (reflectConfig, other.reflectConfig) match {
        case (Some(acc), Some(res)) => Some(acc ++ res)
        case _ => reflectConfig.orElse(other.reflectConfig)
      }
      ArtefactFiles(newRes, newReflect)
    }
  }
}

Then, for your project, add this configuration for "resource generators" in build.sbt:

lazy val root = (project in file("."))
    .enablePlugins(NativeImagePlugin)
    .settings(
        // ....
        // Add this...
        Compile / resourceGenerators += Def.task {
            import NativeImageGenerateMetadataFiles._
            implicit val logger: sbt.util.Logger = sbt.Keys.streams.value.log
            generateResourceFiles(
                // Path needed for cloning the metadata repository
                (Compile / target).value,
                // Path where the metadata files will be generated
                (Compile / resourceManaged).value / "META-INF" / "native-image",
                // List all tranzitive dependencies (can also add our own files)
                update.value
                    .allModules
                    .map(m => Artefact(s"${m.organization}:${m.name}:${m.revision}"))
                    .toList
            )
        }.taskValue
    )

I'm not providing support for it 😉

Cheers,