From 2ac63329c51b7c67423322bd1728fa7d96b2a5f2 Mon Sep 17 00:00:00 2001
From: Benjie Gatt <bgatt@gradle.com>
Date: Thu, 30 May 2024 14:42:23 +0200
Subject: [PATCH 1/6] Update build files to allow better incremental and cached
 builds

---
 api/build.gradle.kts                           |  9 +++------
 .../resources/ai/djl/engine/api.properties     |  1 +
 .../src/main/kotlin/ai/djl/check.gradle.kts    |  1 +
 engines/llama/build.gradle.kts                 | 10 ++++++----
 engines/ml/xgboost/build.gradle.kts            |  3 ++-
 engines/mxnet/jnarator/build.gradle.kts        | 10 ++++++++++
 engines/mxnet/mxnet-engine/build.gradle.kts    | 13 ++++++++-----
 .../src/main/resources/mx-engine.properties    |  2 ++
 .../onnxruntime-engine/build.gradle.kts        |  6 +++---
 .../pytorch/pytorch-engine/build.gradle.kts    |  9 +++------
 .../main/resources/pytorch-engine.properties   |  2 ++
 .../tensorflow-engine/build.gradle.kts         |  8 +++-----
 .../resources/tensorflow-engine.properties     |  2 ++
 engines/tensorrt/build.gradle.kts              | 13 ++++++++-----
 .../src/main/resources/tensorrt.properties     |  1 +
 extensions/fasttext/build.gradle.kts           | 13 +++++++++----
 .../resources/native/lib/fasttext.properties   |  1 +
 extensions/sentencepiece/build.gradle.kts      | 12 +++++++-----
 .../native/lib/sentencepiece.properties        |  1 +
 extensions/tokenizers/build.gradle.kts         | 18 +++++++++++-------
 .../resources/native/lib/tokenizers.properties |  1 +
 tools/gradle/android-formatter.gradle          |  7 ++++++-
 22 files changed, 91 insertions(+), 52 deletions(-)
 create mode 100644 api/src/main/resources/ai/djl/engine/api.properties
 create mode 100644 engines/mxnet/mxnet-engine/src/main/resources/mx-engine.properties
 create mode 100644 engines/pytorch/pytorch-engine/src/main/resources/pytorch-engine.properties
 create mode 100644 engines/tensorflow/tensorflow-engine/src/main/resources/tensorflow-engine.properties
 create mode 100644 engines/tensorrt/src/main/resources/tensorrt.properties
 create mode 100644 extensions/fasttext/src/main/resources/native/lib/fasttext.properties
 create mode 100644 extensions/sentencepiece/src/main/resources/native/lib/sentencepiece.properties
 create mode 100644 extensions/tokenizers/src/main/resources/native/lib/tokenizers.properties

diff --git a/api/build.gradle.kts b/api/build.gradle.kts
index 4d4cc7a2dbe..8fa167e186a 100644
--- a/api/build.gradle.kts
+++ b/api/build.gradle.kts
@@ -23,12 +23,9 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        outputs.file(buildDirectory / "classes/java/main/ai/djl/engine/api.properties")
-        doFirst {
-            val classesDir = file("$buildDirectory/classes/java/main/ai/djl/engine/")
-            classesDir.mkdirs()
-            val propFile = File(classesDir, "api.properties")
-            propFile.text = "djl_version=${project.version}"
+        inputs.properties(mapOf("projectVersion" to project.version))
+        filesMatching("**/api.properties") {
+            expand(mapOf("projectVersion" to project.version))
         }
     }
 
diff --git a/api/src/main/resources/ai/djl/engine/api.properties b/api/src/main/resources/ai/djl/engine/api.properties
new file mode 100644
index 00000000000..0508cf08a2b
--- /dev/null
+++ b/api/src/main/resources/ai/djl/engine/api.properties
@@ -0,0 +1 @@
+djl_version=${projectVersion}
\ No newline at end of file
diff --git a/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts b/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts
index c65cd9f19ca..ccbafe0d29e 100644
--- a/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts
+++ b/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts
@@ -53,6 +53,7 @@ checkstyle {
         "checkstyle.licenseHeader.file" to file("${rootProject.projectDir}/tools/conf/licenseHeader.java")
     )
     configFile = file("${rootProject.projectDir}/tools/conf/checkstyle.xml")
+    
 }
 tasks {
     named<Checkstyle>("checkstyleMain") {
diff --git a/engines/llama/build.gradle.kts b/engines/llama/build.gradle.kts
index 3ca39b38770..d19ac5d2741 100644
--- a/engines/llama/build.gradle.kts
+++ b/engines/llama/build.gradle.kts
@@ -17,7 +17,9 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        outputs.dir(project.projectDir / "build/classes/java/main/native/lib")
+        var path = "${project.projectDir}/build/resources/main"
+        inputs.properties(mapOf("djl_version" to libs.versions.djl.get(), "llamacpp_version" to libs.versions.llamacpp.get()))
+        outputs.dir("$path/native/lib")
         doLast {
             val llamacpp = libs.versions.llamacpp.get()
             val djl = libs.versions.djl.get()
@@ -49,15 +51,15 @@ tasks {
             }
             copy {
                 from(jnilibDir)
-                into(project.projectDir / "build/classes/java/main/native/lib")
+                into("$path/native/lib")
             }
 
             // write properties
-            val propFile = project.projectDir / "build/classes/java/main/native/lib/llama.properties"
+            val propFile = file("$path/native/lib/llama.properties")
             propFile.text = "version=$llamacpp-$version\n"
 
             url = "https://mlrepo.djl.ai/model/nlp/text_generation/ai/djl/huggingface/gguf/models.json.gz"
-            val prefix = project.projectDir / "build/classes/java/main/nlp/text_generation"
+            val prefix = File("$path/nlp/text_generation")
             val file = prefix / "ai.djl.huggingface.gguf.json"
             if (file.exists())
                 project.logger.lifecycle("gguf index file already exists")
diff --git a/engines/ml/xgboost/build.gradle.kts b/engines/ml/xgboost/build.gradle.kts
index fe660797d39..824bd2ea987 100644
--- a/engines/ml/xgboost/build.gradle.kts
+++ b/engines/ml/xgboost/build.gradle.kts
@@ -42,7 +42,8 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        val jnilibDir = buildDirectory / "classes/java/main/lib/linux/aarch64/"
+        val jnilibDir = buildDirectory / "resources/java/main/lib/linux/aarch64/"
+        inputs.properties(mapOf("xgboostVersion" to libs.versions.xgboost.get()))
         outputs.dir(jnilibDir)
         doLast {
             val url =
diff --git a/engines/mxnet/jnarator/build.gradle.kts b/engines/mxnet/jnarator/build.gradle.kts
index b0af9fd453f..3f63a179a19 100644
--- a/engines/mxnet/jnarator/build.gradle.kts
+++ b/engines/mxnet/jnarator/build.gradle.kts
@@ -20,6 +20,7 @@ tasks {
     pmdMain { exclude("ai/djl/mxnet/jnarator/parser/*") }
 
     jar {
+        dependsOn(generateGrammarSource)
         manifest {
             attributes(
                 "Main-Class" to "ai.djl.mxnet.jnarator.Main",
@@ -30,4 +31,13 @@ tasks {
         duplicatesStrategy = DuplicatesStrategy.INCLUDE
         from(configurations.runtimeClasspath.get().map { if (it.isDirectory()) it else zipTree(it) })
     }
+
+    generateGrammarSource {
+        dependsOn(verifyJava)
+    }
+
+    generateTestGrammarSource {
+        dependsOn(verifyJava)
+    }
+
 }
\ No newline at end of file
diff --git a/engines/mxnet/mxnet-engine/build.gradle.kts b/engines/mxnet/mxnet-engine/build.gradle.kts
index 3c0eb2cf72a..0b825755262 100644
--- a/engines/mxnet/mxnet-engine/build.gradle.kts
+++ b/engines/mxnet/mxnet-engine/build.gradle.kts
@@ -25,11 +25,9 @@ sourceSets.main {
 
 tasks {
     processResources {
-        doFirst {
-            val classesDir = buildDirectory / "classes/java/main/"
-            classesDir.mkdirs()
-            val file = classesDir / "mxnet-engine.properties"
-            file.text = "djl_version=${libs.versions.djl.get()}\nmxnet_version=" + libs.versions.mxnet.get()
+        inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), "mxnetVersion" to libs.versions.mxnet.get()))
+        filesMatching("**/mxnet-engine.properties") {
+            expand(mapOf("djlVersion" to libs.versions.djl.get(), "mxnetVersion" to libs.versions.mxnet.get()))
         }
     }
 
@@ -39,7 +37,11 @@ tasks {
     val jnarator by registering {
         val jnaratorJar = project(":engines:mxnet:jnarator").tasks.jar
         dependsOn(jnaratorJar)
+        inputs.file("${project.projectDir}/src/main/jna/mapping.properties")
+            .withPathSensitivity(PathSensitivity.RELATIVE)
+            .withPropertyName("jna/mapping.properties file")
         outputs.dir(buildDirectory / "generated-src")
+        outputs.cacheIf { true }
         doLast {
             val jnaGenerator = jnaratorJar.get().outputs.files.singleFile
             javaexec {
@@ -106,6 +108,7 @@ tasks {
     }
     compileJava { dependsOn(jnarator) }
     javadoc { dependsOn(jnarator) }
+    verifyJava { dependsOn(jnarator) }
 
     publishing {
         publications {
diff --git a/engines/mxnet/mxnet-engine/src/main/resources/mx-engine.properties b/engines/mxnet/mxnet-engine/src/main/resources/mx-engine.properties
new file mode 100644
index 00000000000..cbfb997e870
--- /dev/null
+++ b/engines/mxnet/mxnet-engine/src/main/resources/mx-engine.properties
@@ -0,0 +1,2 @@
+djl_version=${djlVersion}
+mxnet_version=${mxnetVersion}
\ No newline at end of file
diff --git a/engines/onnxruntime/onnxruntime-engine/build.gradle.kts b/engines/onnxruntime/onnxruntime-engine/build.gradle.kts
index 548fda021fc..5c88c2f2177 100644
--- a/engines/onnxruntime/onnxruntime-engine/build.gradle.kts
+++ b/engines/onnxruntime/onnxruntime-engine/build.gradle.kts
@@ -20,7 +20,8 @@ dependencies {
 
 tasks {
     processResources {
-        outputs.dir(projectDir / "build/classes/java/main/nlp")
+        var basePath = "${project.projectDir}/build/resources/main/nlp"
+        outputs.dir(basePath)
         doLast {
             val url = "https://mlrepo.djl.ai/model/nlp"
             val tasks = listOf(
@@ -30,9 +31,8 @@ tasks {
                 "text_embedding",
                 "token_classification"
             )
-            val prefix = projectDir / "build/classes/java/main/nlp"
             for (task in tasks) {
-                val file = prefix / task / "ai.djl.huggingface.onnxruntime.json"
+                val file = File("$basePath/$task/ai.djl.huggingface.onnxruntime.json")
                 if (file.exists())
                     project.logger.lifecycle("model zoo metadata alrady exists: $task")
                 else {
diff --git a/engines/pytorch/pytorch-engine/build.gradle.kts b/engines/pytorch/pytorch-engine/build.gradle.kts
index ba93c92d3ac..fbed0d4cd94 100644
--- a/engines/pytorch/pytorch-engine/build.gradle.kts
+++ b/engines/pytorch/pytorch-engine/build.gradle.kts
@@ -21,12 +21,9 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        outputs.file(buildDirectory / "classes/java/main/pytorch-engine.properties")
-        doFirst {
-            val classesDir = buildDirectory / "classes/java/main/"
-            classesDir.mkdirs()
-            val propFile = classesDir / "pytorch-engine.properties"
-            propFile.text = "djl_version=${libs.versions.djl.get()}\npytorch_version=${libs.versions.pytorch.get()}"
+        inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), "pytorchVersion" to libs.versions.pytorch.get()))
+        filesMatching("**/pytorch-engine.properties") {
+            expand(mapOf("djlVersion" to libs.versions.djl.get(), "pytorchVersion" to libs.versions.pytorch.get()))
         }
     }
 
diff --git a/engines/pytorch/pytorch-engine/src/main/resources/pytorch-engine.properties b/engines/pytorch/pytorch-engine/src/main/resources/pytorch-engine.properties
new file mode 100644
index 00000000000..7ca0f6e1b1f
--- /dev/null
+++ b/engines/pytorch/pytorch-engine/src/main/resources/pytorch-engine.properties
@@ -0,0 +1,2 @@
+djl_version=${djlVersion}
+pytorch_version=${pytorchVersion}
\ No newline at end of file
diff --git a/engines/tensorflow/tensorflow-engine/build.gradle.kts b/engines/tensorflow/tensorflow-engine/build.gradle.kts
index 5d6760f0d6c..ea3009c1fe1 100644
--- a/engines/tensorflow/tensorflow-engine/build.gradle.kts
+++ b/engines/tensorflow/tensorflow-engine/build.gradle.kts
@@ -14,11 +14,9 @@ dependencies {
 }
 
 tasks.processResources {
-    doFirst {
-        val classesDir = buildDirectory / "classes/java/main/"
-        classesDir.mkdirs()
-        val file = classesDir / "tensorflow-engine.properties"
-        file.text = "djl_version=${libs.versions.djl.get()}\ntensorflow_version=${libs.versions.tensorflow.get()}"
+    inputs.properties(mapOf("djlVersion" to libs.versions.djl, "tensorflowVersion" to libs.versions.tensorflow))
+    filesMatching("**/mxnet-engine.properties") {
+        expand(mapOf("djl_version" to libs.versions.djl, "tensorflow_version" to libs.versions.tensorflow))
     }
 }
 
diff --git a/engines/tensorflow/tensorflow-engine/src/main/resources/tensorflow-engine.properties b/engines/tensorflow/tensorflow-engine/src/main/resources/tensorflow-engine.properties
new file mode 100644
index 00000000000..f3ddbf23461
--- /dev/null
+++ b/engines/tensorflow/tensorflow-engine/src/main/resources/tensorflow-engine.properties
@@ -0,0 +1,2 @@
+djl_version=${projectVersion}
+tensorflow_version=${tensorflowVersion}
\ No newline at end of file
diff --git a/engines/tensorrt/build.gradle.kts b/engines/tensorrt/build.gradle.kts
index 3c8a118eeef..4a63913ca15 100644
--- a/engines/tensorrt/build.gradle.kts
+++ b/engines/tensorrt/build.gradle.kts
@@ -21,7 +21,10 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        outputs.dir(buildDirectory / "classes/java/main/native/lib")
+        inputs.properties(mapOf("djlVersion" to libs.versions.djl, "trtVersion" to libs.versions.tensorrt.get(),
+            "version" to version))
+        val baseResourcePath = "${project.projectDir}/build/resources/main"
+        outputs.dir(file("${baseResourcePath}/native/lib"))
         doLast {
             val trtVersion = libs.versions.tensorrt.get()
             val djlVersion = libs.versions.djl.get()
@@ -41,12 +44,12 @@ tasks {
 
             copy {
                 from(jnilibDir)
-                into(buildDirectory / "classes/java/main/native/lib")
+                into("$baseResourcePath/native/lib")
             }
 
-            // write properties
-            val propFile = buildDirectory / "classes/java/main/native/lib/tensorrt.properties"
-            propFile.text = "version=${trtVersion}-${version}\n"
+            filesMatching("**/tensorrt.properties") {
+                expand(mapOf("trtVersion" to libs.versions.tensorrt.get(), "version" to version))
+            }
         }
     }
 
diff --git a/engines/tensorrt/src/main/resources/tensorrt.properties b/engines/tensorrt/src/main/resources/tensorrt.properties
new file mode 100644
index 00000000000..29bc031ee51
--- /dev/null
+++ b/engines/tensorrt/src/main/resources/tensorrt.properties
@@ -0,0 +1 @@
+version=${trtVersion}-${version}
\ No newline at end of file
diff --git a/extensions/fasttext/build.gradle.kts b/extensions/fasttext/build.gradle.kts
index 046947a3394..4bf844093f3 100644
--- a/extensions/fasttext/build.gradle.kts
+++ b/extensions/fasttext/build.gradle.kts
@@ -18,7 +18,11 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        outputs.dir(buildDirectory / "classes/java/main/native/lib")
+        inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(),
+            "fasttextVersion" to libs.versions.fasttext.get(),
+            "version" to version))
+        val baseResourcePath = "${project.projectDir}/build/resources/main"
+        outputs.dir("$baseResourcePath/native/lib")
         doLast {
             val url =
                 "https://publish.djl.ai/fasttext-${libs.versions.fasttext.get()}/jnilib/${libs.versions.djl.get()}"
@@ -41,12 +45,13 @@ tasks {
             }
             copy {
                 from(jnilibDir)
-                into(buildDirectory / "classes/java/main/native/lib")
+                into("$baseResourcePath/native/lib")
             }
 
             // write properties
-            val propFile = buildDirectory / "classes/java/main/native/lib/fasttext.properties"
-            propFile.text = "version=${libs.versions.fasttext.get()}-${version}\n"
+            filesMatching("**/fasttext.properties") {
+                expand(mapOf("fasttextVersion" to libs.versions.fasttext.get(), "version" to version))
+            }
         }
     }
 
diff --git a/extensions/fasttext/src/main/resources/native/lib/fasttext.properties b/extensions/fasttext/src/main/resources/native/lib/fasttext.properties
new file mode 100644
index 00000000000..8b69cf67a61
--- /dev/null
+++ b/extensions/fasttext/src/main/resources/native/lib/fasttext.properties
@@ -0,0 +1 @@
+version=${fasttextVersion}-${version}
\ No newline at end of file
diff --git a/extensions/sentencepiece/build.gradle.kts b/extensions/sentencepiece/build.gradle.kts
index b872b76c83a..71d01b0a256 100644
--- a/extensions/sentencepiece/build.gradle.kts
+++ b/extensions/sentencepiece/build.gradle.kts
@@ -17,7 +17,9 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        outputs.dir(buildDirectory / "classes/java/main/native/lib")
+        val baseResourcePath = "${project.projectDir}/build/resources/main"
+        inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), "sentencepieceVersion" to libs.versions.sentencepiece.get()))
+        outputs.dir("$baseResourcePath/native/lib")
         doLast {
             val url =
                 "https://publish.djl.ai/sentencepiece-${libs.versions.sentencepiece.get()}/jnilib/${libs.versions.djl.get()}"
@@ -42,12 +44,12 @@ tasks {
             }
             copy {
                 from(jnilibDir)
-                into(buildDirectory / "classes/java/main/native/lib")
+                into("$baseResourcePath/native/lib")
             }
 
-            // write properties
-            val propFile = buildDirectory / "classes/java/main/native/lib/sentencepiece.properties"
-            propFile.text = "version=${libs.versions.sentencepiece.get()}-$version\n"
+            filesMatching("**/sentencepiece.properties") {
+                expand(mapOf("sentencepieceVersion" to libs.versions.sentencepiece.get(), "version" to version))
+            }
         }
     }
 
diff --git a/extensions/sentencepiece/src/main/resources/native/lib/sentencepiece.properties b/extensions/sentencepiece/src/main/resources/native/lib/sentencepiece.properties
new file mode 100644
index 00000000000..b889be9fcbb
--- /dev/null
+++ b/extensions/sentencepiece/src/main/resources/native/lib/sentencepiece.properties
@@ -0,0 +1 @@
+version=${sentencepieceVersion}-${version}
\ No newline at end of file
diff --git a/extensions/tokenizers/build.gradle.kts b/extensions/tokenizers/build.gradle.kts
index cd74e88d3b5..e63314f6350 100644
--- a/extensions/tokenizers/build.gradle.kts
+++ b/extensions/tokenizers/build.gradle.kts
@@ -20,7 +20,12 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        outputs.dir(buildDirectory / "classes/java/main/native/lib")
+        inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(),
+            "tokenizersVersion" to libs.versions.tokenizers.get(),
+            "version" to version))
+        val baseResourcePath = "${project.projectDir}/build/resources/main"
+        outputs.dirs(File("${baseResourcePath}/native/lib"), File("${baseResourcePath}/nlp"))
+
         doLast {
             var url = "https://publish.djl.ai/tokenizers"
             val (tokenizers, djl) = libs.versions.tokenizers.get() to libs.versions.djl.get()
@@ -48,13 +53,12 @@ tasks {
             }
             copy {
                 from(jnilibDir)
-                into(buildDirectory / "classes/java/main/native/lib")
+                into("$baseResourcePath/native/lib")
             }
 
-            // write properties
-            val propFile = buildDirectory / "classes/java/main/native/lib/tokenizers.properties"
-            propFile.text = "version=$tokenizers-$version\n"
-
+            filesMatching("**/tokenizers.properties") {
+                expand(mapOf("tokenizersVersion" to tokenizers, "version" to version))
+            }
             url = "https://mlrepo.djl.ai/model/nlp"
             val tasks = listOf(
                 "fill_mask",
@@ -63,7 +67,7 @@ tasks {
                 "text_embedding",
                 "token_classification"
             )
-            val prefix = buildDirectory / "classes/java/main/nlp"
+            val prefix = File("$baseResourcePath/nlp")
             for (task in tasks) {
                 var file = prefix / task / "ai.djl.huggingface.pytorch.json"
                 if (file.exists())
diff --git a/extensions/tokenizers/src/main/resources/native/lib/tokenizers.properties b/extensions/tokenizers/src/main/resources/native/lib/tokenizers.properties
new file mode 100644
index 00000000000..ad2f45efc53
--- /dev/null
+++ b/extensions/tokenizers/src/main/resources/native/lib/tokenizers.properties
@@ -0,0 +1 @@
+version=${tokenizersVersion}-${version}
\ No newline at end of file
diff --git a/tools/gradle/android-formatter.gradle b/tools/gradle/android-formatter.gradle
index 7ffe4063807..a89e9a83ca2 100644
--- a/tools/gradle/android-formatter.gradle
+++ b/tools/gradle/android-formatter.gradle
@@ -20,7 +20,11 @@ import com.google.googlejavaformat.java.RemoveUnusedImports
 
 class JavaFormatterPlugin implements Plugin<Project> {
     void apply(Project project) {
-        project.task('formatJava') {
+        tasks.register('formatJava') {
+            def resultFilePath = "build/verifyJava-result.txt"
+            inputs.files(project.sourceSets*.allSource)
+            inputs.files(project.fileTree('generated-src'))
+            outputs.file(project.file(resultFilePath))
             doLast {
                 if (project.getRootProject() == project) {
                     return
@@ -35,6 +39,7 @@ class JavaFormatterPlugin implements Plugin<Project> {
                     }
                 }
             }
+            project.file(resultFilePath).write('Success')
         }
     }
 }

From a9ba8666057fc14ce7e2e1a675a5225722af56e8 Mon Sep 17 00:00:00 2001
From: Benjie Gatt <bgatt@gradle.com>
Date: Thu, 30 May 2024 15:14:27 +0200
Subject: [PATCH 2/6] Incremental build fixes following changes to plugins

---
 buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts | 5 +++++
 tools/gradle/android-formatter.gradle                    | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts b/buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts
index a8bf488ae71..50eec3ad4c4 100644
--- a/buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts
+++ b/buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts
@@ -20,6 +20,10 @@ tasks {
     }
 
     val verifyJava by registering {
+        val resultFilePath = "build/verifyJava-result.txt"
+        inputs.files(project.sourceSets.flatMap { it.allSource })
+        inputs.files(project.fileTree("generated-src"))
+        outputs.file(project.file(resultFilePath))
         doLast {
             val formatter = Main(PrintWriter(System.out, true), PrintWriter(System.err, true), System.`in`)
             for (item in project.sourceSets)
@@ -35,6 +39,7 @@ tasks {
                                     + "See https://github.com/deepjavalibrary/djl/blob/master/docs/development/development_guideline.md#coding-conventions for more details"
                         )
                 }
+            project.file(resultFilePath).writeText("Success")
         }
     }
 
diff --git a/tools/gradle/android-formatter.gradle b/tools/gradle/android-formatter.gradle
index a89e9a83ca2..6ade6d3e1ad 100644
--- a/tools/gradle/android-formatter.gradle
+++ b/tools/gradle/android-formatter.gradle
@@ -21,7 +21,7 @@ import com.google.googlejavaformat.java.RemoveUnusedImports
 class JavaFormatterPlugin implements Plugin<Project> {
     void apply(Project project) {
         tasks.register('formatJava') {
-            def resultFilePath = "build/verifyJava-result.txt"
+            def resultFilePath = "build/formatJava-result.txt"
             inputs.files(project.sourceSets*.allSource)
             inputs.files(project.fileTree('generated-src'))
             outputs.file(project.file(resultFilePath))

From 4689bda39eaac0907e0aa2e6d534b9a002db6a9a Mon Sep 17 00:00:00 2001
From: Benjie Gatt <bgatt@gradle.com>
Date: Fri, 31 May 2024 07:00:21 +0200
Subject: [PATCH 3/6] Fix Spotbugs having conflicting paths for different tasks

---
 buildSrc/src/main/kotlin/ai/djl/check.gradle.kts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts b/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts
index ccbafe0d29e..b5bc14f8ea3 100644
--- a/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts
+++ b/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts
@@ -26,7 +26,7 @@ tasks {
         enabled = true
         reports.create("html") {
             required = true
-            outputLocation = file("$buildDirectory/reports/spotbugs.html")
+            outputLocation = file("$buildDirectory/reports/test/spotbugs.html")
         }
     }
 }

From 36c6ace28cbaf01d1c49f6f95990dac21cb86061 Mon Sep 17 00:00:00 2001
From: Benjie Gatt <bgatt@gradle.com>
Date: Fri, 31 May 2024 08:46:23 +0200
Subject: [PATCH 4/6] Remove system properties pass-through, as it breaks
 incremental builds for all tests due to constant changes in the properties

---
 buildSrc/src/main/kotlin/ai/djl/javaProject.gradle.kts | 3 ---
 examples/build.gradle.kts                              | 8 --------
 integration/build.gradle.kts                           | 8 --------
 3 files changed, 19 deletions(-)

diff --git a/buildSrc/src/main/kotlin/ai/djl/javaProject.gradle.kts b/buildSrc/src/main/kotlin/ai/djl/javaProject.gradle.kts
index fa234cb13da..3cb23c191e3 100644
--- a/buildSrc/src/main/kotlin/ai/djl/javaProject.gradle.kts
+++ b/buildSrc/src/main/kotlin/ai/djl/javaProject.gradle.kts
@@ -49,9 +49,6 @@ tasks {
         }
 
         jvmArgs("--add-opens", "java.base/jdk.internal.loader=ALL-UNNAMED")
-        systemProperties = System.getProperties().toMap() as Map<String, Any>
-        systemProperties.remove("user.dir")
-        // systemProperty "ai.djl.logging.level", "debug"
         systemProperties(
             "org.slf4j.simpleLogger.defaultLogLevel" to "debug",
             "org.slf4j.simpleLogger.log.org.mortbay.log" to "warn",
diff --git a/examples/build.gradle.kts b/examples/build.gradle.kts
index 41f6df08b59..62d1bfcf3b1 100644
--- a/examples/build.gradle.kts
+++ b/examples/build.gradle.kts
@@ -32,18 +32,10 @@ tasks {
 
     run.configure {
         environment("TF_CPP_MIN_LOG_LEVEL" to "1") // turn off TensorFlow print out
-        // @Niels Doucet
-        // Just a heads-up: gradle support warned me about systemProperties System.getProperties(). It's really
-        // dangerous to just copy over all system properties to a task invocation. You should really be specific about
-        // the properties you'd like to expose inside the task, or you might get very strange issues.
-        systemProperties = System.getProperties().toMap() as Map<String, Any>
-        systemProperties.remove("user.dir")
         systemProperty("file.encoding", "UTF-8")
     }
 
     register<JavaExec>("listmodels") {
-        systemProperties(System.getProperties() as Map<String, Any>)
-        systemProperties.remove("user.dir")
         systemProperty("file.encoding", "UTF-8")
         classpath = sourceSets.main.get().runtimeClasspath
         mainClass = "ai.djl.examples.inference.ListModels"
diff --git a/integration/build.gradle.kts b/integration/build.gradle.kts
index 9b8fe6b54fd..8024c148fc3 100644
--- a/integration/build.gradle.kts
+++ b/integration/build.gradle.kts
@@ -40,20 +40,12 @@ tasks {
 
     run.configure {
         environment("TF_CPP_MIN_LOG_LEVEL" to "1") // turn off TensorFlow print out
-        // @Niels Doucet
-        // Just a heads-up: gradle support warned me about systemProperties System.getProperties(). It's really
-        // dangerous to just copy over all system properties to a task invocation. You should really be specific about
-        // the properties you'd like to expose inside the task, or you might get very strange issues.
-        systemProperties = System.getProperties().toMap() as Map<String, Any>
-        systemProperties.remove("user.dir")
         systemProperty("file.encoding", "UTF-8")
         jvmArgs("-Xverify:none")
     }
 
     register<JavaExec>("debugEnv") {
         classpath = sourceSets.main.get().runtimeClasspath
-        systemProperties = System.getProperties().toMap() as Map<String, Any>
-        systemProperties.remove("user.dir")
         systemProperties["ai.djl.logging.level"] = "debug"
         mainClass = "ai.djl.integration.util.DebugEnvironment"
     }

From caedea652aa74eaec059a94d9397643b1c936b48 Mon Sep 17 00:00:00 2001
From: Frank Liu <frankfliu2000@gmail.com>
Date: Sat, 1 Jun 2024 17:32:28 -0700
Subject: [PATCH 5/6] Add system properties back

---
 .github/workflows/continuous.yml              |  5 ++---
 .../src/main/kotlin/ai/djl/check.gradle.kts   |  4 +++-
 .../main/kotlin/ai/djl/javaProject.gradle.kts |  8 +++++++-
 examples/build.gradle.kts                     | 20 +++++++++++++++----
 integration/build.gradle.kts                  | 14 +++++++++----
 5 files changed, 38 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/continuous.yml b/.github/workflows/continuous.yml
index 5c585c32e2d..6bfd6076732 100644
--- a/.github/workflows/continuous.yml
+++ b/.github/workflows/continuous.yml
@@ -85,10 +85,9 @@ jobs:
         run: ./gradlew :engines:pytorch:pytorch-native:compileJNI
       - name: Test ONNX Runtime
         run: |
-          ./gradlew -Dai.djl.default_engine=OnnxRuntime :integration:test --stacktrace
-          ./gradlew :integration:clean
+          ./gradlew -Dai.djl.default_engine=OnnxRuntime :integration:clean :integration:test --stacktrace
       - name: Build with Gradle
-        run: ./gradlew build :jacoco:testCodeCoverageReport --stacktrace
+        run: ./gradlew clean build :jacoco:testCodeCoverageReport --stacktrace
       - name: Upload test results
         uses: actions/upload-artifact@v3
         if: always()
diff --git a/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts b/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts
index b5bc14f8ea3..06ab615d47e 100644
--- a/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts
+++ b/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts
@@ -15,6 +15,7 @@ spotbugs {
     excludeFilter = file("${rootProject.projectDir}/tools/conf/findbugs-exclude.xml")
     ignoreFailures = false
 }
+
 tasks {
     named<SpotBugsTask>("spotbugsMain") {
         reports.create("html") {
@@ -37,6 +38,7 @@ pmd {
     ruleSets = emptyList() // workaround pmd gradle plugin bug
     ruleSetFiles = files("${rootProject.projectDir}/tools/conf/pmd.xml")
 }
+
 tasks.withType<Pmd> {
     reports {
         xml.required = true
@@ -53,8 +55,8 @@ checkstyle {
         "checkstyle.licenseHeader.file" to file("${rootProject.projectDir}/tools/conf/licenseHeader.java")
     )
     configFile = file("${rootProject.projectDir}/tools/conf/checkstyle.xml")
-    
 }
+
 tasks {
     named<Checkstyle>("checkstyleMain") {
         classpath += configurations["compileClasspath"]
diff --git a/buildSrc/src/main/kotlin/ai/djl/javaProject.gradle.kts b/buildSrc/src/main/kotlin/ai/djl/javaProject.gradle.kts
index 3cb23c191e3..2a3dc07fd71 100644
--- a/buildSrc/src/main/kotlin/ai/djl/javaProject.gradle.kts
+++ b/buildSrc/src/main/kotlin/ai/djl/javaProject.gradle.kts
@@ -49,12 +49,18 @@ tasks {
         }
 
         jvmArgs("--add-opens", "java.base/jdk.internal.loader=ALL-UNNAMED")
+        for (prop in System.getProperties().iterator()) {
+            val key = prop.key.toString()
+            if (key.startsWith("ai.djl.")) {
+                systemProperty(key, prop.value)
+            }
+        }
         systemProperties(
             "org.slf4j.simpleLogger.defaultLogLevel" to "debug",
             "org.slf4j.simpleLogger.log.org.mortbay.log" to "warn",
             "org.slf4j.simpleLogger.log.org.testng" to "info",
             "disableProgressBar" to "true",
-            "nightly" to System.getProperty("nightly", "false")
+            "nightly" to System.getProperty("nightly", "false"),
         )
         if (gradle.startParameter.isOffline)
             systemProperty("ai.djl.offline", "true")
diff --git a/examples/build.gradle.kts b/examples/build.gradle.kts
index 62d1bfcf3b1..939b3f6583b 100644
--- a/examples/build.gradle.kts
+++ b/examples/build.gradle.kts
@@ -1,5 +1,3 @@
-@file:Suppress("UNCHECKED_CAST")
-
 plugins {
     ai.djl.javaProject
     application
@@ -32,14 +30,28 @@ tasks {
 
     run.configure {
         environment("TF_CPP_MIN_LOG_LEVEL" to "1") // turn off TensorFlow print out
-        systemProperty("file.encoding", "UTF-8")
+        for (prop in System.getProperties().iterator()) {
+            val key = prop.key.toString()
+            if (key.startsWith("ai.djl.")) {
+                systemProperty(key, prop.value)
+            }
+        }
     }
 
     register<JavaExec>("listmodels") {
-        systemProperty("file.encoding", "UTF-8")
+        for (prop in System.getProperties().iterator()) {
+            val key = prop.key.toString()
+            if (key.startsWith("ai.djl.")) {
+                systemProperty(key, prop.value)
+            }
+        }
+        if (!systemProperties.containsKey("ai.djl.logging.level")) {
+            systemProperty("ai.djl.logging.level", "debug")
+        }
         classpath = sourceSets.main.get().runtimeClasspath
         mainClass = "ai.djl.examples.inference.ListModels"
     }
+
     distTar { enabled = false }
     distZip { enabled = false }
 }
\ No newline at end of file
diff --git a/integration/build.gradle.kts b/integration/build.gradle.kts
index 8024c148fc3..dd00ebf6e11 100644
--- a/integration/build.gradle.kts
+++ b/integration/build.gradle.kts
@@ -1,5 +1,3 @@
-@file:Suppress("UNCHECKED_CAST")
-
 plugins {
     ai.djl.javaProject
     application
@@ -39,14 +37,22 @@ tasks {
     }
 
     run.configure {
+        for (prop in systemProperties.iterator()) {
+            if (prop.key.startsWith("ai.djl.") || prop.key == "nightly") {
+                systemProperty(prop.key, prop.value)
+            }
+        }
         environment("TF_CPP_MIN_LOG_LEVEL" to "1") // turn off TensorFlow print out
-        systemProperty("file.encoding", "UTF-8")
         jvmArgs("-Xverify:none")
     }
 
     register<JavaExec>("debugEnv") {
+        for (prop in systemProperties.iterator()) {
+            if (prop.key.startsWith("ai.djl.") || prop.key == "nightly") {
+                systemProperty(prop.key, prop.value)
+            }
+        }
         classpath = sourceSets.main.get().runtimeClasspath
-        systemProperties["ai.djl.logging.level"] = "debug"
         mainClass = "ai.djl.integration.util.DebugEnvironment"
     }
 

From a6e181b2688ff9cedc8c3266837afe778167ff5d Mon Sep 17 00:00:00 2001
From: Frank Liu <frankfliu2000@gmail.com>
Date: Sat, 1 Jun 2024 20:04:48 -0700
Subject: [PATCH 6/6] Fixes build script for properties generation

---
 api/build.gradle.kts                          |  4 +--
 .../resources/ai/djl/engine/api.properties    |  2 +-
 engines/llama/build.gradle.kts                |  2 +-
 engines/ml/xgboost/build.gradle.kts           |  1 +
 ...ine.properties => mxnet-engine.properties} |  0
 .../tensorflow-api/build.gradle.kts           |  1 +
 .../tensorflow-engine/build.gradle.kts        |  6 ++---
 .../resources/tensorflow-engine.properties    |  2 +-
 engines/tensorrt/build.gradle.kts             |  6 ++---
 extensions/fasttext/build.gradle.kts          |  7 +++--
 extensions/sentencepiece/build.gradle.kts     |  6 ++---
 .../ai/djl/sentencepiece/SpProcessor.java     | 11 ++++++++
 .../ai/djl/sentencepiece/SpTokenizerTest.java |  3 +++
 extensions/tokenizers/build.gradle.kts        |  7 ++---
 .../tokenizers/HuggingFaceTokenizer.java      | 11 ++++++++
 .../java/ai/djl/engine/rust/RsEngineTest.java | 27 +++++++++++++++++++
 .../tokenizers/HuggingFaceTokenizerTest.java  |  9 +++++++
 17 files changed, 84 insertions(+), 21 deletions(-)
 rename engines/mxnet/mxnet-engine/src/main/resources/{mx-engine.properties => mxnet-engine.properties} (100%)
 create mode 100644 extensions/tokenizers/src/test/java/ai/djl/engine/rust/RsEngineTest.java

diff --git a/api/build.gradle.kts b/api/build.gradle.kts
index 8fa167e186a..3c2cdc758e1 100644
--- a/api/build.gradle.kts
+++ b/api/build.gradle.kts
@@ -23,9 +23,9 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        inputs.properties(mapOf("projectVersion" to project.version))
+        inputs.properties(mapOf("version" to version))
         filesMatching("**/api.properties") {
-            expand(mapOf("projectVersion" to project.version))
+            expand(mapOf("version" to version))
         }
     }
 
diff --git a/api/src/main/resources/ai/djl/engine/api.properties b/api/src/main/resources/ai/djl/engine/api.properties
index 0508cf08a2b..7006b178367 100644
--- a/api/src/main/resources/ai/djl/engine/api.properties
+++ b/api/src/main/resources/ai/djl/engine/api.properties
@@ -1 +1 @@
-djl_version=${projectVersion}
\ No newline at end of file
+djl_version=${version}
\ No newline at end of file
diff --git a/engines/llama/build.gradle.kts b/engines/llama/build.gradle.kts
index d19ac5d2741..68ada8e8dd6 100644
--- a/engines/llama/build.gradle.kts
+++ b/engines/llama/build.gradle.kts
@@ -17,7 +17,7 @@ tasks {
     compileJava { dependsOn(processResources) }
 
     processResources {
-        var path = "${project.projectDir}/build/resources/main"
+        val path = "${project.projectDir}/build/resources/main"
         inputs.properties(mapOf("djl_version" to libs.versions.djl.get(), "llamacpp_version" to libs.versions.llamacpp.get()))
         outputs.dir("$path/native/lib")
         doLast {
diff --git a/engines/ml/xgboost/build.gradle.kts b/engines/ml/xgboost/build.gradle.kts
index 824bd2ea987..4a6e6cfa225 100644
--- a/engines/ml/xgboost/build.gradle.kts
+++ b/engines/ml/xgboost/build.gradle.kts
@@ -11,6 +11,7 @@ val xgbFlavor = if (isGpu) "-gpu" else ""
 
 val exclusion by configurations.registering
 
+@Suppress("UnstableApiUsage")
 dependencies {
     api(project(":api"))
     api(libs.commons.logging)
diff --git a/engines/mxnet/mxnet-engine/src/main/resources/mx-engine.properties b/engines/mxnet/mxnet-engine/src/main/resources/mxnet-engine.properties
similarity index 100%
rename from engines/mxnet/mxnet-engine/src/main/resources/mx-engine.properties
rename to engines/mxnet/mxnet-engine/src/main/resources/mxnet-engine.properties
diff --git a/engines/tensorflow/tensorflow-api/build.gradle.kts b/engines/tensorflow/tensorflow-api/build.gradle.kts
index 9fa98d5f2cf..d17e65c3794 100644
--- a/engines/tensorflow/tensorflow-api/build.gradle.kts
+++ b/engines/tensorflow/tensorflow-api/build.gradle.kts
@@ -9,6 +9,7 @@ group = "ai.djl.tensorflow"
 
 val exclusion by configurations.registering
 
+@Suppress("UnstableApiUsage")
 dependencies {
     api(libs.bytedeco.javacpp)
     api(libs.google.protobuf)
diff --git a/engines/tensorflow/tensorflow-engine/build.gradle.kts b/engines/tensorflow/tensorflow-engine/build.gradle.kts
index ea3009c1fe1..8f4c521b89a 100644
--- a/engines/tensorflow/tensorflow-engine/build.gradle.kts
+++ b/engines/tensorflow/tensorflow-engine/build.gradle.kts
@@ -14,9 +14,9 @@ dependencies {
 }
 
 tasks.processResources {
-    inputs.properties(mapOf("djlVersion" to libs.versions.djl, "tensorflowVersion" to libs.versions.tensorflow))
-    filesMatching("**/mxnet-engine.properties") {
-        expand(mapOf("djl_version" to libs.versions.djl, "tensorflow_version" to libs.versions.tensorflow))
+    inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), "tensorflowVersion" to libs.versions.tensorflow.get()))
+    filesMatching("**/tensorflow-engine.properties") {
+        expand(mapOf("djlVersion" to libs.versions.djl.get(), "tensorflowVersion" to libs.versions.tensorflow.get()))
     }
 }
 
diff --git a/engines/tensorflow/tensorflow-engine/src/main/resources/tensorflow-engine.properties b/engines/tensorflow/tensorflow-engine/src/main/resources/tensorflow-engine.properties
index f3ddbf23461..6e470b04b50 100644
--- a/engines/tensorflow/tensorflow-engine/src/main/resources/tensorflow-engine.properties
+++ b/engines/tensorflow/tensorflow-engine/src/main/resources/tensorflow-engine.properties
@@ -1,2 +1,2 @@
-djl_version=${projectVersion}
+djl_version=${djlVersion}
 tensorflow_version=${tensorflowVersion}
\ No newline at end of file
diff --git a/engines/tensorrt/build.gradle.kts b/engines/tensorrt/build.gradle.kts
index 4a63913ca15..b10a246e957 100644
--- a/engines/tensorrt/build.gradle.kts
+++ b/engines/tensorrt/build.gradle.kts
@@ -46,10 +46,10 @@ tasks {
                 from(jnilibDir)
                 into("$baseResourcePath/native/lib")
             }
+        }
 
-            filesMatching("**/tensorrt.properties") {
-                expand(mapOf("trtVersion" to libs.versions.tensorrt.get(), "version" to version))
-            }
+        filesMatching("**/tensorrt.properties") {
+            expand(mapOf("trtVersion" to libs.versions.tensorrt.get(), "version" to version))
         }
     }
 
diff --git a/extensions/fasttext/build.gradle.kts b/extensions/fasttext/build.gradle.kts
index 4bf844093f3..c17b83d7145 100644
--- a/extensions/fasttext/build.gradle.kts
+++ b/extensions/fasttext/build.gradle.kts
@@ -47,11 +47,10 @@ tasks {
                 from(jnilibDir)
                 into("$baseResourcePath/native/lib")
             }
+        }
 
-            // write properties
-            filesMatching("**/fasttext.properties") {
-                expand(mapOf("fasttextVersion" to libs.versions.fasttext.get(), "version" to version))
-            }
+        filesMatching("**/fasttext.properties") {
+            expand(mapOf("fasttextVersion" to libs.versions.fasttext.get(), "version" to version))
         }
     }
 
diff --git a/extensions/sentencepiece/build.gradle.kts b/extensions/sentencepiece/build.gradle.kts
index 71d01b0a256..c634ba15494 100644
--- a/extensions/sentencepiece/build.gradle.kts
+++ b/extensions/sentencepiece/build.gradle.kts
@@ -46,10 +46,10 @@ tasks {
                 from(jnilibDir)
                 into("$baseResourcePath/native/lib")
             }
+        }
 
-            filesMatching("**/sentencepiece.properties") {
-                expand(mapOf("sentencepieceVersion" to libs.versions.sentencepiece.get(), "version" to version))
-            }
+        filesMatching("**/sentencepiece.properties") {
+            expand(mapOf("sentencepieceVersion" to libs.versions.sentencepiece.get(), "version" to version))
         }
     }
 
diff --git a/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpProcessor.java b/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpProcessor.java
index c7af964e9a5..d2a627255e4 100644
--- a/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpProcessor.java
+++ b/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpProcessor.java
@@ -16,6 +16,7 @@
 import ai.djl.sentencepiece.jni.SentencePieceLibrary;
 import ai.djl.util.Ec2Utils;
 import ai.djl.util.NativeResource;
+import ai.djl.util.Platform;
 
 /** The processor holder for SentencePiece. */
 public final class SpProcessor extends NativeResource<Long> {
@@ -42,6 +43,16 @@ static SpProcessor newInstance() {
         return new SpProcessor();
     }
 
+    /**
+     * Returns the version of the sentencepiece.
+     *
+     * @return the version number of the sentencepiece
+     */
+    public String getVersion() {
+        Platform platform = Platform.detectPlatform("sentencepiece");
+        return platform.getVersion();
+    }
+
     void loadModel(String path) {
         SentencePieceLibrary.LIB.loadModel(getHandle(), path);
     }
diff --git a/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java b/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java
index 92a1b0ab1ea..1d18e0a8a83 100644
--- a/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java
+++ b/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java
@@ -13,6 +13,7 @@
 
 package ai.djl.sentencepiece;
 
+import ai.djl.engine.Engine;
 import ai.djl.testing.TestRequirements;
 import ai.djl.training.util.DownloadUtils;
 
@@ -45,6 +46,8 @@ public void testLoadFromBytes() throws IOException {
         Path modelPath = Paths.get("build/test/sp_model/sp_model.model");
         byte[] bytes = Files.readAllBytes(modelPath);
         try (SpTokenizer tokenizer = new SpTokenizer(bytes)) {
+            String djlVersion = Engine.getDjlVersion();
+            Assert.assertEquals(tokenizer.getProcessor().getVersion(), "0.2.0-" + djlVersion);
             String original = "Hello World";
             List<String> tokens = tokenizer.tokenize(original);
             List<String> expected = Arrays.asList("▁He", "ll", "o", "▁", "W", "or", "l", "d");
diff --git a/extensions/tokenizers/build.gradle.kts b/extensions/tokenizers/build.gradle.kts
index e63314f6350..7510aa48022 100644
--- a/extensions/tokenizers/build.gradle.kts
+++ b/extensions/tokenizers/build.gradle.kts
@@ -56,9 +56,6 @@ tasks {
                 into("$baseResourcePath/native/lib")
             }
 
-            filesMatching("**/tokenizers.properties") {
-                expand(mapOf("tokenizersVersion" to tokenizers, "version" to version))
-            }
             url = "https://mlrepo.djl.ai/model/nlp"
             val tasks = listOf(
                 "fill_mask",
@@ -93,6 +90,10 @@ tasks {
                 }
             }
         }
+
+        filesMatching("**/tokenizers.properties") {
+            expand(mapOf("tokenizersVersion" to libs.versions.tokenizers.get(), "version" to version))
+        }
     }
 
     register("compileJNI") {
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java
index 177366faff2..b04b528d4a3 100644
--- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java
+++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java
@@ -21,6 +21,7 @@
 import ai.djl.util.Ec2Utils;
 import ai.djl.util.NativeResource;
 import ai.djl.util.PairList;
+import ai.djl.util.Platform;
 import ai.djl.util.Utils;
 
 import org.slf4j.Logger;
@@ -188,6 +189,16 @@ public static HuggingFaceTokenizer newInstance(InputStream is, Map<String, Strin
         return new HuggingFaceTokenizer(handle, options);
     }
 
+    /**
+     * Returns the version of the Huggingface tokenizer.
+     *
+     * @return the version number of the Huggingface tokenizer
+     */
+    public String getVersion() {
+        Platform platform = Platform.detectPlatform("tokenizers");
+        return platform.getVersion();
+    }
+
     /** {@inheritDoc} */
     @Override
     public List<String> tokenize(String sentence) {
diff --git a/extensions/tokenizers/src/test/java/ai/djl/engine/rust/RsEngineTest.java b/extensions/tokenizers/src/test/java/ai/djl/engine/rust/RsEngineTest.java
new file mode 100644
index 00000000000..aea38767c78
--- /dev/null
+++ b/extensions/tokenizers/src/test/java/ai/djl/engine/rust/RsEngineTest.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.engine.rust;
+
+import ai.djl.engine.Engine;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class RsEngineTest {
+
+    @Test
+    public void testVersion() {
+        Engine engine = Engine.getEngine("Rust");
+        Assert.assertEquals(engine.getVersion(), Engine.getDjlVersion());
+    }
+}
diff --git a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
index ead688539e2..713e1fbeb88 100644
--- a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
+++ b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
@@ -13,6 +13,7 @@
 
 package ai.djl.huggingface.tokenizers;
 
+import ai.djl.engine.Engine;
 import ai.djl.huggingface.tokenizers.jni.CharSpan;
 import ai.djl.testing.TestRequirements;
 import ai.djl.training.util.DownloadUtils;
@@ -33,6 +34,14 @@
 
 public class HuggingFaceTokenizerTest {
 
+    @Test
+    public void testVersion() {
+        try (HuggingFaceTokenizer tokenizer = HuggingFaceTokenizer.newInstance("bert-base-cased")) {
+            String djlVersion = Engine.getDjlVersion();
+            Assert.assertEquals(tokenizer.getVersion(), "0.19.1-" + djlVersion);
+        }
+    }
+
     @Test
     public void testTokenizer() throws IOException {
         String input = "Hello, y'all! How are you 😁 ?";