diff --git a/.github/workflows/continuous.yml b/.github/workflows/continuous.yml index 5c585c32e2d..6bfd6076732 100644 --- a/.github/workflows/continuous.yml +++ b/.github/workflows/continuous.yml @@ -85,10 +85,9 @@ jobs: run: ./gradlew :engines:pytorch:pytorch-native:compileJNI - name: Test ONNX Runtime run: | - ./gradlew -Dai.djl.default_engine=OnnxRuntime :integration:test --stacktrace - ./gradlew :integration:clean + ./gradlew -Dai.djl.default_engine=OnnxRuntime :integration:clean :integration:test --stacktrace - name: Build with Gradle - run: ./gradlew build :jacoco:testCodeCoverageReport --stacktrace + run: ./gradlew clean build :jacoco:testCodeCoverageReport --stacktrace - name: Upload test results uses: actions/upload-artifact@v3 if: always() diff --git a/api/build.gradle.kts b/api/build.gradle.kts index 4d4cc7a2dbe..3c2cdc758e1 100644 --- a/api/build.gradle.kts +++ b/api/build.gradle.kts @@ -23,12 +23,9 @@ tasks { compileJava { dependsOn(processResources) } processResources { - outputs.file(buildDirectory / "classes/java/main/ai/djl/engine/api.properties") - doFirst { - val classesDir = file("$buildDirectory/classes/java/main/ai/djl/engine/") - classesDir.mkdirs() - val propFile = File(classesDir, "api.properties") - propFile.text = "djl_version=${project.version}" + inputs.properties(mapOf("version" to version)) + filesMatching("**/api.properties") { + expand(mapOf("version" to version)) } } diff --git a/api/src/main/resources/ai/djl/engine/api.properties b/api/src/main/resources/ai/djl/engine/api.properties new file mode 100644 index 00000000000..7006b178367 --- /dev/null +++ b/api/src/main/resources/ai/djl/engine/api.properties @@ -0,0 +1 @@ +djl_version=${version} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts b/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts index c65cd9f19ca..06ab615d47e 100644 --- a/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts +++ b/buildSrc/src/main/kotlin/ai/djl/check.gradle.kts @@ -15,6 +15,7 @@ spotbugs { excludeFilter = file("${rootProject.projectDir}/tools/conf/findbugs-exclude.xml") ignoreFailures = false } + tasks { named("spotbugsMain") { reports.create("html") { @@ -26,7 +27,7 @@ tasks { enabled = true reports.create("html") { required = true - outputLocation = file("$buildDirectory/reports/spotbugs.html") + outputLocation = file("$buildDirectory/reports/test/spotbugs.html") } } } @@ -37,6 +38,7 @@ pmd { ruleSets = emptyList() // workaround pmd gradle plugin bug ruleSetFiles = files("${rootProject.projectDir}/tools/conf/pmd.xml") } + tasks.withType { reports { xml.required = true @@ -54,6 +56,7 @@ checkstyle { ) configFile = file("${rootProject.projectDir}/tools/conf/checkstyle.xml") } + tasks { named("checkstyleMain") { classpath += configurations["compileClasspath"] diff --git a/buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts b/buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts index a8bf488ae71..50eec3ad4c4 100644 --- a/buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts +++ b/buildSrc/src/main/kotlin/ai/djl/javaFormatter.gradle.kts @@ -20,6 +20,10 @@ tasks { } val verifyJava by registering { + val resultFilePath = "build/verifyJava-result.txt" + inputs.files(project.sourceSets.flatMap { it.allSource }) + inputs.files(project.fileTree("generated-src")) + outputs.file(project.file(resultFilePath)) doLast { val formatter = Main(PrintWriter(System.out, true), PrintWriter(System.err, true), System.`in`) for (item in project.sourceSets) @@ -35,6 +39,7 @@ tasks { + "See https://github.com/deepjavalibrary/djl/blob/master/docs/development/development_guideline.md#coding-conventions for more details" ) } + project.file(resultFilePath).writeText("Success") } } diff --git a/buildSrc/src/main/kotlin/ai/djl/javaProject.gradle.kts b/buildSrc/src/main/kotlin/ai/djl/javaProject.gradle.kts index fa234cb13da..2a3dc07fd71 100644 --- a/buildSrc/src/main/kotlin/ai/djl/javaProject.gradle.kts +++ b/buildSrc/src/main/kotlin/ai/djl/javaProject.gradle.kts @@ -49,15 +49,18 @@ tasks { } jvmArgs("--add-opens", "java.base/jdk.internal.loader=ALL-UNNAMED") - systemProperties = System.getProperties().toMap() as Map - systemProperties.remove("user.dir") - // systemProperty "ai.djl.logging.level", "debug" + for (prop in System.getProperties().iterator()) { + val key = prop.key.toString() + if (key.startsWith("ai.djl.")) { + systemProperty(key, prop.value) + } + } systemProperties( "org.slf4j.simpleLogger.defaultLogLevel" to "debug", "org.slf4j.simpleLogger.log.org.mortbay.log" to "warn", "org.slf4j.simpleLogger.log.org.testng" to "info", "disableProgressBar" to "true", - "nightly" to System.getProperty("nightly", "false") + "nightly" to System.getProperty("nightly", "false"), ) if (gradle.startParameter.isOffline) systemProperty("ai.djl.offline", "true") diff --git a/engines/llama/build.gradle.kts b/engines/llama/build.gradle.kts index 3ca39b38770..68ada8e8dd6 100644 --- a/engines/llama/build.gradle.kts +++ b/engines/llama/build.gradle.kts @@ -17,7 +17,9 @@ tasks { compileJava { dependsOn(processResources) } processResources { - outputs.dir(project.projectDir / "build/classes/java/main/native/lib") + val path = "${project.projectDir}/build/resources/main" + inputs.properties(mapOf("djl_version" to libs.versions.djl.get(), "llamacpp_version" to libs.versions.llamacpp.get())) + outputs.dir("$path/native/lib") doLast { val llamacpp = libs.versions.llamacpp.get() val djl = libs.versions.djl.get() @@ -49,15 +51,15 @@ tasks { } copy { from(jnilibDir) - into(project.projectDir / "build/classes/java/main/native/lib") + into("$path/native/lib") } // write properties - val propFile = project.projectDir / "build/classes/java/main/native/lib/llama.properties" + val propFile = file("$path/native/lib/llama.properties") propFile.text = "version=$llamacpp-$version\n" url = "https://mlrepo.djl.ai/model/nlp/text_generation/ai/djl/huggingface/gguf/models.json.gz" - val prefix = project.projectDir / "build/classes/java/main/nlp/text_generation" + val prefix = File("$path/nlp/text_generation") val file = prefix / "ai.djl.huggingface.gguf.json" if (file.exists()) project.logger.lifecycle("gguf index file already exists") diff --git a/engines/ml/xgboost/build.gradle.kts b/engines/ml/xgboost/build.gradle.kts index f9f976bfd59..4a6e6cfa225 100644 --- a/engines/ml/xgboost/build.gradle.kts +++ b/engines/ml/xgboost/build.gradle.kts @@ -43,7 +43,8 @@ tasks { compileJava { dependsOn(processResources) } processResources { - val jnilibDir = buildDirectory / "classes/java/main/lib/linux/aarch64/" + val jnilibDir = buildDirectory / "resources/java/main/lib/linux/aarch64/" + inputs.properties(mapOf("xgboostVersion" to libs.versions.xgboost.get())) outputs.dir(jnilibDir) doLast { val url = diff --git a/engines/mxnet/jnarator/build.gradle.kts b/engines/mxnet/jnarator/build.gradle.kts index b0af9fd453f..3f63a179a19 100644 --- a/engines/mxnet/jnarator/build.gradle.kts +++ b/engines/mxnet/jnarator/build.gradle.kts @@ -20,6 +20,7 @@ tasks { pmdMain { exclude("ai/djl/mxnet/jnarator/parser/*") } jar { + dependsOn(generateGrammarSource) manifest { attributes( "Main-Class" to "ai.djl.mxnet.jnarator.Main", @@ -30,4 +31,13 @@ tasks { duplicatesStrategy = DuplicatesStrategy.INCLUDE from(configurations.runtimeClasspath.get().map { if (it.isDirectory()) it else zipTree(it) }) } + + generateGrammarSource { + dependsOn(verifyJava) + } + + generateTestGrammarSource { + dependsOn(verifyJava) + } + } \ No newline at end of file diff --git a/engines/mxnet/mxnet-engine/build.gradle.kts b/engines/mxnet/mxnet-engine/build.gradle.kts index 3c0eb2cf72a..0b825755262 100644 --- a/engines/mxnet/mxnet-engine/build.gradle.kts +++ b/engines/mxnet/mxnet-engine/build.gradle.kts @@ -25,11 +25,9 @@ sourceSets.main { tasks { processResources { - doFirst { - val classesDir = buildDirectory / "classes/java/main/" - classesDir.mkdirs() - val file = classesDir / "mxnet-engine.properties" - file.text = "djl_version=${libs.versions.djl.get()}\nmxnet_version=" + libs.versions.mxnet.get() + inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), "mxnetVersion" to libs.versions.mxnet.get())) + filesMatching("**/mxnet-engine.properties") { + expand(mapOf("djlVersion" to libs.versions.djl.get(), "mxnetVersion" to libs.versions.mxnet.get())) } } @@ -39,7 +37,11 @@ tasks { val jnarator by registering { val jnaratorJar = project(":engines:mxnet:jnarator").tasks.jar dependsOn(jnaratorJar) + inputs.file("${project.projectDir}/src/main/jna/mapping.properties") + .withPathSensitivity(PathSensitivity.RELATIVE) + .withPropertyName("jna/mapping.properties file") outputs.dir(buildDirectory / "generated-src") + outputs.cacheIf { true } doLast { val jnaGenerator = jnaratorJar.get().outputs.files.singleFile javaexec { @@ -106,6 +108,7 @@ tasks { } compileJava { dependsOn(jnarator) } javadoc { dependsOn(jnarator) } + verifyJava { dependsOn(jnarator) } publishing { publications { diff --git a/engines/mxnet/mxnet-engine/src/main/resources/mxnet-engine.properties b/engines/mxnet/mxnet-engine/src/main/resources/mxnet-engine.properties new file mode 100644 index 00000000000..cbfb997e870 --- /dev/null +++ b/engines/mxnet/mxnet-engine/src/main/resources/mxnet-engine.properties @@ -0,0 +1,2 @@ +djl_version=${djlVersion} +mxnet_version=${mxnetVersion} \ No newline at end of file diff --git a/engines/onnxruntime/onnxruntime-engine/build.gradle.kts b/engines/onnxruntime/onnxruntime-engine/build.gradle.kts index 548fda021fc..5c88c2f2177 100644 --- a/engines/onnxruntime/onnxruntime-engine/build.gradle.kts +++ b/engines/onnxruntime/onnxruntime-engine/build.gradle.kts @@ -20,7 +20,8 @@ dependencies { tasks { processResources { - outputs.dir(projectDir / "build/classes/java/main/nlp") + var basePath = "${project.projectDir}/build/resources/main/nlp" + outputs.dir(basePath) doLast { val url = "https://mlrepo.djl.ai/model/nlp" val tasks = listOf( @@ -30,9 +31,8 @@ tasks { "text_embedding", "token_classification" ) - val prefix = projectDir / "build/classes/java/main/nlp" for (task in tasks) { - val file = prefix / task / "ai.djl.huggingface.onnxruntime.json" + val file = File("$basePath/$task/ai.djl.huggingface.onnxruntime.json") if (file.exists()) project.logger.lifecycle("model zoo metadata alrady exists: $task") else { diff --git a/engines/pytorch/pytorch-engine/build.gradle.kts b/engines/pytorch/pytorch-engine/build.gradle.kts index ba93c92d3ac..fbed0d4cd94 100644 --- a/engines/pytorch/pytorch-engine/build.gradle.kts +++ b/engines/pytorch/pytorch-engine/build.gradle.kts @@ -21,12 +21,9 @@ tasks { compileJava { dependsOn(processResources) } processResources { - outputs.file(buildDirectory / "classes/java/main/pytorch-engine.properties") - doFirst { - val classesDir = buildDirectory / "classes/java/main/" - classesDir.mkdirs() - val propFile = classesDir / "pytorch-engine.properties" - propFile.text = "djl_version=${libs.versions.djl.get()}\npytorch_version=${libs.versions.pytorch.get()}" + inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), "pytorchVersion" to libs.versions.pytorch.get())) + filesMatching("**/pytorch-engine.properties") { + expand(mapOf("djlVersion" to libs.versions.djl.get(), "pytorchVersion" to libs.versions.pytorch.get())) } } diff --git a/engines/pytorch/pytorch-engine/src/main/resources/pytorch-engine.properties b/engines/pytorch/pytorch-engine/src/main/resources/pytorch-engine.properties new file mode 100644 index 00000000000..7ca0f6e1b1f --- /dev/null +++ b/engines/pytorch/pytorch-engine/src/main/resources/pytorch-engine.properties @@ -0,0 +1,2 @@ +djl_version=${djlVersion} +pytorch_version=${pytorchVersion} \ No newline at end of file diff --git a/engines/tensorflow/tensorflow-engine/build.gradle.kts b/engines/tensorflow/tensorflow-engine/build.gradle.kts index 5d6760f0d6c..8f4c521b89a 100644 --- a/engines/tensorflow/tensorflow-engine/build.gradle.kts +++ b/engines/tensorflow/tensorflow-engine/build.gradle.kts @@ -14,11 +14,9 @@ dependencies { } tasks.processResources { - doFirst { - val classesDir = buildDirectory / "classes/java/main/" - classesDir.mkdirs() - val file = classesDir / "tensorflow-engine.properties" - file.text = "djl_version=${libs.versions.djl.get()}\ntensorflow_version=${libs.versions.tensorflow.get()}" + inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), "tensorflowVersion" to libs.versions.tensorflow.get())) + filesMatching("**/tensorflow-engine.properties") { + expand(mapOf("djlVersion" to libs.versions.djl.get(), "tensorflowVersion" to libs.versions.tensorflow.get())) } } diff --git a/engines/tensorflow/tensorflow-engine/src/main/resources/tensorflow-engine.properties b/engines/tensorflow/tensorflow-engine/src/main/resources/tensorflow-engine.properties new file mode 100644 index 00000000000..6e470b04b50 --- /dev/null +++ b/engines/tensorflow/tensorflow-engine/src/main/resources/tensorflow-engine.properties @@ -0,0 +1,2 @@ +djl_version=${djlVersion} +tensorflow_version=${tensorflowVersion} \ No newline at end of file diff --git a/engines/tensorrt/build.gradle.kts b/engines/tensorrt/build.gradle.kts index 3c8a118eeef..b10a246e957 100644 --- a/engines/tensorrt/build.gradle.kts +++ b/engines/tensorrt/build.gradle.kts @@ -21,7 +21,10 @@ tasks { compileJava { dependsOn(processResources) } processResources { - outputs.dir(buildDirectory / "classes/java/main/native/lib") + inputs.properties(mapOf("djlVersion" to libs.versions.djl, "trtVersion" to libs.versions.tensorrt.get(), + "version" to version)) + val baseResourcePath = "${project.projectDir}/build/resources/main" + outputs.dir(file("${baseResourcePath}/native/lib")) doLast { val trtVersion = libs.versions.tensorrt.get() val djlVersion = libs.versions.djl.get() @@ -41,12 +44,12 @@ tasks { copy { from(jnilibDir) - into(buildDirectory / "classes/java/main/native/lib") + into("$baseResourcePath/native/lib") } + } - // write properties - val propFile = buildDirectory / "classes/java/main/native/lib/tensorrt.properties" - propFile.text = "version=${trtVersion}-${version}\n" + filesMatching("**/tensorrt.properties") { + expand(mapOf("trtVersion" to libs.versions.tensorrt.get(), "version" to version)) } } diff --git a/engines/tensorrt/src/main/resources/tensorrt.properties b/engines/tensorrt/src/main/resources/tensorrt.properties new file mode 100644 index 00000000000..29bc031ee51 --- /dev/null +++ b/engines/tensorrt/src/main/resources/tensorrt.properties @@ -0,0 +1 @@ +version=${trtVersion}-${version} \ No newline at end of file diff --git a/examples/build.gradle.kts b/examples/build.gradle.kts index 41f6df08b59..939b3f6583b 100644 --- a/examples/build.gradle.kts +++ b/examples/build.gradle.kts @@ -1,5 +1,3 @@ -@file:Suppress("UNCHECKED_CAST") - plugins { ai.djl.javaProject application @@ -32,22 +30,28 @@ tasks { run.configure { environment("TF_CPP_MIN_LOG_LEVEL" to "1") // turn off TensorFlow print out - // @Niels Doucet - // Just a heads-up: gradle support warned me about systemProperties System.getProperties(). It's really - // dangerous to just copy over all system properties to a task invocation. You should really be specific about - // the properties you'd like to expose inside the task, or you might get very strange issues. - systemProperties = System.getProperties().toMap() as Map - systemProperties.remove("user.dir") - systemProperty("file.encoding", "UTF-8") + for (prop in System.getProperties().iterator()) { + val key = prop.key.toString() + if (key.startsWith("ai.djl.")) { + systemProperty(key, prop.value) + } + } } register("listmodels") { - systemProperties(System.getProperties() as Map) - systemProperties.remove("user.dir") - systemProperty("file.encoding", "UTF-8") + for (prop in System.getProperties().iterator()) { + val key = prop.key.toString() + if (key.startsWith("ai.djl.")) { + systemProperty(key, prop.value) + } + } + if (!systemProperties.containsKey("ai.djl.logging.level")) { + systemProperty("ai.djl.logging.level", "debug") + } classpath = sourceSets.main.get().runtimeClasspath mainClass = "ai.djl.examples.inference.ListModels" } + distTar { enabled = false } distZip { enabled = false } } \ No newline at end of file diff --git a/extensions/fasttext/build.gradle.kts b/extensions/fasttext/build.gradle.kts index 046947a3394..c17b83d7145 100644 --- a/extensions/fasttext/build.gradle.kts +++ b/extensions/fasttext/build.gradle.kts @@ -18,7 +18,11 @@ tasks { compileJava { dependsOn(processResources) } processResources { - outputs.dir(buildDirectory / "classes/java/main/native/lib") + inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), + "fasttextVersion" to libs.versions.fasttext.get(), + "version" to version)) + val baseResourcePath = "${project.projectDir}/build/resources/main" + outputs.dir("$baseResourcePath/native/lib") doLast { val url = "https://publish.djl.ai/fasttext-${libs.versions.fasttext.get()}/jnilib/${libs.versions.djl.get()}" @@ -41,12 +45,12 @@ tasks { } copy { from(jnilibDir) - into(buildDirectory / "classes/java/main/native/lib") + into("$baseResourcePath/native/lib") } + } - // write properties - val propFile = buildDirectory / "classes/java/main/native/lib/fasttext.properties" - propFile.text = "version=${libs.versions.fasttext.get()}-${version}\n" + filesMatching("**/fasttext.properties") { + expand(mapOf("fasttextVersion" to libs.versions.fasttext.get(), "version" to version)) } } diff --git a/extensions/fasttext/src/main/resources/native/lib/fasttext.properties b/extensions/fasttext/src/main/resources/native/lib/fasttext.properties new file mode 100644 index 00000000000..8b69cf67a61 --- /dev/null +++ b/extensions/fasttext/src/main/resources/native/lib/fasttext.properties @@ -0,0 +1 @@ +version=${fasttextVersion}-${version} \ No newline at end of file diff --git a/extensions/sentencepiece/build.gradle.kts b/extensions/sentencepiece/build.gradle.kts index b872b76c83a..c634ba15494 100644 --- a/extensions/sentencepiece/build.gradle.kts +++ b/extensions/sentencepiece/build.gradle.kts @@ -17,7 +17,9 @@ tasks { compileJava { dependsOn(processResources) } processResources { - outputs.dir(buildDirectory / "classes/java/main/native/lib") + val baseResourcePath = "${project.projectDir}/build/resources/main" + inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), "sentencepieceVersion" to libs.versions.sentencepiece.get())) + outputs.dir("$baseResourcePath/native/lib") doLast { val url = "https://publish.djl.ai/sentencepiece-${libs.versions.sentencepiece.get()}/jnilib/${libs.versions.djl.get()}" @@ -42,12 +44,12 @@ tasks { } copy { from(jnilibDir) - into(buildDirectory / "classes/java/main/native/lib") + into("$baseResourcePath/native/lib") } + } - // write properties - val propFile = buildDirectory / "classes/java/main/native/lib/sentencepiece.properties" - propFile.text = "version=${libs.versions.sentencepiece.get()}-$version\n" + filesMatching("**/sentencepiece.properties") { + expand(mapOf("sentencepieceVersion" to libs.versions.sentencepiece.get(), "version" to version)) } } diff --git a/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpProcessor.java b/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpProcessor.java index c7af964e9a5..d2a627255e4 100644 --- a/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpProcessor.java +++ b/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpProcessor.java @@ -16,6 +16,7 @@ import ai.djl.sentencepiece.jni.SentencePieceLibrary; import ai.djl.util.Ec2Utils; import ai.djl.util.NativeResource; +import ai.djl.util.Platform; /** The processor holder for SentencePiece. */ public final class SpProcessor extends NativeResource { @@ -42,6 +43,16 @@ static SpProcessor newInstance() { return new SpProcessor(); } + /** + * Returns the version of the sentencepiece. + * + * @return the version number of the sentencepiece + */ + public String getVersion() { + Platform platform = Platform.detectPlatform("sentencepiece"); + return platform.getVersion(); + } + void loadModel(String path) { SentencePieceLibrary.LIB.loadModel(getHandle(), path); } diff --git a/extensions/sentencepiece/src/main/resources/native/lib/sentencepiece.properties b/extensions/sentencepiece/src/main/resources/native/lib/sentencepiece.properties new file mode 100644 index 00000000000..b889be9fcbb --- /dev/null +++ b/extensions/sentencepiece/src/main/resources/native/lib/sentencepiece.properties @@ -0,0 +1 @@ +version=${sentencepieceVersion}-${version} \ No newline at end of file diff --git a/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java b/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java index 92a1b0ab1ea..1d18e0a8a83 100644 --- a/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java +++ b/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java @@ -13,6 +13,7 @@ package ai.djl.sentencepiece; +import ai.djl.engine.Engine; import ai.djl.testing.TestRequirements; import ai.djl.training.util.DownloadUtils; @@ -45,6 +46,8 @@ public void testLoadFromBytes() throws IOException { Path modelPath = Paths.get("build/test/sp_model/sp_model.model"); byte[] bytes = Files.readAllBytes(modelPath); try (SpTokenizer tokenizer = new SpTokenizer(bytes)) { + String djlVersion = Engine.getDjlVersion(); + Assert.assertEquals(tokenizer.getProcessor().getVersion(), "0.2.0-" + djlVersion); String original = "Hello World"; List tokens = tokenizer.tokenize(original); List expected = Arrays.asList("▁He", "ll", "o", "▁", "W", "or", "l", "d"); diff --git a/extensions/tokenizers/build.gradle.kts b/extensions/tokenizers/build.gradle.kts index cd74e88d3b5..7510aa48022 100644 --- a/extensions/tokenizers/build.gradle.kts +++ b/extensions/tokenizers/build.gradle.kts @@ -20,7 +20,12 @@ tasks { compileJava { dependsOn(processResources) } processResources { - outputs.dir(buildDirectory / "classes/java/main/native/lib") + inputs.properties(mapOf("djlVersion" to libs.versions.djl.get(), + "tokenizersVersion" to libs.versions.tokenizers.get(), + "version" to version)) + val baseResourcePath = "${project.projectDir}/build/resources/main" + outputs.dirs(File("${baseResourcePath}/native/lib"), File("${baseResourcePath}/nlp")) + doLast { var url = "https://publish.djl.ai/tokenizers" val (tokenizers, djl) = libs.versions.tokenizers.get() to libs.versions.djl.get() @@ -48,13 +53,9 @@ tasks { } copy { from(jnilibDir) - into(buildDirectory / "classes/java/main/native/lib") + into("$baseResourcePath/native/lib") } - // write properties - val propFile = buildDirectory / "classes/java/main/native/lib/tokenizers.properties" - propFile.text = "version=$tokenizers-$version\n" - url = "https://mlrepo.djl.ai/model/nlp" val tasks = listOf( "fill_mask", @@ -63,7 +64,7 @@ tasks { "text_embedding", "token_classification" ) - val prefix = buildDirectory / "classes/java/main/nlp" + val prefix = File("$baseResourcePath/nlp") for (task in tasks) { var file = prefix / task / "ai.djl.huggingface.pytorch.json" if (file.exists()) @@ -89,6 +90,10 @@ tasks { } } } + + filesMatching("**/tokenizers.properties") { + expand(mapOf("tokenizersVersion" to libs.versions.tokenizers.get(), "version" to version)) + } } register("compileJNI") { diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java index 177366faff2..b04b528d4a3 100644 --- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java +++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java @@ -21,6 +21,7 @@ import ai.djl.util.Ec2Utils; import ai.djl.util.NativeResource; import ai.djl.util.PairList; +import ai.djl.util.Platform; import ai.djl.util.Utils; import org.slf4j.Logger; @@ -188,6 +189,16 @@ public static HuggingFaceTokenizer newInstance(InputStream is, Map tokenize(String sentence) { diff --git a/extensions/tokenizers/src/main/resources/native/lib/tokenizers.properties b/extensions/tokenizers/src/main/resources/native/lib/tokenizers.properties new file mode 100644 index 00000000000..ad2f45efc53 --- /dev/null +++ b/extensions/tokenizers/src/main/resources/native/lib/tokenizers.properties @@ -0,0 +1 @@ +version=${tokenizersVersion}-${version} \ No newline at end of file diff --git a/extensions/tokenizers/src/test/java/ai/djl/engine/rust/RsEngineTest.java b/extensions/tokenizers/src/test/java/ai/djl/engine/rust/RsEngineTest.java new file mode 100644 index 00000000000..aea38767c78 --- /dev/null +++ b/extensions/tokenizers/src/test/java/ai/djl/engine/rust/RsEngineTest.java @@ -0,0 +1,27 @@ +/* + * Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +package ai.djl.engine.rust; + +import ai.djl.engine.Engine; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class RsEngineTest { + + @Test + public void testVersion() { + Engine engine = Engine.getEngine("Rust"); + Assert.assertEquals(engine.getVersion(), Engine.getDjlVersion()); + } +} diff --git a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java index ead688539e2..713e1fbeb88 100644 --- a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java +++ b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java @@ -13,6 +13,7 @@ package ai.djl.huggingface.tokenizers; +import ai.djl.engine.Engine; import ai.djl.huggingface.tokenizers.jni.CharSpan; import ai.djl.testing.TestRequirements; import ai.djl.training.util.DownloadUtils; @@ -33,6 +34,14 @@ public class HuggingFaceTokenizerTest { + @Test + public void testVersion() { + try (HuggingFaceTokenizer tokenizer = HuggingFaceTokenizer.newInstance("bert-base-cased")) { + String djlVersion = Engine.getDjlVersion(); + Assert.assertEquals(tokenizer.getVersion(), "0.19.1-" + djlVersion); + } + } + @Test public void testTokenizer() throws IOException { String input = "Hello, y'all! How are you 😁 ?"; diff --git a/integration/build.gradle.kts b/integration/build.gradle.kts index 9b8fe6b54fd..dd00ebf6e11 100644 --- a/integration/build.gradle.kts +++ b/integration/build.gradle.kts @@ -1,5 +1,3 @@ -@file:Suppress("UNCHECKED_CAST") - plugins { ai.djl.javaProject application @@ -39,22 +37,22 @@ tasks { } run.configure { + for (prop in systemProperties.iterator()) { + if (prop.key.startsWith("ai.djl.") || prop.key == "nightly") { + systemProperty(prop.key, prop.value) + } + } environment("TF_CPP_MIN_LOG_LEVEL" to "1") // turn off TensorFlow print out - // @Niels Doucet - // Just a heads-up: gradle support warned me about systemProperties System.getProperties(). It's really - // dangerous to just copy over all system properties to a task invocation. You should really be specific about - // the properties you'd like to expose inside the task, or you might get very strange issues. - systemProperties = System.getProperties().toMap() as Map - systemProperties.remove("user.dir") - systemProperty("file.encoding", "UTF-8") jvmArgs("-Xverify:none") } register("debugEnv") { + for (prop in systemProperties.iterator()) { + if (prop.key.startsWith("ai.djl.") || prop.key == "nightly") { + systemProperty(prop.key, prop.value) + } + } classpath = sourceSets.main.get().runtimeClasspath - systemProperties = System.getProperties().toMap() as Map - systemProperties.remove("user.dir") - systemProperties["ai.djl.logging.level"] = "debug" mainClass = "ai.djl.integration.util.DebugEnvironment" } diff --git a/tools/gradle/android-formatter.gradle b/tools/gradle/android-formatter.gradle index 7ffe4063807..6ade6d3e1ad 100644 --- a/tools/gradle/android-formatter.gradle +++ b/tools/gradle/android-formatter.gradle @@ -20,7 +20,11 @@ import com.google.googlejavaformat.java.RemoveUnusedImports class JavaFormatterPlugin implements Plugin { void apply(Project project) { - project.task('formatJava') { + tasks.register('formatJava') { + def resultFilePath = "build/formatJava-result.txt" + inputs.files(project.sourceSets*.allSource) + inputs.files(project.fileTree('generated-src')) + outputs.file(project.file(resultFilePath)) doLast { if (project.getRootProject() == project) { return @@ -35,6 +39,7 @@ class JavaFormatterPlugin implements Plugin { } } } + project.file(resultFilePath).write('Success') } } }