From 0506c41cecc30b4a86898257d40abd05f185fed2 Mon Sep 17 00:00:00 2001 From: handymenny Date: Mon, 18 Nov 2024 00:20:09 +0100 Subject: [PATCH 1/3] reparsing: Add processing time to reparsing index --- .../uecapabilityparser/model/index/ReparsingIndex.kt | 1 + .../uecapabilityparser/server/JavalinApp.kt | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/it/smartphonecombo/uecapabilityparser/model/index/ReparsingIndex.kt b/src/main/java/it/smartphonecombo/uecapabilityparser/model/index/ReparsingIndex.kt index 1e54f242..3164c5bc 100644 --- a/src/main/java/it/smartphonecombo/uecapabilityparser/model/index/ReparsingIndex.kt +++ b/src/main/java/it/smartphonecombo/uecapabilityparser/model/index/ReparsingIndex.kt @@ -64,4 +64,5 @@ data class ReparsingIndexLine( @Required val timestamp: Long = Instant.now().toEpochMilli(), @Required val parserVersion: String = Config.getOrDefault("project.version", ""), val error: String? = null, + var processingTimeMs: Long = 0, ) diff --git a/src/main/java/it/smartphonecombo/uecapabilityparser/server/JavalinApp.kt b/src/main/java/it/smartphonecombo/uecapabilityparser/server/JavalinApp.kt index a7ed52df..ed16c996 100644 --- a/src/main/java/it/smartphonecombo/uecapabilityparser/server/JavalinApp.kt +++ b/src/main/java/it/smartphonecombo/uecapabilityparser/server/JavalinApp.kt @@ -137,7 +137,14 @@ class JavalinApp { index .getAll() .filter { shouldReparse(it, index, !auto, parserVersion) } - .map { async { reparseItem(it, index, store, compression, !auto) } } + .map { + async { + val elapsedTime = measureTimeMillis { + reparseItem(it, index, store, compression, !auto) + } + index.reparsingIndex[it.id]?.processingTimeMs = elapsedTime + } + } .awaitAll() index.reparsingIndex.store("$store/reparsing.json") From 7c5a1a0385435b283dde37de620309994e0e90bf Mon Sep 17 00:00:00 2001 From: handymenny Date: Mon, 18 Nov 2024 00:20:51 +0100 Subject: [PATCH 2/3] tests: update oracles --- .../server/oracleForReparse/auto/reparsing.json | 4 ++-- .../oracleForReparse/autoCompress/reparsing.json | 4 ++-- .../oracleForReparse/autoWithIndex/reparsing.json | 2 +- .../server/oracleForReparse/force/reparsing.json | 10 +++++----- .../oracleForReparse/forceBad/reparsing.json | 14 +++++++------- .../oracleForReparse/forceCompress/reparsing.json | 10 +++++----- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/test/resources/server/oracleForReparse/auto/reparsing.json b/src/test/resources/server/oracleForReparse/auto/reparsing.json index 972aef7a..9a9beb31 100644 --- a/src/test/resources/server/oracleForReparse/auto/reparsing.json +++ b/src/test/resources/server/oracleForReparse/auto/reparsing.json @@ -1,2 +1,2 @@ -{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c907","timestamp":1730748179933,"parserVersion":"staging"} -{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c908","timestamp":1730748179903,"parserVersion":"staging"} +{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c907","timestamp":1731885438628,"parserVersion":"staging","processingTimeMs":20} +{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c908","timestamp":1731885438608,"parserVersion":"staging","processingTimeMs":20} diff --git a/src/test/resources/server/oracleForReparse/autoCompress/reparsing.json b/src/test/resources/server/oracleForReparse/autoCompress/reparsing.json index 8fb2ec54..ceb17e78 100644 --- a/src/test/resources/server/oracleForReparse/autoCompress/reparsing.json +++ b/src/test/resources/server/oracleForReparse/autoCompress/reparsing.json @@ -1,2 +1,2 @@ -{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c907","timestamp":1730748179709,"parserVersion":"staging"} -{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c908","timestamp":1730748179664,"parserVersion":"staging"} +{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c907","timestamp":1731885438410,"parserVersion":"staging","processingTimeMs":25} +{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c908","timestamp":1731885438386,"parserVersion":"staging","processingTimeMs":54} diff --git a/src/test/resources/server/oracleForReparse/autoWithIndex/reparsing.json b/src/test/resources/server/oracleForReparse/autoWithIndex/reparsing.json index e870b39a..f40d9eaf 100644 --- a/src/test/resources/server/oracleForReparse/autoWithIndex/reparsing.json +++ b/src/test/resources/server/oracleForReparse/autoWithIndex/reparsing.json @@ -1,2 +1,2 @@ -{"id":"bc8f5359-c997-47c2-836d-8a4df2d1c417","timestamp":1731876465393,"parserVersion":"staging"} +{"id":"bc8f5359-c997-47c2-836d-8a4df2d1c417","timestamp":1731885437704,"parserVersion":"staging","processingTimeMs":1638} {"id":"bc8f5359-c997-47c2-836d-8a4df2d1c417-0","timestamp":1730748514633,"parserVersion":"staging"} diff --git a/src/test/resources/server/oracleForReparse/force/reparsing.json b/src/test/resources/server/oracleForReparse/force/reparsing.json index 88680dc5..408a206e 100644 --- a/src/test/resources/server/oracleForReparse/force/reparsing.json +++ b/src/test/resources/server/oracleForReparse/force/reparsing.json @@ -1,5 +1,5 @@ -{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c907","timestamp":1730748180760,"parserVersion":"staging"} -{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c908","timestamp":1730748180727,"parserVersion":"staging"} -{"id":"bc8f5359-c997-47c2-836d-8a4df2d1c417","timestamp":1730748180199,"parserVersion":"staging"} -{"id":"c3e783c9-54bc-42ee-970c-d871117399b2","timestamp":1730748180660,"parserVersion":"staging"} -{"id":"cd7eb9db-196d-46a5-938d-68061a100cdb","timestamp":1730748180358,"parserVersion":"staging"} +{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c907","timestamp":1731885439584,"parserVersion":"staging","processingTimeMs":20} +{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c908","timestamp":1731885439565,"parserVersion":"staging","processingTimeMs":20} +{"id":"bc8f5359-c997-47c2-836d-8a4df2d1c417","timestamp":1731885439039,"parserVersion":"staging","processingTimeMs":124} +{"id":"c3e783c9-54bc-42ee-970c-d871117399b2","timestamp":1731885439489,"parserVersion":"staging","processingTimeMs":288} +{"id":"cd7eb9db-196d-46a5-938d-68061a100cdb","timestamp":1731885439235,"parserVersion":"staging","processingTimeMs":196} diff --git a/src/test/resources/server/oracleForReparse/forceBad/reparsing.json b/src/test/resources/server/oracleForReparse/forceBad/reparsing.json index efc8fd14..b34fb84d 100644 --- a/src/test/resources/server/oracleForReparse/forceBad/reparsing.json +++ b/src/test/resources/server/oracleForReparse/forceBad/reparsing.json @@ -1,7 +1,7 @@ -{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c907","timestamp":1731882693822,"parserVersion":"staging","error":"Reparsed 0dfbcad1-41be-4dc1-bcea-8407f6a2c907 Capabilities is null"} -{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c908","timestamp":1731882693820,"parserVersion":"staging","error":"Something weird, inputs list >= rat List"} -{"id":"7dc8d574-7a05-4ee4-b370-059ddd38d144","timestamp":1731882693641,"parserVersion":"staging","error":"Extension count cannot be 0. This procedure is only invoked if there is at least one extension addition being encoded."} -{"id":"bc8f5359-c997-47c2-836d-8a4df2d1836d","timestamp":1731882693810,"parserVersion":"staging","error":"Reparsed bc8f5359-c997-47c2-836d-8a4df2d1836d Capabilities is null"} -{"id":"bc8f5359-c997-47c2-836d-8a4df2d1c417","timestamp":1731882693808,"parserVersion":"staging","error":"Reparsed bc8f5359-c997-47c2-836d-8a4df2d1c417 Capabilities is null"} -{"id":"bc8f5359-c997-47c2-9999-8a4df2d1836d","timestamp":1731882693650,"parserVersion":"staging","error":"Reparsed bc8f5359-c997-47c2-9999-8a4df2d1836d Capabilities is null"} -{"id":"db8f5359-c997-47c2-9999-8a4df2d1836d","timestamp":1731882693642,"parserVersion":"staging","error":"Reparsed db8f5359-c997-47c2-9999-8a4df2d1836d Capabilities is null"} +{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c907","timestamp":1731885438148,"parserVersion":"staging","error":"Reparsed 0dfbcad1-41be-4dc1-bcea-8407f6a2c907 Capabilities is null","processingTimeMs":3} +{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c908","timestamp":1731885438145,"parserVersion":"staging","error":"Something weird, inputs list >= rat List","processingTimeMs":13} +{"id":"7dc8d574-7a05-4ee4-b370-059ddd38d144","timestamp":1731885438115,"parserVersion":"staging","error":"Extension count cannot be 0. This procedure is only invoked if there is at least one extension addition being encoded.","processingTimeMs":152} +{"id":"bc8f5359-c997-47c2-836d-8a4df2d1836d","timestamp":1731885438132,"parserVersion":"staging","error":"Reparsed bc8f5359-c997-47c2-836d-8a4df2d1836d Capabilities is null","processingTimeMs":2} +{"id":"bc8f5359-c997-47c2-836d-8a4df2d1c417","timestamp":1731885438130,"parserVersion":"staging","error":"Reparsed bc8f5359-c997-47c2-836d-8a4df2d1c417 Capabilities is null","processingTimeMs":11} +{"id":"bc8f5359-c997-47c2-9999-8a4df2d1836d","timestamp":1731885438119,"parserVersion":"staging","error":"Reparsed bc8f5359-c997-47c2-9999-8a4df2d1836d Capabilities is null","processingTimeMs":2} +{"id":"db8f5359-c997-47c2-9999-8a4df2d1836d","timestamp":1731885438117,"parserVersion":"staging","error":"Reparsed db8f5359-c997-47c2-9999-8a4df2d1836d Capabilities is null","processingTimeMs":2} diff --git a/src/test/resources/server/oracleForReparse/forceCompress/reparsing.json b/src/test/resources/server/oracleForReparse/forceCompress/reparsing.json index 1e6f2c3b..56153a09 100644 --- a/src/test/resources/server/oracleForReparse/forceCompress/reparsing.json +++ b/src/test/resources/server/oracleForReparse/forceCompress/reparsing.json @@ -1,5 +1,5 @@ -{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c907","timestamp":1730748181215,"parserVersion":"staging"} -{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c908","timestamp":1730748181197,"parserVersion":"staging"} -{"id":"bc8f5359-c997-47c2-836d-8a4df2d1c417","timestamp":1730748180994,"parserVersion":"staging"} -{"id":"c3e783c9-54bc-42ee-970c-d871117399b2","timestamp":1730748181144,"parserVersion":"staging"} -{"id":"cd7eb9db-196d-46a5-938d-68061a100cdb","timestamp":1730748181076,"parserVersion":"staging"} +{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c907","timestamp":1731885440043,"parserVersion":"staging","processingTimeMs":20} +{"id":"0dfbcad1-41be-4dc1-bcea-8407f6a2c908","timestamp":1731885440022,"parserVersion":"staging","processingTimeMs":20} +{"id":"bc8f5359-c997-47c2-836d-8a4df2d1c417","timestamp":1731885439804,"parserVersion":"staging","processingTimeMs":27} +{"id":"c3e783c9-54bc-42ee-970c-d871117399b2","timestamp":1731885439982,"parserVersion":"staging","processingTimeMs":70} +{"id":"cd7eb9db-196d-46a5-938d-68061a100cdb","timestamp":1731885439917,"parserVersion":"staging","processingTimeMs":125} From f33830ae3abce8d4eae600e766cc7d21e8935ade Mon Sep 17 00:00:00 2001 From: handymenny Date: Mon, 18 Nov 2024 15:21:42 +0100 Subject: [PATCH 3/3] tests: raise similardirs tollerance to 5% --- .../it/smartphonecombo/uecapabilityparser/UtilityForTests.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/it/smartphonecombo/uecapabilityparser/UtilityForTests.kt b/src/test/java/it/smartphonecombo/uecapabilityparser/UtilityForTests.kt index c3e33afe..8852c9f3 100644 --- a/src/test/java/it/smartphonecombo/uecapabilityparser/UtilityForTests.kt +++ b/src/test/java/it/smartphonecombo/uecapabilityparser/UtilityForTests.kt @@ -29,7 +29,7 @@ object UtilityForTests { .collect(Collectors.toList()) } - /** Check if the given directories have the same tree and if files have similar sizes (± 3%) */ + /** Check if the given directories have the same tree and if files have similar sizes (± 5%) */ internal fun dirsSimilar(expected: String, actual: String): Boolean { val pathA = Path(expected) val pathB = Path(actual) @@ -57,7 +57,7 @@ object UtilityForTests { } else { val aSize = fileA.readBytes().size val bSize = fileB.readBytes().size - abs(aSize - bSize) < 3 * aSize / 100 + abs(aSize - bSize) < 5 * aSize / 100 } } return result