Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rename install API to index for file in knowledge manager #2634

Merged
merged 19 commits into from
Aug 2, 2024
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ import androidx.room.Room
import ca.uhn.fhir.context.FhirContext
import ca.uhn.fhir.parser.IParser
import com.google.android.fhir.knowledge.db.KnowledgeDatabase
import com.google.android.fhir.knowledge.db.entities.ImplementationGuideEntity
import com.google.android.fhir.knowledge.db.entities.ResourceMetadataEntity
import com.google.android.fhir.knowledge.db.entities.toEntity
import com.google.android.fhir.knowledge.files.NpmFileManager
import com.google.android.fhir.knowledge.npm.NpmPackageDownloader
import com.google.android.fhir.knowledge.npm.OkHttpNpmPackageDownloader
Expand All @@ -49,6 +49,9 @@ import timber.log.Timber
* - database: indexing knowledge artifacts stored in the local file system,
* - file manager: managing files containing the knowledge artifacts, and
* - NPM downloader: downloading from an NPM package server the knowledge artifacts.
jingtang10 marked this conversation as resolved.
Show resolved Hide resolved
*
* Knowledge artifacts are scoped by the application. Multiple applications using the knowledge
* manager will not share the same sets of knowledge artifacts.
*/
class KnowledgeManager
jingtang10 marked this conversation as resolved.
Show resolved Hide resolved
internal constructor(
Expand Down Expand Up @@ -81,7 +84,7 @@ internal constructor(
try {
val localFhirNpmPackageMetadata =
npmFileManager.getLocalFhirNpmPackageMetadata(it.name, it.version)
install(it, localFhirNpmPackageMetadata.rootDirectory)
import(it, localFhirNpmPackageMetadata.rootDirectory)
install(*localFhirNpmPackageMetadata.dependencies.toTypedArray())
} catch (e: Exception) {
Timber.w("Unable to install package ${it.name} ${it.version}")
Expand All @@ -93,9 +96,18 @@ internal constructor(
* Checks if the [fhirNpmPackage] is present in DB. If necessary, populates the database with the
* metadata of FHIR Resource from the provided [rootDirectory].
jingtang10 marked this conversation as resolved.
Show resolved Hide resolved
*/
suspend fun install(fhirNpmPackage: FhirNpmPackage, rootDirectory: File) {
suspend fun import(fhirNpmPackage: FhirNpmPackage, rootDirectory: File) {
// TODO(ktarasenko) copy files to the safe space?
val igId = knowledgeDao.insert(fhirNpmPackage.toEntity(rootDirectory))
val igId =
knowledgeDao.insert(
ImplementationGuideEntity(
0L,
fhirNpmPackage.canonical ?: "",
fhirNpmPackage.name,
fhirNpmPackage.version,
rootDirectory,
),
)
rootDirectory.listFiles()?.sorted()?.forEach { file ->
try {
val resource = jsonParser.parseResource(FileInputStream(file))
jingtang10 marked this conversation as resolved.
Show resolved Hide resolved
Expand All @@ -116,9 +128,43 @@ internal constructor(
}
}

/** Imports the Knowledge Artifact from the provided [file] to the default dependency. */
suspend fun install(file: File) {
importFile(null, file)
/**
* Indexes a knowledge artifact as a JSON object in the provided [file].
*
* This creates a record of the knowledge artifact's metadata and the file's location. When the
* knowledge artifact is requested, knowledge manager will load the content of the file,
* deserialize it and return the resulting FHIR resource.
*
* This operation does not make a copy of the knowledge artifact, nor does it checksum the content
* of the file. Therefore, it cannot be guaranteed that subsequent retrievals of the knowledge
* artifact will produce the same result. Applications using this function must be aware of the
* risk of the content of the file being modified or corrupt, potentially resulting in incorrect
* or inaccurate result of decision support or measure evaluation.
*
* Use this API for knowledge artifacts in immutable files (e.g. in the app's `assets` folder).
*
* resources already indexed and resources already imported by ig?
jingtang10 marked this conversation as resolved.
Show resolved Hide resolved
*/
suspend fun index(file: File) {
val resource =
withContext(Dispatchers.IO) {
try {
FileInputStream(file).use(jsonParser::parseResource)
} catch (exception: Exception) {
Timber.d(exception, "Unable to import file: $file. Parsing to FhirResource failed.")
}
}
when (resource) {
is Resource -> {
jingtang10 marked this conversation as resolved.
Show resolved Hide resolved
val newId = indexResourceFile(null, resource, file)
resource.setId(IdType(resource.resourceType.name, newId))

// Overrides the Id in the file
FileOutputStream(file).use {
it.write(jsonParser.encodeResourceToString(resource).toByteArray())
}
}
}
}

/** Loads resources from IGs listed in dependencies. */
Expand All @@ -141,7 +187,7 @@ internal constructor(
name != null -> knowledgeDao.getResourcesWithName(resType, name)
else -> knowledgeDao.getResources(resType)
}
return resourceEntities.map { loadResource(it) }
return resourceEntities.map { parseResourceFile(it.resourceFile) }
}

/** Deletes Implementation Guide, cleans up files. */
Expand All @@ -155,43 +201,6 @@ internal constructor(
}
}

private suspend fun importFile(igId: Long?, file: File) {
val resource =
withContext(Dispatchers.IO) {
try {
FileInputStream(file).use(jsonParser::parseResource)
} catch (exception: Exception) {
Timber.d(exception, "Unable to import file: $file. Parsing to FhirResource failed.")
}
}
when (resource) {
is Resource -> {
val newId = indexResourceFile(igId, resource, file)
resource.setId(IdType(resource.resourceType.name, newId))

// Overrides the Id in the file
FileOutputStream(file).use {
it.write(jsonParser.encodeResourceToString(resource).toByteArray())
}
}
}
}

private suspend fun indexResourceFile(igId: Long?, resource: Resource, file: File): Long {
val metadataResource = resource as? MetadataResource
val res =
ResourceMetadataEntity(
0L,
resource.resourceType,
metadataResource?.url,
metadataResource?.name,
metadataResource?.version,
file,
)

return knowledgeDao.insertResource(igId, res)
}

/**
* Loads and initializes a worker context with the specified npm packages.
*
Expand Down Expand Up @@ -220,8 +229,28 @@ internal constructor(
}
}

private fun loadResource(resourceEntity: ResourceMetadataEntity): IBaseResource {
return jsonParser.parseResource(FileInputStream(resourceEntity.resourceFile))
/** */
private suspend fun indexResourceFile(igId: Long?, resource: Resource, file: File): Long {
val metadataResource = resource as? MetadataResource
jingtang10 marked this conversation as resolved.
Show resolved Hide resolved
jingtang10 marked this conversation as resolved.
Show resolved Hide resolved
val res =
ResourceMetadataEntity(
0L,
resource.resourceType,
metadataResource?.url,
metadataResource?.name,
metadataResource?.version,
file,
)

return knowledgeDao.insertResource(igId, res)
}

/**
* Parses the content of a file containing a FHIR resource in JSON and returns the parsed
* resource.
*/
private fun parseResourceFile(file: File): IBaseResource {
return jsonParser.parseResource(FileInputStream(file))
}

companion object {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2022-2023 Google LLC
* Copyright 2022-2024 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -19,7 +19,6 @@ package com.google.android.fhir.knowledge.db.entities
import androidx.room.Entity
import androidx.room.Index
import androidx.room.PrimaryKey
import com.google.android.fhir.knowledge.FhirNpmPackage
import java.io.File

/**
Expand All @@ -45,7 +44,3 @@ internal data class ImplementationGuideEntity(
/** Directory where the Implementation Guide files are stored */
val rootDirectory: File,
)

internal fun FhirNpmPackage.toEntity(rootFolder: File): ImplementationGuideEntity {
return ImplementationGuideEntity(0L, canonical ?: "", name, version, rootFolder)
}
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ internal class KnowledgeManagerTest {

@Test
fun `importing IG creates entries in DB`() = runTest {
knowledgeManager.install(fhirNpmPackage, dataFolder)
knowledgeManager.import(fhirNpmPackage, dataFolder)
val implementationGuideId =
knowledgeDb.knowledgeDao().getImplementationGuide("anc-cds", "0.3.0")!!.implementationGuideId

Expand All @@ -83,7 +83,7 @@ internal class KnowledgeManagerTest {
val igRoot = File(dataFolder.parentFile, "anc-cds.copy")
igRoot.deleteOnExit()
dataFolder.copyRecursively(igRoot)
knowledgeManager.install(fhirNpmPackage, igRoot)
knowledgeManager.import(fhirNpmPackage, igRoot)

knowledgeManager.delete(fhirNpmPackage)

Expand All @@ -93,7 +93,7 @@ internal class KnowledgeManagerTest {

@Test
fun `imported entries are readable`() = runTest {
knowledgeManager.install(fhirNpmPackage, dataFolder)
knowledgeManager.import(fhirNpmPackage, dataFolder)

assertThat(knowledgeManager.loadResources(resourceType = "Library", name = "WHOCommon"))
.isNotNull()
Expand Down Expand Up @@ -128,8 +128,8 @@ internal class KnowledgeManagerTest {
version = "A.1.0.1"
}

knowledgeManager.install(writeToFile(libraryAOld))
knowledgeManager.install(writeToFile(libraryANew))
knowledgeManager.index(writeToFile(libraryAOld))
knowledgeManager.index(writeToFile(libraryANew))

val resources = knowledgeDb.knowledgeDao().getResources()
assertThat(resources).hasSize(2)
Expand Down Expand Up @@ -182,8 +182,8 @@ internal class KnowledgeManagerTest {
url = commonUrl
}

knowledgeManager.install(writeToFile(libraryWithSameUrl))
knowledgeManager.install(writeToFile(planDefinitionWithSameUrl))
knowledgeManager.index(writeToFile(libraryWithSameUrl))
knowledgeManager.index(writeToFile(planDefinitionWithSameUrl))

val resources = knowledgeDb.knowledgeDao().getResources()
assertThat(resources).hasSize(2)
Expand Down Expand Up @@ -215,8 +215,8 @@ internal class KnowledgeManagerTest {
version = "0"
}

knowledgeManager.install(writeToFile(libraryWithSameUrl))
knowledgeManager.install(writeToFile(planDefinitionWithSameUrl))
knowledgeManager.index(writeToFile(libraryWithSameUrl))
knowledgeManager.index(writeToFile(planDefinitionWithSameUrl))

val resources = knowledgeDb.knowledgeDao().getResources()
assertThat(resources).hasSize(2)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ class F_CqlEvaluatorBenchmark {
for (entry in patientImmunizationHistory.entry) {
fhirEngine.create(entry.resource)
}
knowledgeManager.install(
knowledgeManager.index(
File(context.filesDir, lib.name).apply {
writeText(jsonParser.encodeResourceToString(lib))
},
Expand Down
1 change: 0 additions & 1 deletion workflow/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,6 @@ dependencies {
testImplementation(libs.androidx.test.core)
testImplementation(libs.junit)
testImplementation(libs.truth)
testImplementation(project(mapOf("path" to ":knowledge")))
testImplementation(project(":workflow-testing"))

constraints {
Expand Down
Loading