From 6d1e9b82f067269a29fbc8e0a62ac5ca26f4f85b Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 18 Dec 2024 13:51:43 +0100 Subject: [PATCH 01/34] TODOs --- kubernetes/loculus/templates/_common-metadata.tpl | 1 + website/src/components/Submission/DataUploadForm.tsx | 2 ++ website/src/pages/api-documentation/index.astro | 1 + 3 files changed, 4 insertions(+) diff --git a/kubernetes/loculus/templates/_common-metadata.tpl b/kubernetes/loculus/templates/_common-metadata.tpl index 44ae6d38f0..89993322e5 100644 --- a/kubernetes/loculus/templates/_common-metadata.tpl +++ b/kubernetes/loculus/templates/_common-metadata.tpl @@ -75,6 +75,7 @@ fields: autocomplete: true displayName: Date released (exact) columnWidth: 100 + # TODO change to be done here - only include these fields if a flag is set - name: dataUseTerms type: string generateIndex: true diff --git a/website/src/components/Submission/DataUploadForm.tsx b/website/src/components/Submission/DataUploadForm.tsx index 4ff60fe469..bb6adab0d5 100644 --- a/website/src/components/Submission/DataUploadForm.tsx +++ b/website/src/components/Submission/DataUploadForm.tsx @@ -327,6 +327,7 @@ const InnerDataUploadForm = ({ + {/* TODO change to be done here */} {action !== 'revise' && (
+ {/* TODO change to be done here */} {dataUseTermsType === restrictedDataUseTermsOption && (

Your data will be available on Pathoplexus, under the restricted use terms until{' '} diff --git a/website/src/pages/api-documentation/index.astro b/website/src/pages/api-documentation/index.astro index b52f2ee0a4..2a4beca8ce 100644 --- a/website/src/pages/api-documentation/index.astro +++ b/website/src/pages/api-documentation/index.astro @@ -48,6 +48,7 @@ const BUTTON_CLASS = WARNING: Swagger incorrectly displays NDJSON examples in JSON format. For endpoints that require NDJSON as input you must convert the JSON examples to NDJSON (e.g. by removing new lines) prior to testing.

+ {/* TODO changes to be done here */}

By using our API you agree to our Date: Wed, 18 Dec 2024 16:03:17 +0100 Subject: [PATCH 02/34] Add backend TODOs --- .../org/loculus/backend/controller/SubmissionController.kt | 3 +++ .../main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt | 1 + 2 files changed, 4 insertions(+) diff --git a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt index 88ba74b6ad..461844ccbe 100644 --- a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt +++ b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt @@ -78,6 +78,7 @@ open class SubmissionController( private val requestIdContext: RequestIdContext, ) { + /* TODO change to be made here - similar to restrictedUntil the parameter will conditionally be mandatory */ @Operation(description = SUBMIT_DESCRIPTION) @ApiResponse(responseCode = "200", description = SUBMIT_RESPONSE_DESCRIPTION) @PostMapping("/submit", consumes = ["multipart/form-data"]) @@ -96,6 +97,8 @@ open class SubmissionController( " Format: YYYY-MM-DD", ) @RequestParam restrictedUntil: String?, ): List { + // in here, just default to open DataUseTerms and then don't worry anymore + // throw an error if the parameter is given val params = SubmissionParams.OriginalSubmissionParams( organism, authenticatedUser, diff --git a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt index 3cccc63b84..3ef235ee85 100644 --- a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt +++ b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt @@ -126,6 +126,7 @@ open class ReleasedDataModel( ("releasedAtTimestamp" to LongNode(rawProcessedData.releasedAtTimestamp.toTimestamp())), ("releasedDate" to TextNode(rawProcessedData.releasedAtTimestamp.toUtcDateString())), ("versionStatus" to TextNode(versionStatus.name)), + // TODO changes made here - don't include these fields ("dataUseTerms" to TextNode(currentDataUseTerms.type.name)), ("dataUseTermsRestrictedUntil" to restrictedDataUseTermsUntil), ("pipelineVersion" to LongNode(rawProcessedData.pipelineVersion)), From 593d33b9cba2654ff578424f54837dce4a9eb270 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 18 Dec 2024 16:04:48 +0100 Subject: [PATCH 03/34] Add more TODO --- backend/src/main/kotlin/org/loculus/backend/config/Config.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/main/kotlin/org/loculus/backend/config/Config.kt b/backend/src/main/kotlin/org/loculus/backend/config/Config.kt index a56907619b..13442efb1d 100644 --- a/backend/src/main/kotlin/org/loculus/backend/config/Config.kt +++ b/backend/src/main/kotlin/org/loculus/backend/config/Config.kt @@ -7,6 +7,7 @@ import org.loculus.backend.api.Organism data class BackendConfig( val organisms: Map, val accessionPrefix: String, + // TODO changes to be made here: add the flag here. we can put the Urls in a "date use terms" section in the config val dataUseTermsUrls: DataUseTermsUrls?, ) { fun getInstanceConfig(organism: Organism) = organisms[organism.name] ?: throw IllegalArgumentException( From 90b4b5831af0f6ec31c8e7cb3c0ddae255b863d3 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 29 Jan 2025 14:36:04 +0100 Subject: [PATCH 04/34] website disable --- .../components/Submission/DataUploadForm.tsx | 147 +++++++++--------- .../components/Submission/RevisionForm.tsx | 3 + .../components/Submission/SubmissionForm.tsx | 3 + website/src/config.ts | 6 + .../submission/[groupId]/revise.astro | 3 +- .../submission/[groupId]/submit.astro | 3 +- website/src/types/config.ts | 1 + 7 files changed, 91 insertions(+), 75 deletions(-) diff --git a/website/src/components/Submission/DataUploadForm.tsx b/website/src/components/Submission/DataUploadForm.tsx index bb6adab0d5..d0789fe1d3 100644 --- a/website/src/components/Submission/DataUploadForm.tsx +++ b/website/src/components/Submission/DataUploadForm.tsx @@ -42,6 +42,7 @@ type DataUploadFormProps = { onSuccess: () => void; onError: (message: string) => void; submissionDataTypes: SubmissionDataTypes; + dataUseTermsEnabled: boolean; }; const logger = getClientLogger('DataUploadForm'); @@ -128,6 +129,7 @@ const InnerDataUploadForm = ({ referenceGenomeSequenceNames, metadataTemplateFields, submissionDataTypes, + dataUseTermsEnabled, }: DataUploadFormProps) => { const [metadataFile, setMetadataFile] = useState(undefined); // The columnMapping can be null; if null -> don't apply mapping. @@ -327,88 +329,87 @@ const InnerDataUploadForm = ({

- {/* TODO change to be done here */} - {action !== 'revise' && ( + {action === 'submit' && dataUseTermsEnabled && ( )} -
-
-

Acknowledgement

-

Acknowledge submission terms

-
-
-
- {/* TODO change to be done here */} - {dataUseTermsType === restrictedDataUseTermsOption && ( -

- Your data will be available on Pathoplexus, under the restricted use terms until{' '} - {restrictedUntil.toFormat('yyyy-MM-dd')}. After the restricted period your data will - additionally be made publicly available through the{' '} - - INSDC - {' '} - databases (ENA, DDBJ, NCBI). -

- )} - {dataUseTermsType === openDataUseTermsOption && ( -

- Your data will be available on Pathoplexus under the open use terms. It will - additionally be made publicly available through the{' '} - - INSDC - {' '} - databases (ENA, DDBJ, NCBI). -

- )} -
- -
-
- + {dataUseTermsEnabled && ( +
+
+

Acknowledgement

+

Acknowledge submission terms

+
+
+
+ {dataUseTermsType === restrictedDataUseTermsOption && ( +

+ Your data will be available on Pathoplexus, under the restricted use terms until{' '} + {restrictedUntil.toFormat('yyyy-MM-dd')}. After the restricted period your data + will additionally be made publicly available through the{' '} + + INSDC + {' '} + databases (ENA, DDBJ, NCBI). +

+ )} + {dataUseTermsType === openDataUseTermsOption && ( +

+ Your data will be available on Pathoplexus under the open use terms. It will + additionally be made publicly available through the{' '} + + INSDC + {' '} + databases (ENA, DDBJ, NCBI). +

+ )} +
+ +
+
+ +
-
- -
+ )} +
From b52bb31ae11f70aa8475c997753002164ed9e3ba Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 29 Jan 2025 17:28:19 +0100 Subject: [PATCH 10/34] Backend changes WIP --- .../org/loculus/backend/config/Config.kt | 2 +- .../controller/SubmissionController.kt | 24 +++++++++++++------ .../SubmissionControllerDescriptions.kt | 4 ++++ .../backend/config/BackendSpringConfigTest.kt | 1 + .../GenerateAccessionFromNumberServiceTest.kt | 7 +++++- .../EmptyProcessedDataProviderTest.kt | 1 + .../loculus/templates/_common-metadata.tpl | 1 + 7 files changed, 31 insertions(+), 9 deletions(-) diff --git a/backend/src/main/kotlin/org/loculus/backend/config/Config.kt b/backend/src/main/kotlin/org/loculus/backend/config/Config.kt index 13442efb1d..e6b123c248 100644 --- a/backend/src/main/kotlin/org/loculus/backend/config/Config.kt +++ b/backend/src/main/kotlin/org/loculus/backend/config/Config.kt @@ -7,7 +7,7 @@ import org.loculus.backend.api.Organism data class BackendConfig( val organisms: Map, val accessionPrefix: String, - // TODO changes to be made here: add the flag here. we can put the Urls in a "date use terms" section in the config + val dataUseTermsEnabled: Boolean, val dataUseTermsUrls: DataUseTermsUrls?, ) { fun getInstanceConfig(organism: Organism) = organisms[organism.name] ?: throw IllegalArgumentException( diff --git a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt index 461844ccbe..ac6a6eb903 100644 --- a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt +++ b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt @@ -33,6 +33,7 @@ import org.loculus.backend.api.SubmittedProcessedData import org.loculus.backend.api.UnprocessedData import org.loculus.backend.auth.AuthenticatedUser import org.loculus.backend.auth.HiddenParam +import org.loculus.backend.config.BackendConfig import org.loculus.backend.controller.LoculusCustomHeaders.X_TOTAL_RECORDS import org.loculus.backend.log.REQUEST_ID_MDC_KEY import org.loculus.backend.log.RequestIdContext @@ -76,11 +77,11 @@ open class SubmissionController( private val submissionDatabaseService: SubmissionDatabaseService, private val iteratorStreamer: IteratorStreamer, private val requestIdContext: RequestIdContext, + private val backendConfig: BackendConfig, ) { - - /* TODO change to be made here - similar to restrictedUntil the parameter will conditionally be mandatory */ @Operation(description = SUBMIT_DESCRIPTION) @ApiResponse(responseCode = "200", description = SUBMIT_RESPONSE_DESCRIPTION) + @ApiResponse(responseCode = "400", description = SUBMIT_ERROR_RESPONSE) @PostMapping("/submit", consumes = ["multipart/form-data"]) fun submit( @PathVariable @Valid organism: Organism, @@ -88,8 +89,10 @@ open class SubmissionController( @Parameter(description = GROUP_ID_DESCRIPTION) @RequestParam groupId: Int, @Parameter(description = METADATA_FILE_DESCRIPTION) @RequestParam metadataFile: MultipartFile, @Parameter(description = SEQUENCE_FILE_DESCRIPTION) @RequestParam sequenceFile: MultipartFile?, - @Parameter(description = "Data Use terms under which data is released.") @RequestParam dataUseTermsType: - DataUseTermsType, + @Parameter( + description = + "Data Use terms under which data is released. Mandatory when data use terms are enabled for this Instance.", + ) @RequestParam dataUseTermsType: DataUseTermsType?, @Parameter( description = "Mandatory when data use terms are set to 'RESTRICTED'." + @@ -97,15 +100,22 @@ open class SubmissionController( " Format: YYYY-MM-DD", ) @RequestParam restrictedUntil: String?, ): List { - // in here, just default to open DataUseTerms and then don't worry anymore - // throw an error if the parameter is given + var dataUseTermsKind = DataUseTermsType.OPEN + if (backendConfig.dataUseTermsEnabled) { + if (dataUseTermsType == null) { + throw BadRequestException("the 'dataUseTermsType' needs to be provided.") + } else { + dataUseTermsKind = dataUseTermsType + } + } + val params = SubmissionParams.OriginalSubmissionParams( organism, authenticatedUser, metadataFile, sequenceFile, groupId, - DataUseTerms.fromParameters(dataUseTermsType, restrictedUntil), + DataUseTerms.fromParameters(dataUseTermsKind, restrictedUntil), ) return submitModel.processSubmissions(UUID.randomUUID().toString(), params) } diff --git a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionControllerDescriptions.kt b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionControllerDescriptions.kt index 66a7711bb1..babbe5c2d8 100644 --- a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionControllerDescriptions.kt +++ b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionControllerDescriptions.kt @@ -8,6 +8,10 @@ The accession is the (globally unique) id that the system assigned to the sequen You can use this response to associate the user provided submissionId with the system assigned accession. """ +const val SUBMIT_ERROR_RESPONSE = """ +The data use terms type have not been provided, even though they are enabled for this Loculus instance. +""" + const val METADATA_FILE_DESCRIPTION = """ A TSV (tab separated values) file containing the metadata of the submitted sequence entries. The file may be compressed with zstd, xz, zip, gzip, lzma, bzip2 (with common extensions). diff --git a/backend/src/test/kotlin/org/loculus/backend/config/BackendSpringConfigTest.kt b/backend/src/test/kotlin/org/loculus/backend/config/BackendSpringConfigTest.kt index 8979cbb94b..bcbf292420 100644 --- a/backend/src/test/kotlin/org/loculus/backend/config/BackendSpringConfigTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/config/BackendSpringConfigTest.kt @@ -70,5 +70,6 @@ fun backendConfig(metadataList: List, earliestReleaseDate: EarliestRel ), ), accessionPrefix = "FOO_", + dataUseTermsEnabled = true, dataUseTermsUrls = null, ) diff --git a/backend/src/test/kotlin/org/loculus/backend/service/GenerateAccessionFromNumberServiceTest.kt b/backend/src/test/kotlin/org/loculus/backend/service/GenerateAccessionFromNumberServiceTest.kt index ad3b53fa80..a9a05704df 100644 --- a/backend/src/test/kotlin/org/loculus/backend/service/GenerateAccessionFromNumberServiceTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/service/GenerateAccessionFromNumberServiceTest.kt @@ -11,7 +11,12 @@ const val PREFIX = "LOC_" class GenerateAccessionFromNumberServiceTest { private val accessionFromNumberService = GenerateAccessionFromNumberService( - BackendConfig(accessionPrefix = PREFIX, organisms = emptyMap(), dataUseTermsUrls = null), + BackendConfig( + accessionPrefix = PREFIX, + organisms = emptyMap(), + dataUseTermsEnabled = true, + dataUseTermsUrls = null, + ), ) @Test diff --git a/backend/src/test/kotlin/org/loculus/backend/service/submission/EmptyProcessedDataProviderTest.kt b/backend/src/test/kotlin/org/loculus/backend/service/submission/EmptyProcessedDataProviderTest.kt index 96a0dea239..2849d4682c 100644 --- a/backend/src/test/kotlin/org/loculus/backend/service/submission/EmptyProcessedDataProviderTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/service/submission/EmptyProcessedDataProviderTest.kt @@ -46,6 +46,7 @@ class EmptyProcessedDataProviderTest { ), ), ), + dataUseTermsEnabled = true, dataUseTermsUrls = null, ), ) diff --git a/kubernetes/loculus/templates/_common-metadata.tpl b/kubernetes/loculus/templates/_common-metadata.tpl index bb9e456b76..81c0527b3c 100644 --- a/kubernetes/loculus/templates/_common-metadata.tpl +++ b/kubernetes/loculus/templates/_common-metadata.tpl @@ -291,6 +291,7 @@ fields: {{- define "loculus.generateBackendConfig" }} accessionPrefix: {{ quote $.Values.accessionPrefix }} name: {{ quote $.Values.name }} +dataUseTermsEnabled: {{$.Values.dataUseTermsEnabled }} dataUseTermsUrls: {{$.Values.dataUseTermsUrls | toYaml | nindent 2}} organisms: From 585f09cdab71d6f82950124a9b7688e65fd238b0 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 31 Jan 2025 16:02:34 +0100 Subject: [PATCH 11/34] fix tests --- .../kotlin/org/loculus/backend/config/BackendSpringConfig.kt | 5 ++++- .../backend/controller/submission/SubmitEndpointTest.kt | 2 ++ backend/src/test/resources/backend_config.json | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt b/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt index 119665129b..6439ed30e4 100644 --- a/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt +++ b/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt @@ -1,5 +1,6 @@ package org.loculus.backend.config +import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import io.swagger.v3.oas.models.headers.Header @@ -146,7 +147,9 @@ internal fun validateEarliestReleaseDateFields(config: BackendConfig): List(File(configPath)) + val config = objectMapper + .enable(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES) + .readValue(File(configPath)) logger.info { "Loaded backend config from $configPath" } logger.info { "Config: $config" } val validationErrors = validateEarliestReleaseDateFields(config) diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt index 57cbddab60..d3875a9ba1 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt @@ -184,6 +184,8 @@ class SubmitEndpointTest( organism: Organism, dataUseTerm: DataUseTerms, ) { + println("Data use terms enabled: ") + println(backendConfig.dataUseTermsEnabled) submissionControllerClient.submit( metadataFile, sequencesFile, diff --git a/backend/src/test/resources/backend_config.json b/backend/src/test/resources/backend_config.json index 3729e7a6bf..e63e12667f 100644 --- a/backend/src/test/resources/backend_config.json +++ b/backend/src/test/resources/backend_config.json @@ -225,5 +225,6 @@ ] } } - } + }, + "dataUseTermsEnabled": true } From dd1b458ab35b2b6c031f4fbd6862dba8c1884c01 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 31 Jan 2025 16:31:18 +0100 Subject: [PATCH 12/34] progress --- .../backend/model/ReleasedDataModel.kt | 61 +++++++++++-------- .../backend_config_single_segment.json | 1 + 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt index 3ef235ee85..b681cef4d9 100644 --- a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt +++ b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.node.TextNode import mu.KotlinLogging import org.loculus.backend.api.DataUseTerms import org.loculus.backend.api.GeneticSequence +import org.loculus.backend.api.MetadataMap import org.loculus.backend.api.Organism import org.loculus.backend.api.ProcessedData import org.loculus.backend.api.VersionStatus @@ -94,6 +95,9 @@ open class ReleasedDataModel( return "\"$lastUpdateTime\"" // ETag must be enclosed in double quotes } + private fun conditionalMetadata(condition: Boolean, values: MetadataMap): MetadataMap = + if (condition) values else emptyMap() + private fun computeAdditionalMetadataFields( rawProcessedData: RawProcessedData, latestVersions: Map, @@ -111,6 +115,13 @@ open class ReleasedDataModel( val earliestReleaseDate = earliestReleaseDateFinder?.calculateEarliestReleaseDate(rawProcessedData) + val dataUseTermsUrl: String? = backendConfig.dataUseTermsUrls?.let { urls -> + when (currentDataUseTerms) { + DataUseTerms.Open -> urls.open + is DataUseTerms.Restricted -> urls.restricted + } + } + var metadata = rawProcessedData.processedData.metadata + mapOf( ("accession" to TextNode(rawProcessedData.accession)), @@ -126,32 +137,34 @@ open class ReleasedDataModel( ("releasedAtTimestamp" to LongNode(rawProcessedData.releasedAtTimestamp.toTimestamp())), ("releasedDate" to TextNode(rawProcessedData.releasedAtTimestamp.toUtcDateString())), ("versionStatus" to TextNode(versionStatus.name)), - // TODO changes made here - don't include these fields - ("dataUseTerms" to TextNode(currentDataUseTerms.type.name)), - ("dataUseTermsRestrictedUntil" to restrictedDataUseTermsUntil), ("pipelineVersion" to LongNode(rawProcessedData.pipelineVersion)), ) + - if (rawProcessedData.isRevocation) { - mapOf("versionComment" to TextNode(rawProcessedData.versionComment)) - } else { - emptyMap() - }.let { - when (backendConfig.dataUseTermsUrls) { - null -> it - else -> { - val url = when (currentDataUseTerms) { - DataUseTerms.Open -> backendConfig.dataUseTermsUrls.open - is DataUseTerms.Restricted -> backendConfig.dataUseTermsUrls.restricted - } - it + ("dataUseTermsUrl" to TextNode(url)) - } - } - } + - if (earliestReleaseDate != null) { - mapOf("earliestReleaseDate" to TextNode(earliestReleaseDate.toUtcDateString())) - } else { - emptyMap() - } + // TODO add a test for this change + conditionalMetadata( + backendConfig.dataUseTermsEnabled, + mapOf( + "dataUseTerms" to TextNode(currentDataUseTerms.type.name), + "dataUseTermsRestrictedUntil" to restrictedDataUseTermsUntil, + ), + ) + + conditionalMetadata( + rawProcessedData.isRevocation, + mapOf( + "versionComment" to TextNode(rawProcessedData.versionComment), + ), + ) + + conditionalMetadata( + earliestReleaseDate != null, + mapOf( + "earliestReleaseDate" to TextNode(earliestReleaseDate!!.toUtcDateString()), + ), + ) + + conditionalMetadata( + dataUseTermsUrl != null, + mapOf( + "dataUseTermsUrl" to TextNode(dataUseTermsUrl!!), + ), + ) return ProcessedData( metadata = metadata, diff --git a/backend/src/test/resources/backend_config_single_segment.json b/backend/src/test/resources/backend_config_single_segment.json index e61b1c5c72..9f532a9b2c 100644 --- a/backend/src/test/resources/backend_config_single_segment.json +++ b/backend/src/test/resources/backend_config_single_segment.json @@ -1,5 +1,6 @@ { "accessionPrefix": "LOC_", + "dataUseTermsEnabled": true, "organisms": { "dummyOrganism": { "referenceGenomes": { From ca884d9ea0b17c8d3a99edaf6204117d59e8bdf0 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Fri, 31 Jan 2025 16:59:33 +0100 Subject: [PATCH 13/34] fix tests --- .../backend/model/ReleasedDataModel.kt | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt index b681cef4d9..82d78e61bc 100644 --- a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt +++ b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt @@ -95,8 +95,8 @@ open class ReleasedDataModel( return "\"$lastUpdateTime\"" // ETag must be enclosed in double quotes } - private fun conditionalMetadata(condition: Boolean, values: MetadataMap): MetadataMap = - if (condition) values else emptyMap() + private fun conditionalMetadata(condition: Boolean, values: () -> MetadataMap): MetadataMap = + if (condition) values() else emptyMap() private fun computeAdditionalMetadataFields( rawProcessedData: RawProcessedData, @@ -142,28 +142,36 @@ open class ReleasedDataModel( // TODO add a test for this change conditionalMetadata( backendConfig.dataUseTermsEnabled, - mapOf( - "dataUseTerms" to TextNode(currentDataUseTerms.type.name), - "dataUseTermsRestrictedUntil" to restrictedDataUseTermsUntil, - ), + { + mapOf( + "dataUseTerms" to TextNode(currentDataUseTerms.type.name), + "dataUseTermsRestrictedUntil" to restrictedDataUseTermsUntil, + ) + }, ) + conditionalMetadata( rawProcessedData.isRevocation, - mapOf( - "versionComment" to TextNode(rawProcessedData.versionComment), - ), + { + mapOf( + "versionComment" to TextNode(rawProcessedData.versionComment), + ) + }, ) + conditionalMetadata( earliestReleaseDate != null, - mapOf( - "earliestReleaseDate" to TextNode(earliestReleaseDate!!.toUtcDateString()), - ), + { + mapOf( + "earliestReleaseDate" to TextNode(earliestReleaseDate!!.toUtcDateString()), + ) + }, ) + conditionalMetadata( dataUseTermsUrl != null, - mapOf( - "dataUseTermsUrl" to TextNode(dataUseTermsUrl!!), - ), + { + mapOf( + "dataUseTermsUrl" to TextNode(dataUseTermsUrl!!), + ) + }, ) return ProcessedData( From aaace05e05c87c439963346eb09a31e8d2d086df Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 3 Feb 2025 13:47:35 +0100 Subject: [PATCH 14/34] conditional field definitions --- kubernetes/loculus/templates/_common-metadata.tpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kubernetes/loculus/templates/_common-metadata.tpl b/kubernetes/loculus/templates/_common-metadata.tpl index 81c0527b3c..13064cac40 100644 --- a/kubernetes/loculus/templates/_common-metadata.tpl +++ b/kubernetes/loculus/templates/_common-metadata.tpl @@ -75,7 +75,7 @@ fields: autocomplete: true displayName: Date released (exact) columnWidth: 100 - # TODO change to be done here - only include these fields if a flag is set + {{- if $.Values.dataUseTermsEnabled }} - name: dataUseTerms type: string generateIndex: true @@ -90,6 +90,7 @@ fields: displayName: Data use terms restricted until hideOnSequenceDetailsPage: true header: Data use terms + {{- end}} {{- if $.Values.dataUseTermsUrls }} - name: dataUseTermsUrl displayName: Data use terms URL From 048ac4fffc7f7e738c521f82b783b1fd1dbf22ff Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 3 Feb 2025 14:10:39 +0100 Subject: [PATCH 15/34] Don't display bulk edit thing --- .../src/pages/[organism]/submission/[groupId]/released.astro | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/src/pages/[organism]/submission/[groupId]/released.astro b/website/src/pages/[organism]/submission/[groupId]/released.astro index 50617c827d..0f1d2edccd 100644 --- a/website/src/pages/[organism]/submission/[groupId]/released.astro +++ b/website/src/pages/[organism]/submission/[groupId]/released.astro @@ -2,7 +2,7 @@ import { cleanOrganism } from '../../../../components/Navigation/cleanOrganism'; import { SearchFullUI } from '../../../../components/SearchPage/SearchFullUI'; import SubmissionPageWrapper from '../../../../components/Submission/SubmissionPageWrapper.astro'; -import { getRuntimeConfig, getSchema } from '../../../../config'; +import { dataUseTermsAreEnabled, getRuntimeConfig, getSchema } from '../../../../config'; import { GROUP_ID_FIELD, VERSION_STATUS_FIELD } from '../../../../settings'; import { versionStatuses } from '../../../../types/lapis'; import { getAccessToken } from '../../../../utils/getAccessToken'; @@ -62,6 +62,6 @@ const { data, totalCount } = await performLapisSearchQueries( initialData={data} initialCount={totalCount} initialQueryDict={initialQueryDict} - showEditDataUseTermsControls + showEditDataUseTermsControls={dataUseTermsAreEnabled()} /> From 2247450ebee9878518fc1b61fcdbb61d555ba0a4 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 3 Feb 2025 16:02:53 +0100 Subject: [PATCH 16/34] Add config --- .../controller/EndpointTestExtension.kt | 15 +++ .../SingleSegmentedSubmitEndpointTest.kt | 9 +- ...ackend_config_data_use_terms_disabled.json | 105 ++++++++++++++++++ 3 files changed, 122 insertions(+), 7 deletions(-) create mode 100644 backend/src/test/resources/backend_config_data_use_terms_disabled.json diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt b/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt index 5a5833b741..222a0c34c7 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt @@ -10,6 +10,7 @@ import org.junit.platform.launcher.TestExecutionListener import org.junit.platform.launcher.TestPlan import org.loculus.backend.api.Address import org.loculus.backend.api.NewGroup +import org.loculus.backend.config.BackendSpringProperty import org.loculus.backend.controller.datauseterms.DataUseTermsControllerClient import org.loculus.backend.controller.groupmanagement.GroupManagementControllerClient import org.loculus.backend.controller.seqsetcitations.SeqSetCitationsControllerClient @@ -48,6 +49,20 @@ import org.testcontainers.containers.PostgreSQLContainer ) annotation class EndpointTest(@get:AliasFor(annotation = SpringBootTest::class) val properties: Array = []) +const val SINGLE_SEGMENTED_REFERENCE_GENOME = "src/test/resources/backend_config_single_segment.json" + +@EndpointTest( + properties = ["${BackendSpringProperty.BACKEND_CONFIG_PATH}=$SINGLE_SEGMENTED_REFERENCE_GENOME"], +) +annotation class SingleSegmentedReferenceGenomeEndpointTest + +const val DATA_USE_TERMS_DISABLED_CONFIG = "src/test/resources/backend_config_data_use_terms_disabled.json" + +@EndpointTest( + properties = ["${BackendSpringProperty.BACKEND_CONFIG_PATH}=$DATA_USE_TERMS_DISABLED_CONFIG"], +) +annotation class DataUseTermsDisabledEndpointTest + private const val SPRING_DATASOURCE_URL = "spring.datasource.url" private const val SPRING_DATASOURCE_USERNAME = "spring.datasource.username" private const val SPRING_DATASOURCE_PASSWORD = "spring.datasource.password" diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SingleSegmentedSubmitEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SingleSegmentedSubmitEndpointTest.kt index 80713614f6..0b7b182b9a 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SingleSegmentedSubmitEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SingleSegmentedSubmitEndpointTest.kt @@ -5,8 +5,7 @@ import org.hamcrest.Matchers.containsString import org.hamcrest.Matchers.hasEntry import org.junit.jupiter.api.Test import org.loculus.backend.config.BackendConfig -import org.loculus.backend.config.BackendSpringProperty -import org.loculus.backend.controller.EndpointTest +import org.loculus.backend.controller.SingleSegmentedReferenceGenomeEndpointTest import org.loculus.backend.controller.groupmanagement.GroupManagementControllerClient import org.loculus.backend.controller.groupmanagement.andGetGroupId import org.springframework.beans.factory.annotation.Autowired @@ -15,13 +14,9 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status -const val SINGLE_SEGMENTED_REFERENCE_GENOME = "src/test/resources/backend_config_single_segment.json" - private const val DEFAULT_SEQUENCE_NAME = "main" -@EndpointTest( - properties = ["${BackendSpringProperty.BACKEND_CONFIG_PATH}=$SINGLE_SEGMENTED_REFERENCE_GENOME"], -) +@SingleSegmentedReferenceGenomeEndpointTest class SingleSegmentedSubmitEndpointTest( @Autowired val submissionControllerClient: SubmissionControllerClient, @Autowired val convenienceClient: SubmissionConvenienceClient, diff --git a/backend/src/test/resources/backend_config_data_use_terms_disabled.json b/backend/src/test/resources/backend_config_data_use_terms_disabled.json new file mode 100644 index 0000000000..3e946d4007 --- /dev/null +++ b/backend/src/test/resources/backend_config_data_use_terms_disabled.json @@ -0,0 +1,105 @@ +{ + "accessionPrefix": "LOC_", + "organisms": { + "dummyOrganism": { + "referenceGenomes": { + "nucleotideSequences": [ + { + "name": "main", + "sequence": "ATTAAAGGTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCT" + } + ], + "genes": [ + { + "name": "someLongGene", + "sequence": "AAAAAAAAAAAAAAAAAAAAAAAAA" + }, + { + "name": "someShortGene", + "sequence": "MADS" + } + ] + }, + "schema": { + "organismName": "Test", + "allowSubmissionOfConsensusSequences": true, + "metadata": [ + { + "name": "date", + "type": "date", + "required": true + }, + { + "name": "dateSubmitted", + "type": "date" + }, + { + "name": "region", + "type": "string", + "autocomplete": true, + "required": true + }, + { + "name": "country", + "type": "string", + "autocomplete": true, + "required": true + }, + { + "name": "division", + "type": "string", + "autocomplete": true + }, + { + "name": "host", + "type": "string", + "autocomplete": true + }, + { + "name": "age", + "type": "int" + }, + { + "name": "sex", + "type": "string", + "autocomplete": true + }, + { + "name": "pangoLineage", + "type": "string", + "autocomplete": true + }, + { + "name": "qc", + "type": "float" + }, + { + "name": "booleanColumn", + "type": "boolean" + }, + { + "name": "insdcAccessionFull", + "type": "string" + }, + { + "name": "other_db_accession", + "type": "string" + } + ], + "externalMetadata": [ + { + "name": "insdcAccessionFull", + "type": "string", + "externalMetadataUpdater": "ena" + }, + { + "name": "other_db_accession", + "type": "string", + "externalMetadataUpdater": "other_db" + } + ] + } + }, + }, + "dataUseTermsEnabled": false +} From 4de74df7690705f4d61e62ea392ed725bc96f156 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 3 Feb 2025 16:08:46 +0100 Subject: [PATCH 17/34] Add stubs --- .../GetReleasedDataDataUseTermsDisabledEndpointTest.kt | 8 ++++++++ .../submission/SubmitEndpointDataUseTermsDisabledTest.kt | 8 ++++++++ ...dpointTest.kt => SubmitEndpointSingleSegmentedTest.kt} | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt create mode 100644 backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt rename backend/src/test/kotlin/org/loculus/backend/controller/submission/{SingleSegmentedSubmitEndpointTest.kt => SubmitEndpointSingleSegmentedTest.kt} (98%) diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt new file mode 100644 index 0000000000..f5d1a4a75d --- /dev/null +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt @@ -0,0 +1,8 @@ +package org.loculus.backend.controller.submission + +import org.loculus.backend.controller.DataUseTermsDisabledEndpointTest + +@DataUseTermsDisabledEndpointTest +class GetReleasedDataDataUseTermsDisabledEndpointTest { + // TODO +} diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt new file mode 100644 index 0000000000..b320c6106f --- /dev/null +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt @@ -0,0 +1,8 @@ +package org.loculus.backend.controller.submission + +import org.loculus.backend.controller.DataUseTermsDisabledEndpointTest + +@DataUseTermsDisabledEndpointTest +class SubmitEndpointDataUseTermsDisabledTest { + // TODO +} diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SingleSegmentedSubmitEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointSingleSegmentedTest.kt similarity index 98% rename from backend/src/test/kotlin/org/loculus/backend/controller/submission/SingleSegmentedSubmitEndpointTest.kt rename to backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointSingleSegmentedTest.kt index 0b7b182b9a..e50c3e26ad 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SingleSegmentedSubmitEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointSingleSegmentedTest.kt @@ -17,7 +17,7 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status private const val DEFAULT_SEQUENCE_NAME = "main" @SingleSegmentedReferenceGenomeEndpointTest -class SingleSegmentedSubmitEndpointTest( +class SubmitEndpointSingleSegmentedTest( @Autowired val submissionControllerClient: SubmissionControllerClient, @Autowired val convenienceClient: SubmissionConvenienceClient, @Autowired val backendConfig: BackendConfig, From be9a517e501706c0e210f30a38f4718c4df98341 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 3 Feb 2025 16:48:08 +0100 Subject: [PATCH 18/34] WIP --- ...sedDataDataUseTermsDisabledEndpointTest.kt | 105 +++++++++++++++++- .../submission/GetReleasedDataEndpointTest.kt | 11 +- ...ackend_config_data_use_terms_disabled.json | 2 +- 3 files changed, 111 insertions(+), 7 deletions(-) diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt index f5d1a4a75d..0dff1373eb 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt @@ -1,8 +1,111 @@ package org.loculus.backend.controller.submission +import com.fasterxml.jackson.databind.node.BooleanNode +import com.fasterxml.jackson.databind.node.IntNode +import com.fasterxml.jackson.databind.node.NullNode +import com.fasterxml.jackson.databind.node.TextNode +import com.ninjasquad.springmockk.MockkBean +import io.mockk.every +import kotlinx.datetime.Clock +import kotlinx.datetime.toLocalDateTime +import org.hamcrest.CoreMatchers.`is` +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.matchesPattern +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.keycloak.representations.idm.UserRepresentation +import org.loculus.backend.api.GeneticSequence +import org.loculus.backend.api.ProcessedData +import org.loculus.backend.controller.DEFAULT_GROUP +import org.loculus.backend.controller.DEFAULT_GROUP_CHANGED +import org.loculus.backend.controller.DEFAULT_GROUP_NAME_CHANGED +import org.loculus.backend.controller.DEFAULT_PIPELINE_VERSION +import org.loculus.backend.controller.DEFAULT_USER_NAME import org.loculus.backend.controller.DataUseTermsDisabledEndpointTest +import org.loculus.backend.controller.datauseterms.DataUseTermsControllerClient +import org.loculus.backend.controller.expectNdjsonAndGetContent +import org.loculus.backend.controller.groupmanagement.GroupManagementControllerClient +import org.loculus.backend.controller.groupmanagement.andGetGroupId +import org.loculus.backend.controller.jwtForDefaultUser +import org.loculus.backend.controller.submission.SubmitFiles.DefaultFiles.NUMBER_OF_SEQUENCES +import org.loculus.backend.service.KeycloakAdapter +import org.loculus.backend.utils.DateProvider +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status @DataUseTermsDisabledEndpointTest -class GetReleasedDataDataUseTermsDisabledEndpointTest { +class GetReleasedDataDataUseTermsDisabledEndpointTest( + @Autowired private val convenienceClient: SubmissionConvenienceClient, + @Autowired private val submissionControllerClient: SubmissionControllerClient, + @Autowired private val groupClient: GroupManagementControllerClient, +) { + private val currentDate = Clock.System.now().toLocalDateTime(DateProvider.timeZone).date.toString() + + + @MockkBean + lateinit var keycloakAdapter: KeycloakAdapter + + @BeforeEach + fun setup() { + every { keycloakAdapter.getUsersWithName(any()) } returns listOf(UserRepresentation()) + } + + @Test + fun `GIVEN released data exists THEN returns it with additional metadata fields`() { + val groupId = groupClient.createNewGroup(group = DEFAULT_GROUP, jwt = jwtForDefaultUser) + .andExpect(status().isOk) + .andGetGroupId() + + convenienceClient.prepareDefaultSequenceEntriesToApprovedForRelease(groupId = groupId) + + groupClient.updateGroup( + groupId = groupId, + group = DEFAULT_GROUP_CHANGED, + jwt = jwtForDefaultUser, + ).andExpect(status().isOk) + + val response = submissionControllerClient.getReleasedData() + + val responseBody = response.expectNdjsonAndGetContent>() + + assertThat(responseBody.size, `is`(NUMBER_OF_SEQUENCES)) + + response.andExpect(header().string("x-total-records", NUMBER_OF_SEQUENCES.toString())) + + responseBody.forEach { + val id = it.metadata["accession"]!!.asText() + val version = it.metadata["version"]!!.asLong() + assertThat(version, `is`(1)) + + val expectedMetadata = defaultProcessedData.metadata + mapOf( + "accession" to TextNode(id), + "version" to IntNode(version.toInt()), + "accessionVersion" to TextNode("$id.$version"), + "isRevocation" to BooleanNode.FALSE, + "submitter" to TextNode(DEFAULT_USER_NAME), + "groupName" to TextNode(DEFAULT_GROUP_NAME_CHANGED), + "versionStatus" to TextNode("LATEST_VERSION"), + "submittedDate" to TextNode(currentDate), + "dataUseTermsRestrictedUntil" to NullNode.getInstance(), + "pipelineVersion" to IntNode(DEFAULT_PIPELINE_VERSION.toInt()), + ) + + for ((key, value) in it.metadata) { + when (key) { + "submittedAtTimestamp" -> expectIsTimestampWithCurrentYear(value) + "releasedAtTimestamp" -> expectIsTimestampWithCurrentYear(value) + "submissionId" -> assertThat(value.textValue(), matchesPattern("^custom\\d$")) + "groupId" -> assertThat(value.intValue(), `is`(groupId)) + else -> assertThat(value, `is`(expectedMetadata[key])) + } + } + assertThat(it.alignedNucleotideSequences, `is`(defaultProcessedData.alignedNucleotideSequences)) + assertThat(it.unalignedNucleotideSequences, `is`(defaultProcessedData.unalignedNucleotideSequences)) + assertThat(it.alignedAminoAcidSequences, `is`(defaultProcessedData.alignedAminoAcidSequences)) + assertThat(it.nucleotideInsertions, `is`(defaultProcessedData.nucleotideInsertions)) + assertThat(it.aminoAcidInsertions, `is`(defaultProcessedData.aminoAcidInsertions)) + } + } // TODO } diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt index a6b7cabc21..7b367f7c49 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt @@ -97,7 +97,6 @@ class GetReleasedDataEndpointTest( @Autowired private val groupClient: GroupManagementControllerClient, @Autowired private val dataUseTermsClient: DataUseTermsControllerClient, ) { - private val currentYear = Clock.System.now().toLocalDateTime(DateProvider.timeZone).year private val currentDate = Clock.System.now().toLocalDateTime(DateProvider.timeZone).date.toString() @MockkBean @@ -421,10 +420,12 @@ class GetReleasedDataEndpointTest( ) } - private fun expectIsTimestampWithCurrentYear(value: JsonNode) { - val dateTime = Instant.fromEpochSeconds(value.asLong()).toLocalDateTime(DateProvider.timeZone) - assertThat(dateTime.year, `is`(currentYear)) - } +} + +fun expectIsTimestampWithCurrentYear(value: JsonNode) { + val currentYear = Clock.System.now().toLocalDateTime(DateProvider.timeZone).year + val dateTime = Instant.fromEpochSeconds(value.asLong()).toLocalDateTime(DateProvider.timeZone) + assertThat(dateTime.year, `is`(currentYear)) } private const val OPEN_DATA_USE_TERMS_URL = "openUrl" diff --git a/backend/src/test/resources/backend_config_data_use_terms_disabled.json b/backend/src/test/resources/backend_config_data_use_terms_disabled.json index 3e946d4007..b5f6a68082 100644 --- a/backend/src/test/resources/backend_config_data_use_terms_disabled.json +++ b/backend/src/test/resources/backend_config_data_use_terms_disabled.json @@ -99,7 +99,7 @@ } ] } - }, + } }, "dataUseTermsEnabled": false } From 5a220c618e06d9f481519498b71e70b7ea090b63 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 10:37:15 +0100 Subject: [PATCH 19/34] WIP --- .../backend/controller/EndpointTestExtension.kt | 12 +----------- ...leasedDataDataUseTermsDisabledEndpointTest.kt | 16 +++++++++++++--- .../SubmitEndpointDataUseTermsDisabledTest.kt | 8 ++++++-- .../SubmitEndpointSingleSegmentedTest.kt | 8 ++++++-- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt b/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt index 222a0c34c7..b37eecb56f 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt @@ -10,7 +10,6 @@ import org.junit.platform.launcher.TestExecutionListener import org.junit.platform.launcher.TestPlan import org.loculus.backend.api.Address import org.loculus.backend.api.NewGroup -import org.loculus.backend.config.BackendSpringProperty import org.loculus.backend.controller.datauseterms.DataUseTermsControllerClient import org.loculus.backend.controller.groupmanagement.GroupManagementControllerClient import org.loculus.backend.controller.seqsetcitations.SeqSetCitationsControllerClient @@ -24,6 +23,7 @@ import org.loculus.backend.service.submission.METADATA_UPLOAD_AUX_TABLE_NAME import org.loculus.backend.service.submission.SEQUENCE_ENTRIES_PREPROCESSED_DATA_TABLE_NAME import org.loculus.backend.service.submission.SEQUENCE_ENTRIES_TABLE_NAME import org.loculus.backend.service.submission.SEQUENCE_UPLOAD_AUX_TABLE_NAME + import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc import org.springframework.boot.test.context.SpringBootTest import org.springframework.context.annotation.Import @@ -51,18 +51,8 @@ annotation class EndpointTest(@get:AliasFor(annotation = SpringBootTest::class) const val SINGLE_SEGMENTED_REFERENCE_GENOME = "src/test/resources/backend_config_single_segment.json" -@EndpointTest( - properties = ["${BackendSpringProperty.BACKEND_CONFIG_PATH}=$SINGLE_SEGMENTED_REFERENCE_GENOME"], -) -annotation class SingleSegmentedReferenceGenomeEndpointTest - const val DATA_USE_TERMS_DISABLED_CONFIG = "src/test/resources/backend_config_data_use_terms_disabled.json" -@EndpointTest( - properties = ["${BackendSpringProperty.BACKEND_CONFIG_PATH}=$DATA_USE_TERMS_DISABLED_CONFIG"], -) -annotation class DataUseTermsDisabledEndpointTest - private const val SPRING_DATASOURCE_URL = "spring.datasource.url" private const val SPRING_DATASOURCE_USERNAME = "spring.datasource.username" private const val SPRING_DATASOURCE_PASSWORD = "spring.datasource.password" diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt index 0dff1373eb..3124876a22 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt @@ -16,13 +16,15 @@ import org.junit.jupiter.api.Test import org.keycloak.representations.idm.UserRepresentation import org.loculus.backend.api.GeneticSequence import org.loculus.backend.api.ProcessedData +import org.loculus.backend.config.BackendConfig +import org.loculus.backend.config.BackendSpringProperty +import org.loculus.backend.controller.DATA_USE_TERMS_DISABLED_CONFIG import org.loculus.backend.controller.DEFAULT_GROUP import org.loculus.backend.controller.DEFAULT_GROUP_CHANGED import org.loculus.backend.controller.DEFAULT_GROUP_NAME_CHANGED import org.loculus.backend.controller.DEFAULT_PIPELINE_VERSION import org.loculus.backend.controller.DEFAULT_USER_NAME -import org.loculus.backend.controller.DataUseTermsDisabledEndpointTest -import org.loculus.backend.controller.datauseterms.DataUseTermsControllerClient +import org.loculus.backend.controller.EndpointTest import org.loculus.backend.controller.expectNdjsonAndGetContent import org.loculus.backend.controller.groupmanagement.GroupManagementControllerClient import org.loculus.backend.controller.groupmanagement.andGetGroupId @@ -34,11 +36,14 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status -@DataUseTermsDisabledEndpointTest +@EndpointTest( + properties = ["${BackendSpringProperty.BACKEND_CONFIG_PATH}=$DATA_USE_TERMS_DISABLED_CONFIG"], +) class GetReleasedDataDataUseTermsDisabledEndpointTest( @Autowired private val convenienceClient: SubmissionConvenienceClient, @Autowired private val submissionControllerClient: SubmissionControllerClient, @Autowired private val groupClient: GroupManagementControllerClient, + @Autowired private val backendConfig: BackendConfig, ) { private val currentDate = Clock.System.now().toLocalDateTime(DateProvider.timeZone).date.toString() @@ -51,6 +56,11 @@ class GetReleasedDataDataUseTermsDisabledEndpointTest( every { keycloakAdapter.getUsersWithName(any()) } returns listOf(UserRepresentation()) } + @Test + fun `config has been read and data use terms are configred to be off`() { + assertThat(backendConfig.dataUseTermsEnabled, `is`(false)); + } + @Test fun `GIVEN released data exists THEN returns it with additional metadata fields`() { val groupId = groupClient.createNewGroup(group = DEFAULT_GROUP, jwt = jwtForDefaultUser) diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt index b320c6106f..69dd27b42a 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt @@ -1,8 +1,12 @@ package org.loculus.backend.controller.submission -import org.loculus.backend.controller.DataUseTermsDisabledEndpointTest +import org.loculus.backend.config.BackendSpringProperty +import org.loculus.backend.controller.DATA_USE_TERMS_DISABLED_CONFIG +import org.loculus.backend.controller.EndpointTest -@DataUseTermsDisabledEndpointTest +@EndpointTest( + properties = ["${BackendSpringProperty.BACKEND_CONFIG_PATH}=$DATA_USE_TERMS_DISABLED_CONFIG"], +) class SubmitEndpointDataUseTermsDisabledTest { // TODO } diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointSingleSegmentedTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointSingleSegmentedTest.kt index e50c3e26ad..3b083abe9e 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointSingleSegmentedTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointSingleSegmentedTest.kt @@ -5,7 +5,9 @@ import org.hamcrest.Matchers.containsString import org.hamcrest.Matchers.hasEntry import org.junit.jupiter.api.Test import org.loculus.backend.config.BackendConfig -import org.loculus.backend.controller.SingleSegmentedReferenceGenomeEndpointTest +import org.loculus.backend.config.BackendSpringProperty +import org.loculus.backend.controller.EndpointTest +import org.loculus.backend.controller.SINGLE_SEGMENTED_REFERENCE_GENOME import org.loculus.backend.controller.groupmanagement.GroupManagementControllerClient import org.loculus.backend.controller.groupmanagement.andGetGroupId import org.springframework.beans.factory.annotation.Autowired @@ -16,7 +18,9 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status private const val DEFAULT_SEQUENCE_NAME = "main" -@SingleSegmentedReferenceGenomeEndpointTest +@EndpointTest( + properties = ["${BackendSpringProperty.BACKEND_CONFIG_PATH}=$SINGLE_SEGMENTED_REFERENCE_GENOME"], +) class SubmitEndpointSingleSegmentedTest( @Autowired val submissionControllerClient: SubmissionControllerClient, @Autowired val convenienceClient: SubmissionConvenienceClient, From 97a67de7267d4c1377fec51a69c65b659ab53a1d Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 10:54:47 +0100 Subject: [PATCH 20/34] test fixed --- .../controller/EndpointTestExtension.kt | 1 - ...sedDataDataUseTermsDisabledEndpointTest.kt | 34 +++++++++++++++---- .../submission/GetReleasedDataEndpointTest.kt | 1 - 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt b/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt index b37eecb56f..31f6eb9bdb 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt @@ -23,7 +23,6 @@ import org.loculus.backend.service.submission.METADATA_UPLOAD_AUX_TABLE_NAME import org.loculus.backend.service.submission.SEQUENCE_ENTRIES_PREPROCESSED_DATA_TABLE_NAME import org.loculus.backend.service.submission.SEQUENCE_ENTRIES_TABLE_NAME import org.loculus.backend.service.submission.SEQUENCE_UPLOAD_AUX_TABLE_NAME - import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc import org.springframework.boot.test.context.SpringBootTest import org.springframework.context.annotation.Import diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt index 3124876a22..4526f7ebd4 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt @@ -2,13 +2,14 @@ package org.loculus.backend.controller.submission import com.fasterxml.jackson.databind.node.BooleanNode import com.fasterxml.jackson.databind.node.IntNode -import com.fasterxml.jackson.databind.node.NullNode import com.fasterxml.jackson.databind.node.TextNode import com.ninjasquad.springmockk.MockkBean import io.mockk.every import kotlinx.datetime.Clock import kotlinx.datetime.toLocalDateTime +import org.hamcrest.CoreMatchers.hasItem import org.hamcrest.CoreMatchers.`is` +import org.hamcrest.CoreMatchers.not import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.matchesPattern import org.junit.jupiter.api.BeforeEach @@ -47,7 +48,6 @@ class GetReleasedDataDataUseTermsDisabledEndpointTest( ) { private val currentDate = Clock.System.now().toLocalDateTime(DateProvider.timeZone).date.toString() - @MockkBean lateinit var keycloakAdapter: KeycloakAdapter @@ -58,11 +58,29 @@ class GetReleasedDataDataUseTermsDisabledEndpointTest( @Test fun `config has been read and data use terms are configred to be off`() { - assertThat(backendConfig.dataUseTermsEnabled, `is`(false)); + assertThat(backendConfig.dataUseTermsEnabled, `is`(false)) + } + + @Test + fun `GIVEN released data exists THEN NOT returns data use terms properties`() { + val groupId = groupClient.createNewGroup(group = DEFAULT_GROUP, jwt = jwtForDefaultUser) + .andExpect(status().isOk) + .andGetGroupId() + + convenienceClient.prepareDefaultSequenceEntriesToApprovedForRelease(groupId = groupId) + + val response = submissionControllerClient.getReleasedData() + + val responseBody = response.expectNdjsonAndGetContent>() + + responseBody.forEach { + assertThat(it.metadata.keys, not(hasItem("dataUseTerms"))) + assertThat(it.metadata.keys, not(hasItem("dataUseTermsRestrictedUntil"))) + } } @Test - fun `GIVEN released data exists THEN returns it with additional metadata fields`() { + fun `GIVEN released data exists THEN returns with additional metadata fields & no data use terms properties`() { val groupId = groupClient.createNewGroup(group = DEFAULT_GROUP, jwt = jwtForDefaultUser) .andExpect(status().isOk) .andGetGroupId() @@ -96,8 +114,8 @@ class GetReleasedDataDataUseTermsDisabledEndpointTest( "submitter" to TextNode(DEFAULT_USER_NAME), "groupName" to TextNode(DEFAULT_GROUP_NAME_CHANGED), "versionStatus" to TextNode("LATEST_VERSION"), + "releasedDate" to TextNode(currentDate), "submittedDate" to TextNode(currentDate), - "dataUseTermsRestrictedUntil" to NullNode.getInstance(), "pipelineVersion" to IntNode(DEFAULT_PIPELINE_VERSION.toInt()), ) @@ -107,7 +125,10 @@ class GetReleasedDataDataUseTermsDisabledEndpointTest( "releasedAtTimestamp" -> expectIsTimestampWithCurrentYear(value) "submissionId" -> assertThat(value.textValue(), matchesPattern("^custom\\d$")) "groupId" -> assertThat(value.intValue(), `is`(groupId)) - else -> assertThat(value, `is`(expectedMetadata[key])) + else -> { + assertThat(expectedMetadata.keys, hasItem(key)) + assertThat(value, `is`(expectedMetadata[key])) + } } } assertThat(it.alignedNucleotideSequences, `is`(defaultProcessedData.alignedNucleotideSequences)) @@ -117,5 +138,4 @@ class GetReleasedDataDataUseTermsDisabledEndpointTest( assertThat(it.aminoAcidInsertions, `is`(defaultProcessedData.aminoAcidInsertions)) } } - // TODO } diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt index 7b367f7c49..23c8eaf797 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt @@ -419,7 +419,6 @@ class GetReleasedDataEndpointTest( latestVersion5 = 5L, ) } - } fun expectIsTimestampWithCurrentYear(value: JsonNode) { From ae80698ef6cdf2592a7f1e5cc426393d9837de03 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 11:01:52 +0100 Subject: [PATCH 21/34] add comment --- .../org/loculus/backend/controller/EndpointTestExtension.kt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt b/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt index 31f6eb9bdb..8275b90b3a 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt @@ -31,6 +31,12 @@ import org.springframework.test.annotation.DirtiesContext import org.springframework.test.context.ActiveProfiles import org.testcontainers.containers.PostgreSQLContainer +/** + * The main annotation for tests. It also loads the [EndpointTestExtension], which initializes + * a PostgreSQL test container. + * You can set additional properties to - for example - override the backend config file, like in + * [org.loculus.backend.controller.submission.GetReleasedDataDataUseTermsDisabledEndpointTest]. + */ @Target(AnnotationTarget.TYPE, AnnotationTarget.CLASS) @Retention(AnnotationRetention.RUNTIME) @AutoConfigureMockMvc From 0afd92c56cd3f2e6d317315cb084a72aa4b940c1 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 11:16:46 +0100 Subject: [PATCH 22/34] Add test to check that there is an error if you omit DUT on normal submit --- .../submission/SubmissionControllerClient.kt | 16 ++++++++++++++++ .../controller/submission/SubmitEndpointTest.kt | 11 +++++++++++ 2 files changed, 27 insertions(+) diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt index eba76f6dd8..1cee3a2b84 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt @@ -56,6 +56,22 @@ class SubmissionControllerClient(private val mockMvc: MockMvc, private val objec .withAuth(jwt), ) + fun submit_without_data_use_terms( + metadataFile: MockMultipartFile, + sequencesFile: MockMultipartFile? = null, + organism: String = DEFAULT_ORGANISM, + groupId: Int, + jwt: String? = jwtForDefaultUser, + ): ResultActions = mockMvc.perform( + multipart(addOrganismToPath("/submit", organism = organism)) + .apply { + sequencesFile?.let { file(sequencesFile) } + } + .file(metadataFile) + .param("groupId", groupId.toString()) + .withAuth(jwt), + ) + fun extractUnprocessedData( numberOfSequenceEntries: Int, organism: String = DEFAULT_ORGANISM, diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt index d3875a9ba1..8b4d9de138 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt @@ -149,6 +149,17 @@ class SubmitEndpointTest( .andExpect(jsonPath("\$[0].version").value(1)) } + @Test + fun `GIVEN submission without data use terms THEN returns an error`() { + submissionControllerClient.submit_without_data_use_terms( + DefaultFiles.metadataFile, + DefaultFiles.sequencesFileMultiSegmented, + organism = OTHER_ORGANISM, + groupId = groupId, + ) + .andExpect(status().isBadRequest) + } + @Test fun `GIVEN fasta data with unknown segment THEN data is accepted to let the preprocessing pipeline verify it`() { submissionControllerClient.submit( From 12d66a878f1351115df16a72c0e83cadc8499fbf Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 11:27:05 +0100 Subject: [PATCH 23/34] Add another test --- .../backend/model/ReleasedDataModel.kt | 1 - ...sedDataDataUseTermsDisabledEndpointTest.kt | 2 +- .../submission/SubmissionControllerClient.kt | 2 +- .../SubmitEndpointDataUseTermsDisabledTest.kt | 50 ++++++++++++++++++- .../submission/SubmitEndpointTest.kt | 2 +- 5 files changed, 51 insertions(+), 6 deletions(-) diff --git a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt index 82d78e61bc..6eb6758869 100644 --- a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt +++ b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt @@ -139,7 +139,6 @@ open class ReleasedDataModel( ("versionStatus" to TextNode(versionStatus.name)), ("pipelineVersion" to LongNode(rawProcessedData.pipelineVersion)), ) + - // TODO add a test for this change conditionalMetadata( backendConfig.dataUseTermsEnabled, { diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt index 4526f7ebd4..e2efc93c50 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt @@ -57,7 +57,7 @@ class GetReleasedDataDataUseTermsDisabledEndpointTest( } @Test - fun `config has been read and data use terms are configred to be off`() { + fun `config has been read and data use terms are configured to be off`() { assertThat(backendConfig.dataUseTermsEnabled, `is`(false)) } diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt index 1cee3a2b84..7b6f7daa3d 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt @@ -56,7 +56,7 @@ class SubmissionControllerClient(private val mockMvc: MockMvc, private val objec .withAuth(jwt), ) - fun submit_without_data_use_terms( + fun submitWithoutDataUseTerms( metadataFile: MockMultipartFile, sequencesFile: MockMultipartFile? = null, organism: String = DEFAULT_ORGANISM, diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt index 69dd27b42a..ac3f4810c5 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt @@ -1,12 +1,58 @@ package org.loculus.backend.controller.submission +import org.hamcrest.CoreMatchers.`is` +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.containsString +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.loculus.backend.config.BackendConfig import org.loculus.backend.config.BackendSpringProperty import org.loculus.backend.controller.DATA_USE_TERMS_DISABLED_CONFIG +import org.loculus.backend.controller.DEFAULT_ORGANISM import org.loculus.backend.controller.EndpointTest +import org.loculus.backend.controller.groupmanagement.GroupManagementControllerClient +import org.loculus.backend.controller.groupmanagement.andGetGroupId +import org.loculus.backend.controller.submission.SubmitFiles.DefaultFiles +import org.loculus.backend.controller.submission.SubmitFiles.DefaultFiles.NUMBER_OF_SEQUENCES +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.http.MediaType.APPLICATION_JSON_VALUE +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status @EndpointTest( properties = ["${BackendSpringProperty.BACKEND_CONFIG_PATH}=$DATA_USE_TERMS_DISABLED_CONFIG"], ) -class SubmitEndpointDataUseTermsDisabledTest { - // TODO +class SubmitEndpointDataUseTermsDisabledTest( + @Autowired val submissionControllerClient: SubmissionControllerClient, + @Autowired val backendConfig: BackendConfig, + @Autowired val groupManagementClient: GroupManagementControllerClient, +) { + var groupId: Int = 0 + + @BeforeEach + fun prepareNewGroup() { + groupId = groupManagementClient.createNewGroup().andGetGroupId() + } + + @Test + fun `config has been read and data use terms are configred to be off`() { + assertThat(backendConfig.dataUseTermsEnabled, `is`(false)) + } + + @Test + fun `GIVEN valid input multi segment data THEN returns mapping of provided custom ids to generated ids`() { + submissionControllerClient.submitWithoutDataUseTerms( + DefaultFiles.metadataFile, + DefaultFiles.sequencesFile, + organism = DEFAULT_ORGANISM, + groupId = groupId, + ) + .andExpect(status().isOk) + .andExpect(content().contentType(APPLICATION_JSON_VALUE)) + .andExpect(jsonPath("\$.length()").value(NUMBER_OF_SEQUENCES)) + .andExpect(jsonPath("\$[0].submissionId").value("custom0")) + .andExpect(jsonPath("\$[0].accession", containsString(backendConfig.accessionPrefix))) + .andExpect(jsonPath("\$[0].version").value(1)) + } } diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt index 8b4d9de138..69a79b1921 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt @@ -151,7 +151,7 @@ class SubmitEndpointTest( @Test fun `GIVEN submission without data use terms THEN returns an error`() { - submissionControllerClient.submit_without_data_use_terms( + submissionControllerClient.submitWithoutDataUseTerms( DefaultFiles.metadataFile, DefaultFiles.sequencesFileMultiSegmented, organism = OTHER_ORGANISM, From 5992ef2ae74939fa5aa89c10ca171c5b505e1f18 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 12:03:34 +0100 Subject: [PATCH 24/34] restructure config --- .../main/kotlin/org/loculus/backend/config/Config.kt | 5 +++-- kubernetes/loculus/templates/_common-metadata.tpl | 11 +++++------ kubernetes/loculus/values.yaml | 9 +++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/backend/src/main/kotlin/org/loculus/backend/config/Config.kt b/backend/src/main/kotlin/org/loculus/backend/config/Config.kt index e6b123c248..3d84ba284a 100644 --- a/backend/src/main/kotlin/org/loculus/backend/config/Config.kt +++ b/backend/src/main/kotlin/org/loculus/backend/config/Config.kt @@ -7,14 +7,15 @@ import org.loculus.backend.api.Organism data class BackendConfig( val organisms: Map, val accessionPrefix: String, - val dataUseTermsEnabled: Boolean, - val dataUseTermsUrls: DataUseTermsUrls?, + val dataUseTerms: DataUseTerms, ) { fun getInstanceConfig(organism: Organism) = organisms[organism.name] ?: throw IllegalArgumentException( "Organism: ${organism.name} not found in backend config. Available organisms: ${organisms.keys}", ) } +data class DataUseTerms(val enabled: Boolean, val urls: DataUseTermsUrls?) + data class DataUseTermsUrls(val open: String, val restricted: String) data class InstanceConfig(val schema: Schema, val referenceGenomes: ReferenceGenome) diff --git a/kubernetes/loculus/templates/_common-metadata.tpl b/kubernetes/loculus/templates/_common-metadata.tpl index 13064cac40..b81eaa78fa 100644 --- a/kubernetes/loculus/templates/_common-metadata.tpl +++ b/kubernetes/loculus/templates/_common-metadata.tpl @@ -75,7 +75,7 @@ fields: autocomplete: true displayName: Date released (exact) columnWidth: 100 - {{- if $.Values.dataUseTermsEnabled }} + {{- if $.Values.dataUseTerms.enabled }} - name: dataUseTerms type: string generateIndex: true @@ -91,7 +91,7 @@ fields: hideOnSequenceDetailsPage: true header: Data use terms {{- end}} - {{- if $.Values.dataUseTermsUrls }} + {{- if $.Values.dataUseTerms.urls }} - name: dataUseTermsUrl displayName: Data use terms URL type: string @@ -162,7 +162,7 @@ enableLoginNavigationItem: {{ $.Values.website.websiteConfig.enableLoginNavigati enableSubmissionNavigationItem: {{ $.Values.website.websiteConfig.enableSubmissionNavigationItem }} enableSubmissionPages: {{ $.Values.website.websiteConfig.enableSubmissionPages }} enableSeqSets: {{ $.Values.seqSets.enabled }} -enableDataUseTerms: {{ $.Values.dataUseTermsEnabled }} +enableDataUseTerms: {{ $.Values.dataUseTerms.enabled }} accessionPrefix: {{ quote $.Values.accessionPrefix }} {{- $commonMetadata := (include "loculus.commonMetadata" . | fromYaml).fields }} organisms: @@ -292,9 +292,8 @@ fields: {{- define "loculus.generateBackendConfig" }} accessionPrefix: {{ quote $.Values.accessionPrefix }} name: {{ quote $.Values.name }} -dataUseTermsEnabled: {{$.Values.dataUseTermsEnabled }} -dataUseTermsUrls: - {{$.Values.dataUseTermsUrls | toYaml | nindent 2}} +dataUseTerms: + {{$.Values.dataUseTerms | toYaml | nindent 2}} organisms: {{- range $key, $instance := (.Values.organisms | default .Values.defaultOrganisms) }} {{ $key }}: diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index c4cc3a3b0c..03b024f6a3 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -20,10 +20,11 @@ ingestLimitSeconds: 1800 getSubmissionListLimitSeconds: 600 preprocessingTimeout: 600 accessionPrefix: "LOC_" -dataUseTermsEnabled: false -dataUseTermsUrls: - open: https://#TODO-MVP/open - restricted: https://#TODO-MVP/restricted +dataUseTerms: + enabled: false + urls: + open: https://#TODO-MVP/open + restricted: https://#TODO-MVP/restricted name: "Loculus" logo: url: "/favicon.svg" From 9802b01e760370146bd6a6ce695ecf2ab3e05650 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 12:10:14 +0100 Subject: [PATCH 25/34] fix configs --- backend/src/test/resources/backend_config.json | 4 +++- .../resources/backend_config_data_use_terms_disabled.json | 4 +++- backend/src/test/resources/backend_config_single_segment.json | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/src/test/resources/backend_config.json b/backend/src/test/resources/backend_config.json index e63e12667f..128c6016fb 100644 --- a/backend/src/test/resources/backend_config.json +++ b/backend/src/test/resources/backend_config.json @@ -226,5 +226,7 @@ } } }, - "dataUseTermsEnabled": true + "dataUseTerms": { + "enabled": true + } } diff --git a/backend/src/test/resources/backend_config_data_use_terms_disabled.json b/backend/src/test/resources/backend_config_data_use_terms_disabled.json index b5f6a68082..d73ddd2d78 100644 --- a/backend/src/test/resources/backend_config_data_use_terms_disabled.json +++ b/backend/src/test/resources/backend_config_data_use_terms_disabled.json @@ -101,5 +101,7 @@ } } }, - "dataUseTermsEnabled": false + "dataUseTerms": { + "enabled": false + } } diff --git a/backend/src/test/resources/backend_config_single_segment.json b/backend/src/test/resources/backend_config_single_segment.json index 9f532a9b2c..00cb719276 100644 --- a/backend/src/test/resources/backend_config_single_segment.json +++ b/backend/src/test/resources/backend_config_single_segment.json @@ -1,6 +1,8 @@ { "accessionPrefix": "LOC_", - "dataUseTermsEnabled": true, + "dataUseTerms": { + "enabled": true + }, "organisms": { "dummyOrganism": { "referenceGenomes": { From 7f15efaf875e30cad40272fa555c37ab94f0e3cc Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 12:44:57 +0100 Subject: [PATCH 26/34] fix usages --- .../loculus/backend/controller/SubmissionController.kt | 2 +- .../org/loculus/backend/model/ReleasedDataModel.kt | 4 ++-- .../loculus/backend/config/BackendSpringConfigTest.kt | 3 +-- .../GetReleasedDataDataUseTermsDisabledEndpointTest.kt | 2 +- .../controller/submission/GetReleasedDataEndpointTest.kt | 9 +++++---- .../submission/SubmitEndpointDataUseTermsDisabledTest.kt | 4 ++-- .../backend/controller/submission/SubmitEndpointTest.kt | 2 -- .../service/GenerateAccessionFromNumberServiceTest.kt | 4 ++-- .../service/submission/EmptyProcessedDataProviderTest.kt | 4 ++-- 9 files changed, 16 insertions(+), 18 deletions(-) diff --git a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt index ac6a6eb903..1ec8f70f43 100644 --- a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt +++ b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt @@ -101,7 +101,7 @@ open class SubmissionController( ) @RequestParam restrictedUntil: String?, ): List { var dataUseTermsKind = DataUseTermsType.OPEN - if (backendConfig.dataUseTermsEnabled) { + if (backendConfig.dataUseTerms.enabled) { if (dataUseTermsType == null) { throw BadRequestException("the 'dataUseTermsType' needs to be provided.") } else { diff --git a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt index 6eb6758869..619bcc0cf7 100644 --- a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt +++ b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt @@ -115,7 +115,7 @@ open class ReleasedDataModel( val earliestReleaseDate = earliestReleaseDateFinder?.calculateEarliestReleaseDate(rawProcessedData) - val dataUseTermsUrl: String? = backendConfig.dataUseTermsUrls?.let { urls -> + val dataUseTermsUrl: String? = backendConfig.dataUseTerms.urls?.let { urls -> when (currentDataUseTerms) { DataUseTerms.Open -> urls.open is DataUseTerms.Restricted -> urls.restricted @@ -140,7 +140,7 @@ open class ReleasedDataModel( ("pipelineVersion" to LongNode(rawProcessedData.pipelineVersion)), ) + conditionalMetadata( - backendConfig.dataUseTermsEnabled, + backendConfig.dataUseTerms.enabled, { mapOf( "dataUseTerms" to TextNode(currentDataUseTerms.type.name), diff --git a/backend/src/test/kotlin/org/loculus/backend/config/BackendSpringConfigTest.kt b/backend/src/test/kotlin/org/loculus/backend/config/BackendSpringConfigTest.kt index bcbf292420..f516000913 100644 --- a/backend/src/test/kotlin/org/loculus/backend/config/BackendSpringConfigTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/config/BackendSpringConfigTest.kt @@ -70,6 +70,5 @@ fun backendConfig(metadataList: List, earliestReleaseDate: EarliestRel ), ), accessionPrefix = "FOO_", - dataUseTermsEnabled = true, - dataUseTermsUrls = null, + dataUseTerms = DataUseTerms(true, null), ) diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt index e2efc93c50..aacfc67f3c 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt @@ -58,7 +58,7 @@ class GetReleasedDataDataUseTermsDisabledEndpointTest( @Test fun `config has been read and data use terms are configured to be off`() { - assertThat(backendConfig.dataUseTermsEnabled, `is`(false)) + assertThat(backendConfig.dataUseTerms.enabled, `is`(false)) } @Test diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt index 23c8eaf797..6e2c41f875 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt @@ -555,11 +555,12 @@ class GetReleasedDataEndpointWithDataUseTermsUrlTest( @Value("\${${BackendSpringProperty.BACKEND_CONFIG_PATH}}") configPath: String, ): BackendConfig { val originalConfig = readBackendConfig(objectMapper = objectMapper, configPath = configPath) - return originalConfig.copy( - dataUseTermsUrls = DataUseTermsUrls( - open = OPEN_DATA_USE_TERMS_URL, - restricted = RESTRICTED_DATA_USE_TERMS_URL, + dataUseTerms = originalConfig.dataUseTerms.copy( + urls = DataUseTermsUrls( + open = OPEN_DATA_USE_TERMS_URL, + restricted = RESTRICTED_DATA_USE_TERMS_URL, + ), ), ) } diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt index ac3f4810c5..89ae7972ab 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointDataUseTermsDisabledTest.kt @@ -36,8 +36,8 @@ class SubmitEndpointDataUseTermsDisabledTest( } @Test - fun `config has been read and data use terms are configred to be off`() { - assertThat(backendConfig.dataUseTermsEnabled, `is`(false)) + fun `config has been read and data use terms are configured to be off`() { + assertThat(backendConfig.dataUseTerms.enabled, `is`(false)) } @Test diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt index 69a79b1921..f9d8a63196 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt @@ -195,8 +195,6 @@ class SubmitEndpointTest( organism: Organism, dataUseTerm: DataUseTerms, ) { - println("Data use terms enabled: ") - println(backendConfig.dataUseTermsEnabled) submissionControllerClient.submit( metadataFile, sequencesFile, diff --git a/backend/src/test/kotlin/org/loculus/backend/service/GenerateAccessionFromNumberServiceTest.kt b/backend/src/test/kotlin/org/loculus/backend/service/GenerateAccessionFromNumberServiceTest.kt index a9a05704df..d8d1387e88 100644 --- a/backend/src/test/kotlin/org/loculus/backend/service/GenerateAccessionFromNumberServiceTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/service/GenerateAccessionFromNumberServiceTest.kt @@ -4,6 +4,7 @@ import org.hamcrest.CoreMatchers.`is` import org.hamcrest.MatcherAssert.assertThat import org.junit.jupiter.api.Test import org.loculus.backend.config.BackendConfig +import org.loculus.backend.config.DataUseTerms import java.lang.Math.random import kotlin.math.pow @@ -14,8 +15,7 @@ class GenerateAccessionFromNumberServiceTest { BackendConfig( accessionPrefix = PREFIX, organisms = emptyMap(), - dataUseTermsEnabled = true, - dataUseTermsUrls = null, + dataUseTerms = DataUseTerms(true, null), ), ) diff --git a/backend/src/test/kotlin/org/loculus/backend/service/submission/EmptyProcessedDataProviderTest.kt b/backend/src/test/kotlin/org/loculus/backend/service/submission/EmptyProcessedDataProviderTest.kt index 2849d4682c..8b679138d4 100644 --- a/backend/src/test/kotlin/org/loculus/backend/service/submission/EmptyProcessedDataProviderTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/service/submission/EmptyProcessedDataProviderTest.kt @@ -6,6 +6,7 @@ import org.hamcrest.MatcherAssert.assertThat import org.junit.jupiter.api.Test import org.loculus.backend.api.Organism import org.loculus.backend.config.BackendConfig +import org.loculus.backend.config.DataUseTerms import org.loculus.backend.config.InstanceConfig import org.loculus.backend.config.Metadata import org.loculus.backend.config.MetadataType @@ -46,8 +47,7 @@ class EmptyProcessedDataProviderTest { ), ), ), - dataUseTermsEnabled = true, - dataUseTermsUrls = null, + dataUseTerms = DataUseTerms(true, null), ), ) From 9b14a1b13251d8ecba95e73242601a63f23a615a Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 13:02:30 +0100 Subject: [PATCH 27/34] Add page stub --- docs/astro.config.mjs | 1 + docs/src/content/docs/for-administrators/data-use-terms.md | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 docs/src/content/docs/for-administrators/data-use-terms.md diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index 6b631f90b7..2d3c98d4a7 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -73,6 +73,7 @@ export default defineConfig({ label: 'Build new preprocessing pipeline', link: '/for-administrators/build-new-preprocessing-pipeline/', }, + 'for-administrators/data-use-terms', { label: 'User administration', link: '/for-administrators/user-administration/' }, ], }, diff --git a/docs/src/content/docs/for-administrators/data-use-terms.md b/docs/src/content/docs/for-administrators/data-use-terms.md new file mode 100644 index 0000000000..85441fd9c6 --- /dev/null +++ b/docs/src/content/docs/for-administrators/data-use-terms.md @@ -0,0 +1,6 @@ +--- +title: Data use terms +description: What's the data use terms concept of Loculus and how to configure it. +--- + +TODO \ No newline at end of file From e78983a468cad58bd5732393726b4bcac45c5b67 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 13:10:32 +0100 Subject: [PATCH 28/34] add trailing slash --- docs/astro.config.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index 2d3c98d4a7..86d85cf33a 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -73,7 +73,7 @@ export default defineConfig({ label: 'Build new preprocessing pipeline', link: '/for-administrators/build-new-preprocessing-pipeline/', }, - 'for-administrators/data-use-terms', + { label: 'Data use terms', link: '/for-administrators/data-use-terms/' }, { label: 'User administration', link: '/for-administrators/user-administration/' }, ], }, From 5dd0cfb04601bfcbdb3914a0f6a8eac1073d109b Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 13:17:30 +0100 Subject: [PATCH 29/34] Add docs text --- .../docs/for-administrators/data-use-terms.md | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/src/content/docs/for-administrators/data-use-terms.md b/docs/src/content/docs/for-administrators/data-use-terms.md index 85441fd9c6..79f301cdac 100644 --- a/docs/src/content/docs/for-administrators/data-use-terms.md +++ b/docs/src/content/docs/for-administrators/data-use-terms.md @@ -3,4 +3,31 @@ title: Data use terms description: What's the data use terms concept of Loculus and how to configure it. --- -TODO \ No newline at end of file +Loculus comes with built-in handling of data use terms for submitted data, which means that data can either be _open_ or _restricted_. You can define, what restricted means yourself. Users can submit data as restricted, but they have to give a date at which point the sequences become open, this date can at most be one year from the submission date. + +When data use terms are enabled, you can also filter for only open or only restricted data, and when downloading you will have to accept the data use terms. The same applies to API usage. + +```yaml +dataUseTerms: + enabled: true +``` + +To configure data use terms, you should specify them somewhere, at a URL. +You can then enable data use terms like so: + +```yaml +dataUseTerms: + enabled: true + urls: + open: https://example.org/open + restricted: https://example.org/restricted +``` + +## Disabling data use terms + +To disable data use terms, set `enabled` to `false` (`urls` can be omitted in that case): + +```yaml +dataUseTerms: + enabled: true +``` From 42ca48b207883a42f1dbd755c5ca833a46b5b42b Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 14:49:11 +0100 Subject: [PATCH 30/34] drill through prop & disable tickbox --- .../DownloadDialog/DownloadDialog.tsx | 54 ++++++++++--------- .../components/SearchPage/SearchFullUI.tsx | 7 ++- .../src/pages/[organism]/search/index.astro | 3 +- .../submission/[groupId]/released.astro | 3 +- 4 files changed, 39 insertions(+), 28 deletions(-) diff --git a/website/src/components/SearchPage/DownloadDialog/DownloadDialog.tsx b/website/src/components/SearchPage/DownloadDialog/DownloadDialog.tsx index 9987795cc5..539ed80d3e 100644 --- a/website/src/components/SearchPage/DownloadDialog/DownloadDialog.tsx +++ b/website/src/components/SearchPage/DownloadDialog/DownloadDialog.tsx @@ -15,6 +15,7 @@ type DownloadDialogProps = { sequenceFilter: SequenceFilter; referenceGenomesSequenceNames: ReferenceGenomesSequenceNames; allowSubmissionOfConsensusSequences: boolean; + dataUseTermsEnabled: boolean; }; export const DownloadDialog: FC = ({ @@ -22,6 +23,7 @@ export const DownloadDialog: FC = ({ sequenceFilter, referenceGenomesSequenceNames, allowSubmissionOfConsensusSequences, + dataUseTermsEnabled, }) => { const [isOpen, setIsOpen] = useState(false); @@ -29,7 +31,9 @@ export const DownloadDialog: FC = ({ const closeDialog = () => setIsOpen(false); const [downloadOption, setDownloadOption] = useState(); - const [agreedToDataUseTerms, setAgreedToDataUseTerms] = useState(false); + const [agreedToDataUseTerms, setAgreedToDataUseTerms] = useState(dataUseTermsEnabled ? false : true); + + // TODO remove DUT stuff return ( <> @@ -42,29 +46,31 @@ export const DownloadDialog: FC = ({ onChange={setDownloadOption} allowSubmissionOfConsensusSequences={allowSubmissionOfConsensusSequences} /> -
- -
+ {dataUseTermsEnabled && ( +
+ +
+ )} { if (!hiddenFieldValues) { hiddenFieldValues = {}; @@ -188,7 +190,7 @@ export const InnerSearchFullUI = ({ }; useEffect(() => { - if (showEditDataUseTermsControls) { + if (showEditDataUseTermsControls && showDataUseTermsControls) { setAColumnVisibility(DATA_USE_TERMS_FIELD, true); } }, []); @@ -353,7 +355,7 @@ export const InnerSearchFullUI = ({
- {showEditDataUseTermsControls && ( + {showEditDataUseTermsControls && showDataUseTermsControls && (
diff --git a/website/src/pages/[organism]/search/index.astro b/website/src/pages/[organism]/search/index.astro index 2f3b4dccd0..7475fdeaf7 100644 --- a/website/src/pages/[organism]/search/index.astro +++ b/website/src/pages/[organism]/search/index.astro @@ -1,7 +1,7 @@ --- import { cleanOrganism } from '../../../components/Navigation/cleanOrganism'; import { SearchFullUI } from '../../../components/SearchPage/SearchFullUI'; -import { getRuntimeConfig, getSchema } from '../../../config'; +import { dataUseTermsAreEnabled, getRuntimeConfig, getSchema } from '../../../config'; import BaseLayout from '../../../layouts/BaseLayout.astro'; import { VERSION_STATUS_FIELD, IS_REVOCATION_FIELD } from '../../../settings'; import { versionStatuses } from '../../../types/lapis'; @@ -57,5 +57,6 @@ const { data, totalCount } = await performLapisSearchQueries( initialData={data} initialCount={totalCount} initialQueryDict={initialQueryDict} + showDataUseTermsControls={dataUseTermsAreEnabled()} /> diff --git a/website/src/pages/[organism]/submission/[groupId]/released.astro b/website/src/pages/[organism]/submission/[groupId]/released.astro index 0f1d2edccd..31125810a2 100644 --- a/website/src/pages/[organism]/submission/[groupId]/released.astro +++ b/website/src/pages/[organism]/submission/[groupId]/released.astro @@ -62,6 +62,7 @@ const { data, totalCount } = await performLapisSearchQueries( initialData={data} initialCount={totalCount} initialQueryDict={initialQueryDict} - showEditDataUseTermsControls={dataUseTermsAreEnabled()} + showDataUseTermsControls={dataUseTermsAreEnabled()} + showEditDataUseTermsControls /> From e39de960c04cd11e2b9e46cf616aa3effb2c1bd3 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 14:57:46 +0100 Subject: [PATCH 31/34] remove DUT controls from download --- .../DownloadDialog/DownloadDialog.tsx | 5 +- .../DownloadDialog/DownloadForm.tsx | 46 ++++++++++--------- .../DownloadDialog/DownloadUrlGenerator.ts | 14 +++--- .../components/SearchPage/SearchFullUI.tsx | 12 ++--- .../src/pages/[organism]/search/index.astro | 2 +- .../submission/[groupId]/released.astro | 2 +- 6 files changed, 41 insertions(+), 40 deletions(-) diff --git a/website/src/components/SearchPage/DownloadDialog/DownloadDialog.tsx b/website/src/components/SearchPage/DownloadDialog/DownloadDialog.tsx index 539ed80d3e..bcc356ff28 100644 --- a/website/src/components/SearchPage/DownloadDialog/DownloadDialog.tsx +++ b/website/src/components/SearchPage/DownloadDialog/DownloadDialog.tsx @@ -33,18 +33,17 @@ export const DownloadDialog: FC = ({ const [downloadOption, setDownloadOption] = useState(); const [agreedToDataUseTerms, setAgreedToDataUseTerms] = useState(dataUseTermsEnabled ? false : true); - // TODO remove DUT stuff - return ( <> - +
{dataUseTermsEnabled && (
diff --git a/website/src/components/SearchPage/DownloadDialog/DownloadForm.tsx b/website/src/components/SearchPage/DownloadDialog/DownloadForm.tsx index 673b5046ba..e4653e6f59 100644 --- a/website/src/components/SearchPage/DownloadDialog/DownloadForm.tsx +++ b/website/src/components/SearchPage/DownloadDialog/DownloadForm.tsx @@ -10,12 +10,14 @@ type DownloadFormProps = { referenceGenomesSequenceNames: ReferenceGenomesSequenceNames; onChange: (value: DownloadOption) => void; allowSubmissionOfConsensusSequences: boolean; + dataUseTermsEnabled: boolean; }; export const DownloadForm: FC = ({ referenceGenomesSequenceNames, onChange, allowSubmissionOfConsensusSequences, + dataUseTermsEnabled, }) => { const [includeRestricted, setIncludeRestricted] = useState(0); const [includeOldData, setIncludeOldData] = useState(0); @@ -136,27 +138,29 @@ export const DownloadForm: FC = ({ return (
- No, only download open data }, - { - label: ( - <> - Yes, include restricted data -
({/* TODO(862) */} - - What does it mean? - - ) - - ), - }, - ]} - selected={includeRestricted} - onSelect={setIncludeRestricted} - /> + {dataUseTermsEnabled && ( + No, only download open data }, + { + label: ( + <> + Yes, include restricted data +
({/* TODO(862) */} + + What does it mean? + + ) + + ), + }, + ]} + selected={includeRestricted} + onSelect={setIncludeRestricted} + /> + )} { if (!hiddenFieldValues) { hiddenFieldValues = {}; @@ -190,13 +190,13 @@ export const InnerSearchFullUI = ({ }; useEffect(() => { - if (showEditDataUseTermsControls && showDataUseTermsControls) { + if (showEditDataUseTermsControls && dataUseTermsEnabled) { setAColumnVisibility(DATA_USE_TERMS_FIELD, true); } }, []); const lapisUrl = getLapisUrl(clientConfig, organism); - const downloadUrlGenerator = new DownloadUrlGenerator(organism, lapisUrl); + const downloadUrlGenerator = new DownloadUrlGenerator(organism, lapisUrl, dataUseTermsEnabled); const hooks = lapisClientHooks(lapisUrl).zodiosHooks; const aggregatedHook = hooks.useAggregated({}, {}); @@ -355,7 +355,7 @@ export const InnerSearchFullUI = ({
- {showEditDataUseTermsControls && showDataUseTermsControls && ( + {showEditDataUseTermsControls && dataUseTermsEnabled && (
diff --git a/website/src/pages/[organism]/search/index.astro b/website/src/pages/[organism]/search/index.astro index 7475fdeaf7..d6ee5b765b 100644 --- a/website/src/pages/[organism]/search/index.astro +++ b/website/src/pages/[organism]/search/index.astro @@ -57,6 +57,6 @@ const { data, totalCount } = await performLapisSearchQueries( initialData={data} initialCount={totalCount} initialQueryDict={initialQueryDict} - showDataUseTermsControls={dataUseTermsAreEnabled()} + dataUseTermsEnabled={dataUseTermsAreEnabled()} /> diff --git a/website/src/pages/[organism]/submission/[groupId]/released.astro b/website/src/pages/[organism]/submission/[groupId]/released.astro index 31125810a2..4bacc58512 100644 --- a/website/src/pages/[organism]/submission/[groupId]/released.astro +++ b/website/src/pages/[organism]/submission/[groupId]/released.astro @@ -62,7 +62,7 @@ const { data, totalCount } = await performLapisSearchQueries( initialData={data} initialCount={totalCount} initialQueryDict={initialQueryDict} - showDataUseTermsControls={dataUseTermsAreEnabled()} + dataUseTermsEnabled={dataUseTermsAreEnabled()} showEditDataUseTermsControls /> From 09676fe7c44913b69ab0210bd14cee52e47212c9 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 15:18:49 +0100 Subject: [PATCH 32/34] remove DUT controls from download --- .../DownloadDialog/DownloadDialog.spec.tsx | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/website/src/components/SearchPage/DownloadDialog/DownloadDialog.spec.tsx b/website/src/components/SearchPage/DownloadDialog/DownloadDialog.spec.tsx index e94b28eeaf..1e9b0a952f 100644 --- a/website/src/components/SearchPage/DownloadDialog/DownloadDialog.spec.tsx +++ b/website/src/components/SearchPage/DownloadDialog/DownloadDialog.spec.tsx @@ -24,16 +24,19 @@ const defaultOrganism = 'ebola'; async function renderDialog({ downloadParams = new SelectFilter(new Set()), allowSubmissionOfConsensusSequences = true, + dataUseTermsEnabled = true, }: { downloadParams?: SequenceFilter; allowSubmissionOfConsensusSequences?: boolean; + dataUseTermsEnabled?: boolean; } = {}) { render( , ); @@ -139,6 +142,27 @@ describe('DownloadDialog', () => { expect(screen.getByLabelText(olderVersionsLabel)).toBeInTheDocument(); expect(screen.getByLabelText(gzipCompressionLabel)).toBeInTheDocument(); }); + + describe('DataUseTerms disabled', () => { + test('download button activated by default', async () => { + await renderDialog({ dataUseTermsEnabled: false }); + + const downloadButton = screen.getByRole('link', { name: 'Download' }); + expect(downloadButton).not.toHaveClass('btn-disabled'); + }); + + test('checkbox not in the document', async () => { + await renderDialog({ dataUseTermsEnabled: false }); + + expect(screen.queryByLabelText('I agree to the data use terms.')).not.toBeInTheDocument(); + }); + + test('restricted data switch is not in the document', async () => { + await renderDialog({ dataUseTermsEnabled: false }); + + expect(screen.queryByLabelText(/restricted data/)).not.toBeInTheDocument(); + }); + }); }); async function checkAgreement() { From 609c880e2f4801943866e97a83b91c21f736dd41 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 15:30:25 +0100 Subject: [PATCH 33/34] Add param docs --- .../components/SearchPage/DownloadDialog/DownloadUrlGenerator.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/website/src/components/SearchPage/DownloadDialog/DownloadUrlGenerator.ts b/website/src/components/SearchPage/DownloadDialog/DownloadUrlGenerator.ts index fb369a2d4e..31df1d1ad9 100644 --- a/website/src/components/SearchPage/DownloadDialog/DownloadUrlGenerator.ts +++ b/website/src/components/SearchPage/DownloadDialog/DownloadUrlGenerator.ts @@ -23,6 +23,7 @@ export class DownloadUrlGenerator { * Create new DownloadUrlGenerator with the given properties. * @param organism The organism, will be part of the filename. * @param lapisUrl The lapis API URL for downloading. + * @param dataUseTermsEnabled If false, the downloaded URLs won't include any data use terms related settings. */ constructor( private readonly organism: string, From a10fe167a47de2606a930d3be733c8424032720e Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 4 Feb 2025 16:57:48 +0100 Subject: [PATCH 34/34] don't add DUT URLs if DUT not enabled --- kubernetes/loculus/templates/_common-metadata.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/loculus/templates/_common-metadata.tpl b/kubernetes/loculus/templates/_common-metadata.tpl index b81eaa78fa..289b052e3c 100644 --- a/kubernetes/loculus/templates/_common-metadata.tpl +++ b/kubernetes/loculus/templates/_common-metadata.tpl @@ -90,7 +90,6 @@ fields: displayName: Data use terms restricted until hideOnSequenceDetailsPage: true header: Data use terms - {{- end}} {{- if $.Values.dataUseTerms.urls }} - name: dataUseTermsUrl displayName: Data use terms URL @@ -101,6 +100,7 @@ fields: type: link url: "__value__" {{- end}} + {{- end}} - name: versionStatus type: string notSearchable: true