feat!: Allow disabling data use terms

backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt

@@ -1,5 +1,6 @@
 package org.loculus.backend.config
 
+import com.fasterxml.jackson.databind.DeserializationFeature
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
 import io.swagger.v3.oas.models.headers.Header
@@ -146,7 +147,9 @@ internal fun validateEarliestReleaseDateFields(config: BackendConfig): List<Stri
 }
 
 fun readBackendConfig(objectMapper: ObjectMapper, configPath: String): BackendConfig {
-    val config = objectMapper.readValue<BackendConfig>(File(configPath))
+    val config = objectMapper
+        .enable(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES)
+        .readValue<BackendConfig>(File(configPath))
     logger.info { "Loaded backend config from $configPath" }
     logger.info { "Config: $config" }
     val validationErrors = validateEarliestReleaseDateFields(config)

backend/src/main/kotlin/org/loculus/backend/config/Config.kt

@@ -7,13 +7,15 @@ import org.loculus.backend.api.Organism
 data class BackendConfig(
     val organisms: Map<String, InstanceConfig>,
     val accessionPrefix: String,
-    val dataUseTermsUrls: DataUseTermsUrls?,
+    val dataUseTerms: DataUseTerms,
 ) {
     fun getInstanceConfig(organism: Organism) = organisms[organism.name] ?: throw IllegalArgumentException(
         "Organism: ${organism.name} not found in backend config. Available organisms: ${organisms.keys}",
     )
 }
 
+data class DataUseTerms(val enabled: Boolean, val urls: DataUseTermsUrls?)
+
 data class DataUseTermsUrls(val open: String, val restricted: String)
 
 data class InstanceConfig(val schema: Schema, val referenceGenomes: ReferenceGenome)

backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt

@@ -33,6 +33,7 @@ import org.loculus.backend.api.SubmittedProcessedData
 import org.loculus.backend.api.UnprocessedData
 import org.loculus.backend.auth.AuthenticatedUser
 import org.loculus.backend.auth.HiddenParam
+import org.loculus.backend.config.BackendConfig
 import org.loculus.backend.controller.LoculusCustomHeaders.X_TOTAL_RECORDS
 import org.loculus.backend.log.REQUEST_ID_MDC_KEY
 import org.loculus.backend.log.RequestIdContext
@@ -76,33 +77,45 @@ open class SubmissionController(
     private val submissionDatabaseService: SubmissionDatabaseService,
     private val iteratorStreamer: IteratorStreamer,
     private val requestIdContext: RequestIdContext,
+    private val backendConfig: BackendConfig,
 ) {
-
     @Operation(description = SUBMIT_DESCRIPTION)
     @ApiResponse(responseCode = "200", description = SUBMIT_RESPONSE_DESCRIPTION)
+    @ApiResponse(responseCode = "400", description = SUBMIT_ERROR_RESPONSE)
     @PostMapping("/submit", consumes = ["multipart/form-data"])
     fun submit(
         @PathVariable @Valid organism: Organism,
         @HiddenParam authenticatedUser: AuthenticatedUser,
         @Parameter(description = GROUP_ID_DESCRIPTION) @RequestParam groupId: Int,
         @Parameter(description = METADATA_FILE_DESCRIPTION) @RequestParam metadataFile: MultipartFile,
         @Parameter(description = SEQUENCE_FILE_DESCRIPTION) @RequestParam sequenceFile: MultipartFile?,
-        @Parameter(description = "Data Use terms under which data is released.") @RequestParam dataUseTermsType:
-        DataUseTermsType,
+        @Parameter(
+            description =
+            "Data Use terms under which data is released. Mandatory when data use terms are enabled for this Instance.",
+        ) @RequestParam dataUseTermsType: DataUseTermsType?,
         @Parameter(
             description =
             "Mandatory when data use terms are set to 'RESTRICTED'." +
                 " It is the date when the sequence entries will become 'OPEN'." +
                 " Format: YYYY-MM-DD",
         ) @RequestParam restrictedUntil: String?,
     ): List<SubmissionIdMapping> {
+        var dataUseTermsKind = DataUseTermsType.OPEN
+        if (backendConfig.dataUseTerms.enabled) {
+            if (dataUseTermsType == null) {
+                throw BadRequestException("the 'dataUseTermsType' needs to be provided.")
+            } else {
+                dataUseTermsKind = dataUseTermsType
+            }
+        }
+
         val params = SubmissionParams.OriginalSubmissionParams(
             organism,
             authenticatedUser,
             metadataFile,
             sequenceFile,
             groupId,
-            DataUseTerms.fromParameters(dataUseTermsType, restrictedUntil),
+            DataUseTerms.fromParameters(dataUseTermsKind, restrictedUntil),
         )
         return submitModel.processSubmissions(UUID.randomUUID().toString(), params)
     }

backend/src/main/kotlin/org/loculus/backend/controller/SubmissionControllerDescriptions.kt

@@ -8,6 +8,10 @@ The accession is the (globally unique) id that the system assigned to the sequen
 You can use this response to associate the user provided submissionId with the system assigned accession.
 """
 
+const val SUBMIT_ERROR_RESPONSE = """
+The data use terms type have not been provided, even though they are enabled for this Loculus instance.
+"""
+
 const val METADATA_FILE_DESCRIPTION = """    
 A TSV (tab separated values) file containing the metadata of the submitted sequence entries. 
 The file may be compressed with zstd, xz, zip, gzip, lzma, bzip2 (with common extensions).

backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt

@@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.node.TextNode
 import mu.KotlinLogging
 import org.loculus.backend.api.DataUseTerms
 import org.loculus.backend.api.GeneticSequence
+import org.loculus.backend.api.MetadataMap
 import org.loculus.backend.api.Organism
 import org.loculus.backend.api.ProcessedData
 import org.loculus.backend.api.VersionStatus
@@ -94,6 +95,9 @@ open class ReleasedDataModel(
         return "\"$lastUpdateTime\"" // ETag must be enclosed in double quotes
     }
 
+    private fun conditionalMetadata(condition: Boolean, values: () -> MetadataMap): MetadataMap =
+        if (condition) values() else emptyMap()
+
     private fun computeAdditionalMetadataFields(
         rawProcessedData: RawProcessedData,
         latestVersions: Map<Accession, Version>,
@@ -111,6 +115,13 @@ open class ReleasedDataModel(
 
         val earliestReleaseDate = earliestReleaseDateFinder?.calculateEarliestReleaseDate(rawProcessedData)
 
+        val dataUseTermsUrl: String? = backendConfig.dataUseTerms.urls?.let { urls ->
+            when (currentDataUseTerms) {
+                DataUseTerms.Open -> urls.open
+                is DataUseTerms.Restricted -> urls.restricted
+            }
+        }
+
         var metadata = rawProcessedData.processedData.metadata +
             mapOf(
                 ("accession" to TextNode(rawProcessedData.accession)),
@@ -126,31 +137,41 @@ open class ReleasedDataModel(
                 ("releasedAtTimestamp" to LongNode(rawProcessedData.releasedAtTimestamp.toTimestamp())),
                 ("releasedDate" to TextNode(rawProcessedData.releasedAtTimestamp.toUtcDateString())),
                 ("versionStatus" to TextNode(versionStatus.name)),
-                ("dataUseTerms" to TextNode(currentDataUseTerms.type.name)),
-                ("dataUseTermsRestrictedUntil" to restrictedDataUseTermsUntil),
                 ("pipelineVersion" to LongNode(rawProcessedData.pipelineVersion)),
             ) +
-            if (rawProcessedData.isRevocation) {
-                mapOf("versionComment" to TextNode(rawProcessedData.versionComment))
-            } else {
-                emptyMap()
-            }.let {
-                when (backendConfig.dataUseTermsUrls) {
-                    null -> it
-                    else -> {
-                        val url = when (currentDataUseTerms) {
-                            DataUseTerms.Open -> backendConfig.dataUseTermsUrls.open
-                            is DataUseTerms.Restricted -> backendConfig.dataUseTermsUrls.restricted
-                        }
-                        it + ("dataUseTermsUrl" to TextNode(url))
-                    }
-                }
-            } +
-            if (earliestReleaseDate != null) {
-                mapOf("earliestReleaseDate" to TextNode(earliestReleaseDate.toUtcDateString()))
-            } else {
-                emptyMap()
-            }
+            conditionalMetadata(
+                backendConfig.dataUseTerms.enabled,
+                {
+                    mapOf(
+                        "dataUseTerms" to TextNode(currentDataUseTerms.type.name),
+                        "dataUseTermsRestrictedUntil" to restrictedDataUseTermsUntil,
+                    )
+                },
+            ) +
+            conditionalMetadata(
+                rawProcessedData.isRevocation,
+                {
+                    mapOf(
+                        "versionComment" to TextNode(rawProcessedData.versionComment),
+                    )
+                },
+            ) +
+            conditionalMetadata(
+                earliestReleaseDate != null,
+                {
+                    mapOf(
+                        "earliestReleaseDate" to TextNode(earliestReleaseDate!!.toUtcDateString()),
+                    )
+                },
+            ) +
+            conditionalMetadata(
+                dataUseTermsUrl != null,
+                {
+                    mapOf(
+                        "dataUseTermsUrl" to TextNode(dataUseTermsUrl!!),
+                    )
+                },
+            )
 
         return ProcessedData(
             metadata = metadata,

backend/src/test/kotlin/org/loculus/backend/config/BackendSpringConfigTest.kt

@@ -70,5 +70,5 @@ fun backendConfig(metadataList: List<Metadata>, earliestReleaseDate: EarliestRel
         ),
     ),
     accessionPrefix = "FOO_",
-    dataUseTermsUrls = null,
+    dataUseTerms = DataUseTerms(true, null),
 )

backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt

@@ -31,6 +31,12 @@ import org.springframework.test.annotation.DirtiesContext
 import org.springframework.test.context.ActiveProfiles
 import org.testcontainers.containers.PostgreSQLContainer
 
+/**
+ * The main annotation for tests. It also loads the [EndpointTestExtension], which initializes
+ * a PostgreSQL test container.
+ * You can set additional properties to - for example - override the backend config file, like in
+ * [org.loculus.backend.controller.submission.GetReleasedDataDataUseTermsDisabledEndpointTest].
+ */
 @Target(AnnotationTarget.TYPE, AnnotationTarget.CLASS)
 @Retention(AnnotationRetention.RUNTIME)
 @AutoConfigureMockMvc
@@ -48,6 +54,10 @@ import org.testcontainers.containers.PostgreSQLContainer
 )
 annotation class EndpointTest(@get:AliasFor(annotation = SpringBootTest::class) val properties: Array<String> = [])
 
+const val SINGLE_SEGMENTED_REFERENCE_GENOME = "src/test/resources/backend_config_single_segment.json"
+
+const val DATA_USE_TERMS_DISABLED_CONFIG = "src/test/resources/backend_config_data_use_terms_disabled.json"
+
 private const val SPRING_DATASOURCE_URL = "spring.datasource.url"
 private const val SPRING_DATASOURCE_USERNAME = "spring.datasource.username"
 private const val SPRING_DATASOURCE_PASSWORD = "spring.datasource.password"

backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataDataUseTermsDisabledEndpointTest.kt

@@ -0,0 +1,141 @@
+package org.loculus.backend.controller.submission
+
+import com.fasterxml.jackson.databind.node.BooleanNode
+import com.fasterxml.jackson.databind.node.IntNode
+import com.fasterxml.jackson.databind.node.TextNode
+import com.ninjasquad.springmockk.MockkBean
+import io.mockk.every
+import kotlinx.datetime.Clock
+import kotlinx.datetime.toLocalDateTime
+import org.hamcrest.CoreMatchers.hasItem
+import org.hamcrest.CoreMatchers.`is`
+import org.hamcrest.CoreMatchers.not
+import org.hamcrest.MatcherAssert.assertThat
+import org.hamcrest.Matchers.matchesPattern
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+import org.keycloak.representations.idm.UserRepresentation
+import org.loculus.backend.api.GeneticSequence
+import org.loculus.backend.api.ProcessedData
+import org.loculus.backend.config.BackendConfig
+import org.loculus.backend.config.BackendSpringProperty
+import org.loculus.backend.controller.DATA_USE_TERMS_DISABLED_CONFIG
+import org.loculus.backend.controller.DEFAULT_GROUP
+import org.loculus.backend.controller.DEFAULT_GROUP_CHANGED
+import org.loculus.backend.controller.DEFAULT_GROUP_NAME_CHANGED
+import org.loculus.backend.controller.DEFAULT_PIPELINE_VERSION
+import org.loculus.backend.controller.DEFAULT_USER_NAME
+import org.loculus.backend.controller.EndpointTest
+import org.loculus.backend.controller.expectNdjsonAndGetContent
+import org.loculus.backend.controller.groupmanagement.GroupManagementControllerClient
+import org.loculus.backend.controller.groupmanagement.andGetGroupId
+import org.loculus.backend.controller.jwtForDefaultUser
+import org.loculus.backend.controller.submission.SubmitFiles.DefaultFiles.NUMBER_OF_SEQUENCES
+import org.loculus.backend.service.KeycloakAdapter
+import org.loculus.backend.utils.DateProvider
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header
+import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status
+
+@EndpointTest(
+    properties = ["${BackendSpringProperty.BACKEND_CONFIG_PATH}=$DATA_USE_TERMS_DISABLED_CONFIG"],
+)
+class GetReleasedDataDataUseTermsDisabledEndpointTest(
+    @Autowired private val convenienceClient: SubmissionConvenienceClient,
+    @Autowired private val submissionControllerClient: SubmissionControllerClient,
+    @Autowired private val groupClient: GroupManagementControllerClient,
+    @Autowired private val backendConfig: BackendConfig,
+) {
+    private val currentDate = Clock.System.now().toLocalDateTime(DateProvider.timeZone).date.toString()
+
+    @MockkBean
+    lateinit var keycloakAdapter: KeycloakAdapter
+
+    @BeforeEach
+    fun setup() {
+        every { keycloakAdapter.getUsersWithName(any()) } returns listOf(UserRepresentation())
+    }
+
+    @Test
+    fun `config has been read and data use terms are configured to be off`() {
+        assertThat(backendConfig.dataUseTerms.enabled, `is`(false))
+    }
+
+    @Test
+    fun `GIVEN released data exists THEN NOT returns data use terms properties`() {
+        val groupId = groupClient.createNewGroup(group = DEFAULT_GROUP, jwt = jwtForDefaultUser)
+            .andExpect(status().isOk)
+            .andGetGroupId()
+
+        convenienceClient.prepareDefaultSequenceEntriesToApprovedForRelease(groupId = groupId)
+
+        val response = submissionControllerClient.getReleasedData()
+
+        val responseBody = response.expectNdjsonAndGetContent<ProcessedData<GeneticSequence>>()
+
+        responseBody.forEach {
+            assertThat(it.metadata.keys, not(hasItem("dataUseTerms")))
+            assertThat(it.metadata.keys, not(hasItem("dataUseTermsRestrictedUntil")))
+        }
+    }
+
+    @Test
+    fun `GIVEN released data exists THEN returns with additional metadata fields & no data use terms properties`() {
+        val groupId = groupClient.createNewGroup(group = DEFAULT_GROUP, jwt = jwtForDefaultUser)
+            .andExpect(status().isOk)
+            .andGetGroupId()
+
+        convenienceClient.prepareDefaultSequenceEntriesToApprovedForRelease(groupId = groupId)
+
+        groupClient.updateGroup(
+            groupId = groupId,
+            group = DEFAULT_GROUP_CHANGED,
+            jwt = jwtForDefaultUser,
+        ).andExpect(status().isOk)
+
+        val response = submissionControllerClient.getReleasedData()
+
+        val responseBody = response.expectNdjsonAndGetContent<ProcessedData<GeneticSequence>>()
+
+        assertThat(responseBody.size, `is`(NUMBER_OF_SEQUENCES))
+
+        response.andExpect(header().string("x-total-records", NUMBER_OF_SEQUENCES.toString()))
+
+        responseBody.forEach {
+            val id = it.metadata["accession"]!!.asText()
+            val version = it.metadata["version"]!!.asLong()
+            assertThat(version, `is`(1))
+
+            val expectedMetadata = defaultProcessedData.metadata + mapOf(
+                "accession" to TextNode(id),
+                "version" to IntNode(version.toInt()),
+                "accessionVersion" to TextNode("$id.$version"),
+                "isRevocation" to BooleanNode.FALSE,
+                "submitter" to TextNode(DEFAULT_USER_NAME),
+                "groupName" to TextNode(DEFAULT_GROUP_NAME_CHANGED),
+                "versionStatus" to TextNode("LATEST_VERSION"),
+                "releasedDate" to TextNode(currentDate),
+                "submittedDate" to TextNode(currentDate),
+                "pipelineVersion" to IntNode(DEFAULT_PIPELINE_VERSION.toInt()),
+            )
+
+            for ((key, value) in it.metadata) {
+                when (key) {
+                    "submittedAtTimestamp" -> expectIsTimestampWithCurrentYear(value)
+                    "releasedAtTimestamp" -> expectIsTimestampWithCurrentYear(value)
+                    "submissionId" -> assertThat(value.textValue(), matchesPattern("^custom\\d$"))
+                    "groupId" -> assertThat(value.intValue(), `is`(groupId))
+                    else -> {
+                        assertThat(expectedMetadata.keys, hasItem(key))
+                        assertThat(value, `is`(expectedMetadata[key]))
+                    }
+                }
+            }
+            assertThat(it.alignedNucleotideSequences, `is`(defaultProcessedData.alignedNucleotideSequences))
+            assertThat(it.unalignedNucleotideSequences, `is`(defaultProcessedData.unalignedNucleotideSequences))
+            assertThat(it.alignedAminoAcidSequences, `is`(defaultProcessedData.alignedAminoAcidSequences))
+            assertThat(it.nucleotideInsertions, `is`(defaultProcessedData.nucleotideInsertions))
+            assertThat(it.aminoAcidInsertions, `is`(defaultProcessedData.aminoAcidInsertions))
+        }
+    }
+}