Fix Dataset Configurations

CHANGELOG.unreleased.md

@@ -18,6 +18,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 
 ### Fixed
 - Fixed failing histogram requests for float layers with NaN values. [#4834](https://github.com/scalableminds/webknossos/pull/4834)
+- Fixed the disappearing of dataset settings after switching between view mode and trace mode. [#4845](https://github.com/scalableminds/webknossos/pull/4845)
 
 ### Removed
 -

MIGRATIONS.unreleased.md

@@ -9,4 +9,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 -
 
 ### Postgres Evolutions:
-- 
+- [056-add-layer-specific-view-configs.sql](conf/evolutions/056-add-layer-specific-view-configs.sql)

app/controllers/ConfigurationController.scala

@@ -4,23 +4,26 @@ import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContex
 import com.scalableminds.util.tools.Fox
 import javax.inject.Inject
 import models.binary.{DataSet, DataSetDAO, DataSetService}
-import models.configuration.{DataSetConfiguration, DataSetConfigurationDefaults, UserConfiguration}
-import models.user.{UserDataSetConfigurationDAO, UserService}
+import models.configuration.{DataSetConfiguration, DataSetConfigurationDefaults, DataSetLayerId, UserConfiguration}
+import models.user.{UserDataSetConfigurationDAO, UserDataSetLayerConfigurationDAO, UserService}
 import oxalis.security.WkEnv
 import com.mohiva.play.silhouette.api.Silhouette
 import com.mohiva.play.silhouette.api.actions.{SecuredRequest, UserAwareRequest}
 import play.api.i18n.{Messages, MessagesApi}
 import play.api.libs.json.{JsObject, JsValue}
 import play.api.libs.json.Json._
+import play.api.mvc.PlayBodyParsers
 
 import scala.concurrent.ExecutionContext
 
-class ConfigurationController @Inject()(userService: UserService,
-                                        dataSetService: DataSetService,
-                                        dataSetDAO: DataSetDAO,
-                                        userDataSetConfigurationDAO: UserDataSetConfigurationDAO,
-                                        dataSetConfigurationDefaults: DataSetConfigurationDefaults,
-                                        sil: Silhouette[WkEnv])(implicit ec: ExecutionContext)
+class ConfigurationController @Inject()(
+    userService: UserService,
+    dataSetService: DataSetService,
+    dataSetDAO: DataSetDAO,
+    userDataSetConfigurationDAO: UserDataSetConfigurationDAO,
+    userDataSetLayerConfigurationDAO: UserDataSetLayerConfigurationDAO,
+    dataSetConfigurationDefaults: DataSetConfigurationDefaults,
+    sil: Silhouette[WkEnv])(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers)
     extends Controller {
 
   def read = sil.UserAwareAction.async { implicit request =>
@@ -41,30 +44,45 @@ class ConfigurationController @Inject()(userService: UserService,
     }
   }
 
-  def readDataSet(organizationName: String, dataSetName: String) = sil.UserAwareAction.async { implicit request =>
-    request.identity.toFox.flatMap { user =>
-      for {
-        configurationJson: JsValue <- userDataSetConfigurationDAO.findOneForUserAndDataset(user._id, dataSetName)
-      } yield DataSetConfiguration(configurationJson.validate[Map[String, JsValue]].getOrElse(Map.empty))
-    }.orElse(
+  def readDataSet(organizationName: String, dataSetName: String) =
+    sil.UserAwareAction.async(validateJson[List[DataSetLayerId]]) { implicit request =>
+      request.identity.toFox.flatMap { user =>
         for {
+          configurationJson: JsValue <- userDataSetConfigurationDAO.findOneForUserAndDataset(user._id, dataSetName)
+          initialConfigurationMap = configurationJson.validate[Map[String, JsValue]].getOrElse(Map.empty)
+          layerConfigJson <- userDataSetLayerConfigurationDAO.findAllByLayerNameForUserAndDataset(
+            request.body.map(_.name),
+            user._id,
+            dataSetName)
           dataSet <- dataSetDAO.findOneByNameAndOrganizationName(dataSetName, organizationName)(GlobalAccessContext)
-          config <- dataSetConfigurationDefaults.constructInitialDefault(dataSet)
-        } yield config
-      )
-      .getOrElse(dataSetConfigurationDefaults.constructInitialDefault(List()))
-      .map(configuration => Ok(toJson(dataSetConfigurationDefaults.configurationOrDefaults(configuration))))
-  }
+          layerSourceDefaultViewConfigs <- dataSetConfigurationDefaults.getAllLayerSourceDefaultViewConfigForDataSet(
+            dataSet)
+          layerSettings = dataSetConfigurationDefaults.layerConfigurationOrDefaults(request.body,
+                                                                                    layerConfigJson,
+                                                                                    layerSourceDefaultViewConfigs)
+        } yield dataSetConfigurationDefaults.buildCompleteConfig(initialConfigurationMap, layerSettings)
+      }.orElse(
+          for {
+            dataSet <- dataSetDAO.findOneByNameAndOrganizationName(dataSetName, organizationName)(GlobalAccessContext)
+            config <- dataSetConfigurationDefaults.constructInitialDefault(dataSet)
+          } yield config
+        )
+        .getOrElse(dataSetConfigurationDefaults.constructInitialDefault(List()))
+        .map(configuration => Ok(toJson(configuration.configuration)))
+    }
 
   def updateDataSet(organizationName: String, dataSetName: String) =
     sil.SecuredAction.async(parse.json(maxLength = 20480)) { implicit request =>
       for {
         jsConfiguration <- request.body.asOpt[JsObject] ?~> "user.configuration.dataset.invalid"
         conf = jsConfiguration.fields.toMap
+        dataSetConf = conf - "layers"
+        layerConf = conf.get("layers")
         _ <- userService.updateDataSetConfiguration(request.identity,
                                                     dataSetName,
                                                     organizationName,
-                                                    DataSetConfiguration(conf))
+                                                    DataSetConfiguration(dataSetConf),
+                                                    layerConf)
       } yield {
         JsonOk(Messages("user.configuration.dataset.updated"))
       }
@@ -77,9 +95,7 @@ class ConfigurationController @Inject()(userService: UserService,
           Fox.successful(
             Ok(toJson(dataSetConfigurationDefaults.configurationOrDefaults(c, dataSet.sourceDefaultConfiguration))))
         case _ =>
-          dataSetConfigurationDefaults
-            .constructInitialDefault(dataSet)
-            .map(c => Ok(toJson(dataSetConfigurationDefaults.configurationOrDefaults(c))))
+          dataSetConfigurationDefaults.constructInitialDefault(dataSet).map(c => Ok(toJson(c.configuration)))
       }
     }
   }

app/models/configuration/DataSetConfiguration.scala

@@ -7,6 +7,9 @@ import javax.inject.Inject
 import models.binary.{DataSet, DataSetService}
 import play.api.libs.json._
 
+case class DataSetLayerId(name: String, isSegmentationLayer: Boolean)
+object DataSetLayerId { implicit val dataSetLayerId = Json.format[DataSetLayerId] }
+
 case class DataSetConfiguration(configuration: Map[String, JsValue])
 
 object DataSetConfiguration { implicit val dataSetConfigurationFormat = Json.format[DataSetConfiguration] }
@@ -40,7 +43,6 @@ class DataSetConfigurationDefaults @Inject()(dataSetService: DataSetService) {
     DataSetConfiguration(
       Map(
         "fourBit" -> JsBoolean(false),
-        "quality" -> JsNumber(0),
         "interpolation" -> JsBoolean(true),
         "highlightHoveredCellId" -> JsBoolean(true),
         "renderMissingDataBlack" -> JsBoolean(true),
@@ -55,6 +57,33 @@ class DataSetConfigurationDefaults @Inject()(dataSetService: DataSetService) {
       sourceDefaultConfiguration.map(_.toMap).getOrElse(Map.empty) ++
       configuration.configuration
 
+  def layerConfigurationOrDefaults(requestedLayer: List[DataSetLayerId],
+                                   existingLayerConfiguration: Map[String, JsValue],
+                                   sourceDefaultConfiguration: Map[String, JsValue]) =
+    requestedLayer.map {
+      case DataSetLayerId(name, isSegmentationLayer) =>
+        (name,
+         existingLayerConfiguration.getOrElse(
+           name,
+           sourceDefaultConfiguration.getOrElse(name,
+                                                Json.toJson(
+                                                  if (isSegmentationLayer) initialDefaultPerSegmentationLayer
+                                                  else initialDefaultPerColorLayer))))
+    }.toMap
+
+  def buildCompleteConfig(initialConfiguration: Map[String, JsValue], layerConfigurations: Map[String, JsValue]) =
+    DataSetConfiguration(initialConfiguration + ("layers" -> Json.toJson(layerConfigurations)))
+
+  def getAllLayerSourceDefaultViewConfigForDataSet(dataSet: DataSet)(
+      implicit ctx: DBAccessContext): Fox[Map[String, JsValue]] =
+    for {
+
+      dataSource <- dataSetService.dataSourceFor(dataSet)
+      dataLayers = dataSource.toUsable.map(d => d.dataLayers).getOrElse(List())
+      layerSourceDefaultViewConfig = dataLayers.flatMap(dl =>
+        dl.defaultViewConfiguration.map(c => (dl.name, Json.toJson(c.toMap))))
+    } yield layerSourceDefaultViewConfig.toMap
+
   val initialDefaultPerColorLayer: Map[String, JsValue] = Map(
     "brightness" -> JsNumber(0),
     "contrast" -> JsNumber(1),

app/models/user/User.scala

@@ -323,7 +323,7 @@ class UserDataSetConfigurationDAO @Inject()(sqlClient: SQLClient, userDAO: UserD
               where d.name = ${dataSetName}
               and c._user = ${userId}
           """.as[String])
-      parsed = rows.map(Json.parse(_))
+      parsed = rows.map(Json.parse)
       result <- parsed.headOption.toFox
     } yield result
 
@@ -370,3 +370,37 @@ class UserDataSetConfigurationDAO @Inject()(sqlClient: SQLClient, userDAO: UserD
     } yield ()
 
 }
+
+class UserDataSetLayerConfigurationDAO @Inject()(sqlClient: SQLClient, userDAO: UserDAO)(implicit ec: ExecutionContext)
+    extends SimpleSQLDAO(sqlClient) {
+
+  def findAllByLayerNameForUserAndDataset(layerNames: List[String], userId: ObjectId, dataSetName: String)(
+      implicit ctx: DBAccessContext): Fox[Map[String, JsValue]] =
+    for {
+      rows <- run(sql"""select layerName, configuration
+              from webknossos.user_dataSetLayerConfigurations
+              where _dataset in (select _id from webknossos.dataSets_ where name = $dataSetName)
+              and _user = $userId
+              and layerName in #${writeStructTupleWithQuotes(layerNames)}
+          """.as[(String, String)])
+      parsed = rows.map(t => (t._1, Json.parse(t._2)))
+    } yield parsed.toMap
+
+  def updateDatasetConfigurationForUserAndDatasetAndLayer(
+      userId: ObjectId,
+      dataSetId: ObjectId,
+      layerName: String,
+      configuration: JsValue)(implicit ctx: DBAccessContext): Fox[Unit] =
+    for {
+      _ <- userDAO.assertUpdateAccess(userId)
+      deleteQuery = sqlu"""delete from webknossos.user_dataSetLayerConfigurations
+               where _user = $userId and _dataSet = $dataSetId and layerName = $layerName"""
+      insertQuery = sqlu"""insert into webknossos.user_dataSetLayerConfigurations(_user, _dataSet, layerName, configuration)
+               values($userId, $dataSetId, $layerName, '#${sanitize(configuration.toString)}')"""
+      _ <- run(
+        DBIO.sequence(List(deleteQuery, insertQuery)).transactionally.withTransactionIsolation(Serializable),
+        retryCount = 50,
+        retryIfErrorContains = List(transactionSerializationError)
+      )
+    } yield ()
+}

app/models/user/UserService.scala

@@ -28,6 +28,7 @@ class UserService @Inject()(conf: WkConf,
                             userTeamRolesDAO: UserTeamRolesDAO,
                             userExperiencesDAO: UserExperiencesDAO,
                             userDataSetConfigurationDAO: UserDataSetConfigurationDAO,
+                            userDataSetLayerConfigurationDAO: UserDataSetLayerConfigurationDAO,
                             organizationDAO: OrganizationDAO,
                             teamDAO: TeamDAO,
                             teamMembershipService: TeamMembershipService,
@@ -146,15 +147,24 @@ class UserService @Inject()(conf: WkConf,
       user: User,
       dataSetName: String,
       organizationName: String,
-      configuration: DataSetConfiguration)(implicit ctx: DBAccessContext, m: MessagesProvider) =
+      dataSetConfiguration: DataSetConfiguration,
+      layerConfiguration: Option[JsValue])(implicit ctx: DBAccessContext, m: MessagesProvider) =
     for {
       dataSet <- dataSetDAO.findOneByNameAndOrganizationName(dataSetName, organizationName) ?~> Messages(
         "dataSet.notFound",
         dataSetName)
+      layerMap = layerConfiguration.map(_.validate[Map[String, JsValue]].getOrElse(Map.empty)).getOrElse(Map.empty)
+      _ <- Fox.serialCombined(layerMap.toList) {
+        case (name, config) =>
+          userDataSetLayerConfigurationDAO.updateDatasetConfigurationForUserAndDatasetAndLayer(user._id,
+                                                                                               dataSet._id,
+                                                                                               name,
+                                                                                               config)
+      }
       _ <- userDataSetConfigurationDAO.updateDatasetConfigurationForUserAndDataset(user._id,
                                                                                    dataSet._id,
-                                                                                   configuration.configuration)
-      _ = userCache.invalidateUser(user._id)
+                                                                                   dataSetConfiguration.configuration)
+      _ = userCache.invalidateUser(user._id) // TODO Why do we need to invalidate the user here?
     } yield ()
 
   def updateLastTaskTypeId(user: User, lastTaskTypeId: Option[String])(implicit ctx: DBAccessContext) =

conf/evolutions/056-add-layer-specific-view-configs.sql

@@ -0,0 +1,36 @@
+-- https://github.com/scalableminds/webknossos/pull/4XXX
+
+START TRANSACTION;
+
+CREATE TABLE webknossos.user_dataSetLayerConfigurations(
+  _user CHAR(24) NOT NULL,
+  _dataSet CHAR(24) NOT NULL,
+  layerName VARCHAR(256) NOT NULL,
+  configuration JSONB NOT NULL,
+  PRIMARY KEY (_user, _dataSet, layerName),
+  CONSTRAINT configurationIsJsonObject CHECK(jsonb_typeof(configuration) = 'object')
+);
+
+ALTER TABLE webknossos.user_dataSetLayerConfigurations
+  ADD CONSTRAINT user_ref FOREIGN KEY(_user) REFERENCES webknossos.users(_id) ON DELETE CASCADE DEFERRABLE,
+  ADD CONSTRAINT dataSet_ref FOREIGN KEY(_dataSet) REFERENCES webknossos.dataSets(_id) ON DELETE CASCADE DEFERRABLE;
+
+-- Insert the layer configs into the new table
+INSERT INTO webknossos.user_dataSetLayerConfigurations(_user, _dataset, layerName, configuration)
+SELECT _user, _dataset, (js).key AS layerName, (js).value AS config FROM (SELECT _user, _dataset, jsonb_each(configuration->'layers') AS js FROM webknossos.user_dataSetConfigurations WHERE configuration ? 'layers') AS sub_q;
+
+
+-- Remove layers field from old table
+UPDATE webknossos.user_dataSetConfigurations
+SET configuration = configuration - 'layers'
+WHERE configuration ? 'layers';
+
+--remove unused field quality
+UPDATE webknossos.user_dataSetConfigurations
+SET configuration = configuration - 'quality'
+WHERE configuration ? 'quality';
+
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 56;
+
+COMMIT TRANSACTION;

conf/evolutions/reversions/056-add-layer-specific-view-configs.sql

@@ -0,0 +1,17 @@
+
+START TRANSACTION;
+
+-- pre-aggregate all layer jsons into one json object, set json object in configuration
+UPDATE webknossos.user_dataSetConfigurations dsC
+SET configuration = jsonb_set(
+        dsC.configuration,
+        array['layers'],
+        subQ.json)
+FROM (select _user, _dataset, jsonb_object_agg(layerName, configuration) AS json FROM webknossos.user_dataSetLayerConfigurations dlC GROUP BY _user, _dataset) AS subQ
+WHERE dsC._user = subQ._user and dsC._dataset = subQ._dataset;
+
+DROP TABLE webknossos.user_dataSetLayerConfigurations;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 55;
+
+COMMIT TRANSACTION;

conf/webknossos.latest.routes

@@ -28,7 +28,7 @@ POST          /auth/createOrganizationWithAdmin
 # Configurations
 GET           /user/userConfiguration                                           controllers.ConfigurationController.read
 PUT           /user/userConfiguration                                           controllers.ConfigurationController.update
-GET           /dataSetConfigurations/:organizationName/:dataSetName             controllers.ConfigurationController.readDataSet(organizationName: String, dataSetName: String)
+POST          /dataSetConfigurations/:organizationName/:dataSetName             controllers.ConfigurationController.readDataSet(organizationName: String, dataSetName: String)
 PUT           /dataSetConfigurations/:organizationName/:dataSetName             controllers.ConfigurationController.updateDataSet(organizationName: String, dataSetName: String)
 GET           /dataSetConfigurations/default/:organizationName/:dataSetName     controllers.ConfigurationController.readDataSetDefault(organizationName: String, dataSetName: String)
 PUT           /dataSetConfigurations/default/:organizationName/:dataSetName     controllers.ConfigurationController.updateDataSetDefault(organizationName: String, dataSetName: String)

frontend/javascripts/admin/admin_rest_api.js

@@ -23,6 +23,7 @@ import {
   type APIProjectProgressReport,
   type APIProjectUpdater,
   type APIProjectWithAssignments,
+  type APIReducedDataLayer,
   type APISampleDataset,
   type APIScript,
   type APIScriptCreator,
@@ -783,9 +784,16 @@ export function updateDataset(datasetId: APIDatasetId, dataset: APIDataset): Pro
   );
 }
 
-export function getDatasetConfiguration(datasetId: APIDatasetId): Promise<Object> {
-  return Request.receiveJSON(
+export function getDatasetViewConfiguration(
+  datasetId: APIDatasetId,
+  displayedLayers: Array<APIReducedDataLayer>,
+): Promise<Object> {
+  return Request.sendJSONReceiveJSON(
     `/api/dataSetConfigurations/${datasetId.owningOrganization}/${datasetId.name}`,
+    {
+      data: displayedLayers,
+      method: "POST",
+    },
   );
 }
 

frontend/javascripts/admin/api_flow_types.js

@@ -62,6 +62,11 @@ export type APISegmentationLayer = {|
 
 export type APIDataLayer = APIColorLayer | APISegmentationLayer;
 
+export type APIReducedDataLayer = {|
+  +name: string,
+  +isSegmentationLayer: boolean,
+|};
+
 export type APIHistogramData = Array<{
   numberOfElements: number,
   elementCounts: Array<number>,

frontend/javascripts/oxalis/default_state.js

@@ -40,7 +40,6 @@ const defaultState: OxalisState = {
     fourBit: false,
     interpolation: false,
     layers: {},
-    quality: 0,
     loadingStrategy: "PROGRESSIVE_QUALITY",
     highlightHoveredCellId: true,
     segmentationPatternOpacity: 40,