From 5c06706676ec9a013239e57ce7ff1dff9ad8c43e Mon Sep 17 00:00:00 2001 From: Youri K Date: Thu, 29 Oct 2020 15:43:02 +0100 Subject: [PATCH] Organization restricted datastore (#4892) * add db column to datastore * backend changes * ignore forbidden orgs when reporting datasets * Update app/models/binary/DataSetService.scala Co-authored-by: Florian M * add readAccessQ * bump schema version Co-authored-by: Florian M --- CHANGELOG.unreleased.md | 1 + MIGRATIONS.unreleased.md | 3 ++- app/controllers/WKDataStoreController.scala | 1 + app/models/binary/DataSetService.scala | 4 ++++ app/models/binary/DataStore.scala | 15 +++++++++++---- .../058-add-onlyAllowedOrganization.sql | 13 +++++++++++++ .../058-add-onlyAllowedOrganization.sql | 11 +++++++++++ conf/messages | 2 ++ test/db/dataStores.csv | 6 +++--- tools/postgres/schema.sql | 5 +++-- .../controllers/DataSourceController.scala | 2 +- 11 files changed, 52 insertions(+), 11 deletions(-) create mode 100644 conf/evolutions/058-add-onlyAllowedOrganization.sql create mode 100644 conf/evolutions/reversions/058-add-onlyAllowedOrganization.sql diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index a2f9f435e81..6f4ba42a918 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -13,6 +13,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released ### Added - The total length of skeletons can now be measured using the dropdown in the tree list tab. Also, the frontend API received the methods `api.tracing.measureTreeLength` and `api.tracing.measureAllTrees`. [#4898](https://github.com/scalableminds/webknossos/pull/4898) - Introduced an indeterminate visibility state for groups in the tree tab if not all but only some of the group's children are visible. Before, the visibility of those groups was shown as not visible which made it hard to find the visible trees. [#4897](https://github.com/scalableminds/webknossos/pull/4897) +- Dataset uploads on a specific Datastore can now be restricted to a single organization. [#4892](https://github.com/scalableminds/webknossos/pull/4892) ### Changed - In the tree tab, all groups but the root group are now collapsed instead of expanded when opening a tracing. [#4897](https://github.com/scalableminds/webknossos/pull/4897) diff --git a/MIGRATIONS.unreleased.md b/MIGRATIONS.unreleased.md index f07fca7c39c..3c07882466e 100644 --- a/MIGRATIONS.unreleased.md +++ b/MIGRATIONS.unreleased.md @@ -9,4 +9,5 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md). - ### Postgres Evolutions: -- [056-add-layer-specific-view-configs.sql](conf/evolutions/056-add-layer-specific-view-configs.sql) +- [057-add-layer-specific-view-configs.sql](conf/evolutions/056-add-layer-specific-view-configs.sql) +- [058-add-onlyAllowedOrganization.sql](conf/evolutions/057-add-onlyAllowedOrganization.sql) diff --git a/app/controllers/WKDataStoreController.scala b/app/controllers/WKDataStoreController.scala index 132442e434b..2f6fd6f666d 100644 --- a/app/controllers/WKDataStoreController.scala +++ b/app/controllers/WKDataStoreController.scala @@ -44,6 +44,7 @@ class WKDataStoreController @Inject()(dataSetService: DataSetService, _ <- bool2Fox(dataSetService.isProperDataSetName(uploadInfo.name)) ?~> "dataSet.name.invalid" _ <- dataSetService .assertNewDataSetName(uploadInfo.name, organization._id)(GlobalAccessContext) ?~> "dataSet.name.alreadyTaken" + _ <- bool2Fox(dataStore.onlyAllowedOrganization.forall(_ == organization._id)) ?~> "dataSet.upload.Datastore.restricted" } yield Ok } } diff --git a/app/models/binary/DataSetService.scala b/app/models/binary/DataSetService.scala index fb6310b3313..259978f8481 100644 --- a/app/models/binary/DataSetService.scala +++ b/app/models/binary/DataSetService.scala @@ -127,6 +127,10 @@ class DataSetService @Inject()(organizationDAO: OrganizationDAO, .findOneByName(orgaTuple._1) .futureBox .flatMap { + case Full(organization) if dataStore.onlyAllowedOrganization.exists(_ != organization._id) => + logger.info( + s"Ignoring ${orgaTuple._2.length} reported datasets for forbidden organization ${orgaTuple._1} from organization-specific datastore ${dataStore.name}") + Fox.successful(List.empty) case Full(organization) => for { foundDatasets <- dataSetDAO.findAllByNamesAndOrganization(orgaTuple._2.map(_.id.name), organization._id) diff --git a/app/models/binary/DataStore.scala b/app/models/binary/DataStore.scala index 8f142042e35..d536ab60018 100644 --- a/app/models/binary/DataStore.scala +++ b/app/models/binary/DataStore.scala @@ -9,7 +9,7 @@ import play.api.libs.json.{Format, JsObject, Json} import play.api.mvc.{Request, Result, Results, WrappedRequest} import slick.jdbc.PostgresProfile.api._ import slick.lifted.Rep -import utils.{SQLClient, SQLDAO} +import utils.{ObjectId, SQLClient, SQLDAO} import scala.concurrent.{ExecutionContext, Future} @@ -23,6 +23,7 @@ case class DataStore( isForeign: Boolean = false, isConnector: Boolean = false, allowsUpload: Boolean = true, + onlyAllowedOrganization: Option[ObjectId] = None ) object DataStore { @@ -45,7 +46,8 @@ object DataStore { isDeleted = false, isForeign.getOrElse(false), isConnector.getOrElse(false), - allowsUpload.getOrElse(true) + allowsUpload.getOrElse(true), + None ) def fromUpdateForm(name: String, @@ -91,6 +93,9 @@ class DataStoreDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext def idColumn(x: Datastores): Rep[String] = x.name def isDeletedColumn(x: Datastores): Rep[Boolean] = x.isdeleted + override def readAccessQ(requestingUserId: ObjectId): String = + s"(onlyAllowedOrganization is null) OR (onlyAllowedOrganization in (select _organization from webknossos.users_ where _id = '$requestingUserId'))" + def parse(r: DatastoresRow): Fox[DataStore] = Fox.successful( DataStore( @@ -102,7 +107,8 @@ class DataStoreDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext r.isdeleted, r.isforeign, r.isconnector, - r.allowsupload + r.allowsupload, + r.onlyallowedorganization.map(ObjectId(_)) )) def findOneByName(name: String)(implicit ctx: DBAccessContext): Fox[DataStore] = @@ -116,7 +122,8 @@ class DataStoreDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext override def findAll(implicit ctx: DBAccessContext): Fox[List[DataStore]] = for { - r <- run(sql"select #${columns} from webknossos.datastores_ order by name".as[DatastoresRow]) + accessQuery <- readAccessQuery + r <- run(sql"select #$columns from webknossos.datastores_ where #$accessQuery order by name".as[DatastoresRow]) parsed <- Fox.combined(r.toList.map(parse)) } yield parsed diff --git a/conf/evolutions/058-add-onlyAllowedOrganization.sql b/conf/evolutions/058-add-onlyAllowedOrganization.sql new file mode 100644 index 00000000000..4bfd5e0f72a --- /dev/null +++ b/conf/evolutions/058-add-onlyAllowedOrganization.sql @@ -0,0 +1,13 @@ +-- https://github.com/scalableminds/webknossos/pull/4892 + +START TRANSACTION; + +DROP VIEW webknossos.dataStores_; + +ALTER TABLE webknossos.dataStores ADD COLUMN onlyAllowedOrganization CHAR(24); + +CREATE VIEW webknossos.dataStores_ AS SELECT * FROM webknossos.dataStores WHERE NOT isDeleted; + +UPDATE webknossos.releaseInformation SET schemaVersion = 58; + +COMMIT TRANSACTION; diff --git a/conf/evolutions/reversions/058-add-onlyAllowedOrganization.sql b/conf/evolutions/reversions/058-add-onlyAllowedOrganization.sql new file mode 100644 index 00000000000..7b5ba8b2a18 --- /dev/null +++ b/conf/evolutions/reversions/058-add-onlyAllowedOrganization.sql @@ -0,0 +1,11 @@ +START TRANSACTION; + +DROP VIEW webknossos.dataStores_; + +ALTER TABLE webknossos.dataStores DROP onlyAllowedOrganization; + +CREATE VIEW webknossos.dataStores_ AS SELECT * FROM webknossos.dataStores WHERE NOT isDeleted; + +UPDATE webknossos.releaseInformation SET schemaVersion = 57; + +COMMIT TRANSACTION; diff --git a/conf/messages b/conf/messages index 61592ad1b5b..6f6521a52b7 100644 --- a/conf/messages +++ b/conf/messages @@ -116,6 +116,8 @@ dataSet.initialTeams.teamsNotEmpty=Dataset already has allowed teams dataset.initialTeams.invalidTeams=Can only assign teams of user dataset.initialTeams.timeout=Timeout while setting initial teams. Was the request sent manually? dataset.delete.disabled=Dataset deletion is disabled for this webKnossos instance +dataSet.upload.Datastore.restricted=Your organization is not allowed to upload datasets to this datastore. Please choose another datastore. +dataSet.upload.validation.failed=Failed to validate Dataset information for upload. dataSource.notFound=Datasource not found on datastore server diff --git a/test/db/dataStores.csv b/test/db/dataStores.csv index 3f7ebe483e9..05c839d56b0 100644 --- a/test/db/dataStores.csv +++ b/test/db/dataStores.csv @@ -1,3 +1,3 @@ -name,url,publicUrl,key,isScratch,isDeleted,isForeign,isConnector,allowsUpload -'localhost','http://localhost:9000','http://localhost:9000','something-secure',f,f,f,f,t -'connect','http://localhost:8000','http://localhost:8000','secret-key',f,f,f,t,f +name,url,publicUrl,key,isScratch,isDeleted,isForeign,isConnector,allowsUpload,onlyAllowedOrganization +'localhost','http://localhost:9000','http://localhost:9000','something-secure',f,f,f,f,t, +'connect','http://localhost:8000','http://localhost:8000','secret-key',f,f,f,t,f, diff --git a/tools/postgres/schema.sql b/tools/postgres/schema.sql index 65782ce1b10..72bc3769c6d 100644 --- a/tools/postgres/schema.sql +++ b/tools/postgres/schema.sql @@ -21,7 +21,7 @@ START TRANSACTION; CREATE TABLE webknossos.releaseInformation ( schemaVersion BIGINT NOT NULL ); -INSERT INTO webknossos.releaseInformation(schemaVersion) values(57); +INSERT INTO webknossos.releaseInformation(schemaVersion) values(58); COMMIT TRANSACTION; CREATE TABLE webknossos.analytics( @@ -160,7 +160,8 @@ CREATE TABLE webknossos.dataStores( isDeleted BOOLEAN NOT NULL DEFAULT false, isForeign BOOLEAN NOT NULL DEFAULT false, isConnector BOOLEAN NOT NULL DEFAULT false, - allowsUpload BOOLEAN NOT NULL DEFAULT true + allowsUpload BOOLEAN NOT NULL DEFAULT true, + onlyAllowedOrganization CHAR(24) ); CREATE TABLE webknossos.tracingStores( diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala index 5df4c56b0b0..6b694c05750 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala @@ -103,7 +103,7 @@ class DataSourceController @Inject()( val resumableUploadInformation = ResumableUploadInformation(chunkSize, totalChunkCount) for { _ <- if (!uploadService.isKnownUpload(uploadId)) - webKnossosServer.validateDataSourceUpload(id) ?~> "dataSet.name.alreadyTaken" + webKnossosServer.validateDataSourceUpload(id) ?~> "dataSet.upload.validation.failed" else Fox.successful(()) chunkFile <- request.body.file("file") ?~> "zip.file.notFound" _ <- uploadService.handleUploadChunk(uploadId,