Merge branch 'master' of github.com:scalableminds/webknossos into add…

…-switch-orga-for-legacy-links

CHANGELOG.unreleased.md

@@ -15,12 +15,14 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 
 ### Changed
 - Renamed "resolution" to "magnification" in more places within the codebase, including local variables. [#8168](https://github.com/scalableminds/webknossos/pull/8168)
+- Layer names are now allowed to contain `$` as special characters. [#8241](https://github.com/scalableminds/webknossos/pull/8241)
 - Datasets can now be renamed and can have duplicate names. [#8075](https://github.com/scalableminds/webknossos/pull/8075)
 - Improved the default colors for skeleton trees. [#8228](https://github.com/scalableminds/webknossos/pull/8228)
 - Allowed to train an AI model using differently sized bounding boxes. We recommend all bounding boxes to have equal dimensions or to have dimensions which are multiples of the smallest bounding box. [#8222](https://github.com/scalableminds/webknossos/pull/8222)
 
 ### Fixed
 - Fixed that listing datasets with the `api/datasets` route without compression failed due to missing permissions regarding public datasets. [#8249](https://github.com/scalableminds/webknossos/pull/8249)
+- Fixed a bug that uploading a zarr dataset with an already existing `datasource-properties.json` file failed. [#8268](https://github.com/scalableminds/webknossos/pull/8268)
 - Fixed the organization switching feature for datasets opened via old links. [#8257](https://github.com/scalableminds/webknossos/pull/8257)
 - Fixed that the frontend did not ensure a minium length for annotation layer names. Moreover, names starting with a `.` are also disallowed now. [#8244](https://github.com/scalableminds/webknossos/pull/8244)
 - Fixed a bug where in the add remote dataset view the dataset name setting was not in sync with the datasource setting of the advanced tab making the form not submittable. [#8245](https://github.com/scalableminds/webknossos/pull/8245)

conf/evolutions/125-allow-dollar-in-layer-names.sql

@@ -0,0 +1,11 @@
+START TRANSACTION;
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 124, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+
+ALTER TABLE webknossos.annotation_layers DROP CONSTRAINT IF EXISTS annotation_layers_name_check;
+ALTER TABLE webknossos.annotation_layers ADD CONSTRAINT annotation_layers_name_check  CHECK (name ~* '^[A-Za-z0-9\-_\.\$]+$');
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 125;
+
+COMMIT TRANSACTION;

conf/evolutions/reversions/125-allow-dollar-in-layer-names.sql

@@ -0,0 +1,13 @@
+START TRANSACTION;
+
+-- This reversion might take a while because it needs to search in all annotation layer names for '$' and replace it with ''
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 125, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+UPDATE webknossos.annotation_layers SET name = regexp_replace(name, '\$', '', 'g') WHERE name ~* '\$';
+
+ALTER TABLE webknossos.annotation_layers DROP CONSTRAINT IF EXISTS annotation_layers_name_check;
+ALTER TABLE webknossos.annotation_layers ADD CONSTRAINT annotation_layers_name_check  CHECK (name ~* '^[A-Za-z0-9\-_\.]+$');
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 124;
+
+COMMIT TRANSACTION;

frontend/javascripts/admin/dataset/dataset_components.tsx

@@ -35,15 +35,10 @@ export function CardContainer({
     );
   }
 }
-export const layerNameRules = [
+const sharedRules = [
   {
     min: 1,
   },
-  // Note that these rules are also checked by the backend
-  {
-    pattern: /^[0-9a-zA-Z_.-]+$/,
-    message: "Only letters, digits and the following characters are allowed: . _ -",
-  },
   {
     validator: syncValidator(
       (value: string | null) => !value || !value.startsWith("."),
@@ -52,13 +47,25 @@ export const layerNameRules = [
   },
 ];
 
+export const layerNameRules = [
+  ...sharedRules,
+  {
+    pattern: /^[0-9a-zA-Z_.\-$.]+$/,
+    message: "Only letters, digits and the following characters are allowed: . _ - $",
+  },
+];
+
 export const getDatasetNameRules = (activeUser: APIUser | null | undefined) => [
   {
     required: true,
     message: messages["dataset.import.required.name"],
   },
   { min: 3, message: messages["dataset.name_length"] },
   ...layerNameRules,
+  {
+    pattern: /^[0-9a-zA-Z_.-]+$/,
+    message: "Only letters, digits and the following characters are allowed: . _ -",
+  },
   {
     validator: async () => {
       if (!activeUser) throw new Error("Can't do operation if no user is logged in.");

frontend/javascripts/dashboard/dataset/dataset_settings_data_tab.tsx

@@ -398,8 +398,9 @@ function SimpleLayerForm({
               {
                 validator: syncValidator(
                   (value: string) =>
-                    dataLayers.filter((someLayer: APIDataLayer) => someLayer.name === value)
-                      .length <= 1,
+                    form
+                      .getFieldValue(["dataSource", "dataLayers"])
+                      .filter((someLayer: APIDataLayer) => someLayer.name === value).length <= 1,
                   "Layer names must be unique.",
                 ),
               },

frontend/javascripts/dashboard/dataset/dataset_settings_view.tsx

@@ -339,12 +339,18 @@ class DatasetSettingsView extends React.PureComponent<PropsWithFormAndRouter, St
       this.state.activeDataSourceEditMode === "simple" ? "advanced" : "simple",
     );
 
-    const afterForceUpdateCallback = () =>
+    const afterForceUpdateCallback = () => {
       // Trigger validation manually, because fields may have been updated
-      form
-        .validateFields()
-        .then((formValues) => this.submit(formValues))
-        .catch((errorInfo) => this.handleValidationFailed(errorInfo));
+      // and defer the validation as it is done asynchronously by antd or so.
+      setTimeout(
+        () =>
+          form
+            .validateFields()
+            .then((formValues) => this.submit(formValues))
+            .catch((errorInfo) => this.handleValidationFailed(errorInfo)),
+        0,
+      );
+    };
 
     // Need to force update of the SimpleAdvancedDataForm as removing a layer in the advanced tab does not update
     // the form items in the simple tab (only the values are updated). The form items automatically update once

frontend/javascripts/oxalis/view/left-border-tabs/modals/add_volume_layer_modal.tsx

@@ -48,9 +48,9 @@ export function checkLayerNameForInvalidCharacters(readableLayerName: string): V
       message: messages["tracing.volume_layer_name_starts_with_dot"],
     };
   }
-  const uriSafeCharactersRegex = /[0-9a-zA-Z-._]+/g;
+  const validLayerNameCharactersRegex = /[0-9a-zA-Z-._$]+/g;
   // Removing all URISaveCharacters from readableLayerName. The leftover chars are all invalid.
-  const allInvalidChars = readableLayerName.replace(uriSafeCharactersRegex, "");
+  const allInvalidChars = readableLayerName.replace(validLayerNameCharactersRegex, "");
   const allUniqueInvalidCharsAsSet = new Set(allInvalidChars);
   const allUniqueInvalidCharsAsString = "".concat(...allUniqueInvalidCharsAsSet.values());
   const isValid = allUniqueInvalidCharsAsString.length === 0;

tools/postgres/schema.sql

@@ -20,7 +20,7 @@ CREATE TABLE webknossos.releaseInformation (
   schemaVersion BIGINT NOT NULL
 );
 
-INSERT INTO webknossos.releaseInformation(schemaVersion) values(124);
+INSERT INTO webknossos.releaseInformation(schemaVersion) values(125);
 COMMIT TRANSACTION;
 
 
@@ -56,7 +56,7 @@ CREATE TABLE webknossos.annotation_layers(
   _annotation CHAR(24) NOT NULL,
   tracingId CHAR(36) NOT NULL UNIQUE,
   typ webknossos.ANNOTATION_LAYER_TYPE NOT NULL,
-  name VARCHAR(256) NOT NULL CHECK (name ~* '^[A-Za-z0-9\-_\.]+$'),
+  name VARCHAR(256) NOT NULL CHECK (name ~* '^[A-Za-z0-9\-_\.\$]+$'),
   statistics JSONB NOT NULL,
   UNIQUE (name, _annotation),
   PRIMARY KEY (_annotation, tracingId),

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala

@@ -466,10 +466,10 @@ class UploadService @Inject()(dataSourceRepository: DataSourceRepository,
   }
 
   private def guessTypeOfUploadedDataSource(dataSourceDir: Path): UploadedDataSourceType.Value =
-    if (looksLikeZarrArray(dataSourceDir, maxDepth = 2).openOr(false)) {
-      UploadedDataSourceType.ZARR
-    } else if (looksLikeExploredDataSource(dataSourceDir).openOr(false)) {
+    if (looksLikeExploredDataSource(dataSourceDir).openOr(false)) {
       UploadedDataSourceType.EXPLORED
+    } else if (looksLikeZarrArray(dataSourceDir, maxDepth = 2).openOr(false)) {
+      UploadedDataSourceType.ZARR
     } else if (looksLikeZarrArray(dataSourceDir, maxDepth = 3).openOr(false)) {
       UploadedDataSourceType.ZARR_MULTILAYER
     } else if (looksLikeNeuroglancerPrecomputed(dataSourceDir, 1).openOr(false)) {