Added support for ComputeSegmentIndexFile worker job

CHANGELOG.unreleased.md

@@ -17,6 +17,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Added a filter to the Task List->Stats column to quickly filter for tasks with "Pending", "In-Progress" or "Finished" instances. [#7430](https://github.com/scalableminds/webknossos/pull/7430)
 - Added support for S3-compliant object storage services (e.g. MinIO) as a storage backend for remote datasets. [#7453](https://github.com/scalableminds/webknossos/pull/7453)
 - Added support for blosc compressed N5 datasets. [#7465](https://github.com/scalableminds/webknossos/pull/7465)
+- Added route for triggering the compute segment index worker job. [#7471](https://github.com/scalableminds/webknossos/pull/7471)
 
 ### Changed
 - An appropriate error is returned when requesting an API version that is higher that the current version. [#7424](https://github.com/scalableminds/webknossos/pull/7424)

app/controllers/JobsController.scala

@@ -161,6 +161,28 @@ class JobsController @Inject()(
       } yield Ok(js)
     }
 
+  def runComputeSegmentIndexFileJob(organizationName: String, dataSetName: String, layerName: String,
+  ): Action[AnyContent] =
+    sil.SecuredAction.async { implicit request =>
+      for {
+        organization <- organizationDAO.findOneByName(organizationName)(GlobalAccessContext) ?~> Messages(
+          "organization.notFound",
+          organizationName)
+        _ <- bool2Fox(request.identity._organization == organization._id) ?~> "job.segmentIndexFile.notAllowed.organization" ~> FORBIDDEN
+        dataSet <- datasetDAO.findOneByNameAndOrganization(dataSetName, organization._id) ?~> Messages(
+          "dataset.notFound",
+          dataSetName) ~> NOT_FOUND
+        command = JobCommand.compute_segment_index_file
+        commandArgs = Json.obj(
+          "organization_name" -> organizationName,
+          "dataset_name" -> dataSetName,
+          "segmentation_layer_name" -> layerName,
+        )
+        job <- jobService.submitJob(command, commandArgs, request.identity, dataSet._dataStore) ?~> "job.couldNotRunSegmentIndexFile"
+        js <- jobService.publicWrites(job)
+      } yield Ok(js)
+    }
+
   def runInferNucleiJob(organizationName: String,
                         dataSetName: String,
                         layerName: String,

app/models/job/JobCommand.scala

@@ -5,6 +5,6 @@ import com.scalableminds.util.enumeration.ExtendedEnumeration
 object JobCommand extends ExtendedEnumeration {
   type JobCommand = Value
 
-  val compute_mesh_file, convert_to_wkw, export_tiff, find_largest_segment_id, globalize_floodfills, infer_nuclei,
-  infer_neurons, materialize_volume_annotation, render_animation = Value
+  val compute_mesh_file, compute_segment_index_file, convert_to_wkw, export_tiff, find_largest_segment_id,
+  globalize_floodfills, infer_nuclei, infer_neurons, materialize_volume_annotation, render_animation = Value
 }

conf/messages

@@ -297,6 +297,7 @@ initialData.organizationsNotEmpty=There are already organizations present in the
 job.couldNotRunCubing = Failed to start WKW conversion job.
 job.couldNotRunTiffExport = Failed to start Tiff export job.
 job.couldNotRunComputeMeshFile = Failed to start mesh file computation job.
+job.couldNotRunComputeSegmentIndexFile = Failed to start segment index computation job.
 job.couldNotRunNucleiInferral = Failed to start nuclei inferral job.
 job.couldNotRunNeuronInferral = Failed to start neuron inferral job.
 job.couldNotRunGlobalizeFloodfills = Failed to start job for globalizing floodfills.
@@ -311,6 +312,7 @@ job.edgeLengthExceeded = An edge length of the selected bounding box is too larg
 job.inferNuclei.notAllowed.organization = Currently nuclei inferral is only allowed for datasets of your own organization.
 job.inferNeurons.notAllowed.organization = Currently neuron inferral is only allowed for datasets of your own organization.
 job.meshFile.notAllowed.organization = Calculating mesh files is only allowed for datasets of your own organization.
+job.segmentIndexFile.notAllowed.organization = Calculating segment index files is only allowed for datasets of your own organization.
 job.globalizeFloodfill.notAllowed.organization = Globalizing floodfills is only allowed for datasets of your own organization.
 job.materializeVolumeAnnotation.notAllowed.organization = Materializing volume annotations is only allowed for datasets of your own organization.
 job.noWorkerForDatastore = Processing jobs are not available for the datastore this dataset belongs to.

conf/webknossos.latest.routes

@@ -267,6 +267,7 @@ GET           /jobs
 GET           /jobs/status                                                                      controllers.JobsController.status()
 POST          /jobs/run/convertToWkw/:organizationName/:dataSetName                             controllers.JobsController.runConvertToWkwJob(organizationName: String, dataSetName: String, scale: String)
 POST          /jobs/run/computeMeshFile/:organizationName/:dataSetName                          controllers.JobsController.runComputeMeshFileJob(organizationName: String, dataSetName: String, layerName: String, mag: String, agglomerateView: Option[String])
+POST          /jobs/run/computeSegmentIndexFile/:organizationName/:dataSetName                  controllers.JobsController.runComputeSegmentIndexFileJob(organizationName: String, dataSetName: String, layerName: String)
 POST          /jobs/run/exportTiff/:organizationName/:dataSetName                               controllers.JobsController.runExportTiffJob(organizationName: String, dataSetName: String, bbox: String, layerName: Option[String], mag: Option[String], annotationLayerName: Option[String], annotationId: Option[String], asOmeTiff: Boolean)
 POST          /jobs/run/inferNuclei/:organizationName/:dataSetName                              controllers.JobsController.runInferNucleiJob(organizationName: String, dataSetName: String, layerName: String, newDatasetName: String)
 POST          /jobs/run/inferNeurons/:organizationName/:dataSetName                             controllers.JobsController.runInferNeuronsJob(organizationName: String, dataSetName: String, layerName: String, bbox: String, newDatasetName: String)

docs/data_formats.md

@@ -152,7 +152,7 @@ WEBKNOSSOS requires several metadata properties for each dataset to properly dis
   + `dataLayers.largestSegmentId`: The highest ID that is currently used in the respective segmentation layer. This is required for volume annotations where new objects with incrementing IDs are created. Only applies to segmentation layers.
   + `dataLayers.dataFormat`: Should be `wkw`.
 
-#### NML Files
+### NML Files
 When working with skeleton annotation data, WEBKNOSSOS uses the NML format.
 It can be [downloaded](./export.md#data-export-and-interoperability) from and uploaded to WEBKNOSSOS, and used for processing in your scripts.
 NML is an XML-based, human-readable file format.

frontend/javascripts/admin/admin_rest_api.ts

@@ -1271,6 +1271,22 @@ export function startComputeMeshFileJob(
   );
 }
 
+export function startComputeSegmentIndexFileJob(
+  organizationName: string,
+  datasetName: string,
+  layerName: string,
+): Promise<APIJob> {
+  const params = new URLSearchParams();
+  params.append("layerName", layerName);
+
+  return Request.receiveJSON(
+    `/api/jobs/run/computeSegmentIndexFile/${organizationName}/${datasetName}?${params}`,
+    {
+      method: "POST",
+    },
+  );
+}
+
 export function startNucleiInferralJob(
   organizationName: string,
   datasetName: string,

frontend/javascripts/admin/job/job_list_view.tsx

@@ -169,6 +169,19 @@ class JobListView extends React.PureComponent<Props, State> {
           </Link>{" "}
         </span>
       );
+    } else if (
+      job.type === APIJobType.COMPUTE_SEGMENT_INDEX_FILE &&
+      job.organizationName &&
+      job.datasetName
+    ) {
+      return (
+        <span>
+          Segment index file computation for{" "}
+          <Link to={`/datasets/${job.organizationName}/${job.datasetName}/view`}>
+            {job.datasetName}
+          </Link>{" "}
+        </span>
+      );
     } else if (
       job.type === APIJobType.FIND_LARGEST_SEGMENT_ID &&
       job.organizationName &&
@@ -253,7 +266,10 @@ class JobListView extends React.PureComponent<Props, State> {
           Cancel
         </AsyncLink>
       );
-    } else if (job.type === APIJobType.CONVERT_TO_WKW) {
+    } else if (
+      job.type === APIJobType.CONVERT_TO_WKW ||
+      job.type === APIJobType.COMPUTE_SEGMENT_INDEX_FILE
+    ) {
       return (
         <span>
           {job.resultLink && (

frontend/javascripts/oxalis/view/left-border-tabs/layer_settings_tab.tsx

@@ -51,6 +51,7 @@ import {
   convertToHybridTracing,
   deleteAnnotationLayer,
   updateDatasetDefaultConfiguration,
+  startComputeSegmentIndexFileJob,
 } from "admin/admin_rest_api";
 import {
   getDefaultValueRangeOfLayer,
@@ -145,6 +146,7 @@ type DatasetSettingsProps = {
   controlMode: ControlMode;
   isArbitraryMode: boolean;
   isAdminOrDatasetManager: boolean;
+  isSuperUser: boolean;
 };
 
 type State = {
@@ -445,6 +447,36 @@ class DatasetSettings extends React.PureComponent<DatasetSettingsProps, State> {
     );
   };
 
+  getComputeSegmentIndexFileButton = (layerName: string, isSegmentation: boolean) => {
+    if (!(this.props.isSuperUser && isSegmentation)) return <></>;
+
+    const triggerComputeSegmentIndexFileJob = async () => {
+      await startComputeSegmentIndexFileJob(
+        this.props.dataset.owningOrganization,
+        this.props.dataset.name,
+        layerName,
+      );
+      Toast.info(
+        <React.Fragment>
+          Started a job for computating a segment index file.
+          <br />
+          See{" "}
+          <a target="_blank" href="/jobs" rel="noopener noreferrer">
+            Processing Jobs
+          </a>{" "}
+          for an overview of running jobs.
+        </React.Fragment>,
+      );
+    };
+
+    return (
+      <div onClick={triggerComputeSegmentIndexFileJob}>
+        <i className="fas fa-database" />
+        Compute a Segment Index file
+      </div>
+    );
+  };
+
   setVisibilityForAllLayers = (isVisible: boolean) => {
     const { layers } = this.props.datasetConfiguration;
     Object.keys(layers).forEach((otherLayerName) =>
@@ -592,6 +624,10 @@ class DatasetSettings extends React.PureComponent<DatasetSettingsProps, State> {
       hasHistogram && !isDisabled
         ? { label: this.getClipButton(layerName, isInEditMode), key: "clipButton" }
         : null,
+      {
+        label: this.getComputeSegmentIndexFileButton(layerName, isSegmentation),
+        key: "computeSegmentIndexFileButton",
+      },
     ];
     const items = possibleItems.filter((el) => el);
     return (
@@ -1439,6 +1475,7 @@ const mapStateToProps = (state: OxalisState) => ({
   isArbitraryMode: Constants.MODES_ARBITRARY.includes(state.temporaryConfiguration.viewMode),
   isAdminOrDatasetManager:
     state.activeUser != null ? Utils.isUserAdminOrDatasetManager(state.activeUser) : false,
+  isSuperUser: state.activeUser?.isSuperUser || false,
 });
 
 const mapDispatchToProps = (dispatch: Dispatch<any>) => ({

frontend/javascripts/types/api_flow_types.ts

@@ -623,6 +623,7 @@ export enum APIJobType {
   "EXPORT_TIFF" = "export_tiff",
   "RENDER_ANIMATION" = "render_animation",
   "COMPUTE_MESH_FILE" = "compute_mesh_file",
+  "COMPUTE_SEGMENT_INDEX_FILE" = "compute_segment_index_file",
   "FIND_LARGEST_SEGMENT_ID" = "find_largest_segment_id",
   "INFER_NUCLEI" = "infer_nuclei",
   "INFER_NEURONS" = "infer_neurons",