From 530720b6994a1f8132d5812af5217291f4ed33b1 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 3 Oct 2023 15:13:56 -0400 Subject: [PATCH 01/10] Adds support for automatically selecting the proper band filter --- app/api/formulas.py | 52 +++++++++++++++++-- app/api/tiler.py | 20 ++++++- app/migrations/0039_task_orthophoto_bands.py | 43 +++++++++++++++ app/models/task.py | 7 ++- .../app/js/components/LayersControlLayer.jsx | 14 +++-- app/static/app/js/components/Map.jsx | 14 +++++ app/tests/test_api_task.py | 2 +- app/tests/test_formulas.py | 2 +- locale | 2 +- 9 files changed, 141 insertions(+), 15 deletions(-) create mode 100644 app/migrations/0039_task_orthophoto_bands.py diff --git a/app/api/formulas.py b/app/api/formulas.py index 457303200..21d4e2971 100644 --- a/app/api/formulas.py +++ b/app/api/formulas.py @@ -156,6 +156,8 @@ 'BGRNReL', 'BGRReNL', + 'RGBNRePL', + 'L', # FLIR camera has a single LWIR band # more? @@ -171,7 +173,7 @@ def lookup_formula(algo, band_order = 'RGB'): if algo not in algos: raise ValueError("Cannot find algorithm " + algo) - + input_bands = tuple(b for b in re.split(r"([A-Z][a-z]*)", band_order) if b != "") def repl(matches): @@ -193,7 +195,7 @@ def get_algorithm_list(max_bands=3): if k.startswith("_"): continue - cam_filters = get_camera_filters_for(algos[k], max_bands) + cam_filters = get_camera_filters_for(algos[k]['expr'], max_bands) if len(cam_filters) == 0: continue @@ -206,9 +208,9 @@ def get_algorithm_list(max_bands=3): return res -def get_camera_filters_for(algo, max_bands=3): +@lru_cache(maxsize=100) +def get_camera_filters_for(expr, max_bands=3): result = [] - expr = algo['expr'] pattern = re.compile("([A-Z]+?[a-z]*)") bands = list(set(re.findall(pattern, expr))) for f in camera_filters: @@ -226,3 +228,45 @@ def get_camera_filters_for(algo, max_bands=3): return result +@lru_cache(maxsize=1) +def get_bands_lookup(): + bands_aliases = { + 'R': ['red', 'r'], + 'G': ['green', 'g'], + 'B': ['blue', 'b'], + 'N': ['nir', 'n'], + 'Re': ['rededge', 're'], + 'P': ['panchro', 'p'], + 'L': ['lwir', 'l'] + } + bands_lookup = {} + for band in bands_aliases: + for a in bands_aliases[band]: + bands_lookup[a] = band + return bands_lookup + +def get_auto_bands(orthophoto_bands, formula): + algo = algos.get(formula) + if not algo: + raise ValueError("Cannot find formula: " + formula) + + max_bands = len(orthophoto_bands) - 1 # minus alpha + filters = get_camera_filters_for(algo['expr'], max_bands) + if not filters: + raise valueError(f"Cannot find filters for {algo} with max bands {max_bands}") + + bands_lookup = get_bands_lookup() + band_order = "" + + for band in orthophoto_bands: + if band['name'] == 'alpha' or (not band['description']): + continue + f_band = bands_lookup.get(band['description'].lower()) + + if f_band is not None: + band_order += f_band + + if band_order in filters: + return band_order, True + else: + return filters[0], False # Fallback diff --git a/app/api/tiler.py b/app/api/tiler.py index 91a496e14..855eef569 100644 --- a/app/api/tiler.py +++ b/app/api/tiler.py @@ -23,7 +23,7 @@ from app.raster_utils import extension_for_export_format, ZOOM_EXTRA_LEVELS from .hsvblend import hsv_blend from .hillshade import LightSource -from .formulas import lookup_formula, get_algorithm_list +from .formulas import lookup_formula, get_algorithm_list, get_auto_bands from .tasks import TaskNestedView from rest_framework import exceptions from rest_framework.response import Response @@ -141,6 +141,12 @@ def get(self, request, pk=None, project_pk=None, tile_type=""): if boundaries_feature == '': boundaries_feature = None if boundaries_feature is not None: boundaries_feature = json.loads(boundaries_feature) + + is_auto_bands_match = False + is_auto_bands = False + if bands == 'auto' and formula: + is_auto_bands = True + bands, is_auto_bands_match = get_auto_bands(task.orthophoto_bands, formula) try: expr, hrange = lookup_formula(formula, bands) if defined_range is not None: @@ -224,6 +230,8 @@ def get(self, request, pk=None, project_pk=None, tile_type=""): colormaps = [] algorithms = [] + auto_bands = {'filter': '', 'match': None} + if tile_type in ['dsm', 'dtm']: colormaps = ['viridis', 'jet', 'terrain', 'gist_earth', 'pastel1'] elif formula and bands: @@ -231,9 +239,14 @@ def get(self, request, pk=None, project_pk=None, tile_type=""): 'better_discrete_ndvi', 'viridis', 'plasma', 'inferno', 'magma', 'cividis', 'jet', 'jet_r'] algorithms = *get_algorithm_list(band_count), + if is_auto_bands: + auto_bands['filter'] = bands + auto_bands['match'] = is_auto_bands_match info['color_maps'] = [] info['algorithms'] = algorithms + info['auto_bands'] = auto_bands + if colormaps: for cmap in colormaps: try: @@ -254,6 +267,7 @@ def get(self, request, pk=None, project_pk=None, tile_type=""): info['maxzoom'] += ZOOM_EXTRA_LEVELS info['minzoom'] -= ZOOM_EXTRA_LEVELS info['bounds'] = {'value': src.bounds, 'crs': src.dataset.crs} + return Response(info) @@ -296,6 +310,8 @@ def get(self, request, pk=None, project_pk=None, tile_type="", z="", x="", y="", if color_map == '': color_map = None if hillshade == '' or hillshade == '0': hillshade = None if tilesize == '' or tilesize is None: tilesize = 256 + if bands == 'auto' and formula: + bands, _ = get_auto_bands(task.orthophoto_bands, formula) try: tilesize = int(tilesize) @@ -611,4 +627,4 @@ def post(self, request, pk=None, project_pk=None, asset_type=None): else: celery_task_id = export_pointcloud.delay(url, epsg=epsg, format=export_format).task_id - return Response({'celery_task_id': celery_task_id, 'filename': filename}) \ No newline at end of file + return Response({'celery_task_id': celery_task_id, 'filename': filename}) diff --git a/app/migrations/0039_task_orthophoto_bands.py b/app/migrations/0039_task_orthophoto_bands.py new file mode 100644 index 000000000..c801ab853 --- /dev/null +++ b/app/migrations/0039_task_orthophoto_bands.py @@ -0,0 +1,43 @@ +# Generated by Django 2.2.27 on 2023-10-02 10:21 + +import rasterio +import os +import django.contrib.postgres.fields.jsonb +from django.db import migrations +from webodm import settings + +def update_orthophoto_bands_fields(apps, schema_editor): + Task = apps.get_model('app', 'Task') + + for t in Task.objects.all(): + + bands = [] + orthophoto_path = os.path.join(settings.MEDIA_ROOT, "project", str(t.project.id), "task", str(t.id), "assets", "odm_orthophoto", "odm_orthophoto.tif") + + if os.path.isfile(orthophoto_path): + try: + with rasterio.open(orthophoto_path) as f: + names = [c.name for c in f.colorinterp] + for i, n in enumerate(names): + bands.append({ + 'name': n, + 'description': f.descriptions[i] + }) + except Exception as e: + print(e) + + print("Updating {} (with orthophoto bands: {})".format(t, str(bands))) + + t.orthophoto_bands = bands + t.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0038_remove_task_console_output'), + ] + + operations = [ + migrations.RunPython(update_orthophoto_bands_fields), + ] diff --git a/app/models/task.py b/app/models/task.py index 1f0c4c0c6..89bcb85c0 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -1024,7 +1024,12 @@ def update_orthophoto_bands_field(self, commit=False): if os.path.isfile(orthophoto_path): with rasterio.open(orthophoto_path) as f: - bands = [c.name for c in f.colorinterp] + names = [c.name for c in f.colorinterp] + for i, n in enumerate(names): + bands.append({ + 'name': n, + 'description': f.descriptions[i] + }) self.orthophoto_bands = bands if commit: self.save() diff --git a/app/static/app/js/components/LayersControlLayer.jsx b/app/static/app/js/components/LayersControlLayer.jsx index 36509ad53..2aaaf531a 100644 --- a/app/static/app/js/components/LayersControlLayer.jsx +++ b/app/static/app/js/components/LayersControlLayer.jsx @@ -134,7 +134,7 @@ export default class LayersControlLayer extends React.Component { // Check if bands need to be switched const algo = this.getAlgorithm(e.target.value); - if (algo && algo['filters'].indexOf(bands) === -1) bands = algo['filters'][0]; // Pick first + if (algo && algo['filters'].indexOf(bands) === -1 && bands !== "auto") bands = algo['filters'][0]; // Pick first this.setState({formula: e.target.value, bands}); } @@ -262,7 +262,7 @@ export default class LayersControlLayer extends React.Component { render(){ const { colorMap, bands, hillshade, formula, histogramLoading, exportLoading } = this.state; const { meta, tmeta } = this; - const { color_maps, algorithms } = tmeta; + const { color_maps, algorithms, auto_bands } = tmeta; const algo = this.getAlgorithm(formula); let cmapValues = null; @@ -298,13 +298,17 @@ export default class LayersControlLayer extends React.Component { {bands !== "" && algo ?
- +
{histogramLoading ? : - + {algo.filters.map(f => )} - } + , + bands == "auto" && !auto_bands.match ? + + : ""]}
: ""} diff --git a/app/static/app/js/components/Map.jsx b/app/static/app/js/components/Map.jsx index b93ff5611..1e78675c4 100644 --- a/app/static/app/js/components/Map.jsx +++ b/app/static/app/js/components/Map.jsx @@ -94,6 +94,16 @@ class Map extends React.Component { return ""; } + hasBands = (bands, orthophoto_bands) => { + if (!orthophoto_bands) return false; + console.log(orthophoto_bands) + for (let i = 0; i < bands.length; i++){ + if (orthophoto_bands.find(b => b.description !== null && b.description.toLowerCase() === bands[i].toLowerCase()) === undefined) return false; + } + + return true; + } + loadImageryLayers(forceAddLayers = false){ // Cancel previous requests if (this.tileJsonRequests) { @@ -131,7 +141,11 @@ class Map extends React.Component { // Single band, probably thermal dataset, in any case we can't render NDVI // because it requires 3 bands metaUrl += "?formula=Celsius&bands=L&color_map=magma"; + }else if (meta.task && meta.task.orthophoto_bands){ + let formula = this.hasBands(["red", "green", "nir"], meta.task.orthophoto_bands) ? "NDVI" : "VARI"; + metaUrl += `?formula=${formula}&bands=auto&color_map=rdylgn`; }else{ + // This should never happen? metaUrl += "?formula=NDVI&bands=RGN&color_map=rdylgn"; } }else if (type == "dsm" || type == "dtm"){ diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index 1abbff4ec..bf3af575a 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -679,7 +679,7 @@ def test_task(self): for k in algos: a = algos[k] - filters = get_camera_filters_for(a) + filters = get_camera_filters_for(a['expr']) for f in filters: params.append(("orthophoto", "formula={}&bands={}&color_map=rdylgn".format(k, f), status.HTTP_200_OK)) diff --git a/app/tests/test_formulas.py b/app/tests/test_formulas.py index 76077d47a..ab9117da0 100644 --- a/app/tests/test_formulas.py +++ b/app/tests/test_formulas.py @@ -38,7 +38,7 @@ def test_algo_list(self): bands = list(set(re.findall(pattern, f))) self.assertTrue(len(bands) <= 3) - self.assertTrue(get_camera_filters_for(algos['VARI']) == ['RGB']) + self.assertTrue(get_camera_filters_for(algos['VARI']['expr']) == ['RGB']) # Request algorithms with more band filters al = get_algorithm_list(max_bands=5) diff --git a/locale b/locale index d253dd577..0a68f6ed5 160000 --- a/locale +++ b/locale @@ -1 +1 @@ -Subproject commit d253dd5770c42a0705d0f861db3314b08f230a68 +Subproject commit 0a68f6ed5172a8838571a9872bcc3cb4f310794c From 43b24eb8b61e41b72180fe2eb51568fc4a873151 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 3 Oct 2023 15:19:28 -0400 Subject: [PATCH 02/10] Bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8c812941d..1018c1b4b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "WebODM", - "version": "2.1.3", + "version": "2.2.0", "description": "User-friendly, extendable application and API for processing aerial imagery.", "main": "index.js", "scripts": { From b68e622234604e9ffda361672013ff450f78273b Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 3 Oct 2023 15:31:06 -0400 Subject: [PATCH 03/10] Update locales --- .../app/js/components/LayersControlLayer.jsx | 4 +- app/static/app/js/components/Map.jsx | 2 +- .../app/js/components/ProjectListItem.jsx | 2 + .../app/js/translations/odm_autogenerated.js | 152 +++++++++--------- locale | 2 +- 5 files changed, 82 insertions(+), 80 deletions(-) diff --git a/app/static/app/js/components/LayersControlLayer.jsx b/app/static/app/js/components/LayersControlLayer.jsx index 2aaaf531a..4996b01fd 100644 --- a/app/static/app/js/components/LayersControlLayer.jsx +++ b/app/static/app/js/components/LayersControlLayer.jsx @@ -302,12 +302,12 @@ export default class LayersControlLayer extends React.Component {
{histogramLoading ? : - [ {algo.filters.map(f => )} , bands == "auto" && !auto_bands.match ? - + : ""]}
: ""} diff --git a/app/static/app/js/components/Map.jsx b/app/static/app/js/components/Map.jsx index 1e78675c4..2d6caacc4 100644 --- a/app/static/app/js/components/Map.jsx +++ b/app/static/app/js/components/Map.jsx @@ -96,7 +96,7 @@ class Map extends React.Component { hasBands = (bands, orthophoto_bands) => { if (!orthophoto_bands) return false; - console.log(orthophoto_bands) + for (let i = 0; i < bands.length; i++){ if (orthophoto_bands.find(b => b.description !== null && b.description.toLowerCase() === bands[i].toLowerCase()) === undefined) return false; } diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 31d6ec429..51023e933 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -573,6 +573,8 @@ class ProjectListItem extends React.Component { render() { const { refreshing, data, filterTags } = this.state; const numTasks = data.tasks.length; + + // const numValidTasks = data.tasks.filter(t => ) const canEdit = this.hasPermission("change"); const userTags = Tags.userTags(data.tags); let deleteWarning = _("All tasks, images and models associated with this project will be permanently deleted. Are you sure you want to continue?"); diff --git a/app/static/app/js/translations/odm_autogenerated.js b/app/static/app/js/translations/odm_autogenerated.js index bb87910bb..cd078b718 100644 --- a/app/static/app/js/translations/odm_autogenerated.js +++ b/app/static/app/js/translations/odm_autogenerated.js @@ -1,94 +1,94 @@ // Auto-generated with extract_odm_strings.py, do not edit! -_("show this help message and exit"); -_("When processing multispectral datasets, ODM will automatically align the images for each band. If the images have been postprocessed and are already aligned, use this option. Default: %(default)s"); -_("Set this parameter if you want to generate a PNG rendering of the orthophoto. Default: %(default)s"); -_("Skip generation of a full 3D model. This can save time if you only need 2D results such as orthophotos and DEMs. Default: %(default)s"); -_("Automatically compute image masks using AI to remove the background. Experimental. Default: %(default)s"); -_("Skip normalization of colors across all images. Useful when processing radiometric data. Default: %(default)s"); -_("The maximum output resolution of extracted video frames in pixels. Default: %(default)s"); -_("Filters the point cloud by removing points that deviate more than N standard deviations from the local mean. Set to 0 to disable filtering. Default: %(default)s"); -_("Set this parameter if you want a striped GeoTIFF. Default: %(default)s"); -_("Perform image matching with the nearest images based on GPS exif data. Set to 0 to match by triangulation. Default: %(default)s"); -_("Automatically set a boundary using camera shot locations to limit the area of the reconstruction. This can help remove far away background artifacts (sky, background landscapes, etc.). See also --boundary. Default: %(default)s"); -_("Path to the project folder. Your project folder should contain subfolders for each dataset. Each dataset should have an \"images\" folder."); -_("End processing at this stage. Can be one of: %(choices)s. Default: %(default)s"); -_("Automatically compute image masks using AI to remove the sky. Experimental. Default: %(default)s"); +_("Matcher algorithm, Fast Library for Approximate Nearest Neighbors or Bag of Words. FLANN is slower, but more stable. BOW is faster, but can sometimes miss valid matches. BRUTEFORCE is very slow but robust.Can be one of: %(choices)s. Default: %(default)s"); +_("Set point cloud quality. Higher quality generates better, denser point clouds, but requires more memory and takes longer. Each step up in quality increases processing time roughly by a factor of 4x.Can be one of: %(choices)s. Default: %(default)s"); +_("Path to the image geolocation file containing the camera center coordinates used for georeferencing. If you don't have values for yaw/pitch/roll you can set them to 0. The file needs to use the following format: EPSG: or <+proj definition>image_name geo_x geo_y geo_z [yaw (degrees)] [pitch (degrees)] [roll (degrees)] [horz accuracy (meters)] [vert accuracy (meters)]Default: %(default)s"); +_("Octree depth used in the mesh reconstruction, increase to get more vertices, recommended values are 8-12. Default: %(default)s"); +_("Maximum number of frames to extract from video files for processing. Set to 0 for no limit. Default: %(default)s"); +_("Choose what to merge in the merge step in a split dataset. By default all available outputs are merged. Options: %(choices)s. Default: %(default)s"); _("Skip generation of the orthophoto. This can save time if you only need 3D results or DEMs. Default: %(default)s"); +_("Simple Morphological Filter slope parameter (rise over run). Default: %(default)s"); +_("Perform image matching with the nearest images based on GPS exif data. Set to 0 to match by triangulation. Default: %(default)s"); +_("Turn on rolling shutter correction. If the camera has a rolling shutter and the images were taken in motion, you can turn on this option to improve the accuracy of the results. See also --rolling-shutter-readout. Default: %(default)s"); _("Use this tag to build a DTM (Digital Terrain Model, ground only) using a simple morphological filter. Check the --dem* and --smrf* parameters for finer tuning. Default: %(default)s"); -_("URL to a ClusterODM instance for distributing a split-merge workflow on multiple nodes in parallel. Default: %(default)s"); -_("Set a value in meters for the GPS Dilution of Precision (DOP) information for all images. If your images are tagged with high precision GPS information (RTK), this value will be automatically set accordingly. You can use this option to manually set it in case the reconstruction fails. Lowering this option can sometimes help control bowling-effects over large areas. Default: %(default)s"); -_("Create Cloud-Optimized GeoTIFFs instead of normal GeoTIFFs. Default: %(default)s"); -_("Simple Morphological Filter window radius parameter (meters). Default: %(default)s"); -_("Generates a polygon around the cropping area that cuts the orthophoto around the edges of features. This polygon can be useful for stitching seamless mosaics with multiple overlapping orthophotos. Default: %(default)s"); +_("Skips dense reconstruction and 3D model generation. It generates an orthophoto directly from the sparse reconstruction. If you just need an orthophoto and do not need a full 3D model, turn on this option. Default: %(default)s"); _("Set the radiometric calibration to perform on images. When processing multispectral and thermal images you should set this option to obtain reflectance/temperature values (otherwise you will get digital number values). [camera] applies black level, vignetting, row gradient gain/exposure compensation (if appropriate EXIF tags are found) and computes absolute temperature values. [camera+sun] is experimental, applies all the corrections of [camera], plus compensates for spectral radiance registered via a downwelling light sensor (DLS) taking in consideration the angle of the sun. Can be one of: %(choices)s. Default: %(default)s"); -_("Specify the distance between camera shot locations and the outer edge of the boundary when computing the boundary with --auto-boundary. Set to 0 to automatically choose a value. Default: %(default)s"); -_("Simple Morphological Filter slope parameter (rise over run). Default: %(default)s"); -_("Turn off camera parameter optimization during bundle adjustment. This can be sometimes useful for improving results that exhibit doming/bowling or when images are taken with a rolling shutter camera. Default: %(default)s"); -_("Do not use GPU acceleration, even if it's available. Default: %(default)s"); -_("Simple Morphological Filter elevation threshold parameter (meters). Default: %(default)s"); -_("Generate OBJs that have a single material and a single texture file instead of multiple ones. Default: %(default)s"); -_("Orthophoto resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate.Default: %(default)s"); +_("Set feature extraction quality. Higher quality generates better features, but requires more memory and takes longer. Can be one of: %(choices)s. Default: %(default)s"); +_("Name of dataset (i.e subfolder name within project folder). Default: %(default)s"); _("Filters the point cloud by keeping only a single point around a radius N (in meters). This can be useful to limit the output resolution of the point cloud and remove duplicate points. Set to 0 to disable sampling. Default: %(default)s"); -_("Copy output results to this folder after processing."); -_("Export the georeferenced point cloud in CSV format. Default: %(default)s"); -_("Perform image matching with the nearest N images based on image filename order. Can speed up processing of sequential images, such as those extracted from video. It is applied only on non-georeferenced datasets. Set to 0 to disable. Default: %(default)s"); -_("Simple Morphological Filter elevation scalar parameter. Default: %(default)s"); +_("End processing at this stage. Can be one of: %(choices)s. Default: %(default)s"); +_("Computes an euclidean raster map for each DEM. The map reports the distance from each cell to the nearest NODATA value (before any hole filling takes place). This can be useful to isolate the areas that have been filled. Default: %(default)s"); +_("Use the camera parameters computed from another dataset instead of calculating them. Can be specified either as path to a cameras.json file or as a JSON string representing the contents of a cameras.json file. Default: %(default)s"); +_("The maximum output resolution of extracted video frames in pixels. Default: %(default)s"); +_("Delete heavy intermediate files to optimize disk space usage. This affects the ability to restart the pipeline from an intermediate stage, but allows datasets to be processed on machines that don't have sufficient disk space available. Default: %(default)s"); +_("Do not attempt to merge partial reconstructions. This can happen when images do not have sufficient overlap or are isolated. Default: %(default)s"); +_("The maximum number of processes to use in various processes. Peak memory requirement is ~1GB per thread and 2 megapixel image resolution. Default: %(default)s"); _("Use this tag to build a DSM (Digital Surface Model, ground + objects) using a progressive morphological filter. Check the --dem* parameters for finer tuning. Default: %(default)s"); -_("Save the georeferenced point cloud in Cloud Optimized Point Cloud (COPC) format. Default: %(default)s"); -_("Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. Starting with a radius equal to the output resolution, N different DEMs are generated with progressively bigger radius using the inverse distance weighted (IDW) algorithm and merged together. Remaining gaps are then merged using nearest neighbor interpolation. Default: %(default)s"); -_("The maximum vertex count of the output mesh. Default: %(default)s"); -_("DSM/DTM resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate. Default: %(default)s"); -_("Decimate the points before generating the DEM. 1 is no decimation (full quality). 100 decimates ~99%% of the points. Useful for speeding up generation of DEM results in very large datasets. Default: %(default)s"); -_("Skip generation of PDF report. This can save time if you don't need a report. Default: %(default)s"); -_("Matcher algorithm, Fast Library for Approximate Nearest Neighbors or Bag of Words. FLANN is slower, but more stable. BOW is faster, but can sometimes miss valid matches. BRUTEFORCE is very slow but robust.Can be one of: %(choices)s. Default: %(default)s"); -_("Choose the structure from motion algorithm. For aerial datasets, if camera GPS positions and angles are available, triangulation can generate better results. For planar scenes captured at fixed altitude with nadir-only images, planar can be much faster. Can be one of: %(choices)s. Default: %(default)s"); -_("Set this parameter if you want to generate a Google Earth (KMZ) rendering of the orthophoto. Default: %(default)s"); _("Use a full 3D mesh to compute the orthophoto instead of a 2.5D mesh. This option is a bit faster and provides similar results in planar areas. Default: %(default)s"); -_("Average number of images per submodel. When splitting a large dataset into smaller submodels, images are grouped into clusters. This value regulates the number of images that each cluster should have on average. Default: %(default)s"); -_("GeoJSON polygon limiting the area of the reconstruction. Can be specified either as path to a GeoJSON file or as a JSON string representing the contents of a GeoJSON file. Default: %(default)s"); -_("Perform ground rectification on the point cloud. This means that wrongly classified ground points will be re-classified and gaps will be filled. Useful for generating DTMs. Default: %(default)s"); -_("The maximum number of processes to use in various processes. Peak memory requirement is ~1GB per thread and 2 megapixel image resolution. Default: %(default)s"); -_("Do not attempt to merge partial reconstructions. This can happen when images do not have sufficient overlap or are isolated. Default: %(default)s"); -_("Name of dataset (i.e subfolder name within project folder). Default: %(default)s"); -_("Use this tag if you have a GCP File but want to use the EXIF information for georeferencing instead. Default: %(default)s"); -_("Set the compression to use for orthophotos. Can be one of: %(choices)s. Default: %(default)s"); -_("Set point cloud quality. Higher quality generates better, denser point clouds, but requires more memory and takes longer. Each step up in quality increases processing time roughly by a factor of 4x.Can be one of: %(choices)s. Default: %(default)s"); +_("Simple Morphological Filter elevation threshold parameter (meters). Default: %(default)s"); +_("DSM/DTM resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate. Default: %(default)s"); +_("When processing multispectral datasets, ODM will automatically align the images for each band. If the images have been postprocessed and are already aligned, use this option. Default: %(default)s"); +_("Set a value in meters for the GPS Dilution of Precision (DOP) information for all images. If your images are tagged with high precision GPS information (RTK), this value will be automatically set accordingly. You can use this option to manually set it in case the reconstruction fails. Lowering this option can sometimes help control bowling-effects over large areas. Default: %(default)s"); +_("Radius of the overlap between submodels. After grouping images into clusters, images that are closer than this radius to a cluster are added to the cluster. This is done to ensure that neighboring submodels overlap. Default: %(default)s"); +_("Set this parameter if you want a striped GeoTIFF. Default: %(default)s"); +_("Skip generation of a full 3D model. This can save time if you only need 2D results such as orthophotos and DEMs. Default: %(default)s"); _("Keep faces in the mesh that are not seen in any camera. Default: %(default)s"); -_("Turn on rolling shutter correction. If the camera has a rolling shutter and the images were taken in motion, you can turn on this option to improve the accuracy of the results. See also --rolling-shutter-readout. Default: %(default)s"); -_("Skips dense reconstruction and 3D model generation. It generates an orthophoto directly from the sparse reconstruction. If you just need an orthophoto and do not need a full 3D model, turn on this option. Default: %(default)s"); +_("Create Cloud-Optimized GeoTIFFs instead of normal GeoTIFFs. Default: %(default)s"); +_("URL to a ClusterODM instance for distributing a split-merge workflow on multiple nodes in parallel. Default: %(default)s"); +_("Generate single file Binary glTF (GLB) textured models. Default: %(default)s"); +_("Perform image matching with the nearest N images based on image filename order. Can speed up processing of sequential images, such as those extracted from video. It is applied only on non-georeferenced datasets. Set to 0 to disable. Default: %(default)s"); +_("Path to the image groups file that controls how images should be split into groups. The file needs to use the following format: image_name group_nameDefault: %(default)s"); +_("Do not use GPU acceleration, even if it's available. Default: %(default)s"); +_("Permanently delete all previous results and rerun the processing pipeline."); _("Automatically crop image outputs by creating a smooth buffer around the dataset boundaries, shrunk by N meters. Use 0 to disable cropping. Default: %(default)s"); -_("Computes an euclidean raster map for each DEM. The map reports the distance from each cell to the nearest NODATA value (before any hole filling takes place). This can be useful to isolate the areas that have been filled. Default: %(default)s"); +_("Use images' GPS exif data for reconstruction, even if there are GCPs present.This flag is useful if you have high precision GPS measurements. If there are no GCPs, this flag does nothing. Default: %(default)s"); +_("Skip normalization of colors across all images. Useful when processing radiometric data. Default: %(default)s"); +_("Override the rolling shutter readout time for your camera sensor (in milliseconds), instead of using the rolling shutter readout database. Note that not all cameras are present in the database. Set to 0 to use the database value. Default: %(default)s"); +_("Minimum number of features to extract per image. More features can be useful for finding more matches between images, potentially allowing the reconstruction of areas with little overlap or insufficient features. More features also slow down processing. Default: %(default)s"); +_("Export the georeferenced point cloud in CSV format. Default: %(default)s"); +_("Generate static tiles for orthophotos and DEMs that are suitable for viewers like Leaflet or OpenLayers. Default: %(default)s"); +_("Displays version number and exits. "); +_("Ignore Ground Sampling Distance (GSD).A memory and processor hungry change relative to the default behavior if set to true. Ordinarily, GSD estimates are used to cap the maximum resolution of image outputs and resizes images when necessary, resulting in faster processing and lower memory usage. Since GSD is an estimate, sometimes ignoring it can result in slightly better image output quality. Never set --ignore-gsd to true unless you are positive you need it, and even then: do not use it. Default: %(default)s"); _("Skip the blending of colors near seams. Default: %(default)s"); +_("Generates a polygon around the cropping area that cuts the orthophoto around the edges of features. This polygon can be useful for stitching seamless mosaics with multiple overlapping orthophotos. Default: %(default)s"); +_("Rerun this stage only and stop. Can be one of: %(choices)s. Default: %(default)s"); +_("Set the compression to use for orthophotos. Can be one of: %(choices)s. Default: %(default)s"); +_("Copy output results to this folder after processing."); +_("The maximum vertex count of the output mesh. Default: %(default)s"); _("Run local bundle adjustment for every image added to the reconstruction and a global adjustment every 100 images. Speeds up reconstruction for very large datasets. Default: %(default)s"); -_("Radius of the overlap between submodels. After grouping images into clusters, images that are closer than this radius to a cluster are added to the cluster. This is done to ensure that neighboring submodels overlap. Default: %(default)s"); -_("Path to the image geolocation file containing the camera center coordinates used for georeferencing. If you don't have values for yaw/pitch/roll you can set them to 0. The file needs to use the following format: EPSG: or <+proj definition>image_name geo_x geo_y geo_z [yaw (degrees)] [pitch (degrees)] [roll (degrees)] [horz accuracy (meters)] [vert accuracy (meters)]Default: %(default)s"); +_("Path to the project folder. Your project folder should contain subfolders for each dataset. Each dataset should have an \"images\" folder."); _("Choose the algorithm for extracting keypoints and computing descriptors. Can be one of: %(choices)s. Default: %(default)s"); -_("Generate OGC 3D Tiles outputs. Default: %(default)s"); -_("Rerun this stage only and stop. Can be one of: %(choices)s. Default: %(default)s"); -_("Export the georeferenced point cloud in Entwine Point Tile (EPT) format. Default: %(default)s"); -_("Path to a GeoTIFF DEM or a LAS/LAZ point cloud that the reconstruction outputs should be automatically aligned to. Experimental. Default: %(default)s"); -_("Use the camera parameters computed from another dataset instead of calculating them. Can be specified either as path to a cameras.json file or as a JSON string representing the contents of a cameras.json file. Default: %(default)s"); -_("Choose what to merge in the merge step in a split dataset. By default all available outputs are merged. Options: %(choices)s. Default: %(default)s"); -_("Delete heavy intermediate files to optimize disk space usage. This affects the ability to restart the pipeline from an intermediate stage, but allows datasets to be processed on machines that don't have sufficient disk space available. Default: %(default)s"); -_("Set a camera projection type. Manually setting a value can help improve geometric undistortion. By default the application tries to determine a lens type from the images metadata. Can be one of: %(choices)s. Default: %(default)s"); +_("Set this parameter if you want to generate a Google Earth (KMZ) rendering of the orthophoto. Default: %(default)s"); +_("Automatically compute image masks using AI to remove the sky. Experimental. Default: %(default)s"); +_("show this help message and exit"); +_("Rerun processing from this stage. Can be one of: %(choices)s. Default: %(default)s"); +_("Perform ground rectification on the point cloud. This means that wrongly classified ground points will be re-classified and gaps will be filled. Useful for generating DTMs. Default: %(default)s"); +_("Choose the structure from motion algorithm. For aerial datasets, if camera GPS positions and angles are available, triangulation can generate better results. For planar scenes captured at fixed altitude with nadir-only images, planar can be much faster. Can be one of: %(choices)s. Default: %(default)s"); +_("Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. Starting with a radius equal to the output resolution, N different DEMs are generated with progressively bigger radius using the inverse distance weighted (IDW) algorithm and merged together. Remaining gaps are then merged using nearest neighbor interpolation. Default: %(default)s"); _("When processing multispectral datasets, you can specify the name of the primary band that will be used for reconstruction. It's recommended to choose a band which has sharp details and is in focus. Default: %(default)s"); -_("Maximum number of frames to extract from video files for processing. Set to 0 for no limit. Default: %(default)s"); -_("Path to the image groups file that controls how images should be split into groups. The file needs to use the following format: image_name group_nameDefault: %(default)s"); -_("Generate single file Binary glTF (GLB) textured models. Default: %(default)s"); +_("Set a camera projection type. Manually setting a value can help improve geometric undistortion. By default the application tries to determine a lens type from the images metadata. Can be one of: %(choices)s. Default: %(default)s"); +_("Path to a GeoTIFF DEM or a LAS/LAZ point cloud that the reconstruction outputs should be automatically aligned to. Experimental. Default: %(default)s"); _("Geometric estimates improve the accuracy of the point cloud by computing geometrically consistent depthmaps but may not be usable in larger datasets. This flag disables geometric estimates. Default: %(default)s"); -_("Permanently delete all previous results and rerun the processing pipeline."); -_("Override the rolling shutter readout time for your camera sensor (in milliseconds), instead of using the rolling shutter readout database. Note that not all cameras are present in the database. Set to 0 to use the database value. Default: %(default)s"); +_("Decimate the points before generating the DEM. 1 is no decimation (full quality). 100 decimates ~99%% of the points. Useful for speeding up generation of DEM results in very large datasets. Default: %(default)s"); +_("Skip alignment of submodels in split-merge. Useful if GPS is good enough on very large datasets. Default: %(default)s"); +_("Automatically set a boundary using camera shot locations to limit the area of the reconstruction. This can help remove far away background artifacts (sky, background landscapes, etc.). See also --boundary. Default: %(default)s"); +_("Filters the point cloud by removing points that deviate more than N standard deviations from the local mean. Set to 0 to disable filtering. Default: %(default)s"); +_("Simple Morphological Filter elevation scalar parameter. Default: %(default)s"); +_("Export the georeferenced point cloud in Entwine Point Tile (EPT) format. Default: %(default)s"); +_("Export the georeferenced point cloud in LAS format. Default: %(default)s"); _("Build orthophoto overviews for faster display in programs such as QGIS. Default: %(default)s"); -_("Set feature extraction quality. Higher quality generates better features, but requires more memory and takes longer. Can be one of: %(choices)s. Default: %(default)s"); -_("Generate static tiles for orthophotos and DEMs that are suitable for viewers like Leaflet or OpenLayers. Default: %(default)s"); -_("Displays version number and exits. "); -_("Minimum number of features to extract per image. More features can be useful for finding more matches between images, potentially allowing the reconstruction of areas with little overlap or insufficient features. More features also slow down processing. Default: %(default)s"); -_("Use images' GPS exif data for reconstruction, even if there are GCPs present.This flag is useful if you have high precision GPS measurements. If there are no GCPs, this flag does nothing. Default: %(default)s"); -_("Octree depth used in the mesh reconstruction, increase to get more vertices, recommended values are 8-12. Default: %(default)s"); -_("Rerun processing from this stage. Can be one of: %(choices)s. Default: %(default)s"); +_("Skip generation of PDF report. This can save time if you don't need a report. Default: %(default)s"); _("Classify the point cloud outputs. You can control the behavior of this option by tweaking the --dem-* parameters. Default: %(default)s"); -_("Export the georeferenced point cloud in LAS format. Default: %(default)s"); -_("Ignore Ground Sampling Distance (GSD).A memory and processor hungry change relative to the default behavior if set to true. Ordinarily, GSD estimates are used to cap the maximum resolution of image outputs and resizes images when necessary, resulting in faster processing and lower memory usage. Since GSD is an estimate, sometimes ignoring it can result in slightly better image output quality. Never set --ignore-gsd to true unless you are positive you need it, and even then: do not use it. Default: %(default)s"); +_("Orthophoto resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate.Default: %(default)s"); +_("Simple Morphological Filter window radius parameter (meters). Default: %(default)s"); +_("GeoJSON polygon limiting the area of the reconstruction. Can be specified either as path to a GeoJSON file or as a JSON string representing the contents of a GeoJSON file. Default: %(default)s"); +_("Average number of images per submodel. When splitting a large dataset into smaller submodels, images are grouped into clusters. This value regulates the number of images that each cluster should have on average. Default: %(default)s"); +_("Use this tag if you have a GCP File but want to use the EXIF information for georeferencing instead. Default: %(default)s"); +_("Turn off camera parameter optimization during bundle adjustment. This can be sometimes useful for improving results that exhibit doming/bowling or when images are taken with a rolling shutter camera. Default: %(default)s"); _("Path to the file containing the ground control points used for georeferencing. The file needs to use the following format: EPSG: or <+proj definition>geo_x geo_y geo_z im_x im_y image_name [gcp_name] [extra1] [extra2]Default: %(default)s"); -_("Skip alignment of submodels in split-merge. Useful if GPS is good enough on very large datasets. Default: %(default)s"); +_("Set this parameter if you want to generate a PNG rendering of the orthophoto. Default: %(default)s"); +_("Automatically compute image masks using AI to remove the background. Experimental. Default: %(default)s"); +_("Generate OBJs that have a single material and a single texture file instead of multiple ones. Default: %(default)s"); +_("Generate OGC 3D Tiles outputs. Default: %(default)s"); +_("Save the georeferenced point cloud in Cloud Optimized Point Cloud (COPC) format. Default: %(default)s"); +_("Specify the distance between camera shot locations and the outer edge of the boundary when computing the boundary with --auto-boundary. Set to 0 to automatically choose a value. Default: %(default)s"); diff --git a/locale b/locale index 0a68f6ed5..64eee8cae 160000 --- a/locale +++ b/locale @@ -1 +1 @@ -Subproject commit 0a68f6ed5172a8838571a9872bcc3cb4f310794c +Subproject commit 64eee8cae41e5fe40a0123400f12600fcd6125b9 From 44b24952912fdda9a4c57c94544d154356f2c28a Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 3 Oct 2023 15:41:05 -0400 Subject: [PATCH 04/10] Fix raster export --- app/api/tiler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/api/tiler.py b/app/api/tiler.py index 855eef569..8ae27bbd3 100644 --- a/app/api/tiler.py +++ b/app/api/tiler.py @@ -560,6 +560,9 @@ def post(self, request, pk=None, project_pk=None, asset_type=None): raise exceptions.ValidationError(_("Both formula and bands parameters are required")) if formula and bands: + if bands == 'auto': + bands, _ = get_auto_bands(task.orthophoto_bands, formula) + try: expr, _discard_ = lookup_formula(formula, bands) except ValueError as e: From 4897d4e52a911f0732ef29813f607785ec4aab42 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 3 Oct 2023 22:37:10 -0400 Subject: [PATCH 05/10] Fix var override --- app/api/tiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/tiler.py b/app/api/tiler.py index 8ae27bbd3..470930cdc 100644 --- a/app/api/tiler.py +++ b/app/api/tiler.py @@ -311,7 +311,7 @@ def get(self, request, pk=None, project_pk=None, tile_type="", z="", x="", y="", if hillshade == '' or hillshade == '0': hillshade = None if tilesize == '' or tilesize is None: tilesize = 256 if bands == 'auto' and formula: - bands, _ = get_auto_bands(task.orthophoto_bands, formula) + bands, _discard_ = get_auto_bands(task.orthophoto_bands, formula) try: tilesize = int(tilesize) @@ -561,7 +561,7 @@ def post(self, request, pk=None, project_pk=None, asset_type=None): if formula and bands: if bands == 'auto': - bands, _ = get_auto_bands(task.orthophoto_bands, formula) + bands, _discard_ = get_auto_bands(task.orthophoto_bands, formula) try: expr, _discard_ = lookup_formula(formula, bands) From 80dcff41ca39cf12c2ce1b4946ba0866822c2893 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 4 Oct 2023 13:04:39 -0400 Subject: [PATCH 06/10] Add unit tests --- .../app/js/components/ProjectListItem.jsx | 2 -- app/tests/test_api_task.py | 16 +++++++++- app/tests/test_formulas.py | 32 +++++++++++++++++-- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 51023e933..31d6ec429 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -573,8 +573,6 @@ class ProjectListItem extends React.Component { render() { const { refreshing, data, filterTags } = this.state; const numTasks = data.tasks.length; - - // const numValidTasks = data.tasks.filter(t => ) const canEdit = this.hasPermission("change"); const userTags = Tags.userTags(data.tags); let deleteWarning = _("All tasks, images and models associated with this project will be permanently deleted. Are you sure you want to continue?"); diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index bf3af575a..a3df03470 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -497,6 +497,10 @@ def test_task(self): self.assertEqual(metadata['algorithms'], []) self.assertEqual(metadata['color_maps'], []) + # Auto bands + self.assertEqual(metadata['auto_bands']['filter'], '') + self.assertEqual(metadata['auto_bands']['match'], None) + # Address key is removed self.assertFalse('address' in metadata) @@ -531,6 +535,10 @@ def test_task(self): self.assertTrue(len(metadata['algorithms']) > 0) self.assertTrue(len(metadata['color_maps']) > 0) + # Auto band is populated + self.assertEqual(metadata['auto_bands']['filter'], 'RGN') + self.assertEqual(metadata['auto_bands']['match'], False) + # Algorithms have valid keys for k in ['id', 'filters', 'expr', 'help']: for a in metadata['algorithms']: @@ -557,6 +565,10 @@ def test_task(self): self.assertEqual(metadata['statistics']['1']['min'], algos['VARI']['range'][0]) self.assertEqual(metadata['statistics']['1']['max'], algos['VARI']['range'][1]) + # Formula can be set to auto + res = client.get("/api/projects/{}/tasks/{}/orthophoto/metadata?formula=VARI&bands=auto".format(project.id, task.id)) + self.assertEqual(res.status_code, status.HTTP_200_OK) + tile_path = { 'orthophoto': '17/32042/46185', 'dsm': '18/64083/92370', @@ -665,7 +677,9 @@ def test_task(self): ("orthophoto", "formula=VARI&bands=RGB", status.HTTP_200_OK), ("orthophoto", "formula=VARI&bands=invalid", status.HTTP_400_BAD_REQUEST), ("orthophoto", "formula=invalid&bands=RGB", status.HTTP_400_BAD_REQUEST), - + ("orthophoto", "formula=NDVI&bands=auto", status.HTTP_200_OK), + ("orthophoto", "formula=NDVI&bands=auto", status.HTTP_200_OK), + ("orthophoto", "formula=NDVI&bands=RGN&color_map=rdylgn&rescale=-1,1", status.HTTP_200_OK), ("orthophoto", "formula=NDVI&bands=RGN&color_map=rdylgn&rescale=1,-1", status.HTTP_200_OK), diff --git a/app/tests/test_formulas.py b/app/tests/test_formulas.py index ab9117da0..70fb9f382 100644 --- a/app/tests/test_formulas.py +++ b/app/tests/test_formulas.py @@ -1,6 +1,6 @@ import re from django.test import TestCase -from app.api.formulas import lookup_formula, get_algorithm_list, get_camera_filters_for, algos +from app.api.formulas import lookup_formula, get_algorithm_list, get_camera_filters_for, algos, get_auto_bands class TestFormulas(TestCase): def setUp(self): @@ -48,4 +48,32 @@ def test_algo_list(self): # Filters are less than 5 bands for f in i['filters']: bands = list(set(re.findall(pattern, f))) - self.assertTrue(len(bands) <= 5) \ No newline at end of file + self.assertTrue(len(bands) <= 5) + + def test_auto_bands(self): + obands = [{'name': 'red', 'description': 'red'}, + {'name': 'green', 'description': 'green'}, + {'name': 'blue', 'description': 'blue'}, + {'name': 'gray', 'description': 'nir'}, + {'name': 'alpha', 'description': None}] + + self.assertEqual(get_auto_bands(obands, "NDVI")[0], "RGBN") + self.assertTrue(get_auto_bands(obands, "NDVI")[1]) + + self.assertEqual(get_auto_bands(obands, "Celsius")[0], "L") + self.assertFalse(get_auto_bands(obands, "Celsius")[1]) + + self.assertEqual(get_auto_bands(obands, "VARI")[0], "RGBN") + self.assertTrue(get_auto_bands(obands, "VARI")[0]) + + obands = [{'name': 'red', 'description': None}, + {'name': 'green', 'description': None}, + {'name': 'blue', 'description': None}, + {'name': 'gray', 'description': None}, + {'name': 'alpha', 'description': None}] + + self.assertEqual(get_auto_bands(obands, "NDVI")[0], "RGN") + self.assertFalse(get_auto_bands(obands, "NDVI")[1]) + + self.assertEqual(get_auto_bands(obands, "VARI")[0], "RGB") + self.assertFalse(get_auto_bands(obands, "VARI")[1]) \ No newline at end of file From 13121566ad909fd466979448e3bbbfce8b045fbc Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 4 Oct 2023 13:13:32 -0400 Subject: [PATCH 07/10] Update README --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 976d2f145..0521f50c0 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,11 @@ A user-friendly, commercial grade software for drone image processing. Generate Windows and macOS users can purchase an automated [installer](https://www.opendronemap.org/webodm/download#installer), which makes the installation process easier. -To install WebODM manually, these steps should get you up and running: +There's also a cloud-hosted version of WebODM available from [webodm.net](https://webodm.net). -* Install the following applications (if they are not installed already): +To install WebODM manually on your machine: + +* Install the following applications: - [Git](https://git-scm.com/downloads) - [Docker](https://www.docker.com/) - [Docker-compose](https://docs.docker.com/compose/install/) From f67f435a1cac618a3b4a20c027a1c63b53c014b4 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 4 Oct 2023 13:34:31 -0400 Subject: [PATCH 08/10] Ignore celery results when appropriate --- worker/tasks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index 392582b79..5ca8f24c2 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -31,7 +31,7 @@ # What class to use for async results, since during testing we need to mock it TestSafeAsyncResult = worker.celery.MockAsyncResult if settings.TESTING else app.AsyncResult -@app.task +@app.task(ignore_result=True) def update_nodes_info(): if settings.NODE_OPTIMISTIC_MODE: return @@ -57,7 +57,7 @@ def update_nodes_info(): processing_node.hostname = check_hostname processing_node.save() -@app.task +@app.task(ignore_result=True) def cleanup_projects(): # Delete all projects that are marked for deletion # and that have no tasks left @@ -68,7 +68,7 @@ def cleanup_projects(): logger.info("Deleted {} projects".format(count_dict['app.Project'])) -@app.task +@app.task(ignore_result=True) def cleanup_tmp_directory(): # Delete files and folder in the tmp directory that are # older than 24 hours @@ -99,7 +99,7 @@ def loop(): t.start() return stopped.set -@app.task +@app.task(ignore_result=True) def process_task(taskId): lock_id = 'task_lock_{}'.format(taskId) cancel_monitor = None @@ -159,7 +159,7 @@ def get_pending_tasks(): processing_node__isnull=False, partial=False) | Q(pending_action__isnull=False, partial=False)) -@app.task +@app.task(ignore_result=True) def process_pending_tasks(): tasks = get_pending_tasks() for task in tasks: @@ -207,7 +207,7 @@ def export_pointcloud(self, input, **opts): logger.error(str(e)) return {'error': str(e)} -@app.task +@app.task(ignore_result=True) def check_quotas(): profiles = Profile.objects.filter(quota__gt=-1) for p in profiles: From 62d5185a79a7857a6da17f26a4d565a962a993db Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 4 Oct 2023 13:39:16 -0400 Subject: [PATCH 09/10] Fix test --- app/tests/test_api_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index a3df03470..d68d4504d 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -536,7 +536,7 @@ def test_task(self): self.assertTrue(len(metadata['color_maps']) > 0) # Auto band is populated - self.assertEqual(metadata['auto_bands']['filter'], 'RGN') + self.assertEqual(metadata['auto_bands']['filter'], '') self.assertEqual(metadata['auto_bands']['match'], False) # Algorithms have valid keys From 49c9f2d7b893a63339a12708f9f180db01011a55 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 4 Oct 2023 15:51:53 -0400 Subject: [PATCH 10/10] Fix test --- app/tests/test_api_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index d68d4504d..3c6d095ff 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -537,7 +537,7 @@ def test_task(self): # Auto band is populated self.assertEqual(metadata['auto_bands']['filter'], '') - self.assertEqual(metadata['auto_bands']['match'], False) + self.assertEqual(metadata['auto_bands']['match'], None) # Algorithms have valid keys for k in ['id', 'filters', 'expr', 'help']: