From c1e79fef7e4b8cbd9179392b76bae0698490e62b Mon Sep 17 00:00:00 2001 From: Chris Harding Date: Sun, 24 Oct 2021 16:55:22 -0500 Subject: [PATCH] version 3.5 --- .gitignore | 3 +- NEWS.md | 15 +- TouchTerrain_jupyter_for_starters.ipynb | 204 +++++++++--------- requirements.txt | 3 +- .../Run_TouchTerrain_debug_server_app.py | 8 + touchterrain/server/templates/touchterrain.js | 3 +- 6 files changed, 129 insertions(+), 107 deletions(-) create mode 100644 touchterrain/server/Run_TouchTerrain_debug_server_app.py diff --git a/.gitignore b/.gitignore index 3446bb5..ed9c196 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ # Created by https://www.toptal.com/developers/gitignore/api/python,visualstudiocode # Edit at https://www.toptal.com/developers/gitignore?templates=python,visualstudiocode -# CH: don't put debug server script on github +# CH: ignore this script in project root (where it has to be run), it belongs in the server folder Run_TouchTerrain_debug_server_app.py # my private API key for testing @@ -41,6 +41,7 @@ terrain !test/test_TouchTerrain_standalone.py !stuff/TouchTerrain_bottom_example.png !stuff/example_config.json +!touchterrain/server/Run_TouchTerrain_debug_server_app.py ### Python ### # Byte-compiled / optimized / DLL files diff --git a/NEWS.md b/NEWS.md index 4ec2ffb..ff2b9b7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,11 @@ +Oct. 24, 2021 (version 3.5) + - rewrote how large STL/OBJ files are created, they are now written during the process to disk + rather first into memory which turned out to be too much for our 4 Gb server which would + kill the process and end in a time-out. Still not sure what happens if several large jobs are + run but this will certainly improve the odds of finishing. + - accepted offset_masks_lower and fill_holes options contributed by Anson Liu (NOT well tested outside his use case) + - some edits to the jupyter starter notebook + Jul. 16, 2021 (version 3.4) - fixed bug with kml/kmz file showing up in the browser but then not actually used for processing (Once again, thank you, Nick!) @@ -11,14 +19,11 @@ Apr. 29, 2021 (version 3.3) Apr. 9, 2021 - added a manual option to switch border optimization on/off (smooth_borders) -Mar. 30, 2021 +Mar. 30, 2021 (version 3.2.1) - fixed double application of z-scale (thanks to Simon Scherrer!) - made version 3.2.1 - -Mar. 12, 2021 +Mar. 12, 2021 (version 3.2) - added Canadian 20m NRCan/CDEM and Australian AU/GA/AUSTRALIA_5M_DEM -- made version 3.2 Mar. 11, 2021 - temporary fix for bad google-api-python-client version diff --git a/TouchTerrain_jupyter_for_starters.ipynb b/TouchTerrain_jupyter_for_starters.ipynb index 150693e..247a3c0 100644 --- a/TouchTerrain_jupyter_for_starters.ipynb +++ b/TouchTerrain_jupyter_for_starters.ipynb @@ -5,12 +5,12 @@ "metadata": {}, "source": [ "# TouchTerrain standalone in a jupyter notebook - starter edition\n", - "Chris Harding, Mar 8 , 2021 ()\n", + "Chris Harding, Oct. 24 , 2021 ()\n", "\n", "- This notebook is meant for users who are new to jupyter notebooks and Python. It walks through the process of using TouchTerrain with many more small steps at a slower pace, makes less assumptions and gives more explanations - but will also hide some advanced details.\n", "- It assumes that you are using it via the touchterrain docker container and that you have downloaded the docker image from https://github.com/ChHarding/TouchTerrain_jupyter_docker (which contains a complete Linux system with all packages and libraries needed for TouchTerrain), have created the touch_terrain docker container, have run the install script and have started a local jupyter server via the run script.\n", "- This notebook is a companion to the more generic TouchTerrain_standalone_jupyter_notebook.ipynb. Once you've worked through the starter notebook and are more familiar with TouchTerrain, you may want to graduate to it as it can be more efficient and faster to work with.\n", - "- Before we start, you may want to rename this notebook into e.g. TouchTerrain_starter_chris.ipynb) to make sure it doesn't get overwritten by accident. " + "- Before we start, you may want to rename this notebook (e.g. into TouchTerrain_starter_chris.ipynb) to make sure it doesn't get overwritten by accident. " ] }, { @@ -40,6 +40,8 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "### One-time Authentication with Earth Engine\n", "- the next cell will initiate an authorization with Google. It's quite painless and uses a browser\n", @@ -54,9 +56,7 @@ "- The web page will have you select a Google account for use with Earth Engine and give you a code (copy it)\n", "- Paste in your code here: `Enter verification code: ` and hit Enter, you should get `Successfully saved authorization token.`\n", "- Important: this needs to be done only once when using Touchterrain. Once done, you should put the `#` back in front of ee.Authenticate() (so the line is `#ee.Authenticate()` again), so you're not running it again by accident. But, if you ever remove or update the image, you'll have to authenticate again." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -99,6 +99,8 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "## Define the values for processing settings\n", "- We now need to define the processing settings for the 3D model we want to create. Unlike the web app, they need to be set via Python variables.\n", @@ -122,24 +124,22 @@ "_#print(123)\n", "_print(123)\n", "```" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ - "### Import a locally stored geotiff DEM file \n", - "__If you don't plan to use a local geotiff and instead want to manually digitze your print area, skip this cell and go to the next__\n", + "### Use a locally stored geotiff DEM file instead of an online DEM\n", + "__If you don't plan to use a local geotiff and instead want to use an online DEM, skip this cell and go to the next__\n", "- You may want create a terrain model from a geotiff DEM file that you've downloaded or created via a GIS. As TouchTerrain needs to import this file during processing, you first need to copy (upload) it via the jupyter file manager (tree).\n", "- Go to the same folder as your .ipynb notebook file and hit _Upload_ in the upper right corner. This will copy a geotiff from your native OS into the container.\n", "- Alternatively, you could make a separate data folder first, again in the same folder as your .ipynb notebook file, something like mydata. Navigate into this folder and upload your geotiff(s) there\n", "- The folder called __stuff__ contains a very small example geotiff file called __pyramid.tif__. From the folder you notebook is in, the path to this file is therefore __stuff/pyramid.tif__ (Unix uses a /, not a \\ like Windows, for folders!)\n", "- The cell below set the value of __importedDEM__ to this path (`stuff/pyramid.tif`). To import your file, replace this with the name of your geotiff (if you uploaded it the notebook's folder) e.g. myDEM.tif. If you're using your own data folder prepend the folder's name in frony of the filename, e.g. mydata/myDEM.tif. Note that this must not start with a / !\n", "- if you do NOT want to use a local geotiff and want to use online DEM rasters instead, set the value to None. Comment/Uncomment the appropriate choice below and hit Shift-Enter\n", - "- you can skip _Select DEM source_ and _Select print area on geemap_ as the geotiff is your DEM source and implicitly defines the print area. Go straight to _Setting the tile width_" - ], - "cell_type": "markdown", - "metadata": {} + "- if you are using a local geotiff, skip _Select DEM source_ and _Select print area on geemap_ as the geotiff is your DEM source and implicitly defines the print area. Go straight to _Setting the tile width_" + ] }, { "cell_type": "code", @@ -158,9 +158,34 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Select the name of the DEM source\n", + "- Before you can select the actual print area you need to select which online DEM source will be used. The setting is called `DEM_name`.\n", + "- There are several options but for starters you only need to decide among two options:\n", + " - If your area is in the lower 48 US states, use `USGS/NED`\n", + " - Anywhere else on the globe use: `JAXA/ALOS/AW3D30/V2_2`\n", + "- Again, comment/uncomment the appropriate line in the cell below and hit Shift-Enter" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Comment out one of the following two lines\n", + "args[\"DEM_name\"] = \"USGS/NED\" # area within the lower 48 (US)\n", + "#args[\"DEM_name\"] = \"JAXA/ALOS/AW3D30/V2_2\" # area is outside the US (worldwide)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, "source": [ "### Import GPX path lines\n", - "__ This is more of an expert option, if you are just starting, just skip this ... __\n", + "__This is more of an expert option, if you are just starting, just skip this and go to the next cell__\n", "- You can drape one or more gpx (path) files over your terrain\n", "- Similar to importedDEM, the file names for the GPX file(s) are stored in the importedGPX setting.\n", "- However, as you can import more than one GPX file, you need to put the path name into a list, i.e. inside brackets, even if you just have one GPX file. Separate multiple GPX files with a comma: \n", @@ -171,9 +196,7 @@ "- `gpxPathThickness` (meters) controls the thickness of the path, which makes it more pronouced when printed\n", "- There are several GPX example files in the stuff folder. To use them, uncomment the first 4 lines in the cell below and comment out the last line.\n", "- If you're not using GPX files, just leave the cell as is and run it. Setting importedGPX to None means that no GPX files are imported.\n" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -191,37 +214,21 @@ ] }, { - "source": [ - "### Select the name of the DEM source\n", - "- Before you can select the actual print area you need to select which online DEM source will be used. The setting is called `DEM_name`.\n", - "- There are several options but for starters you only need to decide among two options:\n", - " - If your area is in the lower 48 US states, use `USGS/NED`\n", - " - Anywhere else on the globe use: `JAXA/ALOS/AW3D30/V2_2`\n", - "- Again, comment/uncomment the appropriate line in the cell below and hit Shift-Enter" - ], "cell_type": "markdown", - "metadata": {} - }, - { - "source": [ - "# Comment out one of the following two lines\n", - "args[\"DEM_name\"] = \"USGS/NED\" # area within the lower 48 (US)\n", - "#args[\"DEM_name\"] = \"JAXA/ALOS/AW3D30/V2_2\" # area is outside the US (worldwide)" - ], - "cell_type": "code", "metadata": {}, - "execution_count": null, - "outputs": [] - }, - { "source": [ - "### Select print area via lat/long coordinates of its corners (optional)\n", + "## Select your print area\n", + "-There are several ways to define the area you want to print. Use only one and skip the cells for the other two options. The go to _Setting th tile width_\n", + "- A) Enter corner coordinates of a box into a code cell\n", + "- B) Use your mouse to digitize a box, circle or polygon on a geemap\n", + "- C) Load a kml file (typically created via Google Earth) that contains a single polygon\n", + "\n", + "\n", + "### A) Select print area via lat/long coordinates of its corners (optional)\n", "- If you'd rather define your area through actual coordinates, change the default values in the cell below accordingly. \n", "- Note that you *must* give it the lat/long of the bottom left (South-West) corner and lat/long of the top right (North-East) corner!\n", "- After you've run the cell, jump over the geemap selection and the kml file outline (which would override your settings) and jump straight to `Setting the tile width`" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -240,16 +247,16 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ - "### Select print area interactively via geemap\n", + "### B) Select print area interactively via geemap\n", "- If you didn't use a local geotiff or set your the print area's corner coordinates, run the next cell to show an interactive map, from which you will select your print area. \n", "- Valid areas are covered by a grey hillshade layer in the background. If your desired area is not covered, go back and select the worldwide DEM source, and re-run the next cell.\n", "- If you're importing GPX files, they will show up on the map as cyan lines.\n", "- To find the desired print area, use left mouse drag to pan and mouse wheel (or +- buttons) to zoom in/out.\n", "- You can also use the search buttons (globe on top left or spyglass) which will place a marker at the result's location." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -277,15 +284,15 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "- Once you found a good general area, hit a Draw button in the center left to digitize the exact outline of your terrain model. \n", "- Your options are: Draw a Polygon or Draw a Rectangle or Draw a Circle. Do NOT use Draw a Marker or a Polyline!\n", "- Draw a Rectangle is the simplest method.\n", "- Note:I do not know how to edit or delete outlines. If you change you mind you can simply draw another outline on top of previous outline, which will then be used. If this gets too cluttered, you can always re-run the cell above.\n", "- Once you're happy with your outline, run the cell below. It will store the outline and use it to create your terrain model. If you change your mind, simply draw another outline and re-run the cell below, it will always take the most recent outline." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -299,15 +306,14 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ - "### Using an optional KML file to define the outline\n", + "### C) Using a KML file to define the outline of the area\n", "- Instead of digitizing an outline in the geemap, you could instead use a polygon stored in a kml file. You can digitize a polygon in Google Earth and store it as a kml file (NOT a kmz file!). \n", "- Again, place the kml file in the same folder as the notebook (or in a subfolder) and set `poly_file` to its location.\n", - "- The stuff folder contains an example kml file\n", - "- Be aware that if you decide to use a kml file, you must set `polygon` to None, otherwise your digitized gee polygon will be used and the kml file will be ignored!" - ], - "cell_type": "markdown", - "metadata": {} + "- The stuff folder contains an example kml file" + ] }, { "cell_type": "code", @@ -324,16 +330,16 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "### Setting the tile width\n", - "- (if you used a local geotiff, resume here)\n", + "- (if you used a local geotiff file as DEM, resume here)\n", "- the tilewidth setting defines how large the selected print area will be after it's been printed. Units are in mm.\n", "- By width, we mean the extent of you model in the East-West direction. The height will be automatically calculated.\n", "- Change the value in the cell below to your liking and hit Shift-Enter\n", "- Although this can be fixed later, you should set the width to be less than the widest dimension of your printer's build plate." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -345,15 +351,15 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "### Setting the number of tiles and their and setup \n", "- It's possible to divide the print are into multiple tiles. This will results in several STK files instead of just one.\n", "- This is useful if you want to print several (smaller) tiles that each fit on you buildplate and later glue them together into one large model.\n", "- If you just want a single STL file, set both, ntilesx and ntilesy, to 1\n", "- For multiple tiles, decide how many tiles you want along East-West (ntilesx) and how many along North-South (ntilesy)" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -366,13 +372,13 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "### Setting the Base thickness (in mm)\n", "- The basethick setting determines how much material is put beneath the thinnest part to the terrain print \n", "- Use a value of at least 0.5 mm, more if you want a beefier base" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -384,14 +390,15 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "### Setting the print resolution\n", "- Set this value to the diameter of you nozzle size or slightly below. \n", - "- Tinker with this only if you know what you're doing. This setting defines how much of the fine details the STL file will contain. The detail you're realistaclly able to print is limited by your nozzle size. If you set setting much lower than your nozzle size, the STL file will be needlessly large, the slicer will take longer but your print will come out the same as if you had used approximately your nozzle size.\n", - "- In almost all cases, 0.4 mm will be fine for 3D printers." - ], - "cell_type": "markdown", - "metadata": {} + "- Tinker with this only if you know what you're doing. This setting defines how much of the fine details the STL file will contain. The detail you're realistically able to print is limited by your nozzle size. If you set setting much lower than your nozzle size, the STL file will be needlessly large, the slicer will take longer but your print will come out the same as if you had used approximately your nozzle size.\n", + "- In almost all cases, 0.4 mm will be fine for 3D printers.\n", + "- If you are using a local geofiff file, setting this to -1 will use the native resolution of the geotiff. This is typically overkill b/c of the nozzle limitation and may lead to huge models that may be too much for you system to handle!" + ] }, { "cell_type": "code", @@ -403,14 +410,15 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "### Setting the z-scale (elevation exaggeration factor)\n", "- To print the terrain in its \"true\" form (without any vertical scaling applied), set zscale to 1.0. This works well for terrain with great elevation differences, such as the Grand Canyon or Mt. Fuji\n", "- Most other terrain usual benefits from a z-scale larger than 1. For gentle hills, use 1.5 to 2.5. For cities use 2.5 to 6. For river deltas and other low relief areas use up to 10.\n", - "- The height of your printed terrain should be at least 20 - 30 mm high to show good detail. You can check the processing log for `top min/max`, max should be least 20 - 30 mm. If this is too low, use a higher zscale and process again." - ], - "cell_type": "markdown", - "metadata": {} + "- The height of your printed terrain should be at least 20 - 30 mm high to show good detail. You can check the processing log for `top min/max`, max should be least 20 - 30 mm. If this is too low, use a higher zscale and process again.\n", + "- Alternatively, you can set this to a __negative number__ which will be interpreted as the desired height of the tallest terrain point in mm. Examples: -12.7 means \"make 1/2 inch tall\", -20 means \"make 20 mm tall\", etc." + ] }, { "cell_type": "code", @@ -422,15 +430,15 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "### Set the name of your terrain model zipfile\n", "- Once a STL file of your terrain model has been created, it will be zipped into a zipfile and copied into the tmp folder. The zipfile also contains a log file and a geotiff.\n", "- It's often a good idea to give the zipfile a name that reflects the terrain, e.g. grand_canyon.zipfile\n", "- Set zip_file_name to the name of your zipfile. Dot not add the .zip extension, this will be done automatically\n", "- To 3D print you model, download the zip file from the container to your native OS" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -442,14 +450,14 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "### Other settings\n", "- There are several more expert settings which we will skip for now.\n", "- If you're interested, look here: https://github.com/ChHarding/TouchTerrain_for_CAGEO#processing-parameters\n", "- If you're interested, run the cell below to list all setting names and their values\n" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -462,14 +470,14 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "## Generating the STL model file\n", "- Running the next cell will convert the DEM (either from a local geotiff file or from a Google Earth online DEM source) into STL file and put it inside a zipfile in the tmp folder. You will also get a log file and the geotiff that was processed.\n", "- This may take some time!\n", "- During processing you'll see a star indicator (In[*]) and a lot of log messages. Those messages will also be in the logfile inside the zip, and may be useful later." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -483,12 +491,12 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "## Preview the model\n", "- run the cell below to get a 3D Preview of the model. If you have multiple tiles they will have different colors." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -520,6 +528,8 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "## Final thoughts\n", "- You can now download your zipfile from the tmp folder and unzip it to 3D print the STL file.\n", @@ -548,9 +558,7 @@ "- After running the container, you will again have to run the install script (`./install_touchterrain.sh`) and then run jupyter with `./run_touchterrain.sh` and copy/paste the `127.0.0.1` URL into your browser.\n", "- For more see https://github.com/ChHarding/TouchTerrain_jupyter_docker#download-the-image-and-run-the-container\n", "- Once jupyter is running, re-upload anything you need (notebook, data) to the container and resume working" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -561,14 +569,12 @@ } ], "metadata": { + "interpreter": { + "hash": "07e731ae0f6763183334093cb54db717ef5dbea7569bfaafd729cfce36c11ce4" + }, "kernelspec": { - "name": "python3", - "display_name": "Python 3.7.3 64-bit (conda)", - "metadata": { - "interpreter": { - "hash": "07e731ae0f6763183334093cb54db717ef5dbea7569bfaafd729cfce36c11ce4" - } - } + "display_name": "Python 3.7.3 64-bit ('base': conda)", + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -580,7 +586,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3-final" + "version": "3.7.3" }, "varInspector": { "cols": { @@ -614,4 +620,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/requirements.txt b/requirements.txt index 89e7639..1450595 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,8 @@ earthengine-api>=0.1.232 Flask>=1.0.2 oauth2client>=4.1.3 numpy>=1.17 +scipy>=1.2 kml2geojson>=4.0.2 geojson>=2.5 defusedxml>=0.6 -six>=1.15.0 \ No newline at end of file +six>=1.15.0 diff --git a/touchterrain/server/Run_TouchTerrain_debug_server_app.py b/touchterrain/server/Run_TouchTerrain_debug_server_app.py new file mode 100644 index 0000000..b55e8bc --- /dev/null +++ b/touchterrain/server/Run_TouchTerrain_debug_server_app.py @@ -0,0 +1,8 @@ +# This script runs a local Flask server, which can be usefull for debugging +# Note that this has to be run in the project root folder! (I put it here b/c it's server related) +# To get a proper Google Map you will need to create a GoogleMapsKey.txt file that contains +# a valid API key as a single string +# Note that this won't run multi-core processing (I think ...) + +from touchterrain.server.TouchTerrain_app import app +app.run(debug=False, port=8080) diff --git a/touchterrain/server/templates/touchterrain.js b/touchterrain/server/templates/touchterrain.js index 074cfba..3eaee16 100644 --- a/touchterrain/server/templates/touchterrain.js +++ b/touchterrain/server/templates/touchterrain.js @@ -421,7 +421,8 @@ window.onload = function () { // help popovers $('#Whats_new__popover').popover({ content: 'Mouse over the question marks to see the help text or click on it to toggle the text on/off

\ - New in 3.4: fixed kml file issues

\ + New in 3.5: large processing jobs should no longer time out
\ + New in 3.4: fixed kml file issues
\ New in 3.3:
\
  • Help popups explain the different options and settings.
    • \
  • Better z-scaling: will let you define how tall you want your printed model to be (model height), \