Data production july 22 (#43)

* Cleans up notebooks * PIF-based image normalization, improved image compositing
wri · Jan 30, 2023 · 976ebfa · 976ebfa
1 parent 319b03f
commit 976ebfa
Show file tree

Hide file tree

Showing 7 changed files with 16 additions and 45 deletions.
diff --git a/notebooks/1a-download-sentinel-2.ipynb b/notebooks/1a-download-sentinel-2.ipynb
@@ -4914,10 +4914,10 @@
     "plots = [x for x in plots if x[:2] == '99']\n",
     "for plot in plots:\n",
     "    i += 1\n",
-    "    if not os.path.exists(\"../data/train-s2/\" + plot + \".hkl\"):\n",
+    "    if not os.path.exists(\"../data/test-s2/\" + plot + \".hkl\"):\n",
     "        print(plot)\n",
     "        try:\n",
-    "            tiles = process_raw(plot, path = 'train')\n",
+    "            tiles = process_raw(plot, path = 'test')\n",
     "            print(i, plot)\n",
     "        except:\n",
     "            continue"

diff --git a/notebooks/1b-download-sentinel-1.ipynb b/notebooks/1b-download-sentinel-1.ipynb
@@ -903,17 +903,10 @@
     "for i in (os.listdir(\"../data/train-csv/\")):\n",
     "    if \"pineapple-2021.csv\" in i:\n",
     "        print(i)\n",
-    "        download_plots(\"../data/train-csv/\" + i, \n",
-    "                       \"../data/train-s1/\",\n",
+    "        download_plots(\"../data/test-csv/\" + i, \n",
+    "                       \"../data/test-s1/\",\n",
     "                       image_format = MimeType.TIFF)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

diff --git a/notebooks/2-data-preprocessing.ipynb b/notebooks/2-data-preprocessing.ipynb
@@ -80,7 +80,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "source = 'train'\n",
+    "source = 'test'\n",
     "sentinel_1 = True\n",
     "s2_path = \"../data/{}-s2/\".format(source)\n",
     "s1_path = \"../data/{}-s1/\".format(source)\n",
@@ -6482,30 +6482,7 @@
    "cell_type": "code",
    "execution_count": 18,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Writing train data\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/Users/jbrandt.terminal/opt/anaconda3/envs/tf/lib/python3.7/site-packages/ipykernel_launcher.py:2: FutureWarning: In a future version of pandas all arguments of DataFrame.drop except for the argument 'labels' will be keyword-only\n",
-      "  \n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Finished!\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import hickle as hkl\n",
     "dataframe = dataframe.drop(0, 0)\n",

diff --git a/src/download_and_predict_job_fast.py b/src/download_and_predict_job_fast.py
@@ -712,6 +712,8 @@ def make_and_smooth_indices(arr, dates):
     #print(np.sum(np.isnan(indices)))
     #indices = normalize_first_last_quarter(indices, dates)
 
+    #ndices = normalize_first_last_quarter(indices, dates)
+
     try:
         indices, _ = calculate_and_save_best_images(indices, dates)
     except:

diff --git a/src/preprocessing/cloud_removal.py b/src/preprocessing/cloud_removal.py
@@ -516,7 +516,6 @@ def remove_cloud_and_shadows(tiles: np.ndarray,
     return tiles, areas_interpolated
 """
 
-
 def detect_pfcp(arr, dem, bbx):
     # Detects potential false cloud pixels
     # By using the paralax effect of the NIR bands as
@@ -711,8 +710,8 @@ def _winsum(in_arr, windowsize):
         shadows_i = binary_dilation(
             1 - (binary_dilation(shadows_i == 0, iterations=2)), iterations=3)
         shadows_i = distance(1 - shadows_i)
-        shadows_i[shadows_i <= 5] = 0.
-        shadows_i[shadows_i > 5] = 1
+        shadows_i[shadows_i <= 4] = 0.
+        shadows_i[shadows_i > 4] = 1
         shadows_i = 1 - shadows_i
         shadows[i] = shadows_i
 
@@ -1208,7 +1207,7 @@ def _indices_month(dates, x, y):
             if len(indices_month) > 1:
                 month_dates = dates[indices_month]
                 month_clouds = probs[indices_month]
-
+                
                 subset_month = True
                 if x == -60:
                     feb_mar = np.argwhere(

diff --git a/src/resegment_tiles_wide.py b/src/resegment_tiles_wide.py
@@ -600,6 +600,7 @@ def preprocess_tile(arr, dates, interp, clm, fname, dem, bbx):
         if len(missing_px) > 0:
             print(f"Deleting {missing_px} from cloud mask")
             clm = np.delete(clm, missing_px, 0)
+            clm[fcps] = 0.
         try:
             clm[fcps] = 0.
             print("CLM", np.mean(clm, axis = (1, 2)))
@@ -1644,10 +1645,10 @@ def cleanup_row_or_col(idx, current_idx, local_path):
             except KeyboardInterrupt:
                 break
 
-            except Exception as e:
-                print(f"Ran into {str(e)}")
-                finished = 0
-                s2_shape = (0, 0)
+            #except Exception as e:
+            #    print(f"Ran into {str(e)}")
+            #    finished = 0
+            #    s2_shape = (0, 0)
 
             if finished == 1:
                 try:

diff --git a/src/tof/tof_downloading.py b/src/tof/tof_downloading.py
@@ -372,7 +372,6 @@ def identify_clouds_big_bbx(
         cloud_dates = np.delete(cloud_dates, to_remove)
         cloud_img = np.delete(cloud_img, to_remove, 0)
         cloud_percent = np.delete(cloud_percent, to_remove)
-
     # Type assertions, size assertions
     if not isinstance(cloud_img.flat[0], np.floating):
         #assert np.max(cloud_img) > 1