Upgrade to tensorflow2

.dockerignore

@@ -5,6 +5,7 @@ cdl_annotations/*
 county/*
 naip/*
 predictions/*
+predictions_without_masking/*
 road_annotations/*
 road_annotations_for_mask/*
 roads/*

README.md

@@ -7,9 +7,8 @@
 ```
 
 ```bash
-export DOCKER_TAG=cnn_land_cover_docker
-sudo docker build ~/cnn_land_cover --tag=$DOCKER_TAG
-sudo docker run -it -v ~/cnn_land_cover:/home/cnn_land_cover $DOCKER_TAG bash
+sudo docker build ~/cnn_land_cover --tag=cnn_land_cover_docker
+sudo docker run -it -v ~/cnn_land_cover:/home/cnn_land_cover cnn_land_cover_docker bash
 cd /home/cnn_land_cover
 python src/annotate_naip_scenes.py
 python src/fit_model.py
@@ -23,7 +22,7 @@ python src/fit_model.py
 * [ ] GPU
 * [ ] Tensorboard
 * [ ] Tune dropout probability, number of filters, number of blocks
-* [ ] Visualizations
+* [ ] Visualizations, including gradients
 
 # Datasets
 

config/model_config.yml

@@ -48,4 +48,4 @@ test_scenes:
   - ./naip/m_4309101_sw_15_1_20171016.tif
   - ./naip/m_4009048_se_15_1_20170725.tif
 
-# TODO Model hyperparams here
+# TODO Put model hyperparams here

requirements.txt

@@ -1,10 +1,10 @@
 Fiona==1.8.6
-Keras==2.2.4
+PyYAML==5.3.1
 Shapely==1.6.4.post2
 black==19.3b0
 matplotlib==3.0.3
 numpy==1.16.3
 pyproj==2.1.3
 rasterio==1.0.22
 scikit-learn==0.21.1
-tensorflow==1.13.1
+tensorflow==2.1.0

src/cnn.py

@@ -1,9 +1,9 @@
 from functools import partial
 
-import keras
-from keras import backend as K
-from keras.models import Model
-from keras.layers import (
+from tensorflow.keras import losses, optimizers
+from tensorflow.keras import backend as K
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import (
     BatchNormalization,
     Conv2D,
     Dense,
@@ -12,6 +12,7 @@
     GlobalAveragePooling2D,
     Input,
     MaxPooling2D,
+    Reshape,
     UpSampling2D,
     concatenate,
 )
@@ -21,6 +22,7 @@
     HAS_BUILDINGS,
     HAS_ROADS,
     PIXELS,
+    PIXELS_RESHAPED,
     IS_MAJORITY_FOREST,
     MODAL_LAND_COVER,
 )
@@ -68,29 +70,9 @@ def add_upsampling_block(input_layer, block_index, downsampling_conv2_layers):
     return BatchNormalization()(conv2)
 
 
-def get_masked_categorical_crossentropy(cdl_indices_to_mask):
-    def masked_categorical_crossentropy(y_true, y_pred):
+def get_keras_model(image_shape, label_encoder, include_reshape=True):
 
-        # Used for pixel classifications: mask pixels whose class is in cdl_indices_to_mask
-        # Note: shapes are (batch, image_shape, image_shape, n_classes)
-        # TODO Speed this up, test it
-
-        mask = K.ones_like(y_true[:, :, :, 0])
-
-        for cdl_index in cdl_indices_to_mask:
-            cdl_index_indicators = K.cast(
-                K.equal(y_true[:, :, :, cdl_index], 1), "float32"
-            )
-            mask -= cdl_index_indicators
-
-        return K.categorical_crossentropy(y_true, y_pred) * mask
-
-    return masked_categorical_crossentropy
-
-
-def get_keras_model(image_shape, label_encoder):
-
-    # Note: model is fully convolutional, so image width and height can be arbitrary
+    # Note: the model is fully convolutional: the input image width and height can be arbitrary
     input_layer = Input(shape=(None, None, image_shape[2]))
 
     # Note: Keep track of conv2 layers so that they can be connected to the upsampling blocks
@@ -122,10 +104,23 @@ def get_keras_model(image_shape, label_encoder):
             current_last_layer, index, downsampling_conv2_layers
         )
 
-    output_pixels = Conv2D(n_classes, 1, activation="softmax", name=PIXELS)(
+    pixels_final_conv = Conv2D(n_classes, 1, activation="softmax", name=PIXELS)(
         current_last_layer
     )
 
+    if include_reshape:
+
+        # Note: we are reshaping so that we can use weights with sample_weight_mode temporal when training
+        #  See https://github.com/keras-team/keras/issues/3653#issuecomment-557844450
+        #  The model is **not** fully convolutional when include_reshape is True
+        output_pixels = Reshape((image_shape[0] * image_shape[1], n_classes), name=PIXELS_RESHAPED)(pixels_final_conv)
+        output_pixels_name = PIXELS_RESHAPED
+
+    else:
+
+        output_pixels = pixels_final_conv
+        output_pixels_name = PIXELS
+
     model = Model(
         inputs=input_layer,
         outputs=[
@@ -139,21 +134,25 @@ def get_keras_model(image_shape, label_encoder):
 
     print(model.summary())
 
-    nadam = keras.optimizers.Nadam()
+    nadam = optimizers.Nadam()
 
     cdl_indices_to_mask = label_encoder.transform(CDL_CLASSES_TO_MASK)
-    masked_categorical_crossentropy = get_masked_categorical_crossentropy(
-        cdl_indices_to_mask
-    )
 
     model.compile(
         optimizer=nadam,
         loss={
-            HAS_BUILDINGS: keras.losses.binary_crossentropy,
-            HAS_ROADS: keras.losses.binary_crossentropy,
-            IS_MAJORITY_FOREST: keras.losses.binary_crossentropy,
-            MODAL_LAND_COVER: keras.losses.categorical_crossentropy,
-            PIXELS: masked_categorical_crossentropy,
+            HAS_BUILDINGS: losses.binary_crossentropy,
+            HAS_ROADS: losses.binary_crossentropy,
+            IS_MAJORITY_FOREST: losses.binary_crossentropy,
+            MODAL_LAND_COVER: losses.categorical_crossentropy,
+            output_pixels_name: losses.categorical_crossentropy,
+        },
+        sample_weight_mode={
+            HAS_BUILDINGS: None,
+            HAS_ROADS: None,
+            IS_MAJORITY_FOREST: None,
+            MODAL_LAND_COVER: None,
+            output_pixels_name: "temporal",
         },
         metrics=["accuracy"],
     )

src/constants.py

@@ -31,8 +31,8 @@
 
 # Note: buffering is applied after projecting road shapes into NAIP's CRS
 # For road type definitions, see https://www2.census.gov/geo/pdfs/maps-data/data/tiger/tgrshp2009/TGRSHP09AF.pdf
-ROAD_BUFFER_METERS_DEFAULT = 3.0
-ROAD_BUFFER_METERS = {"S1100": 5.0, "S1200": 5.0}
+ROAD_BUFFER_METERS_DEFAULT = 4.0
+ROAD_BUFFER_METERS = {"S1100": 6.0, "S1200": 6.0, "S1630": 6.0}
 
 # Note: roads are buffered by this amount in order to implement CDL developed masking logic
 ROAD_BUFFER_METERS_MASK = 30.0
@@ -42,3 +42,4 @@
 IS_MAJORITY_FOREST = "is_majority_forest"
 MODAL_LAND_COVER = "modal_land_cover"
 PIXELS = "pixels"
+PIXELS_RESHAPED = "pixels_reshaped"

src/fit_model.py

@@ -3,9 +3,8 @@
 import json
 import os
 
-import keras
-from keras import callbacks
-from keras.utils import plot_model
+from tensorflow.keras import callbacks
+from tensorflow.keras.utils import plot_model
 import matplotlib.pyplot as plt
 import numpy as np
 import rasterio
@@ -19,7 +18,7 @@
     HAS_ROADS,
     IS_MAJORITY_FOREST,
     MODAL_LAND_COVER,
-    PIXELS,
+    PIXELS_RESHAPED,
 )
 from constants import (
     BUILDING_ANNOTATION_DIR,
@@ -209,23 +208,18 @@ def fit_model(config, label_encoder, cdl_mapping):
 
     validation_generator = get_generator(validation_scenes, label_encoder, IMAGE_SHAPE)
 
-    # TODO Also use class_weight when computing test accuracy stats
-    # TODO Doesn't work for pixels, see https://github.com/keras-team/keras/issues/3653
-    class_weight = get_class_weight(label_encoder)
-    print(f"Class weights used in training: {class_weight}")
-
     # TODO Tensorboard
-    history = model.fit_generator(
-        generator=training_generator,
+    history = model.fit(
+        x=training_generator,
         steps_per_epoch=50,
         epochs=100,
         verbose=True,
+        # TODO EarlyStopping val_loss is not available warning
         callbacks=[
             callbacks.EarlyStopping(
                 patience=20, monitor="val_loss", restore_best_weights=True, verbose=True
             )
         ],
-        class_weight=class_weight,
         validation_data=validation_generator,
         validation_steps=10,
     )
@@ -283,7 +277,7 @@ def print_classification_reports(test_X, test_y, model, label_encoder):
                 name, y_pred, y_true, cdl_indices_to_mask, label_encoder
             )
 
-        elif name == PIXELS:
+        elif name == PIXELS_RESHAPED:
 
             y_pred = test_predictions[index].argmax(axis=-1).flatten()
             y_true = test_y[name].argmax(axis=-1).flatten()
@@ -324,19 +318,6 @@ def print_classification_reports(test_X, test_y, model, label_encoder):
             )
 
 
-def get_class_weight(label_encoder):
-
-    # Note: using the same class weights for pixel classifications is tricky and requires a custom loss fn
-    # See https://github.com/keras-team/keras/issues/3653
-
-    return {
-        MODAL_LAND_COVER: {
-            label_encoder.transform([c])[0]: 0.0 if c in CDL_CLASSES_TO_MASK else 1.0
-            for c in label_encoder.classes_
-        }
-    }
-
-
 def get_model_name():
 
     datetime_now = dt.datetime.now().strftime("%Y_%m_%d_%H")
@@ -378,25 +359,14 @@ def main():
     test_generator = get_generator(
         test_scenes, label_encoder, IMAGE_SHAPE, batch_size=600
     )
-    test_X, test_y = next(test_generator)
+    test_X, test_y, test_weights = next(test_generator)
 
+    # TODO Show test set loss for each objective
+    # Also fit some simple baseline models (null model, regression
+    #  tree that only sees average for each band in the image, nearest neighbors...),
+    #  compute their test set loss and show on a plot with CNN test loss
     print_classification_reports(test_X, test_y, model, label_encoder)
 
-    colormap = get_colormap(label_encoder)
-    print(f"Colormap used for predictions: {colormap}")
-
-    for test_scene in config["test_scenes"]:
-
-        predict_pixels_entire_scene(
-            model,
-            test_scene,
-            X_mean_train,
-            X_std_train,
-            IMAGE_SHAPE,
-            label_encoder,
-            colormap,
-        )
-
 
 if __name__ == "__main__":
     main()