From 8ca881977a27bcfed7c0ce9e87d9502ed964dc97 Mon Sep 17 00:00:00 2001
From: "Noah D. Brenowitz" <nbren12@gmail.com>
Date: Tue, 17 Mar 2020 19:18:40 +0000
Subject: [PATCH 01/13] Refactore online_modules to fv3net

---
 .../runtime}/config.py                        |  2 +-
 .../runtime}/sklearn_interface.py             | 24 +++++++------------
 .../runtime}/state_io.py                      |  0
 .../prognostic_c48_run/sklearn_runfile.py     | 15 ++++++------
 4 files changed, 16 insertions(+), 25 deletions(-)
 rename {workflows/prognostic_c48_run/online_modules => fv3net/runtime}/config.py (93%)
 rename {workflows/prognostic_c48_run/online_modules => fv3net/runtime}/sklearn_interface.py (70%)
 rename {workflows/prognostic_c48_run/online_modules => fv3net/runtime}/state_io.py (100%)

diff --git a/workflows/prognostic_c48_run/online_modules/config.py b/fv3net/runtime/config.py
similarity index 93%
rename from workflows/prognostic_c48_run/online_modules/config.py
rename to fv3net/runtime/config.py
index 13eaed4d45..39fe1794be 100644
--- a/workflows/prognostic_c48_run/online_modules/config.py
+++ b/fv3net/runtime/config.py
@@ -10,7 +10,7 @@ class dotdict(dict):
     __delattr__ = dict.__delitem__
 
 
-def get_config():
+def get_runfile_config():
     with open("fv3config.yml") as f:
         config = yaml.safe_load(f)
     return dotdict(config["scikit_learn"])
diff --git a/workflows/prognostic_c48_run/online_modules/sklearn_interface.py b/fv3net/runtime/sklearn_interface.py
similarity index 70%
rename from workflows/prognostic_c48_run/online_modules/sklearn_interface.py
rename to fv3net/runtime/sklearn_interface.py
index 5a5027f7b7..e6e730b33e 100644
--- a/workflows/prognostic_c48_run/online_modules/sklearn_interface.py
+++ b/fv3net/runtime/sklearn_interface.py
@@ -3,10 +3,16 @@
 from sklearn.externals import joblib
 from sklearn.utils import parallel_backend
 
-import state_io
+from . import state_io
 
+__all__ = [
+    "open_model",
+    "predict",
+    "update"
+]
 
-def open_sklearn_model(url):
+
+def open_model(url):
     # Load the model
     with fsspec.open(url, "rb") as f:
         return joblib.load(f)
@@ -30,17 +36,3 @@ def update(model, state, dt):
     )
 
     return state_io.rename_to_orig(updated), state_io.rename_to_orig(tend)
-
-
-if __name__ == "__main__":
-    import sys
-
-    state_path = sys.argv[1]
-    model = open_sklearn_model(sys.argv[2])
-
-    with open(state_path, "rb") as f:
-        data = state_io.load(f)
-
-    tile = data[0]
-    preds = update(model, tile, dt=1)
-    print(preds)
diff --git a/workflows/prognostic_c48_run/online_modules/state_io.py b/fv3net/runtime/state_io.py
similarity index 100%
rename from workflows/prognostic_c48_run/online_modules/state_io.py
rename to fv3net/runtime/state_io.py
diff --git a/workflows/prognostic_c48_run/sklearn_runfile.py b/workflows/prognostic_c48_run/sklearn_runfile.py
index 77a3c183c1..c6c3bfbea9 100644
--- a/workflows/prognostic_c48_run/sklearn_runfile.py
+++ b/workflows/prognostic_c48_run/sklearn_runfile.py
@@ -3,9 +3,8 @@
 import zarr
 
 import fv3gfs
-import sklearn_interface
-import state_io
 from fv3gfs._wrapper import get_time
+from fv3net import runtime
 from mpi4py import MPI
 import config
 
@@ -32,8 +31,8 @@ def compute_diagnostics(state, diags):
     )
 
 
-args = config.get_config()
-NML = config.get_namelist()
+args = runtime.get_runfile_config()
+NML = runtime.get_namelist()
 TIMESTEP = NML["coupler_nml"]["dt_atmos"]
 
 times = []
@@ -55,7 +54,7 @@ def compute_diagnostics(state, diags):
 
     if rank == 0:
         logger.info("Downloading Sklearn Model")
-        MODEL = sklearn_interface.open_sklearn_model(args.model)
+        MODEL = runtime.sklearn.open_model(args.model)
         logger.info("Model downloaded")
     else:
         MODEL = None
@@ -81,7 +80,7 @@ def compute_diagnostics(state, diags):
 
         if rank == 0:
             logger.debug("Computing RF updated variables")
-        preds, diags = sklearn_interface.update(MODEL, state, dt=TIMESTEP)
+        preds, diags = runtime.sklearn.update(MODEL, state, dt=TIMESTEP)
         if rank == 0:
             logger.debug("Setting Fortran State")
         fv3gfs.set_state(preds)
@@ -91,8 +90,8 @@ def compute_diagnostics(state, diags):
         diagnostics = compute_diagnostics(state, diags)
 
         if i == 0:
-            writers = state_io.init_writers(GROUP, comm, diagnostics)
-        state_io.append_to_writers(writers, diagnostics)
+            writers = runtime.init_writers(GROUP, comm, diagnostics)
+        runtime.append_to_writers(writers, diagnostics)
 
         times.append(get_time())
 

From 5909c73b3877908e263b2a5083e172cf5fde4c14 Mon Sep 17 00:00:00 2001
From: "Noah D. Brenowitz" <nbren12@gmail.com>
Date: Tue, 17 Mar 2020 21:05:23 +0000
Subject: [PATCH 02/13] black

---
 fv3net/runtime/sklearn_interface.py           |  6 +-----
 workflows/prognostic_c48_run/Dockerfile       | 12 +++++------
 workflows/prognostic_c48_run/Makefile         | 20 +++++++++----------
 workflows/prognostic_c48_run/fv3config.yml    |  3 +--
 workflows/prognostic_c48_run/requirements.txt |  5 +++++
 .../prognostic_c48_run/sklearn_runfile.py     |  3 +--
 6 files changed, 22 insertions(+), 27 deletions(-)
 create mode 100644 workflows/prognostic_c48_run/requirements.txt

diff --git a/fv3net/runtime/sklearn_interface.py b/fv3net/runtime/sklearn_interface.py
index e6e730b33e..a77106625e 100644
--- a/fv3net/runtime/sklearn_interface.py
+++ b/fv3net/runtime/sklearn_interface.py
@@ -5,11 +5,7 @@
 
 from . import state_io
 
-__all__ = [
-    "open_model",
-    "predict",
-    "update"
-]
+__all__ = ["open_model", "predict", "update"]
 
 
 def open_model(url):
diff --git a/workflows/prognostic_c48_run/Dockerfile b/workflows/prognostic_c48_run/Dockerfile
index 27d8929fb2..8de1e64a66 100644
--- a/workflows/prognostic_c48_run/Dockerfile
+++ b/workflows/prognostic_c48_run/Dockerfile
@@ -1,10 +1,8 @@
 FROM us.gcr.io/vcm-ml/fv3gfs-python:v0.2.1
 
-ADD fv3net-0.1.0-py3-none-any.whl /wheels/fv3net-0.1.0-py3-none-any.whl
-ADD vcm-0.1.0-py3-none-any.whl /wheels/vcm-0.1.0-py3-none-any.whl
-RUN pip3 install --no-deps /wheels/fv3net-0.1.0-py3-none-any.whl && \
-    pip3 install /wheels/vcm-0.1.0-py3-none-any.whl && \
-    pip3 install scikit-learn==0.22.1 joblib zarr
 
-COPY online_modules/ /online_modules
-ENV PYTHONPATH=/online_modules:${PYTHON_PATH}
+COPY requirements.txt /tmp/requirements.txt
+RUN pip3 install -r /tmp/requirements.txt
+COPY fv3net-0.1.0-py3-none-any.whl /wheels/fv3net-0.1.0-py3-none-any.whl
+COPY vcm-0.1.0-py3-none-any.whl /wheels/vcm-0.1.0-py3-none-any.whl
+RUN pip3 install --no-deps /wheels/fv3net-0.1.0-py3-none-any.whl  && pip3 install /wheels/vcm-0.1.0-py3-none-any.whl
diff --git a/workflows/prognostic_c48_run/Makefile b/workflows/prognostic_c48_run/Makefile
index 82d95a4f7e..30c55c2f0e 100644
--- a/workflows/prognostic_c48_run/Makefile
+++ b/workflows/prognostic_c48_run/Makefile
@@ -7,24 +7,17 @@ RUN_ARGS = --rm $(KEY_ARGS) $(LOCAL_DIR_ARGS) $(IMAGE)
 RUN_INTERACTIVE = docker run -ti $(RUN_ARGS)
 RUN ?= docker run $(RUN_ARGS)
 SKLEARN_MODEL = gs://vcm-ml-data/test-annak/ml-pipeline-output/2020-01-17_rf_40d_run.pkl
-FV3CONFIG = fv3config.yml
-FV3NET_VERSION ?=2020-01-23-prognostic-rf
+FV3CONFIG = gs://vcm-ml-data/end-to-end-experiments/2020-02-26-physics-off/annak-prognostic-physics-off-1773255e/prognostic_run_prognostic_yaml_adjust_prognostic_config.yml_ic_timestep_20160801.001500_docker_image_prognostic-run-orchestration/job_config/fv3config.yml
 
 all: sklearn_run
 
-fv3net-0.1.0-py3-none-any.whl:
-	pip wheel --no-deps git+ssh://git@github.com/VulcanClimateModeling/fv3net.git@$(FV3NET_VERSION)
-
-build: fv3net-0.1.0-py3-none-any.whl
-	docker build . -t $(IMAGE)
-
-fv3net-local:
+fv3net:
 	pip wheel --no-deps ../../.
 
-vcm-local:
+vcm:
 	pip wheel --no-deps ../../external/vcm
 
-build_local: fv3net-local vcm-local
+build: fv3net vcm
 	docker build . -t $(IMAGE)
 
 dev:
@@ -36,9 +29,14 @@ test_run_sklearn: state.pkl
 state.pkl:
 	fv3run --dockerimage test-image --runfile save_state_runfile.py $(FV3CONFIG) save_state/
 	cp save_state/state.pkl .
+	
+sklearn_run_local: #rundir
+	fv3run --dockerimage $(IMAGE) --runfile sklearn_runfile.py $(FV3CONFIG) rundir
 
 sklearn_run: #rundir
 	fv3run --dockerimage us.gcr.io/vcm-ml/prognostic-run-orchestration --runfile sklearn_runfile.py $(FV3CONFIG) ../../scratch/rundir
 
 clean:
 	rm -rf net_precip net_heating/ PW
+
+.PHONY: fv3net vcm build dev sklearn_run
\ No newline at end of file
diff --git a/workflows/prognostic_c48_run/fv3config.yml b/workflows/prognostic_c48_run/fv3config.yml
index e7bf1b4b5d..673325ca16 100644
--- a/workflows/prognostic_c48_run/fv3config.yml
+++ b/workflows/prognostic_c48_run/fv3config.yml
@@ -1,5 +1,4 @@
-scikit_learn:
-  model: gs://vcm-ml-data/test-annak/ml-pipeline-output/2020-01-17_rf_40d_run.pkl
+scikit_learn: model:gs://vcm-ml-data/end-to-end-experiments/2020-02-26-physics-off/annak-prognostic-physics-off/train_sklearn_model_train-config-file_example_base_rf_training_config.yml_delete-local-results-after-upload_False/sklearn_model.pkl
   zarr_output: diags.zarr
 data_table: default
 diag_table: gs://vcm-ml-data/2020-01-15-noahb-exploration/2hr_strong_dampingone_step_config/C48/20160805.000000/diag_table
diff --git a/workflows/prognostic_c48_run/requirements.txt b/workflows/prognostic_c48_run/requirements.txt
new file mode 100644
index 0000000000..9944d45ec4
--- /dev/null
+++ b/workflows/prognostic_c48_run/requirements.txt
@@ -0,0 +1,5 @@
+scikit-learn==0.22.1
+dask
+joblib
+zarr
+scikit-image
\ No newline at end of file
diff --git a/workflows/prognostic_c48_run/sklearn_runfile.py b/workflows/prognostic_c48_run/sklearn_runfile.py
index c6c3bfbea9..8536ec178d 100644
--- a/workflows/prognostic_c48_run/sklearn_runfile.py
+++ b/workflows/prognostic_c48_run/sklearn_runfile.py
@@ -6,14 +6,13 @@
 from fv3gfs._wrapper import get_time
 from fv3net import runtime
 from mpi4py import MPI
-import config
 
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
 
 SPHUM = "specific_humidity"
 DELP = "pressure_thickness_of_atmospheric_layer"
-VARIABLES = list(state_io.CF_TO_RESTART_MAP) + [DELP]
+VARIABLES = list(runtime.CF_TO_RESTART_MAP) + [DELP]
 
 cp = 1004
 gravity = 9.81

From cee28fd12bb9660a7a797393f06d9c12acfd2382 Mon Sep 17 00:00:00 2001
From: "Noah D. Brenowitz" <nbren12@gmail.com>
Date: Tue, 17 Mar 2020 21:21:27 +0000
Subject: [PATCH 03/13] Create us.gcr.io/vcm-ml/prognostic_run:v0.1.0

Builds a docker image with fv3net, vcm, and fv3gfs-python installed.
---
 .../prognostic_c48_run => docker}/Dockerfile  |  0
 docker/Dockerfile.kubernetes                  | 30 -------------------
 docker/Makefile                               | 17 +++++++++++
 docker/download_inputdata.sh                  | 22 --------------
 docker/install_gcloud.sh                      | 18 -----------
 .../requirements.txt                          |  0
 fv3net/runtime/__init__.py                    |  3 ++
 .../end_to_end/example-workflow-config.yaml   |  2 +-
 workflows/prognostic_c48_run/Makefile         | 13 ++------
 9 files changed, 23 insertions(+), 82 deletions(-)
 rename {workflows/prognostic_c48_run => docker}/Dockerfile (100%)
 delete mode 100644 docker/Dockerfile.kubernetes
 create mode 100644 docker/Makefile
 delete mode 100755 docker/download_inputdata.sh
 delete mode 100644 docker/install_gcloud.sh
 rename {workflows/prognostic_c48_run => docker}/requirements.txt (100%)
 create mode 100644 fv3net/runtime/__init__.py

diff --git a/workflows/prognostic_c48_run/Dockerfile b/docker/Dockerfile
similarity index 100%
rename from workflows/prognostic_c48_run/Dockerfile
rename to docker/Dockerfile
diff --git a/docker/Dockerfile.kubernetes b/docker/Dockerfile.kubernetes
deleted file mode 100644
index 681af739b2..0000000000
--- a/docker/Dockerfile.kubernetes
+++ /dev/null
@@ -1,30 +0,0 @@
-FROM google/cloud-sdk:latest
-
-ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
-ENV PATH /opt/conda/bin:$PATH
-
-RUN apt-get update --fix-missing && apt-get install -y wget bzip2 ca-certificates \
-    libglib2.0-0 libxext6 libsm6 libxrender1 \
-    git mercurial subversion
-
-RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
-    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
-    rm ~/miniconda.sh && \
-    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
-    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
-    echo "conda activate base" >> ~/.bashrc
-
-RUN apt-get install -y curl grep sed dpkg && \
-    TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \
-    curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \
-    dpkg -i tini.deb && \
-    rm tini.deb && \
-    apt-get clean
-
-COPY . /fv3net
-
-RUN conda env update --file /fv3net/environment.yml --name base && conda clean -afy
-RUN python /fv3net/setup.py develop
-
-ENTRYPOINT [ "/usr/bin/tini", "--" ]
-CMD [ "/bin/bash" ]
\ No newline at end of file
diff --git a/docker/Makefile b/docker/Makefile
new file mode 100644
index 0000000000..3a21535e9c
--- /dev/null
+++ b/docker/Makefile
@@ -0,0 +1,17 @@
+IMAGE = us.gcr.io/vcm-ml/prognostic_run:v0.1.0
+
+all: build
+
+fv3net:
+	pip wheel --no-deps ../
+
+vcm:
+	pip wheel --no-deps ../external/vcm
+
+build: fv3net vcm
+	docker build . -t $(IMAGE)
+
+push: build
+	docker push $(IMAGE)
+
+.PHONY: fv3net vcm build dev sklearn_run
diff --git a/docker/download_inputdata.sh b/docker/download_inputdata.sh
deleted file mode 100755
index b202a94398..0000000000
--- a/docker/download_inputdata.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/bash
-
-set -e
-set -x
-
-dir=2019-09-27-FV3GFS-docker-input-c48-LH-nml
-filename=fv3gfs-data-docker_2019-09-27.tar.gz
-url=http://storage.googleapis.com/vcm-ml-public/$dir/$filename
-datadir_local=inputdata
-
-mkdir -p $datadir_local
-
-# download data
-[[ -f $filename ]] || wget $url
-
-# unzip/tar input data
-tar xzf $filename -C $datadir_local
-
-rm $filename
-mkdir -p /inputdata/
-mv inputdata/fv3gfs-data-docker/fix.v201702 /inputdata/fix.v201702  
-
diff --git a/docker/install_gcloud.sh b/docker/install_gcloud.sh
deleted file mode 100644
index 36a2644792..0000000000
--- a/docker/install_gcloud.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-# Add the Cloud SDK distribution URI as a package source
-echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
-
-# Import the Google Cloud Platform public key
-curl https://packages.cloud.google.com/apt/doc/apt-key.gpg |  apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
-
-# Update the package list and install the Cloud SDK
-apt-get update &&  apt-get install -y python-dev google-cloud-sdk
-
-# Install fast crc mod for multithreaded downloads
-# the snakemake workflow will fail on gsutil downloads without this
-curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-python2.7 get-pip.py
-PIP="python2.7 -m pip"
-echo "PIP Version"
-$PIP -V
-$PIP uninstall crcmod
-$PIP install --no-cache-dir -U crcmod
diff --git a/workflows/prognostic_c48_run/requirements.txt b/docker/requirements.txt
similarity index 100%
rename from workflows/prognostic_c48_run/requirements.txt
rename to docker/requirements.txt
diff --git a/fv3net/runtime/__init__.py b/fv3net/runtime/__init__.py
new file mode 100644
index 0000000000..80c9e7e8f6
--- /dev/null
+++ b/fv3net/runtime/__init__.py
@@ -0,0 +1,3 @@
+from . import sklearn_interface as sklearn
+from .state_io import init_writers, append_to_writers, CF_TO_RESTART_MAP
+from .config import get_runfile_config, get_namelist
\ No newline at end of file
diff --git a/workflows/end_to_end/example-workflow-config.yaml b/workflows/end_to_end/example-workflow-config.yaml
index 9e1355013f..1e899e5f26 100644
--- a/workflows/end_to_end/example-workflow-config.yaml
+++ b/workflows/end_to_end/example-workflow-config.yaml
@@ -84,4 +84,4 @@ experiment:
       extra_args:
         prognostic_yaml_adjust: workflows/prognostic_c48_run/prognostic_config.yml
         ic_timestep: "20160801.001500"
-        docker_image: us.gcr.io/vcm-ml/prognostic-run-orchestration
+        docker_image: us.gcr.io/vcm-ml/prognostic_run:v0.1.0
diff --git a/workflows/prognostic_c48_run/Makefile b/workflows/prognostic_c48_run/Makefile
index 30c55c2f0e..3b405c7c0f 100644
--- a/workflows/prognostic_c48_run/Makefile
+++ b/workflows/prognostic_c48_run/Makefile
@@ -1,4 +1,4 @@
-IMAGE=test-image
+IMAGE = us.gcr.io/vcm-ml/prognostic_run:v0.1.0
 KEY_ARGS= -v $(GOOGLE_APPLICATION_CREDENTIALS):/key.json \
 	  -e GOOGLE_APPLICATION_CREDENTIALS=/key.json
 LOCAL_DIR_ARGS = -w /code -v $(shell pwd):/code
@@ -11,15 +11,6 @@ FV3CONFIG = gs://vcm-ml-data/end-to-end-experiments/2020-02-26-physics-off/annak
 
 all: sklearn_run
 
-fv3net:
-	pip wheel --no-deps ../../.
-
-vcm:
-	pip wheel --no-deps ../../external/vcm
-
-build: fv3net vcm
-	docker build . -t $(IMAGE)
-
 dev:
 	 $(RUN_INTERACTIVE) bash
 
@@ -39,4 +30,4 @@ sklearn_run: #rundir
 clean:
 	rm -rf net_precip net_heating/ PW
 
-.PHONY: fv3net vcm build dev sklearn_run
\ No newline at end of file
+.PHONY: fv3net vcm build dev sklearn_run

From 305629fc4b175dcc208f197639e7be1ea3cbdaa0 Mon Sep 17 00:00:00 2001
From: "Noah D. Brenowitz" <nbren12@gmail.com>
Date: Tue, 17 Mar 2020 21:35:51 +0000
Subject: [PATCH 04/13] Refactor us.gcr.io/vcm-ml/fv3net image build code

---
 Makefile                                     | 9 ++++-----
 README.md                                    | 6 ++++++
 Dockerfile => docker/fv3net/Dockerfile       | 0
 docker/{ => prognostic_run}/Dockerfile       | 0
 docker/{ => prognostic_run}/Makefile         | 2 +-
 docker/{ => prognostic_run}/requirements.txt | 0
 6 files changed, 11 insertions(+), 6 deletions(-)
 rename Dockerfile => docker/fv3net/Dockerfile (100%)
 rename docker/{ => prognostic_run}/Dockerfile (100%)
 rename docker/{ => prognostic_run}/Makefile (83%)
 rename docker/{ => prognostic_run}/requirements.txt (100%)

diff --git a/Makefile b/Makefile
index 756c1cd9a4..26d34f2d23 100644
--- a/Makefile
+++ b/Makefile
@@ -22,8 +22,10 @@ endif
 #################################################################################
 # COMMANDS                                                                      #
 #################################################################################
-build_image:
-	docker build . -t $(IMAGE) -t $(GCR_IMAGE)
+
+build_images:
+	make -C docker/fv3net build
+	make -C docker/prognostic_run build
 
 enter: build_image
 	docker run -it -v $(shell pwd):/code \
@@ -33,9 +35,6 @@ enter: build_image
 #		-e GOOGLE_APPLICATION_CREDENTIALS=/google_creds.json \
 #		-v $(HOME)/.config/gcloud/application_default_credentials.json:/google_creds.json \
 
-push_image: build_image
-	docker push $(GCR_IMAGE)
-
 
 ## Make Dataset
 .PHONY: data update_submodules create_environment overwrite_baseline_images
diff --git a/README.md b/README.md
index 947c393b02..0020b473a2 100644
--- a/README.md
+++ b/README.md
@@ -117,6 +117,12 @@ If you get an error `Could not create workflow; user does not have write access
 trying to submit the dataflow job, do `gcloud auth application-default login` first and then retry.
 
 
+## Building the fv3net docker images
+
+The pipelines use a pair of common images:
+1. us.gcr.io/vcm-ml/prognostic_run:v0.1.0
+1. us.gcr.io/vcm-ml/fv3net
+
 ## Deploying on k8s with fv3net
 
 Docker images with the python-wrapped model and fv3run are available from the
diff --git a/Dockerfile b/docker/fv3net/Dockerfile
similarity index 100%
rename from Dockerfile
rename to docker/fv3net/Dockerfile
diff --git a/docker/Dockerfile b/docker/prognostic_run/Dockerfile
similarity index 100%
rename from docker/Dockerfile
rename to docker/prognostic_run/Dockerfile
diff --git a/docker/Makefile b/docker/prognostic_run/Makefile
similarity index 83%
rename from docker/Makefile
rename to docker/prognostic_run/Makefile
index 3a21535e9c..bef31690e4 100644
--- a/docker/Makefile
+++ b/docker/prognostic_run/Makefile
@@ -9,7 +9,7 @@ vcm:
 	pip wheel --no-deps ../external/vcm
 
 build: fv3net vcm
-	docker build . -t $(IMAGE)
+	docker build -f Dockerfile ../../ -t $(IMAGE)
 
 push: build
 	docker push $(IMAGE)
diff --git a/docker/requirements.txt b/docker/prognostic_run/requirements.txt
similarity index 100%
rename from docker/requirements.txt
rename to docker/prognostic_run/requirements.txt

From ff9f938d0c7618329ec45adb5b00a0b2d75ac4f5 Mon Sep 17 00:00:00 2001
From: "Noah D. Brenowitz" <nbren12@gmail.com>
Date: Tue, 17 Mar 2020 22:13:32 +0000
Subject: [PATCH 05/13] Add build_images makefile target

---
 Makefile                         | 19 ++++++++++++++++---
 docker/fv3net/Dockerfile         | 10 +++-------
 docker/prognostic_run/Dockerfile |  2 +-
 docker/prognostic_run/Makefile   | 17 -----------------
 4 files changed, 20 insertions(+), 28 deletions(-)
 delete mode 100644 docker/prognostic_run/Makefile

diff --git a/Makefile b/Makefile
index 26d34f2d23..716ce26156 100644
--- a/Makefile
+++ b/Makefile
@@ -3,6 +3,7 @@
 #################################################################################
 # GLOBALS                                                                       #
 #################################################################################
+VERSION = 0.1.0
 ENVIRONMENT_SCRIPTS = .environment-scripts
 PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 BUCKET = [OPTIONAL] your-bucket-for-syncing-data (do not include 's3://')
@@ -22,10 +23,22 @@ endif
 #################################################################################
 # COMMANDS                                                                      #
 #################################################################################
+.PHONY: wheels build_images push_image
+wheels:
+	pip wheel --no-deps .
+	pip wheel --no-deps external/vcm
 
-build_images:
-	make -C docker/fv3net build
-	make -C docker/prognostic_run build
+# pattern rule for building docker images
+build_image_%:
+	docker build -f docker/$*/Dockerfile . -t us.gcr.io/vcm-ml/$*:$(VERSION)
+
+build_image_prognostic_run: wheels
+
+build_images: build_image_fv3net build_image_prognostic_run
+
+push_image:
+	docker push us.gcr.io/vcm-ml/fv3net:$(VERSION)
+	docker push us.gcr.io/vcm-ml/prognostic_run:$(VERSION)
 
 enter: build_image
 	docker run -it -v $(shell pwd):/code \
diff --git a/docker/fv3net/Dockerfile b/docker/fv3net/Dockerfile
index 8a68ea973d..d12deb541c 100644
--- a/docker/fv3net/Dockerfile
+++ b/docker/fv3net/Dockerfile
@@ -9,7 +9,6 @@ ENV PROJECT_NAME=fv3net
 USER root
 RUN apt-get update && apt-get install -y gfortran
 ADD environment.yml  $FV3NET/
-ADD Makefile  $FV3NET/
 ADD .environment-scripts $ENVIRONMENT_SCRIPTS
 RUN fix-permissions $FV3NET
 WORKDIR $FV3NET
@@ -20,21 +19,18 @@ ENV PATH=/opt/conda/envs/fv3net/bin:$PATH
 RUN bash $ENVIRONMENT_SCRIPTS/build_environment.sh $PROJECT_NAME
 RUN jupyter labextension install @pyviz/jupyterlab_pyviz
 
-# Add rest of fv3net directory
 USER root 
-ADD . $FV3NET
 # install gcloud sdk
 RUN cd / && \
     curl  https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-284.0.0-linux-x86_64.tar.gz |\
     tar xz
 ENV PATH=/google-cloud-sdk/bin:${PATH}
-#RUN /google-cloud-sdk/bin/gcloud init
+
+# Add rest of fv3net directory
+ADD . $FV3NET
 
 RUN fix-permissions $FV3NET
 USER $NB_UID
 
-# RUN gcloud init
-
 # setup the local python packages
-
 RUN bash $ENVIRONMENT_SCRIPTS/install_local_packages.sh $PROJECT_NAME
diff --git a/docker/prognostic_run/Dockerfile b/docker/prognostic_run/Dockerfile
index 8de1e64a66..8fb34cc92c 100644
--- a/docker/prognostic_run/Dockerfile
+++ b/docker/prognostic_run/Dockerfile
@@ -1,7 +1,7 @@
 FROM us.gcr.io/vcm-ml/fv3gfs-python:v0.2.1
 
 
-COPY requirements.txt /tmp/requirements.txt
+COPY docker/prognostic_run/requirements.txt /tmp/requirements.txt
 RUN pip3 install -r /tmp/requirements.txt
 COPY fv3net-0.1.0-py3-none-any.whl /wheels/fv3net-0.1.0-py3-none-any.whl
 COPY vcm-0.1.0-py3-none-any.whl /wheels/vcm-0.1.0-py3-none-any.whl
diff --git a/docker/prognostic_run/Makefile b/docker/prognostic_run/Makefile
deleted file mode 100644
index bef31690e4..0000000000
--- a/docker/prognostic_run/Makefile
+++ /dev/null
@@ -1,17 +0,0 @@
-IMAGE = us.gcr.io/vcm-ml/prognostic_run:v0.1.0
-
-all: build
-
-fv3net:
-	pip wheel --no-deps ../
-
-vcm:
-	pip wheel --no-deps ../external/vcm
-
-build: fv3net vcm
-	docker build -f Dockerfile ../../ -t $(IMAGE)
-
-push: build
-	docker push $(IMAGE)
-
-.PHONY: fv3net vcm build dev sklearn_run

From b31a0991c4b9f587a538aa0162375f38591ce015 Mon Sep 17 00:00:00 2001
From: "Noah D. Brenowitz" <nbren12@gmail.com>
Date: Tue, 17 Mar 2020 22:15:15 +0000
Subject: [PATCH 06/13] Add __version__ to fv3net init

---
 fv3net/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fv3net/__init__.py b/fv3net/__init__.py
index 186a2b559f..9e3c8a4193 100644
--- a/fv3net/__init__.py
+++ b/fv3net/__init__.py
@@ -2,3 +2,5 @@
 
 TOP_LEVEL_DIR = pathlib.Path(__file__).parent.parent.absolute()
 COARSENED_DIAGS_ZARR_NAME = "gfsphysics_15min_coarse.zarr"
+
+__version__ = '0.1.0'

From 8f9cb3c87ffe598941d8dc82f3b6ebeb4fd7ab4c Mon Sep 17 00:00:00 2001
From: "Noah D. Brenowitz" <nbren12@gmail.com>
Date: Tue, 17 Mar 2020 22:23:38 +0000
Subject: [PATCH 07/13] update prognostic_run_diags configuration

---
 workflows/prognostic_run_diags/argo.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/workflows/prognostic_run_diags/argo.yaml b/workflows/prognostic_run_diags/argo.yaml
index bc7f51501f..a082022471 100644
--- a/workflows/prognostic_run_diags/argo.yaml
+++ b/workflows/prognostic_run_diags/argo.yaml
@@ -32,7 +32,7 @@ spec:
        value: "climate-sim-pool"
        effect: "NoSchedule"
      container:
-      image: us.gcr.io/vcm-ml/fv3net@sha256:3b884201e8c61e9db248dec0f863dbd4dba4a9b18bc8a9794823e8ea4b7d2e0b
+             image: us.gcr.io/vcm-ml/fv3net:0.1.0
       command: ['bash', 'upload_report.sh']
       workingDir: /home/jovyan/fv3net/workflows/prognostic_run_diags
       env:
@@ -62,7 +62,7 @@ spec:
        value: "climate-sim-pool"
        effect: "NoSchedule"
      container:
-      image: us.gcr.io/vcm-ml/fv3net@sha256:3b884201e8c61e9db248dec0f863dbd4dba4a9b18bc8a9794823e8ea4b7d2e0b
+      image: us.gcr.io/vcm-ml/fv3net:0.1.0
       command:
       - 'python'
       - '-m'

From 31930d9484b95e9df4d4b93eee9286f4e8491cbf Mon Sep 17 00:00:00 2001
From: "Noah D. Brenowitz" <nbren12@gmail.com>
Date: Tue, 17 Mar 2020 22:24:18 +0000
Subject: [PATCH 08/13] black

---
 fv3net/__init__.py         | 2 +-
 fv3net/runtime/__init__.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/fv3net/__init__.py b/fv3net/__init__.py
index 9e3c8a4193..d2f8e47558 100644
--- a/fv3net/__init__.py
+++ b/fv3net/__init__.py
@@ -3,4 +3,4 @@
 TOP_LEVEL_DIR = pathlib.Path(__file__).parent.parent.absolute()
 COARSENED_DIAGS_ZARR_NAME = "gfsphysics_15min_coarse.zarr"
 
-__version__ = '0.1.0'
+__version__ = "0.1.0"
diff --git a/fv3net/runtime/__init__.py b/fv3net/runtime/__init__.py
index 80c9e7e8f6..5cb26a73c1 100644
--- a/fv3net/runtime/__init__.py
+++ b/fv3net/runtime/__init__.py
@@ -1,3 +1,3 @@
 from . import sklearn_interface as sklearn
 from .state_io import init_writers, append_to_writers, CF_TO_RESTART_MAP
-from .config import get_runfile_config, get_namelist
\ No newline at end of file
+from .config import get_runfile_config, get_namelist

From f99476680039bd29cd1dd1327fe922572706adc6 Mon Sep 17 00:00:00 2001
From: "Noah D. Brenowitz" <nbren12@gmail.com>
Date: Tue, 17 Mar 2020 22:28:03 +0000
Subject: [PATCH 09/13] update readme

---
 README.md | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 0020b473a2..8b86d23fc1 100644
--- a/README.md
+++ b/README.md
@@ -72,6 +72,17 @@ The main data processing pipelines for this project currently utilize Google Clo
 Dataflow and Kubernetes with Docker images.  Run scripts to deploy these workflows
 along with information can be found under the `workflows` directory.
 
+## Building the fv3net docker images
+
+The workflows use a pair of common images:
+
+|Image| Description| 
+| `us.gcr.io/vcm-ml/prognostic_run` | fv3gfs-python with minimal fv3net and vcm installed |
+| `us.gcr.io/vcm-ml/fv3net` | fv3net image with all dependencies including plotting |
+
+These images can be built and pushed to GCR using `make build_images` and
+`make push_images`, respectively.
+
 ## Dataflow
 
 Dataflow jobs run in a "serverless" style where data is piped between workers who
@@ -117,11 +128,6 @@ If you get an error `Could not create workflow; user does not have write access
 trying to submit the dataflow job, do `gcloud auth application-default login` first and then retry.
 
 
-## Building the fv3net docker images
-
-The pipelines use a pair of common images:
-1. us.gcr.io/vcm-ml/prognostic_run:v0.1.0
-1. us.gcr.io/vcm-ml/fv3net
 
 ## Deploying on k8s with fv3net
 

From 871be05d813e780f25b2374a6b68af1543786e5f Mon Sep 17 00:00:00 2001
From: "Noah D. Brenowitz" <nbren12@gmail.com>
Date: Tue, 17 Mar 2020 22:42:01 +0000
Subject: [PATCH 10/13] Fix table in README

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 8b86d23fc1..00601d57ae 100644
--- a/README.md
+++ b/README.md
@@ -77,6 +77,7 @@ along with information can be found under the `workflows` directory.
 The workflows use a pair of common images:
 
 |Image| Description| 
+|-----|------------|
 | `us.gcr.io/vcm-ml/prognostic_run` | fv3gfs-python with minimal fv3net and vcm installed |
 | `us.gcr.io/vcm-ml/fv3net` | fv3net image with all dependencies including plotting |
 

From db16a398ce2bae5353ab451fb8e7406e27269945 Mon Sep 17 00:00:00 2001
From: "Noah D. Brenowitz" <nbren12@gmail.com>
Date: Tue, 17 Mar 2020 22:46:00 +0000
Subject: [PATCH 11/13] fix yaml bug

---
 workflows/prognostic_run_diags/argo.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/workflows/prognostic_run_diags/argo.yaml b/workflows/prognostic_run_diags/argo.yaml
index a082022471..58713b68c9 100644
--- a/workflows/prognostic_run_diags/argo.yaml
+++ b/workflows/prognostic_run_diags/argo.yaml
@@ -32,7 +32,7 @@ spec:
        value: "climate-sim-pool"
        effect: "NoSchedule"
      container:
-             image: us.gcr.io/vcm-ml/fv3net:0.1.0
+      image: us.gcr.io/vcm-ml/fv3net:0.1.0
       command: ['bash', 'upload_report.sh']
       workingDir: /home/jovyan/fv3net/workflows/prognostic_run_diags
       env:

From 47d69e3d2037494b90815b1506f04db6042f4573 Mon Sep 17 00:00:00 2001
From: "Noah D. Brenowitz" <nbren12@gmail.com>
Date: Tue, 17 Mar 2020 22:58:05 +0000
Subject: [PATCH 12/13] pin pandas version to 1.0.1

---
 environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/environment.yml b/environment.yml
index 8f2194b35f..593d9ef017 100644
--- a/environment.yml
+++ b/environment.yml
@@ -18,6 +18,7 @@ dependencies:
   - h5netcdf
   - h5py>=2.10
   - hypothesis
+  - pandas=1.0.1
   - intake
   - intake-xarray
   - metpy

From 87a912757f9421d34814c190ae5534759d83e498 Mon Sep 17 00:00:00 2001
From: "Noah D. Brenowitz" <nbren12@gmail.com>
Date: Sat, 21 Mar 2020 00:10:44 +0000
Subject: [PATCH 13/13] correct versions in argo

---
 HISTORY.rst                              | 11 +++++++++++
 workflows/prognostic_run_diags/argo.yaml |  4 ++--
 2 files changed, 13 insertions(+), 2 deletions(-)
 create mode 100644 HISTORY.rst

diff --git a/HISTORY.rst b/HISTORY.rst
new file mode 100644
index 0000000000..59077a786a
--- /dev/null
+++ b/HISTORY.rst
@@ -0,0 +1,11 @@
+=======
+History
+=======
+
+Current
+-------
+
+0.1.0 (2020-03-20)
+------------------
+
+* First release of fv3net
diff --git a/workflows/prognostic_run_diags/argo.yaml b/workflows/prognostic_run_diags/argo.yaml
index 58713b68c9..a184d2174d 100644
--- a/workflows/prognostic_run_diags/argo.yaml
+++ b/workflows/prognostic_run_diags/argo.yaml
@@ -32,7 +32,7 @@ spec:
        value: "climate-sim-pool"
        effect: "NoSchedule"
      container:
-      image: us.gcr.io/vcm-ml/fv3net:0.1.0
+      image: us.gcr.io/vcm-ml/fv3net:v0.1.0
       command: ['bash', 'upload_report.sh']
       workingDir: /home/jovyan/fv3net/workflows/prognostic_run_diags
       env:
@@ -62,7 +62,7 @@ spec:
        value: "climate-sim-pool"
        effect: "NoSchedule"
      container:
-      image: us.gcr.io/vcm-ml/fv3net:0.1.0
+      image: us.gcr.io/vcm-ml/fv3net:v0.1.0
       command:
       - 'python'
       - '-m'