diff --git a/fv3net/diagnostics/sklearn_model_performance/__main__.py b/fv3net/diagnostics/sklearn_model_performance/__main__.py index cae28b02fd..bf6c64afdc 100644 --- a/fv3net/diagnostics/sklearn_model_performance/__main__.py +++ b/fv3net/diagnostics/sklearn_model_performance/__main__.py @@ -36,8 +36,9 @@ help="Output dir to write results to. Can be local or a GCS path.", ) parser.add_argument( - "num_test_zarrs", + "--num_test_zarrs", type=int, + default=4, help="Number of zarrs to concat together for use as test set.", ) parser.add_argument( @@ -61,6 +62,7 @@ help="Factor by which to downsample test set time steps", ) args = parser.parse_args() + args.test_data_path = os.path.join(args.test_data_path, "test") # if output path is remote GCS location, save results to local output dir first proto = get_protocol(args.output_path) diff --git a/fv3net/pipelines/coarsen_restarts/__main__.py b/fv3net/pipelines/coarsen_restarts/__main__.py index b7e332b7bd..157f7896c4 100644 --- a/fv3net/pipelines/coarsen_restarts/__main__.py +++ b/fv3net/pipelines/coarsen_restarts/__main__.py @@ -14,14 +14,6 @@ type=str, help="Full GCS path to input data for downloading timesteps.", ) - parser.add_argument( - "gcs_dst_dir", - type=str, - help=( - "Full GCS path to output coarsened timestep data. Defaults to input path" - "with target resolution appended as a directory" - ), - ) parser.add_argument( "gcs_grid_spec_path", type=str, @@ -36,6 +28,14 @@ parser.add_argument( "target_resolution", type=int, help="Target coarsening resolution to output.", ) + parser.add_argument( + "gcs_dst_dir", + type=str, + help=( + "Full GCS path to output coarsened timestep data. Defaults to input path" + "with target resolution appended as a directory" + ), + ) parser.add_argument( "--no-target-subdir", action="store_true", diff --git a/fv3net/regression/sklearn/test.py b/fv3net/regression/sklearn/test.py index 96605dd7fa..fccbf99653 100644 --- a/fv3net/regression/sklearn/test.py +++ b/fv3net/regression/sklearn/test.py @@ -1,8 +1,10 @@ from vcm.cloud import fsspec import joblib import xarray as xr +import os from ..dataset_handler import stack_and_drop_nan_samples +from .train import MODEL_FILENAME from vcm.convenience import round_time from vcm.cubedsphere.constants import INIT_TIME_DIM from fv3net.pipelines.create_training_data import ( @@ -82,7 +84,7 @@ def load_model(model_path): protocol = fsspec.get_protocol(model_path) if protocol == "gs": fs = fsspec.get_fs(model_path) - fs.get(model_path, "temp_model.pkl") + fs.get(os.path.join(model_path, MODEL_FILENAME), "temp_model.pkl") return joblib.load("temp_model.pkl") else: return joblib.load(model_path) diff --git a/fv3net/regression/sklearn/train.py b/fv3net/regression/sklearn/train.py index bb714cbb84..550a10b38f 100644 --- a/fv3net/regression/sklearn/train.py +++ b/fv3net/regression/sklearn/train.py @@ -160,6 +160,7 @@ def save_output(output_url, model, config): "remove local copy after upload.", ) args = parser.parse_args() + args.train_data_path = os.path.join(args.train_data_path, "train") train_config = load_model_training_config( args.train_config_file, args.train_data_path ) diff --git a/workflows/coarsen_c384_diagnostics/README.md b/workflows/coarsen_c384_diagnostics/README.md index 8e98ba2393..abdb6968e5 100644 --- a/workflows/coarsen_c384_diagnostics/README.md +++ b/workflows/coarsen_c384_diagnostics/README.md @@ -1,7 +1,7 @@ #### Usage Run `workflows/coarsen_c384_diagnostics/coarsen_c384_diagnostics.sh` -` {INPUT_LOCATION} {OUTPUT_LOCATION} {CONFIG_LOCATION}` +` {INPUT_LOCATION} {CONFIG_LOCATION} {OUTPUT_LOCATION}` #### Description This script coarsens a subset of diagnostic variables from the high resolution runs diff --git a/workflows/coarsen_c384_diagnostics/coarsen_c384_diagnostics.py b/workflows/coarsen_c384_diagnostics/coarsen_c384_diagnostics.py index d18c60163c..ae8793425f 100644 --- a/workflows/coarsen_c384_diagnostics/coarsen_c384_diagnostics.py +++ b/workflows/coarsen_c384_diagnostics/coarsen_c384_diagnostics.py @@ -3,7 +3,6 @@ import shutil import argparse import yaml -import fsspec import xarray as xr from vcm import coarsen @@ -18,7 +17,7 @@ VAR_LON_OUTER, VAR_LAT_OUTER, ) -from vcm.fv3_restarts import _split_url +from vcm.cloud.fsspec import get_fs from fv3net import COARSENED_DIAGS_ZARR_NAME logging.basicConfig(level=logging.INFO) @@ -79,8 +78,7 @@ def _get_config(config_path): def _get_remote_diags(diags_path): - proto, path = _split_url(diags_path) - fs = fsspec.filesystem(proto) + fs = get_fs(diags_path) mapper = fs.get_mapper(diags_path) return xr.open_zarr(mapper) @@ -90,13 +88,13 @@ def _get_remote_diags(diags_path): parser.add_argument( "input_path", type=str, help="GCS location of C384 diagnostics data zarrs." ) + parser.add_argument( + "config_path", type=str, help="Location of diagnostics coarsening config yaml." + ) parser.add_argument( "output_path", type=str, help="GCS location where= coarsened diagnostics zarrs will be written.", ) - parser.add_argument( - "config_path", type=str, help="Location of diagnostics coarsening config yaml." - ) args = parser.parse_args() coarsen_c384_diagnostics(args) diff --git a/workflows/coarsen_restarts/README.md b/workflows/coarsen_restarts/README.md index d13857789a..34386cc8a2 100644 --- a/workflows/coarsen_restarts/README.md +++ b/workflows/coarsen_restarts/README.md @@ -8,26 +8,20 @@ using pressure-level coarsening defined in `vcm.coarsen`. ```python fv3net.pipelines.coarsen_restarts -usage: __main__.py [-h] --gcs-src-dir GCS_SRC_DIR [--gcs-dst-dir GCS_DST_DIR] - --gcs-grid-spec-path GCS_GRID_SPEC_PATH --source-resolution - SOURCE_RESOLUTION --target_resolution TARGET_RESOLUTION +usage: __main__.py [-h] GCS_SRC_DIR GCS_GRID_SPEC_PATH SOURCE_RESOLUTION + TARGET_RESOLUTION GCS_DST_DIR -optional arguments: +positional arguments: -h, --help show this help message and exit - --gcs-src-dir GCS_SRC_DIR - Full GCS path to input data for downloading timesteps. - --gcs-dst-dir GCS_DST_DIR - Full GCS path to output coarsened timestep data. - Defaults to input pathwith target resolution appended - as a directory - --gcs-grid-spec-path GCS_GRID_SPEC_PATH - Full path with file wildcard 'grid_spec.tile*.nc' to + GCS_SRC_DIR Full GCS path to input data for downloading timesteps + GCS_GRID_SPEC_PATH Full path with file wildcard 'grid_spec.tile*.nc' to select grid spec files with same resolution as the source data - --source-resolution SOURCE_RESOLUTION - Source data cubed-sphere grid resolution. - --target_resolution TARGET_RESOLUTION - Target coarsening resolution to output + SOURCE_RESOLUTION Source data cubed-sphere grid resolution. + TARGET_RESOLUTION Target coarsening resolution to output + GCS_DST_DIR Full GCS path to output coarsened timestep data. + Defaults to input pathwith target resolution appended + as a directory ``` See `workflows/coarsen_restarts/submit_job.sh` to see an example of calling this diff --git a/workflows/coarsen_restarts/orchestrator_job.sh b/workflows/coarsen_restarts/orchestrator_job.sh deleted file mode 100755 index a31d833438..0000000000 --- a/workflows/coarsen_restarts/orchestrator_job.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -INPUT_PATH=$1 -GCS_GRIDSPEC_PATH=$2 -OUTPUT_PATH=$3 -SRC_RESOLUTION=$4 -TARGET_RESOLUTION=$5 - -user=$(whoami) -user=${user,,} - -python -m fv3net.pipelines.coarsen_restarts\ - $INPUT_PATH \ - $OUTPUT_PATH \ - $GCS_GRIDSPEC_PATH \ - $SRC_RESOLUTION \ - $TARGET_RESOLUTION \ - --no-target-subdir \ - --runner DataflowRunner \ - --job_name coarsen-restarts-$user \ - --project vcm-ml \ - --region us-central1 \ - --temp_location gs://vcm-ml-data/tmp_dataflow \ - --num_workers 3 \ - --max_num_workers 50 \ - --disk_size_gb 50 \ - --worker_machine_type n1-highmem-4 \ - --setup_file ./setup.py \ - --extra_package external/vcm/dist/vcm-0.1.0.tar.gz \ - --extra_package external/vcm/external/mappm/dist/mappm-0.0.0.tar.gz \ No newline at end of file diff --git a/workflows/coarsen_restarts/orchestrator_job_directrunner.sh b/workflows/coarsen_restarts/orchestrator_job_directrunner.sh deleted file mode 100755 index ccf32a4842..0000000000 --- a/workflows/coarsen_restarts/orchestrator_job_directrunner.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -INPUT_PATH=$1 -GRIDSPEC_PATH=$2 -OUTPUT_PATH=$3 -SRC_RESOLUTION=$4 -TARGET_RESOLUTION=$5 - -python -m fv3net.pipelines.coarsen_restarts\ - $INPUT_PATH \ - $OUTPUT_PATH \ - $GRIDSPEC_PATH \ - $SRC_RESOLUTION \ - $TARGET_RESOLUTION \ - --no-target-subdir \ - --runner DirectRunner \ No newline at end of file diff --git a/workflows/coarsen_restarts/submit_job.sh b/workflows/coarsen_restarts/submit_job.sh index 26f5d1cfd3..515f98152f 100755 --- a/workflows/coarsen_restarts/submit_job.sh +++ b/workflows/coarsen_restarts/submit_job.sh @@ -1,20 +1,20 @@ #!/bin/sh GCS_SRC="gs://vcm-ml-data/2019-12-02-40-day-X-SHiELD-simulation-C384-restart-files" -GCS_DST="gs://vcm-ml-data/2020-01-16-X-SHiELD-2019-12-02-pressure-coarsened-rundirs/restarts" GCS_GRIDSPEC="gs://vcm-ml-data/2020-01-06-C384-grid-spec-with-area-dx-dy" SRC_RESOLUTION=384 TARGET_RESOLUTION=48 +GCS_DST="gs://vcm-ml-data/2020-01-16-X-SHiELD-2019-12-02-pressure-coarsened-rundirs/restarts" user=$(whoami) user=${user,,} python -m fv3net.pipelines.coarsen_restarts\ $GCS_SRC \ - $GCS_DST \ $GCS_GRIDSPEC \ $SRC_RESOLUTION \ $TARGET_RESOLUTION \ + $GCS_DST \ --runner DataflowRunner \ --job_name coarsen-restarts-$user \ --project vcm-ml \ diff --git a/workflows/coarsen_restarts/submit_job_directrunner.sh b/workflows/coarsen_restarts/submit_job_directrunner.sh index 9eb0f9b7f8..1e0bc9f6dc 100755 --- a/workflows/coarsen_restarts/submit_job_directrunner.sh +++ b/workflows/coarsen_restarts/submit_job_directrunner.sh @@ -1,15 +1,15 @@ #!/bin/sh GCS_SRC="gs://vcm-ml-data/2019-12-02-40-day-X-SHiELD-simulation-C384-restart-files" -GCS_DST="gs://vcm-ml-data/2020-01-16-X-SHiELD-2019-12-02-pressure-coarsened-rundirs/restarts" GCS_GRIDSPEC="gs://vcm-ml-data/2020-01-06-C384-grid-spec-with-area-dx-dy" SRC_RESOLUTION=384 TARGET_RESOLUTION=48 +GCS_DST="gs://vcm-ml-data/2020-01-16-X-SHiELD-2019-12-02-pressure-coarsened-rundirs/restarts" python -m fv3net.pipelines.coarsen_restarts\ $GCS_SRC \ - $GCS_DST \ $GCS_GRIDSPEC \ $SRC_RESOLUTION \ $TARGET_RESOLUTION \ + $GCS_DST \ --runner DirectRunner \ No newline at end of file diff --git a/workflows/create_training_data/orchestrator_job.sh b/workflows/create_training_data/orchestrator_job.sh deleted file mode 100755 index 80bc09c8dd..0000000000 --- a/workflows/create_training_data/orchestrator_job.sh +++ /dev/null @@ -1,22 +0,0 @@ -DATA_PATH=$1 -DIAG_PATH=$2 -OUTPUT_PATH=$3 - -user=$(whoami) -user=${user,,} - -python -m fv3net.pipelines.create_training_data \ -${DATA_PATH} \ -${DIAG_PATH} \ -${OUTPUT_PATH} \ ---job_name create-training-data-${user} \ ---project vcm-ml \ ---region us-central1 \ ---runner DataflowRunner \ ---temp_location gs://vcm-ml-data/tmp_dataflow \ ---num_workers 4 \ ---max_num_workers 30 \ ---disk_size_gb 30 \ ---worker_machine_type n1-standard-1 \ ---setup_file ./setup.py \ ---extra_package external/vcm/dist/vcm-0.1.0.tar.gz diff --git a/workflows/create_training_data/orchestrator_job_directrunner_test.sh b/workflows/create_training_data/orchestrator_job_directrunner_test.sh deleted file mode 100755 index f90b6dcea6..0000000000 --- a/workflows/create_training_data/orchestrator_job_directrunner_test.sh +++ /dev/null @@ -1,13 +0,0 @@ -DATA_PATH=$1 -DIAG_PATH=$2 -OUTPUT_PATH=$3 -TIMESTEPS_PER_OUTPUT=$4 -TRAIN_FRACTION=$5 - -python -m fv3net.pipelines.create_training_data \ -${DATA_PATH} \ -${DIAG_PATH} \ -${OUTPUT_PATH} \ ---timesteps-per-output-file ${TIMESTEPS_PER_OUTPUT} \ ---train-fraction ${TRAIN_FRACTION} \ ---runner DirectRunner \ No newline at end of file diff --git a/workflows/create_training_data/submit_job.sh b/workflows/create_training_data/submit_job.sh index 4eeae3e93e..b2e794de7e 100755 --- a/workflows/create_training_data/submit_job.sh +++ b/workflows/create_training_data/submit_job.sh @@ -1,8 +1,8 @@ python -m fv3net.pipelines.create_training_data \ -gs://vcm-ml-data/2020-02-28-X-SHiELD-2019-12-02-deep-and-mp-off \ -gs://vcm-ml-data/orchestration-testing/shield-coarsened-diags-2019-12-04 \ -gs://vcm-ml-data/experiments-2020-03/deep-conv-mp-off/train-test-data \ ---job_name test-job-create-training-data-annak \ +gs://vcm-ml-data/2020-01-16-X-SHiELD-2019-12-02-pressure-coarsened-rundirs/one_step_output/C48 \ +gs://vcm-ml-data/2019-12-05-40-day-X-SHiELD-simulation-C384-diagnostics/C48_gfsphysics_15min_coarse.zarr \ +gs://vcm-ml-data/test-annak/2020-02-05_train_data_pipeline/ \ +--job_name test-job-create-training-data-brianh \ --project vcm-ml \ --region us-central1 \ --runner DataflowRunner \ diff --git a/workflows/end_to_end/README.md b/workflows/end_to_end/README.md index 9282442f24..068a8e8026 100644 --- a/workflows/end_to_end/README.md +++ b/workflows/end_to_end/README.md @@ -1,6 +1,6 @@ # End to End Workflow -February 2020 +Updated March 2020 This workflow serves to orchestrate the current set of VCM-ML workflows into one, allowing for going from a completed SHiELD run to a prognostic coarse run with ML @@ -15,6 +15,7 @@ The follwing steps are currently integrated into the workflow for use in experim - training an sklearn model - testing an sklearn model - a prognostic run of the coarse model using the trained sklearn parameterization +- generation of diagnostics for the prognostic run The workflow starting point is flexible, i.e., with any of the steps above, as is its endpoint. If starting at a SHiELD run coarsened to C384, it is required that @@ -22,6 +23,11 @@ the SHiELD C384 restart files and diagnostics are available (locally or remotely If starting at a later point, it is assumed that the outputs of all previous steps are available. +The workflow syntax, specified in the yaml, is designed to be flexible enough to allow +any set of positional and optional arguments to be sent to a `python argparse` script +interface. Thus, intermediate shell scripts to launch steps are unnecessary and +discouraged to reduce workflow maintenance. + ## Usage @@ -55,24 +61,27 @@ experiment: - train_sklearn_model - test_sklearn_model - prognostic_run + - diags_prognostic_run steps_config: coarsen_restarts: - command: workflows/coarsen_restarts/orchestrator_job.sh - inputs: + command: python -m fv3net.pipelines.coarsen_restarts + args: data_to_coarsen: location: gs://vcm-ml-data/orchestration-testing/shield-C384-restarts-2019-12-04 - extra_args: + grid_spec: + location: gs://vcm-ml-data/2020-01-06-C384-grid-spec-with-area-dx-dy source-resolution: 384 target-resolution: 48 + --runner: DataflowRunner one_step_run: - command: python ./workflows/one_step_jobs/orchestrate_submit_jobs.py - inputs: + command: python workflows/one_step_jobs/orchestrate_submit_jobs.py + args: restart_data: - from: coarsen_restarts - extra_args: - experiment_yaml: one_step_yaml - experiment_label: test-orchestration-group + from: coarsen_restart + experiment_yaml: workflows/one_step_jobs/all-physics-off.yml + docker_image: us.gcr.io/vcm-ml/prognostic-run-orchestration --n-steps: 50 + ... ``` #### Top-level arguments: @@ -88,14 +97,17 @@ experiment: #### Step level arguments: -For each step in `steps_to_run`, its configuration is set by its equivalently named block in `step_config`. Step configuration can still be defined here but it will not be executed if it is excluded from `steps_to_run`. *Note:* the order of parameters correspond to the positional commandline arguments appended to the job `command`. Parameters are translated to arguments in the order of _inputs_, _output_, _methodargs_. +For each step in `steps_to_run`, its configuration is set by its equivalently named block in `step_config`. Step configuration can still be defined here but it will not be executed if it is excluded from `steps_to_run`. *Note:* the order of parameters correspond to the positional commandline arguments appended to the job `command`. Parameters are translated to arguments in the order of _args_, _output_. - **command**: command to execute the step, e.g., a python or bash command and script that executes the step (without its arguments) -- **inputs**: a list of required input data sources from either previous steps or pre-existing sources; for each input data type, _one_ (but not both) of the following must be specified: - - **from**: name of the previous step which produces the input data for this step, e.g., `coarsen_restarts` is an input to `one_step_run`. _Note:_ The step referenced in `from` does not have to be included in the `steps_to_run`. Whatever the output location is from that step will be used. However, to make things explicit you should specify the input using `location` instead of `from`. - - **location**: explicit path to required input data for this step, which has been generated prior to the current experiment +- **args**: a dict of positional and optional argument key-value pairs for the _command_ above. Arguments are used to specify step input data, configuration files, and parameters. Values may be literals, names of previous steps which to use output as input for the current step, paths to pre-existing data sources, or lists of multiple values for a given key; each value in the args dict must be one of the following types: + - **value/hashable**: A single argument value + - **dict**: if the arg value is a dict, it is for specifying the source of input data to the step. Supported keys in the dict are as follows (only one may be used for a given step argument): + - **from**: name of the previous step which produces the input data for this step, e.g., `coarsen_restarts` is an input to `one_step_run`. Whatever the output location is from that step will be used. The previous step must be run as part of the experiment. + - **location**: explicit path to required input data for this step, which has been generated prior to the current experiment. + - **list**: multiple values to be passed to the command under the same argument name. This is useful for the `--extra-packages` argument in dataflow jobs which may need to be called on multiple packages. + Argument keys may begin with `--` to denote an optional argument. Optional arguments will be appended to the end of the command as --{param_keyname} {param_value}. Arguments without the leading `--` are treated as positional arguments. - **output_location**: (Optional) explicit path to store output from this step. This parameter is autogenerated if not provided and is used as the source for any input `from` refs. -- **extra_args**: extra key-value pair paremeters required for the workflow step. These are commonly used to reference step-specific configuration files or non-data related step parameters; each value is provided as a command-line argument appended to `command` after I/O arguments. You can also specify python optional command-line arguments here by prefixing the optional variable with "--". These optional arguments will be appended to the end of the command as --{param_keyname} {param_value}. ### Data output locations @@ -107,10 +119,15 @@ If no `output_location` is specified for a step, it will be output via the foll where `experiment_name` is the name plus the UUID (if added), and the `step_name` is defined as the name of the workflow step with the first 3 extra_args key/values appended to it. +### The dataflow `--runner` argument + +For steps using Apache Beam and Google Dataflow, the `--runner` optional argument can be passed. This is a reserved argument key that accepts only two values: `DirectRunner` and `DataflowRunner`. If `DirectRunner`is used, this is passed as the `--runner` value to launch a local job. If `DataflowRunner` is passed, a set of Dataflow arguments are appended to the job submission script to enable launching Dataflow jobs on GCS. Those arguments are stored for specific steps which require them in `workflows/end_to_end/dataflow.py`. + + ## Creating new workflow steps To write a new step in the workflow, create a CLI for the step that follows the following format: -```{COMMAND} {INPUT_1} {INPUT_2} ... {OUTPUT_PATH} {EXTRA_ARG_1} {EXTRA_ARG_2} {EXTRA_ARG_3} ... {--OPTIONAL_ARG} {OPTIONAL_ARG_VALUE} ...``` +```{COMMAND} {POSITIONAL_ARG1} {POSITIONAL_ARG2} ... {POSITIONAL_ARGN} {OUTPUT_PATH} {--OPTIONAL_ARG} {OPTIONAL_ARG_VALUE} ...``` -then add the step to the config YAML file, in both the `steps_to_run` list and the `steps_config` dict. At a minimum, the `command` and `inputs` values must be specified for the step configuration. Additionally, the step must be listed in `steps_to_run` in the order in which it is necessary, i.e., after any steps upon which it depends for input, and before any steps that depend on it for output. \ No newline at end of file +then add the step to the config YAML file, in both the `steps_to_run` list and the `steps_config` dict. At a minimum, the `command` and `args` values must be specified for the step configuration. Additionally, the step must be listed in `steps_to_run` in the order in which it is necessary, i.e., after any steps upon which it depends for input, and before any steps that depend on it for output. \ No newline at end of file diff --git a/workflows/end_to_end/dataflow.py b/workflows/end_to_end/dataflow.py new file mode 100644 index 0000000000..4ab0c94eef --- /dev/null +++ b/workflows/end_to_end/dataflow.py @@ -0,0 +1,35 @@ +from getpass import getuser +from fv3net.pipelines.common import get_alphanumeric_unique_tag + +COARSEN_RESTARTS_DATAFLOW_ARGS = { + "--job_name": ( + f"coarsen-restarts-{getuser().lower()}-{get_alphanumeric_unique_tag(7)}" + ), + "--project": "vcm-ml", + "--region": "us-central1", + "--temp_location": "gs://vcm-ml-data/tmp_dataflow", + "--num_workers": 3, + "--max_num_workers": 50, + "--disk_size_gb": 50, + "--worker_machine_type": "n1-highmem-4", + "--setup_file": "./setup.py", + "--extra_package": [ + "external/vcm/dist/vcm-0.1.0.tar.gz", + "external/vcm/external/mappm/dist/mappm-0.0.0.tar.gz", + ], +} + +CREATE_TRAINING_DATAFLOW_ARGS = COARSEN_RESTARTS_DATAFLOW_ARGS.copy() +CREATE_TRAINING_DATAFLOW_ARGS.update( + { + "--job_name": ( + f"create-training-data-{getuser().lower()}-" + f"{get_alphanumeric_unique_tag(7)}" + ), + "--num_workers": 4, + "--max_num_workers": 30, + "--disk_size_gb": 30, + "--worker_machine_type": "n1-standard-1", + "--extra_package": "external/vcm/dist/vcm-0.1.0.tar.gz", + } +) diff --git a/workflows/end_to_end/example-workflow-config.yaml b/workflows/end_to_end/full-workflow-config.yaml similarity index 66% rename from workflows/end_to_end/example-workflow-config.yaml rename to workflows/end_to_end/full-workflow-config.yaml index 9e1355013f..95e424ff72 100644 --- a/workflows/end_to_end/example-workflow-config.yaml +++ b/workflows/end_to_end/full-workflow-config.yaml @@ -5,83 +5,83 @@ experiment: unique_id: True steps_to_run: - coarsen_restarts - - coarsen_diagnostics +# - coarsen_diagnostics - one_step_run - create_training_data - train_sklearn_model -# - test_sklearn_model + - test_sklearn_model - prognostic_run - - diags_prognostic_run +# - diags_prognostic_run steps_config: coarsen_restarts: - command: workflows/coarsen_restarts/orchestrator_job.sh - inputs: + command: python -m fv3net.pipelines.coarsen_restarts + args: data_to_coarsen: location: gs://vcm-ml-data/orchestration-testing/shield-C384-restarts-2019-12-04 grid_spec: location: gs://vcm-ml-data/2020-01-06-C384-grid-spec-with-area-dx-dy - extra_args: source-resolution: 384 target-resolution: 48 + --runner: DataflowRunner coarsen_diagnostics: command: python ./workflows/coarsen_c384_diagnostics/coarsen_c384_diagnostics.py - inputs: + args: c384_diagnostics: location: gs://vcm-ml-data/2019-12-05-40-day-X-SHiELD-simulation-C384-diagnostics/gfsphysics_15min_coarse.zarr - extra_args: coarsening_config: workflows/coarsen_c384_diagnostics/coarsen-c384-diagnostics.yml one_step_run: command: python workflows/one_step_jobs/orchestrate_submit_jobs.py - inputs: + args: restart_data: from: coarsen_restarts - extra_args: experiment_yaml: ./workflows/one_step_jobs/all-physics-off.yml docker_image: us.gcr.io/vcm-ml/prognostic-run-orchestration create_training_data: - command: workflows/create_training_data/orchestrator_job.sh - inputs: + command: python -m fv3net.pipelines.create_training_data + args: one_step_data: from: one_step_run diagnostics_data: - from: coarsen_diagnostics - extra_args: - timesteps-per-output-file: 2 - mask-to-surface-type: - train-fraction: 0.5 + location: gs://vcm-ml-data/orchestration-testing/shield-coarsened-diags-2019-12-04 + --train-fraction: 0.5 + --runner: DataflowRunner train_sklearn_model: - command: workflows/sklearn_regression/orchestrator_train_sklearn.sh - inputs: + command: python -m fv3net.regression.sklearn.train + args: training_data: from: create_training_data - extra_args: train-config-file: ./workflows/sklearn_regression/example_base_rf_training_config.yml test_sklearn_model: - command: workflows/sklearn_regression/orchestrator_test_sklearn.sh - inputs: + command: python -m fv3net.diagnostics.sklearn_model_performance + args: trained_model: from: train_sklearn_model testing_data: from: create_training_data diagnostics_data: - from: coarsen_diagnostics - extra_args: - num-test-zarrs: 4 + location: gs://vcm-ml-data/orchestration-testing/shield-coarsened-diags-2019-12-04 + --num_test_zarrs: 4 prognostic_run: command: python workflows/prognostic_c48_run/orchestrate_submit_job.py - inputs: - sklearn_model: - from: train_sklearn_model + args: restart_file_dir: - from: one_step_run - extra_args: - prognostic_yaml_adjust: workflows/prognostic_c48_run/prognostic_config.yml + from: one_step_run ic_timestep: "20160801.001500" docker_image: us.gcr.io/vcm-ml/prognostic-run-orchestration + --model_url: + from: train_sklearn_model + --prog_config_yml: workflows/prognostic_c48_run/prognostic_config.yml + + + diags_prognostic_run: + command: bash workflows/prognostic_run_diags/run_all.sh + args: + fv3_output: + from: prognostic_run diff --git a/workflows/end_to_end/get_experiment_steps_and_args.py b/workflows/end_to_end/get_experiment_steps_and_args.py index 0859fcb143..d47ed9b2a2 100644 --- a/workflows/end_to_end/get_experiment_steps_and_args.py +++ b/workflows/end_to_end/get_experiment_steps_and_args.py @@ -3,7 +3,13 @@ import json import os import uuid -from typing import List, Mapping +from typing import List, Mapping, Any, Hashable +from dataflow import COARSEN_RESTARTS_DATAFLOW_ARGS, CREATE_TRAINING_DATAFLOW_ARGS + +DATAFLOW_ARGS_MAPPING = { + "coarsen_restarts": COARSEN_RESTARTS_DATAFLOW_ARGS, + "create_training_data": CREATE_TRAINING_DATAFLOW_ARGS, +} def get_experiment_steps_and_args(config_file: str): @@ -16,12 +22,18 @@ def get_experiment_steps_and_args(config_file: str): config = yaml.safe_load(f) # Resolve inputs, outputs, and other config parameters + workflow_steps = config["experiment"]["steps_to_run"] + if any( + [step not in config["experiment"]["steps_config"] for step in workflow_steps] + ): + raise KeyError( + "'steps_to_run' list contains a step not defined in 'steps_config'." + ) _apply_config_transforms(config) - workflow_steps_config = config["experiment"]["steps_to_run"] - all_step_arguments = _get_all_step_arguments(workflow_steps_config, config) + all_step_arguments = _get_all_step_arguments(config) experiment_steps_and_args = { "name": config["experiment"]["name"], - "workflow": " ".join([step for step in workflow_steps_config]), + "workflow": " ".join([step for step in workflow_steps]), "command_and_args": all_step_arguments, } return json.dumps(experiment_steps_and_args) @@ -32,10 +44,10 @@ def _apply_config_transforms(config: Mapping): Transforms to apply to the configuration dictionary. All transforms are assumed to be in-place. """ - _add_unique_id(config) _resolve_output_location(config) _resolve_input_from(config) + _resolve_dataflow_args(config) def _add_unique_id(config: Mapping): @@ -53,9 +65,9 @@ def _add_unique_id(config: Mapping): def _resolve_output_location(config: Mapping): """Get the step output location if one is not specified""" root_exp_path = _get_experiment_path(config) - all_steps_config = config["experiment"]["steps_config"] + steps_config = config["experiment"]["steps_config"] - for step_name, step_config in all_steps_config.items(): + for step_name, step_config in steps_config.items(): if "output_location" in step_config: continue @@ -71,29 +83,44 @@ def _resolve_input_from(config: Mapping): steps if the "from" keyword is used along with a step name. """ - all_steps_config = config["experiment"]["steps_config"] + steps_config = config["experiment"]["steps_config"] - for step_name, step_config in all_steps_config.items(): - input_config = step_config["inputs"] + for step_name, step_config in steps_config.items(): + args_config = step_config["args"] - for input_source, source_info in input_config.items(): - location = source_info.get("location", None) - from_key = source_info.get("from", None) + for arg, val in args_config.items(): + if isinstance(val, Mapping): + _resolve_input_mapping(val, steps_config, arg) - if location is not None and from_key is not None: - raise ValueError( - f"Ambiguous input location for {step_name}-{input_source}." - f" Both 'from' and 'location' were specified" - ) - if location is not None: - continue - elif from_key is not None: - source_info["location"] = all_steps_config[from_key]["output_location"] - else: - raise KeyError( - f"Input section of {step_name} should have either 'location' " - "or 'from' specified in the orchestration configuration" - ) + +def _resolve_input_mapping(input_mapping: Mapping, steps_config: Mapping, arg: str): + + location = input_mapping.get("location", None) + from_key = input_mapping.get("from", None) + + if location is not None and from_key is not None: + raise ValueError( + f"Ambiguous input location for {arg}." + f" Both 'from' and 'location' were specified" + ) + if location is not None: + return + elif from_key is not None: + previous_step = steps_config.get(from_key, None) + if previous_step is not None: + input_mapping["location"] = previous_step["output_location"] + else: + raise KeyError( + f"A step argument specified 'from' another step requires " + f"that the other step's cofiguration be specified. Add " + f"'{from_key}' to the configuration or specify '{arg}' " + f"with 'location' instead." + ) + else: + raise KeyError( + f"{arg} is provided as a key-value pair," + f" but only 'location' or 'from' may be specified." + ) def _get_experiment_path(config: Mapping): @@ -117,23 +144,43 @@ def _get_experiment_path(config: Mapping): return f"{proto}://{root}/{experiment_name}" -def _get_all_step_arguments(workflow_steps: List[str], config: Mapping): +def _resolve_dataflow_args(config: Mapping): + """Add dataflow arguments to step if it is the job runner""" + + steps_config = config["experiment"]["steps_config"] + for step, step_config in steps_config.items(): + dataflow_arg = step_config["args"].get("--runner", None) + if dataflow_arg == "DataflowRunner": + step_config["args"].update(DATAFLOW_ARGS_MAPPING[step]) + elif dataflow_arg == "DirectRunner": + continue + elif dataflow_arg is not None: + raise ValueError( + f"'runner' arg must be 'DataflowRunner' or 'DirectRunner'; " + f"instead received '{dataflow_arg}'." + ) + + +def _get_all_step_arguments(config: Mapping): """Get a dictionary of each step with i/o and methedological arguments""" steps_config = config["experiment"]["steps_config"] all_step_arguments = {} - for i, step in enumerate(workflow_steps): - curr_config = steps_config[step] - all_input_locations = [ - input_info["location"] for input_info in curr_config["inputs"].values() - ] - output_location = curr_config["output_location"] - command = curr_config["command"] - extra_args = _generate_args(curr_config) - - input_args = " ".join(all_input_locations) - step_args = " ".join([command, input_args, output_location, extra_args]) - all_step_arguments[step] = step_args + for step, step_config in steps_config.items(): + step_args = [step_config["command"]] + required_args = [] + optional_args = [] + for key, value in step_config["args"].items(): + arg_string = _resolve_arg_values(key, value) + if arg_string.startswith("--"): + optional_args.append(arg_string) + else: + required_args.append(arg_string) + output_location = step_config["output_location"] + step_args.extend(required_args) + step_args.append(output_location) + step_args.extend(optional_args) + all_step_arguments[step] = " ".join(step_args) return all_step_arguments @@ -144,48 +191,55 @@ def _generate_output_path_from_config( """generate an output location stub from a step's argument configuration""" output_str = step_name - arg_config = step_config.get("extra_args", None) - if arg_config is not None: - arg_strs = [] - for i, (key, val) in enumerate(arg_config.items()): - if i >= max_config_stubs: - break - val = str(val) - - # get last part of path so string isn't so long - if "/" in val: - val = val.split("/")[-1] - - key = key.strip("--") # remove prefix of optional argument - key_val = f"{key}_{val}" - arg_strs.append(key_val) - arg_output_stub = "_".join(arg_strs) - output_str += "_" + arg_output_stub + arg_config = step_config.get("args", None) + arg_strs = [] + non_map_args = { + key: val for key, val in arg_config.items() if not isinstance(val, Mapping) + } + for n_stubs, (key, val) in enumerate(non_map_args.items(), 1): + if n_stubs > max_config_stubs: + break + val = str(arg_config[key]) + + # get last part of path so string isn't so long + if "/" in val: + val = val.split("/")[-1] + + key = key.strip("--") # remove prefix of optional argument + key_val = f"{key}_{val}" + arg_strs.append(key_val) + arg_output_stub = "_".join(arg_strs) + output_str += "_" + arg_output_stub return output_str -def _generate_args(step_config: Mapping): +def _resolve_arg_values(key: Hashable, value: Any) -> Hashable: + """take a step args key-value pair and process into an appropriate arg string" """ - Generate the arguments for the step as positional arguments - in a string followed by optional arguments. - """ - arg_config = step_config.get("extra_args", None) - - if arg_config is not None: - optional_args = [] - required_args = [] - for arg_key, arg_value in arg_config.items(): - if arg_key[:2] == "--": - optional_args += [arg_key, str(arg_value)] + if isinstance(value, Mapping): + # case for when the arg is a dict {"location" : path} + location_value = value.get("location", None) + if location_value is None: + raise ValueError("Argument 'location' value not specified.") + else: + if key.startswith("--"): + arg_values = " ".join([key, str(location_value)]) else: - required_args.append(str(arg_value)) - - combined_args = " ".join(required_args + optional_args) + arg_values = str(location_value) + elif isinstance(value, List): + # case for when the arg is a list + # i.e., multiple optional args with same key, needed for dataflow packages + multiple_optional_args = [] + for item in value: + multiple_optional_args.extend([key, item]) + arg_values = " ".join(multiple_optional_args) else: - combined_args = "" - - return combined_args + if key.startswith("--"): + arg_values = " ".join([key, str(value)]) + else: + arg_values = str(value) + return arg_values if __name__ == "__main__": diff --git a/workflows/one_step_jobs/README.md b/workflows/one_step_jobs/README.md index 8c48731377..8cef622d9f 100644 --- a/workflows/one_step_jobs/README.md +++ b/workflows/one_step_jobs/README.md @@ -13,18 +13,15 @@ have a total duration of 15 minutes. Workflow call signature: ``` $ python submit_jobs.py -h -usage: submit_jobs.py [-h] --one-step-yaml ONE_STEP_YAML --input-url INPUT_URL - --output-url OUTPUT_URL [--n-steps N_STEPS] [-o] +usage: submit_jobs.py [-h] INPUT_URL ONE_STEP_YAML OUTPUT_URL [--n-steps N_STEPS] [-o] -h, --help show this help message and exit - --one-step-yaml ONE_STEP_YAML - Path to local run configuration yaml. - --input-url INPUT_URL - Remote url to initial conditions. Initial conditions + INPUT_URL Remote url to initial conditions. Initial conditions are assumed to be stored as INPUT_URL/{timestamp}/{tim estamp}.{restart_category}.tile*.nc - --output-url OUTPUT_URL - Remote url where model configuration and output will + ONE_STEP_YAML Path to local run configuration yaml. + DOCKER_IMAGE fv3gfs-python model docker image. + OUTPUT_URL Remote url where model configuration and output will be saved. Specifically, configuration files will be saved to OUTPUT_URL/one_step_config and model output to OUTPUT_URL/one_step_output diff --git a/workflows/one_step_jobs/all-physics-on.yml b/workflows/one_step_jobs/all-physics-on.yml new file mode 100644 index 0000000000..ebe24267b0 --- /dev/null +++ b/workflows/one_step_jobs/all-physics-on.yml @@ -0,0 +1,22 @@ +kubernetes: + docker_image: us.gcr.io/vcm-ml/fv3gfs-python:v0.2.1 +fv3config: + diag_table: workflows/one_step_jobs/diag_table_one_step + namelist: + atmos_model_nml: + fhout: 0.01666 + coupler_nml: + days: 0 + minutes: 15 + seconds: 0 + dt_atmos: 60 # seconds + dt_ocean: 60 # seconds + restart_secs: 60 + fv_core_nml: + external_eta: true + npz: 79 + k_split: 1 + n_split: 1 + gfs_physics_nml: + do_deep: true + fhzero: 0.01666 diff --git a/workflows/one_step_jobs/orchestrate_submit_jobs.py b/workflows/one_step_jobs/orchestrate_submit_jobs.py index a4fc482b9f..421ea4b665 100644 --- a/workflows/one_step_jobs/orchestrate_submit_jobs.py +++ b/workflows/one_step_jobs/orchestrate_submit_jobs.py @@ -20,9 +20,6 @@ def _create_arg_parser(): help="Remote url to initial conditions. Initial conditions are assumed to be " "stored as INPUT_URL/{timestamp}/{timestamp}.{restart_category}.tile*.nc", ) - parser.add_argument( - "output_url", type=str, help="Remote url where model output will be saved." - ) parser.add_argument( "one_step_yaml", type=str, help="Path to local run configuration yaml.", ) @@ -31,6 +28,9 @@ def _create_arg_parser(): type=str, help="Docker image to use for performing the one-step FV3GFS runs.", ) + parser.add_argument( + "output_url", type=str, help="Remote url where model output will be saved." + ) parser.add_argument( "-o", "--overwrite", diff --git a/workflows/prognostic_c48_run/_submit_baseline_jobs.sh b/workflows/prognostic_c48_run/_submit_baseline_jobs.sh new file mode 100644 index 0000000000..6b713956bc --- /dev/null +++ b/workflows/prognostic_c48_run/_submit_baseline_jobs.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -x + +STAMP=$(date +%F)-$(uuid | head -c 6) +output=gs://vcm-ml-data/testing-2020-02 +urls=("gs://vcm-ml-data/2020-03-03-X-SHiELD-2019-12-02-deep-conv-off" "gs://vcm-ml-data/2020-02-26-X-SHiELD-2019-12-02-physics-off") +ic=20160803.061500 +image=us.gcr.io/vcm-ml/prognostic-run-orchestration:fv3py_v2.3-mp-off-switch + +for onestep_url in "${urls[@]}" +do + run=$(basename $onestep_url) + output_url=$output/$run/$STAMP/prognostic_run_baseline + python orchestrate_submit_job.py -d $onestep_url $output_url $ic $image +done \ No newline at end of file diff --git a/workflows/prognostic_c48_run/orchestrate_submit_job.py b/workflows/prognostic_c48_run/orchestrate_submit_job.py index 733431d9d1..61f9ce87f6 100644 --- a/workflows/prognostic_c48_run/orchestrate_submit_job.py +++ b/workflows/prognostic_c48_run/orchestrate_submit_job.py @@ -27,34 +27,44 @@ def _create_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() - parser.add_argument( - "model_url", type=str, help="Remote url to a trained sklearn model.", - ) parser.add_argument( "initial_condition_url", type=str, help="Remote url to directory holding timesteps with model initial conditions.", ) + parser.add_argument( + "ic_timestep", + type=str, + help="Time step to grab from the initial conditions url.", + ) + parser.add_argument( + "docker_image", + type=str, + help="Docker image to pull for the prognostic run kubernetes pod.", + ) parser.add_argument( "output_url", type=str, help="Remote storage location for prognostic run output.", ) parser.add_argument( - "prog_config_yml", + "--model_url", type=str, - help="Path to a config update YAML file specifying the changes (e.g., " - "diag_table, runtime, ...) from the one-step runs for the prognostic run.", + default=None, + help="Remote url to a trained sklearn model.", ) parser.add_argument( - "ic_timestep", + "--prog_config_yml", type=str, - help="Time step to grab from the initial conditions url.", + default="prognostic_config.yml", + help="Path to a config update YAML file specifying the changes (e.g., " + "diag_table, runtime, ...) from the one-step runs for the prognostic run.", ) parser.add_argument( - "docker_image", - type=str, - help="Docker image to pull for the prognostic run kubernetes pod.", + "-d", + "--detach", + action="store_true", + help="Do not wait for the k8s job to complete.", ) return parser @@ -114,26 +124,26 @@ def _update_with_prognostic_model_config(model_config, prognostic_config): ) # Add prognostic config section - model_config["scikit_learn"] = { - "model": os.path.join(args.model_url, MODEL_FILENAME), - "zarr_output": "diags.zarr", - } + if args.model_url: + model_config["scikit_learn"] = { + "model": os.path.join(args.model_url, MODEL_FILENAME), + "zarr_output": "diags.zarr", + } + kube_opts["runfile"] = kube_jobs.transfer_local_to_remote(RUNFILE, config_dir) # Upload the new prognostic config with fsspec.open(job_config_path, "w") as f: f.write(yaml.dump(model_config)) - remote_runfile_path = kube_jobs.transfer_local_to_remote(RUNFILE, config_dir) - fv3config.run_kubernetes( config_location=job_config_path, outdir=args.output_url, jobname=job_name, docker_image=args.docker_image, - runfile=remote_runfile_path, job_labels=job_label, **kube_opts, ) - successful, _ = kube_jobs.wait_for_complete(job_label) - kube_jobs.delete_job_pods(successful) + if not args.detach: + successful, _ = kube_jobs.wait_for_complete(job_label) + kube_jobs.delete_job_pods(successful) diff --git a/workflows/prognostic_c48_run/prognostic_config.yml b/workflows/prognostic_c48_run/prognostic_config.yml index 8397c2b8df..49d278bfc6 100644 --- a/workflows/prognostic_c48_run/prognostic_config.yml +++ b/workflows/prognostic_c48_run/prognostic_config.yml @@ -16,4 +16,4 @@ namelist: gfs_physics_nml: fhzero: 0.25 # hours - frequency at which precip is set back to zero fv_core_nml: - n_split: 6 # num dynamics steps per physics step \ No newline at end of file + n_split: 6 # num dynamics steps per physics step diff --git a/workflows/sklearn_regression/README.md b/workflows/sklearn_regression/README.md index 452c406b71..9684e68f59 100644 --- a/workflows/sklearn_regression/README.md +++ b/workflows/sklearn_regression/README.md @@ -8,9 +8,9 @@ the trained model output as well as a copy of the model configuration. Example shell script: ``` python -m fv3net.regression.sklearn.train \ - --train-config-file example_rf_training_config.yml \ - --output-dir-suffix sklearn_regression \ - --train-data-path gs://vcm-ml-data/test_annak/2020-02-05_train_data_pipeline/train + gs://vcm-ml-data/test_annak/2020-02-05_train_data_pipeline #input data path where "train" folder is located + example_rf_training_config.yml \ + {output_data_path} \ --delete-local-results-after-upload True ``` The last two arguments are optional and allow the user to save the output directory to @@ -51,8 +51,9 @@ a timestamped output directory. Example shell script: ``` python -m fv3net.regression.model_diagnostics \ - --test-data-path gs://vcm-ml-data/test-annak/2020-02-05_train_data_pipeline/test \ - --model-path 20200205.205016_model_training_files/20200205.205016_sklearn_model.pkl \ - --high-res-data-path gs://vcm-ml-data/2019-12-05-40-day-X-SHiELD-simulation-C384-diagnostics/C48_gfsphysics_15min_coarse.zarr \ + gs://vcm-ml-data/test-annak/2020-02-05_train_data_pipeline \ # location of "test" directory + 20200205.205016_model_training_files \ # location of "sklearn_model.pkl" file + gs://vcm-ml-data/2019-12-05-40-day-X-SHiELD-simulation-C384-diagnostics/C48_gfsphysics_15min_coarse.zarr \ + {output_path} \ --num-test-zarrs 8 ``` \ No newline at end of file diff --git a/workflows/sklearn_regression/orchestrator_test_sklearn.sh b/workflows/sklearn_regression/orchestrator_test_sklearn.sh deleted file mode 100755 index 9512aa034b..0000000000 --- a/workflows/sklearn_regression/orchestrator_test_sklearn.sh +++ /dev/null @@ -1,13 +0,0 @@ -MODEL_PATH=$1 -DATA_PATH=$2 -DIAGS_PATH=$3 -OUTPUT_PATH=$4 -NUM_TEST_ZARRS=$5 - - -python -m fv3net.diagnostics.sklearn_model_performance \ - ${MODEL_PATH}"/sklearn_model.pkl" \ - ${DATA_PATH}"/test" \ - ${DIAGS_PATH} \ - ${OUTPUT_PATH} \ - ${NUM_TEST_ZARRS} diff --git a/workflows/sklearn_regression/orchestrator_train_sklearn.sh b/workflows/sklearn_regression/orchestrator_train_sklearn.sh deleted file mode 100755 index 65254c61b4..0000000000 --- a/workflows/sklearn_regression/orchestrator_train_sklearn.sh +++ /dev/null @@ -1,8 +0,0 @@ -INPUT_DATA_PATH=$1 -OUTPUT_DATA_PATH=$2 -TRAINING_CONFIG_PATH=$3 - -python -m fv3net.regression.sklearn.train \ - ${INPUT_DATA_PATH}"/train" \ - ${TRAINING_CONFIG_PATH} \ - ${OUTPUT_DATA_PATH} diff --git a/workflows/sklearn_regression/test_sklearn.sh b/workflows/sklearn_regression/test_sklearn.sh index 924a3104a2..de58e6b1b0 100755 --- a/workflows/sklearn_regression/test_sklearn.sh +++ b/workflows/sklearn_regression/test_sklearn.sh @@ -1,6 +1,6 @@ python -m fv3net.diagnostics.sklearn_model_performance \ - 20200205.205016_model_training_files/20200205.205016_sklearn_model.pkl \ - gs://vcm-ml-data/test-annak/2020-02-20_train_data_pipeline_downsampled/test \ + gs://vcm-ml-data/orchestration-testing/test-experiment-7021f96d/train_sklearn_model_train-config-file_example_base_rf_training_config.yml \ + gs://vcm-ml-data/orchestration-testing/test-experiment-0ec4a4b1/create_training_data_train-fraction_0.5_runner_DataflowRunner \ gs://vcm-ml-data/2019-12-05-40-day-X-SHiELD-simulation-C384-diagnostics/C48_gfsphysics_15min_coarse.zarr \ gs://vcm-ml-public/test-annak/test-new-diags \ - 48 + --num_test_zarrs 4 diff --git a/workflows/sklearn_regression/train_sklearn.sh b/workflows/sklearn_regression/train_sklearn.sh index e23fd5b63f..07f5983442 100755 --- a/workflows/sklearn_regression/train_sklearn.sh +++ b/workflows/sklearn_regression/train_sklearn.sh @@ -1,4 +1,4 @@ python -m fv3net.regression.sklearn.train \ - gs://vcm-ml-data/test-annak/2020-02-05_train_data_pipeline/train/ \ + gs://vcm-ml-data/orchestration-testing/test-experiment-0ec4a4b1/create_training_data_train-fraction_0.5_runner_DataflowRunner \ workflows/sklearn_regression/example_rf_training_config.yml \ gs://vcm-ml-data/test_annak