Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CaseClass can look beyond /glade/scratch/mlevy #34

Merged
merged 6 commits into from
Nov 20, 2020
Merged

CaseClass can look beyond /glade/scratch/mlevy #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b4be3ac
CaseClass can look beyond /glade/scratch/mlevy
mnlevy1981 Nov 5, 2020
1dfd00d
Add run_root to Sanity Check notebook
mnlevy1981 Nov 6, 2020
ee612c7
Rerun plot_suite_003, delete some plot_suite_maps
mnlevy1981 Nov 6, 2020
8c5a316
More CaseClass API cleanup
mnlevy1981 Nov 10, 2020
fe19d13
More clean-up following review
mnlevy1981 Nov 19, 2020
f9b9e1a
update github action hook to setup-miniconda
mnlevy1981 Nov 20, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ jobs:
uses: actions/checkout@v2

- name: Install conda
uses: goanpeca/setup-miniconda@v1
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: true
activate-environment: hires-marbl
Expand Down
2 changes: 1 addition & 1 deletion data_reshaping/cice.h1_t13.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
#SBATCH -t 24:00:00
#SBATCH -p dav
#SBATCH -J Cice.h1_t13
#SBATCH --account=CESM0010
#SBATCH --account=P93300606
#SBATCH --mem 100G
#SBATCH -e logs/Cice.h1_t13.err.%J
#SBATCH -o logs/Cice.h1_t13.out.%J
Expand Down
2 changes: 1 addition & 1 deletion data_reshaping/cice.h_t13.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
#SBATCH -t 24:00:00
#SBATCH -p dav
#SBATCH -J Cice.h_t13
#SBATCH --account=CESM0010
#SBATCH --account=P93300606
#SBATCH --mem 100G
#SBATCH -e logs/Cice.h_t13.err.%J
#SBATCH -o logs/Cice.h_t13.out.%J
Expand Down
2 changes: 1 addition & 1 deletion data_reshaping/pop.h.nday1_t13.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
#SBATCH -t 24:00:00
#SBATCH -p dav
#SBATCH -J Pop.h.nday1_t13
#SBATCH --account=CESM0010
#SBATCH --account=P93300606
#SBATCH --mem 100G
#SBATCH -e logs/Pop.h.nday1_t13.err.%J
#SBATCH -o logs/Pop.h.nday1_t13.out.%J
Expand Down
2 changes: 1 addition & 1 deletion data_reshaping/pop.h.nyear1_t13.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
#SBATCH -t 24:00:00
#SBATCH -p dav
#SBATCH -J Pop.h.nyear1_t13
#SBATCH --account=CESM0010
#SBATCH --account=P93300606
#SBATCH --mem 100G
#SBATCH -e logs/Pop.h.nyear1_t13.err.%J
#SBATCH -o logs/Pop.h.nyear1_t13.out.%J
Expand Down
2 changes: 1 addition & 1 deletion data_reshaping/pop.h_t13.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
#SBATCH -t 24:00:00
#SBATCH -p dav
#SBATCH -J Pop.h_t13
#SBATCH --account=CESM0010
#SBATCH --account=P93300606
#SBATCH --mem 100G
#SBATCH -e logs/Pop.h_t13.err.%J
#SBATCH -o logs/Pop.h_t13.out.%J
Expand Down
73 changes: 60 additions & 13 deletions notebooks/Pull info from logs.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

57 changes: 47 additions & 10 deletions notebooks/Sanity Check.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,7 @@
},
"outputs": [],
"source": [
"%matplotlib inline\n",
"%load_ext autoreload\n",
"%autoreload 2\n",
"import os\n",
"\n",
"import utils"
]
Expand Down Expand Up @@ -38,33 +36,72 @@
}
],
"source": [
"# Set up dictionaries for CaseClass objects and Datasets\n",
"# Also set variable of interest (HMXL_2) and save\n",
"# a few directories that will be used in multiple init calls\n",
"cases = dict()\n",
"ds_dict = dict()\n",
"varnames = \"HMXL_2\"\n",
"caseroot_parent = os.path.join(\n",
" os.sep, \"glade\", \"work\", \"mlevy\", \"hi-res_BGC_JRA\", \"cases\"\n",
")\n",
"campaign_root = os.path.join(\n",
" os.sep,\n",
" \"glade\",\n",
" \"campaign\",\n",
" \"cesm\",\n",
" \"development\",\n",
" \"bgcwg\",\n",
" \"projects\",\n",
" \"hi-res_JRA\",\n",
" \"cases\",\n",
")\n",
"\n",
"# Case 001: output is split between two cases\n",
"casenames = [\n",
" \"g.e22.G1850ECO_JRA_HR.TL319_t13.001\",\n",
" \"g.e22.G1850ECO_JRA_HR.TL319_g17.001\",\n",
"]\n",
"cases[\"001\"] = utils.CaseClass(\n",
" [\n",
" \"g.e22.G1850ECO_JRA_HR.TL319_t13.001\",\n",
" \"g.e22.G1850ECO_JRA_HR.TL319_g17.001\",\n",
" ],\n",
" casenames,\n",
" utils.gen_output_roots_from_caseroot(\n",
" [os.path.join(caseroot_parent, casename) for casename in casenames]\n",
" ),\n",
")\n",
"ds_dict[\"001\"] = cases[\"001\"].gen_dataset(\n",
" stream=\"pop.h.nday1\", varnames=varnames\n",
")\n",
"\n",
"cases[\"002\"] = utils.CaseClass(\"g.e22.G1850ECO_JRA_HR.TL319_t13.002\")\n",
"# Case 002:\n",
"casename = \"g.e22.G1850ECO_JRA_HR.TL319_t13.002\"\n",
"cases[\"002\"] = utils.CaseClass(\n",
" casename,\n",
" utils.gen_output_roots_from_caseroot(\n",
" os.path.join(caseroot_parent, casename)\n",
" ),\n",
")\n",
"ds_dict[\"002\"] = cases[\"002\"].gen_dataset(\n",
" stream=\"pop.h.nday1\", varnames=varnames\n",
")\n",
"\n",
"cases[\"003\"] = utils.CaseClass(\"g.e22.G1850ECO_JRA_HR.TL319_t13.003\")\n",
"# Case 003: get time series files from campaign\n",
"casename = \"g.e22.G1850ECO_JRA_HR.TL319_t13.003\"\n",
"cases[\"003\"] = utils.CaseClass(\n",
" casename,\n",
" os.path.join(campaign_root, casename, \"output\"),\n",
")\n",
"ds_dict[\"003\"] = (\n",
" cases[\"003\"]\n",
" .gen_dataset(stream=\"pop.h.nday1\", varnames=varnames, end_year=1)\n",
" .isel(time=slice(0, 151))\n",
")\n",
"\n",
"cases[\"004\"] = utils.CaseClass(\"g.e22.G1850ECO_JRA_HR.TL319_t13.004\")\n",
"# Case 004: get time series files from campaign\n",
"casename = \"g.e22.G1850ECO_JRA_HR.TL319_t13.004\"\n",
"cases[\"004\"] = utils.CaseClass(\n",
" casename,\n",
" os.path.join(campaign_root, casename, \"output\"),\n",
")\n",
"ds_dict[\"004\"] = (\n",
" cases[\"004\"]\n",
" .gen_dataset(stream=\"pop.h.nday1\", varnames=varnames, end_year=1)\n",
Expand Down
42 changes: 35 additions & 7 deletions notebooks/compare_ts_and_hist_003.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"import utils"
]
},
Expand All @@ -20,48 +22,74 @@
"text": [
"Checking year 0001...\n",
"... checking stream pop.h.nyear1 ...\n",
"Skipping stream pop.h.nyear1 for year 0001 because there are no history files\n",
"... checking stream pop.h.nday1 ...\n",
"Skipping stream pop.h.nday1 for year 0001 because there are no history files\n",
"... checking stream pop.h ...\n",
"Skipping stream pop.h for year 0001 because there are no history files\n",
"... checking stream cice.h ...\n",
"All variables available in time series for year 0001\n",
"Skipping stream cice.h for year 0001 because there are no history files\n",
"Could not find time series for all variables in year 0001\n",
"----\n",
"Checking year 0002...\n",
"... checking stream pop.h.nyear1 ...\n",
"Skipping stream pop.h.nyear1 for year 0002 because there are no history files\n",
"... checking stream pop.h.nday1 ...\n",
"Skipping stream pop.h.nday1 for year 0002 because there are no history files\n",
"... checking stream pop.h ...\n",
"Skipping stream pop.h for year 0002 because there are no history files\n",
"... checking stream cice.h1 ...\n",
"Skipping stream cice.h1 for year 0002 because there are no history files\n",
"... checking stream cice.h ...\n",
"All variables available in time series for year 0002\n",
"Skipping stream cice.h for year 0002 because there are no history files\n",
"Could not find time series for all variables in year 0002\n",
"----\n",
"Checking year 0003...\n",
"... checking stream pop.h.nyear1 ...\n",
"Skipping stream pop.h.nyear1 for year 0003 because there are no history files\n",
"... checking stream pop.h.nday1 ...\n",
"Skipping stream pop.h.nday1 for year 0003 because there are no history files\n",
"... checking stream pop.h ...\n",
"Skipping stream pop.h for year 0003 because there are no history files\n",
"... checking stream cice.h1 ...\n",
"Skipping stream cice.h1 for year 0003 because there are no history files\n",
"... checking stream cice.h ...\n",
"All variables available in time series for year 0003\n",
"Skipping stream cice.h for year 0003 because there are no history files\n",
"Could not find time series for all variables in year 0003\n",
"----\n",
"Checking year 0004...\n",
"... checking stream pop.h.nyear1 ...\n",
"Skipping stream pop.h.nyear1 for year 0004 because there are no history files\n",
"... checking stream pop.h.nday1 ...\n",
"Skipping stream pop.h.nday1 for year 0004 because there are no history files\n",
"... checking stream pop.h ...\n",
"Skipping stream pop.h for year 0004 because there are no history files\n",
"... checking stream cice.h1 ...\n",
"Skipping stream cice.h1 for year 0004 because there are no history files\n",
"... checking stream cice.h ...\n",
"All variables available in time series for year 0004\n",
"Skipping stream cice.h for year 0004 because there are no history files\n",
"Could not find time series for all variables in year 0004\n",
"----\n",
"Checking year 0005...\n",
"... checking stream pop.h.nyear1 ...\n",
"Could not find time series for year 0005\n",
"CPU times: user 57.1 s, sys: 2.92 s, total: 1min\n",
"Wall time: 2min 5s\n"
"CPU times: user 207 ms, sys: 86.5 ms, total: 294 ms\n",
"Wall time: 1.39 s\n"
]
}
],
"source": [
"%%time\n",
"\n",
"casename = \"g.e22.G1850ECO_JRA_HR.TL319_t13.003\"\n",
"utils.timeseries_and_history_comparison(casename)"
"\n",
"# Directories to search for netCDF files\n",
"caseroot = os.path.join(os.sep, \"glade\", \"work\", \"mlevy\", \"hi-res_BGC_JRA\", \"cases\")\n",
"campaign_root = os.path.join(os.sep, \"glade\", \"campaign\", \"cesm\", \"development\", \"bgcwg\", \"projects\", \"hi-res_JRA\", \"cases\")\n",
"output_roots = [os.path.join(campaign_root, casename, \"output\")]\n",
"output_roots += utils.gen_output_roots_from_caseroot(os.path.join(caseroot, casename))\n",
"\n",
"utils.timeseries_and_history_comparison(casename, output_roots)"
]
}
],
Expand Down
73 changes: 69 additions & 4 deletions notebooks/compare_ts_and_hist_004.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"import utils"
]
},
Expand Down Expand Up @@ -99,17 +101,80 @@
"----\n",
"Checking year 0011...\n",
"... checking stream pop.h.nyear1 ...\n",
"Could not find time series for year 0011\n",
"CPU times: user 2min 25s, sys: 11.6 s, total: 2min 36s\n",
"Wall time: 6min 6s\n"
"... checking stream pop.h.nday1 ...\n",
"... checking stream pop.h ...\n",
"... checking stream cice.h1 ...\n",
"... checking stream cice.h ...\n",
"All variables available in time series for year 0011\n",
"----\n",
"Checking year 0012...\n",
"... checking stream pop.h.nyear1 ...\n",
"... checking stream pop.h.nday1 ...\n",
"... checking stream pop.h ...\n",
"... checking stream cice.h1 ...\n",
"... checking stream cice.h ...\n",
"All variables available in time series for year 0012\n",
"----\n",
"Checking year 0013...\n",
"... checking stream pop.h.nyear1 ...\n",
"... checking stream pop.h.nday1 ...\n",
"... checking stream pop.h ...\n",
"... checking stream cice.h1 ...\n",
"... checking stream cice.h ...\n",
"All variables available in time series for year 0013\n",
"----\n",
"Checking year 0014...\n",
"... checking stream pop.h.nyear1 ...\n",
"... checking stream pop.h.nday1 ...\n",
"... checking stream pop.h ...\n",
"... checking stream cice.h1 ...\n",
"... checking stream cice.h ...\n",
"All variables available in time series for year 0014\n",
"----\n",
"Checking year 0015...\n",
"... checking stream pop.h.nyear1 ...\n",
"... checking stream pop.h.nday1 ...\n",
"... checking stream pop.h ...\n",
"... checking stream cice.h1 ...\n",
"... checking stream cice.h ...\n",
"All variables available in time series for year 0015\n",
"----\n",
"Checking year 0016...\n",
"... checking stream pop.h.nyear1 ...\n",
"... checking stream pop.h.nday1 ...\n",
"... checking stream pop.h ...\n",
"... checking stream cice.h1 ...\n",
"... checking stream cice.h ...\n",
"All variables available in time series for year 0016\n",
"----\n",
"Checking year 0017...\n",
"... checking stream pop.h.nyear1 ...\n",
"... checking stream pop.h.nday1 ...\n",
"... checking stream pop.h ...\n",
"... checking stream cice.h1 ...\n",
"... checking stream cice.h ...\n",
"All variables available in time series for year 0017\n",
"----\n",
"Checking year 0018...\n",
"... checking stream pop.h.nyear1 ...\n",
"Could not find time series for year 0018\n",
"CPU times: user 4min 46s, sys: 3min 16s, total: 8min 3s\n",
"Wall time: 11min 3s\n"
]
}
],
"source": [
"%%time\n",
"\n",
"casename = \"g.e22.G1850ECO_JRA_HR.TL319_t13.004\"\n",
"utils.timeseries_and_history_comparison(casename)"
"\n",
"# Directories to search for netCDF files\n",
"caseroot = os.path.join(os.sep, \"glade\", \"work\", \"mlevy\", \"hi-res_BGC_JRA\", \"cases\")\n",
"campaign_root = os.path.join(os.sep, \"glade\", \"campaign\", \"cesm\", \"development\", \"bgcwg\", \"projects\", \"hi-res_JRA\", \"cases\")\n",
"output_roots = [os.path.join(campaign_root, casename, \"output\")]\n",
"output_roots += utils.gen_output_roots_from_caseroot(os.path.join(caseroot, casename))\n",
"\n",
"utils.timeseries_and_history_comparison(casename, output_roots)"
]
}
],
Expand Down
Loading