Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add option to plot utilities to generate PNG output rather than inline plots #42

Merged
merged 17 commits into from
Apr 22, 2021
Merged

Add option to plot utilities to generate PNG output rather than inline plots #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
a3a1a73
Add option summary_plot_maps() to save png files
mnlevy1981 Jan 26, 2021
ebb17db
generate plot catalog
andersy005 Jan 26, 2021
7ffe097
Use absolute paths for the time being
andersy005 Jan 26, 2021
b437e06
Prototype using parent class to help with plotting
mnlevy1981 Jan 27, 2021
d3994cd
Refactor summary_plot_maps() API
mnlevy1981 Jan 27, 2021
a1f1c7e
Add support for writing time series as pngs
mnlevy1981 Jan 28, 2021
b3e584f
Add savefig support for histograms and time series
mnlevy1981 Jan 28, 2021
4986914
Different isel_dict leads to different file names
mnlevy1981 Jan 28, 2021
43be393
bugfix: gen_dataset failed reading multiple vars
mnlevy1981 Jan 29, 2021
ce4ec17
Add support for PNGs from trend_maps notebooks
mnlevy1981 Jan 29, 2021
eefc145
A few changes to plot generation:
mnlevy1981 Mar 12, 2021
a94142f
Updated notebooks changing image dirtree
mnlevy1981 Mar 14, 2021
d99527d
Add kwargs for savefig
mnlevy1981 Apr 13, 2021
60079e5
All plotting utilities now pass kwargs to savefig
mnlevy1981 Apr 14, 2021
643f65e
Instead of isel_dict, metadata lists sel values
mnlevy1981 Apr 16, 2021
00274a3
Re-ran gen_csv
mnlevy1981 Apr 16, 2021
8c6880e
Updates following code review:
mnlevy1981 Apr 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,14 @@
**/__pycache__
notebooks/logs
data_reshaping/logs

# ignore images directory, except for the image catalog
# complicated, but it works... https://stackoverflow.com/a/16318111
notebooks/images/*
!notebooks/images/g.e22.G1850ECO_JRA_HR.TL319_t13.003
!notebooks/images/g.e22.G1850ECO_JRA_HR.TL319_t13.004
!notebooks/images/g.e22b05.G1850ECOIAF_JRA.TL319_g17.cocco.001
notebooks/images/g.e22.G1850ECO_JRA_HR.TL319_t13.003/*
notebooks/images/g.e22.G1850ECO_JRA_HR.TL319_t13.004/*
notebooks/images/g.e22b05.G1850ECOIAF_JRA.TL319_g17.cocco.001/*
!notebooks/images/*/png_catalog.csv
332 changes: 332 additions & 0 deletions notebooks/gen_csv.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,332 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from utils import generate_plot_catalog\n",
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Generating csv for histogram plots in g.e22b05.G1850ECOIAF_JRA.TL319_g17.cocco.001\n",
"Generating csv for summary_map plots in g.e22b05.G1850ECOIAF_JRA.TL319_g17.cocco.001\n",
"Generating csv for time_series plots in g.e22b05.G1850ECOIAF_JRA.TL319_g17.cocco.001\n",
"Generating csv for trend_hist plots in g.e22b05.G1850ECOIAF_JRA.TL319_g17.cocco.001\n",
"Found 0 files with extension=.json in images/g.e22b05.G1850ECOIAF_JRA.TL319_g17.cocco.001/trend_hist.\n",
"Generating csv for trend_map plots in g.e22b05.G1850ECOIAF_JRA.TL319_g17.cocco.001\n",
"Found 0 files with extension=.json in images/g.e22b05.G1850ECOIAF_JRA.TL319_g17.cocco.001/trend_map.\n",
"Generating csv for histogram plots in g.e22.G1850ECO_JRA_HR.TL319_t13.003\n",
"Generating csv for summary_map plots in g.e22.G1850ECO_JRA_HR.TL319_t13.003\n",
"Generating csv for time_series plots in g.e22.G1850ECO_JRA_HR.TL319_t13.003\n",
"Generating csv for trend_hist plots in g.e22.G1850ECO_JRA_HR.TL319_t13.003\n",
"Generating csv for trend_map plots in g.e22.G1850ECO_JRA_HR.TL319_t13.003\n",
"Generating csv for histogram plots in g.e22.G1850ECO_JRA_HR.TL319_t13.004\n",
"Generating csv for summary_map plots in g.e22.G1850ECO_JRA_HR.TL319_t13.004\n",
"Generating csv for time_series plots in g.e22.G1850ECO_JRA_HR.TL319_t13.004\n",
"Generating csv for trend_hist plots in g.e22.G1850ECO_JRA_HR.TL319_t13.004\n",
"Generating csv for trend_map plots in g.e22.G1850ECO_JRA_HR.TL319_t13.004\n"
]
}
],
"source": [
"df = dict()\n",
"for casename in [\n",
" \"g.e22b05.G1850ECOIAF_JRA.TL319_g17.cocco.001\",\n",
" \"g.e22.G1850ECO_JRA_HR.TL319_t13.003\",\n",
" \"g.e22.G1850ECO_JRA_HR.TL319_t13.004\",\n",
"]:\n",
" df[casename] = pd.DataFrame({})\n",
" for plottype in [\n",
" \"histogram\",\n",
" \"summary_map\",\n",
" \"time_series\",\n",
" \"trend_hist\",\n",
" \"trend_map\",\n",
" ]:\n",
" print(f\"Generating csv for {plottype} plots in {casename}\")\n",
" df[casename] = pd.concat(\n",
" [\n",
" df[casename],\n",
" generate_plot_catalog(\n",
" \"./\",\n",
" image_dir_name=f\"images/{casename}/{plottype}\",\n",
" use_full_path=False,\n",
" ),\n",
" ]\n",
" )\n",
" df[casename].to_csv(\n",
" f\"images/{casename}/png_catalog.csv\",\n",
" compression=None,\n",
" index=False,\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>plot_type</th>\n",
" <th>varname</th>\n",
" <th>casename</th>\n",
" <th>apply_log10</th>\n",
" <th>time_period</th>\n",
" <th>sel_dict</th>\n",
" <th>filepath</th>\n",
" <th>date</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>histogram</td>\n",
" <td>CaCO3_FLUX_100m</td>\n",
" <td>g.e22.G1850ECO_JRA_HR.TL319_t13.004</td>\n",
" <td>False</td>\n",
" <td>0001-01-01_0001-12-31</td>\n",
" <td>{}</td>\n",
" <td>histogram/CaCO3_FLUX_100m.0001-01-01_0001-12-3...</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>histogram</td>\n",
" <td>CaCO3_FLUX_100m</td>\n",
" <td>g.e22.G1850ECO_JRA_HR.TL319_t13.004</td>\n",
" <td>True</td>\n",
" <td>0001-01-01_0001-12-31</td>\n",
" <td>{}</td>\n",
" <td>histogram/CaCO3_FLUX_100m.0001-01-01_0001-12-3...</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>histogram</td>\n",
" <td>CaCO3_FLUX_100m</td>\n",
" <td>g.e22.G1850ECO_JRA_HR.TL319_t13.004</td>\n",
" <td>False</td>\n",
" <td>0002-01-01_0002-12-31</td>\n",
" <td>{}</td>\n",
" <td>histogram/CaCO3_FLUX_100m.0002-01-01_0002-12-3...</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>histogram</td>\n",
" <td>CaCO3_FLUX_100m</td>\n",
" <td>g.e22.G1850ECO_JRA_HR.TL319_t13.004</td>\n",
" <td>True</td>\n",
" <td>0002-01-01_0002-12-31</td>\n",
" <td>{}</td>\n",
" <td>histogram/CaCO3_FLUX_100m.0002-01-01_0002-12-3...</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>histogram</td>\n",
" <td>CaCO3_FLUX_100m</td>\n",
" <td>g.e22.G1850ECO_JRA_HR.TL319_t13.004</td>\n",
" <td>False</td>\n",
" <td>0003-01-01_0003-12-31</td>\n",
" <td>{}</td>\n",
" <td>histogram/CaCO3_FLUX_100m.0003-01-01_0003-12-3...</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25</th>\n",
" <td>trend_map</td>\n",
" <td>SiO3</td>\n",
" <td>g.e22.G1850ECO_JRA_HR.TL319_t13.004</td>\n",
" <td>NaN</td>\n",
" <td>0002-01-01_0004-12-31</td>\n",
" <td>{'basins': 'Atlantic'}</td>\n",
" <td>trend_map/SiO3.0002-01-01_0004-12-31.basins--A...</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>26</th>\n",
" <td>trend_map</td>\n",
" <td>SiO3</td>\n",
" <td>g.e22.G1850ECO_JRA_HR.TL319_t13.004</td>\n",
" <td>NaN</td>\n",
" <td>0002-01-01_0004-12-31</td>\n",
" <td>{'basins': 'Global'}</td>\n",
" <td>trend_map/SiO3.0002-01-01_0004-12-31.basins--G...</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27</th>\n",
" <td>trend_map</td>\n",
" <td>SiO3</td>\n",
" <td>g.e22.G1850ECO_JRA_HR.TL319_t13.004</td>\n",
" <td>NaN</td>\n",
" <td>0002-01-01_0004-12-31</td>\n",
" <td>{'basins': 'Indian'}</td>\n",
" <td>trend_map/SiO3.0002-01-01_0004-12-31.basins--I...</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28</th>\n",
" <td>trend_map</td>\n",
" <td>SiO3</td>\n",
" <td>g.e22.G1850ECO_JRA_HR.TL319_t13.004</td>\n",
" <td>NaN</td>\n",
" <td>0002-01-01_0004-12-31</td>\n",
" <td>{'basins': 'Pacific'}</td>\n",
" <td>trend_map/SiO3.0002-01-01_0004-12-31.basins--P...</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>29</th>\n",
" <td>trend_map</td>\n",
" <td>SiO3</td>\n",
" <td>g.e22.G1850ECO_JRA_HR.TL319_t13.004</td>\n",
" <td>NaN</td>\n",
" <td>0002-01-01_0004-12-31</td>\n",
" <td>{'z_t': '35109.35'}</td>\n",
" <td>trend_map/SiO3.0002-01-01_0004-12-31.z_t--3510...</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>1244 rows × 8 columns</p>\n",
"</div>"
],
"text/plain": [
" plot_type varname casename \\\n",
"0 histogram CaCO3_FLUX_100m g.e22.G1850ECO_JRA_HR.TL319_t13.004 \n",
"1 histogram CaCO3_FLUX_100m g.e22.G1850ECO_JRA_HR.TL319_t13.004 \n",
"2 histogram CaCO3_FLUX_100m g.e22.G1850ECO_JRA_HR.TL319_t13.004 \n",
"3 histogram CaCO3_FLUX_100m g.e22.G1850ECO_JRA_HR.TL319_t13.004 \n",
"4 histogram CaCO3_FLUX_100m g.e22.G1850ECO_JRA_HR.TL319_t13.004 \n",
".. ... ... ... \n",
"25 trend_map SiO3 g.e22.G1850ECO_JRA_HR.TL319_t13.004 \n",
"26 trend_map SiO3 g.e22.G1850ECO_JRA_HR.TL319_t13.004 \n",
"27 trend_map SiO3 g.e22.G1850ECO_JRA_HR.TL319_t13.004 \n",
"28 trend_map SiO3 g.e22.G1850ECO_JRA_HR.TL319_t13.004 \n",
"29 trend_map SiO3 g.e22.G1850ECO_JRA_HR.TL319_t13.004 \n",
"\n",
" apply_log10 time_period sel_dict \\\n",
"0 False 0001-01-01_0001-12-31 {} \n",
"1 True 0001-01-01_0001-12-31 {} \n",
"2 False 0002-01-01_0002-12-31 {} \n",
"3 True 0002-01-01_0002-12-31 {} \n",
"4 False 0003-01-01_0003-12-31 {} \n",
".. ... ... ... \n",
"25 NaN 0002-01-01_0004-12-31 {'basins': 'Atlantic'} \n",
"26 NaN 0002-01-01_0004-12-31 {'basins': 'Global'} \n",
"27 NaN 0002-01-01_0004-12-31 {'basins': 'Indian'} \n",
"28 NaN 0002-01-01_0004-12-31 {'basins': 'Pacific'} \n",
"29 NaN 0002-01-01_0004-12-31 {'z_t': '35109.35'} \n",
"\n",
" filepath date \n",
"0 histogram/CaCO3_FLUX_100m.0001-01-01_0001-12-3... NaN \n",
"1 histogram/CaCO3_FLUX_100m.0001-01-01_0001-12-3... NaN \n",
"2 histogram/CaCO3_FLUX_100m.0002-01-01_0002-12-3... NaN \n",
"3 histogram/CaCO3_FLUX_100m.0002-01-01_0002-12-3... NaN \n",
"4 histogram/CaCO3_FLUX_100m.0003-01-01_0003-12-3... NaN \n",
".. ... ... \n",
"25 trend_map/SiO3.0002-01-01_0004-12-31.basins--A... NaN \n",
"26 trend_map/SiO3.0002-01-01_0004-12-31.basins--G... NaN \n",
"27 trend_map/SiO3.0002-01-01_0004-12-31.basins--I... NaN \n",
"28 trend_map/SiO3.0002-01-01_0004-12-31.basins--P... NaN \n",
"29 trend_map/SiO3.0002-01-01_0004-12-31.z_t--3510... NaN \n",
"\n",
"[1244 rows x 8 columns]"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df[\"g.e22.G1850ECO_JRA_HR.TL319_t13.004\"]"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'histogram/CaCO3_FLUX_100m.0001-01-01_0001-12-31.png'"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df[\"g.e22.G1850ECO_JRA_HR.TL319_t13.004\"][\"filepath\"].to_list()[0]"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:miniconda3-hires-marbl]",
"language": "python",
"name": "conda-env-miniconda3-hires-marbl-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.8"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Loading