diff --git a/Config/config_timeseries.xml b/Config/config_timeseries.xml index 42f5da8b..7751fde4 100644 --- a/Config/config_timeseries.xml +++ b/Config/config_timeseries.xml @@ -141,7 +141,7 @@ netcdf4c month_1 years - 1000 + 50 hist @@ -149,7 +149,7 @@ netcdf4c month_1 years - 100 + 50 hist @@ -157,7 +157,7 @@ netcdf4c month_1 years - 100 + 50 hist @@ -165,7 +165,7 @@ netcdf4c day_365 years - 100 + 50 hist @@ -173,7 +173,7 @@ netcdf4c day_365 years - 100 + 50 hist @@ -181,7 +181,7 @@ netcdf4c day_1 years - 100 + 50 hist @@ -189,7 +189,7 @@ netcdf4c day_1 years - 100 + 50 hist @@ -197,7 +197,7 @@ netcdf4c hour_3 years - 100 + 50 hist @@ -205,7 +205,7 @@ netcdf4c day_1 years - 100 + 50 @@ -219,16 +219,6 @@ time_bounds time_written - - HKSAT - ZLAKE - DZLAKE - BSW - SUCSAT - ZSOI - DZSOI - WATSAT - @@ -242,7 +232,7 @@ netcdf4c month_1 years - 1000 + 50 hist @@ -250,7 +240,7 @@ netcdf4c day_1 years - 100 + 50 hist @@ -258,7 +248,7 @@ netcdf4c hour_6 years - 100 + 50 hist @@ -266,7 +256,7 @@ netcdf4c hour_3 years - 100 + 50 @@ -293,7 +283,7 @@ netcdf4c month_1 years - 1000 + 50 hist @@ -301,7 +291,7 @@ netcdf4c day_1 years - 100 + 50 hist @@ -309,7 +299,7 @@ netcdf4c hour_6 years - 100 + 50 hist @@ -317,7 +307,7 @@ netcdf4c hour_3 years - 100 + 50 @@ -344,7 +334,7 @@ netcdf4c month_1 years - 1000 + 50 hist @@ -352,7 +342,7 @@ netcdf4c day_1 years - 100 + 50 @@ -372,7 +362,7 @@ netcdf4c month_1 years - 1000 + 50 hist @@ -380,7 +370,7 @@ netcdf4c day_1 years - 100 + 50 hist @@ -388,7 +378,7 @@ netcdf4c year_1 years - 100 + 50 hist @@ -396,7 +386,7 @@ netcdf4c month_1 years - 1000 + 50 hist @@ -404,7 +394,7 @@ netcdf4c year_1 years - 1000 + 50 hist @@ -412,7 +402,7 @@ netcdf4c day_1 years - 100 + 50 @@ -432,7 +422,7 @@ netcdf4c year_1 years - 1000 + 50 hist @@ -440,7 +430,7 @@ netcdf4c month_1 years - 1000 + 50 @@ -459,7 +449,7 @@ netcdf4c month_1 years - 1000 + 50 diff --git a/Externals.cfg b/Externals.cfg index 4110ad72..aa1d1e17 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -13,7 +13,7 @@ local_path = reshaper/pyReshaper required = True [PyConform] -tag = PyConform-0.2.5 +tag = PyConform-0.2.8 protocol = git repo_url = https://github.com/NCAR/PyConform.git local_path = conformer/conformer diff --git a/conform/conform/cesm_conform_generator.py b/conform/conform/cesm_conform_generator.py index 5da4b097..90c2a166 100755 --- a/conform/conform/cesm_conform_generator.py +++ b/conform/conform/cesm_conform_generator.py @@ -58,6 +58,9 @@ external_mods = ['commonfunctions.py', 'pnglfunctions.py', 'CLM_landunit_to_CMIP6_Lut.py', 'CLM_pft_to_CMIP6_vegtype.py', 'idl.py', 'dynvarmipdiags.py', 'dynvarmipfunctions.py'] +#external_mods = ['commonfunctions.py', 'CLM_landunit_to_CMIP6_Lut.py', +# 'CLM_pft_to_CMIP6_vegtype.py', 'idl.py', 'dynvarmipdiags.py', 'dynvarmipfunctions.py'] + cesmModels = {"atmos": "cam", "land": "clm2", @@ -94,7 +97,7 @@ cesm_cmip6_realms = { "cam":"atmos", "clm2":"land", - "rtm":"land", + "mosart":"land", "cism":"landIce", "pop":"ocean", "cice":"seaIce" @@ -131,8 +134,10 @@ "ice":"d2" } -failures = 0 +no_chunking = ['yeartomonth'] +no_time_chunking = ['burntFraction'] +failures = 0 #===================================================== # commandline_options - parse any command line options @@ -243,23 +248,40 @@ def fill_list(nc_files, root_dir, extra_dir, comm, rank, size): 'ocn':'384x320' } + constants = ['bheatflx','volo'] + variablelist = {} gridfile = None nc_files.append(extra_dir+"/ocn_constants.nc") + nc_files.append(extra_dir+"/glc_constants.nc") nc_files_l = comm.partition(nc_files,func=partition.EqualLength(),involved=True) for fn in nc_files_l: f = nc.Dataset(fn, "r") mt = fn.replace(root_dir,"").split("/")[-5] stri = fn model_type = mt + if len(fn.split('.'))>3: + fvn = v_dims = fn.split('.')[-3] + # Added for decadals + if 'sh' == fvn.split('_')[-1]: + fvn = fvn.replace('_sh','') + elif 'nh' == fvn.split('_')[-1]: + fvn = fvn.replace('_nh','') + else: + for c in constants: + if c in f.variables.keys(): + fvn = c if "ocn_constants" in fn: model_type = "ocn" mt = "ocn" + if "glc_constants" in fn: + model_type = "glc" + mt = "glc" if "lnd" in model_type or "rof" in model_type: model_type = 'lnd,rof' if "glc" in model_type: model_type = 'glc,lnd' - if ("time" not in f.variables.keys() or "tseries" not in fn): + if ("time" not in f.variables.keys() and "tseries" not in fn and "_constants" not in fn): variablelist["skip"] = {} else: lt = "none" @@ -270,12 +292,15 @@ def fill_list(nc_files, root_dir, extra_dir, comm, rank, size): lev_name = None time_name = None # Find which dim variables to use - v_dims = f.variables[fn.split('.')[-3]].dimensions + v_dims = f.variables[fvn].dimensions for i in grids[mt]['lat']: if i in v_dims: if 'nlat' in i or 'nj' in i: - if hasattr(f.variables[fn.split('.')[-3]], 'coordinates'): - lat_name = str(f.variables[fn.split('.')[-3]].coordinates.split()[1]) + if hasattr(f.variables[fvn], 'coordinates'): + if 'LAT' in str(f.variables[fvn].coordinates.split()[1]): + lat_name = str(f.variables[fvn].coordinates.split()[1]) + else: + lat_name = 'TLAT' else: lat_name = 'TLAT' else: @@ -284,8 +309,11 @@ def fill_list(nc_files, root_dir, extra_dir, comm, rank, size): for i in grids[mt]['lon']: if i in v_dims: if 'nlon' in i or 'ni' in i: - if hasattr(f.variables[fn.split('.')[-3]], 'coordinates'): - lon_name = str(f.variables[fn.split('.')[-3]].coordinates.split()[0]) + if hasattr(f.variables[fvn], 'coordinates'): + if 'LON' in str(f.variables[fvn].coordinates.split()[0]): + lon_name = str(f.variables[fvn].coordinates.split()[0]) + else: + lon_name = 'TLONG' else: lon_name = 'TLONG' if 'ULON' in lon_name: @@ -330,8 +358,11 @@ def fill_list(nc_files, root_dir, extra_dir, comm, rank, size): for i in grids[mt]['lat']: if i in v_dims: if 'nlat' in i or 'nj' in i: - if hasattr(f.variables[fn.split('.')[-3]], 'coordinates'): - lat_name = str(f.variables[fn.split('.')[-3]].coordinates.split()[1]) + if hasattr(f.variables[fvn], 'coordinates'): + if 'LAT' in str(f.variables[fvn].coordinates.split()[1]): + lat_name = str(f.variables[fvn].coordinates.split()[1]) + else: + lat_name = 'TLAT' else: lat_name = 'TLAT' else: @@ -340,8 +371,11 @@ def fill_list(nc_files, root_dir, extra_dir, comm, rank, size): for i in grids[mt]['lon']: if i in v_dims: if 'nlon' in i or 'ni' in i: - if hasattr(f.variables[fn.split('.')[-3]], 'coordinates'): - lon_name = str(f.variables[fn.split('.')[-3]].coordinates.split()[0]) + if hasattr(f.variables[fvn], 'coordinates'): + if 'LON' in str(f.variables[fvn].coordinates.split()[0]): + lon_name = str(f.variables[fvn].coordinates.split()[0]) + else: + lon_name = 'TLONG' else: lon_name = 'TLONG' if 'ULON' in lon_name: @@ -356,7 +390,6 @@ def fill_list(nc_files, root_dir, extra_dir, comm, rank, size): lev_name = i lv = len(f.dimensions[i]) - if model_type not in variablelist.keys(): variablelist[model_type] = {} if vn not in variablelist[model_type].keys(): @@ -368,7 +401,7 @@ def fill_list(nc_files, root_dir, extra_dir, comm, rank, size): time_period_freq = f.time_period_freq if time_period_freq not in variablelist[model_type][vn].keys(): variablelist[model_type][vn][time_period_freq] = {} - if 'ocn_constants' in stri: + if 'ocn_constants' in stri or 'glc_constants' in stri: date = "0000" else: date = stri.split('.')[-2] @@ -384,7 +417,21 @@ def fill_list(nc_files, root_dir, extra_dir, comm, rank, size): if "unknown" not in variablelist[model_type][vn].keys(): variablelist[model_type][vn]["unknown"] = {} if stri not in variablelist[model_type][vn]["unknown"]: - variablelist[model_type][vn]["unknown"]["unknown"] = {} + # Modified for decadals + #variablelist[model_type][vn]["unknown"]["unknown"] = {} + time_period_freq = fn.split("/")[-2] + if 'ocn_constants' in stri or 'glc_constants' in stri: + date = "0000" + else: + date = stri.split('.')[-2] + if model_type not in variablelist.keys(): + variablelist[model_type] = {} + if vn not in variablelist[model_type].keys(): + variablelist[model_type][vn] = {} + if time_period_freq not in variablelist[model_type][vn].keys(): + variablelist[model_type][vn][time_period_freq] = {} + if date not in variablelist[model_type][vn][time_period_freq].keys(): + variablelist[model_type][vn][time_period_freq][date] = {} variablelist[model_type][vn][time_period_freq][date]['files']=[stri,gridfile] variablelist[model_type][vn][time_period_freq][date]['lat']=lat_name variablelist[model_type][vn][time_period_freq][date]['lon']=lon_name @@ -503,6 +550,8 @@ def match_tableSpec_to_stream(ts_dir, variable_list, dout_s_root, case): var_defs[j][split[0]]["var_check"] = {} if 'landUse' in var_defs[j][split[0]]["vars"]: var_defs[j][split[0]]["vars"].remove('landUse') + elif 'levsoi' in var_defs[j][split[0]]["vars"]: + var_defs[j][split[0]]["vars"].remove('levsoi') elif 'siline' in var_defs[j][split[0]]["vars"]: var_defs[j][split[0]]["vars"].remove('siline') elif 'basin' in var_defs[j][split[0]]["vars"]: @@ -523,6 +572,8 @@ def match_tableSpec_to_stream(ts_dir, variable_list, dout_s_root, case): else: if 'land' in r and 'h3' in js_fo[var_name]["input_glob"]: f = 'year_1' + elif 'pop' in stream: + f = 'month_1' else: f = var_defs[j][split[0]]["freq"] elif 'Odec' in j: @@ -548,7 +599,8 @@ def match_tableSpec_to_stream(ts_dir, variable_list, dout_s_root, case): elif ',' in cmip6_realms[r]: l_found_all = False if 'glc' in cmip6_realms[r]: - if v in variable_list['lnd,rof'].keys(): + if 'lnd,rof' in variable_list.keys(): + if v in variable_list['lnd,rof'].keys(): l_found_all = False for freq in variable_list['lnd,rof'][v].keys(): if f in freq: @@ -556,7 +608,7 @@ def match_tableSpec_to_stream(ts_dir, variable_list, dout_s_root, case): if not l_found_all: found_all = False missing_vars.append(v) - elif v in variable_list['glc,lnd'].keys(): + elif v in variable_list['glc,lnd'].keys(): l_found_all = False for freq in variable_list['glc,lnd'][v].keys(): if f in freq: @@ -585,6 +637,8 @@ def match_tableSpec_to_stream(ts_dir, variable_list, dout_s_root, case): else: if 'land' in r and 'h3' in js_fo[var_name]["input_glob"]: f = 'year_1' + elif 'pop' in stream: + f = 'month_1' else: f = var_defs[j][split[0]]["freq"] elif 'Odec' in j: @@ -605,6 +659,7 @@ def match_tableSpec_to_stream(ts_dir, variable_list, dout_s_root, case): found_r = cmip6_realms[r] elif ',' in cmip6_realms[r]: if 'glc' in cmip6_realms[r]: + if 'lnd,rof' in variable_list.keys(): if v in variable_list['lnd,rof'].keys(): found_r = 'lnd,rof' elif v in variable_list['glc,lnd'].keys(): @@ -613,7 +668,7 @@ def match_tableSpec_to_stream(ts_dir, variable_list, dout_s_root, case): l_found_all = True #else: #found_all = False - #missing_vars.append(v) + #missing_vars.append(v) if found_r != None and found_all: for freq in variable_list[found_r][v].keys(): if f in freq: @@ -631,6 +686,9 @@ def match_tableSpec_to_stream(ts_dir, variable_list, dout_s_root, case): else: freq_n = freq filename = dout_s_root+"/"+cesm_realms[stream[0]]+"/proc/tseries/"+freq_n+"/"+case+"."+stream[0]+"."+stream[1]+"."+v+"."+date+".nc" + if not os.path.exists(filename): + if ('_constant' in variable_list[found_r][v][freq][date]['files'][0]): + filename = variable_list[found_r][v][freq][date]['files'][0] if "lnd" in cesm_realms[stream[0]] or "rof" in cesm_realms[stream[0]]: gridname = variable_list['lnd,rof'][v][freq][date]['files'][1] elif "glc" in cesm_realms[stream[0]]: @@ -638,6 +696,10 @@ def match_tableSpec_to_stream(ts_dir, variable_list, dout_s_root, case): else: gridname = variable_list[cesm_realms[stream[0]]][v][freq][date]['files'][1] fl1 = [filename,gridname] +# if 'atm' in cesm_realms[stream[0]]: +# ps_fn = dout_s_root+"/"+cesm_realms[stream[0]]+"/proc/tseries/"+freq_n+"/"+case+"."+stream[0]+"."+stream[1]+".PS."+date+".nc" +# if os.path.exists(ps_fn): +# spec_streams[j+">>"+date].append(ps_fn) if os.path.exists(fl1[0]): if (j+">>"+date) not in spec_streams.keys(): spec_streams[j+">>"+date] = [] @@ -647,28 +709,47 @@ def match_tableSpec_to_stream(ts_dir, variable_list, dout_s_root, case): # Add the grid file name if fl1[1] not in spec_streams[j+">>"+date] and fl1[1] is not None: spec_streams[j+">>"+date].append(fl1[1]) + if 'atm' in fl1[0].split('/')[-5]: + ps_fn = fl1[0].replace(v,'PS') + if os.path.exists(ps_fn): + spec_streams[j+">>"+date].append(ps_fn) # Add the correct dimension definitions for k,var in js_fo.iteritems(): - if 'ocean' in j or 'ocn' in j: + if ('ocean' in j or 'ocn' in j) and 'seaIce' not in j: if k in j.split("_")[-2]: if 'lat' in js_f[k]['dimensions'] and 'basin' not in js_f[k]['dimensions']: js_f[k]['dimensions'][js_f[k]['dimensions'].index('lat')] = 'nlat' if 'lon' in js_f[k]['dimensions']: js_f[k]['dimensions'][js_f[k]['dimensions'].index('lon')] = 'nlon' if 'seaIce' in j: - if k in j.split("_")[-2]: - if 'lat' in js_f[k]['dimensions']: - js_f[k]['dimensions'][js_f[k]['dimensions'].index('lat')] = 'nj' - if 'lon' in js_f[k]['dimensions']: - js_f[k]['dimensions'][js_f[k]['dimensions'].index('lon')] = 'ni' + go = True + if stream is not None: + if 'cam' in stream: + go = False + if go: + if 'ocean_seaIce' in j: + if k in j.split("_")[-2]: + if 'lat' in js_f[k]['dimensions'] and 'basin' not in js_f[k]['dimensions']: + js_f[k]['dimensions'][js_f[k]['dimensions'].index('lat')] = 'nlat' + if 'lon' in js_f[k]['dimensions']: + js_f[k]['dimensions'][js_f[k]['dimensions'].index('lon')] = 'nlon' + else: + if k in j.split("_")[-2]: + if 'lat' in js_f[k]['dimensions']: + js_f[k]['dimensions'][js_f[k]['dimensions'].index('lat')] = 'nj' + if 'lon' in js_f[k]['dimensions']: + js_f[k]['dimensions'][js_f[k]['dimensions'].index('lon')] = 'ni' if 'definition' in var.keys(): if 'xxlatxx' in var['definition']: if variable_list[found_r][v][freq][date]['lat'] != None: js_f[k]['definition'] = var['definition'].replace('xxlatxx',variable_list[found_r][v][freq][date]['lat']) if 'TLAT' in variable_list[found_r][v][freq][date]['lat'] or 'ULAT' in variable_list[found_r][v][freq][date]['lat']: - if 'seaIce' in j: - js_f[k]['dimensions']=['nj','ni'] + if 'seaIce' in j: + if 'ocean_seaIce' in j: + js_f[k]['dimensions']=['nlat','nlon'] + else: + js_f[k]['dimensions']=['nj','ni'] elif 'landIce' in j and 'Gre' in j: js_f[k]['dimensions']=['xgre','ygre'] elif 'landIce' in j and 'Ant' in j: @@ -689,7 +770,6 @@ def match_tableSpec_to_stream(ts_dir, variable_list, dout_s_root, case): js_f[k]['definition'] = 'lon' if 'xxxgrexx' in var['definition']: if variable_list[found_r][v][freq][date]['lon'] != None: - print 'testing info:',found_r,v,freq,date,'lon' js_f[k]['definition'] = var['definition'].replace('xxxgrexx',variable_list[found_r][v][freq][date]['lon']) if 'xxygrexx' in var['definition']: if variable_list[found_r][v][freq][date]['lat'] != None: @@ -713,6 +793,12 @@ def match_tableSpec_to_stream(ts_dir, variable_list, dout_s_root, case): js_f[k]['definition'] = "bounds(time, bdim=\"hist_interval\")" if 'time' == var['definition'] and 'Iyr' in j and 'cism' in fl1[0]: js_f[k]['definition'] = 'chunits(time * 365, units=\"days since 0001-01-01\", calendar=\"noleap\")' + if 'diff_axis1_ind0bczero_4d' in var['definition']: + js_f['lev']['definition'] = "z_t" + js_f['lev_bnds']['definition'] = "bounds(z_t, bdim=\"d2\")" + if 'rsdoabsorb' in var['definition']: + js_f['lev']['definition'] = "z_t" + js_f['lev_bnds']['definition'] = "bounds(z_t, bdim=\"d2\")" with open(j,'w') as fp: json.dump(js_f, fp, sort_keys=True, indent=4) @@ -810,6 +896,13 @@ def run_PyConform(spec, file_glob, comm): # load spec json file dsdict = json.load(open(spec_fn,'r'), object_pairs_hook=OrderedDict) + + # look through each file to see if there are any functions in defs we can't chunk over + chunking_ok = True + for v in dsdict.keys(): + for f in no_chunking: + if f in dsdict[v]['definition']: + chunking_ok = False try: # Parse the output dataset @@ -827,7 +920,12 @@ def run_PyConform(spec, file_glob, comm): if 'time'==k or 'time1'==k or 'time2'==k or 'time3'==k: time = k if time is not None: - dataflow.execute(chunks={time:48},serial=True, debug=True, scomm=comm) + if "Oclim" in spec_fn or "oclim" in spec_fn or "Oyr" in spec_fn or "oyr" in spec_fn: + dataflow.execute(chunks={'nlat':10}, serial=True, debug=True, scomm=comm) + elif not chunking_ok: + dataflow.execute(chunks={'lat':10}, serial=True, debug=True, scomm=comm) + else: + dataflow.execute(chunks={time:48},serial=True, debug=True, scomm=comm) else: dataflow.execute(serial=True, debug=True, scomm=comm) diff --git a/conform/conform/cesm_extras b/conform/conform/cesm_extras index dc6728bd..7dd9b9a0 100755 --- a/conform/conform/cesm_extras +++ b/conform/conform/cesm_extras @@ -12,7 +12,8 @@ time = { 'ocnBgchem': 'chunits(rmunits(mean(time_bound, \"d2\"))-366, units=\"days since 0001-01-01 00:00:00\", calendar=\"noleap\")', 'seaIce': 'chunits(mean(rmunits(time_bounds), \"d2\")-1, units=time)', 'land': 'chunits(mean(time_bounds, \"hist_interval\")-1, units=time)', - 'landIce': 'chunits(time * 365, units=\"days since 0001-01-01\", calendar=\"noleap\")' + 'landIce': 'chunits(time * 365, units=\"days since 0001-01-01\", calendar=\"noleap\")', + 'rof': 'chunits(mean(time_bounds, \"hist_interval\")-1, units=time)' # 'landIce': 'yeartomonth_time(chunits(time * 365, units=\"days since 0001-01-01\", calendar=\"noleap\"))', } bounds = { @@ -24,6 +25,7 @@ bounds = { 'seaIce': 'time_bounds', 'land': 'chunits(time_bounds, units=time)', 'landIce': 'bounds(xxtimebndsxx, bdim=\"hist_interval\")', + 'rof': 'chunits(time_bounds, units=time)' } comps = {'atmos': 'atm', @@ -34,12 +36,13 @@ comps = {'atmos': 'atm', 'seaIce': 'ice', 'land': 'lnd', 'landIce': 'glc', + 'rof':'rof' } cesm_realms = { "cam":"atmos", "clm2":"land", - "rtm":"rof", + "mosart":"rof", "cism":"landIce", "pop":"ocean", "cice":"seaIce" @@ -60,6 +63,9 @@ def cli(argv=None): parser.add_argument('-e', '--extra', default='extra_vars.json', type=str, help='Filename of json file that contains the extra definitions to use.') + parser.add_argument('-c', '--case', default='None', type=str, + help='CeSM case name') + parser.add_argument('ifiles', metavar='INFILE', nargs='*', type=str, help='String that points to json files generated by pyconform. Can contain wildcards.') return parser.parse_args(argv) @@ -77,6 +83,9 @@ def main(argv=None): # Get the 'extra' json filename and load it efile_base = args.extra + # Get the casename + cn = args.case + #ifiles = glob.glob(ifiles_dir+"/*.json") # Go through each of the jsons created by iconform that the user wants to modify @@ -108,6 +117,8 @@ def main(argv=None): cvar = fn.split("_")[-2] cr = fn.split("_")[-3] for var,d1 in o_dict_copy.iteritems(): + if "file" in d1.keys(): + o_dict[var]["file"]["attributes"]["cesm_casename"] = cn if "definition" in d1.keys(): if 'basin' in o_dict_copy[var]["dimensions"]: o_dict[var]["dimensions"] = ['latitude' if x=='xxlatxx' else x for x in o_dict_copy[var]["dimensions"]] @@ -117,16 +128,72 @@ def main(argv=None): elif 'yeartomonth_data' in d1["definition"]: if 'input_glob' in d1.keys(): if 'clm2' in d1['input_glob'].split(".")[0]: - o_dict["time"]["definition"] = "yeartomonth_time(chunits(time, units=\"days since 0001-01-01\", calendar=\"noleap\"))" + o_dict["time"]["definition"] = "mean(bounds(yeartomonth_time(time), bdim=\"hist_interval\"), \"hist_interval\")" if "time_bnds" in o_dict.keys(): - o_dict["time_bnds"]["definition"] = 'bounds(yeartomonth_time(chunits(time, units=\"days since 0001-01-01\", calendar=\"noleap\")), bdim=\"hist_interval\")' + o_dict["time_bnds"]["definition"] = 'bounds(yeartomonth_time(time), bdim=\"hist_interval\")' else: o_dict["time_bnds"] = {} - o_dict["time_bnds"]["definition"] = 'bounds(yeartomonth_time(chunits(time, units=\"days since 0001-01-01\", calendar=\"noleap\")), bdim=\"hist_interval\")' + o_dict["time_bnds"]["definition"] = 'bounds(yeartomonth_time(time), bdim=\"hist_interval\")' else: o_dict["time"]["definition"] = "yeartomonth_time(chunits(time * 365, units=\"days since 0001-01-01\", calendar=\"noleap\"))" if "time_bnds" in o_dict.keys(): - o_dict["time_bnds"]["definition"] = 'bounds(yeartomonth_time(chunits(time * 365, units=\"days since 0001-01-01\", calendar=\"noleap\")), bdim=\"hist_interval\")' + o_dict["time_bnds"]["definition"] = 'bounds(yeartomonth_time(time), bdim=\"hist_interval\")' + if 'clm2' in d1['input_glob'].split(".")[0] or 'cism' in d1['input_glob'].split(".")[0]: + o_dict[var]["file"]["autoparse_time_variable"] = "_time" + o_dict["_time"]={ + "attributes": { + "axis": "T", + "direction": "increasing", + "standard_name": "time", + "title": "time", + "type": "double", + "units": "days since 0001-01-01 00:00:00" + }, + "definition": "mean(bounds(yeartomonth_time(time), bdim=\"hist_interval\"), \"hist_interval\")", + "dimensions": [ + "time" + ], + "datatype": "double"} + else: + o_dict["time"]["definition"] = "mean(bounds(yeartomonth_time(time), bdim=\"hist_interval\"), \"hist_interval\")" + if "time_bnds" in o_dict.keys(): + o_dict["time_bnds"]["definition"] = 'bounds(yeartomonth_time(time), bdim=\"hist_interval\")' + else: + o_dict["time_bnds"] = {} + o_dict["time_bnds"]["definition"] = 'bounds(yeartomonth_time(time), bdim=\"hist_interval\")' + elif 'oclim' in d1["definition"] and 'oclim_time' not in d1["definition"]: + o_dict[var]["file"]["autoparse_time_variable"] = "_time" + o_dict["_time"]={ + "attributes": { + "axis": "T", + "direction": "increasing", + "standard_name": "time", + "title": "time", + "type": "double", + "units": "days since 0001-01-01 00:00:00" + }, + "definition": "chunits(rmunits(mean(time_bound, \"d2\"))-366, units=\"days since 0001-01-01 00:00:00\", calendar=\"noleap\")", + "dimensions": [ + "time" + ], + "datatype": "double"} + elif 'monthtoyear_noleap' in d1["definition"] and 'monthtoyear_noleap_time'not in d1["definition"]: + o_dict[var]["file"]["autoparse_time_variable"] = "_time" + o_dict["_time"]={ + "attributes": { + "axis": "T", + "direction": "increasing", + "standard_name": "time", + "title": "time", + "type": "double", + "units": "days since 0001-01-01 00:00:00" + }, + "definition": "chunits(rmunits(mean(time_bound, \"d2\"))-365, units=\"days since 0001-01-01 00:00:00\", calendar=\"noleap\")", + #"definition": "monthtoyear_noleap_time(chunits(rmunits(time_bound), units=\"days since 0001-01-01 00:00:00\", calendar=\"noleap\"))", + "dimensions": [ + "time" + ], + "datatype": "double"} elif 'landUse' in var: o_dict[var]["definition"] = [0,1,2,3] o_dict[var]["attributes"]["requested"] = "primary_and_secondary_land=0, pastures=1, crops=2, urban=3" @@ -163,14 +230,22 @@ def main(argv=None): if 'clm2' in d1['input_glob'].split(".")[0]: o_dict.pop("ygre", None) o_dict.pop("xgre", None) - o_dict['latitude'] = e_dict['latitude'] - o_dict['longitude'] = e_dict['longitude'] - o_dict['latitude_bnds'] = e_dict['latitude_bnds'] - o_dict['longitude_bnds'] = e_dict['longitude_bnds'] + o_dict['lat'] = e_dict['latitude'] + o_dict['lon'] = e_dict['longitude'] + o_dict['lat_bnds'] = e_dict['latitude_bnds'] + o_dict['lon_bnds'] = e_dict['longitude_bnds'] for var,d1 in o_dict_copy.iteritems(): if cvar in var: - o_dict[var]["dimensions"] = [dim.replace("ygre", "latitude") for dim in o_dict[var]["dimensions"]] - o_dict[var]["dimensions"] = [dim.replace("xgre", "longitude") for dim in o_dict[var]["dimensions"]] + o_dict[var]["dimensions"] = [dim.replace("ygre", "lat") for dim in o_dict[var]["dimensions"]] + o_dict[var]["dimensions"] = [dim.replace("xgre", "lon") for dim in o_dict[var]["dimensions"]] + o_dict[var]["attributes"]["coordinates"] = o_dict[var]["attributes"]["coordinates"].replace("ygre", "lat") + o_dict[var]["attributes"]["coordinates"] = o_dict[var]["attributes"]["coordinates"].replace("xgre", "lon") + o_dict['lat']["attributes"]["bounds"]="lat_bnds" + o_dict['lon']["attributes"]["bounds"]="lon_bnds" + o_dict['lat']["dimensions"] = ["lat"] + o_dict['lon']["dimensions"] = ["lon"] + o_dict['lat_bnds']["dimensions"] = ["lat", "hist_interval"] + o_dict['lon_bnds']["dimensions"] = ["lon", "hist_interval"] elif 'cism' in d1['input_glob'].split(".")[0]: o_dict.pop("latitude", None) o_dict.pop("longitude", None) @@ -182,14 +257,22 @@ def main(argv=None): if 'clm2' in d1['input_glob'].split(".")[0]: o_dict.pop("yant", None) o_dict.pop("xant", None) - o_dict['latitude'] = e_dict['latitude'] - o_dict['longitude'] = e_dict['longitude'] - o_dict['latitude_bnds'] = e_dict['latitude_bnds'] - o_dict['longitude_bnds'] = e_dict['longitude_bnds'] + o_dict['lat'] = e_dict['latitude'] + o_dict['lon'] = e_dict['longitude'] + o_dict['lat_bnds'] = e_dict['latitude_bnds'] + o_dict['lon_bnds'] = e_dict['longitude_bnds'] for var,d1 in o_dict_copy.iteritems(): if cvar in var: - o_dict[var]["dimensions"] = [dim.replace("yant", "latitude") for dim in o_dict[var]["dimensions"]] - o_dict[var]["dimensions"] = [dim.replace("xant", "longitude") for dim in o_dict[var]["dimensions"]] + o_dict[var]["dimensions"] = [dim.replace("yant", "lat") for dim in o_dict[var]["dimensions"]] + o_dict[var]["dimensions"] = [dim.replace("xant", "lon") for dim in o_dict[var]["dimensions"]] + o_dict[var]["attributes"]["coordinates"] = o_dict[var]["attributes"]["coordinates"].replace("yant", "lat") + o_dict[var]["attributes"]["coordinates"] = o_dict[var]["attributes"]["coordinates"].replace("xant", "lon") + o_dict['lat']["attributes"]["bounds"]="lat_bnds" + o_dict['lon']["attributes"]["bounds"]="lon_bnds" + o_dict['lat']["dimensions"] = ["lat"] + o_dict['lon']["dimensions"] = ["lon"] + o_dict['lat_bnds']["dimensions"] = ["lat", "hist_interval"] + o_dict['lon_bnds']["dimensions"] = ["lon", "hist_interval"] elif 'cism' in d1['input_glob'].split(".")[0]: o_dict.pop("latitude", None) o_dict.pop("longitude", None) @@ -214,14 +297,15 @@ def main(argv=None): if 'iceband' in var: o_dict['iceband_bnds'] = e_dict['iceband_bnds'] - o_dict['iceband']["dimensions"] = ["nc"] + o_dict['iceband']["dimensions"] = ["iceband"] + o_dict["nc"] = e_dict["nc"] if cvar in var and ('ocean' in cr or 'ocn' in cr): if "nlat" in e_dict.keys(): o_dict["nlat"] = e_dict["nlat"] if "nlon" in e_dict.keys(): o_dict["nlon"] = e_dict["nlon"] - if cvar in var and ('seaIce' in cr): + if cvar in var and ('seaIce' in cr or 'ocean' in cr or 'ocn' in cr): if "nj" in e_dict.keys(): o_dict["nj"] = e_dict["nj"] if "ni" in e_dict.keys(): @@ -246,7 +330,21 @@ def main(argv=None): if 'yeartomonth_time' not in o_dict[d1["attributes"]["bounds"]]["definition"]: o_dict[d1["attributes"]["bounds"]]["definition"] = bounds[realm] if "alevhalf" in d1["dimensions"] and "alevhalf" != var: - o_dict[var]["metavars"] = ["ps", "p0", "a", "b"] + o_dict[var]["file"]["metavars"] = ["ps", "p0", "a", "b"] + if "p0" not in o_dict.keys(): + o_dict["p0"] = e_dict["p0"] + if "ps" not in o_dict.keys(): + o_dict["ps"] = e_dict["ps"] + if "a" not in o_dict.keys(): + o_dict["a"] = e_dict["a"] + if "a_bnds" not in o_dict.keys(): + o_dict["a_bnds"] = e_dict["a_bnds"] + if "b" not in o_dict.keys(): + o_dict["b"] = e_dict["b"] + if "b_bnds" not in o_dict.keys(): + o_dict["b_bnds"] = e_dict["b_bnds"] + if "alevel" in d1["dimensions"] and "alevel" != var: + o_dict[var]["file"]["metavars"] = ["ps", "p0", "a", "b"] if "p0" not in o_dict.keys(): o_dict["p0"] = e_dict["p0"] if "a" not in o_dict.keys(): @@ -257,26 +355,32 @@ def main(argv=None): o_dict["b"] = e_dict["b"] if "b_bnds" not in o_dict.keys(): o_dict["b_bnds"] = e_dict["b_bnds"] + if "ps" not in o_dict.keys(): + o_dict["ps"] = e_dict["ps"] if "alevel" in d1["dimensions"] and "alevel" != var: - o_dict[var]["metavars"] = ["ps", "p0"] + o_dict[var]["file"]["metavars"] = ["ps", "p0"] if "p0" not in o_dict.keys(): o_dict["p0"] = e_dict["p0"] if var == 'time' or var == 'time1' or var == 'time2' or var == 'time3': if 'mon' in freq: if 'input_glob' in d1.keys(): if 'clm2' in d1['input_glob'].split(".")[0]: - o_dict[var]["definition"] = time['land'] + if 'yeartomonth_time' not in o_dict[var]["definition"]: + o_dict[var]["definition"] = time['land'] elif 'cism' in d1['input_glob'].split(".")[0]: o_dict[var]["definition"] = time['landIce'] - else: - o_dict[var]["definition"] = time[realm] else: if "definition" not in o_dict[var].keys(): - o_dict[var]["definition"] = time[realm] - elif 'yeartomonth_time' not in o_dict[var]["definition"]: - o_dict[var]["definition"] = time[realm] + if 'yeartomonth_time' not in o_dict[var]["definition"] and "oclim_time" not in o_dict[var]["definition"] and "monthtoyear_noleap_time" not in o_dict[var]["definition"]: + o_dict[var]["definition"] = time[realm] + elif 'yeartomonth_time' not in o_dict[var]["definition"] and "oclim_time" not in o_dict[var]["definition"] and "monthtoyear_noleap_time" not in o_dict[var]["definition"]: + o_dict[var]["definition"] = time[realm] else: - o_dict[var]["definition"] = "time" + if "definition" not in o_dict[var].keys(): + o_dict[var]["definition"] = "time" + else: + if "oclim_time" not in o_dict[var]["definition"] and "monthtoyear_noleap_time" not in o_dict[var]["definition"]: + o_dict[var]["definition"] = "time" if var == 'alevel' or var == 'alevhalf': o_dict[var]["attributes"]["units"] = "hPa" if var == 'olevel' or var == 'oline': @@ -294,13 +398,15 @@ def main(argv=None): if '366' not in o_dict["time"]["definition"]: o_dict["time"]["definition"] = "chunits(rmunits(time)-365, units=\"days since 0001-01-01 00:00:00\", calendar=\"noleap\")" + # Go through the json file again and correct coord names to_change = {} # find anything with altLabel for v in o_dict.keys(): - if 'altLabel' in o_dict[v]["attributes"]: - to_change[v] = o_dict[v]["attributes"]["altLabel"] - o_dict[v]["attributes"].pop("altLabel") + if "attributes" in o_dict[v].keys(): + if 'altLabel' in o_dict[v]["attributes"]: + to_change[v] = o_dict[v]["attributes"]["altLabel"] + o_dict[v]["attributes"].pop("altLabel") # find a matching bound, if present for o in to_change.keys(): if o+"_bnds" in o_dict.keys(): @@ -322,7 +428,14 @@ def main(argv=None): if o in o_dict[v]["attributes"]["coordinates"]: n = to_change[o] o_dict[v]["attributes"]["coordinates"] = o_dict[v]["attributes"]["coordinates"].replace(o,n) - + for v in o_dict.keys(): + if "definition" in o_dict[v].keys(): + if 'oclim' in o_dict[v]["definition"]: + o_dict["time"]["definition"]="oclim_time(chunits(oclim_timebnds(chunits(time, units=\"days since 0001-01-01\", calendar=\"noleap\"), bdim=\"d2\"), units=\"days since 0001-01-01\", calendar=\"noleap\"))" + o_dict["time_bnds"]["definition"]="oclim_timebnds(chunits(time, units=\"days since 0001-01-01\", calendar=\"noleap\"), bdim=\"d2\")" + elif 'monthtoyear_noleap' in o_dict[v]["definition"]: + o_dict["time"]["definition"]="monthtoyear_noleap_time(chunits(rmunits(time_bound), units=\"days since 0001-01-01 00:00:00\", calendar=\"noleap\"))" + o_dict["time_bnds"]["definition"]="monthtoyear_noleap_timebnds(chunits(rmunits(time_bound), units=\"days since 0001-01-01 00:00:00\", calendar=\"noleap\"))" # Rewrite the file with open(fn, 'w') as outfile: json.dump(o_dict, outfile, sort_keys=True, indent=4)