Skip to content

Commit

Permalink
Squashed 'conformer/conformer/' changes from 9634074..5034a4d
Browse files Browse the repository at this point in the history 
5034a4d Merge pull request NCAR#74 from NCAR/devel
488962b Merge pull request NCAR#73 from NCAR/fix_request_bug
c0159aa Create workaround for bug in data request version 01.00.25
738f0d0 Merge pull request NCAR#72 from NCAR/devel
a5a0962 Merge pull request NCAR#71 from NCAR/more_defs
626ad46 Code needed to get more definitions working
a547f38 Merge pull request NCAR#70 from NCAR/add_functions
96de7cc Add extra functions
0631b1e Merge pull request NCAR#69 from NCAR/updates
2c01173 Additional Functionality
e008005 Bugfix for format string
027653b Formatting only.
44481bc Merge branch 'devel' of https://github.com/NCAR/PyConform.git into devel
f8f7221 Adding manual time conversion test
30d7c67 Merge pull request NCAR#68 from NCAR/updates_for_defs
517453d Updates needed to get more definitions to work and to meet the specifications.
b1e364b Adding allowance to remove chunked sumlike dims if debug enabled
5739755 Bugfix: need to check if dimension in input dataset first
d4f0cce Better error message
0d500bf Forgot loop over args
2a9d138 Fixing missing uses of new parsed types
4f1c581 Adding parsetab back and disabling debug
643c7bd Adding parse tables.
77fd6f8 Forgot to update the vardeps script to work with new parser
56d8149 Only need one OpType
aa032f0 Update to use new lex/yacc parser
4573337 Ignoring ply output files
5f7a298 Replacing pyparsing parser with lex/yacc parser
c5657db Adding lex/yacc-based parser
06f11c5 Merge pull request NCAR#67 from NCAR/zonalmean
fe124bd Added zonal mean fix division function bug Added the zonalmean function The division function had a minus sign as its symbol.  Switched it to a division sign. (Still hanging on the one aerosol equation for emibvoc)
cd2140d Merge pull request NCAR#66 from NCAR/def_updates
b908a51 Add an input_glob attribute to the variable in the json file -Will add this from a comment in the def file. -These are glob streams to search for the input
b7b7f8c Merge pull request NCAR#65 from NCAR/def_updates
62bebc3 Add in modules for land varaibles Remove an underscore in the output file name to match the request
8a75d77 BUGFIX: Need to check for dimensions in input dataset when mapping
84351e2 Exabling writing to temporary file and then renaming
662f973 bump version

git-subtree-dir: conformer/conformer
git-subtree-split: 5034a4d4576db6df84fb76732653b330a3e7bc32
  • Loading branch information
@bertinia
bertinia committed Jun 21, 2018
1 parent 2259606 commit f7b1840
Show file tree
Hide file tree
Showing 18 changed files with 2,030 additions and 806 deletions.
93 changes: 72 additions & 21 deletions scripts/iconform
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ import datetime
from dreqPy import dreq
import uuid


version = 'v'+str(datetime.datetime.now().year)+str(datetime.datetime.now().month).zfill(2)+str(datetime.datetime.now().day).zfill(2)

# Map netcdf types to python types
#data_types = {'char': 'char', 'byte': 'int8', 'short': 'int16', 'int': 'int32',
# 'float': 'float32', 'real': 'float32', 'double': 'float64',
Expand All @@ -25,10 +28,29 @@ data_types = {'char': 'char', 'byte': 'byte', 'short': 'short', 'int': 'int',
'character':'char', 'integer':'int'}

# The way the date should be formatted in the filenames
date_strings = {'yr': '{%Y-%Y}', 'mon': '{%Y%m-%Y%m}', 'monClim': '{%Y%m-%Y%m-clim}',
'day': '{%Y%m%d-%Y%m%d}', '6hr': '{%Y%m%d%H%M-%Y%m%d%H%M}',
'3hr': '{%Y%m%d%H%M-%Y%m%d%H%M}', '1hr': '{%Y%m%d%H%M-%Y%m%d%H%M}',
'subhr': '{%Y%m%d%H%M-%Y%m%d%H%M}', '1hrClimMon': '{%Y%m%d%H%M-%Y%m%d%H%M-clim}'}
#date_strings = {'yr': '{%Y-%Y}', 'mon': '{%Y%m-%Y%m}', 'monClim': '{%Y%m-%Y%m-clim}',
# 'day': '{%Y%m%d-%Y%m%d}', '6hr': '{%Y%m%d%H%M-%Y%m%d%H%M}',
# '3hr': '{%Y%m%d%H%M-%Y%m%d%H%M}', '1hr': '{%Y%m%d%H%M-%Y%m%d%H%M}',
# 'subhr': '{%Y%m%d%H%M-%Y%m%d%H%M}', '1hrClimMon': '{%Y%m%d%H%M-%Y%m%d%H%M-clim}'}
date_strings = {"1hr":'_{%Y%m%d%H%M-%Y%m%d%H%M}',
"1hrCM":'_{%Y%m%d%H%M-%Y%m%d%H%M}-clim',
"1hrPt":'_{%Y%m%d%H%M-%Y%m%d%H%M}',
"3hr":'_{%Y%m%d%H%M-%Y%m%d%H%M}',
"3hrPt":'_{%Y%m%d%H%M-%Y%m%d%H%M}',
"6hr":'_{%Y%m%d%H%M-%Y%m%d%H%M}',
"6hrPt":'_{%Y%m%d%H%M-%Y%m%d%H%M}',
"day":'_{%Y%m%d-%Y%m%d}',
"dec":'_{%Y-%Y}',
"fx":'',
"mon":'_{%Y%m-%Y%m}',
"monC":'_{%Y%m-%Y%m}-clim',
"monPt":'_{%Y%m-%Y%m}',
"subhrPt":'_{%Y%m%d%H%M-%Y%m%d%H%M}',
"yr":'_{%Y-%Y}',
"yrPt":'_{%Y-%Y}'
}



#===================================================================================================
# parseArgs
Expand Down Expand Up @@ -70,26 +92,35 @@ def parseArgs(argv = None):
def load(defs,key=None):

def_dict = {}
ig_dict = {}
if key == 'ga':
def_dict[key] = {}
ig_dict[key] = {}
for line in defs:
input_glob = None
line = line.strip()
# hanle comments
if '#' in line:
if 'TABLE' in line:
key = line.split(':')[1].strip()
def_dict[key] = {}
ig_dict[key] = {}
if 'Coords' in line:
key = 'Coords_'+line.split(':')[1].strip()
def_dict[key] = {}
ig_dict[key] = {}
if len(line.split('#')) == 2:
input_glob = line.split('#')[1]
line = line.split('#')[0].strip()
# slit definition into the two parts
split = line.split('=')
#if (len(split) == 2):
if (len(split) >= 2):
if key == 'ga' and split[1] == '':
def_dict[key][split[0].strip()] = "__FILL__"
ig_dict[key][split[0].strip()] = input_glob
else:
ig_dict[key][split[0].strip()] = input_glob
if len(split) == 2:
def_dict[key][split[0].strip()] = split[1].strip()
else:
Expand All @@ -98,12 +129,16 @@ def load(defs,key=None):
if len(line)>0 :
print 'Could not parse this line: ',line

return def_dict
if key == 'ga':
return def_dict
else:
return def_dict,ig_dict


def fill_missing_glob_attributes(attr, table, v, grids):

for a,d in attr.iteritems():
if d is not None:
if "__FILL__" in d:
if "activity_id" in a:
attr["activity_id"] = table["activity_id"]
Expand Down Expand Up @@ -140,7 +175,7 @@ def fill_missing_glob_attributes(attr, table, v, grids):
if "parent_source_id" in attr.keys():
attr["parent_source_id"] = attr["source_id"]
if "parent_time_units" in attr.keys():
attr["parent_time_units"] = "days since 0000-01-01 00:00:00"
attr["parent_time_units"] = "days since 0001-01-01 00:00:00"
else:
if "branch_time_in_child" in attr.keys():
attr["branch_time_in_child"] = "none"
Expand All @@ -155,13 +190,13 @@ def fill_missing_glob_attributes(attr, table, v, grids):

if "variant_label" in attr.keys():
pre = attr["variant_label"].split('r')[1]
attr["realization_index"] = int(pre.split('i')[0])
attr["realization_index"] = (pre.split('i')[0])
pre = pre.split('i')[1]
attr["initialization_index"] = int(pre.split('p')[0])
attr["initialization_index"] = (pre.split('p')[0])
pre = pre.split('p')[1]
attr["physics_index"] = int(pre.split('f')[0])
attr["physics_index"] = (pre.split('f')[0])
pre = int(pre.split('f')[1])
attr["forcing_index"] = pre
attr["forcing_index"] = str(pre)

if "further_info_url" in attr.keys():
if "__FILL__" in attr["further_info_url"]:
Expand Down Expand Up @@ -202,7 +237,7 @@ def fill_missing_glob_attributes(attr, table, v, grids):
#===================================================================================================
# defineVar
#===================================================================================================
def defineVar(v, varName, attr, table_info, definition, experiment, out_dir):
def defineVar(v, varName, attr, table_info, definition, ig, experiment, out_dir):

v2 = dict(v)
for key,value in v.iteritems(): # remove all attributes that do not have values
Expand Down Expand Up @@ -248,7 +283,7 @@ def defineVar(v, varName, attr, table_info, definition, experiment, out_dir):
source_id = attributes['source_id']
else:
source_id = ''
if 'grid_labels' in attributes.keys():
if 'grid_label' in attributes.keys():
grid = attributes['grid_label']
else:
grid = ''
Expand Down Expand Up @@ -286,16 +321,17 @@ def defineVar(v, varName, attr, table_info, definition, experiment, out_dir):
dst = date_strings[v["frequency"]]
else:
dst = ''
f_name = ("{0}/{1}/{2}/{3}/{4}/{5}/{6}/{7}/{8}/{9}/{10}_{11}_{12}_{13}_{14}_{15}_{16}.nc".format(
f_name = ("{0}/{1}/{2}/{3}/{4}/{5}/{6}/{7}/{8}/{9}/{10}/{11}_{12}_{13}_{14}_{15}_{16}{17}.nc".format(
out_dir, mip_era, activity_id, institution_id, source_id, experiment, ripf, mipTable,
varName, grid,
varName, grid, version,
varName, mipTable, source_id, experiment, ripf, grid, dst))

var = {}

# put together the dictionary entry for this variable
var["attributes"] = v2
var["definition"] = definition
if ig:
var["input_glob"] = ig
var["file"] = {}
var["file"]["attributes"] = attributes
var["file"]["attributes"]["variant_label"] = ripf
Expand All @@ -310,6 +346,17 @@ def defineVar(v, varName, attr, table_info, definition, experiment, out_dir):
var["datatype"] = data_types[v['type']]
else:
var["datatype"] = 'real' # This is done because some of the variables in the request have no type listed yet

#### Needed to get working with netcdf4_classic and netcdf3_classic
# if 'type' in v.keys() and v['type'] != 'None' and v['type'] != '' and v['type'] != None:
# if 'real' in data_types[v['type']]:
# var["datatype"] = "float"
# else:
# var["datatype"] = data_types[v['type']]
# else:
# var["datatype"] = 'float' # This is done because some of the variables in the request have no type listed yet


if 'requested' in v.keys():
if v['requested'] != '':
var['definition'] = v['requested']
Expand Down Expand Up @@ -339,7 +386,7 @@ def defineAxes(v, name):
v2.pop(key,None)
# Hardcode this value in for time. Not ideal, but the request has it listed as "days since ?" and this will fail.
if 'time' in name:
v2["units"] = "days since 0000-01-01 00:00:00"
v2["units"] = "days since 0001-01-01 00:00:00"

# put everything into a variable dictionary
var["attributes"] = v2
Expand Down Expand Up @@ -382,7 +429,7 @@ def getUserVars(fn):
#===================================================================================================
# create_output
#===================================================================================================
def create_output(exp_dict, definitions, attributes, output_path, args, experiment, out_dir, testoutput):
def create_output(exp_dict, definitions, input_glob, attributes, output_path, args, experiment, out_dir, testoutput):

# create the output json files

Expand Down Expand Up @@ -425,9 +472,10 @@ def create_output(exp_dict, definitions, attributes, output_path, args, experime
v_def = ""
else:
v_def = definitions[mip][v]
ig = input_glob[mip][v]
else:
v_def = ""
var_list[v] = defineVar(d, v, attributes, table_info, v_def, experiment, out_dir)
var_list[v] = defineVar(d, v, attributes, table_info, v_def, ig, experiment, out_dir)
realm = d["realm"].replace(' ','_')
ts_key = var_list[v]["file"]["attributes"]["activity_id"]+'_'+var_list[v]["attributes"]["mipTable"]+'_'+realm
if ts_key not in TableSpec.keys():
Expand All @@ -443,7 +491,10 @@ def create_output(exp_dict, definitions, attributes, output_path, args, experime
TableSpec[ts_key][dim] = defineAxes(axes[dim], dim)
if 'Coords_'+t_realm in definitions.keys():
if dim in definitions['Coords_'+t_realm].keys():
TableSpec[ts_key][dim]['definition'] = definitions['Coords_'+t_realm][dim]
if 'landUse' in dim:
TableSpec[ts_key][dim]['definition'] = [0,1,2,3]
else:
TableSpec[ts_key][dim]['definition'] = definitions['Coords_'+t_realm][dim]
else:
if 'definition' not in TableSpec[ts_key][dim].keys():
print "MISSING "+dim+" in "+'Coords_'+t_realm+" (for variable "+v+")"
Expand Down Expand Up @@ -547,7 +598,7 @@ def main(argv=None):
# Open/Read the definition file
if os.path.isfile(args.defFile):
with open(args.defFile) as y_definitions:
definitions = load(y_definitions)
definitions,input_glob = load(y_definitions)
#print 'DEFINITIONS: ',definitions
else:
print 'Definition file does not exist: ',args.defFile
Expand Down Expand Up @@ -603,7 +654,7 @@ def main(argv=None):

if len(exp_dict.keys())>0:
# Write the spec files out to disk
create_output(exp_dict, definitions, attributes, args.outputpath, args, exp, args.outdir, args.testoutput)
create_output(exp_dict, definitions, input_glob, attributes, args.outputpath, args, exp, args.outdir, args.testoutput)


#===================================================================================================
Expand Down
42 changes: 23 additions & 19 deletions scripts/vardeps
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ from argparse import ArgumentParser
from os.path import exists


#===================================================================================================
#=========================================================================
# Command-line Interface
#===================================================================================================
#=========================================================================
def cli(argv=None):
desc = """This tool will analyze a definitions text file or a JSON standardization
file and print out the variables needed for each defined output variable."""
Expand All @@ -37,26 +37,29 @@ def cli(argv=None):
return parser.parse_args(argv)


#===================================================================================================
#=========================================================================
# variable_search
#===================================================================================================
#=========================================================================
def variable_search(obj, vars=None):
if vars is None:
vars = set()
if isinstance(obj, parsing.ParsedVariable):
if isinstance(obj, parsing.VarType):
vars.add(obj.key)
elif isinstance(obj, parsing.ParsedFunction):
elif isinstance(obj, parsing.OpType):
for arg in obj.args:
vars = variable_search(arg, vars=vars)
elif isinstance(obj, parsing.FuncType):
for arg in obj.args:
vars = variable_search(arg, vars=vars)
for kwd in obj.kwds:
vars = variable_search(obj.kwds[kwd], vars=vars)
return vars


#===================================================================================================
#=========================================================================
# print_columnar
#===================================================================================================
def print_columnar(x, textwidth=100, indent=0, header=''):
#=========================================================================
def print_columnar(x, textwidth=10000000, indent=0, header=''):
hrstrp = '{} '.format(str(header).rstrip())
if len(hrstrp) > indent:
indent = len(hrstrp)
Expand All @@ -68,19 +71,19 @@ def print_columnar(x, textwidth=100, indent=0, header=''):
A = [x[i::Nr] for i in xrange(Nr)]
print '{}{}'.format(hrstrp, ' '.join('{: <{Lmax}}'.format(r, Lmax=Lmax) for r in A[0]))
for row in A[1:]:
print '{}{}'.format(' '*indent, ' '.join('{: <{Lmax}}'.format(r, Lmax=Lmax) for r in row))
#===================================================================================================
print '{}{}'.format(' ' * indent, ' '.join('{: <{Lmax}}'.format(r, Lmax=Lmax) for r in row))


#=========================================================================
# Main Script Function
#===================================================================================================
#=========================================================================
def main(argv=None):
args = cli(argv)

# Check that the file exists
if not exists(args.filename):
raise OSError('File {!r} not found'.format(args.filename))

# Read the definitions from the file
vardefs = {}
varfreqs = {}
Expand All @@ -94,7 +97,7 @@ def main(argv=None):
split = line.split('=')
if len(split) == 2:
vardefs[split[0].strip()] = split[1].strip()
elif len(line)>0 :
elif len(line) > 0:
print 'Could not parse this line: {!r}'.format(line)
else:
stddict = load(open(args.filename), object_pairs_hook=OrderedDict)
Expand All @@ -120,7 +123,8 @@ def main(argv=None):
try:
vdeps = variable_search(parsing.parse_definition(vardefs[var]))
alldeps.update(vdeps)
fheader = ' [{},{}]'.format(varfreqs[var], varrealm[var]) if var in varfreqs else ''
fheader = ' [{},{}]'.format(
varfreqs[var], varrealm[var]) if var in varfreqs else ''
vheader = ' {}{}:'.format(var, fheader)
print_columnar(sorted(vdeps), header=vheader)
except:
Expand All @@ -133,8 +137,8 @@ def main(argv=None):
print_columnar(sorted(alldeps), indent=3)


#===================================================================================================
#=========================================================================
# Command-line Operation
#===================================================================================================
#=========================================================================
if __name__ == '__main__':
main()
Loading

0 comments on commit f7b1840

Please sign in to comment.