From f7b184030824cc3e88bc20f0affac591ef27ffd6 Mon Sep 17 00:00:00 2001 From: Alice Bertini Date: Wed, 20 Jun 2018 20:41:42 -0600 Subject: [PATCH] Squashed 'conformer/conformer/' changes from 9634074..5034a4d 5034a4d Merge pull request #74 from NCAR/devel 488962b Merge pull request #73 from NCAR/fix_request_bug c0159aa Create workaround for bug in data request version 01.00.25 738f0d0 Merge pull request #72 from NCAR/devel a5a0962 Merge pull request #71 from NCAR/more_defs 626ad46 Code needed to get more definitions working a547f38 Merge pull request #70 from NCAR/add_functions 96de7cc Add extra functions 0631b1e Merge pull request #69 from NCAR/updates 2c01173 Additional Functionality e008005 Bugfix for format string 027653b Formatting only. 44481bc Merge branch 'devel' of https://github.com/NCAR/PyConform.git into devel f8f7221 Adding manual time conversion test 30d7c67 Merge pull request #68 from NCAR/updates_for_defs 517453d Updates needed to get more definitions to work and to meet the specifications. b1e364b Adding allowance to remove chunked sumlike dims if debug enabled 5739755 Bugfix: need to check if dimension in input dataset first d4f0cce Better error message 0d500bf Forgot loop over args 2a9d138 Fixing missing uses of new parsed types 4f1c581 Adding parsetab back and disabling debug 643c7bd Adding parse tables. 77fd6f8 Forgot to update the vardeps script to work with new parser 56d8149 Only need one OpType aa032f0 Update to use new lex/yacc parser 4573337 Ignoring ply output files 5f7a298 Replacing pyparsing parser with lex/yacc parser c5657db Adding lex/yacc-based parser 06f11c5 Merge pull request #67 from NCAR/zonalmean fe124bd Added zonal mean fix division function bug Added the zonalmean function The division function had a minus sign as its symbol. Switched it to a division sign. (Still hanging on the one aerosol equation for emibvoc) cd2140d Merge pull request #66 from NCAR/def_updates b908a51 Add an input_glob attribute to the variable in the json file -Will add this from a comment in the def file. -These are glob streams to search for the input b7b7f8c Merge pull request #65 from NCAR/def_updates 62bebc3 Add in modules for land varaibles Remove an underscore in the output file name to match the request 8a75d77 BUGFIX: Need to check for dimensions in input dataset when mapping 84351e2 Exabling writing to temporary file and then renaming 662f973 bump version git-subtree-dir: conformer/conformer git-subtree-split: 5034a4d4576db6df84fb76732653b330a3e7bc32 --- scripts/iconform | 93 ++- scripts/vardeps | 42 +- scripts/xconform | 52 +- setup.py | 2 +- source/pyconform/.gitignore | 1 + source/pyconform/dataflow.py | 139 +++-- source/pyconform/flownodes.py | 327 +++++----- source/pyconform/functions.py | 313 +++++++--- source/pyconform/miptableparser.py | 26 +- .../modules/CLM_landunit_to_CMIP6_Lut.py | 173 ++++++ .../modules/CLM_pft_to_CMIP6_vegtype.py | 256 ++++++++ source/pyconform/modules/commonfunctions.py | 265 +++++++- source/pyconform/modules/pnglfunctions.py | 9 +- source/pyconform/parsetab.py | 67 ++ source/pyconform/parsing.py | 434 +++++++------ source/pyconform/version.py | 2 +- source/test/dataflowTests.py | 65 +- source/test/parsingTests.py | 570 ++++++++++-------- 18 files changed, 2030 insertions(+), 806 deletions(-) create mode 100644 source/pyconform/.gitignore create mode 100644 source/pyconform/modules/CLM_landunit_to_CMIP6_Lut.py create mode 100644 source/pyconform/modules/CLM_pft_to_CMIP6_vegtype.py create mode 100644 source/pyconform/parsetab.py diff --git a/scripts/iconform b/scripts/iconform index dd4bae54..5fdc7cfe 100755 --- a/scripts/iconform +++ b/scripts/iconform @@ -16,6 +16,9 @@ import datetime from dreqPy import dreq import uuid + +version = 'v'+str(datetime.datetime.now().year)+str(datetime.datetime.now().month).zfill(2)+str(datetime.datetime.now().day).zfill(2) + # Map netcdf types to python types #data_types = {'char': 'char', 'byte': 'int8', 'short': 'int16', 'int': 'int32', # 'float': 'float32', 'real': 'float32', 'double': 'float64', @@ -25,10 +28,29 @@ data_types = {'char': 'char', 'byte': 'byte', 'short': 'short', 'int': 'int', 'character':'char', 'integer':'int'} # The way the date should be formatted in the filenames -date_strings = {'yr': '{%Y-%Y}', 'mon': '{%Y%m-%Y%m}', 'monClim': '{%Y%m-%Y%m-clim}', - 'day': '{%Y%m%d-%Y%m%d}', '6hr': '{%Y%m%d%H%M-%Y%m%d%H%M}', - '3hr': '{%Y%m%d%H%M-%Y%m%d%H%M}', '1hr': '{%Y%m%d%H%M-%Y%m%d%H%M}', - 'subhr': '{%Y%m%d%H%M-%Y%m%d%H%M}', '1hrClimMon': '{%Y%m%d%H%M-%Y%m%d%H%M-clim}'} +#date_strings = {'yr': '{%Y-%Y}', 'mon': '{%Y%m-%Y%m}', 'monClim': '{%Y%m-%Y%m-clim}', +# 'day': '{%Y%m%d-%Y%m%d}', '6hr': '{%Y%m%d%H%M-%Y%m%d%H%M}', +# '3hr': '{%Y%m%d%H%M-%Y%m%d%H%M}', '1hr': '{%Y%m%d%H%M-%Y%m%d%H%M}', +# 'subhr': '{%Y%m%d%H%M-%Y%m%d%H%M}', '1hrClimMon': '{%Y%m%d%H%M-%Y%m%d%H%M-clim}'} +date_strings = {"1hr":'_{%Y%m%d%H%M-%Y%m%d%H%M}', + "1hrCM":'_{%Y%m%d%H%M-%Y%m%d%H%M}-clim', + "1hrPt":'_{%Y%m%d%H%M-%Y%m%d%H%M}', + "3hr":'_{%Y%m%d%H%M-%Y%m%d%H%M}', + "3hrPt":'_{%Y%m%d%H%M-%Y%m%d%H%M}', + "6hr":'_{%Y%m%d%H%M-%Y%m%d%H%M}', + "6hrPt":'_{%Y%m%d%H%M-%Y%m%d%H%M}', + "day":'_{%Y%m%d-%Y%m%d}', + "dec":'_{%Y-%Y}', + "fx":'', + "mon":'_{%Y%m-%Y%m}', + "monC":'_{%Y%m-%Y%m}-clim', + "monPt":'_{%Y%m-%Y%m}', + "subhrPt":'_{%Y%m%d%H%M-%Y%m%d%H%M}', + "yr":'_{%Y-%Y}', + "yrPt":'_{%Y-%Y}' +} + + #=================================================================================================== # parseArgs @@ -70,18 +92,25 @@ def parseArgs(argv = None): def load(defs,key=None): def_dict = {} + ig_dict = {} if key == 'ga': def_dict[key] = {} + ig_dict[key] = {} for line in defs: + input_glob = None line = line.strip() # hanle comments if '#' in line: if 'TABLE' in line: key = line.split(':')[1].strip() def_dict[key] = {} + ig_dict[key] = {} if 'Coords' in line: key = 'Coords_'+line.split(':')[1].strip() def_dict[key] = {} + ig_dict[key] = {} + if len(line.split('#')) == 2: + input_glob = line.split('#')[1] line = line.split('#')[0].strip() # slit definition into the two parts split = line.split('=') @@ -89,7 +118,9 @@ def load(defs,key=None): if (len(split) >= 2): if key == 'ga' and split[1] == '': def_dict[key][split[0].strip()] = "__FILL__" + ig_dict[key][split[0].strip()] = input_glob else: + ig_dict[key][split[0].strip()] = input_glob if len(split) == 2: def_dict[key][split[0].strip()] = split[1].strip() else: @@ -98,12 +129,16 @@ def load(defs,key=None): if len(line)>0 : print 'Could not parse this line: ',line - return def_dict + if key == 'ga': + return def_dict + else: + return def_dict,ig_dict def fill_missing_glob_attributes(attr, table, v, grids): for a,d in attr.iteritems(): + if d is not None: if "__FILL__" in d: if "activity_id" in a: attr["activity_id"] = table["activity_id"] @@ -140,7 +175,7 @@ def fill_missing_glob_attributes(attr, table, v, grids): if "parent_source_id" in attr.keys(): attr["parent_source_id"] = attr["source_id"] if "parent_time_units" in attr.keys(): - attr["parent_time_units"] = "days since 0000-01-01 00:00:00" + attr["parent_time_units"] = "days since 0001-01-01 00:00:00" else: if "branch_time_in_child" in attr.keys(): attr["branch_time_in_child"] = "none" @@ -155,13 +190,13 @@ def fill_missing_glob_attributes(attr, table, v, grids): if "variant_label" in attr.keys(): pre = attr["variant_label"].split('r')[1] - attr["realization_index"] = int(pre.split('i')[0]) + attr["realization_index"] = (pre.split('i')[0]) pre = pre.split('i')[1] - attr["initialization_index"] = int(pre.split('p')[0]) + attr["initialization_index"] = (pre.split('p')[0]) pre = pre.split('p')[1] - attr["physics_index"] = int(pre.split('f')[0]) + attr["physics_index"] = (pre.split('f')[0]) pre = int(pre.split('f')[1]) - attr["forcing_index"] = pre + attr["forcing_index"] = str(pre) if "further_info_url" in attr.keys(): if "__FILL__" in attr["further_info_url"]: @@ -202,7 +237,7 @@ def fill_missing_glob_attributes(attr, table, v, grids): #=================================================================================================== # defineVar #=================================================================================================== -def defineVar(v, varName, attr, table_info, definition, experiment, out_dir): +def defineVar(v, varName, attr, table_info, definition, ig, experiment, out_dir): v2 = dict(v) for key,value in v.iteritems(): # remove all attributes that do not have values @@ -248,7 +283,7 @@ def defineVar(v, varName, attr, table_info, definition, experiment, out_dir): source_id = attributes['source_id'] else: source_id = '' - if 'grid_labels' in attributes.keys(): + if 'grid_label' in attributes.keys(): grid = attributes['grid_label'] else: grid = '' @@ -286,16 +321,17 @@ def defineVar(v, varName, attr, table_info, definition, experiment, out_dir): dst = date_strings[v["frequency"]] else: dst = '' - f_name = ("{0}/{1}/{2}/{3}/{4}/{5}/{6}/{7}/{8}/{9}/{10}_{11}_{12}_{13}_{14}_{15}_{16}.nc".format( + f_name = ("{0}/{1}/{2}/{3}/{4}/{5}/{6}/{7}/{8}/{9}/{10}/{11}_{12}_{13}_{14}_{15}_{16}{17}.nc".format( out_dir, mip_era, activity_id, institution_id, source_id, experiment, ripf, mipTable, - varName, grid, + varName, grid, version, varName, mipTable, source_id, experiment, ripf, grid, dst)) - var = {} # put together the dictionary entry for this variable var["attributes"] = v2 var["definition"] = definition + if ig: + var["input_glob"] = ig var["file"] = {} var["file"]["attributes"] = attributes var["file"]["attributes"]["variant_label"] = ripf @@ -310,6 +346,17 @@ def defineVar(v, varName, attr, table_info, definition, experiment, out_dir): var["datatype"] = data_types[v['type']] else: var["datatype"] = 'real' # This is done because some of the variables in the request have no type listed yet + + #### Needed to get working with netcdf4_classic and netcdf3_classic +# if 'type' in v.keys() and v['type'] != 'None' and v['type'] != '' and v['type'] != None: +# if 'real' in data_types[v['type']]: +# var["datatype"] = "float" +# else: +# var["datatype"] = data_types[v['type']] +# else: +# var["datatype"] = 'float' # This is done because some of the variables in the request have no type listed yet + + if 'requested' in v.keys(): if v['requested'] != '': var['definition'] = v['requested'] @@ -339,7 +386,7 @@ def defineAxes(v, name): v2.pop(key,None) # Hardcode this value in for time. Not ideal, but the request has it listed as "days since ?" and this will fail. if 'time' in name: - v2["units"] = "days since 0000-01-01 00:00:00" + v2["units"] = "days since 0001-01-01 00:00:00" # put everything into a variable dictionary var["attributes"] = v2 @@ -382,7 +429,7 @@ def getUserVars(fn): #=================================================================================================== # create_output #=================================================================================================== -def create_output(exp_dict, definitions, attributes, output_path, args, experiment, out_dir, testoutput): +def create_output(exp_dict, definitions, input_glob, attributes, output_path, args, experiment, out_dir, testoutput): # create the output json files @@ -425,9 +472,10 @@ def create_output(exp_dict, definitions, attributes, output_path, args, experime v_def = "" else: v_def = definitions[mip][v] + ig = input_glob[mip][v] else: v_def = "" - var_list[v] = defineVar(d, v, attributes, table_info, v_def, experiment, out_dir) + var_list[v] = defineVar(d, v, attributes, table_info, v_def, ig, experiment, out_dir) realm = d["realm"].replace(' ','_') ts_key = var_list[v]["file"]["attributes"]["activity_id"]+'_'+var_list[v]["attributes"]["mipTable"]+'_'+realm if ts_key not in TableSpec.keys(): @@ -443,7 +491,10 @@ def create_output(exp_dict, definitions, attributes, output_path, args, experime TableSpec[ts_key][dim] = defineAxes(axes[dim], dim) if 'Coords_'+t_realm in definitions.keys(): if dim in definitions['Coords_'+t_realm].keys(): - TableSpec[ts_key][dim]['definition'] = definitions['Coords_'+t_realm][dim] + if 'landUse' in dim: + TableSpec[ts_key][dim]['definition'] = [0,1,2,3] + else: + TableSpec[ts_key][dim]['definition'] = definitions['Coords_'+t_realm][dim] else: if 'definition' not in TableSpec[ts_key][dim].keys(): print "MISSING "+dim+" in "+'Coords_'+t_realm+" (for variable "+v+")" @@ -547,7 +598,7 @@ def main(argv=None): # Open/Read the definition file if os.path.isfile(args.defFile): with open(args.defFile) as y_definitions: - definitions = load(y_definitions) + definitions,input_glob = load(y_definitions) #print 'DEFINITIONS: ',definitions else: print 'Definition file does not exist: ',args.defFile @@ -603,7 +654,7 @@ def main(argv=None): if len(exp_dict.keys())>0: # Write the spec files out to disk - create_output(exp_dict, definitions, attributes, args.outputpath, args, exp, args.outdir, args.testoutput) + create_output(exp_dict, definitions, input_glob, attributes, args.outputpath, args, exp, args.outdir, args.testoutput) #=================================================================================================== diff --git a/scripts/vardeps b/scripts/vardeps index 91259a77..ff4e634d 100755 --- a/scripts/vardeps +++ b/scripts/vardeps @@ -15,9 +15,9 @@ from argparse import ArgumentParser from os.path import exists -#=================================================================================================== +#========================================================================= # Command-line Interface -#=================================================================================================== +#========================================================================= def cli(argv=None): desc = """This tool will analyze a definitions text file or a JSON standardization file and print out the variables needed for each defined output variable.""" @@ -37,15 +37,18 @@ def cli(argv=None): return parser.parse_args(argv) -#=================================================================================================== +#========================================================================= # variable_search -#=================================================================================================== +#========================================================================= def variable_search(obj, vars=None): if vars is None: vars = set() - if isinstance(obj, parsing.ParsedVariable): + if isinstance(obj, parsing.VarType): vars.add(obj.key) - elif isinstance(obj, parsing.ParsedFunction): + elif isinstance(obj, parsing.OpType): + for arg in obj.args: + vars = variable_search(arg, vars=vars) + elif isinstance(obj, parsing.FuncType): for arg in obj.args: vars = variable_search(arg, vars=vars) for kwd in obj.kwds: @@ -53,10 +56,10 @@ def variable_search(obj, vars=None): return vars -#=================================================================================================== +#========================================================================= # print_columnar -#=================================================================================================== -def print_columnar(x, textwidth=100, indent=0, header=''): +#========================================================================= +def print_columnar(x, textwidth=10000000, indent=0, header=''): hrstrp = '{} '.format(str(header).rstrip()) if len(hrstrp) > indent: indent = len(hrstrp) @@ -68,19 +71,19 @@ def print_columnar(x, textwidth=100, indent=0, header=''): A = [x[i::Nr] for i in xrange(Nr)] print '{}{}'.format(hrstrp, ' '.join('{: <{Lmax}}'.format(r, Lmax=Lmax) for r in A[0])) for row in A[1:]: - print '{}{}'.format(' '*indent, ' '.join('{: <{Lmax}}'.format(r, Lmax=Lmax) for r in row)) - - -#=================================================================================================== + print '{}{}'.format(' ' * indent, ' '.join('{: <{Lmax}}'.format(r, Lmax=Lmax) for r in row)) + + +#========================================================================= # Main Script Function -#=================================================================================================== +#========================================================================= def main(argv=None): args = cli(argv) # Check that the file exists if not exists(args.filename): raise OSError('File {!r} not found'.format(args.filename)) - + # Read the definitions from the file vardefs = {} varfreqs = {} @@ -94,7 +97,7 @@ def main(argv=None): split = line.split('=') if len(split) == 2: vardefs[split[0].strip()] = split[1].strip() - elif len(line)>0 : + elif len(line) > 0: print 'Could not parse this line: {!r}'.format(line) else: stddict = load(open(args.filename), object_pairs_hook=OrderedDict) @@ -120,7 +123,8 @@ def main(argv=None): try: vdeps = variable_search(parsing.parse_definition(vardefs[var])) alldeps.update(vdeps) - fheader = ' [{},{}]'.format(varfreqs[var], varrealm[var]) if var in varfreqs else '' + fheader = ' [{},{}]'.format( + varfreqs[var], varrealm[var]) if var in varfreqs else '' vheader = ' {}{}:'.format(var, fheader) print_columnar(sorted(vdeps), header=vheader) except: @@ -133,8 +137,8 @@ def main(argv=None): print_columnar(sorted(alldeps), indent=3) -#=================================================================================================== +#========================================================================= # Command-line Operation -#=================================================================================================== +#========================================================================= if __name__ == '__main__': main() diff --git a/scripts/xconform b/scripts/xconform index 30727871..fa85d9e4 100755 --- a/scripts/xconform +++ b/scripts/xconform @@ -28,9 +28,9 @@ from pyconform.dataflow import DataFlow from pyconform.flownodes import ValidationWarning -#=================================================================================================== +#========================================================================= # chunk - Chunksize for a named dimension -#=================================================================================================== +#========================================================================= def chunk(arg): try: name, size_str = arg.split(',') @@ -40,9 +40,9 @@ def chunk(arg): raise ArgumentTypeError("Chunks must be formatted as 'name,size'") -#=================================================================================================== +#========================================================================= # Command-line Interface -#=================================================================================================== +#========================================================================= def cli(argv=None): desc = """This is the PyConform command-line tool. This scripts takes input from the command-line and a predefined output @@ -57,6 +57,8 @@ def cli(argv=None): parser.add_argument('-d', '--deflate', default=None, metavar='DEFLATELEVEL', type=int, help=('Override deflate levels of all output files with given value. ' '(can be any integer from 0 to 9, with 0 meaning no compression)')) + parser.add_argument('--debug', default=False, action='store_true', + help=('Whether to enable rudimentary debug features')) parser.add_argument('-e', '--error', default=False, action='store_true', help=('Whether to error when validation checks do not pass (True) or ' 'simply print a warning message (False) [Default: False]')) @@ -82,57 +84,58 @@ def cli(argv=None): return parser.parse_args(argv) -#=================================================================================================== +#========================================================================= # Main Script Function -#=================================================================================================== +#========================================================================= def main(argv=None): args = cli(argv) - + # Create the necessary SimpleComm scomm = create_comm(serial=args.serial) - + # Do setup only on manager node if scomm.is_manager(): - + # Check that the specfile exists if not exists(args.stdfile): raise OSError(('Output specification file {!r} not ' 'found').format(args.stdfile)) - + # Read the specfile into a dictionary print 'Reading standardization file: {}'.format(args.stdfile) - dsdict = json_load(open(args.stdfile, 'r'), object_pairs_hook=OrderedDict) - + dsdict = json_load(open(args.stdfile, 'r'), + object_pairs_hook=OrderedDict) + # Parse the output Dataset print 'Creating output dataset descriptor from standardization file...' outds = OutputDatasetDesc(dsdict=dsdict) - + else: outds = None - + # Send the output descriptor to all nodes outds = scomm.partition(outds, func=Duplicate(), involved=True) - + # Sync scomm.sync() - + # Continue setup only on manager node if scomm.is_manager(): - + # Gather the list of input files infiles = [] for infile in args.infiles: infiles.extend(glob(infile)) - + # If no input files, stop here if len(infiles) == 0: print 'Standardization file validated.' return - + # Parse the input Dataset print 'Creating input dataset descriptor from {} input files...'.format(len(infiles)) inpds = InputDatasetDesc(filenames=infiles) - + else: inpds = None @@ -150,7 +153,7 @@ def main(argv=None): if args.module is not None: for i, modpath in enumerate(args.module): load_source('user{}'.format(i), modpath) - + # Setup the PyConform data flow on all nodes if scomm.is_manager(): print 'Creating the data flow...'.format(len(infiles)) @@ -158,11 +161,12 @@ def main(argv=None): # Execute the data flow (write to files) history = not args.no_history - dataflow.execute(chunks=dict(args.chunks), scomm=scomm, history=history, deflate=args.deflate) + dataflow.execute(chunks=dict(args.chunks), scomm=scomm, history=history, + deflate=args.deflate, debug=args.debug) -#=================================================================================================== +#========================================================================= # Command-line Operation -#=================================================================================================== +#========================================================================= if __name__ == '__main__': main() diff --git a/setup.py b/setup.py index 9b361b0e..ac741e37 100755 --- a/setup.py +++ b/setup.py @@ -22,5 +22,5 @@ package_dir={'pyconform': 'source/pyconform'}, package_data={'pyconform': ['LICENSE.rst']}, scripts=['scripts/iconform', 'scripts/xconform', 'scripts/vardeps'], - install_requires=['asaptools', 'netCDF4', 'pyparsing'] + install_requires=['asaptools', 'netCDF4', 'ply'] ) diff --git a/source/pyconform/.gitignore b/source/pyconform/.gitignore new file mode 100644 index 00000000..a3146b2a --- /dev/null +++ b/source/pyconform/.gitignore @@ -0,0 +1 @@ +/parser.out diff --git a/source/pyconform/dataflow.py b/source/pyconform/dataflow.py index d5827ba5..2cd6919b 100644 --- a/source/pyconform/dataflow.py +++ b/source/pyconform/dataflow.py @@ -12,13 +12,12 @@ The action associated with each node is not performed until the data is "requested" with the __getitem__ interface, via Node[key]. -Copyright 2017, University Corporation for Atmospheric Research +Copyright 2017-2018, University Corporation for Atmospheric Research LICENSE: See the LICENSE.rst file for details """ from pyconform.datasets import InputDatasetDesc, OutputDatasetDesc, DefinitionWarning -from pyconform.parsing import parse_definition -from pyconform.parsing import ParsedVariable, ParsedFunction, ParsedUniOp, ParsedBinOp +from pyconform.parsing import parse_definition, VarType, FuncType, OpType from pyconform.functions import find_operator, find_function from pyconform.physarray import PhysArray from pyconform.flownodes import DataNode, ReadNode, EvalNode, iter_dfs @@ -30,16 +29,16 @@ import numpy -#======================================================================================================================= +#========================================================================= # VariableNotFoundError -#======================================================================================================================= +#========================================================================= class VariableNotFoundError(ValueError): """Indicate if an input variable could not be found during construction""" -#=================================================================================================== +#========================================================================= # DataFlow -#=================================================================================================== +#========================================================================= class DataFlow(object): """ An object describing the flow of data from input to output @@ -48,7 +47,7 @@ class DataFlow(object): def __init__(self, inpds, outds): """ Initializer - + Parameters: inpds (InputDatasetDesc): The input dataset to use as reference when parsing variable definitions @@ -65,25 +64,28 @@ def __init__(self, inpds, outds): raise TypeError('Output dataset must be of OutputDatasetDesc type') self._ods = outds - # Create a dictionary of DataNodes from variables with non-string definitions + # Create a dictionary of DataNodes from variables with non-string + # definitions datnodes = self._create_data_nodes_() - # Create a dictionary to store FlowNodes for variables with string definitions + # Create a dictionary to store FlowNodes for variables with string + # definitions defnodes = self._create_definition_nodes_(datnodes) # Compute the definition node info objects (zero-sized physarrays) definfos = self._compute_node_infos_(defnodes) - + # Construct the dimension map self._i2omap, self._o2imap = self._compute_dimension_maps_(definfos) - + # Create the map nodes defnodes = self._create_map_nodes_(defnodes, definfos) # Create the validate nodes for each valid output variable self._valnodes = self._create_validate_nodes_(datnodes, defnodes) - - # Get the set of all sum-like dimensions (dimensions that cannot be broken into chunks) + + # Get the set of all sum-like dimensions (dimensions that cannot be + # broken into chunks) self._sumlike_dimensions = self._find_sumlike_dimensions_() # Create the WriteNodes for each time-series output file @@ -106,71 +108,83 @@ def _create_data_nodes_(self): vdata = numpy.asarray(vdesc.definition, dtype=vdesc.dtype) vunits = vdesc.cfunits() vdims = vdesc.dimensions.keys() - varray = PhysArray(vdata, name=vname, units=vunits, dimensions=vdims) + varray = PhysArray(vdata, name=vname, + units=vunits, dimensions=vdims) datnodes[vname] = DataNode(varray) return datnodes - def _create_definition_nodes_(self, datnodes): + def _create_definition_nodes_(self, datnodes): defnodes = {} for vname in self._ods.variables: vdesc = self._ods.variables[vname] if isinstance(vdesc.definition, basestring): try: - vnode = self._construct_flow_(parse_definition(vdesc.definition), datnodes=datnodes) + pdef = parse_definition(vdesc.definition) + vnode = self._construct_flow_(pdef, datnodes=datnodes) except VariableNotFoundError, err: - warn('{}. Skipping output variable {}.'.format(str(err), vname), DefinitionWarning) + warn('{}. Skipping output variable {}.'.format( + str(err), vname), DefinitionWarning) else: defnodes[vname] = vnode - return defnodes + return defnodes def _construct_flow_(self, obj, datnodes={}): - if isinstance(obj, ParsedVariable): + if isinstance(obj, VarType): vname = obj.key if vname in self._ids.variables: - return ReadNode(self._ids.variables[vname], index=obj.args) + indices = numpy.index_exp[tuple(obj.ind)] if len( + obj.ind) > 0 else () + return ReadNode(self._ids.variables[vname], index=indices) elif vname in datnodes: return datnodes[vname] else: - raise VariableNotFoundError('Input variable {!r} not found or cannot be used as input'.format(vname)) + raise VariableNotFoundError( + 'Input variable {!r} not found or cannot be used as input'.format(vname)) - elif isinstance(obj, (ParsedUniOp, ParsedBinOp)): + elif isinstance(obj, OpType): name = obj.key nargs = len(obj.args) op = find_operator(name, numargs=nargs) - args = [self._construct_flow_(arg, datnodes=datnodes) for arg in obj.args] + args = [self._construct_flow_(arg, datnodes=datnodes) + for arg in obj.args] return EvalNode(name, op, *args) - elif isinstance(obj, ParsedFunction): + elif isinstance(obj, FuncType): name = obj.key func = find_function(name) - args = [self._construct_flow_(arg, datnodes=datnodes) for arg in obj.args] - kwds = {k:self._construct_flow_(obj.kwds[k], datnodes=datnodes) for k in obj.kwds} + args = [self._construct_flow_(arg, datnodes=datnodes) + for arg in obj.args] + kwds = {k: self._construct_flow_( + obj.kwds[k], datnodes=datnodes) for k in obj.kwds} return EvalNode(name, func, *args, **kwds) else: return obj def _compute_node_infos_(self, nodes): - # Gather information about each FlowNode's metadata (via empty PhysArrays) + # Gather information about each FlowNode's metadata (via empty + # PhysArrays) infos = {} for name in nodes: - node = nodes[name] + node = nodes[name] try: info = node[None] except Exception, err: ndef = self._ods.variables[name].definition - err_msg = 'Failure in variable {!r} with definition {!r}: {}'.format(name, ndef, str(err)) + err_msg = 'Failure to generate variable {!r} info with definition {!r}: {}'.format( + name, ndef, str(err)) raise RuntimeError(err_msg) else: infos[name] = info return infos - + def _compute_dimension_maps_(self, definfos): # Each output variable FlowNode must be mapped to its output dimensions. # To aid with this, we sort by number of dimensions: - nodeorder = zip(*sorted((len(self._ods.variables[vname].dimensions), vname) for vname in definfos))[1] + nodeorder = zip( + *sorted((len(self._ods.variables[vname].dimensions), vname) for vname in definfos))[1] # Now, we construct the dimension maps i2omap = {} @@ -184,7 +198,8 @@ def _compute_dimension_maps_(self, definfos): unmapped_inp = tuple(d for d in inp_dims if d not in mapped_inp) if len(unmapped_out) != len(unmapped_inp): - map_str = ', '.join('{}-->{}'.format(k, i2omap[k]) for k in i2omap) + map_str = ', '.join( + '{}-->{}'.format(k, i2omap[k]) for k in i2omap) err_msg = ('Cannot map dimensions {} to dimensions {} in output variable {} ' '(MAP: {})').format(inp_dims, out_dims, vname, map_str) raise ValueError(err_msg) @@ -194,13 +209,14 @@ def _compute_dimension_maps_(self, definfos): o2imap[out_dim] = inp_dim i2omap[inp_dim] = out_dim - # Now that we know how dimensions are mapped, compute the output dimension sizes + # Now that we know how dimensions are mapped, compute the output + # dimension sizes for dname, ddesc in self._ods.dimensions.iteritems(): - if dname in o2imap: + if dname in o2imap and o2imap[dname] in self._ids.dimensions: idd = self._ids.dimensions[o2imap[dname]] if (ddesc.is_set() and ddesc.stringlen and ddesc.size < idd.size) or not ddesc.is_set(): ddesc.set(idd) - + return i2omap, o2imap @property @@ -219,7 +235,7 @@ def _create_map_nodes_(self, defnodes, definfos): return mapnodes def _create_validate_nodes_(self, datnodes, defnodes): - valid_vars = datnodes.keys() + defnodes.keys() + valid_vars = datnodes.keys() + defnodes.keys() valnodes = {} for vname in valid_vars: vdesc = self._ods.variables[vname] @@ -229,12 +245,13 @@ def _create_validate_nodes_(self, datnodes, defnodes): validnode = ValidateNode(vdesc, vnode) except Exception, err: vdef = vdesc.definition - err_msg = 'Failure in variable {!r} with definition {!r}: {}'.format(vname, vdef, str(err)) + err_msg = 'Failure in variable {!r} with definition {!r}: {}'.format( + vname, vdef, str(err)) raise RuntimeError(err_msg) valnodes[vname] = validnode return valnodes - + def _find_sumlike_dimensions_(self): unmapped_sumlike_dimensions = set() for vname in self._valnodes: @@ -242,7 +259,7 @@ def _find_sumlike_dimensions_(self): for nd in iter_dfs(vnode): if isinstance(nd, EvalNode): unmapped_sumlike_dimensions.update(nd.sumlike_dimensions) - + # Map the sum-like dimensions to output dimensions return set(self._i2omap[d] for d in unmapped_sumlike_dimensions if d in self._i2omap) @@ -250,12 +267,14 @@ def _create_write_nodes_(self): writenodes = {} for fname in self._ods.files: fdesc = self._ods.files[fname] - vmissing = tuple(vname for vname in fdesc.variables if vname not in self._valnodes) + vmissing = tuple( + vname for vname in fdesc.variables if vname not in self._valnodes) if vmissing: warn('Skipping output file {} due to missing required variables: ' '{}'.format(fname, ', '.join(sorted(vmissing))), DefinitionWarning) else: - vnodes = tuple(self._valnodes[vname] for vname in fdesc.variables) + vnodes = tuple(self._valnodes[vname] + for vname in fdesc.variables) wnode = WriteNode(fdesc, inputs=vnodes) writenodes[wnode.label] = wnode return writenodes @@ -268,17 +287,18 @@ def _compute_variable_sizes_(self): vsize = 1 if vsize == 0 else vsize bytesizes[vname] = vsize * vdesc.dtype.itemsize return bytesizes - + def _compute_file_sizes(self, varsizes): filesizes = {} for fname, wnode in self._writenodes.iteritems(): - filesizes[fname] = sum(varsizes[vnode.label] for vnode in wnode.inputs) + filesizes[fname] = sum(varsizes[vnode.label] + for vnode in wnode.inputs) return filesizes - def execute(self, chunks={}, serial=False, history=False, scomm=None, deflate=None): + def execute(self, chunks={}, serial=False, history=False, scomm=None, deflate=None, debug=False): """ Execute the Data Flow - + Parameters: chunks (dict): A dictionary of output dimension names and chunk sizes for each dimension given. Output dimensions not included in the dictionary will not be @@ -291,6 +311,7 @@ def execute(self, chunks={}, serial=False, history=False, scomm=None, deflate=No scomm (SimpleComm): An externally created SimpleComm object to use for managing parallel operation deflate (int): Override all output file deflate levels with given value + debug (bool): Whether to enable some rudimentary debugging features """ # Check chunks type if not isinstance(chunks, dict): @@ -299,16 +320,22 @@ def execute(self, chunks={}, serial=False, history=False, scomm=None, deflate=No # Make sure that the specified chunking dimensions are valid for odname, odsize in chunks.iteritems(): if odname not in self._o2imap: - raise ValueError('Cannot chunk over unknown output dimension {!r}'.format(odname)) + raise ValueError( + 'Cannot chunk over unknown output dimension {!r}'.format(odname)) if not isinstance(odsize, int): raise TypeError(('Chunk size invalid for output dimension {!r}: ' '{}').format(odname, odsize)) - + # Check that we are not chunking over any "sum-like" dimensions - sumlike_chunk_dims = sorted(d for d in chunks if d in self._sumlike_dimensions) + sumlike_chunk_dims = sorted( + d for d in chunks if d in self._sumlike_dimensions) if len(sumlike_chunk_dims) > 0: - raise ValueError(('Cannot chunk over dimensions that are summed over (or "sum-like")' - ': {}'.format(', '.join(sumlike_chunk_dims)))) + if debug: + for d in sumlike_chunk_dims: + chunks.pop(d) + else: + raise ValueError('Cannot chunk over dimensions that are summed over (or "sum-like")' + ': {}'.format(', '.join(sumlike_chunk_dims))) # Create the simple communicator, if necessary if scomm is None: @@ -318,7 +345,7 @@ def execute(self, chunks={}, serial=False, history=False, scomm=None, deflate=No print 'Inheriting SimpleComm object from parent. (Ignoring serial argument.)' else: raise TypeError('Communication object is not a SimpleComm!') - + # Start general output prefix = '[{}/{}]'.format(scomm.get_rank(), scomm.get_size()) if scomm.is_manager(): @@ -333,8 +360,10 @@ def execute(self, chunks={}, serial=False, history=False, scomm=None, deflate=No else: print 'Not chunking output.' - # Partition the output files/variables over available parallel (MPI) ranks - fnames = scomm.partition(self._filesizes.items(), func=WeightBalanced(), involved=True) + # Partition the output files/variables over available parallel (MPI) + # ranks + fnames = scomm.partition( + self._filesizes.items(), func=WeightBalanced(), involved=True) if scomm.is_manager(): print 'Writing {} files across {} MPI processes.'.format(len(self._filesizes), scomm.get_size()) scomm.sync() @@ -342,7 +371,7 @@ def execute(self, chunks={}, serial=False, history=False, scomm=None, deflate=No # Standard output print '{}: Writing {} files: {}'.format(prefix, len(fnames), ', '.join(fnames)) scomm.sync() - + # Loop over output files and write using given chunking for fname in fnames: print '{}: Writing file: {}'.format(prefix, fname) diff --git a/source/pyconform/flownodes.py b/source/pyconform/flownodes.py index ad11b1e1..cd767120 100644 --- a/source/pyconform/flownodes.py +++ b/source/pyconform/flownodes.py @@ -14,7 +14,7 @@ from cf_units import Unit, num2date from datetime import datetime from os.path import exists, dirname -from os import makedirs +from os import makedirs, rename from netCDF4 import Dataset from collections import OrderedDict from warnings import warn @@ -22,40 +22,40 @@ import numpy -#=================================================================================================== +#========================================================================= # ValidationWarning -#=================================================================================================== +#========================================================================= class ValidationWarning(Warning): """Warning for validation errors""" -#=================================================================================================== +#========================================================================= # UnitsWarning -#=================================================================================================== +#========================================================================= class UnitsWarning(Warning): """Warning for units errors""" -#======================================================================================================================= +#========================================================================= # DateTimeAutoParseWarning -#======================================================================================================================= +#========================================================================= class DateTimeAutoParseWarning(Warning): """Warning for not being able to autoparse new filename based on date-time in the file""" -#======================================================================================================================= +#========================================================================= # iter_dfs - Depth-First Search Iterator -#======================================================================================================================= +#========================================================================= def iter_dfs(node): """ Iterate through graph of FlowNodes from a starting node using a Depth-First Search - + Parameters: node (FlowNode): the starting node from where to begin iterating """ if not isinstance(node, FlowNode): raise TypeError('Can only iterate over FlowNodes') - + visited = set() tosearch = [node] while tosearch: @@ -66,19 +66,19 @@ def iter_dfs(node): yield nd -#======================================================================================================================= +#========================================================================= # iter_bfs - Breadth-First Search Iterator -#======================================================================================================================= +#========================================================================= def iter_bfs(node): """ Iterate through graph of FlowNodes from a starting node using a Breadth-First Search - + Parameters: node (FlowNode): the starting node from where to begin iterating """ if not isinstance(node, FlowNode): raise TypeError('Can only iterate over FlowNodes') - + visited = set() tosearch = [node] while tosearch: @@ -89,13 +89,13 @@ def iter_bfs(node): yield nd -#=================================================================================================== +#========================================================================= # FlowNode -#=================================================================================================== +#========================================================================= class FlowNode(object): """ The base class for objects that can appear in a data flow - + The FlowNode object represents a point in the directed acyclic graph where multiple edges meet. It represents a functional operation on the DataArrays coming into it from its adjacent DataNodes. The FlowNode itself outputs the result of this operation @@ -106,7 +106,7 @@ class FlowNode(object): def __init__(self, label, *inputs): """ Initializer - + Parameters: label: A label to give the FlowNode inputs (list): DataNodes that provide input into this FlowNode @@ -125,20 +125,20 @@ def inputs(self): return self._inputs -#=================================================================================================== +#========================================================================= # DataNode -#=================================================================================================== +#========================================================================= class DataNode(FlowNode): """ FlowNode class to create data in memory - + This is a "source" FlowNode. """ def __init__(self, data): """ Initializer - + Parameters: data (PhysArray): Data to store in this FlowNode """ @@ -160,20 +160,20 @@ def __getitem__(self, index): return self._data[index] -#=================================================================================================== +#========================================================================= # ReadNode -#=================================================================================================== +#========================================================================= class ReadNode(FlowNode): """ FlowNode class for reading data from a NetCDF file - + This is a "source" FlowNode. """ def __init__(self, variable, index=slice(None)): """ Initializer - + Parameters: variable (VariableDesc): A variable descriptor object index (tuple, slice, int, dict): A tuple of slices or ints, or a slice or int, @@ -187,19 +187,22 @@ def __init__(self, variable, index=slice(None)): # Check for associated file if len(variable.files) == 0: - raise ValueError('Variable descriptor {} has no associated files'.format(variable.name)) + raise ValueError( + 'Variable descriptor {} has no associated files'.format(variable.name)) self._filepath = None for fdesc in variable.files.itervalues(): if fdesc.exists(): self._filepath = fdesc.name break if self._filepath is None: - raise OSError('File path not found for input variable: {!r}'.format(variable.name)) + raise OSError( + 'File path not found for input variable: {!r}'.format(variable.name)) # Check that the variable exists in the file with Dataset(self._filepath, 'r') as ncfile: if variable.name not in ncfile.variables: - raise OSError('Variable {!r} not found in NetCDF file: {!r}'.format(variable.name, self._filepath)) + raise OSError('Variable {!r} not found in NetCDF file: {!r}'.format( + variable.name, self._filepath)) self._variable = variable.name # Check if the index means "all" @@ -231,15 +234,16 @@ def __getitem__(self, index): ncvar = ncfile.variables[self._variable] # Get the attributes into a dictionary, for convenience - attrs = {a:ncvar.getncattr(a) for a in ncvar.ncattrs()} - + attrs = {a: ncvar.getncattr(a) for a in ncvar.ncattrs()} + # Read the variable units units_attr = attrs.get('units', 1) calendar_attr = attrs.get('calendar', None) try: units = Unit(units_attr, calendar=calendar_attr) except ValueError: - msg = 'Units {!r} unrecognized in UDUNITS. Assuming unitless.'.format(units_attr) + msg = 'Units {!r} unrecognized in UDUNITS. Assuming unitless.'.format( + units_attr) warn(msg, UnitsWarning) units = Unit(1) except: @@ -255,13 +259,15 @@ def __getitem__(self, index): index1 = align_index(self._index, dimensions0) # Get the dimensions after application of the first index - dimensions1 = tuple(d for d, i in zip(dimensions0, index1) if isinstance(i, slice)) + dimensions1 = tuple(d for d, i in zip( + dimensions0, index1) if isinstance(i, slice)) # Align the second index on the intermediate dimensions index2 = align_index(index, dimensions1) # Get the dimensions after application of the second index - dimensions2 = tuple(d for d, i in zip(dimensions1, index2) if isinstance(i, slice)) + dimensions2 = tuple(d for d, i in zip( + dimensions1, index2) if isinstance(i, slice)) # Compute the joined index object index12 = join(shape0, index1, index2) @@ -277,20 +283,20 @@ def __getitem__(self, index): # Upconvert, if possible if issubclass(ncvar.dtype.type, numpy.float) and ncvar.dtype.itemsize < 8: data = data.astype(numpy.float64) - + # Read the positive attribute, if available pos = attrs.get('positive', None) return PhysArray(data, name=self.label, units=units, dimensions=dimensions2, positive=pos) -#=================================================================================================== +#========================================================================= # EvalNode -#=================================================================================================== +#========================================================================= class EvalNode(FlowNode): """ FlowNode class for evaluating a function on input from neighboring DataNodes - + The EvalNode is constructed with a function reference and any number of arguments to that function. The number of arguments supplied must match the number of arguments accepted by the function. The arguments can be any type, and the order of the arguments will be @@ -304,7 +310,7 @@ class EvalNode(FlowNode): def __init__(self, label, func, *args, **kwds): """ Initializer - + Parameters: label: A label to give the FlowNode func (class): A Function class @@ -313,13 +319,13 @@ def __init__(self, label, func, *args, **kwds): """ # Initialize the function object self._function = func(*args, **kwds) - + # Include all references as input allargs = tuple(args) + tuple(kwds[k] for k in kwds) - + # Call the base class initialization super(EvalNode, self).__init__(label, *allargs) - + @property def sumlike_dimensions(self): """ @@ -337,25 +343,25 @@ def __getitem__(self, index): return self._function[index] -#=================================================================================================== +#========================================================================= # MapNode -#=================================================================================================== +#========================================================================= class MapNode(FlowNode): """ FlowNode class to map input data from a neighboring FlowNode to new dimension names and units - + The MapNode can rename the dimensions of a FlowNode's output data. It does not change the data itself, however. The input dimension names will be changed according to the dimension map given. If an input dimension name is not referenced by the map, then the input dimension name does not change. - + This is a "non-source"/"non-sink" FlowNode. """ def __init__(self, label, dnode, dmap={}): """ Initializer - + Parameters: label: The label given to the FlowNode dnode (FlowNode): FlowNode that provides input into this FlowNode @@ -364,7 +370,8 @@ def __init__(self, label, dnode, dmap={}): """ # Check FlowNode type if not isinstance(dnode, FlowNode): - raise TypeError('MapNode can only act on output from another FlowNode') + raise TypeError( + 'MapNode can only act on output from another FlowNode') # Check dimension map type if not isinstance(dmap, dict): @@ -399,11 +406,13 @@ def __getitem__(self, index): inp_index = None elif isinstance(index, dict): - inp_index = dict((self._o2imap.get(d, d), i) for d, i in index.iteritems()) + inp_index = dict((self._o2imap.get(d, d), i) + for d, i in index.iteritems()) else: out_index = index_tuple(index, len(inp_dims)) - inp_index = dict((self._o2imap.get(d, d), i) for d, i in zip(out_dims, out_index)) + inp_index = dict((self._o2imap.get(d, d), i) + for d, i in zip(out_dims, out_index)) # Return the mapped data idims_str = ','.join(inp_dims) @@ -411,28 +420,29 @@ def __getitem__(self, index): if inp_dims == out_dims: name = inp_info.name else: - name = 'map({}, from=[{}], to=[{}])'.format(inp_info.name, idims_str, odims_str) + name = 'map({}, from=[{}], to=[{}])'.format( + inp_info.name, idims_str, odims_str) return PhysArray(self.inputs[0][inp_index], name=name, dimensions=out_dims) -#=================================================================================================== +#========================================================================= # ValidateNode -#=================================================================================================== +#========================================================================= class ValidateNode(FlowNode): """ FlowNode class to validate input data from a neighboring FlowNode - + The ValidateNode takes additional attributes in its initializer that can effect the behavior of its __getitem__ method. The special attributes are: - + 'valid_min': The minimum value the data should have, if valid 'valid_max': The maximum value the data should have, if valid 'min_mean_abs': The minimum acceptable value of the mean of the absolute value of the data 'max_mean_abs': The maximum acceptable value of the mean of the absolute value of the data - + If these attributes are supplied to the ValidateNode at construction time, then the associated validation checks will be made on the data when __getitem__ is called. - + Additional attributes may be added to the ValidateNode that do not affect functionality. These attributes may be named however the user wishes and can be retrieved from the FlowNode as a dictionary with the 'attributes' property. @@ -443,39 +453,44 @@ class ValidateNode(FlowNode): def __init__(self, vdesc, dnode): """ Initializer - + Parameters: vdesc (VariableDesc): A variable descriptor object for the output variable dnode (FlowNode): FlowNode that provides input into this FlowNode """ # Check Types if not isinstance(vdesc, VariableDesc): - raise TypeError('ValidateNode requires a VariableDesc object as input') + raise TypeError( + 'ValidateNode requires a VariableDesc object as input') if not isinstance(dnode, FlowNode): - raise TypeError('ValidateNode can only act on output from another FlowNode') + raise TypeError( + 'ValidateNode can only act on output from another FlowNode') # Call base class initializer super(ValidateNode, self).__init__(vdesc.name, dnode) - + # Save the variable descriptor object self._vdesc = vdesc - + # Initialize the history attribute, if necessary info = dnode[None] if 'history' not in self.attributes: self.attributes['history'] = info.name - # Else inherit the units and calendar of the input data stream, if necessary + # Else inherit the units and calendar of the input data stream, if + # necessary if 'units' in self.attributes: if info.units.is_time_reference(): - ustr, rstr = [c.strip() for c in str(info.units).split('since')] + ustr, rstr = [c.strip() + for c in str(info.units).split('since')] if self._vdesc.units() is not None: ustr = self._vdesc.units() if self._vdesc.refdatetime() is not None: rstr = self._vdesc.refdatetime() self.attributes['units'] = '{} since {}'.format(ustr, rstr) - - # Calendars must match as convertion between different calendars will fail + + # Calendars must match as convertion between different + # calendars will fail if self._vdesc.calendar() is None and info.units.calendar is not None: self.attributes['calendar'] = info.units.calendar @@ -513,12 +528,13 @@ def __getitem__(self, index): if numpy.can_cast(indata.dtype, odtype, casting='same_kind'): indata = indata.astype(odtype) else: - raise TypeError('Cannot cast datatype {!s} to {!s} in ValidateNode ' - '{!r}').format(indata.dtype, odtype, self.label) + raise TypeError(('Cannot cast datatype {!s} to {!s} in ValidateNode ' + '{!r}').format(indata.dtype, odtype, self.label)) # Check that units match as expected, otherwise convert if 'units' in self.attributes: - ounits = Unit(self.attributes['units'], calendar=self.attributes.get('calendar', None)) + ounits = Unit( + self.attributes['units'], calendar=self.attributes.get('calendar', None)) if ounits != indata.units: if index is None: indata.units = ounits @@ -526,13 +542,14 @@ def __getitem__(self, index): try: indata = indata.convert(ounits) except Exception as err: - err_msg = 'When validating output variable {}: {}'.format(self.label, err) + err_msg = 'When validating output variable {}: {}'.format( + self.label, err) raise err.__class__(err_msg) - + # Check that the dimensions match as expected if self.dimensions != indata.dimensions: indata = indata.transpose(self.dimensions) - + # Check the positive attribute, if specified positive = self.attributes.get('positive', None) if positive is not None and indata.positive != positive: @@ -552,7 +569,8 @@ def __getitem__(self, index): if valid_min: dmin = numpy.min(indata) if dmin < valid_min: - msg = 'valid_min: {} < {} ({!r})'.format(dmin, valid_min, self.label) + msg = 'valid_min: {} < {} ({!r})'.format( + dmin, valid_min, self.label) warn(msg, ValidationWarning) indata = numpy.ma.masked_where(indata <= valid_min, indata) @@ -560,7 +578,8 @@ def __getitem__(self, index): if valid_max: dmax = numpy.max(indata) if dmax > valid_max: - msg = 'valid_max: {} > {} ({!r})'.format(dmax, valid_max, self.label) + msg = 'valid_max: {} > {} ({!r})'.format( + dmax, valid_max, self.label) warn(msg, ValidationWarning) indata = numpy.ma.masked_where(indata >= valid_max, indata) @@ -571,29 +590,31 @@ def __getitem__(self, index): # Validate minimum mean abs if ok_min_mean_abs: if mean_abs < ok_min_mean_abs: - msg = 'ok_min_mean_abs: {} < {} ({!r})'.format(mean_abs, ok_min_mean_abs, self.label) + msg = 'ok_min_mean_abs: {} < {} ({!r})'.format( + mean_abs, ok_min_mean_abs, self.label) warn(msg, ValidationWarning) # Validate maximum mean abs if ok_max_mean_abs: if mean_abs > ok_max_mean_abs: - msg = 'ok_max_mean_abs: {} > {} ({!r})'.format(mean_abs, ok_max_mean_abs, self.label) + msg = 'ok_max_mean_abs: {} > {} ({!r})'.format( + mean_abs, ok_max_mean_abs, self.label) warn(msg, ValidationWarning) return indata -#=================================================================================================== +#========================================================================= # WriteNode -#=================================================================================================== +#========================================================================= class WriteNode(FlowNode): """ FlowNode that writes validated data to a file. - + This is a "sink" node, meaning that the __getitem__ (i.e., [index]) interface does not return anything. Rather, the data "retrieved" through the __getitem__ interface is sent directly to file. - + For this reason, it is possible to "retrieve" data multiple times, resulting in writing and overwriting of data. To eliminate this inefficiency, it is advised that you use the 'execute' method to write data efficiently once (and only once). @@ -602,7 +623,7 @@ class WriteNode(FlowNode): def __init__(self, filedesc, inputs=()): """ Initializer - + Parameters: filedesc (FileDesc): File descriptor for the file to write inputs (tuple): A tuple of ValidateNodes providing input into the file @@ -638,29 +659,30 @@ def __init__(self, filedesc, inputs=()): fname = self._autoparse_filename_(self.label) self._label = fname self._filedesc._name = fname - + self._tmp_ext = '.tmp.nc' + # Set the filehandle self._file = None # Initialize set of inverted dimensions self._idims = set() - + # Initialize set of unwritten attributes self._unwritten_attributes = {'_FillValue', 'direction', 'history'} - + def _autoparse_filename_(self, fname): """ Determine if autoparsing the filename needs to be done - + Parameters: fname (str): The original name of the file - + Returns: str: The new name for the file """ - + if '{' in fname: - + possible_tvars = [] for var in self._filedesc.variables: vdesc = self._filedesc.variables[var] @@ -673,14 +695,16 @@ def _autoparse_filename_(self, fname): elif 'axis' in vdesc.attributes and vdesc.attributes['axis'] == 'T': possible_tvars.append(var) if len(possible_tvars) == 0: - msg = 'Could not identify a time variable to autoparse filename {!r}'.format(fname) + msg = 'Could not identify a time variable to autoparse filename {!r}'.format( + fname) warn(msg, DateTimeAutoParseWarning) return fname - + tvar = 'time' if 'time' in possible_tvars else possible_tvars[0] tnodes = [vnode for vnode in self.inputs if vnode.label == tvar] if len(tnodes) == 0: - raise ValueError('Time variable input missing for file {!r}'.format(fname)) + raise ValueError( + 'Time variable input missing for file {!r}'.format(fname)) tnode = tnodes[0] t1 = tnode[0:1] t2 = tnode[-1:] @@ -689,18 +713,21 @@ def _autoparse_filename_(self, fname): beg = fname.find('{') end = fname.find('}', beg) if end == -1: - raise ValueError('Filename {!r} has unbalanced special characters'.format(fname)) + raise ValueError( + 'Filename {!r} has unbalanced special characters'.format(fname)) prefix = fname[:beg] - fmtstr1, fmtstr2 = fname[beg+1:end].split('-') - suffix = fname[end+1:] + fmtstr1, fmtstr2 = fname[beg + 1:end].split('-') + suffix = fname[end + 1:] + + datestr1 = num2date(t1.data[0], str(t1.units), t1.units.calendar).strftime( + fmtstr1).replace(' ', '0') + datestr2 = num2date(t2.data[0], str(t2.units), t2.units.calendar).strftime( + fmtstr2).replace(' ', '0') - datestr1 = num2date(t1.data[0], str(t1.units), t1.units.calendar).strftime(fmtstr1).replace(' ', '0') - datestr2 = num2date(t2.data[0], str(t2.units), t2.units.calendar).strftime(fmtstr2).replace(' ', '0') - fname = '{}{}-{}{}'.format(prefix, datestr1, datestr2, suffix) - - return fname - + + return fname + def enable_history(self): """ Enable writing of the history attribute to the file @@ -719,24 +746,28 @@ def _open_(self, deflate=None): Open the file for writing, if not open already """ if self._file is None: - + # Make the necessary subdirectories to open the file fname = self.label + tmp_fname = '{}{}'.format(fname, self._tmp_ext) fdir = dirname(fname) + fmt = self._filedesc.format if len(fdir) > 0 and not exists(fdir): try: makedirs(fdir) except: - raise IOError('Failed to create directory for output file {!r}'.format(fname)) - + raise IOError( + 'Failed to create directory for output file {!r}'.format(fname)) + # Try to open the output file for writing try: - self._file = Dataset(fname, 'w', format=self._filedesc.format) + self._file = Dataset(tmp_fname, 'w', format=fmt) except: raise IOError('Failed to open output file {!r}'.format(fname)) # Write the global attributes - self._filedesc.attributes['creation_date'] = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') + self._filedesc.attributes['creation_date'] = datetime.utcnow( + ).strftime('%Y-%m-%dT%H:%M:%SZ') self._file.setncatts(self._filedesc.attributes) # Scan over variables for coordinates and dimension information @@ -754,7 +785,7 @@ def _open_(self, deflate=None): # Determine coordinates and dimensions to invert if len(vdesc.dimensions) == 1 and 'axis' in vnode.attributes: - if 'direction' in vnode.attributes: + if 'direction' in vnode.attributes: vdir_out = vnode.attributes['direction'] if vdir_out not in ['increasing', 'decreasing']: raise ValueError(('Unrecognized direction in output coordinate variable ' @@ -765,7 +796,7 @@ def _open_(self, deflate=None): 'direction').format(vname)) if vdir_inp != vdir_out: self._idims.add(vdesc.dimensions.keys()[0]) - + # Create the required dimensions in the file for dname in req_dims: ddesc = self._filedesc.dimensions[dname] @@ -781,7 +812,8 @@ def _open_(self, deflate=None): for vnode in self.inputs: vname = vnode.label vdesc = self._filedesc.variables[vname] - vattrs = OrderedDict((k, v) for k, v in vnode.attributes.iteritems()) + vattrs = OrderedDict((k, v) + for k, v in vnode.attributes.iteritems()) vdtype = vdesc.dtype fillval = vattrs.get('_FillValue', None) @@ -791,20 +823,25 @@ def _open_(self, deflate=None): clev = self._filedesc.deflate if zlib else 1 else: if not isinstance(deflate, int): - raise TypeError('Override deflate value must be an integer') + raise TypeError( + 'Override deflate value must be an integer') if deflate < 0 or deflate > 9: - raise TypeError('Override deflate value range from 0 to 9') + raise TypeError( + 'Override deflate value range from 0 to 9') zlib = deflate > 0 clev = deflate if zlib else 1 - ncvar = self._file.createVariable(vname, vdtype, vdims, fill_value=fillval, zlib=zlib, complevel=clev) + ncvar = self._file.createVariable( + vname, vdtype, vdims, fill_value=fillval, zlib=zlib, complevel=clev) for aname in vattrs: if aname not in self._unwritten_attributes: avalue = vattrs[aname] if aname == 'history': - idimstr = ','.join(d for d in vdesc.dimensions if d in self._idims) + idimstr = ','.join( + d for d in vdesc.dimensions if d in self._idims) if len(idimstr) > 0: - avalue = 'invdims({}, dims=[{}])'.format(avalue, idimstr) + avalue = 'invdims({}, dims=[{}])'.format( + avalue, idimstr) ncvar.setncattr(aname, avalue) def _close_(self): @@ -815,19 +852,24 @@ def _close_(self): self._file.close() self._idims = set() self._file = None + tmp_fname = '{}{}'.format(self.label, self._tmp_ext) + if exists(tmp_fname): + rename(tmp_fname, self.label) @staticmethod def _chunk_iter_(dsizes, chunks={}): if not isinstance(dsizes, OrderedDict): - raise TypeError('Dimensions must be an ordered dictionary of names and sizes') + raise TypeError( + 'Dimensions must be an ordered dictionary of names and sizes') if not isinstance(chunks, dict): raise TypeError('Dimension chunks must be a dictionary') - chunks_ = {d:chunks[d] if d in chunks else dsizes[d] for d in dsizes} - nchunks = {d:int(dsizes[d]//chunks_[d]) + int(dsizes[d]%chunks_[d]>0) for d in dsizes} + chunks_ = {d: chunks[d] if d in chunks else dsizes[d] for d in dsizes} + nchunks = {d: int(dsizes[d] // chunks_[d]) + + int(dsizes[d] % chunks_[d] > 0) for d in dsizes} ntotal = int(numpy.prod([nchunks[d] for d in nchunks])) - - idx = {d:0 for d in dsizes} + + idx = {d: 0 for d in dsizes} for n in xrange(ntotal): for d in nchunks: n, idx[d] = divmod(n, nchunks[d]) @@ -835,29 +877,32 @@ def _chunk_iter_(dsizes, chunks={}): for d in dsizes: lb = idx[d] * chunks_[d] ub = (idx[d] + 1) * chunks_[d] - chunk[d] = slice(lb, ub if ub < dsizes[d] else None) + chunk[d] = slice(lb, ub if ub < dsizes[d] else None) yield chunk - + @staticmethod def _invert_dims_(dsizes, chunk, idims=set()): if not isinstance(dsizes, OrderedDict): - raise TypeError('Dimensions must be an ordered dictionary of names and sizes') + raise TypeError( + 'Dimensions must be an ordered dictionary of names and sizes') if not isinstance(chunk, OrderedDict): - raise TypeError('Chunk must be an ordered dictionary of names and slices') + raise TypeError( + 'Chunk must be an ordered dictionary of names and slices') if not isinstance(idims, set): raise TypeError('Dimensions to invert must be a set') - + ichunk = OrderedDict() for d in dsizes: s = dsizes[d] c = chunk[d] if d in idims: ub = s if c.stop is None else c.stop - ichunk[d] = slice(s - c.start - 1, s - ub - 1 if ub < s else None, -1) + ichunk[d] = slice(s - c.start - 1, s - ub - + 1 if ub < s else None, -1) else: ichunk[d] = c return ichunk - + @staticmethod def _direction_(data): diff = numpy.diff(data) @@ -871,10 +916,10 @@ def _direction_(data): def execute(self, chunks={}, deflate=None): """ Execute the writing of the WriteNode file at once - + This method efficiently writes all of the data for each file only once, chunking the data according to the 'chunks' parameter, as needed. - + Parameters: chunks (dict): A dictionary of output dimension names and chunk sizes for each dimension given. Output dimensions not included in the dictionary will not be @@ -887,33 +932,37 @@ def execute(self, chunks={}, deflate=None): # Open the file and write the header information self._open_(deflate=deflate) - # Create data structure to keep track of which variable chunks we have written - vchunks = {vnode.label:set() for vnode in self.inputs} - - # Compute the Global Dimension Sizes dictionary from the input variable nodes + # Create data structure to keep track of which variable chunks we have + # written + vchunks = {vnode.label: set() for vnode in self.inputs} + + # Compute the Global Dimension Sizes dictionary from the input variable + # nodes inputdims = [] for vnode in self.inputs: for d in self._filedesc.variables[vnode.label].dimensions: if d not in inputdims: inputdims.append(d) - gdims = OrderedDict((d, self._filedesc.dimensions[d].size) for d in inputdims) - + gdims = OrderedDict( + (d, self._filedesc.dimensions[d].size) for d in inputdims) + # Iterate over the global dimension space for chunk in WriteNode._chunk_iter_(gdims, chunks=chunks): - + # Invert the necessary dimensions to get the read-chunk rchunk = self._invert_dims_(gdims, chunk, idims=self._idims) - + # Loop over all variables and write the data, if necessary for vnode in self.inputs: vname = vnode.label vdesc = self._filedesc.variables[vname] ncvar = self._file.variables[vname] - + # Compute the write-chunk for the given variable wchunk = tuple(chunk[d] for d in vdesc.dimensions) - - # Write the data to the variable, if it hasn't already been written + + # Write the data to the variable, if it hasn't already been + # written if repr(wchunk) not in vchunks[vname]: vdata = vnode[rchunk] if isinstance(vdata, CharArray): diff --git a/source/pyconform/functions.py b/source/pyconform/functions.py index 1cbe3455..19eedcc9 100644 --- a/source/pyconform/functions.py +++ b/source/pyconform/functions.py @@ -9,16 +9,21 @@ from pyconform.physarray import PhysArray, UnitsError from numpy.ma import sqrt, where from cf_units import Unit +import numpy as np -#======================================================================================================================= +#========================================================================= # is_constant - Determine if an argument is a constant (number or string) -#======================================================================================================================= +#========================================================================= + + def is_constant(arg): return isinstance(arg, (basestring, float, int)) or arg is None - -#=================================================================================================== + +#========================================================================= # Find a function or operator based on key and number of arguments -#=================================================================================================== +#========================================================================= + + def find(key, numargs=None): try: fop = find_operator(key, numargs=numargs) @@ -26,9 +31,10 @@ def find(key, numargs=None): pass else: return fop - + if numargs is not None: - raise KeyError('No operator {!r} with {} arguments found'.format(key, numargs)) + raise KeyError( + 'No operator {!r} with {} arguments found'.format(key, numargs)) try: fop = find_function(key) @@ -38,9 +44,9 @@ def find(key, numargs=None): return fop -#=================================================================================================== +#========================================================================= # FunctionBase - base class for Function and Operator Classes -#=================================================================================================== +#========================================================================= class FunctionBase(object): __metaclass__ = ABCMeta key = 'function' @@ -54,14 +60,14 @@ def __getitem__(self): return None -#################################################################################################### -##### OPERATORS #################################################################################### -#################################################################################################### +########################################################################## +##### OPERATORS ########################################################## +########################################################################## -#=================================================================================================== +#========================================================================= # Get the function associated with the given key-symbol -#=================================================================================================== +#========================================================================= def find_operator(key, numargs=None): if key not in __OPERATORS__: raise KeyError('Operator {!r} not found'.format(key)) @@ -75,127 +81,139 @@ def find_operator(key, numargs=None): raise KeyError(('Operator {!r} has multiple definitions, ' 'number of arguments required').format(key)) elif numargs not in ops: - raise KeyError('Operator {!r} with {} arguments not found'.format(key, numargs)) + raise KeyError( + 'Operator {!r} with {} arguments not found'.format(key, numargs)) else: return ops[numargs] -#=================================================================================================== +#========================================================================= # operators -#=================================================================================================== +#========================================================================= def list_operators(): return sorted(__OPERATORS__.keys()) -#=================================================================================================== +#========================================================================= # Operator - From which all 'X op Y'-pattern operators derive -#=================================================================================================== +#========================================================================= class Operator(FunctionBase): key = '?' numargs = 2 - + def __init__(self, *args): super(Operator, self).__init__(*args) -#=================================================================================================== +#========================================================================= # NegationOperator -#=================================================================================================== +#========================================================================= class NegationOperator(Operator): key = '-' numargs = 1 def __init__(self, arg): super(NegationOperator, self).__init__(arg) - + def __getitem__(self, index): - arg = self.arguments[0] if is_constant(self.arguments[0]) else self.arguments[0][index] + arg = self.arguments[0] if is_constant( + self.arguments[0]) else self.arguments[0][index] return -arg -#=================================================================================================== +#========================================================================= # AdditionOperator -#=================================================================================================== +#========================================================================= class AdditionOperator(Operator): key = '+' numargs = 2 def __init__(self, left, right): super(AdditionOperator, self).__init__(left, right) - + def __getitem__(self, index): - left = self.arguments[0] if is_constant(self.arguments[0]) else self.arguments[0][index] - right = self.arguments[1] if is_constant(self.arguments[1]) else self.arguments[1][index] + left = self.arguments[0] if is_constant( + self.arguments[0]) else self.arguments[0][index] + right = self.arguments[1] if is_constant( + self.arguments[1]) else self.arguments[1][index] return left + right -#=================================================================================================== +#========================================================================= # SubtractionOperator -#=================================================================================================== +#========================================================================= class SubtractionOperator(Operator): key = '-' numargs = 2 def __init__(self, left, right): super(SubtractionOperator, self).__init__(left, right) - + def __getitem__(self, index): - left = self.arguments[0] if is_constant(self.arguments[0]) else self.arguments[0][index] - right = self.arguments[1] if is_constant(self.arguments[1]) else self.arguments[1][index] + left = self.arguments[0] if is_constant( + self.arguments[0]) else self.arguments[0][index] + right = self.arguments[1] if is_constant( + self.arguments[1]) else self.arguments[1][index] return left - right -#=================================================================================================== +#========================================================================= # PowerOperator -#=================================================================================================== +#========================================================================= class PowerOperator(Operator): key = '**' numargs = 2 def __init__(self, left, right): super(PowerOperator, self).__init__(left, right) - + def __getitem__(self, index): - left = self.arguments[0] if is_constant(self.arguments[0]) else self.arguments[0][index] - right = self.arguments[1] if is_constant(self.arguments[1]) else self.arguments[1][index] + left = self.arguments[0] if is_constant( + self.arguments[0]) else self.arguments[0][index] + right = self.arguments[1] if is_constant( + self.arguments[1]) else self.arguments[1][index] return left ** right -#=================================================================================================== +#========================================================================= # MultiplicationOperator -#=================================================================================================== +#========================================================================= class MultiplicationOperator(Operator): key = '*' numargs = 2 def __init__(self, left, right): super(MultiplicationOperator, self).__init__(left, right) - + def __getitem__(self, index): - left = self.arguments[0] if is_constant(self.arguments[0]) else self.arguments[0][index] - right = self.arguments[1] if is_constant(self.arguments[1]) else self.arguments[1][index] + left = self.arguments[0] if is_constant( + self.arguments[0]) else self.arguments[0][index] + right = self.arguments[1] if is_constant( + self.arguments[1]) else self.arguments[1][index] return left * right -#=================================================================================================== +#========================================================================= # DivisionOperator -#=================================================================================================== +#========================================================================= class DivisionOperator(Operator): - key = '-' + key = '/' numargs = 2 def __init__(self, left, right): super(DivisionOperator, self).__init__(left, right) - + def __getitem__(self, index): - left = self.arguments[0] if is_constant(self.arguments[0]) else self.arguments[0][index] - right = self.arguments[1] if is_constant(self.arguments[1]) else self.arguments[1][index] + left = self.arguments[0] if is_constant( + self.arguments[0]) else self.arguments[0][index] + right = self.arguments[1] if is_constant( + self.arguments[1]) else self.arguments[1][index] return left / right -#=================================================================================================== +#========================================================================= # Operator map - Fixed to prevent user-redefinition! -#=================================================================================================== +#========================================================================= __OPERATORS__ = {'-': {1: NegationOperator, 2: SubtractionOperator}, '**': {2: PowerOperator}, @@ -203,20 +221,22 @@ def __getitem__(self, index): '*': {2: MultiplicationOperator}, '/': {2: DivisionOperator}} -#################################################################################################### -##### FUNCTIONS #################################################################################### -#################################################################################################### +########################################################################## +##### FUNCTIONS ########################################################## +########################################################################## -#=================================================================================================== +#========================================================================= # Recursively return all subclasses of a given class -#=================================================================================================== +#========================================================================= + + def _all_subclasses_(cls): return cls.__subclasses__() + [c for s in cls.__subclasses__() for c in _all_subclasses_(s)] -#=================================================================================================== +#========================================================================= # Get the function associated with the given key-symbol -#=================================================================================================== +#========================================================================= def find_function(key): func = None for c in _all_subclasses_(Function): @@ -224,44 +244,45 @@ def find_function(key): if func is None: func = c else: - raise RuntimeError('Function {!r} is multiply defined'.format(key)) + raise RuntimeError( + 'Function {!r} is multiply defined'.format(key)) if func is None: raise KeyError('Function {!r} not found'.format(key)) else: return func - -#=================================================================================================== + +#========================================================================= # list_functions -#=================================================================================================== +#========================================================================= def list_functions(): return [c.key for c in _all_subclasses_(Function)] -#=================================================================================================== +#========================================================================= # Function - From which all 'func(...)'-pattern functions derive -#=================================================================================================== +#========================================================================= class Function(FunctionBase): key = 'func' - + def __init__(self, *args, **kwds): super(Function, self).__init__(*args, **kwds) self._sumlike_dimensions = set() - + def __getitem__(self, _): return None - + @property def sumlike_dimensions(self): return self._sumlike_dimensions - + def add_sumlike_dimensions(self, *dims): self._sumlike_dimensions.update(set(dims)) -#=================================================================================================== +#========================================================================= # SquareRoot -#=================================================================================================== +#========================================================================= class SquareRootFunction(Function): key = 'sqrt' @@ -272,7 +293,8 @@ def __init__(self, data): try: units = data_info.units.root(2) except: - raise UnitsError('sqrt: Cannot take square-root of units {!r}'.format(data.units)) + raise UnitsError( + 'sqrt: Cannot take square-root of units {!r}'.format(data.units)) self._units = units else: self._units = None @@ -287,9 +309,9 @@ def __getitem__(self, index): return sqrt(data) -#=================================================================================================== +#========================================================================= # MeanFunction -#=================================================================================================== +#========================================================================= class MeanFunction(Function): key = 'mean' @@ -301,7 +323,7 @@ def __init__(self, data, *dimensions): raise TypeError('mean: Data must be a PhysArray') if not all(isinstance(d, basestring) for d in dimensions): raise TypeError('mean: Dimensions must be strings') - + def __getitem__(self, index): data = self.arguments[0][index] dimensions = self.arguments[1:] @@ -309,45 +331,131 @@ def __getitem__(self, index): return data.mean(dimensions=indims) -#=================================================================================================== +#========================================================================= +# SumFunction +#========================================================================= +class SumFunction(Function): + key = 'sum' + + def __init__(self, data, *dimensions): + super(SumFunction, self).__init__(data, *dimensions) + self.add_sumlike_dimensions(*dimensions) + data_info = data if is_constant(data) else data[None] + if not isinstance(data_info, PhysArray): + raise TypeError('sum: Data must be a PhysArray') + if not all(isinstance(d, basestring) for d in dimensions): + raise TypeError('sum: Dimensions must be strings') + + def __getitem__(self, index): + data = self.arguments[0][index] + dimensions = self.arguments[1:] + indims = [] + for d in dimensions: + #print d, 'in', data.dimensions, '?' + if d in data.dimensions: + #print 'will append ', data.dimensions.index(d) + indims.append(data.dimensions.index(d)) + return np.sum(data, indims[0]) + + +#========================================================================= +# MinFunction +#========================================================================= +class MinFunction(Function): + key = 'min' + + def __init__(self, data, *dimensions): + super(MinFunction, self).__init__(data, *dimensions) + self.add_sumlike_dimensions(*dimensions) + data_info = data if is_constant(data) else data[None] + if not isinstance(data_info, PhysArray): + raise TypeError('min: Data must be a PhysArray') + if not all(isinstance(d, basestring) for d in dimensions): + raise TypeError('min: Dimensions must be strings') + + def __getitem__(self, index): + data = self.arguments[0][index] + dimensions = self.arguments[1:] + indims = [] + if index is None: + return PhysArray(np.zeros((0, 0, 0)), dimensions=[data.dimensions[0], data.dimensions[2], data.dimensions[3]]) + for d in dimensions: + if d in data.dimensions: + indims.append(data.dimensions.index(d)) + new_name = 'min({},{})'.format(data.name, dimensions) + m = np.amin(data, axis=indims[0]) + return PhysArray(m, name=new_name, positive=data.positive, units=data.units, dimensions=[data.dimensions[0], data.dimensions[2], data.dimensions[3]]) + + +#========================================================================= +# MaxFunction +#========================================================================= +class MaxFunction(Function): + key = 'max' + + def __init__(self, data, *dimensions): + super(MaxFunction, self).__init__(data, *dimensions) + self.add_sumlike_dimensions(*dimensions) + data_info = data if is_constant(data) else data[None] + if not isinstance(data_info, PhysArray): + raise TypeError('max: Data must be a PhysArray') + if not all(isinstance(d, basestring) for d in dimensions): + raise TypeError('max: Dimensions must be strings') + + def __getitem__(self, index): + data = self.arguments[0][index] + dimensions = self.arguments[1:] + indims = [] + if index is None: + return PhysArray(np.zeros((0, 0, 0)), units=data.units, dimensions=[data.dimensions[0], data.dimensions[2], data.dimensions[3]]) + for d in dimensions: + if d in data.dimensions: + indims.append(data.dimensions.index(d)) + new_name = 'max({},{})'.format(data.name, dimensions[0]) + m = np.amax(data, axis=indims[0]) + return PhysArray(m, name=new_name, positive=data.positive, units=data.units, dimensions=[data.dimensions[0], data.dimensions[2], data.dimensions[3]]) + + +#========================================================================= # PositiveUpFunction -#=================================================================================================== +#========================================================================= class PositiveUpFunction(Function): key = 'up' def __init__(self, data): super(PositiveUpFunction, self).__init__(data) - + def __getitem__(self, index): data_r = self.arguments[0] data = data_r if is_constant(data_r) else data_r[index] return PhysArray(data).up() -#=================================================================================================== +#========================================================================= # PositiveDownFunction -#=================================================================================================== +#========================================================================= class PositiveDownFunction(Function): key = 'down' def __init__(self, data): super(PositiveDownFunction, self).__init__(data) - + def __getitem__(self, index): data_r = self.arguments[0] data = data_r if is_constant(data_r) else data_r[index] return PhysArray(data).down() -#=================================================================================================== +#========================================================================= # ChangeUnitsFunction -#=================================================================================================== +#========================================================================= class ChangeUnitsFunction(Function): key = 'chunits' def __init__(self, data, units=None, refdate=None, calendar=None): - super(ChangeUnitsFunction, self).__init__(data, units=units, refdate=refdate, calendar=calendar) - + super(ChangeUnitsFunction, self).__init__( + data, units=units, refdate=refdate, calendar=calendar) + dunits = Unit(1) if is_constant(data) else data[None].units dcal = dunits.calendar if dunits.is_time_reference(): @@ -355,7 +463,7 @@ def __init__(self, data, units=None, refdate=None, calendar=None): else: dunit = dunits.origin dref = None - + uobj = Unit(units) if is_constant(units) else units[None].units ucal = uobj.calendar if uobj.is_time_reference(): @@ -363,15 +471,16 @@ def __init__(self, data, units=None, refdate=None, calendar=None): else: uunit = uobj.origin uref = None - + unit = dunit if units is None else uunit - + if isinstance(refdate, basestring): ref = refdate elif refdate is None: ref = dref if uref is None else uref else: - raise ValueError('chunits: Reference date must be a string, if given') + raise ValueError( + 'chunits: Reference date must be a string, if given') if isinstance(calendar, basestring): cal = calendar @@ -379,44 +488,48 @@ def __init__(self, data, units=None, refdate=None, calendar=None): cal = dcal if ucal is None else ucal else: raise ValueError('chunits: Calendar must be a string, if given') - + if ref is None: self._newunits = Unit(unit, calendar=cal) else: - self._newunits = Unit('{} since {}'.format(unit, ref), calendar=cal) - + self._newunits = Unit( + '{} since {}'.format(unit, ref), calendar=cal) + def __getitem__(self, index): - data = self.arguments[0] if is_constant(self.arguments[0]) else self.arguments[0][index] - cal_str = '' if self._newunits.calendar is None else '|{}'.format(self._newunits.calendar) + data = self.arguments[0] if is_constant( + self.arguments[0]) else self.arguments[0][index] + cal_str = '' if self._newunits.calendar is None else '|{}'.format( + self._newunits.calendar) unit_str = '{}{}'.format(self._newunits, cal_str) new_name = 'chunits({}, units={})'.format(data.name, unit_str) return PhysArray(data, name=new_name, units=self._newunits) -#=================================================================================================== +#========================================================================= # LimitFunction -#=================================================================================================== +#========================================================================= class LimitFunction(Function): key = 'limit' def __init__(self, data, below=None, above=None): super(LimitFunction, self).__init__(data, below=below, above=above) - + def __getitem__(self, index): - data = self.arguments[0] if is_constant(self.arguments[0]) else self.arguments[0][index] + data = self.arguments[0] if is_constant( + self.arguments[0]) else self.arguments[0][index] above_val = self.keywords['above'] below_val = self.keywords['below'] if above_val is None and below_val is None: return data - + above_str = '' if above_val is not None: above_ind = where(data > above_val) if len(above_ind) > 0: data[above_ind] = above_val above_str = ', above={}'.format(above_val) - + below_str = '' if below_val is not None: below_ind = where(data < below_val) diff --git a/source/pyconform/miptableparser.py b/source/pyconform/miptableparser.py index 8f2eabd4..11359ef3 100644 --- a/source/pyconform/miptableparser.py +++ b/source/pyconform/miptableparser.py @@ -378,6 +378,8 @@ def parse_table(self,exp,mips,tables,v_list,table_var_fields,table_axes_fields,t return {} activity_id = dq.inx.uid[e_id[0]].mip e_vars = dq.inx.iref_by_sect[e_id[0]].a + if len(e_vars['requestItem']) == 0: + e_vars = dq.inx.iref_by_sect[dq.inx.uid[e_id[0]].egid].a total_request = {} for ri in e_vars['requestItem']: @@ -418,7 +420,7 @@ def parse_table(self,exp,mips,tables,v_list,table_var_fields,table_axes_fields,t if hasattr(c_var,'mipTable'): var['mipTable']=c_var.mipTable if c_var.mipTable in tables or '--ALL--' in tables: - var["_FillValue"] = "9.96921e+36" + var["_FillValue"] = "1e+20" if hasattr(c_var,'deflate'): var['deflate']= c_var.deflate if hasattr(c_var,'deflate_level'): @@ -495,6 +497,17 @@ def parse_table(self,exp,mips,tables,v_list,table_var_fields,table_axes_fields,t if hasattr(t_var,'title'): var['time_title'] = t_var.title + # Is there did? + if hasattr(s_var, 'dids'): + if isinstance(s_var.dids, tuple): + extra_dim = dq.inx.uid[s_var.dids[0]].label + if 'coordinates' not in var.keys(): + var['coordinates'] = extra_dim + else: + var['coordinates'] = extra_dim + "|"+ var['coordinates'] + if extra_dim not in axes_list: + axes_list.append(extra_dim) + # Set what we can from the spatial section if hasattr(s_var, 'spid'): sp_var = dq.inx.uid[s_var.spid] @@ -508,6 +521,8 @@ def parse_table(self,exp,mips,tables,v_list,table_var_fields,table_axes_fields,t var['coordinates'] = sp_var_d + '|' + var['coordinates'] else: var['coordinates'] = sp_var.dimensions + if 'grid_latitude' in var['coordinates']: + var['coordinates'] = var['coordinates'].replace('grid_latitude','gridlatitude') dims = var['coordinates'].split('|') for d in dims: if d not in axes_list and d != '' and d != 'None': @@ -534,9 +549,9 @@ def parse_table(self,exp,mips,tables,v_list,table_var_fields,table_axes_fields,t # Add variable to variable dictionary variables[c_var.label] = var - for a in axes_list: - #print a + if 'grid_latitude' in a: + a = 'gridlatitude' if a in dq.inx.grids.label.keys(): id = dq.inx.grids.label[a] if len(id) > 0: @@ -563,7 +578,10 @@ def parse_table(self,exp,mips,tables,v_list,table_var_fields,table_axes_fields,t else: ax['standard_name'] = a if hasattr(v,'type'): - ax['type'] = v.type + if 'landUse' in a: + ax['type'] = 'int' + else: + ax['type'] = v.type if hasattr(v,'id'): ax['id'] = v.label if hasattr(v,'positive'): diff --git a/source/pyconform/modules/CLM_landunit_to_CMIP6_Lut.py b/source/pyconform/modules/CLM_landunit_to_CMIP6_Lut.py new file mode 100644 index 00000000..5044a122 --- /dev/null +++ b/source/pyconform/modules/CLM_landunit_to_CMIP6_Lut.py @@ -0,0 +1,173 @@ +#! /usr/bin/env python + + +import time, sys +import numpy as np +from pyconform.physarray import PhysArray, UnitsError, DimensionsError +from pyconform.functions import Function, is_constant + + +class CLM_landunit_to_CMIP6_Lut_Function(Function): + key = 'CLM_landunit_to_CMIP6_Lut' + + def __init__(self, EFLX_LH_TOT, vegType, ntim, nlat, nlon, grid1d_ixy, grid1d_jxy, grid1d_lon, + grid1d_lat, land1d_lon, land1d_lat, land1d_ityplunit, + land1d_active, land1d_wtgcell, landUse): + + super(CLM_landunit_to_CMIP6_Lut_Function, self).__init__(EFLX_LH_TOT, vegType, ntim, nlat, nlon, grid1d_ixy, grid1d_jxy, grid1d_lon, + grid1d_lat, land1d_lon, land1d_lat, land1d_ityplunit, + land1d_active, land1d_wtgcell, landUse) + + + def __getitem__(self, index): + + pEFLX_LH_TOT = self.arguments[0][index] + vegType = self.arguments[1] + pntim = self.arguments[2][index] + pnlat = self.arguments[3][index] + pnlon = self.arguments[4][index] + pgrid1d_ixy = self.arguments[5][index] + pgrid1d_jxy = self.arguments[6][index] + pgrid1d_lon = self.arguments[7][index] + pgrid1d_lat = self.arguments[8][index] + pland1d_lon = self.arguments[9][index] + pland1d_lat = self.arguments[10][index] + pland1d_ityplunit = self.arguments[11][index] + pland1d_active = self.arguments[12][index] + pland1d_wtgcell = self.arguments[13][index] + landUse = self.arguments[14][index] + + if index is None: + if 'all' in vegType: + return PhysArray(np.zeros((0,4,0,0)), dimensions=[pntim.dimensions[0],landUse.dimensions[0],pnlat.dimensions[0],pnlon.dimensions[0]]) + else: + return PhysArray(np.zeros((0,0,0)), dimensions=[pntim.dimensions[0],pnlat.dimensions[0],pnlon.dimensions[0]]) + + EFLX_LH_TOT = pEFLX_LH_TOT.data + ntim = pntim.data + nlat = pnlat.data + nlon = pnlon.data + grid1d_ixy = pgrid1d_ixy.data + grid1d_jxy = pgrid1d_jxy.data + grid1d_lon = pgrid1d_lon.data + grid1d_lat = pgrid1d_lat.data + land1d_lon = pland1d_lon.data + land1d_lat = pland1d_lat.data + land1d_ityplunit = pland1d_ityplunit.data + land1d_active = pland1d_active.data + land1d_wtgcell = pland1d_wtgcell.data + + missing = 1e+20 + + long_name = "latent heat flux on land use tile (lut=0:natveg, =1:pasture, =2:crop, =3:urban)" + nlut = 4 + veg = 0 + pasture = 1 + crop = 2 + urban = 3 + + # Tolerance check for weights summing to 1 + eps = 1.e-5 + + # Will contain landunit variables for veg, crop, pasture, and urban on 2d grid + #varo_lut_temp = np.full([len(ntim),4,len(nlat),len(nlon)],fill_value=missing) + #varo_lut = np.ma.masked_values(varo_lut_temp, missing) + varo_lut = np.zeros([len(ntim),4,len(nlat),len(nlon)]) + # Set pasture to fill value + varo_lut[:,pasture,:,:] = 1e+20 + + # If 1, landunit is active + active_lunit = 1 + # If 1, landunit is veg + veg_lunit = 1 + # If 2, landunit is crop + crop_lunit = 2 + # If 7,8, or 9, landunit is urban + beg_urban_lunit = 7 + end_urban_lunit = 9 + + # Set up numpy array to compare against + t = np.stack((land1d_lon,land1d_lat,land1d_active,land1d_ityplunit), axis=1) + tu = np.stack((land1d_lon,land1d_lat,land1d_active), axis=1) + + ind = np.stack((grid1d_ixy,grid1d_jxy), axis=1) + + # Loop over lat/lons + for ixy in range(len(nlon)): + for jxy in range(len(nlat)): + + grid_indx = -99 + # 1d grid index + ind_comp = (ixy+1,jxy+1) + gi = np.where(np.all(ind==ind_comp, axis=1))[0] + if len(gi) > 0: + grid_indx = gi[0] + + landunit_indx_veg = 0.0 + landunit_indx_crop = 0.0 + landunit_indx_urban = 0.0 + # Check for valid land gridcell + if grid_indx != -99: + + # Gridcell lat/lons + grid1d_lon_pt = grid1d_lon[grid_indx] + grid1d_lat_pt = grid1d_lat[grid_indx] + + # veg landunit index for this gridcell + t_var = (grid1d_lon_pt, grid1d_lat_pt, active_lunit, veg_lunit) + landunit_indx_veg = np.where(np.all(t_var == t, axis=1) * (land1d_wtgcell>0))[0] + + # crop landunit index for this gridcell + t_var = (grid1d_lon_pt, grid1d_lat_pt, active_lunit, crop_lunit) + landunit_indx_crop = np.where(np.all(t_var == t, axis=1) * (land1d_wtgcell>0))[0] + + # urban landunit indices for this gridcell + t_var = (grid1d_lon_pt, grid1d_lat_pt, active_lunit) + landunit_indx_urban = np.where( np.all(t_var == tu, axis=1) * (land1d_ityplunit>=beg_urban_lunit) * (land1d_ityplunit<=end_urban_lunit) * (land1d_wtgcell>0))[0] + + # Check for valid veg landunit + if landunit_indx_veg.size > 0: + varo_lut[:,veg,jxy,ixy] = EFLX_LH_TOT[:,landunit_indx_veg].squeeze() + else: + varo_lut[:,veg,jxy,ixy] = 1e+20 + + # Check for valid crop landunit + if landunit_indx_crop.size > 0: + varo_lut[:,crop,jxy,ixy] = EFLX_LH_TOT[:,landunit_indx_crop].squeeze() + else: + varo_lut[:,crop,jxy,ixy] = 1e+20 + + # Check for valid urban landunit and compute weighted-average + if landunit_indx_urban.size > 0: + dum = EFLX_LH_TOT[:,landunit_indx_urban].squeeze() + land1d_wtgcell_pts = (land1d_wtgcell[landunit_indx_urban]).astype(np.float32) + weights = land1d_wtgcell_pts / np.sum(land1d_wtgcell_pts) + if (np.absolute(1. - np.sum(weights)) > eps): + print ("Weights do not sum to 1, exiting") + sys.exit(-1) + varo_lut[:,urban,jxy,ixy] = np.sum(dum * weights) + else: + varo_lut[:,urban,jxy,ixy] = 1e+20 + else: + varo_lut[:,:,jxy,ixy] = 1e+20 + + new_name = 'CLM_landunit_to_CMIP6_Lut({}{}{}{}{}{}{}{}{}{}{}{}{})'.format(pEFLX_LH_TOT.name, + pntim.name, pnlat.name, pnlon.name, pgrid1d_ixy.name, pgrid1d_jxy.name, pgrid1d_lon.name, + pgrid1d_lat.name, pland1d_lon.name, pland1d_lat.name, pland1d_ityplunit.name, + pland1d_active.name, pland1d_wtgcell.name) + + varo_lut[varo_lut>=1e+15] = 1e+20 + mvaro_lut = np.ma.masked_values(varo_lut, 1e+20) + + if 'crop' in vegType: + return PhysArray(mvaro_lut[:,crop,:,:], name=new_name, units=pEFLX_LH_TOT.units) + elif 'veg' in vegType: + return PhysArray(mvaro_lut[:,veg,:,:], name=new_name, units=pEFLX_LH_TOT.units) + elif 'urban' in vegType: + return PhysArray(mvaro_lut[:,urban,:,:], name=new_name, units=pEFLX_LH_TOT.units) + elif 'pasture' in vegType: + return PhysArray(mvaro_lut[:,pasture,:,:], name=new_name, units=pEFLX_LH_TOT.units) + elif 'nlut' in vegType: + return PhysArray(mvaro_lut[:,nlut,:,:], name=new_name, units=pEFLX_LH_TOT.units) + elif 'all' in vegType: + return PhysArray(mvaro_lut, name=new_name, units=pEFLX_LH_TOT.units) diff --git a/source/pyconform/modules/CLM_pft_to_CMIP6_vegtype.py b/source/pyconform/modules/CLM_pft_to_CMIP6_vegtype.py new file mode 100644 index 00000000..e62e8f9d --- /dev/null +++ b/source/pyconform/modules/CLM_pft_to_CMIP6_vegtype.py @@ -0,0 +1,256 @@ +#! /usr/bin/env python + +import time, sys +import numpy as np +from pyconform.physarray import PhysArray, UnitsError, DimensionsError +from pyconform.functions import Function, is_constant + + +class CLM_pft_to_CMIP6_vegtype_Function(Function): + key = 'CLM_pft_to_CMIP6_vegtype' + numargs = 18 + + def __init__(self, GPP, vegType, time, lat, lon, grid1d_ixy, grid1d_jxy, grid1d_lon, + grid1d_lat, land1d_lon, land1d_lat, land1d_ityplunit, + pfts1d_lon, pfts1d_lat, pfts1d_active, pfts1d_itype_veg, + pfts1d_wtgcell, pfts1d_wtlunit): + + super(CLM_pft_to_CMIP6_vegtype_Function, self).__init__(GPP, vegType, time, lat, lon, grid1d_ixy, grid1d_jxy, grid1d_lon, + grid1d_lat, land1d_lon, land1d_lat, land1d_ityplunit, + pfts1d_lon, pfts1d_lat, pfts1d_active, pfts1d_itype_veg, + pfts1d_wtgcell, pfts1d_wtlunit) + + def __getitem__(self, index): + + + pGPP = self.arguments[0][index] + # vegType = grass, shrub, or tree + vegType = self.arguments[1] + ptime = self.arguments[2][index] + plat = self.arguments[3][index] + plon = self.arguments[4][index] + pgrid1d_ixy = self.arguments[5][index] + pgrid1d_jxy = self.arguments[6][index] + pgrid1d_lon = self.arguments[7][index] + pgrid1d_lat = self.arguments[8][index] + pland1d_lon = self.arguments[9][index] + pland1d_lat = self.arguments[10][index] + pland1d_ityplunit = self.arguments[11][index] + ppfts1d_lon = self.arguments[12][index] + ppfts1d_lat = self.arguments[13][index] + ppfts1d_active = self.arguments[14][index] + ppfts1d_itype_veg = self.arguments[15][index] + ppfts1d_wtgcell = self.arguments[16][index] + ppfts1d_wtlunit = self.arguments[17][index] + + if index is None: + return PhysArray(np.zeros((0,0,0)), dimensions=[ptime.dimensions[0],plat.dimensions[0],plon.dimensions[0]]) + + GPP = pGPP.data + time = ptime.data + lat = plat.data + lon = plon.data + grid1d_ixy = pgrid1d_ixy.data + grid1d_jxy = pgrid1d_jxy.data + grid1d_lon = pgrid1d_lon.data + grid1d_lat = pgrid1d_lat.data + land1d_lon = pland1d_lon.data + land1d_lat = pland1d_lat.data + land1d_ityplunit = pland1d_ityplunit.data + pfts1d_lon = ppfts1d_lon.data + pfts1d_lat = ppfts1d_lat.data + pfts1d_active = ppfts1d_active.data + pfts1d_itype_veg = ppfts1d_itype_veg.data + pfts1d_wtgcell = ppfts1d_wtgcell.data + pfts1d_wtlunit = ppfts1d_wtlunit.data + + + # vegType = grass, shrub, or tree + + # Tolerance check for weights summing to 1 + eps = 1.e-5 + + # If 1, pft is active + active_pft = 1 + + # If 1, landunit is veg + veg_lunit = 1 + + # C3 arctic grass, + # C3 non-arctic grass, + # C4 grass + beg_grass_pfts = 12 + end_grass_pfts = 14 + + # broadleaf evergreen shrub - temperate, + # broadleaf deciduous shrub - temperate, + # broadleaf deciduous shrub - boreal + beg_shrub_pfts = 9 + end_shrub_pfts = 11 + + # needleleaf evergreen tree - temperate, + # needleleaf evergreen tree - boreal, + # needleleaf deciduous tree - boreal, + # broadleaf evergreen tree - tropical, + # broadleaf evergreen tree - temperate, + # broadleaf deciduous tree - tropical, + # broadleaf deciduous tree - temperate, + # broadleaf deciduous tree - boreal + beg_tree_pfts = 1 + end_tree_pfts = 8 + + # Will contain weighted average for grass pfts on 2d grid + varo_vegType = np.zeros([len(time),len(lat),len(lon)]) + tu = np.stack((pfts1d_lon,pfts1d_lat, pfts1d_active), axis=1) + + ind = np.stack((grid1d_ixy,grid1d_jxy), axis=1) + + lu = np.stack((land1d_lon,land1d_lat,land1d_ityplunit), axis=1) + + # Loop over lat/lons + for ixy in range(len(lon)): + for jxy in range(len(lat)): + grid_indx = -99 + # 1d grid index + ind_comp = (ixy+1,jxy+1) + gi = np.where(np.all(ind==ind_comp, axis=1))[0] + if len(gi) > 0: + grid_indx = gi[0] + + # Check for valid land gridcell + if grid_indx != -99: + + # Gridcell lat/lons + grid1d_lon_pt = grid1d_lon[grid_indx] + grid1d_lat_pt = grid1d_lat[grid_indx] + + # veg landunit index for this gridcell + t_var = (grid1d_lon_pt, grid1d_lat_pt, veg_lunit) + landunit_indx = np.where(np.all(t_var == lu, axis=1))[0] + + # Check for valid veg landunit + if landunit_indx.size > 0: + if 'grass' in vegType: + t_var = (grid1d_lon_pt,grid1d_lat_pt, active_pft) + pft_indx = np.where( np.all(t_var == tu, axis=1) * (pfts1d_wtgcell > 0.) * (pfts1d_itype_veg >= beg_grass_pfts) * (pfts1d_itype_veg <= end_grass_pfts))[0] + elif 'shrub' in vegType: + t_var = (grid1d_lon_pt,grid1d_lat_pt, active_pft) + pft_indx = np.where( np.all(t_var==tu, axis=1) * (pfts1d_wtgcell > 0.) * (pfts1d_itype_veg >= beg_shrub_pfts) * (pfts1d_itype_veg <= end_shrub_pfts))[0] + elif 'tree' in vegType: + t_var = (grid1d_lon_pt,grid1d_lat_pt, active_pft) + pft_indx = np.where( np.all(t_var==tu, axis=1) * (pfts1d_wtgcell > 0.) * (pfts1d_itype_veg >= beg_tree_pfts) * (pfts1d_itype_veg <= end_tree_pfts))[0] + + # Check for valid pfts and compute weighted average + if pft_indx.size > 0: + if 'grass' in vegType: + pfts1d_wtlunit_grass = (pfts1d_wtlunit[pft_indx]).astype(np.float32) + dum = GPP[:,pft_indx] + weights = pfts1d_wtlunit_grass / np.sum(pfts1d_wtlunit_grass) + if np.absolute(1.-np.sum(weights)) > eps: + print("Weights do not sum to 1, exiting") + sys.exit(-1) + varo_vegType[:,jxy,ixy] = np.sum(dum * weights) + + elif 'shrub' in vegType: + pfts1d_wtlunit_shrub = (pfts1d_wtlunit[pft_indx]).astype(np.float32) + dum = GPP[:,pft_indx] + weights = pfts1d_wtlunit_shrub / np.sum(pfts1d_wtlunit_shrub) + varo_vegType[:,jxy,ixy] = np.sum(dum * weights) + + elif 'tree' in vegType: + pfts1d_wtlunit_tree = (pfts1d_wtlunit[pft_indx]).astype(np.float32) + dum = GPP[:,pft_indx] + weights = pfts1d_wtlunit_tree / np.sum(pfts1d_wtlunit_tree) + varo_vegType[:,jxy,ixy] = np.sum(dum * weights) + + else: + varo_vegType[:,jxy,ixy] = 1e+20 + else: + varo_vegType[:,jxy,ixy] = 1e+20 + else: + varo_vegType[:,jxy,ixy] = 1e+20 + + + new_name = 'CLM_pft_to_CMIP6_vegtype({}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{})'.format(pGPP.name, + vegType, ptime.name, plat.name, plon.name, pgrid1d_ixy.name, pgrid1d_jxy.name, pgrid1d_lon.name, + pgrid1d_lat.name, pland1d_lon.name, pland1d_lat.name, pland1d_ityplunit.name, + ppfts1d_lon.name, ppfts1d_lat.name, ppfts1d_active.name, ppfts1d_itype_veg.name, + ppfts1d_wtgcell.name, ppfts1d_wtlunit.name) + + print 'FINISHED FUNCTION' + + varo_vegType[varo_vegType>=1e+16] = 1e+20 + ma_varo_vegType = np.ma.masked_values(varo_vegType, 1e+20) + + return PhysArray(ma_varo_vegType, name=new_name, units=pGPP.units) + + +def main(argv=None): + + import netCDF4 as nc + + sim = "clm50_r243_1deg_GSWP3V2_cropopt_nsc_emergeV2F_dailyo_hist" + f_in = sim+".clm2.h1.2005-01.nc" + f_out = sim+".clm2.h1veg.0001-01.nc" + f_dir = "/glade2/scratch2/mickelso/CMIP6_LND_SCRIPTS/DATA/" + f_outfir = "/glade2/scratch2/mickelso/CMIP6_LND_SCRIPTS/new/OUTDIR/" + + cdf_file = nc.Dataset(f_dir+f_in,"r") + + ntim = cdf_file.variables['time'][:] + nlat = cdf_file.variables['lat'][:] + nlon = cdf_file.variables['lon'][:] + + grid1d_ixy = cdf_file.variables['grid1d_ixy'][:] + grid1d_jxy = cdf_file.variables['grid1d_jxy'][:] + grid1d_lon = cdf_file.variables['grid1d_lon'][:] + grid1d_lat = cdf_file.variables['grid1d_lat'][:] + land1d_lon = cdf_file.variables['land1d_lon'][:] + land1d_lat = cdf_file.variables['land1d_lat'][:] + land1d_ityplunit = cdf_file.variables['land1d_ityplunit'][:] + pfts1d_lon = cdf_file.variables['pfts1d_lon'][:] + pfts1d_lat = cdf_file.variables['pfts1d_lat'][:] + pfts1d_active = cdf_file.variables['pfts1d_active'][:] + pfts1d_itype_veg = cdf_file.variables['pfts1d_itype_veg'][:] + pfts1d_wtgcell = cdf_file.variables['pfts1d_wtgcell'][:] + pfts1d_wtlunit = cdf_file.variables['pfts1d_wtlunit'][:] + + + GPP = cdf_file.variables['GPP'][:] + + cdf_file.close() + + out_file = nc.Dataset(f_outfir+f_out,"w") + + time = out_file.createDimension('time',None) + lat = out_file.createDimension('lat',len(nlat)) + lon = out_file.createDimension('lon',len(nlon)) + gppGrass = out_file.createVariable('gppGrass', 'f4', ('time', 'lat', 'lon'),fill_value=1.e36) + gppShrub = out_file.createVariable('gppShrub', 'f4', ('time', 'lat', 'lon'),fill_value=1.e36) + gppTree = out_file.createVariable('gppTree', 'f4', ('time', 'lat', 'lon'),fill_value=1.e36) + + print 'Looking for grass' + gppGrass[:] = CLM_pft_to_CMIP6_vegtype(GPP, 'grass', ntim, nlat, nlon, grid1d_ixy, grid1d_jxy, grid1d_lon, + grid1d_lat, land1d_lon, land1d_lat, land1d_ityplunit, + pfts1d_lon, pfts1d_lat, pfts1d_active, pfts1d_itype_veg, + pfts1d_wtgcell, pfts1d_wtlunit) + print 'Looking for shrubs' + gppShrub[:] = CLM_pft_to_CMIP6_vegtype(GPP, 'shrub', ntim, nlat, nlon, grid1d_ixy, grid1d_jxy, grid1d_lon, + grid1d_lat, land1d_lon, land1d_lat, land1d_ityplunit, + pfts1d_lon, pfts1d_lat, pfts1d_active, pfts1d_itype_veg, + pfts1d_wtgcell, pfts1d_wtlunit) + print 'Looking for trees' + gppTree[:] = CLM_pft_to_CMIP6_vegtype(GPP, 'tree', ntim, nlat, nlon, grid1d_ixy, grid1d_jxy, grid1d_lon, + grid1d_lat, land1d_lon, land1d_lat, land1d_ityplunit, + pfts1d_lon, pfts1d_lat, pfts1d_active, pfts1d_itype_veg, + pfts1d_wtgcell, pfts1d_wtlunit) + + + out_file.close() + +if __name__ == '__main__': + main() + + + + diff --git a/source/pyconform/modules/commonfunctions.py b/source/pyconform/modules/commonfunctions.py index 1aa9dfc7..029d3566 100644 --- a/source/pyconform/modules/commonfunctions.py +++ b/source/pyconform/modules/commonfunctions.py @@ -3,7 +3,27 @@ from pyconform.physarray import PhysArray, UnitsError, DimensionsError from pyconform.functions import Function, is_constant from cf_units import Unit -from numpy import diff, empty +from numpy import diff, empty, mean +import numpy as np + +#=================================================================================================== +# ZonalMeanFunction +#=================================================================================================== +class ZonalMeanFunction(Function): + key = 'zonalmean' + + def __init__(self, data): + super(ZonalMeanFunction, self).__init__(data) + data_info = data if is_constant(data) else data[None] + if not isinstance(data_info, PhysArray): + raise TypeError('mean: Data must be a PhysArray') + + def __getitem__(self, index): + data = self.arguments[0][index] + m = mean(data, axis=3) + return m + #return mean(data, axis=3) + #======================================================================================================================= # BoundsFunction @@ -94,4 +114,247 @@ def __getitem__(self, index): return new_data +#=================================================================================================== +# AgeofAirFunction +#=================================================================================================== +class AgeofAirFunction(Function): + key = 'ageofair' + + def __init__(self, spc_zm,date,time,lat,lev): + super(AgeofAirFunction, self).__init__(spc_zm,date,time,lat,lev) + + def __getitem__(self, index): + p_spc_zm = self.arguments[0][index] + p_date = self.arguments[1][index] + p_time = self.arguments[2][index] + p_lat = self.arguments[3][index] + p_lev = self.arguments[4][index] + + if index is None: + return PhysArray(np.zeros((0,0,0)), dimensions=[p_time.dimensions[0],p_lev.dimensions[0],p_lat.dimensions[0]]) + + spc_zm = p_spc_zm.data + date = p_date.data + time = p_time.data + lat = p_lat.data + lev = p_lev.data + + a = np.zeros((len(time),len(lev),len(lat))) + + # Unpack month and year. Adjust to compensate for the output convention in h0 files + year = date/10000 + month = (date/100 % 100) + day = date - 10000*year - 100*month + + month = month - 1 + for m in range(len(month)): + if month[m] == 12: + year[m] = year[m]-1 + month[m] = 0 + + timeyr = year + (month-0.5)/12. + + spc_ref = spc_zm[:,0,0] + for iy in range(len(lat)): + for iz in range(len(lev)): + spc_local = spc_zm[:,iz,iy] + time0 = np.interp(spc_local,spc_ref,timeyr) + a[:,iz,iy] = timeyr - time0 + + + new_name = 'ageofair({}{}{}{}{})'.format(p_spc_zm.name,p_date.name,p_time.name,p_lat.name,p_lev.name) + + return PhysArray(a, name = new_name, units="yr") + + +#=================================================================================================== +# yeartomonth_dataFunction +#=================================================================================================== +class YeartoMonth_dataFunction(Function): + key = 'yeartomonth_data' + + def __init__(self, data, time, lat, lon): + super(YeartoMonth_dataFunction, self).__init__(data, time, lat, lon) + + def __getitem__(self, index): + p_data = self.arguments[0][index] + p_time = self.arguments[1][index] + p_lat = self.arguments[2][index] + p_lon = self.arguments[3][index] + + if index is None: + return PhysArray(np.zeros((0,0,0)), dimensions=[p_time.dimensions[0],p_lat.dimensions[0],p_lon.dimensions[0]]) + + data = p_data.data + time = p_time.data + lat = p_lat.data + lon = p_lon.data + + a = np.zeros((len(time)*12,len(lat),len(lon))) + for i in range(len(time)): + for j in range(12): + a[((i*12)+j),:,:] = data[i,:,:] + + new_name = 'yeartomonth_data({}{}{}{})'.format(p_data.name, p_time.name, p_lat.name, p_lon.name) + + return PhysArray(a, name = new_name, units=p_data.units) + +#=================================================================================================== +# yeartomonth_timeFunction +#=================================================================================================== +class YeartoMonth_timeFunction(Function): + key = 'yeartomonth_time' + + def __init__(self, time): + super(YeartoMonth_timeFunction, self).__init__(time) + + def __getitem__(self, index): + p_time = self.arguments[0][index] + + if index is None: + return PhysArray(np.zeros((0)), dimensions=[p_time.dimensions[0]], units=p_time.units, calendar='noleap') + + time = p_time.data + monLens = [31.0,28.0,31.0,30.0,31.0,30.0,31.0,31.0,30.0,31.0,30.0,31.0] + + a = np.zeros((len(time)*12)) + for i in range(len(time)): + prev = 0 + for j in range(12): + a[((i*12)+j)] = float((time[i]-365)+prev+float(monLens[j]/2.0)) + prev += monLens[j] + + new_name = 'yeartomonth_time({})'.format(p_time.name) + + return PhysArray(a, name = new_name, dimensions=[p_time.dimensions[0]], units=p_time.units, calendar='noleap') + + +#=================================================================================================== +# POP_bottom_layerFunction +#=================================================================================================== +class POP_bottom_layerFunction(Function): + key = 'POP_bottom_layer' + + def __init__(self, KMT, data): + super(POP_bottom_layerFunction, self).__init__(KMT, data) + + def __getitem__(self, index): + p_KMT = self.arguments[0][index] + p_data = self.arguments[1][index] + + if index is None: + return PhysArray(np.zeros((0,0,0)), dimensions=[p_data.dimensions[0],p_data.dimensions[2],p_data.dimensions[3]]) + + data = p_data.data + KMT = p_KMT.data + + a = np.zeros((p_data.shape[0],p_data.shape[2],p_data.shape[3])) + + for j in range(KMT.shape[0]): + for i in range(KMT.shape[1]): + a[:,j,i] = data[:,KMT[j,i]-1,j,i] + + new_name = 'POP_bottom_layer({}{})'.format( p_KMT.name, p_data.name) + + return PhysArray(a, name = new_name, units=p_data.units) + + +#=================================================================================================== +# masked_invalidFunction +#=================================================================================================== +class masked_invalidFunction(Function): + key = 'masked_invalid' + + def __init__(self, data): + super(masked_invalidFunction, self).__init__(data) + + def __getitem__(self, index): + p_data = self.arguments[0][index] + + if index is None: + return PhysArray(np.zeros((0,0,0)), dimensions=[p_data.dimensions[0],p_data.dimensions[1],p_data.dimensions[2]]) + + data = p_data.data + + a = np.ma.masked_invalid(data) + + new_name = 'masked_invalid({})'.format(p_data.name) + + return PhysArray(a, name = new_name, units=p_data.units) + + +#=================================================================================================== +# hemisphereFunction +#=================================================================================================== +class hemisphereFunction(Function): + key = 'hemisphere' + + def __init__(self, data, dim='dim', dr='dr'): + super(hemisphereFunction, self).__init__(data, dim=dim, dr=dr) + + def __getitem__(self, index): + p_data = self.arguments[0][index] + dim = self.keywords['dim'] + dr = self.keywords['dr'] + + data = p_data.data + + a = None + + # dim0? + if dim in p_data.dimensions[0]: + if ">" in dr: + return p_data[(data.shape[0]/2):data.shape[0],:,:] + elif "<" in dr: + return p_data[0:(data.shape[0]/2),:,:] + # dim1? + if dim in p_data.dimensions[1]: + if ">" in dr: + return p_data[:,(data.shape[1]/2):data.shape[1],:] + elif "<" in dr: + return p_data[:,0:(data.shape[1]/2),:] + # dim2? + if dim in p_data.dimensions[2]: + if ">" in dr: + return p_data[:,:,(data.shape[2]/2):data.shape[2]] + elif "<" in dr: + return p_data[:,:,0:(data.shape[2]/2)] + + +#=================================================================================================== +# cice_whereFunction +#=================================================================================================== +class cice_whereFunction(Function): + key = 'cice_where' + + # np.where(x < 5, x, -1) + + def __init__(self, a1, condition, a2, var, value): + super(cice_whereFunction, self).__init__(a1, condition, a2, var, value) + + def __getitem__(self, index): + a1 = self.arguments[0][index] + condition = self.arguments[1] + a2 = self.arguments[2] + var = self.arguments[3][index] + value = self.arguments[4] + + if index is None: + return PhysArray(a1.data, dimensions=[a1.dimensions[0],a1.dimensions[1],a1.dimensions[2]]) + + a = np.ma.zeros(a1.shape) + for t in range(a1.data.shape[0]): + if '>=' in condition: + a[t,:,:] = np.ma.where(a1[t,:,:] >= a2, var, value) + elif '<=' in condition: + a[t,:,:] = np.ma.where(a1[t,:,:] <= a2, var, value) + elif '==' in condition: + a[t,:,:] = np.ma.where(a1[t,:,:] == a2, var, value) + elif '<' in condition: + a[t,:,:] = np.ma.where(a1[t,:,:] < a2, var, value) + elif '>' in condition: + a[t,:,:] = np.ma.where(a1[t,:,:] > a2, var, value) + return PhysArray(a, dimensions=[a1.dimensions[0],a1.dimensions[1],a1.dimensions[2]], units=var.units) + + diff --git a/source/pyconform/modules/pnglfunctions.py b/source/pyconform/modules/pnglfunctions.py index 28091a3a..1231b549 100644 --- a/source/pyconform/modules/pnglfunctions.py +++ b/source/pyconform/modules/pnglfunctions.py @@ -69,6 +69,9 @@ def __getitem__(self, index): intyp = self.keywords['intyp'] ixtrp = self.keywords['ixtrp'] - return PhysArray(vinth2p(datai.data, hbcofa.data, hbcofb.data, plevo.data, - psfc.data, intyp, p0.data, 1, bool(ixtrp)), name=self._new_name, - dimensions=self._new_dims, units=datai.units, positive=datai.positive) + v = vinth2p(datai.data, hbcofa.data, hbcofb.data, plevo.data, + psfc.data, intyp, p0.data, 1, bool(ixtrp)) + + v[v==1e+30] = 1e+20 + + return PhysArray(v, name=self._new_name, dimensions=self._new_dims, units=datai.units, positive=datai.positive) diff --git a/source/pyconform/parsetab.py b/source/pyconform/parsetab.py new file mode 100644 index 00000000..feea1540 --- /dev/null +++ b/source/pyconform/parsetab.py @@ -0,0 +1,67 @@ + +# parsetab.py +# This file is automatically generated. Do not edit. +# pylint: disable=W,C,R +_tabversion = '3.10' + +_lr_method = 'LALR' + +_lr_signature = "leftEQleft<>LEQGEQleft+-left*/rightNEGPOSleftPOWEQ GEQ LEQ NAME POW STRING UFLOAT UINT\n array_like : UFLOAT\n array_like : UINT\n array_like : function\n array_like : variable\n \n array_like : '(' array_like ')'\n \n function : NAME '(' argument_list ',' keyword_dict ')'\n \n function : NAME '(' argument_list ')'\n \n function : NAME '(' keyword_dict ')'\n \n argument_list : argument_list ',' argument\n \n argument_list : argument\n argument_list : \n \n argument : array_like\n argument : STRING\n \n keyword_dict : keyword_dict ',' NAME '=' argument\n \n keyword_dict : NAME '=' argument\n \n variable : NAME '[' index_list ']'\n variable : NAME\n \n index_list : index_list ',' index\n \n index_list : index\n \n index : slice\n index : array_like\n \n slice : slice_argument ':' slice_argument ':' slice_argument\n slice : slice_argument ':' slice_argument\n \n slice_argument : array_like\n slice_argument : \n \n array_like : '-' array_like %prec NEG\n array_like : '+' array_like %prec POS\n \n array_like : array_like POW array_like\n array_like : array_like '-' array_like\n array_like : array_like '+' array_like\n array_like : array_like '*' array_like\n array_like : array_like '/' array_like\n array_like : array_like '<' array_like\n array_like : array_like '>' array_like\n array_like : array_like LEQ array_like\n array_like : array_like GEQ array_like\n array_like : array_like EQ array_like\n " + +_lr_action_items = {'GEQ':([1,5,6,7,8,9,20,21,22,25,26,27,28,29,30,31,32,33,34,35,37,41,45,47,49,54,60,62,],[10,-3,-2,-4,-1,-17,10,-27,-26,-36,-28,-30,-31,-29,-32,-35,10,-33,-34,-5,10,-17,10,-7,-8,-16,10,-6,]),')':([5,6,7,8,9,20,21,22,23,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,47,49,54,55,56,58,62,65,],[-3,-2,-4,-1,-17,35,-27,-26,-11,-36,-28,-30,-31,-29,-32,-35,-37,-33,-34,-5,-10,-12,47,49,-13,-17,-7,-8,-16,-9,62,-15,-6,-14,]),'(':([0,2,3,4,9,10,11,12,13,14,15,16,17,18,19,23,24,41,48,51,52,53,63,64,],[2,2,2,2,23,2,2,2,2,2,2,2,2,2,2,2,2,23,2,2,2,2,2,2,]),'+':([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,37,41,45,47,48,49,51,52,53,54,60,62,63,64,],[3,12,3,3,3,-3,-2,-4,-1,-17,3,3,3,3,3,3,3,3,3,3,12,-27,-26,3,3,12,-28,-30,-31,-29,-32,12,12,12,12,-5,12,-17,12,-7,3,-8,3,3,3,-16,12,-6,3,3,]),'*':([1,5,6,7,8,9,20,21,22,25,26,27,28,29,30,31,32,33,34,35,37,41,45,47,49,54,60,62,],[13,-3,-2,-4,-1,-17,13,-27,-26,13,-28,13,-31,13,-32,13,13,13,13,-5,13,-17,13,-7,-8,-16,13,-6,]),'-':([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,37,41,45,47,48,49,51,52,53,54,60,62,63,64,],[4,14,4,4,4,-3,-2,-4,-1,-17,4,4,4,4,4,4,4,4,4,4,14,-27,-26,4,4,14,-28,-30,-31,-29,-32,14,14,14,14,-5,14,-17,14,-7,4,-8,4,4,4,-16,14,-6,4,4,]),',':([5,6,7,8,9,21,22,23,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,44,45,46,47,49,52,54,55,56,58,59,60,61,62,64,65,66,],[-3,-2,-4,-1,-17,-27,-26,-11,-36,-28,-30,-31,-29,-32,-35,-37,-33,-34,-5,-10,-12,48,50,-13,-17,-19,-20,-21,53,-7,-8,-25,-16,-9,50,-15,-23,-24,-18,-6,-25,-14,-22,]),'/':([1,5,6,7,8,9,20,21,22,25,26,27,28,29,30,31,32,33,34,35,37,41,45,47,49,54,60,62,],[15,-3,-2,-4,-1,-17,15,-27,-26,15,-28,15,-31,15,-32,15,15,15,15,-5,15,-17,15,-7,-8,-16,15,-6,]),':':([5,6,7,8,9,21,22,24,25,26,27,28,29,30,31,32,33,34,35,43,45,47,49,52,53,54,59,60,62,],[-3,-2,-4,-1,-17,-27,-26,-25,-36,-28,-30,-31,-29,-32,-35,-37,-33,-34,-5,52,-24,-7,-8,-25,-25,-16,64,-24,-6,]),'=':([41,57,],[51,63,]),'<':([1,5,6,7,8,9,20,21,22,25,26,27,28,29,30,31,32,33,34,35,37,41,45,47,49,54,60,62,],[18,-3,-2,-4,-1,-17,18,-27,-26,-36,-28,-30,-31,-29,-32,-35,18,-33,-34,-5,18,-17,18,-7,-8,-16,18,-6,]),'$end':([1,5,6,7,8,9,21,22,25,26,27,28,29,30,31,32,33,34,35,47,49,54,62,],[0,-3,-2,-4,-1,-17,-27,-26,-36,-28,-30,-31,-29,-32,-35,-37,-33,-34,-5,-7,-8,-16,-6,]),'STRING':([23,48,51,63,],[40,40,40,40,]),'UINT':([0,2,3,4,10,11,12,13,14,15,16,17,18,19,23,24,48,51,52,53,63,64,],[6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,]),'[':([9,41,],[24,24,]),']':([5,6,7,8,9,21,22,25,26,27,28,29,30,31,32,33,34,35,42,44,45,46,47,49,52,54,59,60,61,62,64,66,],[-3,-2,-4,-1,-17,-27,-26,-36,-28,-30,-31,-29,-32,-35,-37,-33,-34,-5,-19,-20,-21,54,-7,-8,-25,-16,-23,-24,-18,-6,-25,-22,]),'UFLOAT':([0,2,3,4,10,11,12,13,14,15,16,17,18,19,23,24,48,51,52,53,63,64,],[8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,]),'EQ':([1,5,6,7,8,9,20,21,22,25,26,27,28,29,30,31,32,33,34,35,37,41,45,47,49,54,60,62,],[17,-3,-2,-4,-1,-17,17,-27,-26,-36,-28,-30,-31,-29,-32,-35,-37,-33,-34,-5,17,-17,17,-7,-8,-16,17,-6,]),'NAME':([0,2,3,4,10,11,12,13,14,15,16,17,18,19,23,24,48,50,51,52,53,63,64,],[9,9,9,9,9,9,9,9,9,9,9,9,9,9,41,9,41,57,9,9,9,9,9,]),'LEQ':([1,5,6,7,8,9,20,21,22,25,26,27,28,29,30,31,32,33,34,35,37,41,45,47,49,54,60,62,],[16,-3,-2,-4,-1,-17,16,-27,-26,-36,-28,-30,-31,-29,-32,-35,16,-33,-34,-5,16,-17,16,-7,-8,-16,16,-6,]),'POW':([1,5,6,7,8,9,20,21,22,25,26,27,28,29,30,31,32,33,34,35,37,41,45,47,49,54,60,62,],[11,-3,-2,-4,-1,-17,11,11,11,11,-28,11,11,11,11,11,11,11,11,-5,11,-17,11,-7,-8,-16,11,-6,]),'>':([1,5,6,7,8,9,20,21,22,25,26,27,28,29,30,31,32,33,34,35,37,41,45,47,49,54,60,62,],[19,-3,-2,-4,-1,-17,19,-27,-26,-36,-28,-30,-31,-29,-32,-35,19,-33,-34,-5,19,-17,19,-7,-8,-16,19,-6,]),} + +_lr_action = {} +for _k, _v in _lr_action_items.items(): + for _x,_y in zip(_v[0],_v[1]): + if not _x in _lr_action: _lr_action[_x] = {} + _lr_action[_x][_k] = _y +del _lr_action_items + +_lr_goto_items = {'function':([0,2,3,4,10,11,12,13,14,15,16,17,18,19,23,24,48,51,52,53,63,64,],[5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,]),'keyword_dict':([23,48,],[39,56,]),'slice_argument':([24,52,53,64,],[43,59,43,66,]),'slice':([24,53,],[44,44,]),'array_like':([0,2,3,4,10,11,12,13,14,15,16,17,18,19,23,24,48,51,52,53,63,64,],[1,20,21,22,25,26,27,28,29,30,31,32,33,34,37,45,37,37,60,45,37,60,]),'index':([24,53,],[42,61,]),'argument':([23,48,51,63,],[36,55,58,65,]),'index_list':([24,],[46,]),'argument_list':([23,],[38,]),'variable':([0,2,3,4,10,11,12,13,14,15,16,17,18,19,23,24,48,51,52,53,63,64,],[7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,]),} + +_lr_goto = {} +for _k, _v in _lr_goto_items.items(): + for _x, _y in zip(_v[0], _v[1]): + if not _x in _lr_goto: _lr_goto[_x] = {} + _lr_goto[_x][_k] = _y +del _lr_goto_items +_lr_productions = [ + ("S' -> array_like","S'",1,None,None,None), + ('array_like -> UFLOAT','array_like',1,'p_array_like','parsing.py',92), + ('array_like -> UINT','array_like',1,'p_array_like','parsing.py',93), + ('array_like -> function','array_like',1,'p_array_like','parsing.py',94), + ('array_like -> variable','array_like',1,'p_array_like','parsing.py',95), + ('array_like -> ( array_like )','array_like',3,'p_array_like_group','parsing.py',102), + ('function -> NAME ( argument_list , keyword_dict )','function',6,'p_function_with_arguments_and_keywords','parsing.py',109), + ('function -> NAME ( argument_list )','function',4,'p_function_with_arguments_only','parsing.py',116), + ('function -> NAME ( keyword_dict )','function',4,'p_function_with_keywords_only','parsing.py',123), + ('argument_list -> argument_list , argument','argument_list',3,'p_argument_list_append','parsing.py',130), + ('argument_list -> argument','argument_list',1,'p_single_item_argument_list','parsing.py',137), + ('argument_list -> ','argument_list',0,'p_single_item_argument_list','parsing.py',138), + ('argument -> array_like','argument',1,'p_argument','parsing.py',145), + ('argument -> STRING','argument',1,'p_argument','parsing.py',146), + ('keyword_dict -> keyword_dict , NAME = argument','keyword_dict',5,'p_keyword_dict_setitem','parsing.py',153), + ('keyword_dict -> NAME = argument','keyword_dict',3,'p_single_item_keyword_dict','parsing.py',161), + ('variable -> NAME [ index_list ]','variable',4,'p_variable','parsing.py',168), + ('variable -> NAME','variable',1,'p_variable','parsing.py',169), + ('index_list -> index_list , index','index_list',3,'p_index_list_append','parsing.py',177), + ('index_list -> index','index_list',1,'p_single_item_index_list','parsing.py',184), + ('index -> slice','index',1,'p_index','parsing.py',191), + ('index -> array_like','index',1,'p_index','parsing.py',192), + ('slice -> slice_argument : slice_argument : slice_argument','slice',5,'p_slice','parsing.py',199), + ('slice -> slice_argument : slice_argument','slice',3,'p_slice','parsing.py',200), + ('slice_argument -> array_like','slice_argument',1,'p_slice_argument','parsing.py',207), + ('slice_argument -> ','slice_argument',0,'p_slice_argument','parsing.py',208), + ('array_like -> - array_like','array_like',2,'p_expression_unary','parsing.py',215), + ('array_like -> + array_like','array_like',2,'p_expression_unary','parsing.py',216), + ('array_like -> array_like POW array_like','array_like',3,'p_expression_binary','parsing.py',229), + ('array_like -> array_like - array_like','array_like',3,'p_expression_binary','parsing.py',230), + ('array_like -> array_like + array_like','array_like',3,'p_expression_binary','parsing.py',231), + ('array_like -> array_like * array_like','array_like',3,'p_expression_binary','parsing.py',232), + ('array_like -> array_like / array_like','array_like',3,'p_expression_binary','parsing.py',233), + ('array_like -> array_like < array_like','array_like',3,'p_expression_binary','parsing.py',234), + ('array_like -> array_like > array_like','array_like',3,'p_expression_binary','parsing.py',235), + ('array_like -> array_like LEQ array_like','array_like',3,'p_expression_binary','parsing.py',236), + ('array_like -> array_like GEQ array_like','array_like',3,'p_expression_binary','parsing.py',237), + ('array_like -> array_like EQ array_like','array_like',3,'p_expression_binary','parsing.py',238), +] diff --git a/source/pyconform/parsing.py b/source/pyconform/parsing.py index 39844cf3..2b7c87c9 100644 --- a/source/pyconform/parsing.py +++ b/source/pyconform/parsing.py @@ -1,186 +1,276 @@ """ -Parsing Module +Parsing Module - NEW Based on PLY This module defines the necessary elements to parse a string variable definition into the recognized elements that are used to construct an Operation Graph. -Copyright 2017, University Corporation for Atmospheric Research +Copyright 2017-2018, University Corporation for Atmospheric Research LICENSE: See the LICENSE.rst file for details """ -from numpy import index_exp -from pyparsing import nums, alphas, alphanums, oneOf, delimitedList, opAssoc, operatorPrecedence -from pyparsing import Word, Combine, Forward, Suppress, Group, Optional, ParserElement -from pyparsing import Literal, CaselessLiteral, QuotedString - -# To improve performance -ParserElement.enablePackrat() - -#=================================================================================================== -# ParsedFunction -#=================================================================================================== -class ParsedFunction(object): - """ - A parsed function string-type - """ - - def __init__(self, tokens): - token = tokens[0] - self.key = token[0] - self.args = [] - self.kwds = {} - for t in token[1:]: - if isinstance(t, tuple): - self.kwds[t[0]] = t[1] - else: - self.args.append(t) - self.args = tuple(self.args) - def __repr__(self): - clsname = self.__class__.__name__ - argstr = ','.join('{!r}'.format(a) for a in self.args) - kwdstr = ','.join('{}={!r}'.format(k, self.kwds[k]) for k in self.kwds) - if len(self.args) > 0: - paramstr = '{},{}'.format(argstr, kwdstr) if len(self.kwds) > 0 else argstr - else: - paramstr = kwdstr if len(self.kwds) > 0 else '' - return ("<{} {}({}) ('{}') at {}>").format(clsname, self.key, paramstr, str(self), hex(id(self))) - def __str__(self): - argstr = ','.join('{!s}'.format(a) for a in self.args) - kwdstr = ','.join('{}={!s}'.format(k, self.kwds[k]) for k in self.kwds) - if len(self.args) > 0: - paramstr = '{},{}'.format(argstr, kwdstr) if len(self.kwds) > 0 else argstr - else: - paramstr = kwdstr if len(self.kwds) > 0 else '' - return "{}({!s})".format(self.key, paramstr) - def __eq__(self, other): - return ((type(self) == type(other)) and (self.key == other.key) and - (self.args == other.args) and (self.kwds == other.kwds)) - - -#=================================================================================================== -# ParsedUniOp -#=================================================================================================== -class ParsedUniOp(ParsedFunction): - """ - A parsed unary-operator string-type - """ - def __str__(self): - return "({}{!s})".format(self.key, self.args[0]) - - -#=================================================================================================== -# ParsedBinOp -#=================================================================================================== -class ParsedBinOp(ParsedFunction): - """ - A parsed binary-operator string-type - """ - def __str__(self): - return "({!s}{}{!s})".format(self.args[0], self.key, self.args[1]) - - -#=================================================================================================== -# ParsedVariable -#=================================================================================================== -class ParsedVariable(ParsedFunction): - """ - A parsed variable string-type - """ - def __init__(self, tokens): - super(ParsedVariable, self).__init__(tokens) - self.args = index_exp[self.args] if len(self.args) > 0 else () - def __repr__(self): - return "<{} '{}' at {}>".format(self.__class__.__name__, str(self), hex(id(self))) - def __str__(self): - if len(self.args) == 0: - strargs = '' - else: - strargs = str(list(self.args)) - return "{}{}".format(self.key, strargs) +from ply import lex, yacc +from collections import namedtuple + +tokens = ('UINT', 'UFLOAT', 'STRING', 'NAME', 'POW', 'EQ', 'LEQ', 'GEQ') +literals = ('*', '/', '+', '-', '<', '>', '=', ',', ':', '(', ')', '[', ']') +t_ignore = ' \t' + +t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' +t_POW = r'\*\*' +t_LEQ = r'<=' +t_GEQ = r'>=' +t_EQ = r'==' + + +def t_UFLOAT(t): + r'(([0-9]+\.[0-9]*|[0-9]*\.[0-9]+)([eE][+-]?[0-9]+)?|[0-9]+[eE][+-]?[0-9]+)' + t.value = float(t.value) + return t + + +def t_UINT(t): + r'[0-9]+' + t.value = int(t.value) + return t + + +def t_STRING(t): + r'"([^"\\]*(\\.[^"\\]*)*)"|\'([^\'\\]*(\\.[^\'\\]*)*)\'' + t.value = t.value[1:-1] + return t + + +def t_error(t): + raise TypeError('Unexpected string: {!r}'.format(t.value)) -#=================================================================================================== -# Operator Parser Functions -#=================================================================================================== +lex.lex(debug=False) -# Negation operator -def _negop_(tokens): - op, val = tokens[0] - if op == '+': - return val + +def ind_str(index): + if isinstance(index, slice): + ind_list = [index.start, index.stop, index.step] + _str = ':'.join('' if i is None else str(i) for i in ind_list) + return ':' if _str == '::' else _str else: - return ParsedUniOp([[op, val]]) - -# Binary Operators -def _binop_(tokens): - left, op, right = tokens[0] - return ParsedBinOp([[op, left, right]]) - -#=================================================================================================== -# MAIN PARSER (DEFINED AT MODULE LEVEL TO MAKE A SINGLETON) -#=================================================================================================== - -# INTEGERS: Just any word consisting only of numbers -_INT_ = Word(nums) -_INT_.setParseAction(lambda t: int(t[0])) - -# FLOATS: More complicated... can be decimal format or exponential -# format or a combination of the two -_DEC_FLT_ = (Combine(Word(nums) + '.' + Word(nums)) | - Combine(Word(nums) + '.') | - Combine('.' + Word(nums))) -_EXP_FLT_ = (Combine(CaselessLiteral('e') + Optional(oneOf('+ -')) + Word(nums))) -_FLOAT_ = (Combine(Word(nums) + _EXP_FLT_) | Combine(_DEC_FLT_ + Optional(_EXP_FLT_))) -_FLOAT_.setParseAction(lambda t: float(t[0])) - -# QUOTED STRINGS: Any words between quotations -_QSTR_ = QuotedString('"', escChar='\\') - -# String _NAME_s ...identifiers for function or variable _NAME_s -_NAME_ = Word(alphas + "_", alphanums + "_") - -# FUNCTIONS: Function arguments can be empty or any combination of -# ints, _FLOAT_, variables, and even other functions. Hence, -# we need a Forward place-holder to start... -_EXPR_PARSER_ = Forward() - -# Named Arguments -_KWDS_ = Group( _NAME_ + Suppress('=') + (_QSTR_ | _EXPR_PARSER_) ) -_KWDS_.setParseAction(lambda t: tuple(*t)) - -# Functions -_FUNC_ = Group(_NAME_ + (Suppress('(') + Optional(delimitedList(_QSTR_ | _KWDS_ | _EXPR_PARSER_)) + - Suppress(')'))) -_FUNC_.setParseAction(ParsedFunction) - -# VARIABLE NAMES: Can be just string _NAME_s or _NAME_s with blocks -# of indices (e.g., [1,2,-4]) -_INDEX_ = Combine(Optional('-') + Word(nums)) -_INDEX_.setParseAction(lambda t: int(t[0])) - -_IDX_OR_NONE_ = Optional(_INDEX_) -_IDX_OR_NONE_.setParseAction(lambda t: t[0] if len(t) > 0 else [None]) - -# _ISLICE_ = _IDX_OR_NONE_ + Optional(Suppress(':') + _IDX_OR_NONE_ + Optional(Suppress(':') + _IDX_OR_NONE_)) -# _ISLICE_.setParseAction(lambda t: slice(*t) if len(t) > 1 else t[0]) -_ISLICE_ = delimitedList(_IDX_OR_NONE_, delim=':') -_ISLICE_.setParseAction(lambda t: slice(*t) if len(t) > 1 else t[0]) - -_VARIABLE_ = Group(_NAME_ + Optional(Suppress('[') + delimitedList(_ISLICE_ | _INDEX_) + Suppress(']'))) -_VARIABLE_.setParseAction(ParsedVariable) - -# Expression parser -_EXPR_PARSER_ << operatorPrecedence(_FLOAT_ | _INT_ | _FUNC_ | _VARIABLE_, - [(Literal('**'), 2, opAssoc.RIGHT, _binop_), - (oneOf('+ -'), 1, opAssoc.RIGHT, _negop_), - (Literal('/'), 2, opAssoc.RIGHT, _binop_), - (Literal('*'), 2, opAssoc.RIGHT, _binop_), - (Literal('-'), 2, opAssoc.RIGHT, _binop_), - (Literal('+'), 2, opAssoc.RIGHT, _binop_)]) - -#=================================================================================================== -# Function to parse a string definition -#=================================================================================================== + return str(index) + + +def op_str(self): + if len(self.args) == 1: + return '({}{})'.format(self.key, self.args[0]) + elif len(self.args) == 2: + return '({}{}{})'.format(self.args[0], self.key, self.args[1]) + + +OpType = namedtuple('OpType', ['key', 'args']) +OpType.__new__.__defaults__ = (None, []) +OpType.__str__ = lambda self: op_str(self) + +FuncType = namedtuple('FuncType', ['key', 'args', 'kwds']) +FuncType.__new__.__defaults__ = (None, [], {}) +FuncType.__str__ = lambda self: '{}({})'.format( + self.key, ','.join([str(a) for a in self.args] + + ['{}={}'.format(k, self.kwds[k]) for k in self.kwds])) + +VarType = namedtuple('VarType', ['key', 'ind']) +VarType.__new__.__defaults__ = (None, []) +VarType.__str__ = lambda self: '{}{}'.format( + self.key, '' if len(self.ind) == 0 else '[{}]'.format(','.join([ind_str(a) for a in self.ind]))) + + +precedence = (('left', 'EQ'), + ('left', '<', '>', 'LEQ', 'GEQ'), + ('left', '+', '-'), + ('left', '*', '/'), + ('right', 'NEG', 'POS'), + ('left', 'POW')) + + +def p_array_like(p): + """ + array_like : UFLOAT + array_like : UINT + array_like : function + array_like : variable + """ + p[0] = p[1] + + +def p_array_like_group(p): + """ + array_like : '(' array_like ')' + """ + p[0] = p[2] + + +def p_function_with_arguments_and_keywords(p): + """ + function : NAME '(' argument_list ',' keyword_dict ')' + """ + p[0] = FuncType(p[1], p[3], p[5]) + + +def p_function_with_arguments_only(p): + """ + function : NAME '(' argument_list ')' + """ + p[0] = FuncType(p[1], p[3], {}) + + +def p_function_with_keywords_only(p): + """ + function : NAME '(' keyword_dict ')' + """ + p[0] = FuncType(p[1], [], p[3]) + + +def p_argument_list_append(p): + """ + argument_list : argument_list ',' argument + """ + p[0] = p[1] + [p[3]] + + +def p_single_item_argument_list(p): + """ + argument_list : argument + argument_list : + """ + p[0] = [p[1]] if len(p) > 1 else [] + +def p_argument(p): + """ + argument : array_like + argument : STRING + """ + p[0] = p[1] + + +def p_keyword_dict_setitem(p): + """ + keyword_dict : keyword_dict ',' NAME '=' argument + """ + p[1][p[3]] = p[5] + p[0] = p[1] + + +def p_single_item_keyword_dict(p): + """ + keyword_dict : NAME '=' argument + """ + p[0] = {p[1]: p[3]} + + +def p_variable(p): + """ + variable : NAME '[' index_list ']' + variable : NAME + """ + indices = p[3] if len(p) > 3 else [] + p[0] = VarType(p[1], indices) + + +def p_index_list_append(p): + """ + index_list : index_list ',' index + """ + p[0] = p[1] + [p[3]] + + +def p_single_item_index_list(p): + """ + index_list : index + """ + p[0] = [p[1]] + + +def p_index(p): + """ + index : slice + index : array_like + """ + p[0] = p[1] + + +def p_slice(p): + """ + slice : slice_argument ':' slice_argument ':' slice_argument + slice : slice_argument ':' slice_argument + """ + p[0] = slice(*p[1::2]) + + +def p_slice_argument(p): + """ + slice_argument : array_like + slice_argument : + """ + p[0] = p[1] if len(p) > 1 else None + + +def p_expression_unary(p): + """ + array_like : '-' array_like %prec NEG + array_like : '+' array_like %prec POS + """ + if p[1] == '+': + p[0] = p[2] + elif p[1] == '-': + if isinstance(p[2], (OpType, FuncType, VarType)): + p[0] = OpType(p[1], [p[2]]) + else: + p[0] = -p[2] + + +def p_expression_binary(p): + """ + array_like : array_like POW array_like + array_like : array_like '-' array_like + array_like : array_like '+' array_like + array_like : array_like '*' array_like + array_like : array_like '/' array_like + array_like : array_like '<' array_like + array_like : array_like '>' array_like + array_like : array_like LEQ array_like + array_like : array_like GEQ array_like + array_like : array_like EQ array_like + """ + if (isinstance(p[1], (OpType, FuncType, VarType)) or + isinstance(p[3], (OpType, FuncType, VarType))): + p[0] = OpType(p[2], [p[1], p[3]]) + elif p[2] == '**': + p[0] = p[1] ** p[3] + elif p[2] == '-': + p[0] = p[1] - p[3] + elif p[2] == '+': + p[0] = p[1] + p[3] + elif p[2] == '*': + p[0] = p[1] * p[3] + elif p[2] == '/': + p[0] = p[1] / p[3] + elif p[2] == '<': + p[0] = p[1] < p[3] + elif p[2] == '>': + p[0] = p[1] > p[3] + elif p[2] == '<=': + p[0] = p[1] <= p[3] + elif p[2] == '>=': + p[0] = p[1] >= p[3] + elif p[2] == '==': + p[0] = p[1] == p[3] + + +def p_error(p): + raise TypeError('Parsing error at {!r}'.format(p.value)) + + +yacc.yacc(debug=False) + + +#========================================================================= +# Function to parse a string definition +#========================================================================= def parse_definition(strexpr): - return _EXPR_PARSER_.parseString(strexpr)[0] + return yacc.parse(strexpr) # @UndefinedVariable diff --git a/source/pyconform/version.py b/source/pyconform/version.py index 75340b86..ce5f98f0 100644 --- a/source/pyconform/version.py +++ b/source/pyconform/version.py @@ -1,2 +1,2 @@ # Single place for version information -__version__ = '0.2.3' +__version__ = '0.2.5' diff --git a/source/test/dataflowTests.py b/source/test/dataflowTests.py index 2c5163d2..e424b5cf 100644 --- a/source/test/dataflowTests.py +++ b/source/test/dataflowTests.py @@ -16,9 +16,9 @@ import numpy -#======================================================================================================================= +#========================================================================= # DataFlowTests -#======================================================================================================================= +#========================================================================= class DataFlowTests(unittest.TestCase): """ Unit tests for the flownodes.FlowNode class @@ -32,7 +32,8 @@ def setUp(self): self.fattribs = OrderedDict([('a1', 'attribute 1'), ('a2', 'attribute 2')]) - self.dims = OrderedDict([('time', 4), ('lat', 7), ('lon', 9), ('strlen', 6), ('ncat', 3), ('bnds', 2)]) + self.dims = OrderedDict( + [('time', 4), ('lat', 7), ('lon', 9), ('strlen', 6), ('ncat', 3), ('bnds', 2)]) self.vdims = OrderedDict([('time', ('time',)), ('time_bnds', ('time', 'bnds')), ('lat', ('lat',)), @@ -40,7 +41,8 @@ def setUp(self): ('cat', ('ncat', 'strlen')), ('u1', ('time', 'lat', 'lon')), ('u2', ('time', 'lat', 'lon')), - ('u3', ('time', 'lat', 'lon'))]) + ('u3', ('time', 'lat', 'lon')), + ('tyears', ('time',))]) self.vattrs = OrderedDict([('lat', {'units': 'degrees_north', 'standard_name': 'latitude'}), ('lon', {'units': 'degrees_east', @@ -57,19 +59,25 @@ def setUp(self): 'standard_name': 'u variable 2'}), ('u3', {'units': 'kg', 'standard_name': 'u variable 3', - 'positive': 'down'})]) - self.dtypes = {'lat': 'd', 'lon': 'd', 'time': 'd', 'time_bnds': 'd', 'cat': 'c', 'u1': 'f', 'u2': 'f', 'u3': 'f'} - - ulen = reduce(lambda x, y: x * y, (self.dims[d] for d in self.vdims['u1']), 1) + 'positive': 'down'}), + ('tyears', {'units': 'years since 1979-01-01', + 'calendar': 'noleap'})]) + self.dtypes = {'lat': 'd', 'lon': 'd', 'time': 'd', 'tyears': 'd', + 'time_bnds': 'd', 'cat': 'c', 'u1': 'f', 'u2': 'f', 'u3': 'f'} + + ulen = reduce(lambda x, y: x * y, + (self.dims[d] for d in self.vdims['u1']), 1) ushape = tuple(self.dims[d] for d in self.vdims['u1']) + tdata = numpy.arange(self.dims['time'], dtype=self.dtypes['time']) self.vdat = {'lat': numpy.linspace(-90, 90, num=self.dims['lat'], endpoint=True, dtype=self.dtypes['lat']), 'lon': -numpy.linspace(-180, 180, num=self.dims['lon'], dtype=self.dtypes['lon'])[::-1], - 'time': numpy.arange(self.dims['time'], dtype=self.dtypes['time']), - 'time_bnds': numpy.array([[i,i+1] for i in range(self.dims['time'])], dtype=self.dtypes['time_bnds']), + 'time': tdata, + 'time_bnds': numpy.array([[i, i + 1] for i in range(self.dims['time'])], dtype=self.dtypes['time_bnds']), 'cat': numpy.asarray(['left', 'middle', 'right'], dtype='S').view(self.dtypes['cat']), 'u1': numpy.linspace(0, ulen, num=ulen, dtype=self.dtypes['u1']).reshape(ushape), 'u2': numpy.linspace(0, ulen, num=ulen, dtype=self.dtypes['u2']).reshape(ushape), - 'u3': numpy.linspace(0, ulen, num=ulen, dtype=self.dtypes['u3']).reshape(ushape)} + 'u3': numpy.linspace(0, ulen, num=ulen, dtype=self.dtypes['u3']).reshape(ushape), + 'tyears': tdata / 365.0} for vname in self.filenames: fname = self.filenames[vname] @@ -80,8 +88,10 @@ def setUp(self): dsize = self.dims[dname] if dname != 'time' else None ncf.createDimension(dname, dsize) for uname in [u for u in self.vdims if u not in self.filenames]: - ncvars[uname] = ncf.createVariable(uname, self.dtypes[uname], self.vdims[uname]) - ncvars[vname] = ncf.createVariable(vname, self.dtypes[vname], self.vdims[vname]) + ncvars[uname] = ncf.createVariable( + uname, self.dtypes[uname], self.vdims[uname]) + ncvars[vname] = ncf.createVariable( + vname, self.dtypes[vname], self.vdims[vname]) for vnam in ncvars: vobj = ncvars[vnam] for aname in self.vattrs[vnam]: @@ -91,7 +101,8 @@ def setUp(self): print_ncfile(fname) print - self.inpds = datasets.InputDatasetDesc('inpds', self.filenames.values()) + self.inpds = datasets.InputDatasetDesc( + 'inpds', self.filenames.values()) vdicts = OrderedDict() @@ -107,7 +118,7 @@ def setUp(self): vdicts['C'] = OrderedDict() vdicts['C']['datatype'] = 'char' - vdicts['C']['dimensions'] = ('c','n') + vdicts['C']['dimensions'] = ('c', 'n') vdicts['C']['definition'] = 'cat' vattribs = OrderedDict() vattribs['standard_name'] = 'category' @@ -163,6 +174,16 @@ def setUp(self): vattribs['calendar'] = 'noleap' vdicts['T_bnds']['attributes'] = vattribs + vdicts['T2'] = OrderedDict() + vdicts['T2']['datatype'] = 'double' + vdicts['T2']['dimensions'] = ('t',) + vdicts['T2']['definition'] = 'chunits(tyears * 365, units="days since 1979-01-01", calendar="noleap")' + vattribs = OrderedDict() + vattribs['standard_name'] = 'time_years' + vattribs['units'] = 'days since 1979-01-01 00:00:00' + vattribs['calendar'] = 'noleap' + vdicts['T2']['attributes'] = vattribs + vdicts['V1'] = OrderedDict() vdicts['V1']['datatype'] = 'double' vdicts['V1']['dimensions'] = ('t', 'y', 'x') @@ -220,7 +241,7 @@ def setUp(self): vattribs['valid_min'] = 1.0 vattribs['valid_max'] = 200.0 vdicts['V4']['attributes'] = vattribs - + vdicts['V5'] = OrderedDict() vdicts['V5']['datatype'] = 'double' vdicts['V5']['dimensions'] = ('t', 'y') @@ -282,7 +303,7 @@ def setUp(self): vattribs['valid_max'] = -1.0 vattribs['positive'] = 'up' vdicts['V8']['attributes'] = vattribs - + self.dsdict = vdicts self.outds = datasets.OutputDatasetDesc('outds', self.dsdict) @@ -317,7 +338,8 @@ def test_dimension_map(self): testname = 'DataFlow().dimension_map' df = dataflow.DataFlow(self.inpds, self.outds) actual = df.dimension_map - expected = {'lat': 'y', 'strlen': 'n', 'lon': 'x', 'ncat': 'c', 'time': 't', 'bnds': 'd'} + expected = {'lat': 'y', 'strlen': 'n', 'lon': 'x', + 'ncat': 'c', 'time': 't', 'bnds': 'd'} print_test_message(testname, actual=actual, expected=expected) self.assertEqual(actual, expected, '{} failed'.format(testname)) @@ -357,7 +379,8 @@ def test_execute_chunks_2D_x_y(self): df = dataflow.DataFlow(self.inpds, self.outds) expected = ValueError print_test_message(testname, expected=expected) - self.assertRaises(expected, df.execute, chunks=OrderedDict([('x', 4), ('y', 3)])) + self.assertRaises(expected, df.execute, + chunks=OrderedDict([('x', 4), ('y', 3)])) def test_execute_chunks_2D_t_y(self): testname = 'DataFlow().execute()' @@ -372,8 +395,8 @@ def test_execute_chunks_2D_t_y(self): print -#=============================================================================== +#========================================================================= # Command-Line Operation -#=============================================================================== +#========================================================================= if __name__ == "__main__": unittest.main() diff --git a/source/test/parsingTests.py b/source/test/parsingTests.py index 44edd540..448f5617 100644 --- a/source/test/parsingTests.py +++ b/source/test/parsingTests.py @@ -1,7 +1,7 @@ """ Parsing Unit Tests -Copyright 2017, University Corporation for Atmospheric Research +Copyright 2017-2018, University Corporation for Atmospheric Research LICENSE: See the LICENSE.rst file for details """ @@ -9,149 +9,155 @@ from testutils import print_test_message import unittest -import pyparsing -#=============================================================================== +#========================================================================= # ParsedStringTypeTests -#=============================================================================== +#========================================================================= class ParsedStringTypeTests(unittest.TestCase): def test_pst_init(self): - indata = (['x'], {}) - pst = parsing.ParsedFunction(indata) + indata = ['x'] + pst = parsing.FuncType(*indata) actual = type(pst) - expected = parsing.ParsedFunction - testname = 'ParsedFunction.__init__({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + expected = parsing.FuncType + testname = 'FuncType.__init__({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Types do not match') def test_varpst_init(self): - indata = (['x'], {}) - pst = parsing.ParsedVariable(indata) + indata = ['x'] + pst = parsing.VarType(*indata) actual = type(pst) - expected = parsing.ParsedVariable - testname = 'ParsedVariable.__init__({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + expected = parsing.VarType + testname = 'VarType.__init__({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Types do not match') def test_funcpst_init(self): indata = (['x'], {}) - pst = parsing.ParsedFunction(indata) + pst = parsing.FuncType(indata) actual = type(pst) - expected = parsing.ParsedFunction - testname = 'ParsedFunction.__init__({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + expected = parsing.FuncType + testname = 'FuncType.__init__({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Types do not match') def test_operpst_init(self): indata = (['x'], {}) - pst = parsing.ParsedBinOp(indata) + pst = parsing.OpType(indata) actual = type(pst) - expected = parsing.ParsedBinOp - testname = 'ParsedBinOp.__init__({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + expected = parsing.OpType + testname = 'OpType.__init__({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Types do not match') def test_pst_init_args(self): indata = (['x', 1, -3.2], {}) - pst = parsing.ParsedFunction(indata) + pst = parsing.FuncType(indata) actual = type(pst) - expected = parsing.ParsedFunction - testname = 'ParsedFunction.__init__({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + expected = parsing.FuncType + testname = 'FuncType.__init__({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Types do not match') def test_pst_func_key(self): - indata = (['x', 1, -3.2], {}) - pst = parsing.ParsedFunction(indata) + indata = ['x', [1, -3.2], {}] + pst = parsing.FuncType(*indata) actual = pst.key - expected = indata[0][0] - testname = 'ParsedFunction.__init__({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + expected = indata[0] + testname = 'FuncType.__init__({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Key does not match') def test_pst_func_args(self): - indata = (['x', 1, -3.2], {}) - pst = parsing.ParsedFunction(indata) + indata = ['x', [1, -3.2], {}] + pst = parsing.FuncType(*indata) actual = pst.args - expected = tuple(indata[0][1:]) - testname = 'ParsedFunction.__init__({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + expected = indata[1] + testname = 'FuncType.__init__({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Args do not match') def test_pst_func_kwds(self): - indata = (['x', 1, -3.2, ('x', 5)], {}) - pst = parsing.ParsedFunction(indata) + indata = ['x', [1, -3.2], {'x': 5}] + pst = parsing.FuncType(*indata) actual = pst.kwds expected = {'x': 5} - testname = 'ParsedFunction.__init__({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + testname = 'FuncType.__init__({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Args do not match') -#=============================================================================== +#========================================================================= # DefinitionParserTests -#=============================================================================== +#========================================================================= class DefinitionParserTests(unittest.TestCase): -#===== QUOTED STRINGS ========================================================== + #===== QUOTED STRINGS ==================================================== def test_parse_quote_funcarg_int(self): indata = 'f("1")' actual = parsing.parse_definition(indata) - expected = parsing.ParsedFunction([['f', '1']]) + expected = parsing.FuncType('f', ['1']) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'String parsing failed') def test_parse_quote_funcarg_kwd(self): indata = 'f(a="1")' actual = parsing.parse_definition(indata) - expected = parsing.ParsedFunction([['f', ('a', '1')]]) + expected = parsing.FuncType('f', [], {'a': '1'}) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'String parsing failed') - def test_parse_quote_nofunc(self): - indata = '"Hello, World!"' - testname = 'parse_definition({0!r})'.format(indata) - expected = pyparsing.ParseException - print_test_message(testname, indata=indata, expected=expected) - self.assertRaises(expected, parsing.parse_definition, indata) - def test_parse_quote_funcarg(self): indata = 'f("Hello, World!")' actual = parsing.parse_definition(indata) - expected = parsing.ParsedFunction([['f', 'Hello, World!']]) + expected = parsing.FuncType('f', ['Hello, World!']) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'String parsing failed') def test_parse_quote_funcarg_escaped(self): - indata = 'f("\\"1\\"")' + indata = 'f(\'"1"\')' actual = parsing.parse_definition(indata) - expected = parsing.ParsedFunction([['f', '"1"']]) + expected = parsing.FuncType('f', ['"1"']) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'String parsing failed') def test_parse_quote_funcarg_func(self): indata = 'g("f(x,y,z)")' actual = parsing.parse_definition(indata) - expected = parsing.ParsedFunction([['g', 'f(x,y,z)']]) + expected = parsing.FuncType('g', ['f(x,y,z)']) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'String parsing failed') -#===== INTEGERS ================================================================ +#===== INTEGERS ========================================================== def test_parse_integer(self): indata = '1' actual = parsing.parse_definition(indata) expected = int(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Integer parsing failed') def test_parse_integer_large(self): @@ -159,17 +165,19 @@ def test_parse_integer_large(self): actual = parsing.parse_definition(indata) expected = int(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Integer parsing failed') -#===== FLOATS ================================================================== +#===== FLOATS ============================================================ def test_parse_float_dec(self): indata = '1.' actual = parsing.parse_definition(indata) expected = float(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Float parsing failed') def test_parse_float_dec_long(self): @@ -177,7 +185,8 @@ def test_parse_float_dec_long(self): actual = parsing.parse_definition(indata) expected = float(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Float parsing failed') def test_parse_float_dec_nofirst(self): @@ -185,7 +194,8 @@ def test_parse_float_dec_nofirst(self): actual = parsing.parse_definition(indata) expected = float(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Float parsing failed') def test_parse_float_exp(self): @@ -193,7 +203,8 @@ def test_parse_float_exp(self): actual = parsing.parse_definition(indata) expected = float(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Float parsing failed') def test_parse_float_pos_exp(self): @@ -201,7 +212,8 @@ def test_parse_float_pos_exp(self): actual = parsing.parse_definition(indata) expected = float(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Float parsing failed') def test_parse_float_neg_exp(self): @@ -209,7 +221,8 @@ def test_parse_float_neg_exp(self): actual = parsing.parse_definition(indata) expected = float(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Float parsing failed') def test_parse_float_dec_exp(self): @@ -217,7 +230,8 @@ def test_parse_float_dec_exp(self): actual = parsing.parse_definition(indata) expected = float(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Float parsing failed') def test_parse_float_dec_pos_exp(self): @@ -225,7 +239,8 @@ def test_parse_float_dec_pos_exp(self): actual = parsing.parse_definition(indata) expected = float(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Float parsing failed') def test_parse_float_dec_neg_exp(self): @@ -233,7 +248,8 @@ def test_parse_float_dec_neg_exp(self): actual = parsing.parse_definition(indata) expected = float(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Float parsing failed') def test_parse_float_dec_long_exp(self): @@ -241,7 +257,8 @@ def test_parse_float_dec_long_exp(self): actual = parsing.parse_definition(indata) expected = float(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Float parsing failed') def test_parse_float_dec_long_pos_exp(self): @@ -249,7 +266,8 @@ def test_parse_float_dec_long_pos_exp(self): actual = parsing.parse_definition(indata) expected = float(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Float parsing failed') def test_parse_float_dec_long_neg_exp(self): @@ -257,7 +275,8 @@ def test_parse_float_dec_long_neg_exp(self): actual = parsing.parse_definition(indata) expected = float(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Float parsing failed') def test_parse_float_dec_nofirst_exp(self): @@ -265,7 +284,8 @@ def test_parse_float_dec_nofirst_exp(self): actual = parsing.parse_definition(indata) expected = float(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Float parsing failed') def test_parse_float_dec_nofirst_pos_exp(self): @@ -273,7 +293,8 @@ def test_parse_float_dec_nofirst_pos_exp(self): actual = parsing.parse_definition(indata) expected = float(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Float parsing failed') def test_parse_float_dec_nofirst_neg_exp(self): @@ -281,171 +302,182 @@ def test_parse_float_dec_nofirst_neg_exp(self): actual = parsing.parse_definition(indata) expected = float(indata) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Float parsing failed') -#===== FUNCTIONS =============================================================== +#===== FUNCTIONS ========================================================= def test_parse_func(self): indata = 'f()' actual = parsing.parse_definition(indata) - expected = parsing.ParsedFunction(('f', {})) + expected = parsing.FuncType('f') testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Function parsing failed') def test_parse_func_arg(self): indata = 'f(1)' actual = parsing.parse_definition(indata) - expected = parsing.ParsedFunction([['f', 1]]) + expected = parsing.FuncType('f', [1]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Function parsing failed') def test_parse_func_nested(self): - g2 = parsing.ParsedFunction([['g', 2]]) - f1g = parsing.ParsedFunction([['f', 1, g2]]) indata = 'f(1, g(2))' + g2 = parsing.FuncType('g', [2]) + f1g = parsing.FuncType('f', [1, g2]) actual = parsing.parse_definition(indata) expected = f1g testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Function parsing failed') -#===== VARIABLES =============================================================== +#===== VARIABLES ========================================================= def test_parse_var(self): indata = 'x' actual = parsing.parse_definition(indata) - expected = parsing.ParsedVariable([['x']]) + expected = parsing.VarType('x') testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Variable parsing failed') def test_parse_var_index(self): indata = 'x[1]' actual = parsing.parse_definition(indata) - expected = parsing.ParsedVariable([['x', 1]]) + expected = parsing.VarType('x', [1]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Variable parsing failed') def test_parse_var_slice(self): indata = 'x[1:2:3]' actual = parsing.parse_definition(indata) - expected = parsing.ParsedVariable([['x', slice(1, 2, 3)]]) + expected = parsing.VarType('x', [slice(1, 2, 3)]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Variable parsing failed') def test_parse_var_int_slice(self): indata = 'x[3,1:2]' actual = parsing.parse_definition(indata) - expected = parsing.ParsedVariable([['x', 3, slice(1, 2, None)]]) - testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) - self.assertEqual(actual, expected, 'Variable parsing failed') - - def test_parse_var_none_slice(self): - indata = 'x[,1:2:3]' - actual = parsing.parse_definition(indata) - expected = parsing.ParsedVariable([['x', None, slice(1, 2, 3)]]) - testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) - self.assertEqual(actual, expected, 'Variable parsing failed') - - def test_parse_var_slice_none(self): - indata = 'x[]' - actual = parsing.parse_definition(indata) - expected = parsing.ParsedVariable([['x', None]]) + expected = parsing.VarType('x', [3, slice(1, 2, None)]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Variable parsing failed') def test_parse_var_slice_empty(self): indata = 'x[:]' actual = parsing.parse_definition(indata) - expected = parsing.ParsedVariable([['x', slice(None)]]) + expected = parsing.VarType('x', [slice(None)]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Variable parsing failed') - + def test_parse_var_slice_partial_1(self): indata = 'x[1:]' actual = parsing.parse_definition(indata) - expected = parsing.ParsedVariable([['x', slice(1, None)]]) + expected = parsing.VarType('x', [slice(1, None)]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Variable parsing failed') def test_parse_var_slice_partial_2(self): indata = 'x[:2]' actual = parsing.parse_definition(indata) - expected = parsing.ParsedVariable([['x', slice(None, 2)]]) + expected = parsing.VarType('x', [slice(None, 2)]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Variable parsing failed') def test_parse_var_slice_partial_3(self): indata = 'x[::-1]' actual = parsing.parse_definition(indata) - expected = parsing.ParsedVariable([['x', slice(None, None, -1)]]) + expected = parsing.VarType('x', [slice(None, None, -1)]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Variable parsing failed') - + def test_parse_var_slice_partial_4(self): indata = 'x[::-1::::]' expected = TypeError testname = 'parse_definition({0!r})'.format(indata) print_test_message(testname, indata=indata, expected=expected) self.assertRaises(expected, parsing.parse_definition, indata) - -#===== NEGATION ================================================================ + def test_parse_var_time(self): + indata = 'time' + actual = parsing.parse_definition(indata) + expected = parsing.VarType('time') + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Variable time parsing failed') + + +#===== NEGATION ========================================================== def test_parse_neg_integer(self): indata = '-1' actual = parsing.parse_definition(indata) - expected = parsing.ParsedUniOp([['-', 1]]) + expected = -1 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Negation parsing failed') def test_parse_neg_float(self): indata = '-1.4' actual = parsing.parse_definition(indata) - expected = parsing.ParsedUniOp([['-', 1.4]]) + expected = -1.4 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Negation parsing failed') def test_parse_neg_var(self): indata = '-x' actual = parsing.parse_definition(indata) - x = parsing.ParsedVariable([['x']]) - expected = parsing.ParsedUniOp([['-', x]]) + x = parsing.VarType('x') + expected = parsing.OpType('-', [x]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Negation parsing failed') def test_parse_neg_func(self): indata = '-f()' actual = parsing.parse_definition(indata) - f = parsing.ParsedFunction([['f']]) - expected = parsing.ParsedUniOp([['-', f]]) + f = parsing.FuncType('f') + expected = parsing.OpType('-', [f]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Negation parsing failed') -#===== POSITIVE ================================================================ +#===== POSITIVE ========================================================== def test_parse_pos_integer(self): indata = '+1' actual = parsing.parse_definition(indata) expected = 1 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Positive operator parsing failed') def test_parse_pos_float(self): @@ -453,265 +485,313 @@ def test_parse_pos_float(self): actual = parsing.parse_definition(indata) expected = 1e7 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Positive operator parsing failed') def test_parse_pos_func(self): indata = '+f()' actual = parsing.parse_definition(indata) - expected = parsing.ParsedFunction([['f']]) + expected = parsing.FuncType('f') testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Positive operator parsing failed') def test_parse_pos_var(self): indata = '+x[1]' actual = parsing.parse_definition(indata) - expected = parsing.ParsedVariable([['x', 1]]) + expected = parsing.VarType('x', [1]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Positive operator parsing failed') -#===== POWER =================================================================== +#===== POWER ============================================================= def test_parse_int_pow_int(self): indata = '2**1' actual = parsing.parse_definition(indata) - expected = parsing.ParsedBinOp([['**', 2, 1]]) + expected = 2 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Power operator parsing failed') def test_parse_float_pow_float(self): - indata = '2.4 ** 1e7' + indata = '2.4 ** 3.5' actual = parsing.parse_definition(indata) - expected = parsing.ParsedBinOp([['**', 2.4, 1e7]]) + expected = 2.4 ** 3.5 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Power operator parsing failed') def test_parse_func_pow_func(self): indata = 'f() ** g(1)' actual = parsing.parse_definition(indata) - f = parsing.ParsedFunction([['f']]) - g1 = parsing.ParsedFunction([['g', 1]]) - expected = parsing.ParsedBinOp([['**', f, g1]]) + f = parsing.FuncType('f') + g1 = parsing.FuncType('g', [1]) + expected = parsing.OpType('**', [f, g1]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Power operator parsing failed') def test_parse_var_pow_var(self): indata = 'x[1] ** y' actual = parsing.parse_definition(indata) - x1 = parsing.ParsedVariable([['x', 1]]) - y = parsing.ParsedVariable([['y']]) - expected = parsing.ParsedBinOp([['**', x1, y]]) + x1 = parsing.VarType('x', [1]) + y = parsing.VarType('y') + expected = parsing.OpType('**', [x1, y]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Power operator parsing failed') -#===== DIV ===================================================================== +#===== DIV =============================================================== def test_parse_int_div_int(self): indata = '2/1' actual = parsing.parse_definition(indata) - expected = parsing.ParsedBinOp([['/', 2, 1]]) + expected = 2 / 1 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Division operator parsing failed') def test_parse_float_div_float(self): indata = '2.4/1e7' actual = parsing.parse_definition(indata) - expected = parsing.ParsedBinOp([['/', 2.4, 1e7]]) + expected = 2.4 / 1e7 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Division operator parsing failed') def test_parse_func_div_func(self): indata = 'f() / g(1)' actual = parsing.parse_definition(indata) - f = parsing.ParsedFunction([['f']]) - g1 = parsing.ParsedFunction([['g', 1]]) - expected = parsing.ParsedBinOp([['/', f, g1]]) + f = parsing.FuncType('f') + g1 = parsing.FuncType('g', [1]) + expected = parsing.OpType('/', [f, g1]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Division operator parsing failed') def test_parse_var_div_var(self): indata = 'x[1] / y' actual = parsing.parse_definition(indata) - x1 = parsing.ParsedVariable([['x', 1]]) - y = parsing.ParsedVariable([['y']]) - expected = parsing.ParsedBinOp([['/', x1, y]]) + x1 = parsing.VarType('x', [1]) + y = parsing.VarType('y') + expected = parsing.OpType('/', [x1, y]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Division operator parsing failed') -#===== MUL ===================================================================== +#===== MUL =============================================================== def test_parse_int_mul_int(self): indata = '2*1' actual = parsing.parse_definition(indata) - expected = parsing.ParsedBinOp([['*', 2, 1]]) + expected = 2 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) - self.assertEqual(actual, expected, 'Multiplication operator parsing failed') + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Multiplication operator parsing failed') def test_parse_float_mul_float(self): indata = '2.4*1e7' actual = parsing.parse_definition(indata) - expected = parsing.ParsedBinOp([['*', 2.4, 1e7]]) + expected = 2.4 * 1e7 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) - self.assertEqual(actual, expected, 'Multiplication operator parsing failed') + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Multiplication operator parsing failed') def test_parse_func_mul_func(self): indata = 'f() * g(1)' actual = parsing.parse_definition(indata) - f = parsing.ParsedFunction([['f']]) - g1 = parsing.ParsedFunction([['g', 1]]) - expected = parsing.ParsedBinOp([['*', f, g1]]) + f = parsing.FuncType('f') + g1 = parsing.FuncType('g', [1]) + expected = parsing.OpType('*', [f, g1]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) - self.assertEqual(actual, expected, 'Multiplication operator parsing failed') + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Multiplication operator parsing failed') def test_parse_var_mul_var(self): indata = 'x[1] * y' actual = parsing.parse_definition(indata) - x1 = parsing.ParsedVariable([['x', 1]]) - y = parsing.ParsedVariable([['y']]) - expected = parsing.ParsedBinOp([['*', x1, y]]) + x1 = parsing.VarType('x', [1]) + y = parsing.VarType('y') + expected = parsing.OpType('*', [x1, y]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) - self.assertEqual(actual, expected, 'Multiplication operator parsing failed') + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Multiplication operator parsing failed') -#===== ADD ===================================================================== +#===== ADD =============================================================== def test_parse_int_add_int(self): indata = '2+1' actual = parsing.parse_definition(indata) - expected = parsing.ParsedBinOp([['+', 2, 1]]) + expected = 3 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Addition operator parsing failed') def test_parse_float_add_float(self): indata = '2.4+1e7' actual = parsing.parse_definition(indata) - expected = parsing.ParsedBinOp([['+', 2.4, 1e7]]) + expected = 1e7 + 2.4 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Addition operator parsing failed') def test_parse_func_add_func(self): indata = 'f() + g(1)' actual = parsing.parse_definition(indata) - f = parsing.ParsedFunction([['f']]) - g1 = parsing.ParsedFunction([['g', 1]]) - expected = parsing.ParsedBinOp([['+', f, g1]]) + f = parsing.FuncType('f') + g1 = parsing.FuncType('g', [1]) + expected = parsing.OpType('+', [f, g1]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Addition operator parsing failed') def test_parse_var_add_var(self): indata = 'x[1] + y' actual = parsing.parse_definition(indata) - x1 = parsing.ParsedVariable([['x', 1]]) - y = parsing.ParsedVariable([['y']]) - expected = parsing.ParsedBinOp([['+', x1, y]]) + x1 = parsing.VarType('x', [1]) + y = parsing.VarType('y') + expected = parsing.OpType('+', [x1, y]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) self.assertEqual(actual, expected, 'Addition operator parsing failed') -#===== SUB ===================================================================== +#===== SUB =============================================================== def test_parse_int_sub_int(self): indata = '2-1' actual = parsing.parse_definition(indata) - expected = parsing.ParsedBinOp([['-', 2, 1]]) + expected = 1 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) - self.assertEqual(actual, expected, 'Subtraction operator parsing failed') + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Subtraction operator parsing failed') def test_parse_float_sub_float(self): indata = '2.4-1e7' actual = parsing.parse_definition(indata) - expected = parsing.ParsedBinOp([['-', 2.4, 1e7]]) + expected = 2.4 - 1e7 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) - self.assertEqual(actual, expected, 'Subtraction operator parsing failed') + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Subtraction operator parsing failed') def test_parse_func_sub_func(self): indata = 'f() - g(1)' actual = parsing.parse_definition(indata) - f = parsing.ParsedFunction([['f']]) - g1 = parsing.ParsedFunction([['g', 1]]) - expected = parsing.ParsedBinOp([['-', f, g1]]) + f = parsing.FuncType('f') + g1 = parsing.FuncType('g', [1]) + expected = parsing.OpType('-', [f, g1]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) - self.assertEqual(actual, expected, 'Subtraction operator parsing failed') + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Subtraction operator parsing failed') def test_parse_var_sub_var(self): indata = 'x[1] - y' actual = parsing.parse_definition(indata) - x1 = parsing.ParsedVariable([['x', 1]]) - y = parsing.ParsedVariable([['y']]) - expected = parsing.ParsedBinOp([['-', x1, y]]) + x1 = parsing.VarType('x', [1]) + y = parsing.VarType('y') + expected = parsing.OpType('-', [x1, y]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) - self.assertEqual(actual, expected, 'Subtraction operator parsing failed') + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Subtraction operator parsing failed') -#===== Integration ============================================================= +#===== Integration ======================================================= def test_parse_integrated_1(self): indata = '2-17.3*x**2' actual = parsing.parse_definition(indata) - x = parsing.ParsedVariable([['x']]) - x2 = parsing.ParsedBinOp([['**', x, 2]]) - m17p3x2 = parsing.ParsedBinOp([['*', 17.3, x2]]) - expected = parsing.ParsedBinOp([['-', 2, m17p3x2]]) + x = parsing.VarType('x') + x2 = parsing.OpType('**', [x, 2]) + m17p3x2 = parsing.OpType('*', [17.3, x2]) + expected = parsing.OpType('-', [2, m17p3x2]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) - self.assertEqual(actual, expected, 'Integrated #1 operator parsing failed') + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Integrated #1 operator parsing failed') def test_parse_integrated_2(self): indata = '2-17.3*x / f(2.3, x[2:5])' actual = parsing.parse_definition(indata) - x = parsing.ParsedVariable([['x']]) - x25 = parsing.ParsedVariable([['x', slice(2, 5)]]) - f = parsing.ParsedFunction([['f', 2.3, x25]]) - dxf = parsing.ParsedBinOp([['/', x, f]]) - m17p3dxf = parsing.ParsedBinOp([['*', 17.3, dxf]]) - expected = parsing.ParsedBinOp([['-', 2, m17p3dxf]]) + x = parsing.VarType('x') + x25 = parsing.VarType('x', [slice(2, 5)]) + f = parsing.FuncType('f', [2.3, x25]) + m17p3x = parsing.OpType('*', [17.3, x]) + dm17p3xf = parsing.OpType('/', [m17p3x, f]) + expected = parsing.OpType('-', [2, dm17p3xf]) testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) - self.assertEqual(actual, expected, 'Integrated #2 operator parsing failed') + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Integrated #2 operator parsing failed') def test_parse_integrated_3(self): indata = '2-3+1' actual = parsing.parse_definition(indata) - m23 = parsing.ParsedBinOp([['-', 2, 3]]) - expected = parsing.ParsedBinOp([['+', m23, 1]]) + expected = 0 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) - self.assertEqual(actual, expected, 'Integrated #3 operator parsing failed') + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Integrated #3 operator parsing failed') def test_parse_integrated_4(self): indata = '2-3/4*2+1' actual = parsing.parse_definition(indata) - d34 = parsing.ParsedBinOp([['/', 3, 4]]) - d34m2 = parsing.ParsedBinOp([['*', d34, 2]]) - s2d34m2 = parsing.ParsedBinOp([['-', 2, d34m2]]) - expected = parsing.ParsedBinOp([['+', s2d34m2, 1]]) + expected = 3 testname = 'parse_definition({0!r})'.format(indata) - print_test_message(testname, indata=indata, actual=actual, expected=expected) - self.assertEqual(actual, expected, 'Integrated #4 operator parsing failed') - - -#=============================================================================== + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Integrated #4 operator parsing failed') + + def test_parse_integrated_5(self): + indata = 'mean(chunits(time_bnds, units=time), "bnds")' + actual = parsing.parse_definition(indata) + time = parsing.VarType('time') + time_bnds = parsing.VarType('time_bnds') + chunits = parsing.FuncType('chunits', [time_bnds], {'units': time}) + expected = parsing.FuncType('mean', [chunits, 'bnds']) + testname = 'parse_definition({0!r})'.format(indata) + print_test_message(testname, indata=indata, + actual=actual, expected=expected) + self.assertEqual(actual, expected, + 'Integrated #5 operator parsing failed') + + +#========================================================================= # Command-Line Operation -#=============================================================================== +#========================================================================= if __name__ == "__main__": # import sys;sys.argv = ['', 'Test.testName'] unittest.main()