From 99d489b10def142ad93aa1959c31d797cfa46896 Mon Sep 17 00:00:00 2001 From: Charles Doutriaux Date: Tue, 26 Apr 2016 08:53:03 -0700 Subject: [PATCH 01/11] cdscan removed use of old modules string and types --- Packages/cdms2/Script/cdscan | 84 ++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/Packages/cdms2/Script/cdscan b/Packages/cdms2/Script/cdscan index 59896b1b43..8914abbddf 100755 --- a/Packages/cdms2/Script/cdscan +++ b/Packages/cdms2/Script/cdscan @@ -7,11 +7,9 @@ from cdms2.grid import lookupArray from cdms2.axis import calendarToTag, tagToCalendar from cdms2.cdmsobj import CdFromObject,CdString,CdScalar,CdFloat,CdDouble,CdShort,CdInt,CdLong import numpy -import string import cdtime import os.path import copy -import types from cdms2 import cdmsNode import re @@ -258,7 +256,7 @@ def timeindex(value, units, basetime, delta, calendar): where t = reltime(value, units) and delu is the time interval (delta, delunits) (e.g., 1 month). """ - if string.find(units," as ")==-1: + if units.find(" as ")==-1: tval = cdtime.reltime(value, units) else: tval = cdtime.abstime(value, units) @@ -426,7 +424,7 @@ def disambig(name, dict, num, comparator, value): uniqname = '%s_%d'%(name,num) if dict.has_key(uniqname) and comparator(value, dict[uniqname]): trial_name = uniqname - for letter in string.lowercase: + for letter in 'abcdefghijklmnopqrstuvwxyz': uniqname = '%s_%s'%(trial_name,letter) if not dict.has_key(uniqname) or not comparator(value, dict[uniqname]): break @@ -448,7 +446,7 @@ def comparedomains(domain1, domain2): item2 = domain2[i] if type(item1)!=type(item2): return 1 - if type(item1)==types.StringType: + if isinstance(item1, basestring): return item1!=item2 elif compareaxes(item1, item2): return 1 @@ -486,14 +484,14 @@ def validateAttrs(node): if reqtype==CdFromObject: reqtype = parenttype if reqtype!=datatype and datatype==CdString and scaletype==CdScalar: - if reqtype in (CdFloat,CdDouble) and type(attval)!=types.FloatType: + if reqtype in (CdFloat,CdDouble) and not isinstance(attval, float): try: - attval = string.atof(attval) + attval = float(attval) except: if verbose: print >> sys.stderr, "Warning: %s=%s should be a float, id=%s"%(attname,attval,node.id), try: - attval = string.atoi(attval) + attval = int(attval) attval = float(attval) if verbose: print "(Recasting)" @@ -507,14 +505,14 @@ def validateAttrs(node): else: if verbose: print "" - elif reqtype in (CdShort,CdInt,CdLong) and type(attval)!=types.IntType: + elif reqtype in (CdShort,CdInt,CdLong) and not isinstance(attval, int): try: - attval = string.atoi(attval) + attval = int(attval) except: if verbose: print >> sys.stderr, "Warning: %s=%s should be an integer, id=%s"%(attname,attval,node.id), try: - attval = string.atof(attval) + attval = float(attval) attval = int(attval) if verbose: print "(Recasting)" @@ -530,7 +528,7 @@ def cloneWithLatCheck(axis): global notrimlat axisvals = origvals = axis[:] - if axis.isLatitude() and hasattr(axis,"units") and string.lower(axis.units[0:6])=="degree": + if axis.isLatitude() and hasattr(axis,"units") and axis.units[0:6].lower()=="degree": if notrimlat==0: axisvals = numpy.maximum(-90.0, numpy.minimum(90.0,axisvals)) if not numpy.ma.allclose(axisvals, origvals) and verbose: @@ -563,9 +561,9 @@ def addAttrs(fobj, eattrs): def setNodeDict(node, dict): for key in dict.keys(): value = dict[key] - if (isinstance(value, numpy.integer) or isinstance(value, types.IntType)): + if (isinstance(value, numpy.integer) or isinstance(value, int)): datatype = CdLong - elif (isinstance(value, numpy.floating) or isinstance(value, types.FloatType)): + elif (isinstance(value, numpy.floating) or isinstance(value, float)): datatype = CdDouble else: datatype = CdString @@ -696,7 +694,7 @@ def main(argv): if flag=='-a': aliasMapFile = arg elif flag=='-c': - calenkey = string.lower(arg) + calenkey = arg.lower() calendar = calendarMap[calenkey] overrideCalendar = 1 elif flag=='-d': @@ -717,7 +715,7 @@ def main(argv): elif flag=='--exclude': if arg[0]=='-': raise RuntimeError, "--exclude option requires an argument" - excludeList = string.split(arg,',') + excludeList = arg.split(',') elif flag=='--exclude-file': excludePattern = arg elif flag=='-f': @@ -732,7 +730,7 @@ def main(argv): sys.exit(0) elif flag=='-i': splitOnTime = 1 - referenceDelta = string.atof(arg) + referenceDelta = float(arg) timeIsLinear = 1 timeIsVector = None elif flag=='--ignore-open-error': @@ -740,7 +738,7 @@ def main(argv): elif flag=='--include': if arg[0]=='-': raise RuntimeError, "--include option requires an argument" - includeList = string.split(arg,',') + includeList = arg.split(',') elif flag=='--include-file': includePattern = arg elif flag=='-j': @@ -748,8 +746,8 @@ def main(argv): timeIsLinear = None elif flag=='-l': splitOnLevel = 1 - levelstr = string.split(arg,',') - levellist = map(string.atof, levelstr) + levelstr = arg.split(',') + levellist = map(float, levelstr) levels = numpy.array(levellist) levels = numpy.sort(levels) elif flag=='-m': @@ -771,19 +769,19 @@ def main(argv): timeid = arg args.append(('-e','%s.axis=T'%timeid)) # Add axis=T attribute elif flag=='--time-linear': - targlist = string.split(arg,',') - ttzero = string.atof(targlist[0]) - tdelta = string.atof(targlist[1]) - tunits = string.strip(targlist[2]) + targlist = arg.split(',') + ttzero = float(targlist[0]) + tdelta = float(targlist[1]) + tunits = targlist[2].strip() if len(targlist)==4: - tcalendar = string.strip(targlist[3]) + tcalendar = targlist[3].strip() else: tcalendar = None overrideTimeLinear = [ttzero,tdelta,tunits,tcalendar] elif flag=='--var-locate': if varLocate is None: varLocate = {} - vname, pattern = string.split(arg,',') + vname, pattern = arg.split(',') varLocate[vname]=pattern elif flag=='-x': writeToStdout = 0 @@ -806,7 +804,7 @@ def main(argv): # Ignore blank paths realargs = [] for arg in lastargs: - sarg = string.strip(arg) + sarg = arg.strip() if len(sarg)>0: realargs.append(sarg) lastargs = realargs @@ -816,7 +814,7 @@ def main(argv): dsetargs = [] for arg in lastargs: base, suffix = os.path.splitext(arg) - if string.lower(suffix) in ['.xml','.cdml']: + if suffix.lower() in ['.xml','.cdml']: dsetargs.append(arg) else: fileargs.append(arg) @@ -872,7 +870,7 @@ def main(argv): modelMap = {} modelDirs = [] for line in mfile.readlines(): - mdirec, model = string.split(line) + mdirec, model = line.split() modelMap[mdirec] = model modelDirs.append(mdirec) mfile.close() @@ -882,10 +880,10 @@ def main(argv): aliasMap = {} for line in afile.readlines(): if line[0] not in ["'",'"']: #" - varid, alias = string.split(line) + varid, alias = line.split() else: - dummy, varid, alias = string.split(line,line[0]) - alias = string.strip(alias) + dummy, varid, alias = line.split(line[0]) + alias = alias.strip() aliasMap[varid] = alias afile.close() @@ -983,7 +981,7 @@ def main(argv): boundsmap = {} # boundsmap : varid => timebounds_id boundsdict = {} # Same as vardict for time bounds for path in fileargs: - path = string.strip(path) + path = path.strip() # Check if the path is included if includePattern is not None: @@ -1021,7 +1019,7 @@ def main(argv): if mo is not None: suffixPattern = modelMap[direc] def gensuffix(m, mo=mo): - i = string.atoi(m.group(1)) + i = int(m.group(1)) return mo.group(i) varsuffix = re.sub(r'\\g<(\d)>', gensuffix, suffixPattern) break @@ -1191,7 +1189,7 @@ def main(argv): if verbose and not forecast: print 'Setting reference time units to', referenceTime if timeIsLinear is None and timeIsVector is None: - timeIsLinear = (string.lower(string.split(referenceTime)[0]) in ['hour','hours','minute','minutes','second','seconds']) + timeIsLinear = referenceTime.split()[0].lower() in ['hour','hours','minute','minutes','second','seconds'] if timeIsLinear and verbose: print 'Setting time representation to "linear"' #' if timeIsLinear and referenceDelta is None: @@ -1301,7 +1299,7 @@ def main(argv): if prevcal is not None and calendar != prevcal: sameCalendars = 0 prevcal = calendar - if string.find(units," as ")==-1: + if units.find(" as ")==-1: time0 = cdtime.reltime(values[0],units) else: time0 = cdtime.abstime(values[0],units) @@ -1319,7 +1317,7 @@ def main(argv): if prevcal is not None and calendar != prevcal: sameCalendars = 0 prevcal = calendar - if string.find(units," as ")==-1: + if units.find(" as ")==-1: time0 = cdtime.reltime(values[0],units) else: time0 = cdtime.abstime(values[0],units) @@ -1460,7 +1458,7 @@ def main(argv): domain, attributes, tcode = vardict[varname] for i in range(len(domain)): item = domain[i] - if type(item)==types.StringType and item==name: + if isinstance(item, basestring) and item==name: domain[i] = axisobj # Add bounds variables to vardict, varindex @@ -1479,7 +1477,7 @@ def main(argv): if reprVar in varids: varids.append(boundsname) tmpdom = boundsinfo[0] - if type(tmpdom[1])==types.StringType: + if isinstance(tmpdom[1], basestring): bndsobj = tmpdom[0] boundsdomain = (bndsobj, axisobj) else: @@ -1547,10 +1545,10 @@ def main(argv): # Generate the cdms_filemap attribute cdms_filemap = str(cdms_filemap_list) - cdms_filemap = string.replace(cdms_filemap, ' ', '') - cdms_filemap = string.replace(cdms_filemap, 'None', '-') - cdms_filemap = string.replace(cdms_filemap, '"', '') #" - cdms_filemap = string.replace(cdms_filemap, "'", '') + cdms_filemap = cdms_filemap.replace(' ', '') + cdms_filemap = cdms_filemap.replace('None', '-') + cdms_filemap = cdms_filemap.replace('"', '') + cdms_filemap = cdms_filemap.replace("'", '') # Dump to XML datasetnode = cdmsNode.DatasetNode(datasetid) From c9da8fbba9ae83414658ee040f63c41c0eef6f6c Mon Sep 17 00:00:00 2001 From: Charles Doutriaux Date: Tue, 26 Apr 2016 09:07:43 -0700 Subject: [PATCH 02/11] cleaned up avariable from using types or string module --- Packages/cdms2/Lib/avariable.py | 55 ++++++++++++++------------------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/Packages/cdms2/Lib/avariable.py b/Packages/cdms2/Lib/avariable.py index dd2a2792db..bc20b8be53 100644 --- a/Packages/cdms2/Lib/avariable.py +++ b/Packages/cdms2/Lib/avariable.py @@ -3,8 +3,6 @@ "CDMS Variable objects, abstract interface" import numpy -import types -import string import re import warnings import cdmsNode @@ -404,7 +402,7 @@ def getMissing(self, asarray=0): if asarray==0 and isinstance(mv, numpy.ndarray): mv = mv[0] - if type(mv) is types.StringType and self.dtype.char not in ['?','c','O','S']: + if isinstance(mv, basestring) and self.dtype.char not in ['?','c','O','S']: mv = float(mv) return mv @@ -423,17 +421,16 @@ def setMissing(self, value): return selftype = self.typecode() - valuetype = type(value) - if valuetype is numpy.ndarray: + if isnstance(value, numpy.ndarray): value = value.astype(selftype).item() - elif isinstance(value, numpy.floating) or isinstance(value, numpy.integer): + elif isinstance(value, (numpy.floating, numpy.integer)): value = numpy.array([value], selftype) - elif valuetype in [types.FloatType, types.IntType, types.LongType, types.ComplexType]: + elif isinstance(value, (float, int, long, complex)): try: value = numpy.array([value], selftype) except: # Set fill value when ar[i:j] returns a masked value value = numpy.array([numpy.ma.default_fill_value(self)], selftype) - elif isinstance(value,(str,numpy.string_,numpy.str,numpy.string0,numpy.str_)) and selftype in ['?','c','O','S']: # '?' for Boolean and object + elif isinstance(value,(basestring,numpy.string_,numpy.str,numpy.string0,numpy.str_)) and selftype in ['?','c','O','S']: # '?' for Boolean and object pass else: raise CDMSError, 'Invalid missing value %s'%`value` @@ -1089,7 +1086,7 @@ def regrid (self, togrid, missing=None, order=None, mask=None, **keywords): if re.search('^regrid', regridTool, re.I): if keywords.has_key('diag') and \ - type(keywords['diag']) == types.DictType: + isinstance(keywords['diag'], dict): keywords['diag']['regridTool'] = 'regrid' # the original cdms2 regridder @@ -1206,7 +1203,7 @@ def _single_specs (self, specs): if specs[i] is Ellipsis: j = myrank - (nsupplied - (i+1)) else: - if isinstance(specs[i], types.IntType): + if isinstance(specs[i], int): singles.append(j) j = j + 1 i = i + 1 @@ -1227,15 +1224,15 @@ def specs2slices (self, speclist, force=None): slicelist = [] for i in range(self.rank()): key = speclist[i] - if isinstance(key, types.IntType): # x[i] + if isinstance(key, int): # x[i] slicelist.append (slice(key,key+1)) - elif type(key) is types.SliceType: # x[i:j:k] + elif isinstance(key, slice): # x[i:j:k] slicelist.append(key) elif key is unspecified or key is None or key == ':': slicelist.append (slice(0, len(self.getAxis(i)))) elif key is Ellipsis: raise CDMSError, "Misuse of ellipsis in specification." - elif type(key) is types.TupleType: + elif sinstance(key, tuple): slicelist.append(slice(*key)) else: raise CDMSError, 'invalid index: %s'% str(key) @@ -1287,13 +1284,13 @@ def reg_specs2slices(self, initspeclist,force=None): for i in range(self.rank()): item = speclist[i] - if isinstance(item, types.SliceType): + if isinstance(item, slice): newitem = item elif item==':' or item is None or item is unspecified: axis = self.getAxis(i) newitem = slice(0,len(axis)) - elif isinstance(item, types.ListType) or \ - isinstance(item, types.TupleType): + elif isinstance(item, list) or \ + isinstance(item, tuple): axis = self.getAxis(i) if len(item)==2: # (start,end) indexInterval = axis.mapIntervalExt(item) @@ -1314,13 +1311,7 @@ def reg_specs2slices(self, initspeclist,force=None): if indexInterval is None: raise CDMSError, OutOfRange + str(item) newitem = slice(indexInterval[0],indexInterval[1],indexInterval[2]) - elif isinstance(item, numpy.floating) or \ - isinstance(item, types.FloatType) or \ - isinstance(item, numpy.integer) or \ - isinstance(item, types.IntType) or \ - isinstance(item, types.LongType) or \ - isinstance(item, types.StringType) or \ - type(item) in CdtimeTypes: + elif isinstance(item, (numpy.floating, float, numpy.integer, int, long, basestring)) or type(item) in CdtimeTypes: axis = self.getAxis(i) # # default is 'ccn' in axis.mapIntervalExt @@ -1397,10 +1388,10 @@ def getGridIndices(self): # numpy.ma overrides def __getitem__(self, key): - if type(key) is types.TupleType: + if isinstance(key, tuple): speclist = self._process_specs(key, {}) else: - if isinstance(key, types.IntType) and key>=len(self): + if isinstance(key, int) and key>=len(self): raise IndexError, "Index too large: %d"%key speclist = self._process_specs([key], {}) @@ -1509,7 +1500,7 @@ def orderparse (order): remaining axes. (name) meaning an axis whose id is name """ - if not isinstance(order, types.StringType): + if not isinstance(order, basestring): raise CDMSError, 'order arguments must be strings.' pos = 0 result=[] @@ -1523,8 +1514,8 @@ def orderparse (order): elif r == '...': r = Ellipsis elif len(r) == 1: - if r in string.digits: - r = string.atoi(r) + if r in '0123456789': + r = int(r) result.append(r) pos = m.end(0) @@ -1544,9 +1535,9 @@ def order2index (axes, order): remaining axes. (name) meaning an axis whose id is name """ - if isinstance(order, types.StringType): + if isinstance(order, basestring): result = orderparse(order) - elif isinstance(order, types.ListType): + elif isinstance(order, list): result = order else: raise CDMSError, 'order2index, order specified of bad type:' + str(type(order)) @@ -1557,7 +1548,7 @@ def order2index (axes, order): pos = 0 while j < len(result): item = result[j] - if isinstance(item, types.StringType): + if isinstance(item, basestring): if item == 't': spec = 'time' elif item == 'x': @@ -1581,7 +1572,7 @@ def order2index (axes, order): break else: raise CDMSError, 'No axis matching order spec %s' %str(item) - elif isinstance(item, types.IntType): + elif isinstance(item, int): if item in permutation: raise CDMSError, 'Duplicate item in order %s' % order if item >= n: From c9956674eb09da35755504d376765dd4f47630bb Mon Sep 17 00:00:00 2001 From: Charles Doutriaux Date: Tue, 26 Apr 2016 09:16:39 -0700 Subject: [PATCH 03/11] cleaned up cdmsobj from using types or string module --- Packages/cdms2/Lib/cdmsobj.py | 59 +++++++++++++++++------------------ 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/Packages/cdms2/Lib/cdmsobj.py b/Packages/cdms2/Lib/cdmsobj.py index 14c2d9bf0a..b66cf73079 100644 --- a/Packages/cdms2/Lib/cdmsobj.py +++ b/Packages/cdms2/Lib/cdmsobj.py @@ -7,10 +7,7 @@ import glob import os import re -import string import sys -import types -#import internattr # Data types @@ -204,7 +201,7 @@ def generateTime(matchobj,timespecs): if yrspec: pat,name,dimtype,pos = _specifierMap[yrspec] yrstr = matchobj.group(name) - iyr = string.atoi(yrstr) + iyr = int(yrstr) # Map 2-digit year to [1950,2049) if yrspec in ('%y','%ey'): @@ -220,23 +217,23 @@ def generateTime(matchobj,timespecs): elif mospec in ('%g','%eg'): imo = _monthMapLower[mostr] elif mospec in ('%m','%em','%n','%en'): - imo = string.atoi(mostr) + imo = int(mostr) if dyspec: pat,name,dimtype,pos = _specifierMap[dyspec] dystr = matchobj.group(name) - idy = string.atoi(dystr) + idy = int(dystr) if hrspec: pat,name,dimtype,pos = _specifierMap[hrspec] hrstr = matchobj.group(name) - ihr = string.atoi(hrstr) + ihr = int(hrstr) if mispec: pat,name,dimtype,pos = _specifierMap[mispec] mistr = matchobj.group(name) - imi = string.atoi(mistr) + imi = int(mistr) if sespec: pat,name,dimtype,pos = _specifierMap[sespec] sestr = matchobj.group(name) - ise = string.atoi(sestr) + ise = int(sestr) return cdtime.comptime(iyr,imo,idy,ihr,imi,ise) # Find all files in 'direc' which match 'template'. @@ -280,9 +277,9 @@ def matchingFiles(direc,template): if dimtypes['etime'] != [None,None,None,None,None,None]: matchnames[_etime] = generateTime(matchobj,dimtypes['etime']) if dimtypes['level'] is not None: - matchnames[_level] = string.atoi(matchobj.group('level')) + matchnames[_level] = int(matchobj.group('level')) if dimtypes['elevel'] is not None: - matchnames[_elevel] = string.atoi(matchobj.group('elevel')) + matchnames[_elevel] = int(matchobj.group('elevel')) matchfiles.append((candfile,matchnames)) return matchfiles @@ -300,22 +297,22 @@ def getTimeAsString(spec,time): elif spec in ('%S','%eS'): specstr = str(int(time.second)) elif spec in ('%Y','%eY'): - specstr = string.zfill(str(time.year),4) + specstr = str(time.year).zfill(4) elif spec in ('%d','%ed'): specstr = str(time.day) elif spec in ('%f','%ef'): - specstr = string.zfill(str(time.day),2) + specstr = str(time.day).zfill(2) elif spec in ('%g','%eg'): imo = time.month specstr = _monthListLower[imo-1] elif spec in ('%h','%eh'): - specstr = string.zfill(str(time.hour),2) + specstr = str(time.hour).zfill(2) elif spec in ('%m','%em'): specstr = str(time.month) elif spec in ('%n','%en'): - specstr = string.zfill(str(time.month),2) + specstr = str(time.month).zfill(2) elif spec in ('%y','%ey'): - specstr = string.zfill(str(time.year%100),2) + specstr = str(time.year%100).zfill(2) elif spec in ('%z','%ez'): specstr = getTimeAsString('%H',time)+'Z'+getTimeAsString('%Y',time)+getTimeAsString('%n',time)+getTimeAsString('%d',time) return specstr @@ -374,9 +371,9 @@ def retpath(matchobj, matchnames=matchnames): def searchPattern(objlist, pattern, attribute=None, tag=None): if tag is not None: - tag = string.lower(tag) + tag = tag.lower() regexp = re.compile(pattern) - if type(objlist) is not types.ListType: + if not isinstance(objlist, list): objlist = [objlist] returnlist = [] @@ -395,9 +392,9 @@ def searchPattern(objlist, pattern, attribute=None, tag=None): def matchPattern(objlist, pattern, attribute=None, tag=None): if tag is not None: - tag = string.lower(tag) + tag = tag.lower() regexp = re.compile(pattern) - if type(objlist) is not types.ListType: + if not isinstance(objlist, list): objlist = [objlist] returnlist = [] @@ -425,8 +422,8 @@ def matchPattern(objlist, pattern, attribute=None, tag=None): def searchPredicate(objlist, predicate, tag=None): if tag is not None: - tag = string.lower(tag) - if type(objlist) is not types.ListType: + tag = tag.lower() + if not isinstance(objlist, list): objlist = [objlist] returnlist = [] @@ -490,12 +487,12 @@ def __init__(self, node = None): if reqtype!=datatype and datatype==CdString and scaletype==CdScalar: if reqtype in (CdFloat,CdDouble): try: - attval = string.atof(attval) + attval = float(attval) except: raise RuntimeError,"%s=%s must be a float"%(attname,attval) elif reqtype in (CdShort,CdInt,CdLong,CdInt64): try: - attval = string.atoi(attval) + attval = int(attval) except: raise RuntimeError,"%s=%s must be an integer"%(attname,attval) adict[attname] = attval @@ -519,12 +516,12 @@ def searchone(self, pattern, attname): """ if attname is None: for attval in self.attributes.values(): - if type(attval) is types.StringType and pattern.search(attval) is not None: + if isinstance(attval,basestring) and pattern.search(attval) is not None: return 1 return 0 elif self.attributes.has_key(attname): attval = self.attributes[attname] - return (type(attval) is types.StringType and pattern.search(attval) is not None) + return isinstance(attval,basestring) and pattern.search(attval) is not None else: return 0 @@ -549,12 +546,12 @@ def matchone(self, pattern, attname): """ if attname is None: for attval in self.attributes.values(): - if type(attval) is types.StringType and pattern.match(attval) is not None: + if isinstance(attval,basestring) and pattern.search(attval) is not None: return 1 return 0 elif self.attributes.has_key(attname): attval = self.attributes[attname] - return (type(attval) is types.StringType and pattern.match(attval) is not None) + return isinstance(attval,basestring) and pattern.search(attval) is not None else: return 0 @@ -573,7 +570,7 @@ def searchPattern(self,pattern,attribute,tag): result :: (list) (0) ::: """ - if tag is None or string.lower(tag)==self._node_.tag: + if tag is None or tag.lower()==self._node_.tag: if self.searchone(pattern,attribute): return [self] else: @@ -596,7 +593,7 @@ def matchPattern(self,pattern,attribute,tag): result :: (list) (0) ::: """ - if tag is None or string.lower(tag)==self._node_.tag: + if tag is None or tag.lower()==self._node_.tag: if self.matchone(pattern,attribute): return [self] else: @@ -619,7 +616,7 @@ def searchPredicate(self,predicate,tag): result :: (list) (0) ::: """ - if tag is None or string.lower(tag)==self._node_.tag: + if tag is None or tag.lower()==self._node_.tag: try: if apply(predicate,(self,))==1: result = [self] From fb948093cd0f4d5eea14203c56fad2c707f1a019 Mon Sep 17 00:00:00 2001 From: Charles Doutriaux Date: Tue, 26 Apr 2016 09:40:16 -0700 Subject: [PATCH 04/11] cleaned up axis.py --- Packages/cdms2/Lib/axis.py | 92 +++++++++--------- Packages/cdms2/Lib/internattr.py | 157 ------------------------------- 2 files changed, 46 insertions(+), 203 deletions(-) delete mode 100644 Packages/cdms2/Lib/internattr.py diff --git a/Packages/cdms2/Lib/axis.py b/Packages/cdms2/Lib/axis.py index 703c0e938c..458096c221 100644 --- a/Packages/cdms2/Lib/axis.py +++ b/Packages/cdms2/Lib/axis.py @@ -6,7 +6,7 @@ """ _debug = 0 std_axis_attributes = ['name', 'units', 'length', 'values', 'bounds'] -import string, sys, types, copy +import sys, copy import numpy # import regrid2._regrid import cdmsNode @@ -22,13 +22,13 @@ class AliasList (UserList): def __init__(self, alist): UserList.__init__(self,alist) def __setitem__ (self, i, value): - self.data[i] = string.lower(value) + self.data[i] = value.lower() def __setslice(self, i, j, values): - self.data[i:j] = map(lambda x: string.lower(x), values) + self.data[i:j] = map(lambda x: x.lower(), values) def append(self, value): - self.data.append(string.lower(value)) + self.data.append(value.lower()) def extend(self, values): - self.data.extend(map(string.lower, values)) + self.data.extend(map("".lower, values)) level_aliases = AliasList(['plev']) longitude_aliases = AliasList([]) @@ -228,7 +228,7 @@ def mapLinearExt(axis, bounds, interval, indicator ='ccn', epsilon=None, stride= intersection is empty. """ - indicator = string.lower(indicator) + indicator = indicator.lower() length = len(axis) # Make the interval and search array non-decreasing @@ -615,7 +615,7 @@ def __init__ (self, parent, node): self._doubledata_ = None def __str__ (self): - return string.join(self.listall(), "\n") + "\n" + return "\n".join(self.listall() + "\n" __repr__ = __str__ @@ -812,14 +812,14 @@ def asDTGTime(self, calendar=None): for val in self[:]: comptime=cdtime.reltime(val, self.units).tocomp(calendar) s=repr(comptime) - tt=string.split(s,' ') + tt=s.split(' ') - ttt=string.split(tt[0],'-') + ttt=tt[0].split('-') yr=int(ttt[0]) mo=int(ttt[1]) da=int(ttt[2]) - ttt=string.split(tt[1],':') + ttt=tt[1].split(':') hr=int(ttt[0]) dtg="%04d%02d%02d%02d"%(yr,mo,da,hr) result.append(dtg) @@ -974,7 +974,7 @@ def setBounds(self, bounds): # calendar. def getCalendar(self): if hasattr(self,'calendar'): - calendar = string.lower(self.calendar) + calendar = self.calendar.lower() else: calendar = None @@ -1009,7 +1009,7 @@ def _time2value(self, value): if self.isTime(): if type(value) in CdtimeTypes: value = value.torel(self.units, self.getCalendar()).value - elif type(value) is types.StringType and value not in [':',unspecified]: + elif isinstance(value, basestring) and value not in [':',unspecified]: cal = self.getCalendar() value = cdtime.s2c(value, cal).torel(self.units, cal).value return value @@ -1022,7 +1022,7 @@ def getModuloCycle(self): # # mf 20010419 test if attribute is a string (non CF), set to 360.0 # - if(type(cycle) == types.StringType): + if isnstance(cycle, basestring): cycle = 360.0 else: cycle = 360.0 @@ -1106,7 +1106,7 @@ def mapIntervalExt(self,interval,indicator='ccn',cycle=None,epsilon=None): # check length of indicator if overridden by user # - indicator = string.lower(indicator) + indicator = indicator.lower() if len(indicator)==2: indicator += 'n' if( ( len(indicator) != 3 ) or @@ -1530,24 +1530,24 @@ def __getitem__(self, key): length = len(node) # Allow key of form (slice(i,j),) etc. - if type(key) is types.TupleType and len(key)==1: + if isinstance(key, tuple) and len(key)==1: key = key[0] - if isinstance(key, (types.IntType, numpy.int,numpy.int32)): # x[i] + if isinstance(key, (int, numpy.int,numpy.int32)): # x[i] if key>=length: raise IndexError, 'index out of bounds' else: # Don't generate the entire array (if linear) just for one value return node.data[key%length] - elif type(key) is types.SliceType: # x[i:j:k] + elif isinstance(key, slice): # x[i:j:k] if self._data_ is None: self._data_ = node.getData() return self._data_[key.start:key.stop:key.step] - elif type(key) is types.EllipsisType: # x[...] + elif isinstance(key, Ellipsis.__class__): # x[...] if self._data_ is None: self._data_ = node.getData() return self._data_ - elif type(key) is types.TupleType: + elif isinstance(key, tuple): raise IndexError,'axis is one-dimensional' else: raise IndexError,'index must be an integer: %s'%`key` @@ -1597,9 +1597,9 @@ def getExplicitBounds(self): def getCalendar(self): if hasattr(self,'calendar'): - calendar = string.lower(self.calendar) + calendar = self.calendar.lower() elif self.parent is not None and hasattr(self.parent, 'calendar'): - calendar = string.lower(self.parent.calendar) + calendar = self.parent.calendar.lower() else: calendar = None @@ -1781,7 +1781,7 @@ def __init__(self, parent, axisname, obj=None): if obj is not None: for attname in self._obj_.__dict__.keys(): attval = getattr(self._obj_,attname) - if type(attval)!=types.BuiltinFunctionType: + if not callable(attval): self.__dict__[attname] = attval att = self.attributes att[attname]=attval @@ -1879,29 +1879,29 @@ def __getitem__(self, key): if (self._obj_ is not None) and (self.parent._mode_!='r') and not (hasattr(self.parent,'format') and self.parent.format=="DRS"): # For negative strides, get the equivalent slice with positive stride, # then reverse the result. - if (type(key) is types.SliceType) and (key.step is not None) and key.step<0: + if (isinstance(key, slice) and (key.step is not None) and key.step<0: posslice = reverseSlice(key,len(self)) result = apply(self._obj_.getitem, (posslice,)) return result[::-1] else: - if isinstance(key, types.IntType) and key>=len(self): + if isinstance(key, int) and key>=len(self): raise IndexError, 'Index out of bounds: %d'%key - if type(key) is not types.TupleType: + if isinstance(key, tuple): key = (key,) return apply(self._obj_.getitem, key) if self._data_ is None: self._data_ = self.getData() length = len(self._data_) - if isinstance(key, types.IntType): # x[i] + if isinstance(key, int): # x[i] if key>=length: raise IndexError, 'index out of bounds' else: return self._data_[key%length] - elif type(key) is types.SliceType: # x[i:j:k] + elif isinstance(key, slice): # x[i:j:k] return self._data_[key.start:key.stop:key.step] - elif type(key) is types.EllipsisType: # x[...] + elif isinstance(key, Ellipsis.__class__): # x[...] return self._data_ - elif type(key) is types.TupleType: + elif isinstance(key, tuple): raise IndexError,'axis is one-dimensional' else: raise IndexError,'index must be an integer or slice: %s'%`key` @@ -2027,9 +2027,9 @@ def setBounds(self, bounds, persistent=0, validate=0, index=None, boundsid=None) def getCalendar(self): if hasattr(self,'calendar'): - calendar = string.lower(self.calendar) + calendar = self.calendar.lower() elif self.parent is not None and hasattr(self.parent, 'calendar'): - calendar = string.lower(self.parent.calendar) + calendar = self.parent.calendar.lower() else: calendar = None @@ -2126,15 +2126,15 @@ def axisMatchIndex (axes, specifications=None, omit=None, order=None): """ if specifications is None: speclist = axes - elif isinstance(specifications, types.StringType): + elif isinstance(specifications, basestring): speclist = [specifications] - elif isinstance(specifications, types.ListType): + elif isinstance(specifications, list): speclist = specifications - elif isinstance(specifications, types.TupleType): + elif isinstance(specifications, tuple): speclist=list(specifications) - elif isinstance(specifications, types.IntType): + elif isinstance(specifications, int): speclist = [specifications] - elif isinstance(specifications, types.FunctionType): + elif callable(specifications): speclist = [specifications] else: # to allow arange, etc. speclist = list(numpy.ma.filled(specifications)) @@ -2142,7 +2142,7 @@ def axisMatchIndex (axes, specifications=None, omit=None, order=None): candidates = [] for i in range(len(axes)): for s in speclist: - if isinstance(s, types.IntType): + if isinstance(s, int): r = (s == i) else: r = axisMatches(axes[i], s) @@ -2155,15 +2155,15 @@ def axisMatchIndex (axes, specifications=None, omit=None, order=None): if omit is None: omitlist = [] - elif isinstance(omit, types.StringType): + elif isinstance(omit, basestring): omitlist = [omit] - elif isinstance(omit, types.ListType): + elif isinstance(omit, list): omitlist = omit - elif isinstance(omit, types.TupleType): + elif isinstance(omit, tuple): omitlist=list(omit) - elif isinstance(omit, types.IntType): + elif isinstance(omit, int): omitlist = [omit] - elif isinstance(omit, types.FunctionType): + elif callable(omit): omitlist = [omit] elif isinstance(omit, AbstractAxis): omitlist = [omit] @@ -2171,7 +2171,7 @@ def axisMatchIndex (axes, specifications=None, omit=None, order=None): raise CDMSError, 'Unknown type of omit specifier.' for s in omitlist: - if isinstance(s, types.IntType): + if isinstance(s, int): for i in range(len(candidates)): if axes[candidates[i]] is axes[s]: del candidates[i] @@ -2251,13 +2251,13 @@ def axisMatches(axis, specification): 3. an axis object; will match if it is the same object as axis. """ if isinstance(specification, basestring): - s = string.lower(specification) + s = specification.lower() s = s.strip() while s[0] == '(': if s[-1] != ')': raise CDMSError, 'Malformed axis spec, ' + specification s = s[1:-1].strip() - if string.lower(axis.id) == s: + if axis.id.lower() == s: return 1 elif (s == 'time') or (s in time_aliases): return axis.isTime() @@ -2272,7 +2272,7 @@ def axisMatches(axis, specification): else: return 0 - elif isinstance(specification, types.FunctionType): + elif callable(specification): r = specification(axis) if r: return 1 diff --git a/Packages/cdms2/Lib/internattr.py b/Packages/cdms2/Lib/internattr.py deleted file mode 100644 index ff6ebeb0c9..0000000000 --- a/Packages/cdms2/Lib/internattr.py +++ /dev/null @@ -1,157 +0,0 @@ -"InternalAttributes (implmentation class for CDMS)" -import types -import PropertiedClasses -_PCLASS = PropertiedClasses.PropertiedClass -class AttributeDict: - """An attribute dictionary.""" - def __init__ (self, owner): - self._owner = owner - - def __getitem__ (self, name): - if self.has_key(name): - return self._owner.__dict__[name] - else: - raise KeyError, "%s instance has no external attribute %s" % \ - (self._owner.__class__.__name__, name) - - def __setitem__ (self, name, value): - if self._owner.is_internal_attribute(name): - raise RuntimeError, 'Cannot set internal name in external attribute dictionary.' - self._owner.__dict__[name] = value - - def clear (self): - self._owner.__dict__.clear() - - def get (self, name, default=None): - if self.has_key(name): - return self._owner.__dict__[name] - else: - return default - - def has_key(self, name): - d = self._owner.__dict__ - if d.has_key(name) and not self._owner.is_internal_attribute(name): - return 1 - else: - return 0 - - def items (self): - result = [] - for name, value in self._owner.__dict__.items(): - if self._owner.is_internal_attribute(name): continue - result.append((name, value)) - return result - - def keys (self): - result = [] - for name in self._owner.__dict__.keys(): - if self._owner.is_internal_attribute(name): continue - result.append(name) - return result - - def update(self, d): - for name, value in d.items(): - if self._owner.is_internal_attribute(name): - raise RuntimeError, "Cannot update attribute dict with internal name" - self._owner.__dict__[name] = value - - def values (self): - result = [] - for name, value in self._owner.__dict__.items(): - if self._owner.is_internal_attribute(name): continue - result.append(value) - return result - - def __repr__(self): - return 'AttributeDict (' + \ - repr(self._owner.__dict__) + \ - ')' - - def __str__(self): - return str(self._owner.__dict__) - -class InternalAttributesClass (_PCLASS): - def _getattributes (self, name): - """Return a dictionary-like object of the non-internal attributes.""" - return AttributeDict(self) - - def is_internal_attribute (self, name): - """is_internal_attribute(name) is true if name is internal.""" - if name[0] == '_' or name in self.__class__._internal: - return 1 - return 0 - - def replace_external_attributes(self, newAttributes): - """replace_external_attributes(newAttributes) - Replace the external attributes with dictionary newAttributes. - """ - if not isinstance(newAttributes, types.DictType) and \ - not isinstance(newAttributes, AttributeDict): - raise ValueError, "Argument must be a dictionary" - for n in self.__dict__.keys(): - if not self.is_internal_attribute(n): - del self.__dict__[n] - for n, v in newAttributes.items(): - self.__dict__[n] = v - -def initialize_internal_attributes (C): - "Prepare a class for life as a child of InternalAttributesClass." - if C.__dict__.has_key('_internal'): return - if not issubclass(C, InternalAttributesClass): - raise ValueError, 'Must be subclass of InternalAttributesClass' - PropertiedClasses.initialize_property_class (C) - C._internal = [] - for CP in C.__bases__: - if issubclass(CP, InternalAttributesClass): - initialize_internal_attributes(CP) - for name in CP._internal: - C._internal.append(name) - -def add_internal_attribute (C, *aname): - """add_internal_attribute (C, name, ...) - Make attributes name, ... internal in class C. - """ - initialize_internal_attributes(C) - for name in aname: - if not name in C._internal: - C._internal.append(name) - -PropertiedClasses.set_property(InternalAttributesClass, 'attributes', - InternalAttributesClass._getattributes, - nowrite=1, nodelete=1) - -if __name__ == '__main__': - class Test(InternalAttributesClass): - def __init__ (self): - self.node = None - self.parent = None - self.__dict__['ro'] = 1 - self.__hide = 3 - self._p = 4 - self.value = 1 - - PropertiedClasses.set_property(Test, 'ro', nowrite=1, nodelete=1) - add_internal_attribute(Test, 'node', 'parent', 'ro') - - t1 = Test() - assert t1.value == 1 - assert not t1.attributes.has_key('__hide') - assert not t1.attributes.has_key('_p') - assert t1._p == 4 - t1.value = 2 - assert t1.value == 2 - assert 'value' in t1.attributes.keys() - t1.b = t1.value + 1 - assert t1.b == 3 - assert t1.b == t1.attributes['b'] - t1.node = 'me' - t1.parent = 'dad' - assert t1.node == 'me' - assert 'node' not in t1.attributes.keys() - assert t1.ro == 1 - try: - t1.ro == 2 - except AttributeError: - pass - assert t1.ro == 1 - print "Test passed." From d40cb17afc176ecc96b35b563a36b8c27966812e Mon Sep 17 00:00:00 2001 From: Charles Doutriaux Date: Tue, 26 Apr 2016 09:51:59 -0700 Subject: [PATCH 05/11] cleaned up cdmsNode --- Packages/cdms2/Lib/cdmsNode.py | 48 +++++++++++++++------------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/Packages/cdms2/Lib/cdmsNode.py b/Packages/cdms2/Lib/cdmsNode.py index 794e4ed3b4..5d82a95265 100644 --- a/Packages/cdms2/Lib/cdmsNode.py +++ b/Packages/cdms2/Lib/cdmsNode.py @@ -9,9 +9,7 @@ import CDML import cdtime import re -import string import sys -from types import * from error import CDMSError # Regular expressions @@ -309,9 +307,9 @@ def write_ldif(self, parentdn, userAttrs=[], fd=None, format=1): if (validAttrs and (attname in validAttrs)) or (not validAttrs): (attval,datatype)=self.attribute[attname] # attvalstr = _Illegal.sub(mapIllegalToEntity,str(attval)) # Map illegal chars to entities - if type(attval)!=StringType: + if not isinstance(attval, basestring): attval = `attval` - attvalstr = string.strip(attval) + attvalstr = attval.strip() attvalstr = re.sub('\n','\n ',attvalstr) # Make sure continuation lines are preceded with a space if attvalstr=='': attvalstr = "none" fd.write("%s: %s\n"%(attname,attvalstr)) @@ -320,7 +318,7 @@ def write_ldif(self, parentdn, userAttrs=[], fd=None, format=1): for attname in self.attribute.keys(): if validAttrs and (attname not in validAttrs): (attval,datatype)=self.attribute[attname] - if type(attval)!=StringType: + if not isinstance(attval, basestring): attval = `attval` attval = re.sub('\n','\n ',attval) # Make sure continuation lines are preceded with a space fd.write("attr: %s=%s\n"%(attname,attval)) @@ -332,7 +330,7 @@ def write_ldif(self, parentdn, userAttrs=[], fd=None, format=1): # fd.write("value: %s"%(content,)) # Write user attributes - if type(userAttrs)==StringType: + if isinstance(userAttrs, basestring): newAttrs = [userAttrs] else: newAttrs = userAttrs @@ -356,7 +354,7 @@ def validate(self,idtable=None): for attname in self.attribute.keys(): if attname in validKeys: (atttype,default)=self.dtd[attname] - if type(atttype) is TupleType: + if isinstance(atttype,tuple): attval=self.getExternalAttr(attname) assert attval in atttype, 'Invalid attribute %s=%s must be in %s'%(attname,attval,`atttype`) elif atttype==CDML.Idref: @@ -426,7 +424,7 @@ class VariableNode(CdmsNode): # Create a variable. # If validate is true, validate immediately def __init__(self, id, datatype, domain): - assert type(datatype) is StringType, 'Invalid datatype: '+`datatype` + assert isinstance(datatype,basestring), 'Invalid datatype: '+`datatype` assert datatype in CdDatatypes, 'Invalid datatype: '+`datatype` CdmsNode.__init__(self,"variable",id) self.datatype = datatype @@ -456,8 +454,8 @@ class AxisNode(CdmsNode): # If datatype is None, assume values [0,1,..,length-1] # data is a numpy array, if specified def __init__(self, id, length, datatype=CdLong,data=None): - assert isinstance(length, IntType), 'Invalid length: '+`length` - assert type(datatype) is StringType, 'Invalid datatype: '+`datatype` + assert isinstance(length, int), 'Invalid length: '+`length` + assert isinstance(datatype, basestring), 'Invalid datatype: '+`datatype` assert datatype in CdDatatypes, 'Invalid datatype: '+`datatype` if data is not None: assert isinstance(data, numpy.ndarray), 'data must be a 1-D Numeric array' CdmsNode.__init__(self,"axis",id) @@ -492,7 +490,7 @@ def setContentFromString(self,datastring): numlist = [] for numstring in stringlist: if numstring=='': continue - numlist.append(string.atof(numstring)) + numlist.append(float(numstring)) if len(numlist)>0: # NB! len(zero-length array) causes IndexError on Linux! dataArray = numpy.array(numlist,numericType) @@ -506,7 +504,7 @@ def setPartitionFromString(self,partstring): numlist = [] for numstring in stringlist: if numstring=='': continue - numlist.append(string.atoi(numstring)) + numlist.append(int(numstring)) dataArray = numpy.array(numlist,numpy.int) if len(dataArray)>0: self.partition = dataArray @@ -722,13 +720,13 @@ def __len__(self): # Linear data element class LinearDataNode(CdmsNode): - validStartTypes = [IntType,FloatType,type(cdtime.comptime(0)),type(cdtime.reltime(0,"hours"))] - validDeltaTypes = [IntType,FloatType,ListType] + validStartTypes = [int, float, type(cdtime.comptime(0)), type(cdtime.reltime(0,"hours"))] + validDeltaTypes = [int, float, list] def __init__(self, start, delta, length): assert isinstance(start, numpy.floating) or isinstance(start, numpy.integer) or (type(start) in self.validStartTypes), 'Invalid start argument: '+`start` assert isinstance(start, numpy.floating) or isinstance(start, numpy.integer) or (type(delta) in self.validDeltaTypes), 'Invalid delta argument: '+`delta` - assert isinstance(length, IntType), 'Invalid length argument: '+`length` + assert isinstance(length, int), 'Invalid length argument: '+`length` CdmsNode.__init__(self,"linear") self.delta = delta self.length = length @@ -932,12 +930,8 @@ class AttrNode(CdmsNode): def __init__(self, name, value=None): CdmsNode.__init__(self,"attr") - if not (isinstance(value,IntType) - or isinstance(value,numpy.integer) - or isinstance(value,FloatType) - or isinstance(value,numpy.floating) - or isinstance(value,StringType) - or isinstance(value,NoneType)): + if not (isinstance(value,(int, numpy.integer, float, numpy.floating, basestring)) \ + or value is None: raise CDMSError, 'Invalid attribute type: '+`value` self.name = name self.value = value @@ -954,11 +948,11 @@ def mapToExternal(self): def getDatatype(self): if self.datatype: return self.datatype - elif type(self.value) is StringType: + elif isinstance(self.value, basestring): return CdString - elif isinstance(self.value, FloatType) or isinstance(self.value,numpy.floating): + elif isinstance(self.value, (float,numpy.floating)): return CdDouble - elif isinstance(self.value, IntType) or isinstance(self.value,numpy.integer): + elif isinstance(self.value, (int, numpy.integer)): return CdLong else: raise CDMSError, 'Invalid attribute type: '+`self.value` @@ -970,18 +964,18 @@ def getLength(self): # Returns ValueError if the conversion fails def setValueFromString(self,valString,datatype): val = None - if type(valString) is not StringType: + if not isinstance(valString, basestring): raise CDMSError, 'input value is not a string' if datatype == CdString: val=valString elif datatype in (CdShort,CdInt,CdLong): try: - val=string.atoi(valString) + val=int(valString) except ValueError: raise CDMSError, 'value is not an integer: '+valString elif datatype in (CdFloat,CdDouble): try: - val=string.atof(valString) + val=float(valString) except ValueError: raise CDMSError, 'value is not floating-point: '+valString self.value=val From bf68e04105ca4367cbd048732fa6ecd26b531732 Mon Sep 17 00:00:00 2001 From: Charles Doutriaux Date: Tue, 26 Apr 2016 09:54:00 -0700 Subject: [PATCH 06/11] cleaned up coord --- Packages/cdms2/Lib/coord.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Packages/cdms2/Lib/coord.py b/Packages/cdms2/Lib/coord.py index 6f7773fd1b..8460485e66 100644 --- a/Packages/cdms2/Lib/coord.py +++ b/Packages/cdms2/Lib/coord.py @@ -10,8 +10,6 @@ import copy import numpy #import internattr -import types -import string from cdmsobj import CdmsObj from axis import createAxis, TransientVirtualAxis from error import CDMSError @@ -118,7 +116,7 @@ def designateTime(self, persistent=0, calendar=None): # calendar. def getCalendar(self): if hasattr(self,'calendar'): - calendar = string.lower(self.calendar) + calendar = self.calendar.lower() else: calendar = None @@ -147,7 +145,7 @@ def isLatitude(self): # Return true iff the axis is a level axis def isLevel(self): - id = string.lower(self.id) + id = self.id.lower() if (hasattr(self,'axis') and self.axis=='Z'): return 1 return ((id[0:3] == 'lev') or (id[0:5] == 'depth') or (id in level_aliases)) @@ -158,13 +156,13 @@ def isLongitude(self): # Return true iff the axis is a time axis def isTime(self): - id = string.lower(self.id) + id = self.id.lower() if (hasattr(self,'axis') and self.axis=='T'): return 1 return (id[0:4] == 'time') or (id in time_aliases) # Return true iff the axis is a forecast axis def isForecast(self): - id = string.lower(self.id) + id = self.id.lower() if (hasattr(self,'axis') and self.axis=='F'): return 1 return (id[0:6] == 'fctau0') or (id in forecast_aliases) From f570c2cf6112133f777fd6073769814c99b2b5e1 Mon Sep 17 00:00:00 2001 From: Charles Doutriaux Date: Tue, 26 Apr 2016 11:23:01 -0700 Subject: [PATCH 07/11] cleaned up all (most?) files --- Packages/cdms2/Lib/cudsinterface.py | 6 +++--- Packages/cdms2/Lib/database.py | 8 +++----- Packages/cdms2/Lib/dataset.py | 19 +++++++++---------- Packages/cdms2/Lib/fvariable.py | 1 - Packages/cdms2/Lib/grid.py | 5 ++--- Packages/cdms2/Lib/gsStaticVariable.py | 1 - Packages/cdms2/Lib/mvCdmsRegrid.py | 3 +-- Packages/cdms2/Lib/mvSphereMesh.py | 11 +++++------ Packages/cdms2/Lib/selectors.py | 4 ++-- Packages/cdms2/Lib/slabinterface.py | 6 +++--- Packages/cdms2/Lib/tvariable.py | 9 ++++----- Packages/cdms2/Lib/variable.py | 1 - 12 files changed, 32 insertions(+), 42 deletions(-) diff --git a/Packages/cdms2/Lib/cudsinterface.py b/Packages/cdms2/Lib/cudsinterface.py index c97f5fdefc..4232bdf442 100644 --- a/Packages/cdms2/Lib/cudsinterface.py +++ b/Packages/cdms2/Lib/cudsinterface.py @@ -2,7 +2,7 @@ ## Further modified to be pure new numpy June 24th 2008 "Emulation of old cu package" -import string, types, sys +import sys from error import CDMSError from dataset import openDataset, createDataset from tvariable import createVariable @@ -215,7 +215,7 @@ def showall (self, vname=None, all=None, device=None): if device is None: device=sys.stdout if vname is None: vname = self.default_variable_name alist = self.listall(vname, all=all) - device.write(string.join(alist, "\n")) + device.write("\n".join(alist)) device.write("\n") def dimensionobject (self, dname, vname=None): @@ -357,7 +357,7 @@ def getslab (self, vname, *args,**keys): idim = ndims - (nargs - i - 1) i = i + 1 ne = 1 - elif type(x) == types.TupleType: + elif isinstance(x,tuple): cdms_args[idim] = x idim = idim + 1 i = i + 1 diff --git a/Packages/cdms2/Lib/database.py b/Packages/cdms2/Lib/database.py index 80f749fb43..ac8b0dc7ad 100644 --- a/Packages/cdms2/Lib/database.py +++ b/Packages/cdms2/Lib/database.py @@ -7,9 +7,7 @@ import copy import os import re -import string import sys -import types from CDMLParser import CDMLParser from cdmsobj import CdmsObj from dataset import Dataset @@ -205,7 +203,7 @@ def __del__(self): def normalizedn(self, dn): explodeddn = ldap.explode_dn(dn) - return string.join(explodeddn,',') + return ','.join(explodeddn) def cachecdml(self, name, cdml, datapath): normaldn = self.normalizedn(name) @@ -239,7 +237,7 @@ def getObjFromDataset(self, dn): # Get the parent dataset explodeddn = ldap.explode_dn(dn) - dsetdn = string.join(explodeddn[1:],',') # Dataset node is parent of variable + dsetdn = ','.join(explodeddn[1:]) # Dataset node is parent of variable dset = self.getDataset(dsetdn) rdn = explodeddn[0] matchobj = _Att.match(rdn) @@ -468,7 +466,7 @@ def searchPredicate(self, predicate, tag=None): """ if tag is not None: - tag = string.lower(tag) + tag = tag.lower() resultlist = [] for entry in self: diff --git a/Packages/cdms2/Lib/dataset.py b/Packages/cdms2/Lib/dataset.py index 977b1d8bb2..6074615dea 100644 --- a/Packages/cdms2/Lib/dataset.py +++ b/Packages/cdms2/Lib/dataset.py @@ -7,7 +7,6 @@ import numpy import cdmsNode import os, sys -import string import urllib import cdmsURLopener # Import after urllib, to handle errors import urlparse @@ -280,7 +279,7 @@ def openDataset(uri,mode='r',template=None,dods=1,dpath=None, hostObj=None): file :: (cdms2.dataset.CdmsFile) (0) file to read from ::: """ - uri = string.strip(uri) + uri = uri.strip() (scheme,netloc,path,parameters,query,fragment)=urlparse.urlparse(uri) if scheme in ('','file'): if netloc: @@ -417,7 +416,7 @@ def parseIndexList(text): for i in range(nindices): s = m.group(i+1) if s!='-': - result[i] = string.atoi(s) + result[i] = int(s) result[nindices] = m.group(nindices+1) return result, m.end() @@ -744,7 +743,7 @@ def createVariable(self,name,datatype,axisnames): def searchPattern(self,pattern,attribute,tag): resultlist = [] if tag is not None: - tag = string.lower(tag) + tag = tag.lower() if tag in ('dataset',None): if self.searchone(pattern,attribute)==1: resultlist = [self] @@ -769,7 +768,7 @@ def searchPattern(self,pattern,attribute,tag): def matchPattern(self,pattern,attribute,tag): resultlist = [] if tag is not None: - tag = string.lower(tag) + tag = tag.lower() if tag in ('dataset',None): if self.matchone(pattern,attribute)==1: resultlist = [self] @@ -797,7 +796,7 @@ def matchPattern(self,pattern,attribute,tag): def searchPredicate(self,predicate,tag): resultlist = [] if tag is not None: - tag = string.lower(tag) + tag = tag.lower() if tag in ('dataset',None): try: if apply(predicate,(self,))==1: @@ -1548,7 +1547,7 @@ def searchPattern(self,pattern,attribute,tag): """ resultlist = [] if tag is not None: - tag = string.lower(tag) + tag = tag.lower() if tag in ('cdmsFile',None,'dataset'): if self.searchone(pattern,attribute)==1: resultlist = [self] @@ -1585,7 +1584,7 @@ def matchPattern(self,pattern,attribute,tag): """ resultlist = [] if tag is not None: - tag = string.lower(tag) + tag = tag.lower() if tag in ('cdmsFile',None,'dataset'): if self.matchone(pattern,attribute)==1: resultlist = [self] @@ -1624,7 +1623,7 @@ def searchPredicate(self,predicate,tag): """ resultlist = [] if tag is not None: - tag = string.lower(tag) + tag = tag.lower() if tag in ('cdmsFile',None,'dataset'): try: if apply(predicate,(self,))==1: @@ -2017,7 +2016,7 @@ def getBoundsAxis(self, n,boundid=None): def __repr__(self): filerep = `self._file_` - loc = string.find(filerep,"file") + loc = filerep.find("file") if loc==-1: loc=0 return ""%self._status_ diff --git a/Packages/cdms2/Lib/fvariable.py b/Packages/cdms2/Lib/fvariable.py index 4271e12c62..6ca32df157 100644 --- a/Packages/cdms2/Lib/fvariable.py +++ b/Packages/cdms2/Lib/fvariable.py @@ -4,7 +4,6 @@ "CDMS File-based variables." import numpy import typeconv -import types import re from cdmsobj import Max32int diff --git a/Packages/cdms2/Lib/grid.py b/Packages/cdms2/Lib/grid.py index f11ca1764d..05fdf9b7c0 100644 --- a/Packages/cdms2/Lib/grid.py +++ b/Packages/cdms2/Lib/grid.py @@ -2,7 +2,6 @@ ## Further modified to be pure new numpy June 24th 2008 """CDMS Grid objects""" -import types import re from error import CDMSError import numpy #, PropertiedClasses, internattr @@ -129,14 +128,14 @@ def setRegionSpecs(grid, coordSpec, coordType, resultSpec): if (coordSpec is None) or (coordSpec==':'): canonSpec = None - elif type(coordSpec) is types.TupleType: + elif isinstance(coordSpec, tuple): if len(coordSpec)==2: canonSpec = (coordSpec[0],coordSpec[1],'cc',None) elif len(coordSpec)==3: canonSpec = (coordSpec[0],coordSpec[1],coordSpec[2],None) elif len(coordSpec)!=4: raise CDMSError, 'Invalid coordinate specification: %s'%`coordSpec` - elif type(coordSpec) in [types.IntType, types.FloatType]: + elif isinstance(coordSpec, (int, float)): canonSpec = (coordSpec, coordSpec, 'cc', None) else: raise CDMSError, 'Invalid coordinate specification: %s'%`coordSpec` diff --git a/Packages/cdms2/Lib/gsStaticVariable.py b/Packages/cdms2/Lib/gsStaticVariable.py index 0332e14158..fdd740c865 100644 --- a/Packages/cdms2/Lib/gsStaticVariable.py +++ b/Packages/cdms2/Lib/gsStaticVariable.py @@ -9,7 +9,6 @@ import operator import cdms2 -import types from cdms2.error import CDMSError from cdms2.hgrid import AbstractCurveGrid, TransientCurveGrid, FileCurveGrid from cdms2.coord import TransientAxis2D, TransientVirtualAxis diff --git a/Packages/cdms2/Lib/mvCdmsRegrid.py b/Packages/cdms2/Lib/mvCdmsRegrid.py index 6cc0adf8a3..5a1f34361e 100644 --- a/Packages/cdms2/Lib/mvCdmsRegrid.py +++ b/Packages/cdms2/Lib/mvCdmsRegrid.py @@ -5,7 +5,6 @@ This code is provided with the hope that it will be useful. No guarantee is provided whatsoever. Use at your own risk. """ -import types import operator import re import numpy @@ -450,7 +449,7 @@ def __call__(self, srcVar, **args): attrs = {} for a in srcVar.attributes: v = srcVar.attributes[a] - if type(v) is types.StringType: + if isinstance(v, basestring): attrs[a] = v # if the missing value is present in the destination data, set diff --git a/Packages/cdms2/Lib/mvSphereMesh.py b/Packages/cdms2/Lib/mvSphereMesh.py index 2d4b7abadb..7b95922e29 100644 --- a/Packages/cdms2/Lib/mvSphereMesh.py +++ b/Packages/cdms2/Lib/mvSphereMesh.py @@ -8,7 +8,6 @@ """ import numpy -from types import NoneType class SphereMesh: @@ -34,7 +33,7 @@ def __init__(self, var, sphereThickness=0.1): # compute the min/max of elevation, needed # for normalization - if type(elvs) != NoneType: + if elvs is not None: self.minElv = min(elvs[:]) self.maxElv = max(elvs[:]) if hasattr(elvs, 'positive'): @@ -44,7 +43,7 @@ def __init__(self, var, sphereThickness=0.1): # determine the dimensionality and # whether the grid is rectilinear for axis in lons, lats, elvs: - if type(axis) != NoneType: + if axis is not None: self.ndims += 1 if len(axis.shape) != 1: self.isRectilinear = False @@ -53,7 +52,7 @@ def __init__(self, var, sphereThickness=0.1): if self.isRectilinear: self.shape = [] for axis in lons, lats, elvs: - if type(axis) != NoneType: + if axis is not None: self.shape.append( len(axis) ) self.shape.reverse() @@ -63,7 +62,7 @@ def __init__(self, var, sphereThickness=0.1): # store lon, lat, elv as a curvilinear grid if self.isRectilinear: # apply tensore product of axes to generat curvilinear coordinates - if type(elvs) != NoneType: + if elvs is not None: self.elvs = numpy.outer(numpy.outer( numpy.ones(self.shape[:0], numpy.float32), elvs), numpy.ones(self.shape[0+1:], numpy.float32)).reshape(self.shape) else: @@ -77,7 +76,7 @@ def __init__(self, var, sphereThickness=0.1): # already in curvilinear form self.lons = lons[:] self.lats = lats[:] - if type(elvs) != NoneType: + if elvs is not None: self.elvs = elvs[:] else: self.elvs = numpy.zeros( self.shape, numpy.float32 ) diff --git a/Packages/cdms2/Lib/selectors.py b/Packages/cdms2/Lib/selectors.py index b2ee04bc6a..f224439762 100644 --- a/Packages/cdms2/Lib/selectors.py +++ b/Packages/cdms2/Lib/selectors.py @@ -1,6 +1,6 @@ """Classes to support easy selection of climate data""" -import string, types, cdtime +import cdtime from axis import axisMatches from error import CDMSError from grid import AbstractRectGrid, defaultRegion, setRegionSpecs, LongitudeType, LatitudeType, TimeType, VerticalType @@ -393,7 +393,7 @@ def required(values): """Creates a selector that requires a certain axis to be present.""" if values is None: return all - if isinstance(values, types.StringType): + if isinstance(values, basestring): values = (values,) return Selector(requiredComponent(values)) diff --git a/Packages/cdms2/Lib/slabinterface.py b/Packages/cdms2/Lib/slabinterface.py index e08b49ea59..642aea1312 100644 --- a/Packages/cdms2/Lib/slabinterface.py +++ b/Packages/cdms2/Lib/slabinterface.py @@ -3,7 +3,7 @@ "Read part of the old cu slab interface implemented over CDMS" import numpy -import string, types, sys +import sys from error import CDMSError from axis import std_axis_attributes import cdms2 as cdms @@ -143,7 +143,7 @@ def showdim(self): for nd in range(self.rank()): result.append('** Dimension ' + str(nd+1) + ' **') result = result + self.getAxis(nd).listall(1) - print string.join(result, '\n') + print '\n'.join(result) def listdimnames(self): "Return a list of the names of the dimensions." @@ -179,7 +179,7 @@ def listall (self, all=None): def info(self, flag=None, device=None): "Write info about slab; include dimension values and weights if flag" if device is None: device = sys.stdout - device.write(string.join(self.listall(all=flag), "\n")) + device.write('\n'.join(self.listall(all=flag))) device.write("\n") def cdms_bounds2cu_bounds (b): diff --git a/Packages/cdms2/Lib/tvariable.py b/Packages/cdms2/Lib/tvariable.py index 27cab81568..7d9db8b9db 100644 --- a/Packages/cdms2/Lib/tvariable.py +++ b/Packages/cdms2/Lib/tvariable.py @@ -8,7 +8,6 @@ """ import json import re -import types import typeconv import numpy from numpy import sctype2char @@ -166,7 +165,7 @@ def __init__(self,data, typecode=None, copy=1, savespace=0, if dtype is None and typecode is not None: dtype = typeconv.convtypecode2(typecode) typecode = sctype2char(dtype) - if type(data) is types.TupleType: + if isinstance(data, tuple): data = list(data) AbstractVariable.__init__ (self) @@ -252,7 +251,7 @@ def __new__(cls, data, typecode=None, copy=0, savespace=0, if dtype is None and typecode is not None: dtype = typeconv.convtypecode2(typecode) typecode = sctype2char(dtype) - if type(data) is types.TupleType: + if isinstance(data, tuple): data = list(data) if isinstance(data, AbstractVariable): if not isinstance(data, TransientVariable): @@ -479,7 +478,7 @@ def setdimattribute(self, dim, field, value): raise CDMSError, "setdimattribute, dim out of bounds." d = self.getAxis(dim) if field == "name": - if not type(value) == types.StringType: + if not isinstance(value, basestring): raise CDMSError, "setdimattribute: name not a string" d.id = value @@ -492,7 +491,7 @@ def setdimattribute(self, dim, field, value): self.setAxis(dim, a) elif field == "units": - if not type(value) == types.StringType: + if not isinstance(value, basestring): raise CDMSError, "setdimattribute: units not a string" d.units = value diff --git a/Packages/cdms2/Lib/variable.py b/Packages/cdms2/Lib/variable.py index a1f3f7f7da..396dbdf06f 100644 --- a/Packages/cdms2/Lib/variable.py +++ b/Packages/cdms2/Lib/variable.py @@ -11,7 +11,6 @@ import os import string import sys -import types import cdmsobj from cdmsobj import CdmsObj, getPathFromTemplate, Max32int from avariable import AbstractVariable From 05e2675ee1e395a89b18dd6afa713e5d08535aa4 Mon Sep 17 00:00:00 2001 From: Charles Doutriaux Date: Tue, 26 Apr 2016 11:39:47 -0700 Subject: [PATCH 08/11] removed string odule wheen possible --- Packages/cdms2/Lib/CDMLParser.py | 15 +++++++-------- Packages/cdms2/Lib/cdxmllib.py | 1 - Packages/cdms2/Lib/convention.py | 27 +++++++++++++-------------- Packages/cdms2/Lib/gengrid.py | 9 ++++----- Packages/cdms2/Lib/hgrid.py | 9 ++++----- Packages/cdms2/Lib/variable.py | 5 ++--- 6 files changed, 30 insertions(+), 36 deletions(-) diff --git a/Packages/cdms2/Lib/CDMLParser.py b/Packages/cdms2/Lib/CDMLParser.py index 1ff9abc56a..e741e44d26 100644 --- a/Packages/cdms2/Lib/CDMLParser.py +++ b/Packages/cdms2/Lib/CDMLParser.py @@ -6,7 +6,6 @@ import CDML import re import cdmsNode -import string # Error constants InvalidAttribute = "Invalid attribute" @@ -52,7 +51,7 @@ def handle_data(self, data): if not matchObj: if self.verbose: print 'data:',data if self.root: - self.getCurrentNode().setContentFromString(string.strip(data)) + self.getCurrentNode().setContentFromString(data.strip()) def handle_cdata(self, data): if self.verbose: print 'cdata:', `data` @@ -125,7 +124,7 @@ def start_axis(self,attrs): datatype = attrs.get('datatype') if _Integer.match(length_s) is None: raise InvalidAttribute, 'length='+length_s - length = string.atoi(length_s) + length = int(length_s) axis = cdmsNode.AxisNode(id,length,datatype) partstring = attrs.get('partition') if partstring is not None: @@ -202,11 +201,11 @@ def start_domElem(self, attrs): start_s = attrs.get('start') length_s = attrs.get('length') if start_s is not None: - start = string.atoi(start_s) + start = int(start_s) else: start = None if length_s is not None: - length = string.atoi(length_s) + length = int(length_s) else: length = None domElem = cdmsNode.DomElemNode(name,start,length) @@ -250,15 +249,15 @@ def start_linear(self, attrs): delta_s = attrs['delta'] length_s = attrs['length'] try: - start=string.atof(start_s) + start=float(start_s) except ValueError: raise InvalidAttribute, 'start='+start_s try: - delta=string.atof(delta_s) + delta=float(delta_s) except ValueError: raise InvalidAttribute, 'delta='+delta_s try: - length=string.atoi(length_s) + length=int(length_s) except ValueError: raise InvalidAttribute, 'length='+length_s linear = cdmsNode.LinearDataNode(start,delta,length) diff --git a/Packages/cdms2/Lib/cdxmllib.py b/Packages/cdms2/Lib/cdxmllib.py index baa0f6dfbf..7c1aff3082 100644 --- a/Packages/cdms2/Lib/cdxmllib.py +++ b/Packages/cdms2/Lib/cdxmllib.py @@ -3,7 +3,6 @@ # Author: Sjoerd Mullender. import re -import string # import warnings # warnings.warn("The xmllib module is obsolete. Use xml.sax instead.", diff --git a/Packages/cdms2/Lib/convention.py b/Packages/cdms2/Lib/convention.py index 4e8c9053e2..0ed475877c 100644 --- a/Packages/cdms2/Lib/convention.py +++ b/Packages/cdms2/Lib/convention.py @@ -1,6 +1,5 @@ """ metadata conventions """ -import string from error import CDMSError from UserList import UserList @@ -13,11 +12,11 @@ class AliasList (UserList): def __init__(self, alist): UserList.__init__(self,alist) def __setitem__ (self, i, value): - self.data[i] = string.lower(value) + self.data[i] = value.lower() def __setslice(self, i, j, values): - self.data[i:j] = map(lambda x: string.lower(x), values) + self.data[i:j] = map(lambda x: x.lower(), values) def append(self, value): - self.data.append(string.lower(value)) + self.data.append(value.lower()) level_aliases = AliasList(['plev']) longitude_aliases = AliasList([]) @@ -37,11 +36,11 @@ def getDsetnodeAuxAxisIds(self, dsetnode): raise CDMSError, MethodNotImplemented def axisIsLatitude(self, axis): - id = string.lower(axis.id) + id = axis.id.lower() return (id[0:3] == 'lat') or (id in latitude_aliases) def axisIsLongitude(self, axis): - id = string.lower(axis.id) + id = axis.id.lower() return (id[0:3] == 'lon') or (id in longitude_aliases) def getVarLatId(self, var, vardict=None): @@ -99,7 +98,7 @@ def getAxisAuxIds(self, vardict, axiskeys): coorddict = {} for var in vardict.values(): if hasattr(var, 'coordinates'): - coordnames = string.split(var.coordinates) + coordnames = var.coordinates.lower() for item in coordnames: # Don't include if already a 1D coordinate axis. if item in axiskeys: @@ -127,7 +126,7 @@ def getDsetnodeAuxAxisIds(self, dsetnode): for node in dsetdict.values(): coordnames = node.getExternalAttr('coordinates') if coordnames is not None: - coordnames = string.split(coordnames) + coordnames = coordnames.split() for item in coordnames: # Don't include if already a 1D coordinate axis. if dsetdict.has_key(item) and dsetdict[item].tag=='axis': @@ -149,7 +148,7 @@ def getVarLatId(self, var, vardict): return (lat, nlat) if hasattr(var, 'coordinates'): - coordnames = string.split(var.coordinates) + coordnames = var.coordinates.split() for name in coordnames: coord = vardict.get(name) @@ -173,7 +172,7 @@ def getVarLonId(self, var, vardict): return (lon, nlon) if hasattr(var, 'coordinates'): - coordnames = string.split(var.coordinates) + coordnames = var.coordinates.split() for name in coordnames: coord = vardict.get(name) @@ -191,9 +190,9 @@ def getVarLonId(self, var, vardict): def axisIsLatitude(self, axis): if (hasattr(axis,'axis') and axis.axis=='Y'): return 1 - elif (hasattr(axis, 'units') and string.lower(axis.units) in ['degrees_north', 'degree_north', 'degree_n', 'degrees_n', 'degreen', 'degreesn']): + elif (hasattr(axis, 'units') and axis.units.lower() in ['degrees_north', 'degree_north', 'degree_n', 'degrees_n', 'degreen', 'degreesn']): return 1 - elif (hasattr(axis, 'standard_name') and string.lower(axis.standard_name)=='latitude'): + elif (hasattr(axis, 'standard_name') and axis.standard_name.lower()=='latitude'): return 1 else: return AbstractConvention.axisIsLatitude(self, axis) @@ -201,9 +200,9 @@ def axisIsLatitude(self, axis): def axisIsLongitude(self, axis): if (hasattr(axis,'axis') and axis.axis=='X'): return 1 - elif (hasattr(axis, 'units') and string.lower(axis.units) in ['degrees_east', 'degree_east', 'degree_e', 'degrees_e', 'degreee', 'degreese']): + elif (hasattr(axis, 'units') and axis.units.lower() in ['degrees_east', 'degree_east', 'degree_e', 'degrees_e', 'degreee', 'degreese']): return 1 - elif (hasattr(axis, 'standard_name') and string.lower(axis.standard_name)=='longitude'): + elif (hasattr(axis, 'standard_name') and axis.standard_name.lower()=='longitude'): return 1 else: return AbstractConvention.axisIsLongitude(self, axis) diff --git a/Packages/cdms2/Lib/gengrid.py b/Packages/cdms2/Lib/gengrid.py index 1204733bcf..95e6d18715 100644 --- a/Packages/cdms2/Lib/gengrid.py +++ b/Packages/cdms2/Lib/gengrid.py @@ -338,7 +338,6 @@ def readScripGenericGrid(fileobj, dims, whichType, whichGrid): whichType is the type of file, either "grid" or "mapping" if whichType is "mapping", whichGrid is the choice of grid, either "source" or "destination" """ - import string from auxcoord import TransientAuxAxis1D from coord import TransientVirtualAxis @@ -400,7 +399,7 @@ def readScripGenericGrid(fileobj, dims, whichType, whichGrid): ni = dims[0] boundsshape = (ni, ncorners) - if hasattr(cornerLat, 'units') and string.lower(cornerLat.units)[0:6]=='radian': + if hasattr(cornerLat, 'units') and cornerLat.units.lower()[0:6]=='radian': cornerLat = (cornerLat*(180.0/numpy.pi)).reshape(boundsshape) cornerLon = (cornerLon*(180.0/numpy.pi)).reshape(boundsshape) @@ -415,21 +414,21 @@ def readScripGenericGrid(fileobj, dims, whichType, whichGrid): if vardict.has_key(gridCenterLatName): centerLat = fileobj(gridCenterLatName) - if hasattr(centerLat, "units") and string.lower(centerLat.units)=='radians': + if hasattr(centerLat, "units") and centerLat.units.lower()=='radians': centerLat *= (180.0/numpy.pi) else: centerLat = cornerLat[:,:,0] if vardict.has_key(gridCenterLonName): centerLon = fileobj(gridCenterLonName) - if hasattr(centerLon, "units") and string.lower(centerLon.units)=='radians': + if hasattr(centerLon, "units") and centerLon.units.lower()=='radians': centerLon *= (180.0/numpy.pi) else: centerLon = cornerLon[:,:,0] if hasattr(fileobj,titleName): gridid = getattr(fileobj, titleName) - gridid = string.replace(string.strip(gridid), ' ','_') + gridid = sgridid.strip().replace(' ','_') else: gridid="" diff --git a/Packages/cdms2/Lib/hgrid.py b/Packages/cdms2/Lib/hgrid.py index 56758ad724..1f53d37026 100644 --- a/Packages/cdms2/Lib/hgrid.py +++ b/Packages/cdms2/Lib/hgrid.py @@ -758,7 +758,6 @@ def readScripCurveGrid(fileobj, dims, whichType, whichGrid): whichType is the type of file, either "grid" or "mapping" if whichType is "mapping", whichGrid is the choice of grid, either "source" or "destination" """ - import string from coord import TransientAxis2D if 'S' in fileobj.variables.keys(): @@ -814,7 +813,7 @@ def readScripCurveGrid(fileobj, dims, whichType, whichGrid): nj = dims[0] gridshape = (ni, nj) boundsshape = (ni, nj, ncorners) - if hasattr(cornerLat, 'units') and string.lower(cornerLat.units)[0:6]=='radian': + if hasattr(cornerLat, 'units') and cornerLat.units.lower()[0:6]=='radian': cornerLat = (cornerLat*(180.0/numpy.pi)).reshape(boundsshape) cornerLon = (cornerLon*(180.0/numpy.pi)).reshape(boundsshape) else: @@ -835,7 +834,7 @@ def readScripCurveGrid(fileobj, dims, whichType, whichGrid): if vardict.has_key(gridCenterLatName): centerLat = fileobj(gridCenterLatName).reshape(gridshape) gclat = fileobj[gridCenterLatName] - if hasattr(gclat, "units") and string.lower(gclat.units)=='radians': + if hasattr(gclat, "units") and gclat.units.lower()=='radians': centerLat *= (180.0/numpy.pi) else: centerLat = cornerLat[:,:,0] @@ -843,14 +842,14 @@ def readScripCurveGrid(fileobj, dims, whichType, whichGrid): if vardict.has_key(gridCenterLonName): centerLon = fileobj(gridCenterLonName).reshape(gridshape) gclon = fileobj[gridCenterLonName] - if hasattr(gclon, "units") and string.lower(gclon.units)=='radians': + if hasattr(gclon, "units") and gclon.units.lower()=='radians': centerLon *= (180.0/numpy.pi) else: centerLon = cornerLon[:,:,0] if hasattr(fileobj,titleName): gridid = getattr(fileobj, titleName) - gridid = string.replace(string.strip(gridid), ' ','_') + gridid = gridid.strip().replace(' ','_') else: gridid="" diff --git a/Packages/cdms2/Lib/variable.py b/Packages/cdms2/Lib/variable.py index 396dbdf06f..23650e3d84 100644 --- a/Packages/cdms2/Lib/variable.py +++ b/Packages/cdms2/Lib/variable.py @@ -9,7 +9,6 @@ import cdtime import copy import os -import string import sys import cdmsobj from cdmsobj import CdmsObj, getPathFromTemplate, Max32int @@ -139,7 +138,7 @@ def initDomain(self, axisdict, griddict): raise CDMSError, InvalidGridElement + dename partlenstr = denode.getExternalAttr('partition_length') if partlenstr is not None: - truelen = string.atoi(partlenstr) + truelen = int(partlenstr) else: truelen = denode.length self.domain.append((domelem, denode.start, denode.length, truelen)) @@ -224,7 +223,7 @@ def genMatch(self, axis, interval, matchnames): else: # Use template method time0 = axis[interval[0]] time1 = axis[interval[1]-1] - isabs = (string.find(axis.units," as ")!=-1) + isabs = (axis.units.find(" as ")!=-1) if isabs: start = cdtime.abstime(time0,axis.units) end = cdtime.abstime(time1,axis.units) From d5a9a226ca5fafb65e95fa793cf8144a349adbe3 Mon Sep 17 00:00:00 2001 From: Charles Doutriaux Date: Wed, 27 Apr 2016 08:46:49 -0700 Subject: [PATCH 09/11] Address @danlipsa comments --- Packages/cdms2/Lib/avariable.py | 4 ++-- Packages/cdms2/Lib/cdmsobj.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Packages/cdms2/Lib/avariable.py b/Packages/cdms2/Lib/avariable.py index bc20b8be53..db174105eb 100644 --- a/Packages/cdms2/Lib/avariable.py +++ b/Packages/cdms2/Lib/avariable.py @@ -1232,7 +1232,7 @@ def specs2slices (self, speclist, force=None): slicelist.append (slice(0, len(self.getAxis(i)))) elif key is Ellipsis: raise CDMSError, "Misuse of ellipsis in specification." - elif sinstance(key, tuple): + elif isinstance(key, tuple): slicelist.append(slice(*key)) else: raise CDMSError, 'invalid index: %s'% str(key) @@ -1514,7 +1514,7 @@ def orderparse (order): elif r == '...': r = Ellipsis elif len(r) == 1: - if r in '0123456789': + if r.isdigit(): r = int(r) result.append(r) pos = m.end(0) diff --git a/Packages/cdms2/Lib/cdmsobj.py b/Packages/cdms2/Lib/cdmsobj.py index b66cf73079..8fad6b3ab0 100644 --- a/Packages/cdms2/Lib/cdmsobj.py +++ b/Packages/cdms2/Lib/cdmsobj.py @@ -546,12 +546,12 @@ def matchone(self, pattern, attname): """ if attname is None: for attval in self.attributes.values(): - if isinstance(attval,basestring) and pattern.search(attval) is not None: + if isinstance(attval,basestring) and pattern.match(attval) is not None: return 1 return 0 elif self.attributes.has_key(attname): attval = self.attributes[attname] - return isinstance(attval,basestring) and pattern.search(attval) is not None + return isinstance(attval,basestring) and pattern.match(attval) is not None else: return 0 From 6fc54c9a9537d0e4f47ab84a04bdad2c2ea486fa Mon Sep 17 00:00:00 2001 From: Charles Doutriaux Date: Wed, 27 Apr 2016 09:08:08 -0700 Subject: [PATCH 10/11] autopep8ed --- Packages/cdms2/Lib/CDML.py | 425 +++--- Packages/cdms2/Lib/CDMLParser.py | 180 ++- Packages/cdms2/Lib/MV2.py | 524 ++++--- Packages/cdms2/Lib/__init__.py | 62 +- Packages/cdms2/Lib/auxcoord.py | 80 +- Packages/cdms2/Lib/avariable.py | 962 +++++++------ Packages/cdms2/Lib/axis.py | 1827 +++++++++++++----------- Packages/cdms2/Lib/bindex.py | 41 +- Packages/cdms2/Lib/cache.py | 167 ++- Packages/cdms2/Lib/cdmsNode.py | 763 +++++----- Packages/cdms2/Lib/cdmsURLopener.py | 17 +- Packages/cdms2/Lib/cdmsobj.py | 438 +++--- Packages/cdms2/Lib/cdurllib.py | 72 +- Packages/cdms2/Lib/cdurlparse.py | 339 ++--- Packages/cdms2/Lib/cdxmllib.py | 249 ++-- Packages/cdms2/Lib/convention.py | 62 +- Packages/cdms2/Lib/coord.py | 201 +-- Packages/cdms2/Lib/cudsinterface.py | 182 +-- Packages/cdms2/Lib/database.py | 220 +-- Packages/cdms2/Lib/dataset.py | 1245 +++++++++------- Packages/cdms2/Lib/error.py | 4 +- Packages/cdms2/Lib/forecast.py | 226 +-- Packages/cdms2/Lib/fvariable.py | 137 +- Packages/cdms2/Lib/gengrid.py | 106 +- Packages/cdms2/Lib/grid.py | 385 ++--- Packages/cdms2/Lib/gsHost.py | 212 +-- Packages/cdms2/Lib/gsMosaic.py | 251 ++-- Packages/cdms2/Lib/gsStaticVariable.py | 107 +- Packages/cdms2/Lib/gsTimeVariable.py | 160 ++- Packages/cdms2/Lib/gui.py | 31 +- Packages/cdms2/Lib/hgrid.py | 247 ++-- Packages/cdms2/Lib/mvBaseWriter.py | 19 +- Packages/cdms2/Lib/mvCdmsRegrid.py | 26 +- Packages/cdms2/Lib/mvSphereMesh.py | 170 ++- Packages/cdms2/Lib/mvVTKSGWriter.py | 29 +- Packages/cdms2/Lib/mvVTKUGWriter.py | 47 +- Packages/cdms2/Lib/mvVsWriter.py | 36 +- Packages/cdms2/Lib/restApi.py | 965 +++++++------ Packages/cdms2/Lib/selectors.py | 286 ++-- Packages/cdms2/Lib/slabinterface.py | 130 +- Packages/cdms2/Lib/sliceut.py | 214 +-- Packages/cdms2/Lib/tvariable.py | 609 ++++---- Packages/cdms2/Lib/typeconv.py | 30 +- Packages/cdms2/Lib/variable.py | 345 ++--- Packages/cdms2/Lib/xlink.py | 10 +- 45 files changed, 7050 insertions(+), 5788 deletions(-) diff --git a/Packages/cdms2/Lib/CDML.py b/Packages/cdms2/Lib/CDML.py index 7a6c013f37..963f70ac68 100644 --- a/Packages/cdms2/Lib/CDML.py +++ b/Packages/cdms2/Lib/CDML.py @@ -37,6 +37,7 @@ # Note: at some point, this should be created dynamically # from the XML DTD file. For now, it is built statically. + class CDML: cache = {} @@ -47,246 +48,254 @@ def __init__(self, uri=None): self.extra = self.extraCache.get(uri) if not self.dtd: self.dtd = self.buildDTD(uri) - self.cache[uri]=self.dtd + self.cache[uri] = self.dtd self.extra = self.buildExtra(uri) - self.extraCache[uri]=self.extra + self.extraCache[uri] = self.extra - def buildDTD(self,uri): + def buildDTD(self, uri): dtd = {} dtd['attr'] = { - 'name': (Cdata,Required), - 'datatype': (("Char","Byte","Short","Int","Long","Int64","Float","Double","String"),Required), - } + 'name': (Cdata, Required), + 'datatype': + (("Char", "Byte", "Short", "Int", "Long", + "Int64", "Float", "Double", "String"), Required), + } dtd['axis'] = { - 'id': (Id,Required), - 'associate': (Idref,Implied), - 'axis': (("X","Y","Z","T"),Implied), - 'bounds': (Idref,Implied), - 'calendar': (Cdata,Implied), - 'comment': (Cdata,Implied), - 'component': (Cdata,Implied), - 'compress': (Cdata,Implied), - 'datatype': (("Char","Byte","Short","Int","Long","Int64","Float","Double","String"),Required), - 'expand': (Idref,Implied), - 'interval': (Cdata,Implied), - 'isvar': ( ("true","false"),"true"), - 'length': (Cdata,Required), - 'long_name': (Cdata,Implied), - 'modulo': (Cdata,Implied), - 'name_in_file': (Cdata,Implied), - 'partition': (Cdata,Implied), - 'partition_length': (Cdata,Implied), - 'positive': (("up","down"),Implied), - 'spacing': (("uniform","variable","disjoint"),Implied), - 'topology': (("linear","circular"),Implied), - 'weights': (Idref,Implied), - 'units': (Cdata,Required), - } + 'id': (Id, Required), + 'associate': (Idref, Implied), + 'axis': (("X", "Y", "Z", "T"), Implied), + 'bounds': (Idref, Implied), + 'calendar': (Cdata, Implied), + 'comment': (Cdata, Implied), + 'component': (Cdata, Implied), + 'compress': (Cdata, Implied), + 'datatype': + (("Char", "Byte", "Short", "Int", "Long", + "Int64", "Float", "Double", "String"), Required), + 'expand': (Idref, Implied), + 'interval': (Cdata, Implied), + 'isvar': (("true", "false"), "true"), + 'length': (Cdata, Required), + 'long_name': (Cdata, Implied), + 'modulo': (Cdata, Implied), + 'name_in_file': (Cdata, Implied), + 'partition': (Cdata, Implied), + 'partition_length': (Cdata, Implied), + 'positive': (("up", "down"), Implied), + 'spacing': (("uniform", "variable", "disjoint"), Implied), + 'topology': (("linear", "circular"), Implied), + 'weights': (Idref, Implied), + 'units': (Cdata, Required), + } dtd['component'] = { - 'name':(Idref,Required), - } + 'name': (Idref, Required), + } dtd['dataset'] = { - 'id': (Id,Required), - 'Conventions': (Cdata,Required), - 'appendices': (Cdata,Implied), - 'calendar': (Cdata,Implied), - 'cdms_filemap': (Cdata,Implied), - 'comment': (Cdata,Implied), - 'directory': (Cdata,Implied), - 'frequency': (Cdata,Implied), - 'history': (Cdata,Implied), - 'institution': (Cdata,Implied), - 'production': (Cdata,Implied), - 'project': (Cdata,Implied), - 'template': (Cdata,Implied), - } + 'id': (Id, Required), + 'Conventions': (Cdata, Required), + 'appendices': (Cdata, Implied), + 'calendar': (Cdata, Implied), + 'cdms_filemap': (Cdata, Implied), + 'comment': (Cdata, Implied), + 'directory': (Cdata, Implied), + 'frequency': (Cdata, Implied), + 'history': (Cdata, Implied), + 'institution': (Cdata, Implied), + 'production': (Cdata, Implied), + 'project': (Cdata, Implied), + 'template': (Cdata, Implied), + } dtd['doclink'] = { - 'id': (Id,Implied), - 'xml:link': (Cdata,(Fixed,"simple")), - 'href': (Cdata,Required), - 'role': (Cdata,Implied), - 'title': (Cdata,Implied), - 'show': (("embed","replace","new"),"replace"), - 'actuate': (("auto","user"),Implied), - 'behavior':(Cdata,Implied), - 'content-role': (Cdata,Implied), - 'content-title': (Cdata,Implied), - 'inline':(("true","false"),"true"), - } + 'id': (Id, Implied), + 'xml:link': (Cdata, (Fixed, "simple")), + 'href': (Cdata, Required), + 'role': (Cdata, Implied), + 'title': (Cdata, Implied), + 'show': (("embed", "replace", "new"), "replace"), + 'actuate': (("auto", "user"), Implied), + 'behavior': (Cdata, Implied), + 'content-role': (Cdata, Implied), + 'content-title': (Cdata, Implied), + 'inline': (("true", "false"), "true"), + } dtd['domain'] = {} dtd['domElem'] = { - 'name':(Idref,Required), - 'length':(Cdata,Implied), - 'partition_length':(Cdata,Implied), - 'start':(Cdata,Implied), - } + 'name': (Idref, Required), + 'length': (Cdata, Implied), + 'partition_length': (Cdata, Implied), + 'start': (Cdata, Implied), + } dtd['rectGrid'] = { - 'id': (Id,Required), - 'type':(("gaussian","uniform","equalarea","unknown"),"unknown"), - 'latitude':(Idref,Required), - 'longitude':(Idref,Required), - 'mask':(Idref,Implied), - 'order':(("xy","yx"),"yx"), - } + 'id': (Id, Required), + 'type': + (("gaussian", "uniform", "equalarea", "unknown"), "unknown"), + 'latitude': (Idref, Required), + 'longitude': (Idref, Required), + 'mask': (Idref, Implied), + 'order': (("xy", "yx"), "yx"), + } dtd['linear'] = { - 'delta': (Cdata,Required), - 'length': (Cdata,Required), - 'start': (Cdata,Required), - } + 'delta': (Cdata, Required), + 'length': (Cdata, Required), + 'start': (Cdata, Required), + } dtd['variable'] = { - 'id': (Id,Required), - 'add_offset': (Cdata,Implied), - 'associate': (Cdata,Implied), - 'axis': (Cdata,Implied), - 'comments': (Cdata,Implied), - 'datatype': (("Char","Byte","Short","Int","Long","Int64","Float","Double","String"),Required), - 'grid_name': (Cdata,Implied), - 'grid_type': (Cdata,Implied), - 'long_name': (Cdata,Implied), + 'id': (Id, Required), + 'add_offset': (Cdata, Implied), + 'associate': (Cdata, Implied), + 'axis': (Cdata, Implied), + 'comments': (Cdata, Implied), + 'datatype': + (("Char", "Byte", "Short", "Int", "Long", + "Int64", "Float", "Double", "String"), Required), + 'grid_name': (Cdata, Implied), + 'grid_type': (Cdata, Implied), + 'long_name': (Cdata, Implied), 'missing_value': (Cdata, Implied), - 'name_in_file': (Cdata,Implied), - 'scale_factor': (Cdata,Implied), - 'subgrid': (Cdata,Implied), - 'template': (Cdata,Implied), - 'units': (Cdata,Implied), - 'valid_max': (Cdata,Implied), - 'valid_min': (Cdata,Implied), - 'valid_range': (Cdata,Implied), - } + 'name_in_file': (Cdata, Implied), + 'scale_factor': (Cdata, Implied), + 'subgrid': (Cdata, Implied), + 'template': (Cdata, Implied), + 'units': (Cdata, Implied), + 'valid_max': (Cdata, Implied), + 'valid_min': (Cdata, Implied), + 'valid_range': (Cdata, Implied), + } dtd['xlink'] = { - 'id': (Id,Implied), - 'xml:link': (Cdata,(Fixed,"simple")), - 'href': (Cdata,Required), - 'role': (Cdata,Implied), - 'title': (Cdata,Implied), - 'show': (("embed","replace","new"),"embed"), - 'actuate': (("auto","user"),Implied), - 'behavior':(Cdata,Implied), - 'content-role': (("dataset","axis","grid","variable","object"),"object"), - 'content-title': (Cdata,Implied), - 'inline':(("true","false"),"true"), - } + 'id': (Id, Implied), + 'xml:link': (Cdata, (Fixed, "simple")), + 'href': (Cdata, Required), + 'role': (Cdata, Implied), + 'title': (Cdata, Implied), + 'show': (("embed", "replace", "new"), "embed"), + 'actuate': (("auto", "user"), Implied), + 'behavior': (Cdata, Implied), + 'content-role': + (("dataset", "axis", "grid", "variable", "object"), "object"), + 'content-title': (Cdata, Implied), + 'inline': (("true", "false"), "true"), + } return dtd # Extra datatype information not included in the formal DTD. - def buildExtra(self,uri): + def buildExtra(self, uri): extra = {} extra['attr'] = { - 'name': (CdScalar,CdString), - 'datatype': (CdScalar,CdString), - } + 'name': (CdScalar, CdString), + 'datatype': (CdScalar, CdString), + } extra['axis'] = { - 'id': (CdScalar,CdString), - 'associate': (CdScalar,CdString), - 'axis': (CdScalar,CdString), - 'bounds': (CdArray,CdFromObject), - 'calendar': (CdScalar,CdString), - 'comment': (CdScalar,CdString), - 'component': (CdScalar,CdString), - 'compress': (CdScalar,CdString), - 'datatype': (CdScalar,CdString), - 'expand': (CdScalar,CdString), - 'interval': (CdScalar,CdFromObject), - 'isvar': (CdScalar,CdString), - 'length': (CdScalar,CdInt), - 'long_name': (CdScalar,CdString), - 'modulo': (CdScalar,CdAny), - 'name_in_file': (CdScalar,CdString), - 'partition': (CdArray,CdInt), - 'partition_length': (CdScalar,CdInt), - 'positive': (CdScalar,CdString), - 'spacing': (CdScalar,CdFromObject), - 'topology': (CdScalar,CdString), - 'weights': (CdArray,CdDouble), - 'units': (CdScalar,CdString), - } + 'id': (CdScalar, CdString), + 'associate': (CdScalar, CdString), + 'axis': (CdScalar, CdString), + 'bounds': (CdArray, CdFromObject), + 'calendar': (CdScalar, CdString), + 'comment': (CdScalar, CdString), + 'component': (CdScalar, CdString), + 'compress': (CdScalar, CdString), + 'datatype': (CdScalar, CdString), + 'expand': (CdScalar, CdString), + 'interval': (CdScalar, CdFromObject), + 'isvar': (CdScalar, CdString), + 'length': (CdScalar, CdInt), + 'long_name': (CdScalar, CdString), + 'modulo': (CdScalar, CdAny), + 'name_in_file': (CdScalar, CdString), + 'partition': (CdArray, CdInt), + 'partition_length': (CdScalar, CdInt), + 'positive': (CdScalar, CdString), + 'spacing': (CdScalar, CdFromObject), + 'topology': (CdScalar, CdString), + 'weights': (CdArray, CdDouble), + 'units': (CdScalar, CdString), + } extra['component'] = { - 'name': (CdScalar,CdString), - } + 'name': (CdScalar, CdString), + } extra['dataset'] = { - 'id': (CdScalar,CdString), - 'Conventions': (CdScalar,CdString), - 'appendices': (CdScalar,CdString), - 'calendar': (CdScalar,CdString), - 'cdms_filemap': (CdScalar,CdString), - 'comment': (CdScalar,CdString), - 'directory': (CdScalar,CdString), - 'frequency': (CdScalar,CdString), - 'history': (CdScalar,CdString), - 'institution': (CdScalar,CdString), - 'production': (CdScalar,CdString), - 'project': (CdScalar,CdString), - 'template': (CdScalar,CdString), - } + 'id': (CdScalar, CdString), + 'Conventions': (CdScalar, CdString), + 'appendices': (CdScalar, CdString), + 'calendar': (CdScalar, CdString), + 'cdms_filemap': (CdScalar, CdString), + 'comment': (CdScalar, CdString), + 'directory': (CdScalar, CdString), + 'frequency': (CdScalar, CdString), + 'history': (CdScalar, CdString), + 'institution': (CdScalar, CdString), + 'production': (CdScalar, CdString), + 'project': (CdScalar, CdString), + 'template': (CdScalar, CdString), + } extra['doclink'] = { - 'id': (CdScalar,CdString), - 'xml:link': (CdScalar,CdString), - 'href': (CdScalar,CdString), - 'role': (CdScalar,CdString), - 'title': (CdScalar,CdString), - 'show': (CdScalar,CdString), - 'actuate': (CdScalar,CdString), - 'behavior': (CdScalar,CdString), - 'content-role': (CdScalar,CdString), - 'content-title': (CdScalar,CdString), - 'inline': (CdScalar,CdString), - } + 'id': (CdScalar, CdString), + 'xml:link': (CdScalar, CdString), + 'href': (CdScalar, CdString), + 'role': (CdScalar, CdString), + 'title': (CdScalar, CdString), + 'show': (CdScalar, CdString), + 'actuate': (CdScalar, CdString), + 'behavior': (CdScalar, CdString), + 'content-role': (CdScalar, CdString), + 'content-title': (CdScalar, CdString), + 'inline': (CdScalar, CdString), + } extra['domain'] = {} extra['domElem'] = { - 'name': (CdScalar,CdString), - 'length': (CdScalar,CdInt), - 'partition_length': (CdScalar,CdInt), - 'start': (CdScalar,CdInt), - } + 'name': (CdScalar, CdString), + 'length': (CdScalar, CdInt), + 'partition_length': (CdScalar, CdInt), + 'start': (CdScalar, CdInt), + } extra['rectGrid'] = { - 'id': (CdScalar,CdString), - 'type': (CdScalar,CdString), - 'latitude': (CdScalar,CdString), - 'longitude': (CdScalar,CdString), - 'mask': (CdScalar,CdString), - 'order': (CdScalar,CdString), - } + 'id': (CdScalar, CdString), + 'type': (CdScalar, CdString), + 'latitude': (CdScalar, CdString), + 'longitude': (CdScalar, CdString), + 'mask': (CdScalar, CdString), + 'order': (CdScalar, CdString), + } extra['linear'] = { - 'delta': (CdScalar,CdFromObject), - 'length': (CdScalar,CdInt), - 'start': (CdScalar,CdInt), - } + 'delta': (CdScalar, CdFromObject), + 'length': (CdScalar, CdInt), + 'start': (CdScalar, CdInt), + } extra['variable'] = { - 'id': (CdScalar,CdString), - 'add_offset': (CdScalar,CdDouble), - 'associate': (CdScalar,CdString), - 'axis': (CdScalar,CdString), - 'comments': (CdScalar,CdString), - 'datatype': (CdScalar,CdString), - 'grid_name': (CdScalar,CdString), - 'grid_type': (CdScalar,CdString), - 'long_name': (CdScalar,CdString), - 'missing_value': (CdScalar,CdFromObject), - 'name_in_file': (CdScalar,CdString), - 'scale_factor': (CdScalar,CdDouble), - 'subgrid': (CdScalar,CdString), - 'template': (CdScalar,CdString), - 'units': (CdScalar,CdString), - 'valid_max': (CdScalar,CdFromObject), - 'valid_min': (CdScalar,CdFromObject), - 'valid_range': (CdArray,CdFromObject), - } + 'id': (CdScalar, CdString), + 'add_offset': (CdScalar, CdDouble), + 'associate': (CdScalar, CdString), + 'axis': (CdScalar, CdString), + 'comments': (CdScalar, CdString), + 'datatype': (CdScalar, CdString), + 'grid_name': (CdScalar, CdString), + 'grid_type': (CdScalar, CdString), + 'long_name': (CdScalar, CdString), + 'missing_value': (CdScalar, CdFromObject), + 'name_in_file': (CdScalar, CdString), + 'scale_factor': (CdScalar, CdDouble), + 'subgrid': (CdScalar, CdString), + 'template': (CdScalar, CdString), + 'units': (CdScalar, CdString), + 'valid_max': (CdScalar, CdFromObject), + 'valid_min': (CdScalar, CdFromObject), + 'valid_range': (CdArray, CdFromObject), + } extra['xlink'] = { - 'id': (CdScalar,CdString), - 'xml:link': (CdScalar,CdString), - 'href': (CdScalar,CdString), - 'role': (CdScalar,CdString), - 'title': (CdScalar,CdString), - 'show': (CdScalar,CdString), - 'actuate': (CdScalar,CdString), - 'behavior': (CdScalar,CdString), - 'content-role': (CdScalar,CdString), - 'content-title': (CdScalar,CdString), - 'inline': (CdScalar,CdString), - } + 'id': (CdScalar, CdString), + 'xml:link': (CdScalar, CdString), + 'href': (CdScalar, CdString), + 'role': (CdScalar, CdString), + 'title': (CdScalar, CdString), + 'show': (CdScalar, CdString), + 'actuate': (CdScalar, CdString), + 'behavior': (CdScalar, CdString), + 'content-role': (CdScalar, CdString), + 'content-title': (CdScalar, CdString), + 'inline': (CdScalar, CdString), + } return extra -if __name__=='__main__': +if __name__ == '__main__': cdml = CDML() print cdml.extra cdml2 = CDML() diff --git a/Packages/cdms2/Lib/CDMLParser.py b/Packages/cdms2/Lib/CDMLParser.py index e741e44d26..6a88eda9c1 100644 --- a/Packages/cdms2/Lib/CDMLParser.py +++ b/Packages/cdms2/Lib/CDMLParser.py @@ -2,10 +2,10 @@ Parse a CDML/XML file """ -from cdxmllib import XMLParser -import CDML +from .cdxmllib import XMLParser +from . import CDML import re -import cdmsNode +from . import cdmsNode # Error constants InvalidAttribute = "Invalid attribute" @@ -13,21 +13,23 @@ # Regular expressions _S = re.compile('[ \t\r\n]+$') _opS = '[ \t\r\n]*' -_Integer = re.compile(_opS+'[0-9]+$'+_opS) +_Integer = re.compile(_opS + '[0-9]+$' + _opS) + class CDMLParser(XMLParser): def __init__(self, verbose=0): - XMLParser.__init__(self) + XMLParser.__init__(self) self.root = None self.currentPath = [] # Current path, a stack self.dtd = CDML.CDML().dtd self.verbose = verbose # Push current node on the stack - def pushCurrentNode(self,node): + def pushCurrentNode(self, node): self.currentPath.append(node) - if not self.root: self.root = node + if not self.root: + self.root = node # Pop the current node off the stack def popCurrentNode(self): @@ -49,62 +51,67 @@ def getRoot(self): def handle_data(self, data): matchObj = _S.match(data) if not matchObj: - if self.verbose: print 'data:',data + if self.verbose: + print 'data:', data if self.root: self.getCurrentNode().setContentFromString(data.strip()) def handle_cdata(self, data): - if self.verbose: print 'cdata:', `data` + if self.verbose: + print 'cdata:', repr(data) def handle_proc(self, name, data): - if self.verbose: print 'processing:',name,`data` + if self.verbose: + print 'processing:', name, repr(data) def handle_special(self, data): - if self.verbose: print 'special:',`data` + if self.verbose: + print 'special:', repr(data) def handle_starttag(self, tag, method, attrs): - if self.dtd.has_key(tag): + if tag in self.dtd: # Check that attributes are valid - validDict = self.dtd[tag] + validDict = self.dtd[tag] validAttrs = validDict.keys() attrnames = attrs.keys() for attrname in attrnames: if attrname not in validAttrs: self.cdml_syntax_error(self.lineno, - 'unknown attribute %s of element %s' % - (attrname, tag)) + 'unknown attribute %s of element %s' % + (attrname, tag)) else: - (atttype,attdefault)=validDict[attrname] - if type(atttype)==type((0,)): + (atttype, attdefault) = validDict[attrname] + if isinstance(atttype, tuple): attrval = attrs[attrname] if attrval not in atttype: self.cdml_syntax_error(self.lineno, - 'invalid attribute value %s=%s of element %s, must be one of %s' % - (attrname,attrval,tag,atttype)) + 'invalid attribute value %s=%s of element %s, must be one of %s' % + (attrname, attrval, tag, atttype)) # Check that required attributes are present, # and add default values for attrname in validAttrs: - (atttype,attdefault)=validDict[attrname] - if attdefault==CDML.Required and attrname not in attrnames: + (atttype, attdefault) = validDict[attrname] + if attdefault == CDML.Required and attrname not in attrnames: self.cdml_syntax_error(self.lineno, - 'element %s requires an attribute %s' % - (tag,attrname)) - if type(attdefault)==type("") and attrname not in attrnames: - attrs[attrname]=attdefault - - method(attrs) + 'element %s requires an attribute %s' % + (tag, attrname)) + if isinstance(attdefault, basestring) and attrname not in attrnames: + attrs[attrname] = attdefault + + method(attrs) #------------------------------------------------------------------------ # CDML tags - def start_attr(self,attrs): - if self.verbose: print 'attr:',attrs + def start_attr(self, attrs): + if self.verbose: + print 'attr:', attrs name = attrs['name'] datatype = attrs['datatype'] - attr = cdmsNode.AttrNode(name,None) + attr = cdmsNode.AttrNode(name, None) attr.datatype = datatype self.pushCurrentNode(attr) @@ -112,33 +119,35 @@ def start_attr(self,attrs): def end_attr(self): attr = self.popCurrentNode() var = self.getCurrentNode() - attr.setValueFromString(attr.getContent(),attr.datatype) + attr.setValueFromString(attr.getContent(), attr.datatype) var.setExternalAttrFromAttr(attr) #------------------------------------------------------------------------ - def start_axis(self,attrs): - if self.verbose: print 'axis:',attrs + def start_axis(self, attrs): + if self.verbose: + print 'axis:', attrs id = attrs['id'] length_s = attrs['length'] datatype = attrs.get('datatype') if _Integer.match(length_s) is None: - raise InvalidAttribute, 'length='+length_s + raise InvalidAttribute('length=' + length_s) length = int(length_s) - axis = cdmsNode.AxisNode(id,length,datatype) + axis = cdmsNode.AxisNode(id, length, datatype) partstring = attrs.get('partition') if partstring is not None: axis.setPartitionFromString(partstring) axis.setExternalDict(attrs) - self.getCurrentNode().addId(id,axis) + self.getCurrentNode().addId(id, axis) self.pushCurrentNode(axis) - + def end_axis(self): self.popCurrentNode() #------------------------------------------------------------------------ def start_cdml(self, attrs): - if self.verbose: print 'cdml:',attrs + if self.verbose: + print 'cdml:', attrs def end_cdml(self): pass @@ -146,36 +155,40 @@ def end_cdml(self): #------------------------------------------------------------------------ def start_component(self, attrs): - if self.verbose: print 'component:',attrs + if self.verbose: + print 'component:', attrs def end_component(self): pass #------------------------------------------------------------------------ def start_compoundAxis(self, attrs): - if self.verbose: print 'compoundAxis:',attrs + if self.verbose: + print 'compoundAxis:', attrs def end_compoundAxis(self): pass #------------------------------------------------------------------------ def start_data(self, attrs): - if self.verbose: print 'data:',attrs + if self.verbose: + print 'data:', attrs def end_data(self): pass #------------------------------------------------------------------------ - def start_dataset(self,attrs): - if self.verbose: print 'dataset:',attrs + def start_dataset(self, attrs): + if self.verbose: + print 'dataset:', attrs id = attrs['id'] dataset = cdmsNode.DatasetNode(id) dataset.setExternalDict(attrs) if self.root: - self.getCurrentNode().addId(id,dataset) + self.getCurrentNode().addId(id, dataset) self.pushCurrentNode(dataset) - + def end_dataset(self): dataset = self.popCurrentNode() dataset.validate() @@ -183,7 +196,8 @@ def end_dataset(self): #------------------------------------------------------------------------ def start_doclink(self, attrs): - if self.verbose: print 'docLink:',attrs + if self.verbose: + print 'docLink:', attrs uri = attrs['href'] doclink = cdmsNode.DocLinkNode(uri) doclink.setExternalDict(attrs) @@ -196,7 +210,8 @@ def end_doclink(self): #------------------------------------------------------------------------ def start_domElem(self, attrs): - if self.verbose: print 'domElem:',attrs + if self.verbose: + print 'domElem:', attrs name = attrs['name'] start_s = attrs.get('start') length_s = attrs.get('length') @@ -208,7 +223,7 @@ def start_domElem(self, attrs): length = int(length_s) else: length = None - domElem = cdmsNode.DomElemNode(name,start,length) + domElem = cdmsNode.DomElemNode(name, start, length) domElem.setExternalDict(attrs) self.getCurrentNode().add(domElem) @@ -217,7 +232,8 @@ def end_domElem(self): #------------------------------------------------------------------------ def start_domain(self, attrs): - if self.verbose: print 'domain:',attrs + if self.verbose: + print 'domain:', attrs domain = cdmsNode.DomainNode() self.getCurrentNode().setDomain(domain) self.pushCurrentNode(domain) @@ -227,40 +243,42 @@ def end_domain(self): #------------------------------------------------------------------------ - def start_rectGrid(self,attrs): - if self.verbose: print 'rectGrid:',attrs + def start_rectGrid(self, attrs): + if self.verbose: + print 'rectGrid:', attrs id = attrs['id'] gridtype = attrs['type'] latitude = attrs['latitude'] longitude = attrs['longitude'] - grid = cdmsNode.RectGridNode(id,latitude,longitude,gridtype) + grid = cdmsNode.RectGridNode(id, latitude, longitude, gridtype) grid.setExternalDict(attrs) - self.getCurrentNode().addId(id,grid) + self.getCurrentNode().addId(id, grid) self.pushCurrentNode(grid) - + def end_rectGrid(self): self.popCurrentNode() #------------------------------------------------------------------------ def start_linear(self, attrs): - if self.verbose: print 'linear:',attrs + if self.verbose: + print 'linear:', attrs start_s = attrs['start'] delta_s = attrs['delta'] length_s = attrs['length'] try: - start=float(start_s) + start = float(start_s) except ValueError: - raise InvalidAttribute, 'start='+start_s + raise InvalidAttribute('start=' + start_s) try: - delta=float(delta_s) + delta = float(delta_s) except ValueError: - raise InvalidAttribute, 'delta='+delta_s + raise InvalidAttribute('delta=' + delta_s) try: - length=int(length_s) + length = int(length_s) except ValueError: - raise InvalidAttribute, 'length='+length_s - linear = cdmsNode.LinearDataNode(start,delta,length) + raise InvalidAttribute('length=' + length_s) + linear = cdmsNode.LinearDataNode(start, delta, length) self.getCurrentNode().setLinearData(linear) def end_linear(self): @@ -268,28 +286,30 @@ def end_linear(self): #------------------------------------------------------------------------ - def start_variable(self,attrs): - if self.verbose: print 'variable:',attrs + def start_variable(self, attrs): + if self.verbose: + print 'variable:', attrs id = attrs['id'] datatype = attrs['datatype'] - variable = cdmsNode.VariableNode(id,datatype,None) + variable = cdmsNode.VariableNode(id, datatype, None) variable.setExternalDict(attrs) - self.getCurrentNode().addId(id,variable) + self.getCurrentNode().addId(id, variable) self.pushCurrentNode(variable) - + def end_variable(self): self.popCurrentNode() #------------------------------------------------------------------------ def start_xlink(self, attrs): - if self.verbose: print 'xlink:',attrs + if self.verbose: + print 'xlink:', attrs id = attrs['id'] uri = attrs['href'] contentRole = attrs['content-role'] - xlink = cdmsNode.XLinkNode(id,uri,contentRole) + xlink = cdmsNode.XLinkNode(id, uri, contentRole) xlink.setExternalDict(attrs) - self.getCurrentNode().addId(id,xlink) + self.getCurrentNode().addId(id, xlink) self.pushCurrentNode(xlink) def end_xlink(self): @@ -301,30 +321,33 @@ def cdml_syntax_error(self, lineno, message): print 'error near line %d:' % lineno, message def unknown_starttag(self, tag, attrs): - if self.verbose: print '**'+tag+'**:',attrs + if self.verbose: + print '**' + tag + '**:', attrs def unknown_endtag(self, tag): pass def unknown_entityref(self, ref): - self.flush() - if self.verbose: print '*** unknown entity ref: &' + ref + ';' + self.flush() + if self.verbose: + print '*** unknown entity ref: &' + ref + ';' def unknown_charref(self, ref): - self.flush() - if self.verbose: print '*** unknown char ref: &#' + ref + ';' + self.flush() + if self.verbose: + print '*** unknown char ref: &#' + ref + ';' def close(self): - XMLParser.close(self) + XMLParser.close(self) if __name__ == '__main__': import sys - + sampfile = open(sys.argv[1]) text = sampfile.read() sampfile.close() - if len(sys.argv)==2: + if len(sys.argv) == 2: verbose = 0 else: verbose = 1 @@ -332,4 +355,3 @@ def close(self): p.feed(text) p.close() p.root.dump() - diff --git a/Packages/cdms2/Lib/MV2.py b/Packages/cdms2/Lib/MV2.py index b203bc6af7..297cd73626 100644 --- a/Packages/cdms2/Lib/MV2.py +++ b/Packages/cdms2/Lib/MV2.py @@ -1,31 +1,31 @@ -## Automatically adapted for numpy.oldnumeric Aug 01, 2007 by -## Further modified to be pure new numpy June 24th 2008 +# Automatically adapted for numpy.oldnumeric Aug 01, 2007 by +# Further modified to be pure new numpy June 24th 2008 "CDMS Variable objects, MaskedArray interface" import numpy -import typeconv +from . import typeconv from numpy import character, float, float32, float64, int, int8, int16, int32 from numpy.ma import allclose, allequal, common_fill_value, compress, make_mask_none, dot, filled, \ - getmask, getmaskarray, identity, indices, innerproduct, masked, put, putmask, rank, ravel, \ + getmask, getmaskarray, identity, indices, innerproduct, masked, put, putmask, rank, ravel, \ set_fill_value, shape, size, isMA, isMaskedArray, is_mask, isarray, \ make_mask, make_mask_none, mask_or, nomask from numpy import sctype2char, get_printoptions, set_printoptions -from avariable import AbstractVariable, getNumericCompatibility -from tvariable import TransientVariable, asVariable -from grid import AbstractRectGrid -from error import CDMSError -#from numpy.ma import * -from axis import allclose as axisAllclose, TransientAxis, concatenate as axisConcatenate, take as axisTake - +from .avariable import AbstractVariable, getNumericCompatibility +from .tvariable import TransientVariable, asVariable +from .grid import AbstractRectGrid +from .error import CDMSError +# from numpy.ma import * +from .axis import allclose as axisAllclose, TransientAxis, concatenate as axisConcatenate, take as axisTake create_mask = make_mask_none e = numpy.e pi = numpy.pi -#NewAxis = numpy.oldnumeric.NewAxis +# NewAxis = numpy.oldnumeric.NewAxis newaxis = numpy.newaxis counter = 0 + def fill_value(ar): return ar.fill_value @@ -34,13 +34,14 @@ def _makeMaskedArg(x): """If x is a variable, turn it into a TransientVariable.""" if isinstance(x, AbstractVariable) and not isinstance(x, TransientVariable): return x.subSlice() - elif isinstance(x,TransientVariable): + elif isinstance(x, TransientVariable): return x else: return array(x) -def _extractMetadata(a, axes=None, attributes=None, id=None, omit=None, omitall=False): +def _extractMetadata( + a, axes=None, attributes=None, id=None, omit=None, omitall=False): """Extract axes, attributes, id from 'a', if arg is None.""" resultgrid = None if isinstance(a, AbstractVariable): @@ -52,7 +53,7 @@ def _extractMetadata(a, axes=None, attributes=None, id=None, omit=None, omitall= attributes = a.attributes if id is None: id = "variable_%i" % TransientVariable.variable_count - TransientVariable.variable_count+=1 + TransientVariable.variable_count += 1 # If the grid is rectilinear, don't return an explicit grid: it's implicitly defined # by the axes. @@ -60,13 +61,16 @@ def _extractMetadata(a, axes=None, attributes=None, id=None, omit=None, omitall= if (resultgrid is None) or (isinstance(resultgrid, AbstractRectGrid)) or (axes is None): resultgrid = None - # If the omitted axis was associated with the grid, the result will not be gridded. + # If the omitted axis was associated with the grid, the result will not + # be gridded. elif (omit is not None) and (resultgrid is not None) and (a.getAxis(omit) in resultgrid.getAxisList()): resultgrid = None return axes, attributes, id, resultgrid + class var_unary_operation: + def __init__(self, mafunc): """ var_unary_operation(mafunc) mafunc is an numpy.ma masked_unary_function. @@ -74,26 +78,31 @@ def __init__(self, mafunc): self.mafunc = mafunc self.__doc__ = mafunc.__doc__ - def __call__ (self, a): + def __call__(self, a): axes, attributes, id, grid = _extractMetadata(a) maresult = self.mafunc(_makeMaskedArg(a)) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid) + class var_unary_operation_with_axis: + def __init__(self, mafunc): """ var_unary_operation(mafunc) mafunc is an numpy.ma masked_unary_function. """ self.mafunc = mafunc self.__doc__ = mafunc.__doc__ - def __call__ (self, a, axis=0): + + def __call__(self, a, axis=0): axis = _conv_axis_arg(axis) ta = _makeMaskedArg(a) maresult = self.mafunc(ta, axis=axis) - axes, attributes, id, grid = _extractMetadata(a, omit=axis, omitall=(axis is None)) + axes, attributes, id, grid = _extractMetadata( + a, omit=axis, omitall=(axis is None)) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid) -def commonDomain(a,b,omit=None): + +def commonDomain(a, b, omit=None): """commonDomain(a,b) tests that the domains of variables/arrays a and b are equal, and returns the common domain if equal, or None if not equal. The domains may differ in that one domain may have leading axes not common @@ -101,14 +110,15 @@ def commonDomain(a,b,omit=None): If is specified, as an integer i, skip comparison of the ith dimension and return None for the ith (common) dimension. """ - + if isinstance(b, AbstractVariable): bdom = b.getAxisList() else: bdom = None - return commonAxes(a,bdom,omit=omit) + return commonAxes(a, bdom, omit=omit) -def commonAxes(a,bdom,omit=None): + +def commonAxes(a, bdom, omit=None): """Helper function for commonDomain. 'a' is a variable or array, 'b' is an axislist or None. """ @@ -116,40 +126,40 @@ def commonAxes(a,bdom,omit=None): adom = a.getAxisList() arank = len(adom) brank = len(bdom) - if arank>brank: + if arank > brank: maxrank = arank minrank = brank else: maxrank = brank minrank = arank - diffrank = maxrank-minrank - if maxrank==arank: + diffrank = maxrank - minrank + if maxrank == arank: maxdom = adom else: maxdom = bdom - common = [None]*maxrank + common = [None] * maxrank if omit is None: iomit = None else: - iomit = omit-minrank + iomit = omit - minrank # Check shared dimensions, last to first for i in range(minrank): - j = -i-1 - if j==iomit: + j = -i - 1 + if j == iomit: continue aj = adom[j] bj = bdom[j] - if len(aj)!=len(bj): + if len(aj) != len(bj): return None - elif axisAllclose(aj,bj): + elif axisAllclose(aj, bj): common[j] = aj else: common[j] = TransientAxis(numpy.arange(len(aj))) # Copy leading (non-shared) axes for i in range(diffrank): - common[i]=maxdom[i] + common[i] = maxdom[i] return common elif isinstance(a, AbstractVariable): @@ -162,6 +172,7 @@ def commonAxes(a,bdom,omit=None): bdom[omit] = None return bdom + def commonGrid(a, b, axes): """commonGrid(a,b,axes) tests if the grids associated with variables a, b are equal, and consistent with the list of axes. If so, the common grid is returned, else None @@ -182,6 +193,7 @@ def commonGrid(a, b, axes): return commonGrid1(a, gb, axes) + def commonGrid1(a, gb, axes): """Helper function for commonGrid.""" if isinstance(a, AbstractVariable): @@ -210,7 +222,9 @@ def commonGrid1(a, gb, axes): return result + class var_binary_operation: + def __init__(self, mafunc): """ var_binary_operation(mafunc) mafunc is an numpy.ma masked_binary_function. @@ -218,29 +232,29 @@ def __init__(self, mafunc): self.mafunc = mafunc self.__doc__ = mafunc.__doc__ - def __call__ (self, a, b): + def __call__(self, a, b): id = "variable_%i" % TransientVariable.variable_count - TransientVariable.variable_count+=1 - axes = commonDomain(a,b) - grid = commonGrid(a,b,axes) + TransientVariable.variable_count += 1 + axes = commonDomain(a, b) + grid = commonGrid(a, b, axes) ta = _makeMaskedArg(a) tb = _makeMaskedArg(b) - maresult = self.mafunc(ta,tb) - return TransientVariable(maresult, axes=axes, grid=grid,no_update_from=True,id=id) + maresult = self.mafunc(ta, tb) + return TransientVariable(maresult, axes=axes, grid=grid, no_update_from=True, id=id) - def reduce (self, target, axis=0): + def reduce(self, target, axis=0): ttarget = _makeMaskedArg(target) maresult = self.mafunc.reduce(ttarget, axis=axis) axes, attributes, id, grid = _extractMetadata(target, omit=axis) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid) - def accumulate (self, target, axis=0): + def accumulate(self, target, axis=0): ttarget = _makeMaskedArg(target) maresult = self.mafunc.accumulate(ttarget, axis=axis) axes, attributes, id, grid = _extractMetadata(target, omit=axis) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid) - - def outer (self, a, b): + + def outer(self, a, b): """Return the function applied to the outer product of a and b""" a1 = _makeMaskedArg(a) b1 = _makeMaskedArg(b) @@ -248,11 +262,13 @@ def outer (self, a, b): return TransientVariable(maresult) -def compress(a,b): - __doc__=numpy.ma.__doc__ - import warnings - warnings.warn("arguments order for compress function has changed\nit is now: MV2.copmress(array,condition), if your code seems to not react or act wrong to a call to compress, please check this", Warning) - return TransientVariable(numpy.ma.compress(a,b),copy=1) +def compress(a, b): + __doc__ = numpy.ma.__doc__ + import warnings + warnings.warn( + "arguments order for compress function has changed\nit is now: MV2.copmress(array,condition), if your code seems to not react or act wrong to a call to compress, please check this", + Warning) + return TransientVariable(numpy.ma.compress(a, b), copy=1) sqrt = var_unary_operation(numpy.ma.sqrt) @@ -269,27 +285,31 @@ def compress(a,b): greater_equal = var_binary_operation(numpy.ma.greater_equal) less = var_binary_operation(numpy.ma.less) greater = var_binary_operation(numpy.ma.greater) -def power (a, b, third=None): + + +def power(a, b, third=None): "a**b" ta = _makeMaskedArg(a) tb = _makeMaskedArg(b) - maresult = numpy.ma.power(ta,tb,third) + maresult = numpy.ma.power(ta, tb, third) axes, attributes, id, grid = _extractMetadata(a) return TransientVariable(maresult, axes=axes, attributes=attributes, grid=grid, id=id) -def left_shift (a, n): + +def left_shift(a, n): "Left shift n bits" ta = _makeMaskedArg(a) tb = _makeMaskedArg(n) - maresult = numpy.ma.left_shift(ta,numpy.ma.filled(tb)) + maresult = numpy.ma.left_shift(ta, numpy.ma.filled(tb)) axes, attributes, id, grid = _extractMetadata(a) return TransientVariable(maresult, axes=axes, attributes=attributes, grid=grid, id=id) -def right_shift (a, n): + +def right_shift(a, n): "Right shift n bits" ta = _makeMaskedArg(a) tb = _makeMaskedArg(n) - maresult = numpy.ma.right_shift(ta,numpy.ma.filled(tb)) + maresult = numpy.ma.right_shift(ta, numpy.ma.filled(tb)) axes, attributes, id, grid = _extractMetadata(a) return TransientVariable(maresult, axes=axes, attributes=attributes, grid=grid, id=id) @@ -300,27 +320,33 @@ def _convdtype(dtype, typecode): dtype = typeconv.convtypecode2(typecode) return dtype + def _conv_axis_arg(axis): "Handle backward compatibility with numpy for axis arg" if getNumericCompatibility() and axis is None: - axis=0 + axis = 0 return axis + def is_masked(x): "Is x a 0-D masked value?" - return isMaskedArray(x) and x.size==1 and x.ndim==0 and x.mask.item() + return isMaskedArray(x) and x.size == 1 and x.ndim == 0 and x.mask.item() + def is_floating(x): "Is x a scalar float, either python or numpy?" return (isinstance(x, numpy.floating) or isinstance(x, float)) + def is_integer(x): "Is x a scalar integer, either python or numpy?" return (isinstance(x, numpy.integer) or isinstance(x, int) or isinstance(x, long)) + def get_print_limit(): return get_printoptions()['threshold'] + def set_print_limit(limit=numpy.inf): set_printoptions(threshold=limit) @@ -369,75 +395,113 @@ def set_print_limit(limit=numpy.inf): bitwise_xor = var_binary_operation(numpy.ma.bitwise_xor) -def count (a, axis = None): - "Count of the non-masked elements in a, or along a certain axis." +def count(a, axis=None): + "Count of the non-masked elements in a, or along a certain axis." if axis is None: - return numpy.ma.count(a,axis) + return numpy.ma.count(a, axis) else: ta = _makeMaskedArg(a) - maresult = numpy.ma.count(ta,axis) - axes, attributes, id, grid = _extractMetadata(a,omit=axis) - F=getattr(a,"fill_value",1.e20) + maresult = numpy.ma.count(ta, axis) + axes, attributes, id, grid = _extractMetadata(a, omit=axis) + F = getattr(a, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, grid=grid, id=id, fill_value=F) -def sum (a, axis = None, fill_value=0, dtype=None): + +def sum(a, axis=None, fill_value=0, dtype=None): "Sum of elements along a certain axis." axis = _conv_axis_arg(axis) ta = _makeMaskedArg(a) maresult = numpy.ma.sum(ta, axis, dtype=dtype) - axes, attributes, id, grid = _extractMetadata(a, omit=axis, omitall=(axis is None)) - F=getattr(a,"fill_value",1.e20) + axes, attributes, id, grid = _extractMetadata( + a, omit=axis, omitall=(axis is None)) + F = getattr(a, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, grid=grid, id=id, fill_value=F) -def product (a, axis = 0, dtype=None): + +def product(a, axis=0, dtype=None): "Product of elements along axis." ta = _makeMaskedArg(a) maresult = numpy.ma.product(ta, axis, dtype=dtype) axes, attributes, id, grid = _extractMetadata(a, omit=axis) - F=getattr(a,"fill_value",1.e20) + F = getattr(a, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, grid=grid, id=id, fill_value=F) -def average (a, axis=None, weights=None, returned=False): + +def average(a, axis=None, weights=None, returned=False): axis = _conv_axis_arg(axis) ta = _makeMaskedArg(a) maresult = numpy.ma.average(ta, axis, weights, returned) - axes, attributes, id, grid = _extractMetadata(a, omit=axis, omitall=(axis is None)) + axes, attributes, id, grid = _extractMetadata( + a, omit=axis, omitall=(axis is None)) if returned: - if isinstance(maresult,tuple): - maresult, wresult = maresult - else: - #ok it's masked constant need to return both things by hand - wresult = numpy.ma.masked - F=getattr(a,"fill_value",1.e20) - r1 = TransientVariable(maresult, axes=axes, attributes=attributes, grid=grid, id=id,no_update_from=True, fill_value=F) + if isinstance(maresult, tuple): + maresult, wresult = maresult + else: + # ok it's masked constant need to return both things by hand + wresult = numpy.ma.masked + F = getattr(a, "fill_value", 1.e20) + r1 = TransientVariable( + maresult, + axes=axes, + attributes=attributes, + grid=grid, + id=id, + no_update_from=True, + fill_value=F) if returned: - F=getattr(a,"fill_value",1.e20) - w1 = TransientVariable(wresult, axes=axes, grid=grid, id=id,no_update_from=True, fill_value=F) + F = getattr(a, "fill_value", 1.e20) + w1 = TransientVariable( + wresult, + axes=axes, + grid=grid, + id=id, + no_update_from=True, + fill_value=F) return r1, w1 else: return r1 average.__doc__ = numpy.ma.average.__doc__ -def max (a, axis=None): + +def max(a, axis=None): axis = _conv_axis_arg(axis) ta = _makeMaskedArg(a) maresult = numpy.ma.max(ta, axis) - axes, attributes, id, grid = _extractMetadata(a, omit=axis, omitall=(axis is None)) - F=getattr(a,"fill_value",1.e20) - r1 = TransientVariable(maresult, axes=axes, attributes=attributes, grid=grid, id=id,no_update_from=True, fill_value=F) + axes, attributes, id, grid = _extractMetadata( + a, omit=axis, omitall=(axis is None)) + F = getattr(a, "fill_value", 1.e20) + r1 = TransientVariable( + maresult, + axes=axes, + attributes=attributes, + grid=grid, + id=id, + no_update_from=True, + fill_value=F) return r1 max.__doc__ = numpy.ma.max.__doc__ -def min (a, axis=None): + + +def min(a, axis=None): axis = _conv_axis_arg(axis) ta = _makeMaskedArg(a) maresult = numpy.ma.min(ta, axis) - axes, attributes, id, grid = _extractMetadata(a, omit=axis, omitall=(axis is None)) - F=getattr(a,"fill_value",1.e20) - r1 = TransientVariable(maresult, axes=axes, attributes=attributes, grid=grid, id=id,no_update_from=True, fill_value=F) + axes, attributes, id, grid = _extractMetadata( + a, omit=axis, omitall=(axis is None)) + F = getattr(a, "fill_value", 1.e20) + r1 = TransientVariable( + maresult, + axes=axes, + attributes=attributes, + grid=grid, + id=id, + no_update_from=True, + fill_value=F) return r1 min.__doc__ = numpy.ma.min.__doc__ -def sort (a, axis=-1): + +def sort(a, axis=-1): ta = _makeMaskedArg(a) maresult = numpy.ma.sort(a.asma(), axis) axes, attributes, id, grid = _extractMetadata(a) @@ -445,11 +509,13 @@ def sort (a, axis=-1): if (grid is not None) and (sortaxis in grid.getAxisList()): grid = None axes[axis] = TransientAxis(numpy.arange(len(sortaxis))) - F=getattr(a,"fill_value",1.e20) + F = getattr(a, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, grid=grid, id=id, fill_value=F) -sort.__doc__ = numpy.ma.sort.__doc__ + "The sort axis is replaced with a dummy axis." +sort.__doc__ = numpy.ma.sort.__doc__ + \ + "The sort axis is replaced with a dummy axis." -def choose (indices, t): + +def choose(indices, t): """Returns an array shaped like indices containing elements chosen from t. If an element of t is the special element masked, any element @@ -458,35 +524,38 @@ def choose (indices, t): The result has only the default axes. """ maresult = numpy.ma.choose(indices, map(_makeMaskedArg, t)) - F=getattr(t,"fill_value",1.e20) + F = getattr(t, "fill_value", 1.e20) return TransientVariable(maresult, fill_value=F) -def where (condition, x, y): - "where(condition, x, y) is x where condition is true, y otherwise" -## axes = commonDomain(x,y) -## grid = commonGrid(x,y,axes) + +def where(condition, x, y): + "where(condition, x, y) is x where condition is true, y otherwise" +# axes = commonDomain(x,y) +# grid = commonGrid(x,y,axes) maresult = numpy.ma.where(condition, _makeMaskedArg(x), _makeMaskedArg(y)) axes, attributes, id, grid = _extractMetadata(condition) - F=getattr(x,"fill_value",1.e20) + F = getattr(x, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, grid=grid, id=id, fill_value=F) + def masked_where(condition, x, copy=1): - """Return x as an array masked where condition is true. + """Return x as an array masked where condition is true. Also masked where x or condition masked. """ tx = _makeMaskedArg(x) tcondition = _makeMaskedArg(condition) maresult = numpy.ma.masked_where(tcondition, tx, copy) axes, attributes, id, grid = _extractMetadata(x) - F=getattr(x,"fill_value",1.e20) + F = getattr(x, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, fill_value=F) + def masked_greater(x, value): "masked_greater(x, value) = x masked where x > value" tx = _makeMaskedArg(x) maresult = numpy.ma.masked_greater(tx, value) axes, attributes, id, grid = _extractMetadata(x) - F=getattr(x,"fill_value",1.e20) + F = getattr(x, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, fill_value=F) @@ -495,33 +564,37 @@ def masked_greater_equal(x, value): tx = _makeMaskedArg(x) maresult = numpy.ma.masked_greater_equal(tx, value) axes, attributes, id, grid = _extractMetadata(x) - F=getattr(x,"fill_value",1.e20) + F = getattr(x, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, fill_value=F) + def masked_less(x, value): "masked_less(x, value) = x masked where x < value" tx = _makeMaskedArg(x) maresult = numpy.ma.masked_less(tx, value) axes, attributes, id, grid = _extractMetadata(x) - F=getattr(x,"fill_value",1.e20) + F = getattr(x, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, fill_value=F) + def masked_less_equal(x, value): "masked_less_equal(x, value) = x masked where x <= value" tx = _makeMaskedArg(x) maresult = numpy.ma.masked_less_equal(tx, value) axes, attributes, id, grid = _extractMetadata(x) - F=getattr(x,"fill_value",1.e20) + F = getattr(x, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, fill_value=F) + def masked_not_equal(x, value): "masked_not_equal(x, value) = x masked where x != value" tx = _makeMaskedArg(x) maresult = numpy.ma.masked_not_equal(tx, value) axes, attributes, id, grid = _extractMetadata(x) - F=getattr(x,"fill_value",1.e20) + F = getattr(x, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, fill_value=F) + def masked_equal(x, value): """masked_equal(x, value) = x masked where x == value For floating point consider masked_values(x, value) instead. @@ -529,40 +602,43 @@ def masked_equal(x, value): tx = _makeMaskedArg(x) maresult = numpy.ma.masked_equal(tx, value) axes, attributes, id, grid = _extractMetadata(x) - F=getattr(x,"fill_value",1.e20) + F = getattr(x, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, fill_value=F) + def masked_outside(x, v1, v2): "x with mask of all values of x that are outside [v1,v2]" tx = _makeMaskedArg(x) maresult = numpy.ma.masked_outside(tx, v1, v2) axes, attributes, id, grid = _extractMetadata(x) - F=getattr(x,"fill_value",1.e20) + F = getattr(x, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, fill_value=F) + def masked_inside(x, v1, v2): "x with mask of all values of x that are inside [v1,v2]" tx = _makeMaskedArg(x) maresult = numpy.ma.masked_inside(tx, v1, v2) axes, attributes, id, grid = _extractMetadata(x) - F=getattr(x,"fill_value",1.e20) + F = getattr(x, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, fill_value=F) -def concatenate (arrays, axis=0, axisid=None, axisattributes=None): + +def concatenate(arrays, axis=0, axisid=None, axisattributes=None): """Concatenate the arrays along the given axis. Give the extended axis the id and attributes provided - by default, those of the first array.""" tarrays = [_makeMaskedArg(a) for a in arrays] maresult = numpy.ma.concatenate(arrays, axis=axis) - if len(arrays)>1: + if len(arrays) > 1: varattributes = None varid = None - axes = commonDomain(tarrays[0],tarrays[1],omit=axis) + axes = commonDomain(tarrays[0], tarrays[1], omit=axis) grid = commonGrid(tarrays[0], tarrays[1], axes) - for i in range(len(arrays)-2): + for i in range(len(arrays) - 2): if axes is None: break - axes = commonAxes(tarrays[i+2],axes,omit=axis) + axes = commonAxes(tarrays[i + 2], axes, omit=axis) grid = commonGrid1(a, grid, axes) else: axes = tarrays[0].getAxisList() @@ -575,32 +651,37 @@ def concatenate (arrays, axis=0, axisid=None, axisattributes=None): if axes is not None: if axisid is None: axisid = tarrays[0].getAxis(axis).id - allunitsequal=True + allunitsequal = True try: - allunits=tarrays[0].getAxis(axis).units + allunits = tarrays[0].getAxis(axis).units except: - allunits=None + allunits = None for t in tarrays[1:]: try: - tunits=t.getAxis(axis).units + tunits = t.getAxis(axis).units except: - tunits=None - if tunits!=allunits: - allunitsequal=False + tunits = None + if tunits != allunits: + allunitsequal = False if allunitsequal: if axisattributes is None: axisattributes = tarrays[0].getAxis(axis).attributes - axes[axis] = axisConcatenate([t.getAxis(axis) for t in tarrays], axisid, axisattributes) + axes[axis] = axisConcatenate( + [t.getAxis(axis) for t in tarrays], + axisid, + axisattributes) - # If the grid doesn't match the axislist (e.g., catenation was on latitude) then omit it. + # If the grid doesn't match the axislist (e.g., catenation was on + # latitude) then omit it. if grid is not None: for item in grid.getAxisList(): if item not in axes: grid = None - F=getattr(arrays[0],"fill_value",1.e20) - return TransientVariable(maresult, axes=axes, attributes=varattributes,id=varid,grid=grid, fill_value=F) + F = getattr(arrays[0], "fill_value", 1.e20) + return TransientVariable(maresult, axes=axes, attributes=varattributes, id=varid, grid=grid, fill_value=F) + -def take (a, indices, axis=None): +def take(a, indices, axis=None): "take(a, indices, axis=None) returns selection of items from a." axis = _conv_axis_arg(axis) ta = _makeMaskedArg(a) @@ -608,15 +689,16 @@ def take (a, indices, axis=None): # ma compatibility interface has a bug maresult = numpy.ma.take(ta, indices, axis=axis) axes, attributes, id, grid = _extractMetadata(a, omitall=(axis is None)) - + # If the take is on a grid axis, omit the grid. if (grid is not None) and (axes[axis] in grid.getAxisList()): grid = None if axes is not None: axes[axis] = axisTake(axes[axis], indices) - F=getattr(a,"fill_value",1.e20) + F = getattr(a, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, fill_value=F) + def transpose(a, axes=None): "transpose(a, axes=None) reorder dimensions per tuple axes" ta = _makeMaskedArg(a) @@ -628,49 +710,53 @@ def transpose(a, axes=None): newaxes = None if oldaxes is not None: newaxes = [oldaxes[i] for i in axes] - F=getattr(a,"fill_value",1.e20) + F = getattr(a, "fill_value", 1.e20) return TransientVariable(maresult, axes=newaxes, attributes=attributes, id=id, grid=grid, copy=1, fill_value=F) + class _minimum_operation: + "Object to calculate minima" - def __init__ (self): + + def __init__(self): """minimum(a, b) or minimum(a) In one argument case returns the scalar minimum. """ pass - def __call__ (self, a, b=None): + def __call__(self, a, b=None): "Execute the call behavior." a = _makeMaskedArg(a) if b is None: m = getmask(a) - if m is nomask: + if m is nomask: d = numpy.min(filled(a).ravel()) return d -## ac = a.compressed() -## if len(ac) == 0: -## return masked +# ac = a.compressed() +# if len(ac) == 0: +# return masked else: return numpy.ma.min(a) else: return where(less(a, b), a, b)[...] - - def reduce (self, target, axis=0): + + def reduce(self, target, axis=0): """Reduce target along the given axis.""" a = _makeMaskedArg(target) axes, attributes, id, grid = _extractMetadata(a, omit=axis) m = getmask(a) if m is nomask: t = filled(a) - result = masked_array (numpy.minimum.reduce (t, axis)) + result = masked_array(numpy.minimum.reduce(t, axis)) else: - t = numpy.minimum.reduce(filled(a, numpy.ma.minimum_fill_value(a)), axis) + t = numpy.minimum.reduce( + filled(a, numpy.ma.minimum_fill_value(a)), axis) m = numpy.logical_and.reduce(m, axis) result = masked_array(t, m, fill_value(a)) return TransientVariable(result, axes=axes, copy=0, - fill_value=fill_value(a), grid=grid, id=id) + fill_value=fill_value(a), grid=grid, id=id) - def outer (self, a, b): + def outer(self, a, b): "Return the function applied to the outer product of a and b." a = _makeMaskedArg(a) b = _makeMaskedArg(b) @@ -685,47 +771,51 @@ def outer (self, a, b): d = numpy.minimum.outer(filled(a), filled(b)) return TransientVariable(d, mask=m, copy=0) -minimum = _minimum_operation () - +minimum = _minimum_operation() + + class _maximum_operation: + "Object to calculate maxima" - def __init__ (self): + + def __init__(self): """maximum(a, b) or maximum(a) In one argument case returns the scalar maximum. """ pass - def __call__ (self, a, b=None): + def __call__(self, a, b=None): "Execute the call behavior." a = _makeMaskedArg(a) if b is None: m = getmask(a) - if m is nomask: + if m is nomask: d = numpy.max(filled(a).ravel()) return d -## ac = a.compressed() -## if len(ac) == 0: -## return masked +# ac = a.compressed() +# if len(ac) == 0: +# return masked else: return numpy.ma.max(a) else: return where(greater(a, b), a, b)[...] - - def reduce (self, target, axis=0): + + def reduce(self, target, axis=0): """Reduce target along the given axis.""" axes, attributes, id, grid = _extractMetadata(target, omit=axis) a = _makeMaskedArg(target) m = getmask(a) if m is nomask: t = filled(a) - return masked_array (numpy.maximum.reduce (t, axis)) + return masked_array(numpy.maximum.reduce(t, axis)) else: - t = numpy.maximum.reduce(filled(a, numpy.ma.maximum_fill_value(a)), axis) + t = numpy.maximum.reduce( + filled(a, numpy.ma.maximum_fill_value(a)), axis) m = numpy.logical_and.reduce(m, axis) return TransientVariable(t, mask=m, fill_value=fill_value(a), - axes = axes, grid=grid, id=id) + axes=axes, grid=grid, id=id) - def outer (self, a, b): + def outer(self, a, b): "Return the function applied to the outer product of a and b." a = _makeMaskedArg(a) b = _makeMaskedArg(b) @@ -740,8 +830,9 @@ def outer (self, a, b): d = numpy.maximum.outer(filled(a), filled(b)) return TransientVariable(d, mask=m) -maximum = _maximum_operation () - +maximum = _maximum_operation() + + def asarray(data, typecode=None, dtype=None): """asarray(data, typecode=None, dtype=None) is equivalent to array(data, dtype=None, copy=0) Returns data if dtype is None or data is a MaskedArray of the same dtype. @@ -751,10 +842,12 @@ def asarray(data, typecode=None, dtype=None): if isinstance(data, AbstractVariable) and (dtype is None or sctype2char(dtype) == data.dtype.char): return data else: - F=getattr(data,"fill_value",1.e20) + F = getattr(data, "fill_value", 1.e20) return TransientVariable(data, dtype=dtype, copy=0, fill_value=F) -def arrayrange(start, stop=None, step=1, typecode=None, axis=None, attributes=None, id=None, dtype=None): + +def arrayrange(start, stop=None, step=1, typecode=None, + axis=None, attributes=None, id=None, dtype=None): """Just like range() except it returns a variable whose type can be specfied by the keyword argument typecode. The axis of the result variable may be specified. """ @@ -767,15 +860,19 @@ def arrayrange(start, stop=None, step=1, typecode=None, axis=None, attributes=No arange = arrayrange -def zeros (shape, typecode=float, savespace=0, axes=None, attributes=None, id=None, grid=None, dtype=None): - """zeros(n, typecode=float, savespace=0, axes=None, attributes=None, id=None) = + +def zeros(shape, typecode=float, savespace=0, axes=None, + attributes=None, id=None, grid=None, dtype=None): + """zeros(n, typecode=float, savespace=0, axes=None, attributes=None, id=None) = an array of all zeros of the given length or shape.""" dtype = _convdtype(dtype, typecode) maresult = numpy.ma.zeros(shape, dtype=dtype) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid) - -def ones (shape, typecode=float, savespace=0, axes=None, attributes=None, id=None, grid=None, dtype=None): - """ones(n, typecode=float, savespace=0, axes=None, attributes=None, id=None) = + + +def ones(shape, typecode=float, savespace=0, axes=None, + attributes=None, id=None, grid=None, dtype=None): + """ones(n, typecode=float, savespace=0, axes=None, attributes=None, id=None) = an array of all ones of the given length or shape.""" dtype = _convdtype(dtype, typecode) maresult = numpy.ma.ones(shape, dtype=dtype) @@ -783,28 +880,31 @@ def ones (shape, typecode=float, savespace=0, axes=None, attributes=None, id=Non as_masked = numpy.ma.array + def outerproduct(a, b): """outerproduct(a,b) = {a[i]*b[j]}, has shape (len(a),len(b))""" - ta = asVariable(a,writeable=1) - tb = asVariable(b,writeable=1) - maresult = numpy.ma.outerproduct(ta,tb) - axes = (ta.getAxis(0),tb.getAxis(0)) - F=getattr(a,"fill_value",1.e20) + ta = asVariable(a, writeable=1) + tb = asVariable(b, writeable=1) + maresult = numpy.ma.outerproduct(ta, tb) + axes = (ta.getAxis(0), tb.getAxis(0)) + F = getattr(a, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, fill_value=F) -def argsort (x, axis = -1, fill_value=None): + +def argsort(x, axis=-1, fill_value=None): """Treating masked values as if they have the value fill_value, return sort indices for sorting along given axis. if fill_value is None, use fill_value(x) - """ + """ tx = _makeMaskedArg(x) - maresult = numpy.ma.argsort(tx,axis=axis,fill_value=fill_value) + maresult = numpy.ma.argsort(tx, axis=axis, fill_value=fill_value) axes, attributes, id, grid = _extractMetadata(x) - F=getattr(x,"fill_value",1.e20) + F = getattr(x, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, fill_value=F) array = TransientVariable + def repeat(a, repeats, axis=None): """repeat elements of a repeats times along axis repeats is a sequence of length a.shape[axis] @@ -818,68 +918,85 @@ def repeat(a, repeats, axis=None): grid = None if axes is not None: axes[axis] = None - F=getattr(a,"fill_value",1.e20) + F = getattr(a, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, no_update_from=True, fill_value=F) -def reshape (a, newshape, axes=None, attributes=None, id=None, grid=None): + +def reshape(a, newshape, axes=None, attributes=None, id=None, grid=None): ignore, attributes, id, ignore = _extractMetadata(a, axes, attributes, id) if axes is not None: axesshape = [len(item) for item in axes] - if axesshape!=list(newshape): - raise CDMSError, 'axes must be shaped %s'%`newshape` + if axesshape != list(newshape): + raise CDMSError('axes must be shaped %s' % repr(newshape)) ta = _makeMaskedArg(a) maresult = numpy.ma.reshape(ta, newshape) - F=getattr(a,"fill_value",1.e20) + F = getattr(a, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, no_update_from=True, fill_value=F) -reshape.__doc__="numpy doc: %s\naxes/attributes/grid are applied onto the new variable" % numpy.reshape.__doc__ +reshape.__doc__ = "numpy doc: %s\naxes/attributes/grid are applied onto the new variable" % numpy.reshape.__doc__ + -def resize (a, new_shape, axes=None, attributes=None, id=None, grid=None): +def resize(a, new_shape, axes=None, attributes=None, id=None, grid=None): """resize(a, new_shape) returns a new array with the specified shape. The original array's total size can be any size.""" ignore, attributes, id, ignore = _extractMetadata(a, axes, attributes, id) if axes is not None: axesshape = [len(item) for item in axes] - if axesshape!=list(new_shape): - raise CDMSError, 'axes must be shaped %s'%`newshape` + if axesshape != list(new_shape): + raise CDMSError('axes must be shaped %s' % repr(newshape)) ta = _makeMaskedArg(a) maresult = numpy.ma.resize(ta, new_shape) - F=getattr(a,"fill_value",1.e20) + F = getattr(a, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, fill_value=F) -def masked_array (a, mask=None, fill_value=None, axes=None, attributes=None, id=None): - """masked_array(a, mask=None) = + +def masked_array(a, mask=None, fill_value=None, + axes=None, attributes=None, id=None): + """masked_array(a, mask=None) = array(a, mask=mask, copy=0, fill_value=fill_value) Use fill_value(a) if None. """ - maresult = numpy.ma.masked_array(_makeMaskedArg(a), mask=mask, fill_value=fill_value) + maresult = numpy.ma.masked_array( + _makeMaskedArg(a), + mask=mask, + fill_value=fill_value) axes, attributes, id, grid = _extractMetadata(a, axes, attributes, id) - F=getattr(a,"fill_value",1.e20) + F = getattr(a, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, fill_value=F) -def masked_values (data, value, rtol=1.e-5, atol=1.e-8, copy=1, - savespace=0, axes=None, attributes=None, id=None): + +def masked_values(data, value, rtol=1.e-5, atol=1.e-8, copy=1, + savespace=0, axes=None, attributes=None, id=None): """ masked_values(data, value, rtol=1.e-5, atol=1.e-8) Create a masked array; mask is None if possible. May share data values with original array, but not recommended. Masked where abs(data-value)<= atol + rtol * abs(value) """ - maresult = numpy.ma.masked_values(_makeMaskedArg(data), value, rtol=rtol, atol=atol, copy=copy) + maresult = numpy.ma.masked_values( + _makeMaskedArg(data), + value, + rtol=rtol, + atol=atol, + copy=copy) axes, attributes, id, grid = _extractMetadata(data, axes, attributes, id) - F=getattr(data,"fill_value",1.e20) + F = getattr(data, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, fill_value=F) -def masked_object (data, value, copy=1, savespace=0, axes=None, attributes=None, id=None): + +def masked_object(data, value, copy=1, + savespace=0, axes=None, attributes=None, id=None): "Create array masked where exactly data equal to value" maresult = numpy.ma.masked_object(_makeMaskedArg(data), value, copy=copy) axes, attributes, id, grid = _extractMetadata(data, axes, attributes, id) - F=getattr(data,"fill_value",1.e20) + F = getattr(data, "fill_value", 1.e20) return TransientVariable(maresult, axes=axes, attributes=attributes, id=id, grid=grid, fill_value=F) - -def isMaskedVariable (x): + + +def isMaskedVariable(x): "Is x a masked variable, that is, an instance of AbstractVariable?" return isinstance(x, AbstractVariable) + def set_default_fill_value(value_type, value): """Set the default fill value for value_type to value. value_type is a string: 'real','complex','character','integer',or 'object'. @@ -896,24 +1013,23 @@ def set_default_fill_value(value_type, value): elif value_type == 'object': numpy.ma.default_object_fill_value = value -def fromfunction (f, dimensions): + +def fromfunction(f, dimensions): """Apply f to s to create an array as in numpy.""" return TransientVariable(numpy.ma.fromfunction(f, dimensions)) -def diagonal (a, offset = 0, axis1=0, axis2 = 1): - """diagonal(a, offset=0, axis1=0, axis2 = 1) returns the given + +def diagonal(a, offset=0, axis1=0, axis2=1): + """diagonal(a, offset=0, axis1=0, axis2 = 1) returns the given diagonals defined by the two dimensions of the array. """ - F=getattr(a,"fill_value",1.e20) - return TransientVariable(numpy.ma.diagonal(_makeMaskedArg(a), - offset, axis1, axis2), fill_value=F) + F = getattr(a, "fill_value", 1.e20) + return TransientVariable(numpy.ma.diagonal(_makeMaskedArg(a), + offset, axis1, axis2), fill_value=F) -def fromstring (s, t): + +def fromstring(s, t): """Construct a masked array from a string. Result will have no mask. t is a typecode. """ - return TransientArray(numpy.ma.fromstring(s,t)) - - - - + return TransientArray(numpy.ma.fromstring(s, t)) diff --git a/Packages/cdms2/Lib/__init__.py b/Packages/cdms2/Lib/__init__.py index 8746016325..e3e53ff098 100644 --- a/Packages/cdms2/Lib/__init__.py +++ b/Packages/cdms2/Lib/__init__.py @@ -2,35 +2,35 @@ CDMS module-level API """ import cdat_info -cdat_info.pingPCMDIdb("cdat","cdms2") +cdat_info.pingPCMDIdb("cdat", "cdms2") -__all__ = ["cdmsobj", "axis", "coord", "grid", "hgrid", "avariable", \ -"sliceut", "error", "variable", "fvariable", "tvariable", "dataset", \ -"database", "cache", "selectors", "MV2", "convention", "bindex", \ -"auxcoord", "gengrid", "gsHost", "gsStaticVariable", "gsTimeVariable", \ -"mvBaseWriter", "mvSphereMesh", "mvVsWriter", "mvCdmsRegrid"] +__all__ = ["cdmsobj", "axis", "coord", "grid", "hgrid", "avariable", + "sliceut", "error", "variable", "fvariable", "tvariable", "dataset", + "database", "cache", "selectors", "MV2", "convention", "bindex", + "auxcoord", "gengrid", "gsHost", "gsStaticVariable", "gsTimeVariable", + "mvBaseWriter", "mvSphereMesh", "mvVsWriter", "mvCdmsRegrid"] # Errors -from error import CDMSError +from .error import CDMSError # CDMS datatypes -from cdmsobj import CdArray, CdChar, CdByte, CdDouble, CdFloat, CdFromObject, CdInt, CdLong, CdScalar, CdShort, CdString +from .cdmsobj import CdArray, CdChar, CdByte, CdDouble, CdFloat, CdFromObject, CdInt, CdLong, CdScalar, CdShort, CdString # Functions which operate on all objects or groups of objects -from cdmsobj import Unlimited, getPathFromTemplate, matchPattern, matchingFiles, searchPattern, searchPredicate, setDebugMode +from .cdmsobj import Unlimited, getPathFromTemplate, matchPattern, matchingFiles, searchPattern, searchPredicate, setDebugMode # Axis functions and classes -from axis import AbstractAxis, axisMatches, axisMatchAxis, axisMatchIndex -from axis import createAxis, createEqualAreaAxis, createGaussianAxis, createUniformLatitudeAxis, createUniformLongitudeAxis, setAutoBounds, getAutoBounds +from .axis import AbstractAxis, axisMatches, axisMatchAxis, axisMatchIndex +from .axis import createAxis, createEqualAreaAxis, createGaussianAxis, createUniformLatitudeAxis, createUniformLongitudeAxis, setAutoBounds, getAutoBounds # Grid functions -from grid import createGenericGrid, createGlobalMeanGrid, createRectGrid, createUniformGrid, createZonalGrid, setClassifyGrids, createGaussianGrid, writeScripGrid, isGrid +from .grid import createGenericGrid, createGlobalMeanGrid, createRectGrid, createUniformGrid, createZonalGrid, setClassifyGrids, createGaussianGrid, writeScripGrid, isGrid # Dataset functions -from dataset import createDataset, openDataset, useNetcdf3, \ - getNetcdfClassicFlag, getNetcdfShuffleFlag, getNetcdfDeflateFlag, getNetcdfDeflateLevelFlag,\ +from .dataset import createDataset, openDataset, useNetcdf3, \ + getNetcdfClassicFlag, getNetcdfShuffleFlag, getNetcdfDeflateFlag, getNetcdfDeflateLevelFlag,\ setNetcdfClassicFlag, setNetcdfShuffleFlag, setNetcdfDeflateFlag, setNetcdfDeflateLevelFlag,\ - setNetcdfUseNCSwitchModeFlag,getNetcdfUseNCSwitchModeFlag,\ + setNetcdfUseNCSwitchModeFlag, getNetcdfUseNCSwitchModeFlag,\ setCompressionWarnings,\ setNetcdf4Flag, getNetcdf4Flag,\ setNetcdfUseParallelFlag, getNetcdfUseParallelFlag, \ @@ -39,31 +39,31 @@ open = openDataset # Database functions -from database import connect, Base, Onelevel, Subtree +from .database import connect, Base, Onelevel, Subtree -#Selectors -import selectors -from selectors import longitude, latitude, time, level, required, \ - longitudeslice, latitudeslice, levelslice, timeslice +# Selectors +from . import selectors +from .selectors import longitude, latitude, time, level, required, \ + longitudeslice, latitudeslice, levelslice, timeslice -from avariable import order2index, orderparse, setNumericCompatibility, getNumericCompatibility +from .avariable import order2index, orderparse, setNumericCompatibility, getNumericCompatibility # TV -from tvariable import asVariable, createVariable, isVariable +from .tvariable import asVariable, createVariable, isVariable -from mvSphereMesh import SphereMesh -from mvBaseWriter import BaseWriter -from mvVsWriter import VsWriter -from mvVTKSGWriter import VTKSGWriter -from mvVTKUGWriter import VTKUGWriter -from mvCdmsRegrid import CdmsRegrid +from .mvSphereMesh import SphereMesh +from .mvBaseWriter import BaseWriter +from .mvVsWriter import VsWriter +from .mvVTKSGWriter import VTKSGWriter +from .mvVTKUGWriter import VTKUGWriter +from .mvCdmsRegrid import CdmsRegrid # Gridspec is not installed by default so just pass on if it isn't installed try: - from gsStaticVariable import StaticFileVariable - from gsTimeVariable import TimeFileVariable + from .gsStaticVariable import StaticFileVariable + from .gsTimeVariable import TimeFileVariable except: pass -from restApi import esgfConnection,esgfDataset,FacetConnection +from .restApi import esgfConnection, esgfDataset, FacetConnection MV = MV2 diff --git a/Packages/cdms2/Lib/auxcoord.py b/Packages/cdms2/Lib/auxcoord.py index 24a97e8c3f..334831dce0 100644 --- a/Packages/cdms2/Lib/auxcoord.py +++ b/Packages/cdms2/Lib/auxcoord.py @@ -4,50 +4,66 @@ Note: In contrast to Axis objects (concrete classes subclassed from AbstractAxis), auxiliary coordinate variables are not monotonic in value, and do not share a name with the dimension. """ -## import internattr -from error import CDMSError -from coord import AbstractCoordinateAxis -from fvariable import FileVariable -from variable import DatasetVariable -from tvariable import TransientVariable -from avariable import AbstractVariable +# import internattr +from .error import CDMSError +from .coord import AbstractCoordinateAxis +from .fvariable import FileVariable +from .variable import DatasetVariable +from .tvariable import TransientVariable +from .avariable import AbstractVariable + class AbstractAuxAxis1D(AbstractCoordinateAxis): - def __init__ (self, parent=None, variableNode=None, bounds=None): - AbstractCoordinateAxis.__init__(self, parent, variableNode, bounds=bounds) + def __init__(self, parent=None, variableNode=None, bounds=None): + AbstractCoordinateAxis.__init__( + self, + parent, + variableNode, + bounds=bounds) - def clone (self, copyData=1): + def clone(self, copyData=1): """clone (self, copyData=1) Return a copy of self as a transient axis. If copyData is 1, make a separate copy of the data.""" - result = TransientAuxAxis1D(self[:], copy=copyData, axes=self.getAxisList(), attributes=self.attributes, bounds=self.getBounds()) + result = TransientAuxAxis1D( + self[:], + copy=copyData, + axes=self.getAxisList(), + attributes=self.attributes, + bounds=self.getBounds()) return result def setBounds(self, bounds): if bounds is not None: - if len(bounds.shape)!=2: - raise CDMSError, 'Bounds must have rank=2' - if bounds.shape[0:1]!=self.shape: - raise CDMSError, 'Bounds shape %s is inconsistent with axis shape %s'%(`bounds.shape`,`self.shape`) + if len(bounds.shape) != 2: + raise CDMSError('Bounds must have rank=2') + if bounds.shape[0:1] != self.shape: + raise CDMSError( + 'Bounds shape %s is inconsistent with axis shape %s' % + (repr(bounds.shape), repr(self.shape))) AbstractCoordinateAxis.setBounds(self, bounds) - def subSlice (self, *specs, **keys): + def subSlice(self, *specs, **keys): # Take a subslice, returning a TransientAuxAxis1D avar = AbstractVariable.subSlice(self, *specs, **keys) bounds = self.getBounds() if bounds is None: newbounds = None else: - newbounds = bounds[specs] # bounds can be a numarray or DatasetVariable + newbounds = bounds[ + specs] # bounds can be a numarray or DatasetVariable - # Note: disable axis copy to preserve identity of grid and variable domains - result = TransientAuxAxis1D(avar, bounds=newbounds, copyaxes=0) + # Note: disable axis copy to preserve identity of grid and variable + # domains + result = TransientAuxAxis1D(avar, bounds=newbounds, copyaxes=0) return result + class DatasetAuxAxis1D(AbstractAuxAxis1D, DatasetVariable): # Note: node is a VariableNode + def __init__(self, parent, id=None, variableNode=None, bounds=None): AbstractAuxAxis1D.__init__(self, parent, variableNode, bounds=bounds) DatasetVariable.__init__(self, parent, id, variableNode) @@ -55,11 +71,13 @@ def __init__(self, parent, id=None, variableNode=None, bounds=None): def __repr__(self): if self.parent is not None: - return ""%(self.id, self.parent.id, `self.shape`) + return "" % (self.id, self.parent.id, repr(self.shape)) else: - return ""%self.id + return "" % self.id + +# internattr.initialize_internal_attributes(DatasetAuxAxis1D) # Copy +# internal attrs from parents -## internattr.initialize_internal_attributes(DatasetAuxAxis1D) # Copy internal attrs from parents class FileAuxAxis1D(AbstractAuxAxis1D, FileVariable): @@ -70,15 +88,18 @@ def __init__(self, parent, id, obj=None, bounds=None): def __repr__(self): if self.parent is not None: - return ""%(self.id, self.parent.id, `self.shape`) + return "" % (self.id, self.parent.id, repr(self.shape)) else: - return ""%self.id + return "" % self.id + +# internattr.initialize_internal_attributes(FileAuxAxis1D) # Copy internal +# attrs from parents -## internattr.initialize_internal_attributes(FileAuxAxis1D) # Copy internal attrs from parents class TransientAuxAxis1D(AbstractAuxAxis1D, TransientVariable): - def __init__(self, data, typecode=None, copy=0, savespace=0, mask=None, fill_value=None, + def __init__( + self, data, typecode=None, copy=0, savespace=0, mask=None, fill_value=None, axes=None, attributes=None, id=None, copyaxes=1, bounds=None): """Create a transient, auxiliary 1-D axis. All arguments are as for TransientVariable. @@ -86,11 +107,12 @@ def __init__(self, data, typecode=None, copy=0, savespace=0, mask=None, fill_val nvert is the max number of vertices per cell. """ AbstractAuxAxis1D.__init__(self, None, None, bounds=bounds) - TransientVariable.__init__(self, data, typecode=typecode, copy=copy, savespace=savespace, + TransientVariable.__init__( + self, data, typecode=typecode, copy=copy, savespace=savespace, mask=mask, fill_value=fill_value, axes=axes, attributes=attributes, id=id, copyaxes=copyaxes) if axes is not None: self.setBounds(bounds) -## internattr.initialize_internal_attributes(TransientAuxAxis1D) # Copy internal attrs from parents - +# internattr.initialize_internal_attributes(TransientAuxAxis1D) # Copy +# internal attrs from parents diff --git a/Packages/cdms2/Lib/avariable.py b/Packages/cdms2/Lib/avariable.py index db174105eb..ae31c11718 100644 --- a/Packages/cdms2/Lib/avariable.py +++ b/Packages/cdms2/Lib/avariable.py @@ -1,26 +1,26 @@ -## Automatically adapted for numpy.oldnumeric Aug 01, 2007 by -## Further modified to be pure new numpy June 24th 2008 +# Automatically adapted for numpy.oldnumeric Aug 01, 2007 by +# Further modified to be pure new numpy June 24th 2008 "CDMS Variable objects, abstract interface" import numpy import re import warnings -import cdmsNode -from cdmsobj import CdmsObj +from . import cdmsNode +from .cdmsobj import CdmsObj import cdms2 -from slabinterface import Slab -from sliceut import * -from error import CDMSError -from axis import axisMatchIndex, axisMatchAxis, axisMatches, unspecified, CdtimeTypes, AbstractAxis -import selectors +from .slabinterface import Slab +from .sliceut import * +from .error import CDMSError +from .axis import axisMatchIndex, axisMatchAxis, axisMatches, unspecified, CdtimeTypes, AbstractAxis +from . import selectors import copy # from regrid2 import Regridder, PressureRegridder, CrossSectionRegridder -from mvCdmsRegrid import CdmsRegrid +from .mvCdmsRegrid import CdmsRegrid from regrid2.mvGenericRegrid import guessPeriodicity -#import PropertiedClasses -from convention import CF1 -from grid import AbstractRectGrid -#import internattr +# import PropertiedClasses +from .convention import CF1 +from .grid import AbstractRectGrid +# import internattr InvalidRegion = "Invalid region: " OutOfRange = "Coordinate interval is out of range or intersection has no data: " @@ -32,9 +32,10 @@ # True: return 0-D arrays # MV axis=1 by default + def getMinHorizontalMask(var): """ - Get the minimum mask associated with 'x' and 'y' (i.e. with the + Get the minimum mask associated with 'x' and 'y' (i.e. with the min number of ones) across all axes @param var CDMS variable with a mask @return mask array or None if order 'x' and 'y' were not found @@ -46,7 +47,7 @@ def getMinHorizontalMask(var): shp = var.shape ndims = len(shp) - order = var.getOrder() # e.g. 'zxty-', ndims = 5 + order = var.getOrder() # e.g. 'zxty-', ndims = 5 # run a few checks numX = order.count('x') @@ -57,43 +58,43 @@ def getMinHorizontalMask(var): msg = """ Not able to locate the horizontal (y, x) axes for order = %s in getMinHorizontalMask """ % str(order) - raise CDMSError, msg - - - ps = [] # index position of x/y, e.g. [1,3] - es = [] # end indices, sizes of x/y axes + raise CDMSError(msg) + + ps = [] # index position of x/y, e.g. [1,3] + es = [] # end indices, sizes of x/y axes nonHorizShape = [] found = False - for i in range(ndims-1, -1, -1): - # iterate backwards because the horizontal + for i in range(ndims - 1, -1, -1): + # iterate backwards because the horizontal # axes are more likely to be last o = order[i] # curvilinear coordinates have '-' in place of - # x or y, also check for '-' but exit if we think + # x or y, also check for '-' but exit if we think # we found the x and y coords if not found and (o in 'xy') or (not hasXY and o == '-'): - ps = [i,] + ps - es = [shp[i],] + es + ps = [i, ] + ps + es = [shp[i], ] + es if len(ps) == 2: found = True else: - nonHorizShape = [shp[i],] + nonHorizShape + nonHorizShape = [shp[i], ] + nonHorizShape if len(ps) == 2: - # found all the horizontal axes, start with mask + # found all the horizontal axes, start with mask # set to invalid everywhere mask = numpy.ones(es, numpy.bool8) # iterate over all non-horizontal axes, there can be as # many as you want... for it in MultiArrayIter(nonHorizShape): - inds = it.getIndices() # (i0, i1, i2) + inds = it.getIndices() # (i0, i1, i2) # build the slice operator, there are three parts - # (head, middle, and tail), some parts may be + # (head, middle, and tail), some parts may be # missing # slce = 'i0,' + ':,' + 'i1,' + ':,' + 'i2,' - slce = ('%d,'*ps[0]) % tuple(inds[:ps[0]]) + ':,' \ - + ('%d,'*(ps[1]-ps[0]-1)) % tuple(inds[ps[0]:ps[1]-1]) \ - + ':,' + ('%d,'*(ndims-ps[1]-1)) % tuple(inds[ps[1]-1:]) + slce = ('%d,' * ps[0]) % tuple(inds[:ps[0]]) + ':,' \ + + ('%d,' * (ps[1] - ps[0] - 1)) % tuple(inds[ps[0]:ps[1] - 1]) \ + + ':,' + \ + ('%d,' * (ndims - ps[1] - 1)) % tuple(inds[ps[1] - 1:]) # evaluate the slice for this time, level.... mask &= eval('var.mask[' + slce + ']') return mask @@ -101,45 +102,49 @@ def getMinHorizontalMask(var): msg = """ Could not find all the horizontal axes for order = %s in getMinHorizontalMask """ % str(order) - raise CDMSError, msg - return None + raise CDMSError(msg) + return None + def setNumericCompatibility(mode): global _numeric_compatibility - if mode==True or mode=='on': + if mode or mode == 'on': _numeric_compatibility = True - elif mode==False or mode=='off': + elif mode == False or mode == 'off': _numeric_compatibility = False + def getNumericCompatibility(): return _numeric_compatibility + class AbstractVariable(CdmsObj, Slab): - def __init__ (self, parent=None, variableNode=None): + + def __init__(self, parent=None, variableNode=None): """Not to be called by users. variableNode is the variable tree node, if any. parent is the containing dataset instance. """ - if variableNode is not None and variableNode.tag !='variable': - raise CDMSError, 'Node is not a variable node' + if variableNode is not None and variableNode.tag != 'variable': + raise CDMSError('Node is not a variable node') CdmsObj.__init__(self, variableNode) - val = self.__cdms_internals__ + ['id','domain'] - self.___cdms_internals__ = val + val = self.__cdms_internals__ + ['id', 'domain'] + self.___cdms_internals__ = val Slab.__init__(self) self.id = None # Transient variables key on this to create a default ID self.parent = parent self._grid_ = None # Variable grid, if any - if not hasattr(self,'missing_value'): + if not hasattr(self, 'missing_value'): self.missing_value = None elif numpy.isnan(self.missing_value): - self.missing_value = None + self.missing_value = None # Reminder: children to define self.shape and set self.id - def __array__ (self, t=None, context=None): #Numeric, ufuncs call this + def __array__(self, t=None, context=None): # Numeric, ufuncs call this return numpy.ma.filled(self.getValue(squeeze=0)) - def __call__ (self, *args, **kwargs): + def __call__(self, *args, **kwargs): "Selection of a subregion using selectors" # separate options from selector specs d = kwargs.copy() @@ -152,13 +157,13 @@ def __call__ (self, *args, **kwargs): s = selectors.Selector(*args, **d) # get the selection return s.unmodified_select(self, raw=raw, - squeeze=squeeze, - order=order, - grid=grid) + squeeze=squeeze, + order=order, + grid=grid) select = __call__ - def rank (self): + def rank(self): return len(self.shape) def _returnArray(self, ar, squeeze, singles=None): @@ -166,85 +171,90 @@ def _returnArray(self, ar, squeeze, singles=None): # job is to make sure we return an numpy.ma or a scalar. # If singles is not None, squeeze dimension indices in singles inf = 1.8e308 - if isinstance(ar,cdms2.tvariable.TransientVariable): - result = numpy.ma.array(ar._data,mask=ar.mask) - elif numpy.ma.isMaskedArray(ar): #already numpy.ma, only need squeeze. + if isinstance(ar, cdms2.tvariable.TransientVariable): + result = numpy.ma.array(ar._data, mask=ar.mask) + elif numpy.ma.isMaskedArray(ar): # already numpy.ma, only need squeeze. result = ar elif isinstance(ar, numpy.ndarray): missing = self.getMissing() if missing is None: result = numpy.ma.masked_array(ar) - elif missing==inf or missing!=missing: # (x!=x) ==> x is NaN + elif missing == inf or missing != missing: # (x!=x) ==> x is NaN result = numpy.ma.masked_object(ar, missing, copy=0) - elif ar.dtype.char=='c': + elif ar.dtype.char == 'c': # umath.equal is not implemented - resultmask = (ar==missing) + resultmask = (ar == missing) if not resultmask.any(): resultmask = numpy.ma.nomask - result = numpy.ma.masked_array(ar, mask=resultmask, fill_value=missing) + result = numpy.ma.masked_array( + ar, + mask=resultmask, + fill_value=missing) else: result = numpy.ma.masked_values(ar, missing, copy=0) elif ar is numpy.ma.masked: - return ar - else: # scalar, but it might be the missing value + return ar + else: # scalar, but it might be the missing value missing = self.getMissing() if missing is None: - return ar #scalar + return ar # scalar else: result = numpy.ma.masked_values(ar, missing, copy=0) squoze = 0 if squeeze: n = 1 - newshape=[] + newshape = [] for s in result.shape: - if s == 1: - squoze = 1 - continue - else: - n = n * s - newshape.append(s) + if s == 1: + squoze = 1 + continue + else: + n = n * s + newshape.append(s) elif singles is not None: n = 1 - newshape=[] + newshape = [] oldshape = result.shape for i in range(len(oldshape)): - if i in singles: - squoze = 1 - continue - else: - s = oldshape[i] - n = n * s - newshape.append(s) - + if i in singles: + squoze = 1 + continue + else: + s = oldshape[i] + n = n * s + newshape.append(s) + else: n = numpy.ma.size(result) if n == 1 and squeeze: - return numpy.ma.ravel(result)[0] # scalar or masked + return numpy.ma.ravel(result)[0] # scalar or masked if squoze: result.shape = newshape return result def generateGridkey(self, convention, vardict): - """ generateGridkey(): Determine if the variable is gridded, + """ generateGridkey(): Determine if the variable is gridded, and generate ((latname, lonname, order, maskname, class), lat, lon) if gridded, or (None, None, None) if not gridded. vardict is the variable dictionary of the parent""" lat, nlat = convention.getVarLatId(self, vardict) lon, nlon = convention.getVarLonId(self, vardict) if (lat is not None) and (lat is lon): - raise CDMSError, "Axis %s is both a latitude and longitude axis! Check standard_name and/or axis attributes."%lat.id + raise CDMSError( + "Axis %s is both a latitude and longitude axis! Check standard_name and/or axis attributes." % + lat.id) # Check for 2D grid if (lat is None) or (lon is None): return None, lat, lon # Check for a rectilinear grid - if isinstance(lat, AbstractAxis) and isinstance(lon, AbstractAxis) and (lat.rank()==lon.rank()==1): + if isinstance(lat, AbstractAxis) and isinstance(lon, AbstractAxis) and (lat.rank() == lon.rank() == 1): return self.generateRectGridkey(lat, lon), lat, lon # Check for a curvilinear grid: - if lat.rank()==lon.rank()==2: + if lat.rank() == lon.rank() == 2: # check that they are defined on the same indices as self vardomain = self.getAxisIds() @@ -258,7 +268,7 @@ def generateGridkey(self, convention, vardict): if axisid not in vardomain: allok = 0 break - + # It's a curvilinear grid if allok: if hasattr(lat, 'maskid'): @@ -268,7 +278,7 @@ def generateGridkey(self, convention, vardict): return (lat.id, lon.id, 'yx', maskid, 'curveGrid'), lat, lon # Check for a generic grid: - if lat.rank()==lon.rank()==1: + if lat.rank() == lon.rank() == 1: # check that they are defined on the same indices as self vardomain = self.getAxisIds() @@ -282,7 +292,7 @@ def generateGridkey(self, convention, vardict): if axisid not in vardomain: allok = 0 break - + # It's a generic grid if allok: if hasattr(lat, 'maskid'): @@ -307,12 +317,14 @@ def generateRectGridkey(self, lat, lon): ilat = k k += 1 - if ilat==-1: - raise CDMSError, "Cannot find latitude axis; check standard_name and/or axis attributes" - if ilon==-1: - raise CDMSError, "Cannot find longitude axis; check standard_name and/or axis attributes" + if ilat == -1: + raise CDMSError( + "Cannot find latitude axis; check standard_name and/or axis attributes") + if ilon == -1: + raise CDMSError( + "Cannot find longitude axis; check standard_name and/or axis attributes") - if ilat0: + elif len(axes) > 0: # If forcing use of input axes, make sure they are not copied. - # Same if the grid is not rectilinear - this is when forceaxes is set. + # Same if the grid is not rectilinear - this is when forceaxes is + # set. copyaxes = (forceaxes is None) and (resultgrid is None) - result = TransientVariable(resultArray, - copy=0, - fill_value = newmissing, - axes=axes, - copyaxes = copyaxes, - grid = resultgrid, - attributes=self.attributes, - id = self.id) + result = TransientVariable(resultArray, + copy=0, + fill_value=newmissing, + axes=axes, + copyaxes=copyaxes, + grid=resultgrid, + attributes=self.attributes, + id=self.id) if grid is not None: order2 = grid.getOrder() if order is None: order = order2 elif order != order2: - raise CDMSError, 'grid, order options not compatible.' + raise CDMSError('grid, order options not compatible.') result = result.reorder(order).regrid(grid) if raw == 0: return result @@ -627,7 +653,7 @@ def subSlice (self, *specs, **keys): else: # Return numpy.ma for zero rank, so that __cmp__ works. return resultArray - def getSlice (self, *specs, **keys): + def getSlice(self, *specs, **keys): """x.getSlice takes arguments of the following forms and produces a return array. The keyword argument squeeze determines whether or not the shape of the returned array contains dimensions whose @@ -652,21 +678,21 @@ def getSlice (self, *specs, **keys): of that dimension, as in normal Python indexing. """ # Turn on squeeze and raw options by default. - keys['numericSqueeze'] = keys.get('numericSqueeze',0) - keys['squeeze'] = keys.get('squeeze',1-keys['numericSqueeze']) - keys['raw'] = keys.get('raw',1) + keys['numericSqueeze'] = keys.get('numericSqueeze', 0) + keys['squeeze'] = keys.get('squeeze', 1 - keys['numericSqueeze']) + keys['raw'] = keys.get('raw', 1) keys['order'] = keys.get('order', None) keys['grid'] = keys.get('grid', None) isitem = keys.get('isitem', 0) result = self.subSlice(*specs, **keys) # return a scalar for 0-D slices - if isitem and result.size==1 and (not _numeric_compatibility) and not result.mask.item(): + if isitem and result.size == 1 and (not _numeric_compatibility) and not result.mask.item(): result = result.item() return result def expertSlice(self, slicelist): - raise CDMSError, NotImplemented + 'expertSlice' + raise CDMSError(NotImplemented + 'expertSlice') def getRegion(self, *specs, **keys): """getRegion @@ -702,65 +728,67 @@ def getRegion(self, *specs, **keys): """ # By default, squeeze and raw options are on - keys['squeeze'] = keys.get ('squeeze', 1) - keys['raw'] = keys.get('raw',1) + keys['squeeze'] = keys.get('squeeze', 1) + keys['raw'] = keys.get('raw', 1) keys['order'] = keys.get('order', None) keys['grid'] = keys.get('grid', None) return self.subRegion(*specs, **keys) - def subRegion (self, *specs, **keys): + def subRegion(self, *specs, **keys): - speclist = self._process_specs (specs, keys) - slicelist = self.reg_specs2slices (speclist) + speclist = self._process_specs(specs, keys) + slicelist = self.reg_specs2slices(speclist) - squeeze = keys.get ('squeeze', 0) - raw = keys.get('raw',0) + squeeze = keys.get('squeeze', 0) + raw = keys.get('raw', 0) order = keys.get('order', None) grid = keys.get('grid', None) - raweasy = raw==1 and order is None and grid is None + raweasy = raw == 1 and order is None and grid is None if grid is not None and order is None: order = grid.getOrder() - # Check if any slice wraps around. wrapdim = -1 - + axes = [] circulardim = None - + for idim in range(len(slicelist)): item = slicelist[idim] axis = self.getAxis(idim) axislen = len(axis) - if(axis.isCircular()): circulardim=idim + if(axis.isCircular()): + circulardim = idim - wraptest1 = ( axis.isCircular() and speclist[idim] != unspecified) + wraptest1 = (axis.isCircular() and speclist[idim] != unspecified) start, stop = item.start, item.stop - wraptest2 = not ((start is None or (0<=start= 0: - raise CDMSError, "Too many dimensions wrap around." + raise CDMSError("Too many dimensions wrap around.") wrapdim = idim break - + else: # No wraparound, just read the data - # redo the speclist -> slice if passed circular test but not wrapped test + # redo the speclist -> slice if passed circular test but not + # wrapped test if(circulardim is not None): - slicelist = self.reg_specs2slices (speclist,force=circulardim) - - d = {'raw':raw, - 'squeeze':squeeze, - 'order':order, - 'grid':grid, - } + slicelist = self.reg_specs2slices(speclist, force=circulardim) + + d = {'raw': raw, + 'squeeze': squeeze, + 'order': order, + 'grid': grid, + } return self.subSlice(*slicelist, **d) # @@ -775,11 +803,11 @@ def subRegion (self, *specs, **keys): # shift the wrap slice to the positive side and calc number of cycles shifted # - wb=wrapslice.start - we=wrapslice.stop - ws=wrapslice.step - size=length - cycle=self.getAxis(wrapdim).getModulo() + wb = wrapslice.start + we = wrapslice.stop + ws = wrapslice.step + size = length + cycle = self.getAxis(wrapdim).getModulo() # # ncycle: @@ -789,92 +817,93 @@ def subRegion (self, *specs, **keys): # ncyclesrev: # resetting the world coordinate for reversed direction # - - ncycles=0 - ncyclesrev=0 - - if(ws>0): - - if(wb>0): - ncycles=1 - while(wb>=0): - wb=wb-size - we=we-size - ncycles=ncycles-1 + + ncycles = 0 + ncyclesrev = 0 + + if(ws > 0): + + if(wb > 0): + ncycles = 1 + while(wb >= 0): + wb = wb - size + we = we - size + ncycles = ncycles - 1 else: - ncycles=0 - while(wb<0): - wb=wb+size - we=we+size - ncycles=ncycles+1 - + ncycles = 0 + while(wb < 0): + wb = wb + size + we = we + size + ncycles = ncycles + 1 + if(wb < 0): - wb=wb+size - we=we+size - + wb = wb + size + we = we + size + # reversed direction - + else: # do the ncycles for resetting world coordinate - wbrev=wb - werev=we - werevNoneTest=0 + wbrev = wb + werev = we + werevNoneTest = 0 if(werev is None): - werev=0 - werevNoneTest=1 + werev = 0 + werevNoneTest = 1 - ncycleRevStart=1 + ncycleRevStart = 1 if(wbrev > 0): - ncyclesrev=ncycleRevStart - while(wbrev>=0): - wbrev=wbrev-size - werev=werev-size - ncyclesrev=ncyclesrev-1 + ncyclesrev = ncycleRevStart + while(wbrev >= 0): + wbrev = wbrev - size + werev = werev - size + ncyclesrev = ncyclesrev - 1 else: - ncyclesrev=0 - while(wbrev<0): - wbrev=wbrev+size - werev=werev+size - ncyclesrev=ncyclesrev+1 + ncyclesrev = 0 + while(wbrev < 0): + wbrev = wbrev + size + werev = werev + size + ncyclesrev = ncyclesrev + 1 while(werev < 0): - wbrev=wbrev+size - werev=werev+size + wbrev = wbrev + size + werev = werev + size # number of cycles to make the slice positive - while( we<0 and we != None ): - wb=wb+size - we=we+size - ncycles=ncycles+1 - - wb=wbrev - we=werev - if(werevNoneTest): we=None - - wrapslice=slice(wb,we,ws) - + while(we < 0 and we is not None): + wb = wb + size + we = we + size + ncycles = ncycles + 1 + + wb = wbrev + we = werev + if(werevNoneTest): + we = None + + wrapslice = slice(wb, we, ws) + # # calc the actual positive slices and create data array # - donew=1 + donew = 1 if(donew): wraps = splitSliceExt(wrapslice, length) - for kk in range(0,len(wraps)): - sl=wraps[kk] - + for kk in range(0, len(wraps)): + sl = wraps[kk] + slicelist[wrapdim] = sl if(kk == 0): ar1 = self.getSlice(squeeze=0, *slicelist) - result=ar1 + result = ar1 else: ar2 = self.getSlice(squeeze=0, *slicelist) - result = numpy.ma.concatenate((result,ar2),axis=wrapdim) + result = numpy.ma.concatenate((result, ar2), axis=wrapdim) else: @@ -883,8 +912,7 @@ def subRegion (self, *specs, **keys): ar1 = self.getSlice(squeeze=0, *slicelist) slicelist[wrapdim] = wrap2 ar2 = self.getSlice(squeeze=0, *slicelist) - result = numpy.ma.concatenate((ar1,ar2),axis=wrapdim) - + result = numpy.ma.concatenate((ar1, ar2), axis=wrapdim) if raweasy: return self._returnArray(result, squeeze) @@ -895,26 +923,28 @@ def subRegion (self, *specs, **keys): # #---------------------------------------------------------------------- - wrapspec=speclist[wrapdim] - + wrapspec = speclist[wrapdim] + axes = [] for i in range(self.rank()): - if squeeze and numpy.ma.size(result, i) == 1: continue + if squeeze and numpy.ma.size(result, i) == 1: + continue sl = slicelist[i] if i == wrapdim: axis = self.getAxis(i).subAxis(wb, we, ws) - + if(ws > 0): - delta_beg_wrap_dimvalue = ncycles*cycle + delta_beg_wrap_dimvalue = ncycles * cycle else: - delta_beg_wrap_dimvalue = ncyclesrev*cycle + delta_beg_wrap_dimvalue = ncyclesrev * cycle axis.setBounds(axis.getBounds() - delta_beg_wrap_dimvalue) - - axis[:]= (axis[:] - delta_beg_wrap_dimvalue).astype(axis.typecode()) + + axis[:] = (axis[:] - delta_beg_wrap_dimvalue).astype( + axis.typecode()) else: axis = self.getAxis(i).subaxis(sl.start, sl.stop, sl.step) @@ -922,18 +952,18 @@ def subRegion (self, *specs, **keys): result = self._returnArray(result, squeeze) result = TransientVariable(result, - copy=0, - fill_value = self.missing_value, - axes=axes, - attributes=self.attributes, - id = self.id) + copy=0, + fill_value=self.missing_value, + axes=axes, + attributes=self.attributes, + id=self.id) if grid is not None: order2 = grid.getOrder() if order is None: order = order2 elif order != order2: - raise CDMSError, 'grid, order options not compatible.' - + raise CDMSError('grid, order options not compatible.') + result = result.reorder(order).regrid(grid) if raw == 0: return result @@ -943,22 +973,22 @@ def subRegion (self, *specs, **keys): def getValue(self, squeeze=1): """Return the entire set of values.""" return self.getSlice(Ellipsis, squeeze=squeeze) - - def assignValue(self,data): - raise CDMSError, NotImplemented + 'assignValue' - def reorder (self, order): + def assignValue(self, data): + raise CDMSError(NotImplemented + 'assignValue') + + def reorder(self, order): """return self reordered per the specification order""" - if order is None: + if order is None: return self axes = self.getAxisList() permutation = order2index(axes, order) if permutation == range(len(axes)): return self - return MV.transpose (self, permutation) + return MV.transpose(self, permutation) - def regrid (self, togrid, missing=None, order=None, mask=None, **keywords): - """return self regridded to the new grid. + def regrid(self, togrid, missing=None, order=None, mask=None, **keywords): + """return self regridded to the new grid. One can use the regrid2.Regridder optional arguments as well. Example: @@ -973,48 +1003,49 @@ def regrid (self, togrid, missing=None, order=None, mask=None, **keywords): @param keywords optional keyword arguments dependent on regridTool @return regridded variable """ - # there is a circular dependency between cdms2 and regrid2. In + # there is a circular dependency between cdms2 and regrid2. In # principle, cdms2 files should not import regrid2, we're bending # rules here... import regrid2 from regrid2 import Horizontal - if togrid is None: + if togrid is None: return self else: - fromgrid = self.getGrid() # this returns the horizontal grid only + fromgrid = self.getGrid() # this returns the horizontal grid only # default w/o bounds - regridTool = 'libcf' + regridTool = 'libcf' regridMethod = 'linear' - if self.getAxis(-1).attributes.has_key('topology'): + if 'topology' in self.getAxis(-1).attributes: if self.getAxis(-1).attributes['topology'] == 'circular': # for the ESMF regridders - keywords['periodicity'] = guessPeriodicity(self.getAxis(-1).getBounds()) + keywords['periodicity'] = guessPeriodicity( + self.getAxis(-1).getBounds()) keywords['mkCyclic'] = 1 # for LibCF regridder # check if there are bounds and we have esmf - if fromgrid.getBounds() is not None and hasattr(regrid2,"ESMFRegrid"): + if fromgrid.getBounds() is not None and hasattr(regrid2, "ESMFRegrid"): regridTool = 'esmf' regridMethod = 'linear' # Hum ok if only 1 longitude regrid fails, let's check - if len(togrid.getLongitude())==1: - # esmf can't deal with this - regridTool = "regrid2" + if len(togrid.getLongitude()) == 1: + # esmf can't deal with this + regridTool = "regrid2" # let user override userSpecifiesMethod = False for rm in 'rm', 'method', 'regridmethod', 'regrid_method', 'regridMethod': - if keywords.has_key(rm): + if rm in keywords: regridMethod = keywords[rm] del keywords[rm] userSpecifiesMethod = True userSpecifiesTool = False for rt in 'rt', 'tool', 'regridtool', 'regrid_tool', 'regridTool': - if keywords.has_key(rt): + if rt in keywords: regridTool = keywords[rt] del keywords[rt] userSpecifiesTool = True @@ -1027,8 +1058,8 @@ def regrid (self, togrid, missing=None, order=None, mask=None, **keywords): # make sure the tool can do it if re.search('^regrid', regridTool, re.I) is not None and \ - ( len(fromgrid.getLatitude().shape) > 1 or \ - len(togrid.getLatitude().shape) > 1 ): + (len(fromgrid.getLatitude().shape) > 1 or + len(togrid.getLatitude().shape) > 1): message = """ avariable.regrid: regrid2 cannot do curvilinear, will switch to esmf..." """ @@ -1038,7 +1069,7 @@ def regrid (self, togrid, missing=None, order=None, mask=None, **keywords): if re.search('esmf', regridTool, re.I): # make sure source grids have bounds haveBounds = True - for g in fromgrid,: + for g in fromgrid, : for c in g.getLatitude(), g.getLongitude(): haveBounds &= (c.getBounds() is not None) if not haveBounds: @@ -1048,19 +1079,18 @@ def regrid (self, togrid, missing=None, order=None, mask=None, **keywords): warnings.warn(message, Warning) regridTool = 'libcf' regridMethod = 'linear' - if not hasattr(regrid2,"ESMFRegrid"): - message = """ + if not hasattr(regrid2, "ESMFRegrid"): + message = """ avariable.regrid: regridTool = 'esmf' but your version does not seems to be built with esmf, will switch to regridTool = 'libcf' """ - warnings.warn(message, Warning) - regridTool = 'libcf' - regridMethod = 'linear' - + warnings.warn(message, Warning) + regridTool = 'libcf' + regridMethod = 'linear' if re.search('conserv', regridMethod, re.I): # make sure destination grid has bounds haveBounds = True - for g in togrid,: + for g in togrid, : for c in g.getLatitude(), g.getLongitude(): haveBounds &= (c.getBounds() is not None) if not haveBounds: @@ -1074,47 +1104,48 @@ def regrid (self, togrid, missing=None, order=None, mask=None, **keywords): message = """ avariable.regrid: We chose regridTool = %s for you among the following choices: Tools -> 'regrid2' (old behavior) - 'esmf' (conserve, patch, linear) or + 'esmf' (conserve, patch, linear) or 'libcf' (linear)""" % regridTool warnings.warn(message, Warning) if not userSpecifiesMethod and re.search('^regrid', regridTool, re.I) is None: message = """ -avariable.regrid: We chose regridMethod = %s for you among the following choices: +avariable.regrid: We chose regridMethod = %s for you among the following choices: 'conserve' or 'linear' or 'patch'""" % regridMethod warnings.warn(message, Warning) if re.search('^regrid', regridTool, re.I): - if keywords.has_key('diag') and \ + if 'diag' in keywords and \ isinstance(keywords['diag'], dict): keywords['diag']['regridTool'] = 'regrid' # the original cdms2 regridder regridf = Horizontal(fromgrid, togrid) - return regridf(self, missing=missing, order=order, + return regridf(self, missing=missing, order=order, mask=mask, **keywords) # emsf or libcf... srcGridMask = None # set the source mask if a mask is defined with the source data - if numpy.any(self.mask == True): + if numpy.any(self.mask): srcGridMask = getMinHorizontalMask(self) # compute the interpolation weights - ro = CdmsRegrid(fromgrid, togrid, - dtype = self.dtype, - regridMethod = regridMethod, - regridTool = regridTool, - srcGridMask = srcGridMask, - srcGridAreas = None, - dstGridMask = None, - dstGridAreas = None, + ro = CdmsRegrid(fromgrid, togrid, + dtype=self.dtype, + regridMethod=regridMethod, + regridTool=regridTool, + srcGridMask=srcGridMask, + srcGridAreas=None, + dstGridMask=None, + dstGridAreas=None, **keywords) # now interpolate return ro(self, **keywords) - def pressureRegrid (self, newLevel, missing=None, order=None, method="log"): + def pressureRegrid( + self, newLevel, missing=None, order=None, method="log"): """Return the variable regridded to new pressure levels. The variable should be a function of lat, lon, pressure, and (optionally) time. is an axis of the result pressure levels. @@ -1126,12 +1157,13 @@ def pressureRegrid (self, newLevel, missing=None, order=None, method="log"): fromlevel = self.getLevel() if fromlevel is None: - raise CDMSError, 'No pressure level' + raise CDMSError('No pressure level') pregridf = PressureRegridder(fromlevel, newLevel) result = pregridf(self, missing=missing, order=order, method=method) return result - def crossSectionRegrid(self, newLevel, newLatitude, missing=None, order=None, method="log"): + def crossSectionRegrid( + self, newLevel, newLatitude, missing=None, order=None, method="log"): """Return the variable regridded to new pressure levels and latitudes. The variable should be a function of lat, level, and (optionally) time. is an axis of the result pressure levels. @@ -1145,14 +1177,18 @@ def crossSectionRegrid(self, newLevel, newLatitude, missing=None, order=None, me fromlevel = self.getLevel() fromlat = self.getLatitude() if fromlevel is None: - raise CDMSError, 'No pressure level' + raise CDMSError('No pressure level') if fromlat is None: - raise CDMSError, 'No latitude level' - xregridf = CrossSectionRegridder(fromlat, newLatitude, fromlevel, newLevel) + raise CDMSError('No latitude level') + xregridf = CrossSectionRegridder( + fromlat, + newLatitude, + fromlevel, + newLevel) result = xregridf(self, missing=missing, order=order, method=method) return result - def _process_specs (self, specs, keys): + def _process_specs(self, specs, keys): """Process the arguments for a getSlice, getRegion, etc. Returns an array of specifications for all dimensions. Any Ellipsis has been eliminated. @@ -1163,36 +1199,39 @@ def _process_specs (self, specs, keys): if Ellipsis in specs: nellipses = 1 elif numpy.newaxis in specs: - raise CDMSError, 'Sorry, you cannot use NewAxis in this context ' + str(specs) + raise CDMSError( + 'Sorry, you cannot use NewAxis in this context ' + str(specs)) else: nellipses = 0 - if nsupplied-nellipses > myrank: - raise CDMSError, InvalidRegion + \ - "too many dimensions: %d, for variable %s"%(len(specs),self.id) + if nsupplied - nellipses > myrank: + raise CDMSError(InvalidRegion + + "too many dimensions: %d, for variable %s" % (len(specs), self.id)) - speclist = [unspecified]*myrank + speclist = [unspecified] * myrank i = 0 j = 0 while i < nsupplied: if specs[i] is Ellipsis: - j = myrank - (nsupplied - (i+1)) + j = myrank - (nsupplied - (i + 1)) else: - speclist[j] = specs[i] - j = j + 1 + speclist[j] = specs[i] + j = j + 1 i = i + 1 for k, v in keys.items(): - if k in ['squeeze','raw','grid','order']: + if k in ['squeeze', 'raw', 'grid', 'order']: continue i = self.getAxisIndex(k) if i >= 0: if speclist[i] is not unspecified: - raise CDMSError, 'Conflict between specifier %s and %s'%(`speclist[i]`,`keys`) + raise CDMSError( + 'Conflict between specifier %s and %s' % + (repr(speclist[i]), repr(keys))) speclist[i] = v return speclist - def _single_specs (self, specs): + def _single_specs(self, specs): """Return a list of dimension indices where the spec is an index.""" myrank = self.rank() nsupplied = len(specs) @@ -1201,7 +1240,7 @@ def _single_specs (self, specs): singles = [] while i < nsupplied: if specs[i] is Ellipsis: - j = myrank - (nsupplied - (i+1)) + j = myrank - (nsupplied - (i + 1)) else: if isinstance(specs[i], int): singles.append(j) @@ -1209,7 +1248,7 @@ def _single_specs (self, specs): i = i + 1 return singles - def specs2slices (self, speclist, force=None): + def specs2slices(self, speclist, force=None): """Create an equivalent list of slices from an index specification An index specification is a list of acceptable items, which are -- an integer @@ -1220,22 +1259,22 @@ def specs2slices (self, speclist, force=None): The size of the speclist must be self.rank() """ if len(speclist) != self.rank(): - raise CDMSError, "Incorrect length of speclist in specs2slices." + raise CDMSError("Incorrect length of speclist in specs2slices.") slicelist = [] for i in range(self.rank()): key = speclist[i] if isinstance(key, int): # x[i] - slicelist.append (slice(key,key+1)) - elif isinstance(key, slice): # x[i:j:k] + slicelist.append(slice(key, key + 1)) + elif isinstance(key, slice): # x[i:j:k] slicelist.append(key) elif key is unspecified or key is None or key == ':': - slicelist.append (slice(0, len(self.getAxis(i)))) + slicelist.append(slice(0, len(self.getAxis(i)))) elif key is Ellipsis: - raise CDMSError, "Misuse of ellipsis in specification." + raise CDMSError("Misuse of ellipsis in specification.") elif isinstance(key, tuple): slicelist.append(slice(*key)) else: - raise CDMSError, 'invalid index: %s'% str(key) + raise CDMSError('invalid index: %s' % str(key)) # Change default or negative start, stop to positive for i in range(self.rank()): axis = self.getAxis(i) @@ -1246,86 +1285,106 @@ def specs2slices (self, speclist, force=None): # # allow negative indices in a wrapped (isCircular() = 1) axis # - circular=(axis.isCircular() and force is None) + circular = (axis.isCircular() and force is None) altered = 0 - if step is None: + if step is None: altered = 1 - step=1 + step = 1 - if ( ( start is None or stop is None or start<0 or stop<0 ) and ( circular == 0 ) ): + if ((start is None or stop is None or start < 0 or stop < 0) and (circular == 0)): altered = 1 adjustit = 1 - if step>0: - if start is None: - start=0 - if stop is None: - stop=length - if start==-1 and stop==0: - stop=length + if step > 0: + if start is None: + start = 0 + if stop is None: + stop = length + if start == -1 and stop == 0: + stop = length else: - if start is None: - start=length-1 + if start is None: + start = length - 1 if stop is None: # stop=-1 adjustit = 0 - if start<0: - start=start%length - if stop<0 and adjustit: - stop=stop%length - if altered: + if start < 0: + start = start % length + if stop < 0 and adjustit: + stop = stop % length + if altered: slicelist[i] = slice(start, stop, step) return slicelist - def reg_specs2slices(self, initspeclist,force=None): + def reg_specs2slices(self, initspeclist, force=None): # Don't use input to store return value - speclist=copy.copy(initspeclist) + speclist = copy.copy(initspeclist) for i in range(self.rank()): item = speclist[i] if isinstance(item, slice): newitem = item - elif item==':' or item is None or item is unspecified: + elif item == ':' or item is None or item is unspecified: axis = self.getAxis(i) - newitem = slice(0,len(axis)) + newitem = slice(0, len(axis)) elif isinstance(item, list) or \ - isinstance(item, tuple): + isinstance(item, tuple): axis = self.getAxis(i) - if len(item)==2: # (start,end) + if len(item) == 2: # (start,end) indexInterval = axis.mapIntervalExt(item) - elif len(item)==3: # (start,end,'xxx') - coordInterval = (item[0],item[1]) - indexInterval = axis.mapIntervalExt(coordInterval,item[2]) - elif len(item)==4: - coordInterval = (item[0],item[1]) - indexInterval = axis.mapIntervalExt(coordInterval,item[2],item[3]) - elif len(item)==5: - coordInterval = (item[0],item[1]) - indexInterval = axis.mapIntervalExt(coordInterval,item[2],item[3],item[4]) - elif len(item)==6: - coordInterval = (item[0],item[1]) - indexInterval = axis.mapIntervalExt(coordInterval,item[2],item[3],item[4],item[5]) + elif len(item) == 3: # (start,end,'xxx') + coordInterval = (item[0], item[1]) + indexInterval = axis.mapIntervalExt(coordInterval, item[2]) + elif len(item) == 4: + coordInterval = (item[0], item[1]) + indexInterval = axis.mapIntervalExt( + coordInterval, item[2], item[3]) + elif len(item) == 5: + coordInterval = (item[0], item[1]) + indexInterval = axis.mapIntervalExt( + coordInterval, + item[2], + item[3], + item[4]) + elif len(item) == 6: + coordInterval = (item[0], item[1]) + indexInterval = axis.mapIntervalExt( + coordInterval, + item[2], + item[3], + item[4], + item[5]) else: - raise CDMSError, InvalidRegion + "invalid format for coordinate interval: %s"%str(item) + raise CDMSError( + InvalidRegion + "invalid format for coordinate interval: %s" % + str(item)) if indexInterval is None: - raise CDMSError, OutOfRange + str(item) - newitem = slice(indexInterval[0],indexInterval[1],indexInterval[2]) + raise CDMSError(OutOfRange + str(item)) + newitem = slice( + indexInterval[0], + indexInterval[1], + indexInterval[2]) elif isinstance(item, (numpy.floating, float, numpy.integer, int, long, basestring)) or type(item) in CdtimeTypes: axis = self.getAxis(i) # # default is 'ccn' in axis.mapIntervalExt # - indexInterval = axis.mapIntervalExt((item,item)) + indexInterval = axis.mapIntervalExt((item, item)) if indexInterval is None: - raise CDMSError, OutOfRange + str(item) - newitem = slice(indexInterval[0],indexInterval[1],indexInterval[2]) + raise CDMSError(OutOfRange + str(item)) + newitem = slice( + indexInterval[0], + indexInterval[1], + indexInterval[2]) else: - raise CDMSError, InvalidRegion + "invalid format for coordinate interval: %s"%str(item) + raise CDMSError( + InvalidRegion + "invalid format for coordinate interval: %s" % + str(item)) speclist[i] = newitem - slicelist = self.specs2slices(speclist,force) + slicelist = self.specs2slices(speclist, force) return slicelist def _decodedType(self): @@ -1340,7 +1399,7 @@ def _decodedType(self): def isEncoded(self): "True iff self is represented as packed data." - return (hasattr(self,"scale_factor") or hasattr(self,"add_offset")) + return (hasattr(self, "scale_factor") or hasattr(self, "add_offset")) def decode(self, ar): "Decode compressed data. ar is a masked array, scalar, or numpy.ma.masked" @@ -1348,18 +1407,20 @@ def decode(self, ar): if hasattr(self, 'scale_factor'): scale_factor = self.scale_factor else: - scale_factor = numpy.array([1.0],resulttype) - + scale_factor = numpy.array([1.0], resulttype) + if hasattr(self, 'add_offset'): add_offset = self.add_offset else: - add_offset = numpy.array([0.0],resulttype) - + add_offset = numpy.array([0.0], resulttype) + if ar is not numpy.ma.masked: - result = scale_factor*ar + add_offset - if isinstance(result,numpy.ma.MaskedArray): + result = scale_factor * ar + add_offset + if isinstance(result, numpy.ma.MaskedArray): result = result.astype(resulttype) - numpy.ma.set_fill_value(result, numpy.ma.default_fill_value(0.)) + numpy.ma.set_fill_value( + result, + numpy.ma.default_fill_value(0.)) else: tmp = numpy.array(result) result = tmp.astype(resulttype)[0] @@ -1381,7 +1442,9 @@ def getGridIndices(self): result.append(j) break else: - raise CDMSError, 'Variable and grid do not share common dimensions: %s'%self.id + raise CDMSError( + 'Variable and grid do not share common dimensions: %s' % + self.id) return tuple(result) @@ -1391,106 +1454,109 @@ def __getitem__(self, key): if isinstance(key, tuple): speclist = self._process_specs(key, {}) else: - if isinstance(key, int) and key>=len(self): - raise IndexError, "Index too large: %d"%key + if isinstance(key, int) and key >= len(self): + raise IndexError("Index too large: %d" % key) speclist = self._process_specs([key], {}) # Note: raw=0 ensures that a TransientVariable is returned return self.getSlice(numericSqueeze=1, raw=0, isitem=1, *speclist) - + def __getslice__(self, low, high): # Note: raw=0 ensures that a TransientVariable is returned - return self.getSlice (slice(low, high), numericSqueeze = 1, raw=0) + return self.getSlice(slice(low, high), numericSqueeze=1, raw=0) def typecode(self): - raise CDMSError, NotImplemented + 'typecode' + raise CDMSError(NotImplemented + 'typecode') - def __abs__(self): + def __abs__(self): return MV.absolute(self) - def __neg__(self): + def __neg__(self): return MV.negative(self) def __add__(self, other): return MV.add(self, other) - + __radd__ = __add__ - def __lshift__ (self, n): + def __lshift__(self, n): return MV.left_shift(self, n) - def __rshift__ (self, n): + def __rshift__(self, n): return MV.right_shift(self, n) - - def __sub__(self, other): + + def __sub__(self, other): return MV.subtract(self, other) - def __rsub__(self, other): + def __rsub__(self, other): return MV.subtract(other, self) def __mul__(self, other): return MV.multiply(self, other) - + __rmul__ = __mul__ - def __div__(self, other): + def __div__(self, other): return MV.divide(self, other) - def __rdiv__(self, other): + def __rdiv__(self, other): return MV.divide(other, self) - def __pow__(self,other, third=None): + def __pow__(self, other, third=None): return MV.power(self, other, third) - def __iadd__(self, other): + def __iadd__(self, other): "Add other to self in place." return MV.add(self, other) # See if we can improve these later. - def __isub__(self, other): + def __isub__(self, other): "Subtract other from self in place." return MV.subtract(self, other) # See if we can improve these later. - def __imul__(self, other): + def __imul__(self, other): "Multiply self by other in place." return MV.multiply(self, other) # See if we can improve these later. - def __idiv__(self, other): + def __idiv__(self, other): "Divide self by other in place." return MV.divide(self, other) # See if we can improve these later. - def __eq__(self,other): - return MV.equal(self,other) + def __eq__(self, other): + return MV.equal(self, other) - def __ne__(self,other): - return MV.not_equal(self,other) + def __ne__(self, other): + return MV.not_equal(self, other) - def __lt__(self,other): - return MV.less(self,other) + def __lt__(self, other): + return MV.less(self, other) - def __le__(self,other): - return MV.less_equal(self,other) + def __le__(self, other): + return MV.less_equal(self, other) - def __gt__(self,other): - return MV.greater(self,other) + def __gt__(self, other): + return MV.greater(self, other) - def __ge__(self,other): - return MV.greater_equal(self,other) + def __ge__(self, other): + return MV.greater_equal(self, other) - def __sqrt__(self): + def __sqrt__(self): return MV.sqrt(self) - def astype (self, tc): + def astype(self, tc): "return self as array of given type." return self.subSlice().astype(tc) - -## internattr.add_internal_attribute(AbstractVariable, 'id', 'parent') -#PropertiedClasses.set_property(AbstractVariable, 'missing_value', acts=AbstractVariable._setmissing, nodelete=1) + +# internattr.add_internal_attribute(AbstractVariable, 'id', 'parent') +# PropertiedClasses.set_property(AbstractVariable, 'missing_value', +# acts=AbstractVariable._setmissing, nodelete=1) __rp = r'\s*([-txyz0-9]{1,1}|\(\s*\w+\s*\)|[.]{3,3})\s*' __crp = re.compile(__rp) -def orderparse (order): + + +def orderparse(order): """Parse an order string. Returns a list of axes specifiers. Order elements can be: Letters t, x, y, z meaning time, longitude, latitude, level @@ -1501,13 +1567,14 @@ def orderparse (order): (name) meaning an axis whose id is name """ if not isinstance(order, basestring): - raise CDMSError, 'order arguments must be strings.' + raise CDMSError('order arguments must be strings.') pos = 0 - result=[] + result = [] order = order.strip() while pos < len(order): m = __crp.match(order, pos) - if m is None: break + if m is None: + break r = m.group(1) if r[0] == '(': pass @@ -1520,11 +1587,12 @@ def orderparse (order): pos = m.end(0) if pos != len(order): - raise CDMSError, 'Order string "' + order + \ - '" malformed, index '+str(pos) + raise CDMSError('Order string "' + order + + '" malformed, index ' + str(pos)) return result -def order2index (axes, order): + +def order2index(axes, order): """Find the index permutation of axes to match order. The argument order is a string. Order elements can be: @@ -1540,22 +1608,23 @@ def order2index (axes, order): elif isinstance(order, list): result = order else: - raise CDMSError, 'order2index, order specified of bad type:' + str(type(order)) + raise CDMSError( + 'order2index, order specified of bad type:' + str(type(order))) n = len(axes) ie = n - permutation = [None]*n + permutation = [None] * n j = 0 pos = 0 while j < len(result): item = result[j] if isinstance(item, basestring): - if item == 't': + if item == 't': spec = 'time' - elif item == 'x': + elif item == 'x': spec = 'longitude' - elif item == 'y': + elif item == 'y': spec = 'latitude' - elif item == 'z': + elif item == 'z': spec = 'level' elif item == '-': pos += 1 @@ -1566,25 +1635,26 @@ def order2index (axes, order): for k in range(n): if axisMatches(axes[k], spec): if k in permutation: - raise CDMSError, 'Duplicate item in order %s' % order + raise CDMSError('Duplicate item in order %s' % order) permutation[pos] = k pos += 1 break else: - raise CDMSError, 'No axis matching order spec %s' %str(item) + raise CDMSError('No axis matching order spec %s' % str(item)) elif isinstance(item, int): if item in permutation: - raise CDMSError, 'Duplicate item in order %s' % order + raise CDMSError('Duplicate item in order %s' % order) if item >= n: - raise CDMSError, 'Index %d out of range in order %s' %\ - (item,order) + raise CDMSError('Index %d out of range in order %s' % + (item, order)) permutation[pos] = item pos += 1 elif item is Ellipsis: nleft = len(result) - j - 1 pos = n - nleft else: - raise CDMSError, 'List specified for order contains bad item: ' + repr(item) + raise CDMSError( + 'List specified for order contains bad item: ' + repr(item)) j += 1 for i in range(n): @@ -1593,7 +1663,7 @@ def order2index (axes, order): if permutation[j] is None: permutation[j] = i break - return permutation + return permutation -from tvariable import TransientVariable -import MV2 as MV +from .tvariable import TransientVariable +from . import MV2 as MV diff --git a/Packages/cdms2/Lib/axis.py b/Packages/cdms2/Lib/axis.py index 458096c221..1f9561313a 100644 --- a/Packages/cdms2/Lib/axis.py +++ b/Packages/cdms2/Lib/axis.py @@ -1,12 +1,13 @@ -## Automatically adapted for numpy.oldnumeric Aug 01, 2007 by -## Further modified to be pure new numpy June 24th 2008 +# Automatically adapted for numpy.oldnumeric Aug 01, 2007 by +# Further modified to be pure new numpy June 24th 2008 """ CDMS Axis objects """ _debug = 0 std_axis_attributes = ['name', 'units', 'length', 'values', 'bounds'] -import sys, copy +import sys +import copy import numpy # import regrid2._regrid import cdmsNode @@ -16,17 +17,24 @@ from sliceut import reverseSlice, splitSlice, splitSliceExt from error import CDMSError import forecast -#import internattr +# import internattr from UserList import UserList + + class AliasList (UserList): + def __init__(self, alist): - UserList.__init__(self,alist) - def __setitem__ (self, i, value): + UserList.__init__(self, alist) + + def __setitem__(self, i, value): self.data[i] = value.lower() + def __setslice(self, i, j, values): self.data[i:j] = map(lambda x: x.lower(), values) + def append(self, value): self.data.append(value.lower()) + def extend(self, values): self.data.extend(map("".lower, values)) @@ -46,35 +54,35 @@ def extend(self, values): InvalidNCycles = "Invalid number of cycles requested for wrapped dimension: " ComptimeType = type(cdtime.comptime(0)) -ReltimeType = type(cdtime.reltime(0,"days")) +ReltimeType = type(cdtime.reltime(0, "days")) CdtimeTypes = (ComptimeType, ReltimeType) # Map between cdtime calendar and CF tags calendarToTag = { - cdtime.MixedCalendar : 'gregorian', - cdtime.NoLeapCalendar : 'noleap', - cdtime.GregorianCalendar : 'proleptic_gregorian', - cdtime.JulianCalendar : 'julian', - cdtime.Calendar360 : '360_day', - cdtime.ClimCalendar : 'clim_noncf', - cdtime.ClimLeapCalendar : 'climleap_noncf', - cdtime.DefaultCalendar : 'gregorian', - cdtime.StandardCalendar : 'proleptic_gregorian', + cdtime.MixedCalendar: 'gregorian', + cdtime.NoLeapCalendar: 'noleap', + cdtime.GregorianCalendar: 'proleptic_gregorian', + cdtime.JulianCalendar: 'julian', + cdtime.Calendar360: '360_day', + cdtime.ClimCalendar: 'clim_noncf', + cdtime.ClimLeapCalendar: 'climleap_noncf', + cdtime.DefaultCalendar: 'gregorian', + cdtime.StandardCalendar: 'proleptic_gregorian', } tagToCalendar = { - 'gregorian' : cdtime.MixedCalendar, - 'standard' : cdtime.GregorianCalendar, - 'noleap' : cdtime.NoLeapCalendar, - 'julian' : cdtime.JulianCalendar, - 'proleptic_gregorian' : cdtime.GregorianCalendar, - '360_day' : cdtime.Calendar360, - '360' : cdtime.Calendar360, - '365_day' : cdtime.NoLeapCalendar, - 'clim' : cdtime.ClimCalendar, - 'clim_noncf' : cdtime.ClimCalendar, - 'climleap_noncf' : cdtime.ClimLeapCalendar, - 'climleap' : cdtime.ClimLeapCalendar, + 'gregorian': cdtime.MixedCalendar, + 'standard': cdtime.GregorianCalendar, + 'noleap': cdtime.NoLeapCalendar, + 'julian': cdtime.JulianCalendar, + 'proleptic_gregorian': cdtime.GregorianCalendar, + '360_day': cdtime.Calendar360, + '360': cdtime.Calendar360, + '365_day': cdtime.NoLeapCalendar, + 'clim': cdtime.ClimCalendar, + 'clim_noncf': cdtime.ClimCalendar, + 'climleap_noncf': cdtime.ClimLeapCalendar, + 'climleap': cdtime.ClimLeapCalendar, } # This is not an error message, it is used to detect which things have @@ -86,64 +94,77 @@ def extend(self, values): # Modes: # 0 : off (not bounds generation) # 1 : on (generate bounds) - # 2 : grid (generate bounds for lat/lon grids only) + # 2 : grid (generate bounds for lat/lon + # grids only) # Set autobounds mode to 'on' or 'off'. If on, getBounds will automatically # generate boundary information for an axis or grid, if not explicitly defined. # If 'off', and no boundary data is explicitly defined, the bounds will NOT # be generated; getBounds will return None for the boundaries. + + def setAutoBounds(mode): global _autobounds - if mode=='on' or mode==1: - _autobounds=1 - elif mode=='off' or mode==0: - _autobounds=0 - elif mode=='grid' or mode==2: - _autobounds=2 + if mode == 'on' or mode == 1: + _autobounds = 1 + elif mode == 'off' or mode == 0: + _autobounds = 0 + elif mode == 'grid' or mode == 2: + _autobounds = 2 + def getAutoBounds(): return _autobounds # Create a transient axis + + def createAxis(data, bounds=None, id=None, copy=0): return TransientAxis(data, bounds, id, copy=copy) # Generate a Gaussian latitude axis, north-to-south + + def createGaussianAxis(nlat): import regrid2._regrid - lats,wts,bnds = regrid2._regrid.gridattr(nlat,'gaussian') - - # For odd number of latitudes, gridattr returns 0 in the second half of lats - if nlat%2: - mid = nlat/2 - lats[mid+1:] = -lats[:mid][::-1] - - latBounds = numpy.zeros((nlat,2),numpy.float) - latBounds[:,0] = bnds[:-1] - latBounds[:,1] = bnds[1:] - lat = createAxis(lats,latBounds,id="latitude") + lats, wts, bnds = regrid2._regrid.gridattr(nlat, 'gaussian') + + # For odd number of latitudes, gridattr returns 0 in the second half of + # lats + if nlat % 2: + mid = nlat / 2 + lats[mid + 1:] = -lats[:mid][::-1] + + latBounds = numpy.zeros((nlat, 2), numpy.float) + latBounds[:, 0] = bnds[:-1] + latBounds[:, 1] = bnds[1:] + lat = createAxis(lats, latBounds, id="latitude") lat.designateLatitude() lat.units = "degrees_north" return lat # Generate an equal-area latitude axis, north-to-south + + def createEqualAreaAxis(nlat): import regrid2._regrid - lats,wts,bnds = regrid2._regrid.gridattr(nlat,'equalarea') - latBounds = numpy.zeros((nlat,2),numpy.float) - latBounds[:,0] = bnds[:-1] - latBounds[:,1] = bnds[1:] - lat = createAxis(lats,latBounds,id="latitude") + lats, wts, bnds = regrid2._regrid.gridattr(nlat, 'equalarea') + latBounds = numpy.zeros((nlat, 2), numpy.float) + latBounds[:, 0] = bnds[:-1] + latBounds[:, 1] = bnds[1:] + lat = createAxis(lats, latBounds, id="latitude") lat.designateLatitude() lat.units = "degrees_north" return lat # Generate a uniform latitude axis + + def createUniformLatitudeAxis(startLat, nlat, deltaLat): - latArray = startLat + deltaLat*numpy.arange(nlat) - lat = createAxis(latArray,id="latitude") + latArray = startLat + deltaLat * numpy.arange(nlat) + lat = createAxis(latArray, id="latitude") lat.designateLatitude() lat.units = "degrees_north" latBounds = lat.genGenericBounds(width=deltaLat) @@ -151,18 +172,21 @@ def createUniformLatitudeAxis(startLat, nlat, deltaLat): return lat # Generate a uniform longitude axis + + def createUniformLongitudeAxis(startLon, nlon, deltaLon): - lonArray = startLon + deltaLon*numpy.arange(nlon) - lon = createAxis(lonArray,id="longitude") + lonArray = startLon + deltaLon * numpy.arange(nlon) + lon = createAxis(lonArray, id="longitude") lon.designateLongitude() lon.units = "degrees_east" lonBounds = lon.genGenericBounds(width=deltaLon) lon.setBounds(lonBounds) return lon -def mapLinearIntersection(xind,yind,iind, - aMinusEps,aPlusEps,bPlusEps,bMinusEps, - boundLeft,nodeSubI,boundRight): + +def mapLinearIntersection(xind, yind, iind, + aMinusEps, aPlusEps, bPlusEps, bMinusEps, + boundLeft, nodeSubI, boundRight): """ Return true iff the coordinate interval (a,b) intersects the node @@ -182,34 +206,35 @@ def mapLinearIntersection(xind,yind,iind, """ if(iind == 'n' or iind == 'e'): - testC_ = ( aMinusEps <= nodeSubI ) - test_C = ( nodeSubI <= bPlusEps ) - testO_ = ( aPlusEps < nodeSubI ) - test_O = ( nodeSubI < bMinusEps ) + testC_ = (aMinusEps <= nodeSubI) + test_C = (nodeSubI <= bPlusEps) + testO_ = (aPlusEps < nodeSubI) + test_O = (nodeSubI < bMinusEps) elif(iind == 'b'): - testC_ = ( aMinusEps <= boundRight ) - test_C = ( boundLeft <= bPlusEps ) - testO_ = ( aPlusEps < boundRight ) - test_O = ( boundLeft < bMinusEps ) + testC_ = (aMinusEps <= boundRight) + test_C = (boundLeft <= bPlusEps) + testO_ = (aPlusEps < boundRight) + test_O = (boundLeft < bMinusEps) elif(iind == 's'): - testC_ = ( aMinusEps <= boundLeft ) - test_C = ( boundRight <= bPlusEps ) - testO_ = ( aPlusEps < boundLeft ) - test_O = ( boundRight < bMinusEps ) + testC_ = (aMinusEps <= boundLeft) + test_C = (boundRight <= bPlusEps) + testO_ = (aPlusEps < boundLeft) + test_O = (boundRight < bMinusEps) if(xind == 'c' and yind == 'c'): - test=(testC_ and test_C) + test = (testC_ and test_C) elif(xind == 'c' and yind == 'o'): - test=(testC_ and test_O) + test = (testC_ and test_O) elif(xind == 'o' and yind == 'c'): - test=(testO_ and test_C) + test = (testO_ and test_C) elif(xind == 'o' and yind == 'o'): - test=(testO_ and test_O) + test = (testO_ and test_O) return(test) -def mapLinearExt(axis, bounds, interval, indicator ='ccn', epsilon=None, stride=1, wrapped=0): +def mapLinearExt(axis, bounds, interval, + indicator='ccn', epsilon=None, stride=1, wrapped=0): """Map coordinate interval to index interval, without wraparound. interval has the form (x,y) where x and y are the endpoints in coordinate space. indicator is a three-character @@ -222,137 +247,139 @@ def mapLinearExt(axis, bounds, interval, indicator ='ccn', epsilon=None, stride= 'b' - the interval intersects the cell bounds 's' - the cell bounds are a subset of the interval 'e' - same as 'n', plus an extra node on either side. - + Returns the corresponding index interval (i,j), where iy: - x,y = y,x - xind = indicator[1] - yind = indicator[0] - + x, y = interval + + iind = indicator[2] + + if x > y: + x, y = y, x + xind = indicator[1] + yind = indicator[0] + else: xind = indicator[0] yind = indicator[1] - if axis[0]>axis[-1]: + if axis[0] > axis[-1]: ar = axis[::-1] - if bounds[0,0]1: - epsilon = eps * min(abs(ar[1]-ar[0]), abs(ar[-1]-ar[-2])) + eps = 1.0e-5 + if len(ar) > 1: + epsilon = eps * min(abs(ar[1] - ar[0]), abs(ar[-1] - ar[-2])) else: - epsilon=eps + epsilon = eps # # interval bound +/- epsilon # - aMinusEps=(x-epsilon) - aPlusEps=(x+epsilon) - bMinusEps=(y-epsilon) - bPlusEps=(y+epsilon) - + aMinusEps = (x - epsilon) + aPlusEps = (x + epsilon) + bMinusEps = (y - epsilon) + bPlusEps = (y + epsilon) - #oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo + # oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo # # out-of-bounds requests # - #oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo + # oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo - if iind in ['n','e']: + if iind in ['n', 'e']: mina = ar[0] maxa = ar[-1] else: - mina = bd[0,0] - maxa = bd[-1,1] - + mina = bd[0, 0] + maxa = bd[-1, 1] + if(bPlusEps < mina or aMinusEps > maxa): return None - #nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn + # nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn # # empty node check # - #nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn + # nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn # Handle empty intersections if ( - ( ((aPlusEps) > ar[-1]) and (iind == 'n') and (xind == 'o') ) or - ( ((aMinusEps) >= ar[-1]) and (iind == 'n') and (xind == 'c') ) or - ( ((bMinusEps) < ar[0] ) and (iind == 'n') and (yind == 'o') ) or - ( ((bPlusEps) <= ar[0] ) and (iind == 'n') and (yind == 'c') ) + (((aPlusEps) > ar[-1]) and (iind == 'n') and (xind == 'o')) or + (((aMinusEps) >= ar[-1]) and (iind == 'n') and (xind == 'c')) or + (((bMinusEps) < ar[0]) and (iind == 'n') and (yind == 'o')) or + (((bPlusEps) <= ar[0]) and (iind == 'n') and (yind == 'c')) ): return None - - bdMaxRight=max(bd[-1][0],bd[-1][1]) - bdMinLeft=min(bd[0][0],bd[0][1]) + bdMaxRight = max(bd[-1][0], bd[-1][1]) + bdMinLeft = min(bd[0][0], bd[0][1]) if ( - ( ( (aMinusEps) > bdMaxRight ) and (iind != 'n') and (xind == 'o') ) or - ( ( (aMinusEps) >= bdMaxRight ) and (iind != 'n') and (xind == 'c') ) or - ( ( (bPlusEps) < bdMinLeft ) and (iind != 'n') and (yind == 'o') ) or - ( ( (bPlusEps) <= bdMinLeft ) and (iind != 'n') and (yind == 'c') ) - ): + (((aMinusEps) > bdMaxRight) and (iind != 'n') and (xind == 'o')) or + (((aMinusEps) >= bdMaxRight) and (iind != 'n') and (xind == 'c')) or + (((bPlusEps) < bdMinLeft) and (iind != 'n') and (yind == 'o')) or + (((bPlusEps) <= bdMinLeft) and (iind != 'n') and (yind == 'c')) + ): return None - - # The intersection is nonempty; use searchsorted to get left/right limits for testing - ii,jj = numpy.searchsorted(ar,(x,y)) + # The intersection is nonempty; use searchsorted to get left/right limits + # for testing + + ii, jj = numpy.searchsorted(ar, (x, y)) # # find index range for left (iStart,iEnd) and right (jStart,jEnd) # - - # iEnd + 2 because last point in loop not done - iStart=ii-1 - iEnd=ii+2 - if(iStart < 0): iStart=0 - if( iEnd >= length ): iEnd = length - 1 - jStart=jj-1 - jEnd=jj+2 - if( jStart < 0 ): jStart=0 - if( jEnd >= length ): jEnd = length - 1 + # iEnd + 2 because last point in loop not done + iStart = ii - 1 + iEnd = ii + 2 + if(iStart < 0): + iStart = 0 + if(iEnd >= length): + iEnd = length - 1 + + jStart = jj - 1 + jEnd = jj + 2 + if(jStart < 0): + jStart = 0 + if(jEnd >= length): + jEnd = length - 1 # # initialise the index to -1 (does not exist) # - iInterval=-1 - jInterval=-1 - iIntervalB=-1 - jIntervalB=-1 + iInterval = -1 + jInterval = -1 + iIntervalB = -1 + jIntervalB = -1 - #pppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp + # pppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp # # preliminary checks # - #pppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp - + # pppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp if(iStart == jStart == iEnd == jEnd): iInterval = jInterval = iStart @@ -362,172 +389,173 @@ def mapLinearExt(axis, bounds, interval, indicator ='ccn', epsilon=None, stride= else: - #llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll + # llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll # # left interval check # - #llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll + # llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll # - user check - - for i in range(iStart,iEnd+1): - nodeSubI=ar[i] - boundLeft=bd[i][0] - boundRight=bd[i][1] + for i in range(iStart, iEnd + 1): + + nodeSubI = ar[i] + boundLeft = bd[i][0] + boundRight = bd[i][1] - test=mapLinearIntersection(xind,yind,iind, - aMinusEps,aPlusEps,bPlusEps,bMinusEps, - boundLeft,nodeSubI,boundRight) + test = mapLinearIntersection(xind, yind, iind, + aMinusEps, aPlusEps, bPlusEps, bMinusEps, + boundLeft, nodeSubI, boundRight) - if( iInterval == -1 and test ): + if(iInterval == -1 and test): iInterval = i break # - "B" check for extension - - for i in range(iStart,iEnd+1): - nodeSubI=ar[i] - boundLeft=bd[i][0] - boundRight=bd[i][1] + for i in range(iStart, iEnd + 1): - testB=mapLinearIntersection(xind,yind,'b', - aMinusEps,aPlusEps,bPlusEps,bMinusEps, - boundLeft,nodeSubI,boundRight) + nodeSubI = ar[i] + boundLeft = bd[i][0] + boundRight = bd[i][1] - if( iIntervalB == -1 and testB ): + testB = mapLinearIntersection(xind, yind, 'b', + aMinusEps, aPlusEps, bPlusEps, bMinusEps, + boundLeft, nodeSubI, boundRight) + + if(iIntervalB == -1 and testB): iIntervalB = i break - #rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr + # rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr # # right interval check # - #rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr + # rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr - for j in range(jStart,jEnd+1): + for j in range(jStart, jEnd + 1): - nodeSubI=ar[j] - boundLeft=bd[j][0] - boundRight=bd[j][1] + nodeSubI = ar[j] + boundLeft = bd[j][0] + boundRight = bd[j][1] # # user test # - test=mapLinearIntersection(xind,yind,iind, - aMinusEps,aPlusEps,bPlusEps,bMinusEps, - boundLeft,nodeSubI,boundRight) + test = mapLinearIntersection(xind, yind, iind, + aMinusEps, aPlusEps, bPlusEps, bMinusEps, + boundLeft, nodeSubI, boundRight) - if( ( jInterval == -1 and iInterval != -1 and test == 0 and j <= jEnd ) ): - jInterval = j-1 + if((jInterval == -1 and iInterval != -1 and test == 0 and j <= jEnd)): + jInterval = j - 1 - if( (j == length-1 and test == 1) ): + if((j == length - 1 and test == 1)): jInterval = j - + # no break here... # - # B test on right + # B test on right # - for j in range(jStart,jEnd+1): + for j in range(jStart, jEnd + 1): - nodeSubI=ar[j] - boundLeft=bd[j][0] - boundRight=bd[j][1] + nodeSubI = ar[j] + boundLeft = bd[j][0] + boundRight = bd[j][1] - testB=mapLinearIntersection(xind,yind,'b', - aMinusEps,aPlusEps,bPlusEps,bMinusEps, - boundLeft,nodeSubI,boundRight) + testB = mapLinearIntersection(xind, yind, 'b', + aMinusEps, aPlusEps, bPlusEps, bMinusEps, + boundLeft, nodeSubI, boundRight) - if( ( jIntervalB == -1 and iIntervalB != -1 and testB == 0 and j <= jEnd ) ): - jIntervalB = j-1 + if((jIntervalB == -1 and iIntervalB != -1 and testB == 0 and j <= jEnd)): + jIntervalB = j - 1 - if( ( j == length-1 and testB == 1) ): + if((j == length - 1 and testB == 1)): jIntervalB = j - - #eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + # eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee # # extension check # - #eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + # eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee if(iind == 'e'): # if B index does not exist return - if(iIntervalB < 0 or jIntervalB <0): + if(iIntervalB < 0 or jIntervalB < 0): return None # if user index exists: - elif ( ( iInterval > -1 and jInterval > -1 ) ): + elif ((iInterval > -1 and jInterval > -1)): if(jInterval < iInterval): - - npoints=iInterval-jInterval + + npoints = iInterval - jInterval if(npoints > 0): - (iInterval,jInterval)=(jInterval+1,iInterval+1) - + (iInterval, jInterval) = (jInterval + 1, iInterval + 1) + else: - jInterval=iInterval - iInterval=jInterval+1 - + jInterval = iInterval + iInterval = jInterval + 1 + else: - iInterval = iInterval-1 - jInterval = jInterval+1 - + iInterval = iInterval - 1 + jInterval = jInterval + 1 + # else set index interval to B index interval else: - - iInterval=iIntervalB - jInterval=jIntervalB + + iInterval = iIntervalB + jInterval = jIntervalB if(iInterval == jInterval): - if( x < ar[iInterval] and iInterval > 0 ): - iInterval=jInterval-1 - elif( jIntervalB < length-1 ): - jInterval=iInterval+1 + if(x < ar[iInterval] and iInterval > 0): + iInterval = jInterval - 1 + elif(jIntervalB < length - 1): + jInterval = iInterval + 1 if(jInterval < iInterval): - npoints=jInterval-iInterval + npoints = jInterval - iInterval if(npoints > 2): - jInterval=iIntervalB - iInterval=jIntervalB + jInterval = iIntervalB + iInterval = jIntervalB else: - jInterval=iIntervalB - iInterval=jIntervalB+1 - - # Since the lookup is linear, ensure that the result is in range [0..length) - iInterval = max(iInterval,0) - jInterval = min(jInterval,length-1) - - #ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + jInterval = iIntervalB + iInterval = jIntervalB + 1 + + # Since the lookup is linear, ensure that the result is in range + # [0..length) + iInterval = max(iInterval, 0) + jInterval = min(jInterval, length - 1) + + # ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff # # final checks # - #ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + # ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff # if jInteval < iInterval have a single point; set to iInterval if(jInterval < iInterval): - jInterval=iInterval - + jInterval = iInterval + elif(jInterval < 0 and iInterval < 0): return None - + # Reverse back if necessary - if direc=='dec': - iInterval,jInterval = length-jInterval-1,length-iInterval-1 - - iReturn=iInterval - jReturn=jInterval+1 + if direc == 'dec': + iInterval, jInterval = length - jInterval - 1, length - iInterval - 1 + + iReturn = iInterval + jReturn = jInterval + 1 + + return (iReturn, jReturn) - return (iReturn,jReturn) -def lookupArray(ar,value): +def lookupArray(ar, value): """Lookup value in array ar. Return index such that: (a) ar is monotonically increasing: value <= ar[index], index==0..len(ar)-1 @@ -537,42 +565,46 @@ def lookupArray(ar,value): value < ar[index], index==len(ar) """ ar = numpy.ma.filled(ar) - ascending = (ar[0](len(vec2)-len(vec1)): - return (0,-1) # vec1 is too large, cannot be a subset - issubset = numpy.alltrue(numpy.less(numpy.absolute(vec1-vec2[index:index+len(vec1)]),tol)) + index = lookupArray(vec2, vec1[0]) + if index > (len(vec2) - len(vec1)): + return (0, -1) # vec1 is too large, cannot be a subset + issubset = numpy.alltrue( + numpy.less(numpy.absolute(vec1 - vec2[index:index + len(vec1)]), tol)) if issubset: - return (issubset,index) + return (issubset, index) else: - return (0,-1) + return (0, -1) + def isOverlapVector(vec1, vec2, atol=1.e-8): """Returns (isoverlap, index) where: @@ -580,32 +612,35 @@ def isOverlapVector(vec1, vec2, atol=1.e-8): index is the index such that vec1[0]<=vec2[index]. If index==len(vec2), then vec1[0]>vec2[len(vec2)-1] """ - index = lookupArray(vec2,vec1[0]) - if index==0 and abs(vec1[0]-vec2[0]): - return (0,index) - elif index==len(vec2): - return (1,index) + index = lookupArray(vec2, vec1[0]) + if index == 0 and abs(vec1[0] - vec2[0]): + return (0, index) + elif index == len(vec2): + return (1, index) else: - ar2 = vec2[index:index+len(vec1)] + ar2 = vec2[index:index + len(vec1)] ar1 = vec1[:len(ar2)] isoverlap = numpy.ma.allclose(ar1, ar2, atol=atol) if isoverlap: - return (isoverlap,index) + return (isoverlap, index) else: - return (0,index) + return (0, index) + def allclose(ax1, ax2, rtol=1.e-5, atol=1.e-8): """True if all elements of axes ax1 and ax2 are close, in the sense of numpy.ma.allclose.""" - return ((ax1 is ax2) or numpy.ma.allclose(ax1[:],ax2[:],rtol=rtol,atol=atol)) + return ((ax1 is ax2) or numpy.ma.allclose(ax1[:], ax2[:], rtol=rtol, atol=atol)) -# AbstractAxis defines the common axis interface. +# AbstractAxis defines the common axis interface. # Concrete axis classes are derived from this class. + class AbstractAxis(CdmsObj): - def __init__ (self, parent, node): - CdmsObj.__init__ (self, node) - val = self.__cdms_internals__ + ['id',] + + def __init__(self, parent, node): + CdmsObj.__init__(self, node) + val = self.__cdms_internals__ + ['id', ] self.___cdms_internals__ = val self.parent = parent self.id = id @@ -613,11 +648,11 @@ def __init__ (self, parent, node): self._data_ = None # Cached wraparound values for circular axes self._doubledata_ = None - - def __str__ (self): + + def __str__(self): return "\n".join(self.listall() + "\n" - __repr__ = __str__ + __repr__=__str__ def __len__(self): raise CDMSError, MethodNotImplemented @@ -626,7 +661,7 @@ def _getshape(self): return (len(self),) def _getdtype(self, name): - tc = self.typecode() + tc=self.typecode() return numpy.dtype(tc) def __getitem__(self, key): @@ -648,17 +683,17 @@ def rank(self): # If persistent is true, write metadata to the container. def designateLatitude(self, persistent=0): if persistent: - self.axis = "Y" + self.axis="Y" else: - self.__dict__['axis'] = "Y" + self.__dict__['axis']="Y" self.attributes['axis']="Y" # Return true iff the axis is a latitude axis def isLatitude(self): - id = self.id.strip().lower() - if (hasattr(self,'axis') and self.axis=='Y'): return 1 - units = getattr(self,"units","").strip().lower() - if units in ["degrees_north","degree_north","degree_n","degrees_n","degreen","degreesn"]: + id=self.id.strip().lower() + if (hasattr(self, 'axis') and self.axis == 'Y'): return 1 + units=getattr(self, "units", "").strip().lower() + if units in ["degrees_north", "degree_north", "degree_n", "degrees_n", "degreen", "degreesn"]: return 1 return (id[0:3] == 'lat') or (id in latitude_aliases) @@ -666,25 +701,25 @@ def isLatitude(self): # If persistent is true, write metadata to the container. def designateLevel(self, persistent=0): if persistent: - self.axis = "Z" + self.axis="Z" else: - self.__dict__['axis'] = "Z" + self.__dict__['axis']="Z" self.attributes['axis']="Z" # Return true iff the axis is a level axis def isLevel(self): - id = self.id.strip().lower() - if (hasattr(self,'axis') and self.axis=='Z'): return 1 - if getattr(self,"positive","").strip().lower() in ["up","down"]: + id=self.id.strip().lower() + if (hasattr(self, 'axis') and self.axis == 'Z'): return 1 + if getattr(self, "positive", "").strip().lower() in ["up", "down"]: return 1 try: - #Ok let's see if this thing as pressure units + # Ok let's see if this thing as pressure units import genutil - p=genutil.udunits(1,"Pa") - units=getattr(self,'units',"").strip() + p=genutil.udunits(1, "Pa") + units=getattr(self, 'units', "").strip() p2=p.to(units) return 1 - except Exception,err: + except Exception, err: pass return ((id[0:3] == 'lev') or (id[0:5] == 'depth') or (id in level_aliases)) @@ -693,30 +728,30 @@ def isLevel(self): # If modulo is defined, set as circular def designateLongitude(self, persistent=0, modulo=360.0): if persistent: - self.axis = "X" + self.axis="X" if modulo is None: - self.topology = 'linear' + self.topology='linear' else: - self.modulo = modulo - self.topology = 'circular' + self.modulo=modulo + self.topology='circular' else: - self.__dict__['axis'] = "X" + self.__dict__['axis']="X" self.attributes['axis']="X" if modulo is None: - self.__dict__['topology'] = 'linear' - self.attributes['topology'] = 'linear' + self.__dict__['topology']='linear' + self.attributes['topology']='linear' else: - self.__dict__['modulo'] = modulo - self.__dict__['topology'] = 'circular' - self.attributes['modulo'] = modulo - self.attributes['topology'] = 'circular' + self.__dict__['modulo']=modulo + self.__dict__['topology']='circular' + self.attributes['modulo']=modulo + self.attributes['topology']='circular' # Return true iff the axis is a longitude axis def isLongitude(self): - id = self.id.strip().lower() - if (hasattr(self,'axis') and self.axis=='X'): return 1 - units = getattr(self,"units","").strip().lower() - if units in ["degrees_east","degree_east","degree_e","degrees_e","degreee","degreese"]: + id=self.id.strip().lower() + if (hasattr(self, 'axis') and self.axis == 'X'): return 1 + units=getattr(self, "units", "").strip().lower() + if units in ["degrees_east", "degree_east", "degree_e", "degrees_e", "degreee", "degreese"]: return 1 return (id[0:3] == 'lon') or (id in longitude_aliases) @@ -724,63 +759,65 @@ def isLongitude(self): # If persistent is true, write metadata to the container. def designateTime(self, persistent=0, calendar=None): if calendar is None: - calendar = cdtime.DefaultCalendar + calendar=cdtime.DefaultCalendar if persistent: - self.axis = "T" + self.axis="T" if calendar is not None: self.setCalendar(calendar, persistent) else: - self.__dict__['axis'] = "T" - self.attributes['axis'] = "T" + self.__dict__['axis']="T" + self.attributes['axis']="T" if calendar is not None: self.setCalendar(calendar, persistent) # For isTime(), keep track of whether each id is for a time axis or not, for better performance. - # This dictionary is a class variable (not a member of any particular instance). - idtaxis = {} # id:type where type is 'T' for time, 'O' for other + # This dictionary is a class variable (not a member of any particular + # instance). + idtaxis={} # id:type where type is 'T' for time, 'O' for other # Return true iff the axis is a time axis def isTime(self): - id = self.id.strip().lower() - if hasattr(self,'axis'): - if self.axis=='T': return 1 + id=self.id.strip().lower() + if hasattr(self, 'axis'): + if self.axis == 'T': return 1 elif self.axis is not None: return 0 # Have we saved the id-to-axis type information already? if id in self.idtaxis: - if self.idtaxis[id]=='T': + if self.idtaxis[id] == 'T': return 1 else: return 0 - ## Try to figure it out from units + # Try to figure it out from units try: import genutil - units=getattr(self,"units","").lower() - sp = units.split("since") - if len(sp)>1: - t=genutil.udunits(1,"day") - s = sp[0].strip() - if s in t.available_units() and t.known_units()[s]=="TIME": - self.idtaxis[id] = 'T' + units=getattr(self, "units", "").lower() + sp=units.split("since") + if len(sp) > 1: + t=genutil.udunits(1, "day") + s=sp[0].strip() + if s in t.available_units() and t.known_units()[s] == "TIME": + self.idtaxis[id]='T' return 1 - #try the plural version since udunits only as singular (day noy days) - s=s+"s" - if s in t.available_units() and t.known_units()[s]=="TIME": - self.idtaxis[id] = 'T' + # try the plural version since udunits only as singular (day noy + # days) + s=s + "s" + if s in t.available_units() and t.known_units()[s] == "TIME": + self.idtaxis[id]='T' return 1 except: pass - #return (id[0:4] == 'time') or (id in time_aliases) + # return (id[0:4] == 'time') or (id in time_aliases) if (id[0:4] == 'time') or (id in time_aliases): self.idtaxis[id]='T' return 1 else: - self.idtaxis[id] = 'O' + self.idtaxis[id]='O' return 0 # Return true iff the axis is a forecast axis def isForecast(self): - id = self.id.strip().lower() - if (hasattr(self,'axis') and self.axis=='F'): return 1 + id=self.id.strip().lower() + if (hasattr(self, 'axis') and self.axis == 'F'): return 1 return (id[0:6] == 'fctau0') or (id in forecast_aliases) def isForecastTime(self): return self.isForecast() @@ -790,15 +827,15 @@ def asComponentTime(self, calendar=None): if not hasattr(self, 'units'): raise CDMSError, "No time units defined" if calendar is None: - calendar = self.getCalendar() + calendar=self.getCalendar() if self.isForecast(): - result = [ forecast.comptime(t) for t in self[:] ] + result=[forecast.comptime(t) for t in self[:]] else: - result = [] + result=[] for val in self[:]: result.append(cdtime.reltime(val, self.units).tocomp(calendar)) return result - + # # mf 20010418 -- output DTGs (YYYYMMDDHH) # @@ -806,22 +843,22 @@ def asDTGTime(self, calendar=None): "Array version of cdtime tocomp. Returns a list of component times in DTG format." if not hasattr(self, 'units'): raise CDMSError, "No time units defined" - result = [] + result=[] if calendar is None: - calendar = self.getCalendar() + calendar=self.getCalendar() for val in self[:]: comptime=cdtime.reltime(val, self.units).tocomp(calendar) s=repr(comptime) tt=s.split(' ') - + ttt=tt[0].split('-') yr=int(ttt[0]) mo=int(ttt[1]) da=int(ttt[2]) - + ttt=tt[1].split(':') hr=int(ttt[0]) - dtg="%04d%02d%02d%02d"%(yr,mo,da,hr) + dtg="%04d%02d%02d%02d" % (yr, mo, da, hr) result.append(dtg) return result @@ -831,27 +868,35 @@ def asdatetime(self, calendar=None): import datetime if not hasattr(self, 'units'): raise CDMSError, "No time units defined" - result = [] + result=[] if calendar is None: - calendar = self.getCalendar() + calendar=self.getCalendar() for val in self[:]: c=cdtime.reltime(val, self.units).tocomp(calendar) - dtg = datetime.datetime(c.year,c.month,c.day,c.hour,c.minute,int(c.second),int((c.second-int(c.second))*1000)) + dtg=datetime.datetime( + c.year, + c.month, + c.day, + c.hour, + c.minute, + int(c.second), + int((c.second - int(c.second)) * 1000)) result.append(dtg) return result - def asRelativeTime( self, units=None ): + def asRelativeTime(self, units=None): "Array version of cdtime torel. Returns a list of relative times." - sunits = getattr(self,'units',None) - if sunits==None or sunits=='None': + sunits=getattr(self, 'units', None) + if sunits is None or sunits == 'None': raise CDMSError, "No time units defined" - if units==None or units=='None': + if units is None or units == 'None': units=sunits if self.isForecast(): - result = [ forecast.comptime(t).torel(units) for t in self[:] ] + result=[forecast.comptime(t).torel(units) for t in self[:]] else: - cal = self.getCalendar() - result = [ cdtime.reltime(t,sunits).torel(units,cal) for t in self[:] ] + cal=self.getCalendar() + result=[cdtime.reltime(t, sunits).torel(units, cal) + for t in self[:]] return result def toRelativeTime(self, units, calendar=None): @@ -860,31 +905,35 @@ def toRelativeTime(self, units, calendar=None): raise CDMSError, "No time units defined" n=len(self[:]) b=self.getBounds() - scal = self.getCalendar() + scal=self.getCalendar() if calendar is None: - calendar = scal + calendar=scal else: self.setCalendar(calendar) for i in range(n): tmp=cdtime.reltime(self[i], self.units).tocomp(scal) - tmp2 = numpy.array(float(tmp.torel(units, calendar).value)).astype(self[:].dtype.char) - ## if i==1 : print self[:].dtype.char,'tmp2:',tmp2,tmp2.astype('f'),self[i],self[i].astype('f') + tmp2=numpy.array(float(tmp.torel(units, calendar).value)).astype( + self[:].dtype.char) + # if i==1 : print + # self[:].dtype.char,'tmp2:',tmp2,tmp2.astype('f'),self[i],self[i].astype('f') self[i]=tmp2 if b is not None: - tmp=cdtime.reltime(b[i,0], self.units).tocomp(scal) - b[i,0]=numpy.array(float(tmp.torel(units, calendar).value)).astype(b.dtype.char) - tmp=cdtime.reltime(b[i,1], self.units).tocomp(scal) - b[i,1]=numpy.array(float(tmp.torel(units, calendar).value)).astype(b.dtype.char) + tmp=cdtime.reltime(b[i, 0], self.units).tocomp(scal) + b[i, 0]=numpy.array( + float(tmp.torel(units, calendar).value)).astype(b.dtype.char) + tmp=cdtime.reltime(b[i, 1], self.units).tocomp(scal) + b[i, 1]=numpy.array( + float(tmp.torel(units, calendar).value)).astype(b.dtype.char) if b is not None: self.setBounds(b) self.units=units return -#mfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmf +# mfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmf # # mf 20010412 -- test if an Axis is intrinsically circular # -#mfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmf +# mfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmf # Return true iff the axis wraps around @@ -892,22 +941,22 @@ def toRelativeTime(self, units, calendar=None): # (1) self.topology=='circular', or # (2) self.topology is undefined, and the axis is a longitude def isCircularAxis(self): - - if hasattr(self,'topology'): - iscircle = (self.topology=='circular') + + if hasattr(self, 'topology'): + iscircle=(self.topology == 'circular') elif self.isLongitude(): - iscircle = 1 + iscircle=1 else: - iscircle = 0 + iscircle=0 return iscircle -#mfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmf +# mfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmf # # mf 20010405 -- test if an transient Axis is REALLY circular # -#mfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmf +# mfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmf # Return true iff the axis wraps around @@ -916,45 +965,45 @@ def isCircularAxis(self): # (2) self.topology is undefined, and the axis is a longitude def isCircular(self): - if hasattr(self,'realtopology'): - if self.realtopology=='circular': return 1 - elif self.realtopology=='linear': return 0 + if hasattr(self, 'realtopology'): + if self.realtopology == 'circular': return 1 + elif self.realtopology == 'linear': return 0 if(len(self) < 2): return 0 - - baxis = self[0] - eaxis = self[-1] - deltaend = self[-1] - self[-2] - eaxistest = eaxis + deltaend - baxis + + baxis=self[0] + eaxis=self[-1] + deltaend=self[-1] - self[-2] + eaxistest=eaxis + deltaend - baxis cycle=self.getModuloCycle() - tol=0.01*deltaend + tol=0.01 * deltaend test=0 if(abs(eaxistest - cycle) < tol): test=1 - - if hasattr(self,'topology') and test == 1: - iscircle = (self.topology=='circular') + + if hasattr(self, 'topology') and test == 1: + iscircle=(self.topology == 'circular') elif (self.isLongitude() and test == 1): - iscircle = 1 + iscircle=1 else: - iscircle = 0 + iscircle=0 # save realtopology attribute in __dict__, don't write it to the file - if iscircle==1: self.__dict__['realtopology'] = 'circular' - elif iscircle==0: self.__dict__['realtopology'] = 'linear' + if iscircle == 1: self.__dict__['realtopology']='circular' + elif iscircle == 0: self.__dict__['realtopology']='linear' return iscircle def designateCircular(self, modulo, persistent=0): if persistent: - self.topology = 'circular' - self.modulo = modulo + self.topology='circular' + self.modulo=modulo else: - self.__dict__['topology'] = 'circular' - self.__dict__['modulo'] = modulo - self.attributes['modulo'] = modulo - self.attributes['topology'] = 'linear' + self.__dict__['topology']='circular' + self.__dict__['modulo']=modulo + self.attributes['modulo']=modulo + self.attributes['topology']='linear' def isLinear(self): raise CDMSError, MethodNotImplemented @@ -973,62 +1022,62 @@ def setBounds(self, bounds): # or None. If the axis does not have a calendar attribute, return the global # calendar. def getCalendar(self): - if hasattr(self,'calendar'): - calendar = self.calendar.lower() + if hasattr(self, 'calendar'): + calendar=self.calendar.lower() else: - calendar = None + calendar=None - cdcal = tagToCalendar.get(calendar, cdtime.DefaultCalendar) + cdcal=tagToCalendar.get(calendar, cdtime.DefaultCalendar) return cdcal # Set the calendar def setCalendar(self, calendar, persistent=1): if persistent: - self.calendar = calendarToTag.get(calendar, None) + self.calendar=calendarToTag.get(calendar, None) self.attributes['calendar']=self.calendar if self.calendar is None: raise CDMSError, InvalidCalendar % calendar else: - self.__dict__['calendar'] = calendarToTag.get(calendar, None) + self.__dict__['calendar']=calendarToTag.get(calendar, None) self.attributes['calendar']=self.calendar if self.__dict__['calendar'] is None: raise CDMSError, InvalidCalendar % calendar def getData(self): raise CDMSError, MethodNotImplemented - + # Return the entire array def getValue(self): return self.__getitem__(slice(None)) - def assignValue(self,data): - self.__setitem__(slice(None),data) + def assignValue(self, data): + self.__setitem__(slice(None), data) def _time2value(self, value): """ Map value of type comptime, reltime, or string of form "yyyy-mm-dd hh:mi:ss" to value""" if self.isTime(): if type(value) in CdtimeTypes: - value = value.torel(self.units, self.getCalendar()).value - elif isinstance(value, basestring) and value not in [':',unspecified]: - cal = self.getCalendar() - value = cdtime.s2c(value, cal).torel(self.units, cal).value + value=value.torel(self.units, self.getCalendar()).value + elif isinstance(value, basestring) and value not in [':', unspecified]: + cal=self.getCalendar() + value=cdtime.s2c(value, cal).torel(self.units, cal).value return value def getModuloCycle(self): - if hasattr(self,'modulo'): - cycle = self.modulo + if hasattr(self, 'modulo'): + cycle=self.modulo # # mf 20010419 test if attribute is a string (non CF), set to 360.0 # if isnstance(cycle, basestring): - cycle = 360.0 + cycle=360.0 else: - cycle = 360.0 + cycle=360.0 if isinstance(cycle, numpy.ndarray): - cycle = cycle[0] + cycle=cycle[0] return(cycle) @@ -1040,7 +1089,7 @@ def getModulo(self): return(self.getModuloCycle()) - def mapInterval(self,interval,indicator='ccn',cycle=None): + def mapInterval(self, interval, indicator='ccn', cycle=None): """ Map coordinate interval to index interval. interval has one of the forms: @@ -1071,295 +1120,301 @@ def mapInterval(self,interval,indicator='ccn',cycle=None): Note: if the interval is interior to the axis, but does not span any axis element, a singleton (i,i+1) indicating an adjacent index is returned. """ - i,j,k = self.mapIntervalExt(interval,indicator,cycle) - j = min(j, i+len(self)) - #i=i-1 - return (i,j) + i, j, k=self.mapIntervalExt(interval, indicator, cycle) + j=min(j, i + len(self)) + # i=i-1 + return (i, j) -#mfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmf +# mfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmf # # mf 20010308 - 20010412 -- general handing of wrapping # -#mfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmf +# mfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmfmf - def mapIntervalExt(self,interval,indicator='ccn',cycle=None,epsilon=None): + def mapIntervalExt( + self, interval, indicator='ccn', cycle=None, epsilon=None): """Like mapInterval, but returns (i,j,k) where k is stride, and (i,j) is not restricted to one cycle.""" # nCycleMax : max number of cycles a user a specify in wrapping nCycleMax=6 - + # interval is None returns the full interval - if interval is None or interval==':': + if interval is None or interval == ':': return (0, len(self), 1) # Allow intervals of the same form as getRegion. - if len(interval)==3: - x,y,indicator = interval - interval = (x,y) - elif len(interval)==4: - x,y,indicator,cycle = interval - interval = (x,y) + if len(interval) == 3: + x, y, indicator=interval + interval=(x, y) + elif len(interval) == 4: + x, y, indicator, cycle=interval + interval=(x, y) # check length of indicator if overridden by user # - indicator = indicator.lower() - if len(indicator)==2: indicator += 'n' + indicator=indicator.lower() + if len(indicator) == 2: indicator += 'n' - if( ( len(indicator) != 3 ) or - ( (indicator[0] != 'c' and indicator[0] != 'o') or + if((len(indicator) != 3) or + ((indicator[0] != 'c' and indicator[0] != 'o') or (indicator[1] != 'c' and indicator[1] != 'o') or (indicator[2] != 'n' and indicator[2] != 'b' and indicator[2] != 's' and indicator[2] != 'e') ) ): raise CDMSError, "EEE: 3-character interval/intersection indicator incomplete or incorrect = "\ - +indicator - + + indicator + if self._data_ is None: - self._data_ = self.getData() + self._data_=self.getData() - #ttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt + # ttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt # Handle time types - interval = (self._time2value(interval[0]), self._time2value(interval[1])) + interval=( + self._time2value(interval[0]), + self._time2value(interval[1])) # If the interval is reversed wrt self, reverse the interval and # set the stride to -1 - if (interval[0]<=interval[1])==(self[0]<=self[-1]): + if (interval[0] <= interval[1]) == (self[0] <= self[-1]): stride=1 else: stride=-1 - interval = (interval[1],interval[0]) - indicator = indicator[1]+indicator[0]+indicator[2] + interval=(interval[1], interval[0]) + indicator=indicator[1] + indicator[0] + indicator[2] - #mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm + # mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm # # basic test for wrapping - is axis REALLY circular? # - #ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + # ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff - xind = indicator[0] - yind = indicator[1] - iind = indicator[2] + xind=indicator[0] + yind=indicator[1] + iind=indicator[2] - xi,yi = interval + xi, yi=interval - length = len(self) - ar = self[:] - ar0 = ar[0] - arn = ar[-1] - armin = min(ar0,arn) - armax = max(ar0,arn) + length=len(self) + ar=self[:] + ar0=ar[0] + arn=ar[-1] + armin=min(ar0, arn) + armax=max(ar0, arn) # Wrapped if circular and at least one value is outside the axis range. - wraptest1 = self.isCircular() - wraptest2 = not ((armin <= xi <= armax) and (armin <= yi <= armax)) - + wraptest1=self.isCircular() + wraptest2=not ((armin <= xi <= armax) and (armin <= yi <= armax)) + if (wraptest1 and wraptest2): # - # find cycle and calc # of cycles in the interval + # find cycle and calc # of cycles in the interval # - + cycle=self.getModulo() - - intervalLength=yi-xi - intervalCycles=intervalLength/cycle - - bd = self.getBounds() - - nPointsCycle = len(ar) - ar0 = ar[0] - ar1 = ar[-1] + intervalLength=yi - xi + intervalCycles=intervalLength / cycle + + bd=self.getBounds() + + nPointsCycle=len(ar) + + ar0=ar[0] + ar1=ar[-1] # # test for reversed coordinates # - - if ar0>ar1: - cycle = -1 * abs(cycle) - #eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + if ar0 > ar1: + cycle=-1 * abs(cycle) + + # eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee # # make sure xi0 and yixi: xi,yi = yi,xi + + if cycle > 0 and yi < xi: xi, yi=yi, xi + if cycle < 0 and yi > xi: xi, yi=yi, xi # calculate the number of cycles to shift to positive side - - nCycleShift=numpy.floor((xi-ar0)/cycle) - xp = xi - cycle * nCycleShift - yp = xp + intervalLength - # Extend the data vector with wraparound number of cycles in interval and shifts + nCycleShift=numpy.floor((xi - ar0) / cycle) + xp=xi - cycle * nCycleShift + yp=xp + intervalLength + + # Extend the data vector with wraparound number of cycles in + # interval and shifts + + nCycle=int(intervalCycles + 1.0 + 0.5) + abs(nCycleShift) + - nCycle = int(intervalCycles + 1.0 + 0.5) + abs(nCycleShift) - - # # check if nCycle is > nCycleMax # if(nCycle >= nCycleMax): raise CDMSError, InvalidNCycles + repr(nCycle) - self._doubledata_ = numpy.concatenate(( ar, ar + cycle )) + self._doubledata_=numpy.concatenate((ar, ar + cycle)) k=2 - while(k= vice i > + # mf 20010328 negative stride i >= vice i > #---------------------------------------------------------------------- - - if wrap and ((k>0 and j>size) or (k<0 and i >= size)) and self.isCircular(): - modulo = self.getModuloCycle() + + if wrap and ((k > 0 and j > size) or (k < 0 and i >= size)) and self.isCircular(): + modulo=self.getModuloCycle() if modulo is not None: # If self is decreasing and stride is positive, # or self is increasing and stride is negative, subtract the modulus, # otherwise add it. - if (self[0]>self[-1])==(k>0): - modulo = -modulo + if (self[0] > self[-1]) == (k > 0): + modulo=-modulo #---------------------------------------------------------------------- # # mf 20010329 -- N vice two slice scheme (more general) # - #---------------------------------------------------------------------- + #------------------------------------------------------------------ donew=1 if(donew): - sn = splitSliceExt(slice(i,j,k),size) - if(_debug): print "SSSS1-------------------- ",sn,len(sn) + sn=splitSliceExt(slice(i, j, k), size) + if(_debug): print "SSSS1-------------------- ", sn, len(sn) - for kk in range(0,len(sn)): + for kk in range(0, len(sn)): sl=sn[kk] - if(_debug): print "SSSSSSSS kk = ",kk,sl - part = self[sl] + kk*modulo - if(_debug): print "SSSSSSSSSSSSSSS modulo",part[0],part[-1],modulo + if(_debug): print "SSSSSSSS kk = ", kk, sl + part=self[sl] + kk * modulo + if(_debug): print "SSSSSSSSSSSSSSS modulo", part[0], part[-1], modulo if(kk == 0): - data = part + data=part else: - data = numpy.concatenate((data,part)) + data=numpy.concatenate((data, part)) if fullBounds is not None: - bound = fullBounds[sl] + kk*modulo + bound=fullBounds[sl] + kk * modulo if (kk == 0): - bounds = bound + bounds=bound else: - bounds = numpy.concatenate((bounds,bound)) + bounds=numpy.concatenate((bounds, bound)) else: - bounds = None - - + bounds=None + + else: - - s1, s2 = splitSlice(slice(i,j,k),size) - if(_debug): print "SSSS0: original ",s1,s2 - - part1 = self[s1] - part2 = self[s2]+modulo - if(_debug): print "SSSSSSSSSSSSSSS modulo",self[0],self[-1],modulo - data = numpy.concatenate((part1,part2)) + + s1, s2=splitSlice(slice(i, j, k), size) + if(_debug): print "SSSS0: original ", s1, s2 + + part1=self[s1] + part2=self[s2] + modulo + if(_debug): print "SSSSSSSSSSSSSSS modulo", self[0], self[-1], modulo + data=numpy.concatenate((part1, part2)) if fullBounds is not None: - bounds1 = fullBounds[s1] - bounds2 = fullBounds[s2]+modulo - bounds = numpy.concatenate((bounds1,bounds2)) + bounds1=fullBounds[s1] + bounds2=fullBounds[s2] + modulo + bounds=numpy.concatenate((bounds1, bounds2)) else: - bounds = None - + bounds=None + else: # no wraparound - data = self[i:j:k] + data=self[i:j:k] if fullBounds is not None: - bounds = fullBounds[i:j:k] + bounds=fullBounds[i:j:k] else: - bounds = None - - newaxis = TransientAxis(data, bounds, id=self.id, copy=1) + bounds=None + + newaxis=TransientAxis(data, bounds, id=self.id, copy=1) if self.isLatitude(): newaxis.designateLatitude() if self.isLongitude(): newaxis.designateLongitude() @@ -1367,190 +1422,201 @@ def subaxis(self,i,j,k=1, wrap=True): if self.isTime(): newaxis.designateTime() for attname in self.attributes.keys(): - if attname not in ["datatype", "length","isvar","name_in_file","partition","partition_length"]: + if attname not in ["datatype", "length", "isvar", "name_in_file", "partition", "partition_length"]: setattr(newaxis, attname, getattr(self, attname)) - newaxis.attributes[attname]=getattr(self,attname) + newaxis.attributes[attname]=getattr(self, attname) # Change circular topology to linear if a strict subset was copied - if hasattr(self,"topology") and self.topology=="circular" and len(newaxis)=self[i]>=bounds[i,1]: + if not bounds[i, 0] >= self[i] >= bounds[i, 1]: raise CDMSError, InvalidBoundsArray + \ -'bounds[%i]=%f is not in the range [%f,%f]'%(i,self[i],bounds[i,1],bounds[i,0]) +'bounds[%i]=%f is not in the range [%f,%f]' % ( + i, self[i], bounds[i, 1], bounds[i, 0]) return bounds - # Generate bounds from midpoints. width is the width of the zone if the axis has one value. + # Generate bounds from midpoints. width is the width of the zone if the + # axis has one value. def genGenericBounds(self, width=1.0): if self._data_ is None: - self._data_ = self.getData() - ar = self._data_ - if len(self)>1: - leftPoint = numpy.array([1.5*ar[0]-0.5*ar[1]]) - midArray = (ar[0:-1]+ar[1:])/2.0 - rightPoint = numpy.array([1.5*ar[-1]-0.5*ar[-2]]) - bnds = numpy.concatenate((leftPoint,midArray,rightPoint)) + self._data_=self.getData() + ar=self._data_ + if len(self) > 1: + leftPoint=numpy.array([1.5 * ar[0] - 0.5 * ar[1]]) + midArray=(ar[0:-1] + ar[1:]) / 2.0 + rightPoint=numpy.array([1.5 * ar[-1] - 0.5 * ar[-2]]) + bnds=numpy.concatenate((leftPoint, midArray, rightPoint)) else: - delta = width/2.0 - bnds = numpy.array([self[0]-delta,self[0]+delta]) + delta=width / 2.0 + bnds=numpy.array([self[0] - delta, self[0] + delta]) # Transform to (n,2) array - retbnds = numpy.zeros((len(ar),2),numpy.float) - retbnds[:,0] = bnds[:-1] - retbnds[:,1] = bnds[1:] + retbnds=numpy.zeros((len(ar), 2), numpy.float) + retbnds[:, 0]=bnds[:-1] + retbnds[:, 1]=bnds[1:] if self.isLatitude(): - retbnds[0,:] = numpy.maximum(-90.0, numpy.minimum(90.0,retbnds[0,:])) - retbnds[-1,:] = numpy.maximum(-90.0, numpy.minimum(90.0,retbnds[-1,:])) + retbnds[0, :]=numpy.maximum( + -90.0, numpy.minimum(90.0, retbnds[0, :])) + retbnds[-1, :]=numpy.maximum( + -90.0, numpy.minimum(90.0, retbnds[-1, :])) return retbnds - def clone (self, copyData=1): + def clone(self, copyData=1): """clone (self, copyData=1) Return a copy of self as a transient axis. If copyData is 1, make a separate copy of the data.""" - b = self.getBounds() - if copyData==1: - mycopy = createAxis(copy.copy(self[:])) + b=self.getBounds() + if copyData == 1: + mycopy=createAxis(copy.copy(self[:])) else: - mycopy = createAxis(self[:]) - mycopy.id = self.id + mycopy=createAxis(self[:]) + mycopy.id=self.id try: mycopy.setBounds(b) except CDMSError: - b = mycopy.genGenericBounds() + b=mycopy.genGenericBounds() mycopy.setBounds(b) for k, v in self.attributes.items(): setattr(mycopy, k, v) return mycopy - def listall (self, all=None): + def listall(self, all=None): "Get list of info about this axis." - aname = self.id - result = [] + aname=self.id + result=[] result.append(' id: ' + aname) if self.isLatitude(): result.append(' Designated a latitude axis.') if self.isLongitude(): result.append(' Designated a longitude axis.') if self.isTime(): result.append(' Designated a time axis.') if self.isLevel(): result.append(' Designated a level axis.') try: - units = self.units + units=self.units result.append(' units: ' + units) except: pass - d = self.getValue() + d=self.getValue() result.append(' Length: ' + str(len(d))) result.append(' First: ' + str(d[0])) result.append(' Last: ' + str(d[-1])) - flag = 1 + flag=1 for k in self.attributes.keys(): if k in std_axis_attributes: continue if flag: result.append(' Other axis attributes:') - flag = 0 - result.append(' '+k+': '+str(self.attributes[k])) + flag=0 + result.append(' ' + k + ': ' + str(self.attributes[k])) result.append(' Python id: %s' % hex(id(self))) if all: result.append(" Values:") result.append(str(d)) - b = self.getBounds() + b=self.getBounds() result.append(" Bounds:") result.append(str(b)) return result def info(self, flag=None, device=None): "Write info about axis; include dimension values and weights if flag" - if device is None: device = sys.stdout + if device is None: device=sys.stdout device.write(str(self)) def isVirtual(self): "Return true iff coordinate values are implicitly defined." return 0 - shape = property(_getshape,None) - dtype = _getdtype + shape=property(_getshape, None) + dtype=_getdtype -## PropertiedClasses.set_property(AbstractAxis, 'shape', -## AbstractAxis._getshape, nowrite=1, nodelete=1) -## PropertiedClasses.set_property(AbstractAxis, 'dtype', -## AbstractAxis._getdtype, nowrite=1, nodelete=1) -## internattr.add_internal_attribute (AbstractAxis, 'id', 'parent') +# PropertiedClasses.set_property(AbstractAxis, 'shape', +# AbstractAxis._getshape, nowrite=1, nodelete=1) +# PropertiedClasses.set_property(AbstractAxis, 'dtype', +# AbstractAxis._getdtype, nowrite=1, nodelete=1) +# internattr.add_internal_attribute (AbstractAxis, 'id', 'parent') # One-dimensional coordinate axis in a dataset class Axis(AbstractAxis): - def __init__(self,parent,axisNode=None): + def __init__(self, parent, axisNode=None): if axisNode is not None and axisNode.tag != 'axis': raise CDMSError, 'Creating axis, node is not an axis node.' AbstractAxis.__init__(self, parent, axisNode) if axisNode is not None: if axisNode.partition is not None: - flatpart = axisNode.partition - self.__dict__['partition']=numpy.reshape(flatpart,(len(flatpart)/2,2)) + flatpart=axisNode.partition + self.__dict__['partition']=numpy.reshape( + flatpart, (len(flatpart) / 2, 2)) self.attributes['partition']=self.partition - self.id = axisNode.id - + self.id=axisNode.id + def typecode(self): return cdmsNode.CdToNumericType.get(self._node_.datatype) # Handle slices of the form x[i], x[i:j:k], x[(slice(i,j,k),)], and x[...] def __getitem__(self, key): - node = self._node_ - length = len(node) + node=self._node_ + length=len(node) # Allow key of form (slice(i,j),) etc. - if isinstance(key, tuple) and len(key)==1: - key = key[0] + if isinstance(key, tuple) and len(key) == 1: + key=key[0] - if isinstance(key, (int, numpy.int,numpy.int32)): # x[i] - if key>=length: + if isinstance(key, (int, numpy.int, numpy.int32)): # x[i] + if key >= length: raise IndexError, 'index out of bounds' else: - # Don't generate the entire array (if linear) just for one value - return node.data[key%length] - elif isinstance(key, slice): # x[i:j:k] + # Don't generate the entire array (if linear) just for one + # value + return node.data[key % length] + elif isinstance(key, slice): # x[i:j:k] if self._data_ is None: - self._data_ = node.getData() + self._data_=node.getData() return self._data_[key.start:key.stop:key.step] - elif isinstance(key, Ellipsis.__class__): # x[...] + elif isinstance(key, Ellipsis.__class__): # x[...] if self._data_ is None: - self._data_ = node.getData() + self._data_=node.getData() return self._data_ elif isinstance(key, tuple): - raise IndexError,'axis is one-dimensional' + raise IndexError, 'axis is one-dimensional' else: - raise IndexError,'index must be an integer: %s'%`key` + raise IndexError, 'index must be an integer: %s' % `key` # Get axis data def getData(self): @@ -1559,7 +1625,7 @@ def getData(self): # Handle slices of the form x[i:j] def __getslice__(self, low, high): if self._data_ is None: - self._data_ = self.getData() + self._data_=self.getData() return self._data_[low:high] def __len__(self): @@ -1567,85 +1633,85 @@ def __len__(self): # Return true iff the axis representation is linear def isLinear(self): - return self._node_.dataRepresent==cdmsNode.CdLinear + return self._node_.dataRepresent == cdmsNode.CdLinear # Return the bounds array, or generate a default if autoBounds mode is on def getBounds(self): - boundsArray = self.getExplicitBounds() + boundsArray=self.getExplicitBounds() try: self.validateBounds(boundsArray) except: - boundsArray = None - abopt = getAutoBounds() - if boundsArray is None and (abopt==1 or (abopt==2 and (self.isLatitude() or self.isLongitude()))) : - boundsArray = self.genGenericBounds() - + boundsArray=None + abopt=getAutoBounds() + if boundsArray is None and (abopt == 1 or (abopt == 2 and (self.isLatitude() or self.isLongitude()))): + boundsArray=self.genGenericBounds() + return boundsArray # Return the bounds array, or None def getExplicitBounds(self): - boundsArray = None - if hasattr(self,'bounds'): - boundsName = self.bounds + boundsArray=None + if hasattr(self, 'bounds'): + boundsName=self.bounds try: - boundsVar = self.parent.variables[boundsName] - boundsArray = numpy.ma.filled(boundsVar.getSlice()) + boundsVar=self.parent.variables[boundsName] + boundsArray=numpy.ma.filled(boundsVar.getSlice()) except KeyError: - boundsArray = None + boundsArray=None return boundsArray def getCalendar(self): - if hasattr(self,'calendar'): - calendar = self.calendar.lower() + if hasattr(self, 'calendar'): + calendar=self.calendar.lower() elif self.parent is not None and hasattr(self.parent, 'calendar'): - calendar = self.parent.calendar.lower() + calendar=self.parent.calendar.lower() else: - calendar = None + calendar=None - cdcal = tagToCalendar.get(calendar, cdtime.DefaultCalendar) + cdcal=tagToCalendar.get(calendar, cdtime.DefaultCalendar) return cdcal # In-memory coordinate axis class TransientAxis(AbstractAxis): - axis_count = 0 + axis_count=0 def __init__(self, data, bounds=None, id=None, attributes=None, copy=0): AbstractAxis.__init__(self, None, None) if id is None: - TransientAxis.axis_count = TransientAxis.axis_count + 1 - id = 'axis_' + str(TransientAxis.axis_count) + TransientAxis.axis_count=TransientAxis.axis_count + 1 + id='axis_' + str(TransientAxis.axis_count) if attributes is None: - if hasattr(data, 'attributes'): attributes = data.attributes + if hasattr(data, 'attributes'): attributes=data.attributes if attributes is not None: for name, value in attributes.items(): if name not in ['missing_value', 'name']: setattr(self, name, value) - self.id = id + self.id=id if isinstance(data, AbstractAxis): if copy == 0: - self._data_ = data[:] + self._data_=data[:] else: - self._data_ = numpy.array(data[:]) + self._data_=numpy.array(data[:]) elif isinstance(data, numpy.ndarray): if copy == 0: - self._data_ = data + self._data_=data else: - self._data_ = numpy.array(data) + self._data_=numpy.array(data) elif isinstance(data, numpy.ma.MaskedArray): if numpy.ma.getmask(data) is not numpy.ma.nomask: raise CDMSError, \ 'Cannot construct an axis with a missing value.' - data = data.data + data=data.data if copy == 0: - self._data_ = data + self._data_=data else: - self._data_ = numpy.array(data) + self._data_=numpy.array(data) elif data is None: - self._data_ = None + self._data_=None else: - self._data_ = numpy.array(data) + self._data_=numpy.array(data) - self._doubledata_ = None + self._doubledata_=None self.setBounds(bounds) def __getitem__(self, key): @@ -1655,10 +1721,10 @@ def __getslice__(self, low, high): return self._data_[low:high] def __setitem__(self, index, value): - self._data_[index] = numpy.ma.filled(value) + self._data_[index]=numpy.ma.filled(value) def __setslice__(self, low, high, value): - self._data_[low:high] = numpy.ma.filled(value) + self._data_[low:high]=numpy.ma.filled(value) def __len__(self): return len(self._data_) @@ -1666,7 +1732,7 @@ def __len__(self): def getBounds(self): if self._bounds_ is not None: return copy.copy(self._bounds_) - elif (getAutoBounds()==1 or (getAutoBounds()==2 and (self.isLatitude() or self.isLongitude()))): + elif (getAutoBounds() == 1 or (getAutoBounds() == 2 and (self.isLatitude() or self.isLongitude()))): return self.genGenericBounds() else: return None @@ -1682,29 +1748,33 @@ def getExplicitBounds(self): # # mf 20010308 - add validate key word, by default do not validate # - def setBounds(self, bounds, persistent=0, validate=0, index=None, boundsid=None): + def setBounds(self, bounds, persistent=0, + validate=0, index=None, boundsid=None): if bounds is not None: if isinstance(bounds, numpy.ma.MaskedArray): - bounds = numpy.ma.filled(bounds) + bounds=numpy.ma.filled(bounds) if validate: - bounds = self.validateBounds(bounds) + bounds=self.validateBounds(bounds) else: # Just do the absolute minimum validation - requiredShape = (len(self),2) - requiredShape2 = (len(self)+1,) - if bounds.shape!=requiredShape and bounds.shape!=requiredShape2: + requiredShape=(len(self), 2) + requiredShape2=(len(self) + 1,) + if bounds.shape != requiredShape and bounds.shape != requiredShape2: raise CDMSError, InvalidBoundsArray + \ - 'shape is %s, should be %s or %s'%(`bounds.shape`,`requiredShape`,`requiredShape2`) - if bounds.shape==requiredShape2: # case of "n+1" bounds + 'shape is %s, should be %s or %s' % ( + `bounds.shape`, + `requiredShape`, + `requiredShape2`) + if bounds.shape == requiredShape2: # case of "n+1" bounds bounds2=numpy.zeros(requiredShape) - bounds2[:,0]=bounds[:-1] - bounds2[:,1]=bounds[1::] + bounds2[:, 0]=bounds[:-1] + bounds2[:, 1]=bounds[1::] bounds=bounds2 - self._bounds_ = copy.copy(bounds) + self._bounds_=copy.copy(bounds) else: - if (getAutoBounds()==1 or (getAutoBounds()==2 and (self.isLatitude() or self.isLongitude()))): - self._bounds_ = self.genGenericBounds() + if (getAutoBounds() == 1 or (getAutoBounds() == 2 and (self.isLatitude() or self.isLongitude()))): + self._bounds_=self.genGenericBounds() else: - self._bounds_ = None + self._bounds_=None def isLinear(self): return 0 @@ -1719,17 +1789,17 @@ class TransientVirtualAxis(TransientAxis): def __init__(self, axisname, axislen): TransientAxis.__init__(self, None, id=axisname) - self._virtualLength = axislen # length of the axis + self._virtualLength=axislen # length of the axis def __len__(self): return self._virtualLength - def __str__ (self): - return ""%(self.id, self._virtualLength) + def __str__(self): + return "" % (self.id, self._virtualLength) - __repr__ = __str__ + __repr__=__str__ - def clone (self, copyData=1): + def clone(self, copyData=1): """clone (self, copyData=1) Return a copy of self as a transient virtual axis. If copyData is 1, make a separate copy of the data.""" @@ -1747,7 +1817,7 @@ def isVirtual(self): def setBounds(self, bounds): "No boundaries on virtual axes" - self._bounds_ = None + self._bounds_=None def __getitem__(self, key): return self.getData()[key] @@ -1755,79 +1825,84 @@ def __getitem__(self, key): def __getslice__(self, low, high): return self.getData()[low:high] -## PropertiedClasses.initialize_property_class (TransientVirtualAxis) +# PropertiedClasses.initialize_property_class (TransientVirtualAxis) # One-dimensional coordinate axis in a CdmsFile. class FileAxis(AbstractAxis): - + def __init__(self, parent, axisname, obj=None): - AbstractAxis.__init__ (self, parent, None) - val = self.__cdms_internals__ +['name_in_file',] - self.___cdms_internals__ = val - self.id = axisname - self._obj_ = obj + AbstractAxis.__init__(self, parent, None) + val=self.__cdms_internals__ + ['name_in_file', ] + self.___cdms_internals__=val + self.id=axisname + self._obj_=obj # Overshadows file boundary data, if not None - self._boundsArray_ = None - (units,typecode,name_in_file,parent_varname,dimtype,ncid) = \ - parent._file_.dimensioninfo[axisname] - self.__dict__['_units'] = units - att = self.attributes + self._boundsArray_=None + (units, typecode, name_in_file, parent_varname, + dimtype, ncid)=parent._file_.dimensioninfo[axisname] + self.__dict__['_units']=units + att=self.attributes att['units']=units - self.attributes = att - self.name_in_file = self.id + self.attributes=att + self.name_in_file=self.id if name_in_file: - self.name_in_file = name_in_file + self.name_in_file=name_in_file # Combine the attributes of the variable object, if any if obj is not None: for attname in self._obj_.__dict__.keys(): - attval = getattr(self._obj_,attname) + attval=getattr(self._obj_, attname) if not callable(attval): - self.__dict__[attname] = attval - att = self.attributes + self.__dict__[attname]=attval + att=self.attributes att[attname]=attval - self.attributes= att - + self.attributes=att + def getData(self): - if cdmsobj._debug==1: - print 'Getting array for axis',self.id + if cdmsobj._debug == 1: + print 'Getting array for axis', self.id if self.parent is None: raise CDMSError, FileWasClosed + self.id try: - result = self.parent._file_.readDimension(self.id) + result=self.parent._file_.readDimension(self.id) except: try: - result = apply(self._obj_.getitem, (slice(None,None),)) + result=apply(self._obj_.getitem, (slice(None, None),)) except: - raise CDMSError,'Data for dimension %s not found'%self.id + raise CDMSError, 'Data for dimension %s not found' % self.id return result def typecode(self): if self.parent is None: raise CDMSError, FileWasClosed + self.id - (units,typecode,name_in_file,parent_varname,dimtype,ncid) = \ - self.parent._file_.dimensioninfo[self.id] + (units, typecode, name_in_file, parent_varname, + dimtype, ncid)=self.parent._file_.dimensioninfo[self.id] return typecode - + def _setunits(self, value): - self._units = value + self._units=value self.attributes['units']=value if self.parent is None: raise CDMSError, FileWasClosed + self.id setattr(self._obj_, 'units', value) - (units,typecode,name_in_file,parent_varname,dimtype,ncid) = \ - self.parent._file_.dimensioninfo[self.id] - self.parent._file_.dimensioninfo[self.id] = \ - (value,typecode,name_in_file,parent_varname,dimtype,ncid) + (units, typecode, name_in_file, parent_varname, + dimtype, ncid)=self.parent._file_.dimensioninfo[self.id] + self.parent._file_.dimensioninfo[self.id]=( + value, + typecode, + name_in_file, + parent_varname, + dimtype, + ncid) def _getunits(self): return self._units def _delunits(self): del(self._units) del(self.attributes['units']) - delattr(self._obj_,'units') + delattr(self._obj_, 'units') - def __getattr__(self,name): + def __getattr__(self, name): if name == 'units': return self._units try: @@ -1835,27 +1910,27 @@ def __getattr__(self,name): except: raise AttributeError # setattr writes external attributes to the file - def __setattr__ (self, name, value): + def __setattr__(self, name, value): if name == 'units': self._setunits(value) return if hasattr(self, 'parent') and self.parent is None: raise CDMSError, FileWasClosed + self.id -## s = self.get_property_s (name) -## if s is not None: -## s(self, name, value) -## return - if not name in self.__cdms_internals__ and name[0]!='_': +# s = self.get_property_s (name) +# if s is not None: +# s(self, name, value) +# return + if not name in self.__cdms_internals__ and name[0] != '_': setattr(self._obj_, name, value) self.attributes[name]=value - self.__dict__[name] = value + self.__dict__[name]=value # delattr deletes external global attributes in the file def __delattr__(self, name): -## d = self.get_property_d(name) -## if d is not None: -## d(self, name) -## return +# d = self.get_property_d(name) +# if d is not None: +# d(self, name) +# return if name == "units": self._delunits() return @@ -1870,83 +1945,84 @@ def __delattr__(self, name): # Read data # If the axis has a related Cdunif variable object, just read that variable - # otherwise, cache the Cdunif (read-only) data values in self._data_. in this case, - # the axis is not extensible, so it is not necessary to reread it each time. + # otherwise, cache the Cdunif (read-only) data values in self._data_. in this case, + # the axis is not extensible, so it is not necessary to reread it each + # time. def __getitem__(self, key): if self.parent is None: raise CDMSError, FileWasClosed + self.id # See __getslice__ comment below. - if (self._obj_ is not None) and (self.parent._mode_!='r') and not (hasattr(self.parent,'format') and self.parent.format=="DRS"): + if (self._obj_ is not None) and (self.parent._mode_ != 'r') and not (hasattr(self.parent, 'format') and self.parent.format == "DRS"): # For negative strides, get the equivalent slice with positive stride, # then reverse the result. - if (isinstance(key, slice) and (key.step is not None) and key.step<0: - posslice = reverseSlice(key,len(self)) - result = apply(self._obj_.getitem, (posslice,)) + if (isinstance(key, slice) and (key.step is not None) and key.step < 0: + posslice=reverseSlice(key, len(self)) + result=apply(self._obj_.getitem, (posslice,)) return result[::-1] else: - if isinstance(key, int) and key>=len(self): - raise IndexError, 'Index out of bounds: %d'%key + if isinstance(key, int) and key >= len(self): + raise IndexError, 'Index out of bounds: %d' % key if isinstance(key, tuple): - key = (key,) + key=(key,) return apply(self._obj_.getitem, key) if self._data_ is None: - self._data_ = self.getData() - length = len(self._data_) + self._data_=self.getData() + length=len(self._data_) if isinstance(key, int): # x[i] - if key>=length: + if key >= length: raise IndexError, 'index out of bounds' else: - return self._data_[key%length] - elif isinstance(key, slice): # x[i:j:k] + return self._data_[key % length] + elif isinstance(key, slice): # x[i:j:k] return self._data_[key.start:key.stop:key.step] - elif isinstance(key, Ellipsis.__class__): # x[...] + elif isinstance(key, Ellipsis.__class__): # x[...] return self._data_ elif isinstance(key, tuple): - raise IndexError,'axis is one-dimensional' + raise IndexError, 'axis is one-dimensional' else: - raise IndexError,'index must be an integer or slice: %s'%`key` + raise IndexError, 'index must be an integer or slice: %s' % `key` def __getslice__(self, low, high): # Hack to prevent netCDF overflow error on 64-bit architectures - high = min(Max32int, high) - + high=min(Max32int, high) + # Hack to fix a DRS bug: force use of readDimension for DRS axes. # Since DRS is read-only here, it is OK just to cache all dimensions. if self.parent is None: raise CDMSError, FileWasClosed + self.id - if (self._obj_ is not None) and (self.parent._mode_!='r') and not (hasattr(self.parent,'format') and self.parent.format=="DRS"): - return apply(self._obj_.getslice,(low,high)) + if (self._obj_ is not None) and (self.parent._mode_ != 'r') and not (hasattr(self.parent, 'format') and self.parent.format == "DRS"): + return apply(self._obj_.getslice, (low, high)) else: if self._data_ is None: - self._data_ = self.getData() + self._data_=self.getData() return self._data_[low:high] def __setitem__(self, index, value): if self._obj_ is None: raise CDMSError, ReadOnlyAxis + self.id if self.parent is None: - raise CDMSError, FileWasClosed+self.id - return apply(self._obj_.setitem,(index,numpy.ma.filled(value))) + raise CDMSError, FileWasClosed + self.id + return apply(self._obj_.setitem, (index, numpy.ma.filled(value))) def __setslice__(self, low, high, value): # Hack to prevent netCDF overflow error on 64-bit architectures - high = min(Max32int, high) + high=min(Max32int, high) if self._obj_ is None: raise CDMSError, ReadOnlyAxis + self.id if self.parent is None: - raise CDMSError, FileWasClosed+self.id - return apply(self._obj_.setslice,(low,high,numpy.ma.filled(value))) + raise CDMSError, FileWasClosed + self.id + return apply(self._obj_.setslice, (low, high, numpy.ma.filled(value))) def __len__(self): if self.parent is None: raise CDMSError, FileWasClosed + self.id if self._obj_ is not None: - length = len(self._obj_) + length=len(self._obj_) elif self._data_ is None: - self._data_ = self.getData() - length = len(self._data_) + self._data_=self.getData() + length=len(self._data_) else: - length = len(self._data_) + length=len(self._data_) return length def isLinear(self): @@ -1954,31 +2030,31 @@ def isLinear(self): # Return the bounds array, or generate a default if autobounds mode is set def getBounds(self): - boundsArray = self.getExplicitBounds() + boundsArray=self.getExplicitBounds() try: - boundsArray = self.validateBounds(boundsArray) - except Exception,err: - boundsArray = None - if boundsArray is None and (getAutoBounds()==1 or (getAutoBounds()==2 and (self.isLatitude() or self.isLongitude()))): - boundsArray = self.genGenericBounds() - + boundsArray=self.validateBounds(boundsArray) + except Exception, err: + boundsArray=None + if boundsArray is None and (getAutoBounds() == 1 or (getAutoBounds() == 2 and (self.isLatitude() or self.isLongitude()))): + boundsArray=self.genGenericBounds() + return boundsArray # Return the bounds array, or None def getExplicitBounds(self): if self._boundsArray_ is None: - boundsArray = None - if hasattr(self,'bounds'): - boundsName = self.bounds + boundsArray=None + if hasattr(self, 'bounds'): + boundsName=self.bounds try: - boundsVar = self.parent[boundsName] - boundsArray = numpy.ma.filled(boundsVar) - self._boundsArray_ = boundsArray # for climatology performance - except KeyError,err: + boundsVar=self.parent[boundsName] + boundsArray=numpy.ma.filled(boundsVar) + self._boundsArray_=boundsArray # for climatology performance + except KeyError, err: print err - boundsArray = None + boundsArray=None else: - boundsArray = self._boundsArray_ + boundsArray=self._boundsArray_ return boundsArray @@ -1989,58 +2065,63 @@ def getExplicitBounds(self): # index in the extended dimension (default is index=0). # If the bounds variable is new, use the name boundsid, or 'bounds_' # if unspecified. - def setBounds(self, bounds, persistent=0, validate=0, index=None, boundsid=None): + def setBounds(self, bounds, persistent=0, + validate=0, index=None, boundsid=None): if persistent: if index is None: if validate: - bounds = self.validateBounds(bounds) - index = 0 + bounds=self.validateBounds(bounds) + index=0 # Create the bound axis, if necessary - file = self.parent + file=self.parent if file._boundAxis_ is None: # First look for 'bound' of length two - if file.axes.has_key("bound") and len(file.axes["bound"])==2: - file._boundAxis_ = file.axes["bound"] + if file.axes.has_key("bound") and len(file.axes["bound"]) == 2: + file._boundAxis_=file.axes["bound"] else: - file._boundAxis_ = file.createVirtualAxis("bound",2) + file._boundAxis_=file.createVirtualAxis("bound", 2) # Create the boundary variable if necessary - if hasattr(self,'bounds'): - boundName = self.bounds - boundVar = file.variables[boundName] + if hasattr(self, 'bounds'): + boundName=self.bounds + boundVar=file.variables[boundName] else: if boundsid is None: - boundName = "bounds_"+self.id + boundName="bounds_" + self.id else: - boundName = boundsid - boundVar = file.createVariable(boundName, cdmsNode.NumericToCdType.get(bounds.dtype.char), (self,file._boundAxis_)) + boundName=boundsid + boundVar=file.createVariable( + boundName, + cdmsNode.NumericToCdType.get(bounds.dtype.char), + (self, + file._boundAxis_)) # And link to self - self.bounds = boundName - self._boundsArray_ = None + self.bounds=boundName + self._boundsArray_=None - boundVar[index:index+len(bounds)] = bounds + boundVar[index:index + len(bounds)]=bounds else: - self._boundsArray_ = copy.copy(bounds) + self._boundsArray_=copy.copy(bounds) def getCalendar(self): - if hasattr(self,'calendar'): - calendar = self.calendar.lower() + if hasattr(self, 'calendar'): + calendar=self.calendar.lower() elif self.parent is not None and hasattr(self.parent, 'calendar'): - calendar = self.parent.calendar.lower() + calendar=self.parent.calendar.lower() else: - calendar = None + calendar=None - cdcal = tagToCalendar.get(calendar, cdtime.DefaultCalendar) + cdcal=tagToCalendar.get(calendar, cdtime.DefaultCalendar) return cdcal def isVirtual(self): "Return true iff coordinate values are implicitly defined." # No virtual axes in GrADS files - if self.parent is not None and hasattr(self.parent, 'format') and self.parent.format=='GRADS': + if self.parent is not None and hasattr(self.parent, 'format') and self.parent.format == 'GRADS': return 0 return (self._obj_ is None) @@ -2050,11 +2131,11 @@ def isUnlimited(self): return (self.parent._file_.dimensions[self.id] is None) else: return False -## PropertiedClasses.set_property (FileAxis, 'units', -## acts=FileAxis._setunits, -## nodelete=1 -## ) -## internattr.add_internal_attribute(FileAxis, 'name_in_file') +# PropertiedClasses.set_property (FileAxis, 'units', +# acts=FileAxis._setunits, +# nodelete=1 +# ) +# internattr.add_internal_attribute(FileAxis, 'name_in_file') class FileVirtualAxis(FileAxis): """An axis with no explicit representation of data values in the file. @@ -2066,8 +2147,8 @@ class FileVirtualAxis(FileAxis): def __init__(self, parent, axisname, axislen): FileAxis.__init__(self, parent, axisname) - self._virtualLength = axislen # length of the axis - + self._virtualLength=axislen # length of the axis + def __len__(self): return self._virtualLength @@ -2078,95 +2159,95 @@ def isVirtual(self): "Return true iff coordinate values are implicitly defined." return 1 -## PropertiedClasses.initialize_property_class (FileVirtualAxis) +# PropertiedClasses.initialize_property_class (FileVirtualAxis) -######## Functions for selecting axes -def axisMatchAxis (axes, specifications=None, omit=None, order=None): - """Given a list of axes and a specification or list of +# Functions for selecting axes +def axisMatchAxis(axes, specifications=None, omit=None, order=None): + """Given a list of axes and a specification or list of specificatons, and a specification or list of specifications - of those axes to omit, return a list of - those axes in the list that match the specification but - do not include in the list any axes that matches an omit + of those axes to omit, return a list of + those axes in the list that match the specification but + do not include in the list any axes that matches an omit specification. If specifications is None, include all axes less the omitted ones. - Individual specifications must be integer indices into axes or + Individual specifications must be integer indices into axes or matching criteria as detailed in axisMatches. Axes are returned in the order they occur in the axes argument unless - order is given. + order is given. - order can be a string containing the symbols t,x,y,z, or -. - If a - is given, any elements of the result not chosen otherwise are + order can be a string containing the symbols t,x,y,z, or -. + If a - is given, any elements of the result not chosen otherwise are filled in from left to right with remaining candidates. """ return [axes[i] for i in \ axisMatchIndex(axes, specifications, omit, order)] -def axisMatchIndex (axes, specifications=None, omit=None, order=None): - """Given a list of axes and a specification or list of +def axisMatchIndex(axes, specifications=None, omit=None, order=None): + """Given a list of axes and a specification or list of specificatons, and a specification or list of specifications - of those axes to omit, return a list of the indices of - those axes in the list that match the specification but - do not include in the list any axes that matches an omit + of those axes to omit, return a list of the indices of + those axes in the list that match the specification but + do not include in the list any axes that matches an omit specification. If specifications is None, include all axes less the omitted ones. - Individual specifications must be integer indices into axes or + Individual specifications must be integer indices into axes or matching criteria as detailed in axisMatches. The indices of axes are returned in the order the axes occur in the axes argument, unless order is given. - order can be a string containing the symbols t,x,y,z, or -. - If a - is given, any elements of the result not chosen otherwise are + order can be a string containing the symbols t,x,y,z, or -. + If a - is given, any elements of the result not chosen otherwise are filled in from left to right with remaining candidates. """ if specifications is None: - speclist = axes + speclist=axes elif isinstance(specifications, basestring): - speclist = [specifications] + speclist=[specifications] elif isinstance(specifications, list): - speclist = specifications + speclist=specifications elif isinstance(specifications, tuple): speclist=list(specifications) elif isinstance(specifications, int): - speclist = [specifications] + speclist=[specifications] elif callable(specifications): - speclist = [specifications] - else: # to allow arange, etc. - speclist = list(numpy.ma.filled(specifications)) + speclist=[specifications] + else: # to allow arange, etc. + speclist=list(numpy.ma.filled(specifications)) - candidates = [] + candidates=[] for i in range(len(axes)): for s in speclist: if isinstance(s, int): - r = (s == i) + r=(s == i) else: - r = axisMatches(axes[i], s) + r=axisMatches(axes[i], s) if r: candidates.append(i) break if not candidates: - return candidates #list empty + return candidates # list empty if omit is None: - omitlist = [] + omitlist=[] elif isinstance(omit, basestring): - omitlist = [omit] + omitlist=[omit] elif isinstance(omit, list): - omitlist = omit + omitlist=omit elif isinstance(omit, tuple): omitlist=list(omit) elif isinstance(omit, int): - omitlist = [omit] + omitlist=[omit] elif callable(omit): - omitlist = [omit] + omitlist=[omit] elif isinstance(omit, AbstractAxis): - omitlist = [omit] + omitlist=[omit] else: raise CDMSError, 'Unknown type of omit specifier.' @@ -2179,11 +2260,11 @@ def axisMatchIndex (axes, specifications=None, omit=None, order=None): elif isinstance(s, AbstractAxis): for i in range(len(candidates)): if s is axes[candidates[i]]: - del candidates[i] + del candidates[i] break else: for i in range(len(candidates)): - r = axisMatches(axes[candidates[i]], s) + r=axisMatches(axes[candidates[i]], s) if r: del candidates[i] break @@ -2191,31 +2272,31 @@ def axisMatchIndex (axes, specifications=None, omit=None, order=None): if order is None: return candidates - n = len(candidates) - m = len(order) - result = [None]*n + n=len(candidates) + m=len(order) + result=[None] * n # this loop is done this way for future escapes where elements of order # are not single chars. - j = 0 - io = 0 + j=0 + io=0 while j < n: if j >= m or order[io] == '-': - result[j] = candidates[0] + result[j]=candidates[0] del candidates[0] j += 1 io += 1 continue elif order[j] == 't': - oj = 'time' + oj='time' io += 1 elif order[j] == 'x': - oj = 'longitude' + oj='longitude' io += 1 elif order[j] == 'y': - oj = 'latitude' + oj='latitude' io += 1 elif order[j] == 'z': - oj = 'level' + oj='level' io += 1 else: # later could put in escaped ids or indices @@ -2223,21 +2304,21 @@ def axisMatchIndex (axes, specifications=None, omit=None, order=None): for i in range(n): if axisMatches(axes[candidates[i]], oj): - result[j] = candidates[i] + result[j]=candidates[i] del candidates[i] break else: raise CDMSError, "Axis requested in order specification not there" - j += 1 + j += 1 return result - + def axisMatches(axis, specification): """Return 1 or 0 depending on whether axis matches the specification. Specification must be one of: - 1. a string representing an axis id or one of - the keywords time, fctau0, latitude or lat, longitude or lon, or - lev or level. + 1. a string representing an axis id or one of + the keywords time, fctau0, latitude or lat, longitude or lon, or + lev or level. axis may be surrounded with parentheses or spaces. @@ -2249,20 +2330,20 @@ def axisMatches(axis, specification): if the value returned is true, the axis matches. 3. an axis object; will match if it is the same object as axis. - """ + """ if isinstance(specification, basestring): - s = specification.lower() - s = s.strip() + s=specification.lower() + s=s.strip() while s[0] == '(': if s[-1] != ')': raise CDMSError, 'Malformed axis spec, ' + specification - s = s[1:-1].strip() + s=s[1:-1].strip() if axis.id.lower() == s: return 1 elif (s == 'time') or (s in time_aliases): - return axis.isTime() + return axis.isTime() elif (s == 'fctau0') or (s in forecast_aliases): - return axis.isForecast() + return axis.isForecast() elif (s[0:3] == 'lat') or (s in latitude_aliases): return axis.isLatitude() elif (s[0:3] == 'lon') or (s in longitude_aliases): @@ -2273,10 +2354,10 @@ def axisMatches(axis, specification): return 0 elif callable(specification): - r = specification(axis) - if r: + r=specification(axis) + if r: return 1 - else: + else: return 0 elif isinstance(specification, AbstractAxis): @@ -2284,26 +2365,26 @@ def axisMatches(axis, specification): raise CDMSError, "Specification not acceptable: "\ + str(type(specification)) + ', ' + str(specification) - + def concatenate(axes, id=None, attributes=None): """Concatenate the axes, return a transient axis.""" - - data = numpy.ma.concatenate([ax[:] for ax in axes]) - boundsArray = [ax.getBounds() for ax in axes] + + data=numpy.ma.concatenate([ax[:] for ax in axes]) + boundsArray=[ax.getBounds() for ax in axes] if None in boundsArray: - bounds = None + bounds=None else: - bounds = numpy.ma.concatenate(boundsArray) + bounds=numpy.ma.concatenate(boundsArray) return TransientAxis(data, bounds=bounds, id=id, attributes=attributes) def take(ax, indices): """Take values indicated by indices list, return a transient axis.""" # Bug in ma compatibility module - data = numpy.ma.take(ax[:], indices) - abounds = ax.getBounds() + data=numpy.ma.take(ax[:], indices) + abounds=ax.getBounds() if abounds is not None: - bounds = numpy.ma.take(abounds, indices, axis=0) + bounds=numpy.ma.take(abounds, indices, axis=0) else: - bounds = None + bounds=None return TransientAxis(data, bounds=bounds, id=ax.id, attributes=ax.attributes) diff --git a/Packages/cdms2/Lib/bindex.py b/Packages/cdms2/Lib/bindex.py index 9156c9709f..913d6fdbdc 100644 --- a/Packages/cdms2/Lib/bindex.py +++ b/Packages/cdms2/Lib/bindex.py @@ -1,9 +1,11 @@ -## Automatically adapted for numpy.oldnumeric Aug 01, 2007 by -## Further modified to be pure new numpy June 24th 2008 +# Automatically adapted for numpy.oldnumeric Aug 01, 2007 by +# Further modified to be pure new numpy June 24th 2008 """Bin index for non-rectilinear grids""" -import _bindex, numpy +import _bindex +import numpy + def bindexHorizontalGrid(latlin, lonlin): """Create a bin index for a horizontal grid. @@ -12,14 +14,16 @@ def bindexHorizontalGrid(latlin, lonlin): Returns the index. """ - lonlin = numpy.mod(lonlin,360) - NI,NJ = _bindex.getLens() - head = numpy.zeros(NI*NJ,dtype='l') # This should match NBINI, NBINJ in bindex.c - next = numpy.zeros(len(latlin),dtype='l') + lonlin = numpy.mod(lonlin, 360) + NI, NJ = _bindex.getLens() + head = numpy.zeros(NI * NJ, dtype='l') + # This should match NBINI, NBINJ in bindex.c + next = numpy.zeros(len(latlin), dtype='l') _bindex.bindex(latlin, lonlin, head, next) - + return (head, next) + def intersectHorizontalGrid(latspecs, lonspecs, latlin, lonlin, index): """Intersect a horizontal grid with a lat-lon region. @@ -31,7 +35,7 @@ def intersectHorizontalGrid(latspecs, lonspecs, latlin, lonlin, index): Returns an array of indices, in latlin/lonlin, of the points in the intersection. """ - points = numpy.zeros(len(latlin),dtype='l') + points = numpy.zeros(len(latlin), dtype='l') if latspecs is None: slat = -90.0 elat = 90.0 @@ -41,14 +45,14 @@ def intersectHorizontalGrid(latspecs, lonspecs, latlin, lonlin, index): elat = latspecs[1] latopt = latspecs[2] - if slat>elat: + if slat > elat: tmp = slat slat = elat elat = tmp # If the longitude range is >=360.0, just intersect with the full range. # Otherwise, the points array could overflow and generate a seg fault. - if lonspecs is None or abs(lonspecs[1]-lonspecs[0])>=360.0: + if lonspecs is None or abs(lonspecs[1] - lonspecs[0]) >= 360.0: slon = 0.0 elon = 360.0 lonopt = 'co' @@ -57,11 +61,22 @@ def intersectHorizontalGrid(latspecs, lonspecs, latlin, lonlin, index): elon = lonspecs[1] lonopt = lonspecs[2] - if slon>elon: + if slon > elon: tmp = slon slon = elon elon = tmp - npoints = _bindex.intersect(slat, slon, elat, elon, latlin, lonlin, index[0], index[1], points, latopt, lonopt) + npoints = _bindex.intersect( + slat, + slon, + elat, + elon, + latlin, + lonlin, + index[0], + index[1], + points, + latopt, + lonopt) return points[:npoints] diff --git a/Packages/cdms2/Lib/cache.py b/Packages/cdms2/Lib/cache.py index c7456b5032..12163fe94b 100644 --- a/Packages/cdms2/Lib/cache.py +++ b/Packages/cdms2/Lib/cache.py @@ -1,8 +1,16 @@ """ CDMS cache management and file movement objects """ -import cdurllib, urlparse, tempfile, os, time, cdmsobj, sys, errno, shelve -from error import CDMSError +import cdurllib +import urlparse +import tempfile +import os +import time +import cdmsobj +import sys +import errno +import shelve +from .error import CDMSError MethodNotImplemented = "Method not yet implemented" SchemeNotSupported = "Scheme not supported: " LockError = "Lock error:" @@ -14,6 +22,7 @@ _lock_naptime = 1 # Seconds between lock tries _cache_tempdir = None # Default temporary directory + def lock(filename): """ Acquire a file-based lock with the given name. @@ -32,48 +41,50 @@ def lock(filename): while (not success) and (tries < _lock_max_tries): try: if cdmsobj._debug: - print 'Process %d: Trying to acquire lock %s'%(os.getpid(),path) - fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_EXCL, 0666) + print 'Process %d: Trying to acquire lock %s' % (os.getpid(), path) + fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_EXCL, 0o666) # If the open failed because the file already exists, keep trying, otherwise # reraise the error except OSError: - if sys.exc_value.errno!=errno.EEXIST: + if sys.exc_value.errno != errno.EEXIST: raise tries = tries + 1 else: if cdmsobj._debug: - print 'Process %d: Acquired lock %s after %d tries'%(os.getpid(),path,tries) + print 'Process %d: Acquired lock %s after %d tries' % (os.getpid(), path, tries) success = 1 break # Sleep until next retry if cdmsobj._debug: - print 'Process %d: Failed to acquire lock %s, sleeping'%(os.getpid(),path) + print 'Process %d: Failed to acquire lock %s, sleeping' % (os.getpid(), path) time.sleep(_lock_naptime) # Error if the lock could not be acquired if not success: - raise CDMSError, LockError + 'Could not acquire a lock on %s'%path + raise CDMSError(LockError + 'Could not acquire a lock on %s' % path) # The lock succeeded, so just close the file - we don't need to write # anything here else: os.close(fd) + def unlock(filename): """ Delete a file-based lock with the given name. - Usage:unlock(filename) + Usage:unlock(filename) If the function returns, the lock was successfully deleted. Note: This function is UNIX-specific. """ path = lockpath(filename) if cdmsobj._debug: - print 'Process %d: Unlocking %s'%(os.getpid(),path) + print 'Process %d: Unlocking %s' % (os.getpid(), path) os.unlink(path) + def lockpath(filename): """ Generate the pathname of a lock. Creates the directory containing the lock @@ -84,12 +95,12 @@ def lockpath(filename): if not _cache_tempdir: tempfile.mktemp() - _cache_tempdir = os.path.join(tempfile.tempdir,'cdms') + _cache_tempdir = os.path.join(tempfile.tempdir, 'cdms') if not os.path.isdir(_cache_tempdir): if cdmsobj._debug: - print 'Process %d: Creating cache directory %s'%(os.getpid(),_cache_tempdir) - os.mkdir(_cache_tempdir,0777) - return os.path.join(_cache_tempdir,filename) + print 'Process %d: Creating cache directory %s' % (os.getpid(), _cache_tempdir) + os.mkdir(_cache_tempdir, 0o777) + return os.path.join(_cache_tempdir, filename) _useWindow = 0 # If true, use a progress dialog _pythonTransfer = 0 @@ -97,6 +108,7 @@ def lockpath(filename): _requestManagerTransfer = 2 _transferMethod = _pythonTransfer # Method of transferring files + def useWindow(): """ Specify that dialog windows should be used if possible. Do not call this directly, use @@ -105,6 +117,7 @@ def useWindow(): global _useWindow _useWindow = 1 + def useTTY(): """ Informational messages such as FTP status should be sent to the terminal. See useWindow. @@ -112,6 +125,7 @@ def useTTY(): global _useWindow _useWindow = 0 + def useGlobusTransfer(): """ Specify that file transfers should use the Globus storage API (SC-API). See usePythonTransfer. @@ -119,6 +133,7 @@ def useGlobusTransfer(): global _transferMethod _transferMethod = _globusTransfer + def usePythonTransfer(): """ Specify that file transfers should use the Python libraries urllib, ftplib. See useGlobusTransfer. @@ -126,15 +141,18 @@ def usePythonTransfer(): global _transferMethod _transferMethod = _pythonTransfer + def useRequestManagerTransfer(): try: import reqm except ImportError: - raise CDMSError, RequestManagerNotSupported + raise CDMSError(RequestManagerNotSupported) global _transferMethod _transferMethod = _requestManagerTransfer -def copyFile(fromURL, toURL, callback=None, lcpath=None, userid=None, useReplica=1): + +def copyFile(fromURL, toURL, callback=None, + lcpath=None, userid=None, useReplica=1): """ Copy file to local file . For FTP transfers, if cache._useWindow is true, display a progress dialog, otherwise just print progress messages. @@ -145,15 +163,16 @@ def copyFile(fromURL, toURL, callback=None, lcpath=None, userid=None, useReplica """ if callback is None: if _useWindow: - import gui + from . import gui dialogParent = gui.getProgressParent() dialog = gui.CdProgressDialog(dialogParent, fromURL) callback = gui.updateProgressGui else: callback = cdurllib.sampleReportHook - (scheme,netloc,path,parameters,query,fragment)=urlparse.urlparse(fromURL) - if scheme=='ftp': - if _transferMethod==_pythonTransfer: + (scheme, netloc, path, parameters, query, + fragment) = urlparse.urlparse(fromURL) + if scheme == 'ftp': + if _transferMethod == _pythonTransfer: urlopener = cdurllib.CDURLopener() # In window environment, attach the dialog to the opener. This will @@ -167,51 +186,57 @@ def copyFile(fromURL, toURL, callback=None, lcpath=None, userid=None, useReplica except: if _useWindow: dialog.Destroy() - raise - elif _transferMethod==_globusTransfer: # Transfer via Globus SC-API + raise + elif _transferMethod == _globusTransfer: # Transfer via Globus SC-API try: import globus.storage except ImportError: - raise CDMSError, GlobusNotSupported + raise CDMSError(GlobusNotSupported) - globus.storage.transfer(fromURL, "file:"+toURL) + globus.storage.transfer(fromURL, "file:" + toURL) else: - raise CDMSError, SchemeNotSupported + scheme + raise CDMSError(SchemeNotSupported + scheme) return - elif _transferMethod==_requestManagerTransfer: # Request manager gransfer - import reqm, signal + elif _transferMethod == _requestManagerTransfer: # Request manager gransfer + import reqm + import signal # Define an alarm handler, to poll the request manager def handler(signum, frame): pass - # Obtain server reference from environment variable ESG_REQM_REF if present + # Obtain server reference from environment variable ESG_REQM_REF if + # present serverRef = os.environ.get('ESG_REQM_REF', '/tmp/esg_rqm.ref') server = reqm.RequestManager(iorFile=serverRef) - result, token = server.requestFile(userid, lcpath, path, toURL, useReplica) + result, token = server.requestFile( + userid, lcpath, path, toURL, useReplica) server.execute(token) # Poll the request manager for completion, signalled by estim<=0.0 - while 1: + while True: signal.signal(signal.SIGALRM, handler) estim = server.estimate(token) - print 'Estimate: ',estim - if estim<=0.0: break + print 'Estimate: ', estim + if estim <= 0.0: + break signal.alarm(3) # Number of seconds between polls signal.pause() #!!!! Remove this when gsincftp uses the right target name !!! - -## oldpath = os.path.join(os.path.dirname(toURL),path) -## os.rename(oldpath,toURL) + +# oldpath = os.path.join(os.path.dirname(toURL),path) +# os.rename(oldpath,toURL) #!!!! - + return else: - raise CDMSError, SchemeNotSupported + scheme + raise CDMSError(SchemeNotSupported + scheme) # A simple data cache + + class Cache: indexpath = None # Path of data cache index @@ -227,9 +252,10 @@ def __init__(self): except: pass lock("index_lock") - self.index = shelve.open(self.indexpath) # Persistent cache index + self.index = shelve.open(self.indexpath) # Persistent cache index try: - os.chmod(self.indexpath,0666) # Make index file world writeable + os.chmod(self.indexpath, 0o666) + # Make index file world writeable except: pass self.index.close() @@ -237,7 +263,7 @@ def __init__(self): # Clean up pending read notifications in the cache. This will also # mess up tranfers in progress... self.clean() - self.direc = os.path.dirname(self.indexpath) # Cache directory + self.direc = os.path.dirname(self.indexpath) # Cache directory def get(self, filekey): """ @@ -269,7 +295,7 @@ def put(self, filekey, path): lock("index_lock") try: if cdmsobj._debug: - print 'Process %d: Adding cache file %s,\n key %s'%(os.getpid(),path,filekey) + print 'Process %d: Adding cache file %s,\n key %s' % (os.getpid(), path, filekey) self.index = shelve.open(self.indexpath) self.index[filekey] = path except: @@ -294,7 +320,8 @@ def deleteEntry(self, filekey): pass unlock("index_lock") - def copyFile(self, fromURL, filekey, lcpath=None, userid=None, useReplica=None): + def copyFile(self, fromURL, filekey, + lcpath=None, userid=None, useReplica=None): """ Copy the file into the cache. Return the result path. @@ -302,9 +329,9 @@ def copyFile(self, fromURL, filekey, lcpath=None, userid=None, useReplica=None): is the string user ID, is true iff the request manager should search the replica catalog for the actual file to transfer. """ - + # Put a notification into the cache, that this file is being read. - self.put(filekey,"__READ_PENDING__") + self.put(filekey, "__READ_PENDING__") # Get a temporary file in the cache tempdir = tempfile.tempdir @@ -314,21 +341,29 @@ def copyFile(self, fromURL, filekey, lcpath=None, userid=None, useReplica=None): # Copy to the temporary file try: - copyFile(fromURL, toPath, lcpath=lcpath, userid=userid, useReplica=useReplica) - os.chmod(toPath,0666) # Make cache files world writeable + copyFile( + fromURL, + toPath, + lcpath=lcpath, + userid=userid, + useReplica=useReplica) + os.chmod(toPath, 0o666) + # Make cache files world writeable except: - # Remove the notification on error, and the temp file, then re-raise + # Remove the notification on error, and the temp file, then + # re-raise self.deleteEntry(filekey) if os.path.isfile(toPath): os.unlink(toPath) raise # Add to the cache index - self.put(filekey,toPath) + self.put(filekey, toPath) return toPath - def getFile(self, fromURL, filekey, naptime=5, maxtries=60, lcpath=None, userid=None, useReplica=None): + def getFile(self, fromURL, filekey, naptime=5, + maxtries=60, lcpath=None, userid=None, useReplica=None): """ Get the file with . If the file is in the cache, read it. If another process is transferring it into the cache, wait for the @@ -351,23 +386,34 @@ def getFile(self, fromURL, filekey, naptime=5, maxtries=60, lcpath=None, userid= """ # If the file is being read into the cache, just wait for it tempname = self.get(filekey) - # Note: This is not bulletproof: another process could set the cache at this point + # Note: This is not bulletproof: another process could set the cache at + # this point if tempname is None: - fpath = self.copyFile(fromURL,filekey,lcpath=lcpath,userid=userid,useReplica=useReplica) - elif tempname=="__READ_PENDING__": + fpath = self.copyFile( + fromURL, + filekey, + lcpath=lcpath, + userid=userid, + useReplica=useReplica) + elif tempname == "__READ_PENDING__": success = 0 for i in range(maxtries): if cdmsobj._debug: - print 'Process %d: Waiting for read completion, %s'%(os.getpid(),`filekey`) + print 'Process %d: Waiting for read completion, %s' % (os.getpid(), repr(filekey)) time.sleep(naptime) tempname = self.get(filekey) # The read failed, or the entry was deleted. if tempname is None: - fpath = self.copyFile(fromURL,filekey,lcpath=lcpath,userid=userid,useReplica=useReplica) + fpath = self.copyFile( + fromURL, + filekey, + lcpath=lcpath, + userid=userid, + useReplica=useReplica) # The read is not yet complete - elif tempname=="__READ_PENDING__": + elif tempname == "__READ_PENDING__": continue # The read is finished. @@ -376,13 +422,13 @@ def getFile(self, fromURL, filekey, naptime=5, maxtries=60, lcpath=None, userid= success = 1 break if not success: - raise CDMSError, TimeOutError +`filekey` + raise CDMSError(TimeOutError + repr(filekey)) else: fpath = tempname if cdmsobj._debug: - print 'Process %d: Got file %s from cache %s'%(os.getpid(),fromURL,fpath) + print 'Process %d: Got file %s from cache %s' % (os.getpid(), fromURL, fpath) return fpath def delete(self): @@ -394,10 +440,11 @@ def delete(self): self.index = shelve.open(self.indexpath) for key in self.index.keys(): path = self.index[key] - if path=="__READ_PENDING__": continue # Don't remove read-pending notifications + if path == "__READ_PENDING__": + continue # Don't remove read-pending notifications try: if cdmsobj._debug: - print 'Process %d: Deleting cache file %s'%(os.getpid(),path) + print 'Process %d: Deleting cache file %s' % (os.getpid(), path) os.unlink(path) except: pass @@ -415,7 +462,7 @@ def clean(self): self.index = shelve.open(self.indexpath) for key in self.index.keys(): path = self.index[key] - if path=="__READ_PENDING__": + if path == "__READ_PENDING__": del self.index[key] self.index.close() unlock("index_lock") diff --git a/Packages/cdms2/Lib/cdmsNode.py b/Packages/cdms2/Lib/cdmsNode.py index 5d82a95265..3e34f34f73 100644 --- a/Packages/cdms2/Lib/cdmsNode.py +++ b/Packages/cdms2/Lib/cdmsNode.py @@ -1,5 +1,5 @@ -## Automatically adapted for numpy.oldnumeric Aug 01, 2007 by -## Further modified to be pure new numpy June 24th 2008 +# Automatically adapted for numpy.oldnumeric Aug 01, 2007 by +# Further modified to be pure new numpy June 24th 2008 """ CDMS node classes @@ -13,10 +13,12 @@ from error import CDMSError # Regular expressions -_Name = re.compile('[a-zA-Z0-9_:][-a-zA-Z0-9._:]*$') # Note: allows digit as first character +_Name = re.compile('[a-zA-Z0-9_:][-a-zA-Z0-9._:]*$') + # Note: allows digit as first character _Integer = re.compile('[0-9]+$') _ArraySep = re.compile('[\[\],\s]+') -_Illegal = re.compile('([<>&\"\'])|([^\t\r\n -\176\240-\377])') #" illegal chars in content +_Illegal = re.compile('([<>&\"\'])|([^\t\r\n -\176\240-\377])') + #" illegal chars in content # Data types @@ -31,42 +33,51 @@ CdString = CDML.CdString CdFromObject = CDML.CdFromObject CdAny = CDML.CdAny -CdDatatypes = [CdChar,CdByte,CdShort,CdInt,CdLong,CdInt64,CdFloat,CdDouble,CdString] +CdDatatypes = [ + CdChar, + CdByte, + CdShort, + CdInt, + CdLong, + CdInt64, + CdFloat, + CdDouble, + CdString] CdScalar = CDML.CdScalar CdArray = CDML.CdArray -NumericToCdType = {numpy.sctype2char(numpy.float32):CdFloat, - numpy.sctype2char(numpy.float):CdDouble, - numpy.sctype2char(numpy.int16):CdShort, - numpy.sctype2char(numpy.int32):CdInt, - numpy.sctype2char(numpy.int):CdLong, - numpy.sctype2char(numpy.int64):CdInt64, - numpy.sctype2char(numpy.intc):CdLong, - numpy.sctype2char(numpy.int8):CdByte, - 'c':CdChar, - 'B':'B', - 'H':'H', - 'L':'L', - 'q':CdInt64, - 'Q':'Q', - 'S':'S' +NumericToCdType = {numpy.sctype2char(numpy.float32): CdFloat, + numpy.sctype2char(numpy.float): CdDouble, + numpy.sctype2char(numpy.int16): CdShort, + numpy.sctype2char(numpy.int32): CdInt, + numpy.sctype2char(numpy.int): CdLong, + numpy.sctype2char(numpy.int64): CdInt64, + numpy.sctype2char(numpy.intc): CdLong, + numpy.sctype2char(numpy.int8): CdByte, + 'c': CdChar, + 'B': 'B', + 'H': 'H', + 'L': 'L', + 'q': CdInt64, + 'Q': 'Q', + 'S': 'S' } -CdToNumericType = {CdChar:'c', - CdByte:numpy.int8, - CdShort:numpy.int16, - CdInt:numpy.int32, - CdLong:numpy.int, - CdInt64:numpy.int64, - CdFloat:numpy.float32, - CdDouble:numpy.float} +CdToNumericType = {CdChar: 'c', + CdByte: numpy.int8, + CdShort: numpy.int16, + CdInt: numpy.int32, + CdLong: numpy.int, + CdInt64: numpy.int64, + CdFloat: numpy.float32, + CdDouble: numpy.float} # Grid types UnknownGridType = "unknown" GaussianGridType = "gaussian" UniformGridType = "uniform" -CdGridtypes = [UnknownGridType,GaussianGridType,UniformGridType] +CdGridtypes = [UnknownGridType, GaussianGridType, UniformGridType] DuplicateIdError = "Duplicate identifier: " InvalidArgumentError = "Invalid argument: " @@ -74,6 +85,8 @@ InvalidGridtype = "Invalid grid type: " InvalidIdError = "Invalid identifier: " NotMonotonic = "Result array is not monotonic " + + class NotMonotonicError(CDMSError): pass @@ -94,6 +107,8 @@ class NotMonotonicError(CDMSError): # '"' --> " # "'" --> ' # all other illegal characters are removed #" + + def mapIllegalToEntity(matchobj): s = matchobj.group(0) if s == '<': @@ -102,49 +117,55 @@ def mapIllegalToEntity(matchobj): return '>' elif s == '&': return '&' - elif s == '"': #" + elif s == '"': # " return '"' - elif s=="'": + elif s == "'": return ''' else: return "" # Named node + + class CdmsNode: def __init__(self, tag, id=None, parent=None): if id and _Name.match(id) is None: raise CDMSError, InvalidIdError + id - self.attribute = {} # External attributes, attribute[name]=(value,cdDatatype) + self.attribute = {} + # External attributes, attribute[name]=(value,cdDatatype) self.child = [] # Children self.id = id # Identifier string self.parent = parent # Parent node in a tree, None for root self.tag = tag # XML tag string self.content = None # XML content string - self.dtd = CDML.CDML().dtd.get(self.tag) # CDML Document Type Definition for this tag - self.extra = CDML.CDML().extra.get(self.tag) # Extra datatype constraints + self.dtd = CDML.CDML().dtd.get(self.tag) + # CDML Document Type Definition for this tag + self.extra = CDML.CDML().extra.get( + self.tag) # Extra datatype constraints CdmsNode.mapToExternal(self) # Don't call subclass mapToExternal! # Map to external attributes def mapToExternal(self): if self.id is not None and _Name.match(self.id) is None: raise CDMSError, InvalidIdError + self.id - if self.id is not None: self.setExternalAttr('id',self.id) + if self.id is not None: + self.setExternalAttr('id', self.id) # Set content from a string. The interpretation # of content is class-dependent - def setContentFromString(self,content): - self.content=content + def setContentFromString(self, content): + self.content = content # Get content def getContent(self): return self.content # Add a child node - def add(self,child): + def add(self, child): if child is not None: self.child.append(child) - child.parent=self + child.parent = self return child # Return a list of child nodes @@ -152,13 +173,13 @@ def children(self): return self.child # Get the child node at index k - def getChildAt(self,index): + def getChildAt(self, index): return self.child[index] # Remove and return the child at index k - def removeChildAt(self,index): + def removeChildAt(self, index): child = self.child[index] - self.child = self.child[:index]+self.child[index+1:] + self.child = self.child[:index] + self.child[index + 1:] return child # Get the number of children @@ -166,7 +187,7 @@ def getChildCount(self): return len(self.child) # Get the index of a node - def getIndex(self,node): + def getIndex(self, node): index = -1 for i in range(len(self.child)): if node is self.child[i]: @@ -180,20 +201,21 @@ def getParent(self): # True iff node is a leaf node def isLeaf(self): - return self.child==[] + return self.child == [] # Set an external attribute # 'attr' is an Attr object def setExternalAttrFromAttr(self, attr): - if attr.value is None: return - self.attribute[attr.name]=(attr.value,attr.getDatatype()) + if attr.value is None: + return + self.attribute[attr.name] = (attr.value, attr.getDatatype()) # Get an external attribute, as an Attr instance def getExternalAttrAsAttr(self, name): attrPair = self.attribute.get(name) if attrPair: - (value,datatype) = attrPair - attr = AttrNode(name,value) + (value, datatype) = attrPair + attr = AttrNode(name, value) attr.datatype = datatype return attr else: @@ -201,7 +223,7 @@ def getExternalAttrAsAttr(self, name): # Set an external attribute def setExternalAttr(self, name, value, datatype=None): - attr = AttrNode(name,value) + attr = AttrNode(name, value) attr.datatype = datatype self.setExternalAttrFromAttr(attr) @@ -209,7 +231,7 @@ def setExternalAttr(self, name, value, datatype=None): def getExternalAttr(self, name): attrPair = self.attribute.get(name) if attrPair: - (value,datatype) = attrPair + (value, datatype) = attrPair return value else: return None @@ -220,71 +242,88 @@ def getExternalDict(self): # Set the external attribute dictionary. The input dictionary # is of the form {name:value,...} where value is a string. - def setExternalDict(self,dict): + def setExternalDict(self, dict): for key in dict.keys(): - self.attribute[key]=(dict[key],CdString) + self.attribute[key] = (dict[key], CdString) # Write to a file, with formatting. # tablevel is the start number of tabs - def write(self,fd=None,tablevel=0,format=1): - if fd is None: fd = sys.stdout + def write(self, fd=None, tablevel=0, format=1): + if fd is None: + fd = sys.stdout printLimit = get_printoptions()['threshold'] - set_printoptions(threshold=inf) # Ensure that all Numeric array values will be printed + set_printoptions(threshold=inf) + # Ensure that all Numeric array values will be printed if self.dtd: validAttrs = self.dtd.keys() else: validAttrs = None - if format: fd.write(tablevel*'\t') - fd.write('<'+self.tag) - if format: fd.write('\n') + if format: + fd.write(tablevel * '\t') + fd.write('<' + self.tag) + if format: + fd.write('\n') # Write valid attributes for attname in self.attribute.keys(): if (validAttrs and (attname in validAttrs)) or (not validAttrs): - if format: fd.write((tablevel+1)*'\t') - (attval,datatype)=self.attribute[attname] + if format: + fd.write((tablevel + 1) * '\t') + (attval, datatype) = self.attribute[attname] # attvalstr = string.replace(str(attval),'"',"'") # Map " to ' - attvalstr = _Illegal.sub(mapIllegalToEntity,str(attval)) # Map illegal chars to entities + attvalstr = _Illegal.sub( + mapIllegalToEntity, + str(attval)) # Map illegal chars to entities if format: - fd.write(attname+'\t="'+attvalstr+'"') + fd.write(attname + '\t="' + attvalstr + '"') else: - fd.write(' '+attname+'="'+attvalstr+'"') - if format: fd.write('\n') - if format: fd.write((tablevel+1)*'\t') + fd.write(' ' + attname + '="' + attvalstr + '"') + if format: + fd.write('\n') + if format: + fd.write((tablevel + 1) * '\t') fd.write('>') - if format: fd.write('\n') + if format: + fd.write('\n') # Write extra attributes for attname in self.attribute.keys(): if validAttrs and (attname not in validAttrs): - (attval,datatype)=self.attribute[attname] - attr = AttrNode(attname,attval) - attr.datatype=datatype + (attval, datatype) = self.attribute[attname] + attr = AttrNode(attname, attval) + attr.datatype = datatype attr.mapToExternal() - attr.write(fd,tablevel+1,format) + attr.write(fd, tablevel + 1, format) # Write content content = self.getContent() if content is not None: - content = _Illegal.sub(mapIllegalToEntity,content) # Map illegal chars to entities - if format: fd.write((tablevel+1)*'\t') + content = _Illegal.sub( + mapIllegalToEntity, + content) # Map illegal chars to entities + if format: + fd.write((tablevel + 1) * '\t') fd.write(content) - if format: fd.write('\n') + if format: + fd.write('\n') # Write children for node in self.child: - node.write(fd,tablevel+1,format) + node.write(fd, tablevel + 1, format) - if format: fd.write((tablevel+1)*'\t') - fd.write('') - if format: fd.write('\n') + if format: + fd.write((tablevel + 1) * '\t') + fd.write('') + if format: + fd.write('\n') set_printoptions(threshold=printLimit) # Restore original - # Write to a file without formatting. - def write_raw(self,fd=None): - if fd is None: fd = sys.stdout - self.write(fd,0,0) + # Write to a file without formatting. + def write_raw(self, fd=None): + if fd is None: + fd = sys.stdout + self.write(fd, 0, 0) # Write an LDIF (LDAP interchange format) entry # parentdn is the parent LDAP distinguished name @@ -292,41 +331,50 @@ def write_raw(self,fd=None): # A trailing newline is added iff format==1 # Note: unlike write, this does not write children as well def write_ldif(self, parentdn, userAttrs=[], fd=None, format=1): - if fd is None: fd = sys.stdout + if fd is None: + fd = sys.stdout if self.dtd: validAttrs = self.dtd.keys() else: validAttrs = None # Write distinguished name - newdn = "%s=%s,%s"%(self.tag,self.id,parentdn) - fd.write("dn: %s\n"%newdn) + newdn = "%s=%s,%s" % (self.tag, self.id, parentdn) + fd.write("dn: %s\n" % newdn) # Write valid attributes for attname in self.attribute.keys(): if (validAttrs and (attname in validAttrs)) or (not validAttrs): - (attval,datatype)=self.attribute[attname] - # attvalstr = _Illegal.sub(mapIllegalToEntity,str(attval)) # Map illegal chars to entities + (attval, datatype) = self.attribute[attname] + # attvalstr = _Illegal.sub(mapIllegalToEntity,str(attval)) # + # Map illegal chars to entities if not isinstance(attval, basestring): attval = `attval` attvalstr = attval.strip() - attvalstr = re.sub('\n','\n ',attvalstr) # Make sure continuation lines are preceded with a space - if attvalstr=='': attvalstr = "none" - fd.write("%s: %s\n"%(attname,attvalstr)) - + attvalstr = re.sub( + '\n', + '\n ', + attvalstr) # Make sure continuation lines are preceded with a space + if attvalstr == '': + attvalstr = "none" + fd.write("%s: %s\n" % (attname, attvalstr)) + # Write extra attributes for attname in self.attribute.keys(): if validAttrs and (attname not in validAttrs): - (attval,datatype)=self.attribute[attname] + (attval, datatype) = self.attribute[attname] if not isinstance(attval, basestring): attval = `attval` - attval = re.sub('\n','\n ',attval) # Make sure continuation lines are preceded with a space - fd.write("attr: %s=%s\n"%(attname,attval)) + attval = re.sub( + '\n', + '\n ', + attval) # Make sure continuation lines are preceded with a space + fd.write("attr: %s=%s\n" % (attname, attval)) # Write content # content = self.getContent() # if content is not None: - # content = _Illegal.sub(mapIllegalToEntity,content) # Map illegal chars to entities + # content = _Illegal.sub(mapIllegalToEntity,content) # Map illegal chars to entities # fd.write("value: %s"%(content,)) # Write user attributes @@ -335,61 +383,64 @@ def write_ldif(self, parentdn, userAttrs=[], fd=None, format=1): else: newAttrs = userAttrs for entry in list(newAttrs): - fd.write("%s\n"%entry) + fd.write("%s\n" % entry) # Write classes fd.write("objectclass: top\n") - fd.write("objectclass: %s\n"%(self.tag)) + fd.write("objectclass: %s\n" % (self.tag)) - if format==1: + if format == 1: fd.write('\n') return newdn # Validate attributes - def validate(self,idtable=None): + def validate(self, idtable=None): # Check validity of enumerated values and references validKeys = self.dtd.keys() for attname in self.attribute.keys(): if attname in validKeys: - (atttype,default)=self.dtd[attname] - if isinstance(atttype,tuple): - attval=self.getExternalAttr(attname) - assert attval in atttype, 'Invalid attribute %s=%s must be in %s'%(attname,attval,`atttype`) - elif atttype==CDML.Idref: - attval=self.getExternalAttr(attname) + (atttype, default) = self.dtd[attname] + if isinstance(atttype, tuple): + attval = self.getExternalAttr(attname) + assert attval in atttype, 'Invalid attribute %s=%s must be in %s' % ( + attname, attval, `atttype`) + elif atttype == CDML.Idref: + attval = self.getExternalAttr(attname) if idtable: if not idtable.has_key(attval): - print 'Warning: ID reference not found: %s=%s'%(attname,attval) - + print 'Warning: ID reference not found: %s=%s' % (attname, attval) + # Validate children for node in self.children(): node.validate(idtable) - + # Container object for other CDMS objects + + class DatasetNode(CdmsNode): def __init__(self, id): - CdmsNode.__init__(self,"dataset",id ) + CdmsNode.__init__(self, "dataset", id) self.idtable = {} # Validate the dataset and all child nodes - def validate(self,idtable=None): + def validate(self, idtable=None): if not idtable: - idtable=self.idtable - CdmsNode.validate(self,idtable) + idtable = self.idtable + CdmsNode.validate(self, idtable) # Add a child node with an ID - def addId(self,id,child): - if self.idtable.has_key(id): - raise CDMSError, DuplicateIdError +id - CdmsNode.add(self,child) - self.idtable[id]=child + def addId(self, id, child): + if self.idtable.has_key(id): + raise CDMSError, DuplicateIdError + id + CdmsNode.add(self, child) + self.idtable[id] = child return child # Get a child node from its ID - def getChildNamed(self,id): + def getChildNamed(self, id): return self.idtable.get(id) # Get the ID table @@ -399,66 +450,80 @@ def getIdDict(self): # Dump to a CDML file. # path is the file to dump to, or None for standard output. # if format is true, write with tab, newline formatting - def dump(self,path=None,format=1): + def dump(self, path=None, format=1): if path: try: - fd = open(path,'w') + fd = open(path, 'w') except IOError: - raise IOError,'%s: %s'%(sys.exc_value,path) + raise IOError, '%s: %s' % (sys.exc_value, path) else: fd = sys.stdout fd.write('') - if format: fd.write('\n') - fd.write('') - if format: fd.write('\n') - self.write(fd,0,format) - if fd!=sys.stdout: fd.close() + if format: + fd.write('\n') + fd.write( + '') + if format: + fd.write('\n') + self.write(fd, 0, format) + if fd != sys.stdout: + fd.close() # Spatio-temporal variable # Two ways to create a variable: # (1) var = VariableNode(id,datatype,domain) # (2) var = VariableNode(id,datatype) # var.setDomain(domain) + + class VariableNode(CdmsNode): # Create a variable. # If validate is true, validate immediately + def __init__(self, id, datatype, domain): - assert isinstance(datatype,basestring), 'Invalid datatype: '+`datatype` - assert datatype in CdDatatypes, 'Invalid datatype: '+`datatype` - CdmsNode.__init__(self,"variable",id) + assert isinstance( + datatype, basestring), 'Invalid datatype: ' + `datatype` + assert datatype in CdDatatypes, 'Invalid datatype: ' + `datatype` + CdmsNode.__init__(self, "variable", id) self.datatype = datatype self.setDomain(domain) VariableNode.mapToExternal(self) # Set the domain - def setDomain(self,domain): + def setDomain(self, domain): if not self.isLeaf(): self.removeChildAt(0) self.add(domain) # Get the domain def getDomain(self): - if self.getChildCount()>0: + if self.getChildCount() > 0: return self.getChildAt(0) else: return None # Map to external attributes def mapToExternal(self): - self.setExternalAttr('datatype',self.datatype) - + self.setExternalAttr('datatype', self.datatype) + # Coordinate axis + + class AxisNode(CdmsNode): # If datatype is None, assume values [0,1,..,length-1] # data is a numpy array, if specified - def __init__(self, id, length, datatype=CdLong,data=None): - assert isinstance(length, int), 'Invalid length: '+`length` - assert isinstance(datatype, basestring), 'Invalid datatype: '+`datatype` - assert datatype in CdDatatypes, 'Invalid datatype: '+`datatype` - if data is not None: assert isinstance(data, numpy.ndarray), 'data must be a 1-D Numeric array' - CdmsNode.__init__(self,"axis",id) + + def __init__(self, id, length, datatype=CdLong, data=None): + assert isinstance(length, int), 'Invalid length: ' + `length` + assert isinstance( + datatype, basestring), 'Invalid datatype: ' + `datatype` + assert datatype in CdDatatypes, 'Invalid datatype: ' + `datatype` + if data is not None: + assert isinstance( + data, numpy.ndarray), 'data must be a 1-D Numeric array' + CdmsNode.__init__(self, "axis", id) self.datatype = datatype self.data = data # data representation is CdLinear or CdVector @@ -477,48 +542,51 @@ def __init__(self, id, length, datatype=CdLong,data=None): # Map to external attributes def mapToExternal(self): - self.setExternalAttr('datatype',self.datatype) - self.setExternalAttr('length',self.length) + self.setExternalAttr('datatype', self.datatype) + self.setExternalAttr('length', self.length) # Set data from content string # The content of an axis is the data array. - def setContentFromString(self,datastring): + def setContentFromString(self, datastring): datatype = self.datatype numericType = CdToNumericType.get(datatype) - if numericType is None: raise CDMSError, InvalidDatatype + datatype + if numericType is None: + raise CDMSError, InvalidDatatype + datatype stringlist = _ArraySep.split(datastring) numlist = [] for numstring in stringlist: - if numstring=='': continue + if numstring == '': + continue numlist.append(float(numstring)) - if len(numlist)>0: + if len(numlist) > 0: # NB! len(zero-length array) causes IndexError on Linux! - dataArray = numpy.array(numlist,numericType) + dataArray = numpy.array(numlist, numericType) self.data = dataArray self.length = len(self.data) # Set the partition from a string. This does not # set the external string representation - def setPartitionFromString(self,partstring): + def setPartitionFromString(self, partstring): stringlist = _ArraySep.split(partstring) numlist = [] for numstring in stringlist: - if numstring=='': continue + if numstring == '': + continue numlist.append(int(numstring)) - dataArray = numpy.array(numlist,numpy.int) - if len(dataArray)>0: + dataArray = numpy.array(numlist, numpy.int) + if len(dataArray) > 0: self.partition = dataArray # Get the content string: the data values if the representation # is as a vector, or ane empty string otherwise def getContent(self): - if self.data is None or self.dataRepresent==CdLinear: + if self.data is None or self.dataRepresent == CdLinear: return '' else: return str(self.data) # Set the data as an array, check for monotonicity - def setData(self,data): + def setData(self, data): # If this axis is currently linear, remove the linear node if self.dataRepresent == CdLinear: @@ -527,8 +595,8 @@ def setData(self,data): self.data = data self.dataRepresent = CdVector self.length = len(data) - self.setExternalAttr('length',self.length) - if self.monotonicity()==CdNotMonotonic: + self.setExternalAttr('length', self.length) + if self.monotonicity() == CdNotMonotonic: raise NotMonotonicError, NotMonotonic # Get the data as an array @@ -540,9 +608,9 @@ def getData(self): # Set the data as a linear vector # If the partition is set, derive the vector length from it - def setLinearData(self,linearNode, partition=None): + def setLinearData(self, linearNode, partition=None): self.data = linearNode - if self.getChildCount()>0: + if self.getChildCount() > 0: self.removeChildAt(0) # Remove the previous linear node self.add(linearNode) self.dataRepresent = CdLinear @@ -553,23 +621,23 @@ def setLinearData(self,linearNode, partition=None): self.partition = partition self.length = partition[-1] linearNode.length = self.length - self.setExternalAttr('partition',str(self.partition)) - self.setExternalAttr('length',self.length) + self.setExternalAttr('partition', str(self.partition)) + self.setExternalAttr('length', self.length) # Test if axis data vectors are equal - def equal(self,axis): + def equal(self, axis): # Require that partitions (if any) are equal if self.partition is not None and axis.partition is not None: - if len(self.partition)!=len(axis.partition): + if len(self.partition) != len(axis.partition): return 0 - if not numpy.alltrue(numpy.equal(self.partition,axis.partition)): + if not numpy.alltrue(numpy.equal(self.partition, axis.partition)): return 0 elif self.partition is not None or axis.partition is not None: return 0 - + if self.dataRepresent == axis.dataRepresent == CdVector: try: - return numpy.alltrue(numpy.equal(self.data,axis.data)) + return numpy.alltrue(numpy.equal(self.data, axis.data)) except ValueError: return 0 elif self.dataRepresent == axis.dataRepresent == CdLinear: @@ -581,34 +649,34 @@ def equal(self,axis): # Test if axis data vectors are element-wise close # True iff for each respective element a and b, abs((b-a)/b)<=eps - def isClose(self,axis,eps): - if eps==0: + def isClose(self, axis, eps): + if eps == 0: return self.equal(axis) if self.dataRepresent == axis.dataRepresent == CdVector: try: - return numpy.alltrue(numpy.less_equal(numpy.absolute(self.data-axis.data),numpy.absolute(eps*self.data))) + return numpy.alltrue(numpy.less_equal(numpy.absolute(self.data - axis.data), numpy.absolute(eps * self.data))) except ValueError: return 0 elif self.dataRepresent == axis.dataRepresent == CdLinear: - return self.data.isClose(axis.data,eps) + return self.data.isClose(axis.data, eps) elif self.dataRepresent == CdVector: - return axis.data.isCloseVector(self.data,eps) + return axis.data.isCloseVector(self.data, eps) else: - return self.data.isCloseVector(axis.data,eps) + return self.data.isCloseVector(axis.data, eps) # Test for strict monotonicity. # Returns CdNotMonotonic, CdIncreasing, CdDecreasing, or CdSingleton def monotonicity(self): if self.dataRepresent == CdLinear: return self.data.monotonicity() - elif self.length==1: + elif self.length == 1: return CdSingleton else: first = self.data[:-1] second = self.data[1:] - if numpy.alltrue(numpy.less(first,second)): + if numpy.alltrue(numpy.less(first, second)): return CdIncreasing - elif numpy.alltrue(numpy.greater(first,second)): + elif numpy.alltrue(numpy.greater(first, second)): return CdDecreasing else: return CdNotMonotonic @@ -616,24 +684,24 @@ def monotonicity(self): # Extend axes. 'isreltime' is true iff # the axes are relative time axes # If allowgaps is true, allow gaps when extending linear vectors - def extend(self,axis,isreltime=0,allowgaps=0): + def extend(self, axis, isreltime=0, allowgaps=0): # Set trylin true if should try to catenate linear vectors - if self.dataRepresent==CdLinear: + if self.dataRepresent == CdLinear: anode = self.data - if axis.dataRepresent==CdLinear: + if axis.dataRepresent == CdLinear: bnode = axis.data trylin = 1 - elif axis.length==1: + elif axis.length == 1: bnode = LinearDataNode(axis.data[0], 0.0, 1) trylin = 1 else: trylin = 0 - elif self.length==1: + elif self.length == 1: anode = LinearDataNode(self.data[0], 0.0, 1) - if axis.dataRepresent==CdLinear: + if axis.dataRepresent == CdLinear: bnode = axis.data trylin = 1 - elif axis.length==1: + elif axis.length == 1: bnode = LinearDataNode(axis.data[0], 0.0, 1) trylin = 1 else: @@ -641,25 +709,25 @@ def extend(self,axis,isreltime=0,allowgaps=0): else: trylin = 0 - if isreltime==1: + if isreltime == 1: units1 = self.getExternalAttr('units') units2 = axis.getExternalAttr('units') else: units1 = units2 = None - if trylin==1: + if trylin == 1: try: aindex = 0 alength = anode.length bindex = alength blength = bnode.length - if isreltime==1 and units1 and units2 and units1!=units2: - rtime = cdtime.reltime(bnode.start,units2) + if isreltime == 1 and units1 and units2 and units1 != units2: + rtime = cdtime.reltime(bnode.start, units2) offset = rtime.torel(units1).value - bnode.start = bnode.start+offset + bnode.start = bnode.start + offset else: offset = None - linNode = anode.concatenate(bnode,allowgaps) + linNode = anode.concatenate(bnode, allowgaps) except NotMonotonicError: # The dimensions cannot be extended as linear arrays, # so try to extend them as vectors @@ -667,15 +735,20 @@ def extend(self,axis,isreltime=0,allowgaps=0): else: # Extend the partition attribute if offset is not None: - bindex = int(offset/linNode.delta+0.5) - if self.partition is None: - partition = numpy.array([aindex,aindex+alength,bindex,bindex+blength]) - self.partition_length = alength+blength + bindex = int(offset / linNode.delta + 0.5) + if self.partition is None: + partition = numpy.array( + [aindex, + aindex + alength, + bindex, + bindex + blength]) + self.partition_length = alength + blength else: - partition = numpy.concatenate((self.partition,[bindex,bindex+blength])) - self.partition_length = self.partition_length+blength - self.setLinearData(linNode,partition) - self.setExternalAttr('partition_length',self.partition_length) + partition = numpy.concatenate( + (self.partition, [bindex, bindex + blength])) + self.partition_length = self.partition_length + blength + self.setLinearData(linNode, partition) + self.setExternalAttr('partition_length', self.partition_length) return self # Else get both axis vectors, concatenate @@ -688,29 +761,31 @@ def extend(self,axis,isreltime=0,allowgaps=0): blength = len(ar2) # Adjust array2 if relative time and units differ - if isreltime==1: + if isreltime == 1: if units1 and units2 and units1 != units2: - rtime = cdtime.reltime(0.0,units2) + rtime = cdtime.reltime(0.0, units2) delta = rtime.torel(units1).value - ar2 = ar2+delta + ar2 = ar2 + delta - ar = numpy.concatenate((ar1,ar2)) + ar = numpy.concatenate((ar1, ar2)) try: self.setData(ar) except NotMonotonicError: # Restore original array and resignal self.setData(ar1) - raise NotMonotonicError, NotMonotonic+`ar` + raise NotMonotonicError, NotMonotonic + `ar` # Extend the partition attribute - if self.partition is None: - self.partition = numpy.array([aindex,aindex+alength,bindex,bindex+blength]) - self.partition_length = alength+blength + if self.partition is None: + self.partition = numpy.array( + [aindex, aindex + alength, bindex, bindex + blength]) + self.partition_length = alength + blength else: - self.partition = numpy.concatenate((self.partition,[bindex,bindex+blength])) - self.partition_length = self.partition_length+blength - self.setExternalAttr('partition',str(self.partition)) - self.setExternalAttr('partition_length',self.partition_length) + self.partition = numpy.concatenate( + (self.partition, [bindex, bindex + blength])) + self.partition_length = self.partition_length + blength + self.setExternalAttr('partition', str(self.partition)) + self.setExternalAttr('partition_length', self.partition_length) return self @@ -718,16 +793,21 @@ def __len__(self): return len(self.data) # Linear data element + + class LinearDataNode(CdmsNode): - validStartTypes = [int, float, type(cdtime.comptime(0)), type(cdtime.reltime(0,"hours"))] + validStartTypes = [ + int, float, type(cdtime.comptime(0)), type(cdtime.reltime(0, "hours"))] validDeltaTypes = [int, float, list] def __init__(self, start, delta, length): - assert isinstance(start, numpy.floating) or isinstance(start, numpy.integer) or (type(start) in self.validStartTypes), 'Invalid start argument: '+`start` - assert isinstance(start, numpy.floating) or isinstance(start, numpy.integer) or (type(delta) in self.validDeltaTypes), 'Invalid delta argument: '+`delta` - assert isinstance(length, int), 'Invalid length argument: '+`length` - CdmsNode.__init__(self,"linear") + assert isinstance(start, numpy.floating) or isinstance(start, numpy.integer) or ( + type(start) in self.validStartTypes), 'Invalid start argument: ' + `start` + assert isinstance(start, numpy.floating) or isinstance(start, numpy.integer) or ( + type(delta) in self.validDeltaTypes), 'Invalid delta argument: ' + `delta` + assert isinstance(length, int), 'Invalid length argument: ' + `length` + CdmsNode.__init__(self, "linear") self.delta = delta self.length = length self.start = start @@ -735,53 +815,57 @@ def __init__(self, start, delta, length): # Get an indexed value def __getitem__(self, index): - return self.start + index*self.delta + return self.start + index * self.delta # Map to external attributes def mapToExternal(self): - self.setExternalAttr("start",self.start) - self.setExternalAttr("delta",self.delta) - self.setExternalAttr("length",self.length) + self.setExternalAttr("start", self.start) + self.setExternalAttr("delta", self.delta) + self.setExternalAttr("length", self.length) # Equality of linear vectors - def equal(self,axis): - return self.delta==axis.delta and self.length==axis.length and self.start==axis.start + def equal(self, axis): + return self.delta == axis.delta and self.length == axis.length and self.start == axis.start # Closeness of linear vectors - def isClose(self,axis,eps): - if eps==0: + def isClose(self, axis, eps): + if eps == 0: return self.equal(axis) else: - return self.delta==axis.delta and self.length==axis.length and abs(self.start-axis.start)<=abs(eps*self.start) + return self.delta == axis.delta and self.length == axis.length and abs(self.start - axis.start) <= abs(eps * self.start) # Equality of linear vector and array - def equalVector(self,ar): - diff = ar[1:]-ar[:-1] + def equalVector(self, ar): + diff = ar[1:] - ar[:-1] try: - comp = numpy.alltrue(numpy.equal((self.delta)*numpy.ones(self.length-1),diff)) + comp = numpy.alltrue( + numpy.equal((self.delta) * numpy.ones(self.length - 1), diff)) except ValueError: return 0 return comp # Closeness of linear vector and array - def isCloseVector(self,ar,eps): - if eps==0: + def isCloseVector(self, ar, eps): + if eps == 0: return self.equalVector(ar) - diff = ar[1:]-ar[:-1] - diff2 = self.delta*numpy.ones(self.length-1) + diff = ar[1:] - ar[:-1] + diff2 = self.delta * numpy.ones(self.length - 1) try: - comp = numpy.alltrue(numpy.less_equal(numpy.absolute(diff2-diff),numpy.absolute(eps*diff2))) + comp = numpy.alltrue( + numpy.less_equal(numpy.absolute(diff2 - diff), + numpy.absolute(eps * diff2))) except ValueError: return 0 return comp - # Return monotonicity: CdNotMonotonic, CdIncreasing, CdDecreasing, or CdSingleton + # Return monotonicity: CdNotMonotonic, CdIncreasing, CdDecreasing, or + # CdSingleton def monotonicity(self): - if self.length==1: + if self.length == 1: return CdSingleton - elif self.delta>0.0: + elif self.delta > 0.0: return CdIncreasing - elif self.delta<0.0: + elif self.delta < 0.0: return CdDecreasing else: return CdNotMonotonic @@ -789,34 +873,39 @@ def monotonicity(self): # Return a vector representation, given a CDMS datatype def toVector(self, datatype): numericType = CdToNumericType.get(datatype) - if numericType is None: raise CDMSError, InvalidDatatype + datatype + if numericType is None: + raise CDMSError, InvalidDatatype + datatype start = self.start delta = self.delta length = self.length - if length>1: - stop = start + (length-0.99)*delta - if delta==0.0: delta=1.0 - ar = numpy.arange(start,stop,delta,numericType) + if length > 1: + stop = start + (length - 0.99) * delta + if delta == 0.0: + delta = 1.0 + ar = numpy.arange(start, stop, delta, numericType) else: - ar = numpy.array([start],numericType) + ar = numpy.array([start], numericType) return ar # Concatenate linear arrays, preserving linearity # If allowgaps is set, don't require that the linear arrays be contiguous # Return a new linear node - def concatenate(self,linearNode,allowgaps=0): - if self.length>1 and linearNode.length>1 and self.delta != linearNode.delta: - raise NotMonotonicError, NotMonotonic + 'linear vector deltas do not match: %s,%s'%(`self.delta`,`linearNode.delta`) + def concatenate(self, linearNode, allowgaps=0): + if self.length > 1 and linearNode.length > 1 and self.delta != linearNode.delta: + raise NotMonotonicError, NotMonotonic + \ + 'linear vector deltas do not match: %s,%s' % ( + `self.delta`, `linearNode.delta`) - if self.length>1: + if self.length > 1: delta = self.delta - elif linearNode.length>1: + elif linearNode.length > 1: delta = linearNode.delta else: delta = linearNode.start - self.start - if allowgaps==0: - if linearNode.start-self.start != self.length*delta: - raise NotMonotonicError, NotMonotonic + 'linear vectors are not contiguous' + if allowgaps == 0: + if linearNode.start - self.start != self.length * delta: + raise NotMonotonicError, NotMonotonic + \ + 'linear vectors are not contiguous' length = self.length + linearNode.length return LinearDataNode(self.start, delta, length) @@ -824,12 +913,16 @@ def __len__(self): return self.length # Rectilinear lat-lon grid + + class RectGridNode(CdmsNode): # Create a grid # All arguments are strings - def __init__(self, id, latitude, longitude, gridtype=UnknownGridType, order="yx", mask=None): - CdmsNode.__init__(self,"rectGrid",id) + + def __init__(self, id, latitude, longitude, + gridtype=UnknownGridType, order="yx", mask=None): + CdmsNode.__init__(self, "rectGrid", id) self.latitude = latitude self.longitude = longitude self.gridtype = gridtype @@ -839,17 +932,20 @@ def __init__(self, id, latitude, longitude, gridtype=UnknownGridType, order="yx" # Map to external attributes def mapToExternal(self): - self.setExternalAttr('type',self.gridtype) + self.setExternalAttr('type', self.gridtype) self.setExternalAttr('latitude', self.latitude) - self.setExternalAttr('longitude',self.longitude) - self.setExternalAttr('order',self.order) - if self.mask is not None: self.setExternalAttr('mask',self.mask) + self.setExternalAttr('longitude', self.longitude) + self.setExternalAttr('order', self.order) + if self.mask is not None: + self.setExternalAttr('mask', self.mask) # Link to an external element + + class XLinkNode(CdmsNode): def __init__(self, id, uri, contentRole, content=''): - CdmsNode.__init__(self,"xlink",id) + CdmsNode.__init__(self, "xlink", id) self.uri = uri self.contentRole = contentRole self.content = content @@ -857,33 +953,39 @@ def __init__(self, id, uri, contentRole, content=''): # Map to external attributes def mapToExternal(self): - self.setExternalAttr("href",self.uri,CdString) - self.setExternalAttr("content-role",self.contentRole,CdString) + self.setExternalAttr("href", self.uri, CdString) + self.setExternalAttr("content-role", self.contentRole, CdString) # Link to a document + + class DocLinkNode(CdmsNode): def __init__(self, uri, content=''): - CdmsNode.__init__(self,"doclink") + CdmsNode.__init__(self, "doclink") self.uri = uri self.content = content DocLinkNode.mapToExternal(self) # Map to external attributes def mapToExternal(self): - self.setExternalAttr("href",self.uri,CdString) + self.setExternalAttr("href", self.uri, CdString) # Domain + + class DomainNode(CdmsNode): def __init__(self): - CdmsNode.__init__(self,"domain") + CdmsNode.__init__(self, "domain") # Domain element + + class DomElemNode(CdmsNode): def __init__(self, name, start=None, length=None): - CdmsNode.__init__(self,"domElem") + CdmsNode.__init__(self, "domElem") self.name = name self.start = start self.length = length @@ -891,14 +993,16 @@ def __init__(self, name, start=None, length=None): # Map to external attributes def mapToExternal(self): - self.setExternalAttr('name',self.name) - if self.start is not None: self.setExternalAttr('start',self.start) - if self.length is not None: self.setExternalAttr('length',self.length) + self.setExternalAttr('name', self.name) + if self.start is not None: + self.setExternalAttr('start', self.start) + if self.length is not None: + self.setExternalAttr('length', self.length) # Set the name - def setName(self,name): + def setName(self, name): self.name = name - self.setExternalAttr('name',self.name) + self.setExternalAttr('name', self.name) # Get the name def getName(self): @@ -906,78 +1010,85 @@ def getName(self): # Write to a file, with formatting. # tablevel is the start number of tabs - def write(self,fd=None,tablevel=0,format=1): - if fd is None: fd = sys.stdout - if format: fd.write(tablevel*'\t') - fd.write('<'+self.tag) + def write(self, fd=None, tablevel=0, format=1): + if fd is None: + fd = sys.stdout + if format: + fd.write(tablevel * '\t') + fd.write('<' + self.tag) for attname in self.attribute.keys(): - (attval,datatype)=self.attribute[attname] + (attval, datatype) = self.attribute[attname] # attvalstr = string.replace(str(attval),'"',"'") # Map " to ' - attvalstr = _Illegal.sub(mapIllegalToEntity,str(attval)) # Map illegal chars to entities - fd.write(' '+attname+'="'+attvalstr+'"') + attvalstr = _Illegal.sub( + mapIllegalToEntity, + str(attval)) # Map illegal chars to entities + fd.write(' ' + attname + '="' + attvalstr + '"') fd.write('/>') - if format: fd.write('\n') + if format: + fd.write('\n') # Attribute node - only used as a placeholder during parse and write # Attr nodes are not placed on the tree # # Two ways to create an Attr object: # (1) attr = AttrNode(name,value) -# datatype = sometype # optionally, to override intrinsic type +# datatype = sometype # optionally, to override intrinsic type # (2) attr = AttrNode(name,None) # attr.setValueFromString(somestring,sometype) + + class AttrNode(CdmsNode): def __init__(self, name, value=None): - CdmsNode.__init__(self,"attr") - if not (isinstance(value,(int, numpy.integer, float, numpy.floating, basestring)) \ + CdmsNode.__init__(self, "attr") + if not (isinstance(value, (int, numpy.integer, float, numpy.floating, basestring)) or value is None: - raise CDMSError, 'Invalid attribute type: '+`value` - self.name = name - self.value = value - self.datatype = None # CDMS datatype, use getDatatype to retrieve - self.content = '' # string content + raise CDMSError, 'Invalid attribute type: ' + `value` + self.name=name + self.value=value + self.datatype=None # CDMS datatype, use getDatatype to retrieve + self.content='' # string content # Note: mapToExternal is not called at init time, must be called explicitly # if needed def mapToExternal(self): - self.attribute['name']=(self.name,CdString) - self.attribute['datatype']=(self.getDatatype(),CdString) - self.content = self.getValueAsString() + self.attribute['name']=(self.name, CdString) + self.attribute['datatype']=(self.getDatatype(), CdString) + self.content=self.getValueAsString() def getDatatype(self): if self.datatype: return self.datatype elif isinstance(self.value, basestring): return CdString - elif isinstance(self.value, (float,numpy.floating)): + elif isinstance(self.value, (float, numpy.floating)): return CdDouble elif isinstance(self.value, (int, numpy.integer)): return CdLong else: - raise CDMSError, 'Invalid attribute type: '+`self.value` + raise CDMSError, 'Invalid attribute type: ' + `self.value` def getLength(self): return 1 # Map a string of a given datatype to a value # Returns ValueError if the conversion fails - def setValueFromString(self,valString,datatype): - val = None + def setValueFromString(self, valString, datatype): + val=None if not isinstance(valString, basestring): raise CDMSError, 'input value is not a string' if datatype == CdString: val=valString - elif datatype in (CdShort,CdInt,CdLong): + elif datatype in (CdShort, CdInt, CdLong): try: val=int(valString) except ValueError: - raise CDMSError, 'value is not an integer: '+valString - elif datatype in (CdFloat,CdDouble): + raise CDMSError, 'value is not an integer: ' + valString + elif datatype in (CdFloat, CdDouble): try: val=float(valString) except ValueError: - raise CDMSError, 'value is not floating-point: '+valString + raise CDMSError, 'value is not floating-point: ' + valString self.value=val self.datatype=datatype return val @@ -987,36 +1098,40 @@ def getValueAsString(self): # Set content # This may be called multiple times, so append - def setContentFromString(self,content): - self.content = self.content+content + def setContentFromString(self, content): + self.content=self.content + content # Write to a file, with formatting. # tablevel is the start number of tabs - def write(self,fd=None,tablevel=0,format=1): - if fd is None: fd = sys.stdout + def write(self, fd=None, tablevel=0, format=1): + if fd is None: fd=sys.stdout if self.dtd: - validAttrs = self.dtd.keys() + validAttrs=self.dtd.keys() else: - validAttrs = None + validAttrs=None - if format: fd.write(tablevel*'\t') - fd.write('<'+self.tag) + if format: fd.write(tablevel * '\t') + fd.write('<' + self.tag) # Write valid attributes for attname in self.attribute.keys(): if (validAttrs and (attname in validAttrs)) or (not validAttrs): - (attval,datatype)=self.attribute[attname] + (attval, datatype)=self.attribute[attname] # attvalstr = string.replace(str(attval),'"',"'") # Map " to ' - attvalstr = _Illegal.sub(mapIllegalToEntity,str(attval)) # Map illegal chars to entities - fd.write(' '+attname+'="'+attvalstr+'"') + attvalstr=_Illegal.sub( + mapIllegalToEntity, + str(attval)) # Map illegal chars to entities + fd.write(' ' + attname + '="' + attvalstr + '"') fd.write('>') # Write content if self.content is not None: - content = _Illegal.sub(mapIllegalToEntity,self.content) # Map illegal chars to entities + content=_Illegal.sub( + mapIllegalToEntity, + self.content) # Map illegal chars to entities fd.write(content) - fd.write('') + fd.write('') if format: fd.write('\n') if __name__ == '__main__': @@ -1049,7 +1164,7 @@ def write(self,fd=None,tablevel=0,format=1): # hAxis = AxisNode('h',len(h),CdDouble,h) # jAxis = AxisNode('j',len(j),CdDouble); jAxis.setLinearData(j) # kAxis = AxisNode('k',len(k),CdDouble,k) - + # print aAxis.monotonicity() # print hAxis.monotonicity() # print kAxis.monotonicity() @@ -1057,29 +1172,29 @@ def write(self,fd=None,tablevel=0,format=1): # print dAxis.monotonicity() # print jAxis.monotonicity() - m = LinearDataNode(1,2,3) - n = LinearDataNode(11,2,3) - p = LinearDataNode(15,-4,3) - q = numpy.array([4.,2.,1.]) - r = numpy.array([11.,9.,8.]) - s = numpy.array([7.]) - t = numpy.array([9.]) - v = numpy.array([5.]) - - mAxis = AxisNode('m',len(m),CdLong); mAxis.setLinearData(m) - nAxis = AxisNode('n',len(n),CdLong); nAxis.setLinearData(n) - pAxis = AxisNode('p',len(p),CdLong); pAxis.setLinearData(p) - qAxis = AxisNode('q',len(q),CdDouble,q) - rAxis = AxisNode('r',len(r),CdDouble,r) - sAxis = AxisNode('s',len(s),CdDouble,s) - tAxis = AxisNode('t',len(t),CdDouble,t) - vAxis = AxisNode('v',len(v),CdDouble,v) + m=LinearDataNode(1, 2, 3) + n=LinearDataNode(11, 2, 3) + p=LinearDataNode(15, -4, 3) + q=numpy.array([4., 2., 1.]) + r=numpy.array([11., 9., 8.]) + s=numpy.array([7.]) + t=numpy.array([9.]) + v=numpy.array([5.]) + + mAxis=AxisNode('m', len(m), CdLong); mAxis.setLinearData(m) + nAxis=AxisNode('n', len(n), CdLong); nAxis.setLinearData(n) + pAxis=AxisNode('p', len(p), CdLong); pAxis.setLinearData(p) + qAxis=AxisNode('q', len(q), CdDouble, q) + rAxis=AxisNode('r', len(r), CdDouble, r) + sAxis=AxisNode('s', len(s), CdDouble, s) + tAxis=AxisNode('t', len(t), CdDouble, t) + vAxis=AxisNode('v', len(v), CdDouble, v) def printType(axis): - if axis.dataRepresent==CdLinear: print 'linear' + if axis.dataRepresent == CdLinear: print 'linear' else: print 'vector' - - def testit(a,b): + + def testit(a, b): import copy x=copy.copy(a) print x.extend(b).getData() @@ -1095,6 +1210,6 @@ def testit(a,b): # testit(mAxis,rAxis) # testit(vAxis,nAxis) # testit(sAxis,rAxis) - + # Errors: # testit(mAxis,nAxis) diff --git a/Packages/cdms2/Lib/cdmsURLopener.py b/Packages/cdms2/Lib/cdmsURLopener.py index f401bf8d55..e0b7101319 100644 --- a/Packages/cdms2/Lib/cdmsURLopener.py +++ b/Packages/cdms2/Lib/cdmsURLopener.py @@ -3,15 +3,16 @@ import urllib + class CDMSURLopener(urllib.FancyURLopener): - # Override FancyURLopener error handling - raise an exception - # Can also define function http_error_DDD where DDD is the 3-digit error code, - # to handle specific errors. - def http_error_default(self, url, fp, errcode, errmsg, headers): - void = fp.read() - fp.close() - raise IOError, ('http error', errcode, errmsg, headers) + # Override FancyURLopener error handling - raise an exception + # Can also define function http_error_DDD where DDD is the 3-digit error code, + # to handle specific errors. -urllib._urlopener = CDMSURLopener() + def http_error_default(self, url, fp, errcode, errmsg, headers): + void = fp.read() + fp.close() + raise IOError('http error', errcode, errmsg, headers) +urllib._urlopener = CDMSURLopener() diff --git a/Packages/cdms2/Lib/cdmsobj.py b/Packages/cdms2/Lib/cdmsobj.py index 8fad6b3ab0..30961e26ba 100644 --- a/Packages/cdms2/Lib/cdmsobj.py +++ b/Packages/cdms2/Lib/cdmsobj.py @@ -2,7 +2,7 @@ CDMS module-level functions and definitions """ -import cdmsNode +from . import cdmsNode import cdtime import glob import os @@ -26,7 +26,7 @@ Unlimited = 1 # Unlimited axis designator -Max32int = 2**31-1 # Maximum 32-bit integer +Max32int = 2**31 - 1 # Maximum 32-bit integer # Regular expressions for each template specifier _Daynum = '[0-3][0-9]' @@ -41,7 +41,7 @@ _Second = '[0-5][0-9]' _Year2 = '[0-9][0-9]' _Year4 = '[0-9]{4,4}' -_Zulu = _Hour+'[z|Z]'+_Year4+_Monthnum+_Daynum +_Zulu = _Hour + '[z|Z]' + _Year4 + _Monthnum + _Daynum # Positions for time lists _yr = 0 @@ -93,42 +93,67 @@ '%v': (_Name, 'name', 'var', None), '%y': (_Year2, 'year2', 'time', _yr), '%z': (_Zulu, 'zulu', 'time', None), - } - -_monthListUpper = ['JAN','FEB','MAR','APR','MAY','JUN','JUL','AUG','SEP','OCT','NOV','DEC'] +} + +_monthListUpper = [ + 'JAN', + 'FEB', + 'MAR', + 'APR', + 'MAY', + 'JUN', + 'JUL', + 'AUG', + 'SEP', + 'OCT', + 'NOV', + 'DEC'] _monthMapUpper = { - 'JAN':1, - 'FEB':2, - 'MAR':3, - 'APR':4, - 'MAY':5, - 'JUN':6, - 'JUL':7, - 'AUG':8, - 'SEP':9, - 'OCT':10, - 'NOV':11, - 'DEC':12, - } - -_monthListLower = ['jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec'] + 'JAN': 1, + 'FEB': 2, + 'MAR': 3, + 'APR': 4, + 'MAY': 5, + 'JUN': 6, + 'JUL': 7, + 'AUG': 8, + 'SEP': 9, + 'OCT': 10, + 'NOV': 11, + 'DEC': 12, +} + +_monthListLower = [ + 'jan', + 'feb', + 'mar', + 'apr', + 'may', + 'jun', + 'jul', + 'aug', + 'sep', + 'oct', + 'nov', + 'dec'] _monthMapLower = { - 'jan':1, - 'feb':2, - 'mar':3, - 'apr':4, - 'may':5, - 'jun':6, - 'jul':7, - 'aug':8, - 'sep':9, - 'oct':10, - 'nov':11, - 'dec':12, - } - -_specre = re.compile('(%%|%G|%H|%L|%M|%S|%Y|%d|%eG|%eH|%eL|%eM|%eS|%eY|%ed|%ef|%eg|%eh|%em|%en|%ey|%ez|%f|%g|%h|%m|%n|%v|%y|%z)') -_filere = re.compile('[^'+os.sep+']+') + 'jan': 1, + 'feb': 2, + 'mar': 3, + 'apr': 4, + 'may': 5, + 'jun': 6, + 'jul': 7, + 'aug': 8, + 'sep': 9, + 'oct': 10, + 'nov': 11, + 'dec': 12, +} + +_specre = re.compile( + '(%%|%G|%H|%L|%M|%S|%Y|%d|%eG|%eH|%eL|%eM|%eS|%eY|%ed|%ef|%eg|%eh|%em|%en|%ey|%ez|%f|%g|%h|%m|%n|%v|%y|%z)') +_filere = re.compile('[^' + os.sep + ']+') _debug = 0 # Print debug info @@ -137,12 +162,14 @@ # Module-level functions # Set debug mode, to 'on' or 'off' + + def setDebugMode(mode): global _debug - if mode=='on': - _debug=1 - elif mode=='off': - _debug=0 + if mode == 'on': + _debug = 1 + elif mode == 'off': + _debug = 0 # Map a template to a regular expression # Returns (regex,dimtypes), where regex is the regular expression @@ -151,206 +178,226 @@ def setDebugMode(mode): # where specStrings is the specifier associated with the dimension type, # or for time, the list of specifiers in the order (yr,mo,dy,hr,mi,se) # where each element is the specifier for that time element + + def templateToRegex(template): matchspec = {} - dimtypes = {'var':None, - 'time':[None,None,None,None,None,None], - 'etime':[None,None,None,None,None,None], - 'level':None, - 'elevel':None, + dimtypes = {'var': None, + 'time': [None, None, None, None, None, None], + 'etime': [None, None, None, None, None, None], + 'level': None, + 'elevel': None, } # Map spec to regex # (The default arg bring matchspec and dimtypes into the local scope) def retspec(matchobj, matchspec=matchspec, dimtypes=dimtypes): spec = matchobj.group(0) - pat,name,dimtype,pos = _specifierMap[spec] + pat, name, dimtype, pos = _specifierMap[spec] - if dimtype=='var': - dimtypes['var']=spec - elif dimtype in ('time','etime'): + if dimtype == 'var': + dimtypes['var'] = spec + elif dimtype in ('time', 'etime'): if pos is not None: - dimtypes[dimtype][pos]=spec - elif name in ('zulu','ezulu'): - pass # Crack Zulu time - elif dimtype in ('level','elevel'): - dimtypes[dimtype]=spec - - if matchspec.has_key(spec): - return '(?P='+name+')' + dimtypes[dimtype][pos] = spec + elif name in ('zulu', 'ezulu'): + pass # Crack Zulu time + elif dimtype in ('level', 'elevel'): + dimtypes[dimtype] = spec + + if spec in matchspec: + return '(?P=' + name + ')' else: matchspec[spec] = 1 - return '(?P<'+name+'>'+pat+')' - - templatere = _specre.sub(retspec,template) - return (templatere,dimtypes) + return '(?P<' + name + '>' + pat + ')' + + templatere = _specre.sub(retspec, template) + return (templatere, dimtypes) + def retglob(matchobj): return '*' # Generate a component time from a matchobj and list of specs -def generateTime(matchobj,timespecs): + + +def generateTime(matchobj, timespecs): iyr = 0 imo = 1 idy = 1 ihr = 0 imi = 0 ise = 0 - yrspec,mospec,dyspec,hrspec,mispec,sespec = timespecs + yrspec, mospec, dyspec, hrspec, mispec, sespec = timespecs if yrspec: - pat,name,dimtype,pos = _specifierMap[yrspec] + pat, name, dimtype, pos = _specifierMap[yrspec] yrstr = matchobj.group(name) iyr = int(yrstr) # Map 2-digit year to [1950,2049) - if yrspec in ('%y','%ey'): - if iyr<50: - iyr = iyr+2000 + if yrspec in ('%y', '%ey'): + if iyr < 50: + iyr = iyr + 2000 else: - iyr = iyr+1900 + iyr = iyr + 1900 if mospec: - pat,name,dimtype,pos = _specifierMap[mospec] + pat, name, dimtype, pos = _specifierMap[mospec] mostr = matchobj.group(name) - if mospec in ('%G','%eG'): + if mospec in ('%G', '%eG'): imo = _monthMapUpper[mostr] - elif mospec in ('%g','%eg'): + elif mospec in ('%g', '%eg'): imo = _monthMapLower[mostr] - elif mospec in ('%m','%em','%n','%en'): + elif mospec in ('%m', '%em', '%n', '%en'): imo = int(mostr) if dyspec: - pat,name,dimtype,pos = _specifierMap[dyspec] + pat, name, dimtype, pos = _specifierMap[dyspec] dystr = matchobj.group(name) idy = int(dystr) if hrspec: - pat,name,dimtype,pos = _specifierMap[hrspec] + pat, name, dimtype, pos = _specifierMap[hrspec] hrstr = matchobj.group(name) ihr = int(hrstr) if mispec: - pat,name,dimtype,pos = _specifierMap[mispec] + pat, name, dimtype, pos = _specifierMap[mispec] mistr = matchobj.group(name) imi = int(mistr) if sespec: - pat,name,dimtype,pos = _specifierMap[sespec] + pat, name, dimtype, pos = _specifierMap[sespec] sestr = matchobj.group(name) ise = int(sestr) - return cdtime.comptime(iyr,imo,idy,ihr,imi,ise) + return cdtime.comptime(iyr, imo, idy, ihr, imi, ise) # Find all files in 'direc' which match 'template'. # template is a relative path, and may contain specifiers # in directory names. Returns a list [(f,m),..,(f,m)] where # f is a matching file name, and m is a list [var,time,etime,level,elevel] # of matching values in f. Any or all elems of the list may be None. -def matchingFiles(direc,template): + + +def matchingFiles(direc, template): if os.path.isabs(template): - raise AbsoluteTemplate, template + raise AbsoluteTemplate(template) # Generate a glob pattern normTemplate = os.path.normpath(template) - globPattern = _filere.sub(retglob,normTemplate) + globPattern = _filere.sub(retglob, normTemplate) # Map the template to a regular expression - templatere,dimtypes = templateToRegex(template) + templatere, dimtypes = templateToRegex(template) ctre = re.compile(templatere) # Get a list of candidate files try: os.chdir(direc) except os.error: - raise IOError,'%s: %s'%(sys.exc_value,direc) - + raise IOError('%s: %s' % (sys.exc_info()[1], direc)) + candlist = glob.glob(globPattern) - # Look for matches + # Look for matches matchfiles = [] for candfile in candlist: matchobj = ctre.match(candfile) # Create matching values - if matchobj is None: continue - matchnames = [None,None,None,None,None] + if matchobj is None: + continue + matchnames = [None, None, None, None, None] if dimtypes['var'] is not None: matchnames[_var] = matchobj.group('name') - if dimtypes['time'] != [None,None,None,None,None,None]: - matchnames[_time] = generateTime(matchobj,dimtypes['time']) - if dimtypes['etime'] != [None,None,None,None,None,None]: - matchnames[_etime] = generateTime(matchobj,dimtypes['etime']) + if dimtypes['time'] != [None, None, None, None, None, None]: + matchnames[_time] = generateTime(matchobj, dimtypes['time']) + if dimtypes['etime'] != [None, None, None, None, None, None]: + matchnames[_etime] = generateTime(matchobj, dimtypes['etime']) if dimtypes['level'] is not None: matchnames[_level] = int(matchobj.group('level')) if dimtypes['elevel'] is not None: matchnames[_elevel] = int(matchobj.group('elevel')) - matchfiles.append((candfile,matchnames)) + matchfiles.append((candfile, matchnames)) return matchfiles # Get a string time component from a spec and a component time -def getTimeAsString(spec,time): - if spec in ('%G','%eG'): + +def getTimeAsString(spec, time): + + if spec in ('%G', '%eG'): imo = time.month - specstr = _monthListUpper[imo-1] - elif spec in ('%H','%eH'): + specstr = _monthListUpper[imo - 1] + elif spec in ('%H', '%eH'): specstr = str(time.hour) - elif spec in ('%M','%eM'): + elif spec in ('%M', '%eM'): specstr = str(time.minute) - elif spec in ('%S','%eS'): + elif spec in ('%S', '%eS'): specstr = str(int(time.second)) - elif spec in ('%Y','%eY'): + elif spec in ('%Y', '%eY'): specstr = str(time.year).zfill(4) - elif spec in ('%d','%ed'): + elif spec in ('%d', '%ed'): specstr = str(time.day) - elif spec in ('%f','%ef'): + elif spec in ('%f', '%ef'): specstr = str(time.day).zfill(2) - elif spec in ('%g','%eg'): + elif spec in ('%g', '%eg'): imo = time.month - specstr = _monthListLower[imo-1] - elif spec in ('%h','%eh'): + specstr = _monthListLower[imo - 1] + elif spec in ('%h', '%eh'): specstr = str(time.hour).zfill(2) - elif spec in ('%m','%em'): + elif spec in ('%m', '%em'): specstr = str(time.month) - elif spec in ('%n','%en'): + elif spec in ('%n', '%en'): specstr = str(time.month).zfill(2) - elif spec in ('%y','%ey'): - specstr = str(time.year%100).zfill(2) - elif spec in ('%z','%ez'): - specstr = getTimeAsString('%H',time)+'Z'+getTimeAsString('%Y',time)+getTimeAsString('%n',time)+getTimeAsString('%d',time) + elif spec in ('%y', '%ey'): + specstr = str(time.year % 100).zfill(2) + elif spec in ('%z', '%ez'): + specstr = getTimeAsString( + '%H', + time) + 'Z' + getTimeAsString( + '%Y', + time) + getTimeAsString( + '%n', + time) + getTimeAsString( + '%d', + time) return specstr # Generate a file path, given a template and matchname list. # matchnames is a list [varname,time,etime,level,elevel], where # any or all elems may be None. If matchnames be a longer list, # it is not an error but the additional elements are ignored. -def getPathFromTemplate(template,matchnames): + + +def getPathFromTemplate(template, matchnames): # Map spec to value string # (Default arg brings matchnames into the local scope) def retpath(matchobj, matchnames=matchnames): spec = matchobj.group(0) - pat,name,dimtype,pos = _specifierMap[spec] - var,time,etime,level,elevel = matchnames[0:5] + pat, name, dimtype, pos = _specifierMap[spec] + var, time, etime, level, elevel = matchnames[0:5] - if dimtype=='var': + if dimtype == 'var': if var is None: specstr = spec else: specstr = var - elif dimtype=='time': + elif dimtype == 'time': if time is None: specstr = spec else: - specstr = getTimeAsString(spec,time) - elif dimtype=='etime': + specstr = getTimeAsString(spec, time) + elif dimtype == 'etime': if etime is None: specstr = spec else: - specstr = getTimeAsString(spec,etime) - elif dimtype=='level': + specstr = getTimeAsString(spec, etime) + elif dimtype == 'level': if level is None: specstr = spec else: specstr = str(level) - elif dimtype=='elevel': + elif dimtype == 'elevel': if elevel is None: specstr = spec else: @@ -358,7 +405,7 @@ def retpath(matchobj, matchnames=matchnames): return specstr - path = _specre.sub(retpath,template) + path = _specre.sub(retpath, template) return path # Search an object or list of objects for a string attribute which @@ -369,6 +416,7 @@ def retpath(matchobj, matchnames=matchnames): # 'rectGrid','xlink', or None. If tag is None, all object classes are # searched. If attribute is None, all attributes are searched. + def searchPattern(objlist, pattern, attribute=None, tag=None): if tag is not None: tag = tag.lower() @@ -378,7 +426,7 @@ def searchPattern(objlist, pattern, attribute=None, tag=None): returnlist = [] for obj in objlist: - returnlist = returnlist + obj.searchPattern(regexp,attribute,tag) + returnlist = returnlist + obj.searchPattern(regexp, attribute, tag) return returnlist @@ -390,6 +438,7 @@ def searchPattern(objlist, pattern, attribute=None, tag=None): # 'rectGrid','xlink', or None. If tag is None, all object classes are # searched. If attribute is None, all attributes are searched. + def matchPattern(objlist, pattern, attribute=None, tag=None): if tag is not None: tag = tag.lower() @@ -399,7 +448,7 @@ def matchPattern(objlist, pattern, attribute=None, tag=None): returnlist = [] for obj in objlist: - returnlist = returnlist + obj.matchPattern(regexp,attribute,tag) + returnlist = returnlist + obj.matchPattern(regexp, attribute, tag) return returnlist @@ -418,7 +467,9 @@ def matchPattern(objlist, pattern, attribute=None, tag=None): # # lambda obj: obj.partition_length > 1000 # -# is sufficient, it is not necessary to test for the existence of the attribute. +# is sufficient, it is not necessary to test for the existence of the +# attribute. + def searchPredicate(objlist, predicate, tag=None): if tag is not None: @@ -428,7 +479,7 @@ def searchPredicate(objlist, predicate, tag=None): returnlist = [] for obj in objlist: - returnlist = returnlist + obj.searchPredicate(predicate,tag) + returnlist = returnlist + obj.searchPredicate(predicate, tag) return returnlist @@ -436,74 +487,80 @@ def searchPredicate(objlist, predicate, tag=None): # Classes # Generic CDMS object has a tree node, attributes + + class CdmsObj (object): -## def __setattr__(self,name,value): -## object.__setattr__(self,name,value) -## if not name in self.__cdms_internals__ and not name[0]=='_': -## self.attributes[name]=value -## ## if name == 'shape' : -## ## print self.__class__,name,value +# def __setattr__(self,name,value): +# object.__setattr__(self,name,value) +# if not name in self.__cdms_internals__ and not name[0]=='_': +# self.attributes[name]=value +# if name == 'shape' : +# print self.__class__,name,value def _listatts(self): - dic={} - for nm,val in self.__dict__.items(): - if (nm[0]!='_' and not nm in self.__cdms_internals__) or nm in ['_FillValue']: - dic[nm]=val + dic = {} + for nm, val in self.__dict__.items(): + if (nm[0] != '_' and not nm in self.__cdms_internals__) or nm in ['_FillValue']: + dic[nm] = val if nm == '_units': - dic['units']=val + dic['units'] = val return dic - def _setatts(self,value): + + def _setatts(self, value): return - attributes = property(_listatts,_setatts) - - - def __init__(self, node = None): - if not hasattr(self,'___cdms_internals__'): - self.__dict__['___cdms_internals__']=[ - '__cdms_internals__','___cdms_internals__', - '_node_','_obj_', - '_numericType_','_grid_','_bounds_', - 'parent','attributes','shape','autoApiInfo'] - self.attributes={} + attributes = property(_listatts, _setatts) + + def __init__(self, node=None): + if not hasattr(self, '___cdms_internals__'): + self.__dict__['___cdms_internals__'] = [ + '__cdms_internals__', '___cdms_internals__', + '_node_', '_obj_', + '_numericType_', '_grid_', '_bounds_', + 'parent', 'attributes', 'shape', 'autoApiInfo'] + self.attributes = {} self._node_ = node if node is not None: - # Build an attribute dictionary from the node, + # Build an attribute dictionary from the node, # CDML datatype constraints - if hasattr(node,'datatype'): + if hasattr(node, 'datatype'): parenttype = node.datatype else: parenttype = None atts = node.getExternalDict() adict = self.__dict__ for attname in atts.keys(): - (attval,datatype)=atts[attname] # (XML value, datatype) + (attval, datatype) = atts[attname] # (XML value, datatype) constraint = node.extra.get(attname) if constraint is not None: - (scaletype,reqtype)=constraint # (CdScalar|CdArray, required type) - if reqtype==CdFromObject: + (scaletype, + reqtype) = constraint # (CdScalar|CdArray, required type) + if reqtype == CdFromObject: reqtype = parenttype - if reqtype!=datatype and datatype==CdString and scaletype==CdScalar: - if reqtype in (CdFloat,CdDouble): + if reqtype != datatype and datatype == CdString and scaletype == CdScalar: + if reqtype in (CdFloat, CdDouble): try: attval = float(attval) except: - raise RuntimeError,"%s=%s must be a float"%(attname,attval) - elif reqtype in (CdShort,CdInt,CdLong,CdInt64): + raise RuntimeError( + "%s=%s must be a float" % + (attname, attval)) + elif reqtype in (CdShort, CdInt, CdLong, CdInt64): try: attval = int(attval) except: - raise RuntimeError,"%s=%s must be an integer"%(attname,attval) + raise RuntimeError( + "%s=%s must be an integer" % + (attname, attval)) adict[attname] = attval self.attributes[attname] = attval - def searchone(self, pattern, attname): """Return true if the attribute with name attname is a string attribute which contains the compiled regular expression pattern, or if attname is None and pattern matches at least one string - attribute. Return false if the attribute is not found or is not + attribute. Return false if the attribute is not found or is not a string. ::: Input::: @@ -516,12 +573,12 @@ def searchone(self, pattern, attname): """ if attname is None: for attval in self.attributes.values(): - if isinstance(attval,basestring) and pattern.search(attval) is not None: + if isinstance(attval, basestring) and pattern.search(attval) is not None: return 1 return 0 - elif self.attributes.has_key(attname): + elif attname in self.attributes: attval = self.attributes[attname] - return isinstance(attval,basestring) and pattern.search(attval) is not None + return isinstance(attval, basestring) and pattern.search(attval) is not None else: return 0 @@ -546,18 +603,19 @@ def matchone(self, pattern, attname): """ if attname is None: for attval in self.attributes.values(): - if isinstance(attval,basestring) and pattern.match(attval) is not None: + if isinstance(attval, basestring) and pattern.match(attval) is not None: return 1 return 0 - elif self.attributes.has_key(attname): + elif attname in self.attributes: attval = self.attributes[attname] - return isinstance(attval,basestring) and pattern.match(attval) is not None + return isinstance(attval, basestring) and pattern.match(attval) is not None else: return 0 # Search for a pattern in a string-valued attribute. If attribute is None, - # search all string attributes. If tag is not None, it must match the internal node tag. - def searchPattern(self,pattern,attribute,tag): + # search all string attributes. If tag is not None, it must match the + # internal node tag. + def searchPattern(self, pattern, attribute, tag): """ Search for a pattern in a string-valued attribute. If attribute is None, search all string attributes. If tag is not None, it must match the internal node tag. ::: @@ -567,11 +625,11 @@ def searchPattern(self,pattern,attribute,tag): tag :: (str/None) (2) node tag ::: Output::: - result :: (list) (0) + result :: (list) (0) ::: """ - if tag is None or tag.lower()==self._node_.tag: - if self.searchone(pattern,attribute): + if tag is None or tag.lower() == self._node_.tag: + if self.searchone(pattern, attribute): return [self] else: return [] @@ -579,8 +637,9 @@ def searchPattern(self,pattern,attribute,tag): return [] # Match a pattern in a string-valued attribute. If attribute is None, - # search all string attributes. If tag is not None, it must match the internal node tag. - def matchPattern(self,pattern,attribute,tag): + # search all string attributes. If tag is not None, it must match the + # internal node tag. + def matchPattern(self, pattern, attribute, tag): """ Match for a pattern in a string-valued attribute. If attribute is None, search all string attributes. If tag is not None, it must match the internal node tag. ::: @@ -590,11 +649,11 @@ def matchPattern(self,pattern,attribute,tag): tag :: (str/None) (2) node tag ::: Output::: - result :: (list) (0) + result :: (list) (0) ::: """ - if tag is None or tag.lower()==self._node_.tag: - if self.matchone(pattern,attribute): + if tag is None or tag.lower() == self._node_.tag: + if self.matchone(pattern, attribute): return [self] else: return [] @@ -604,7 +663,7 @@ def matchPattern(self,pattern,attribute,tag): # Apply a truth-valued predicate. Return a list containing a single instance: [self] # if the predicate is true and either tag is None or matches the object node tag. # If the predicate returns false, return an empty list - def searchPredicate(self,predicate,tag): + def searchPredicate(self, predicate, tag): """ Apply a truth-valued predicate. Return a list containing a single instance: [self] if the predicate is true and either tag is None or matches the object node tag. If the predicate returns false, return an empty list ::: @@ -613,12 +672,12 @@ def searchPredicate(self,predicate,tag): tag :: (str/None) (1) node tag ::: Output::: - result :: (list) (0) + result :: (list) (0) ::: """ - if tag is None or tag.lower()==self._node_.tag: + if tag is None or tag.lower() == self._node_.tag: try: - if apply(predicate,(self,))==1: + if predicate(*(self,)) == 1: result = [self] except: result = [] @@ -626,7 +685,7 @@ def searchPredicate(self,predicate,tag): result = [] return result - def dump(self,path=None,format=1): + def dump(self, path=None, format=1): """ dump(self,path=None,format=1) Dump an XML representation of this object to a file. 'path' is the result file name, None for standard output. @@ -641,24 +700,21 @@ def dump(self,path=None,format=1): ::: """ if self._node_ is None: - raise CDMSError, "No tree node found" - self._node_.dump(path,format) + raise CDMSError("No tree node found") + self._node_.dump(path, format) def _getinternals(self): return self.___cdms_internals__ - def _setinternals(self,value): + + def _setinternals(self, value): self.___cdms_internals__ = value - __cdms_internals__ = property(_getinternals,_setinternals) -#internattr.add_internal_attribute(CdmsObj) + __cdms_internals__ = property(_getinternals, _setinternals) +# internattr.add_internal_attribute(CdmsObj) if __name__ == '__main__': x = CdmsObj(None) x.someatt = 1 assert x.attributes['someatt'] == x.someatt - assert not x.attributes.has_key('_node') + assert '_node' not in x.attributes # need tests for the search routines... print "Test passed." - - - - diff --git a/Packages/cdms2/Lib/cdurllib.py b/Packages/cdms2/Lib/cdurllib.py index 6ae1848b3c..eefccb114b 100644 --- a/Packages/cdms2/Lib/cdurllib.py +++ b/Packages/cdms2/Lib/cdurllib.py @@ -1,13 +1,18 @@ """Customized URLopener""" -import urllib, getpass, socket, string, sys +import urllib +import getpass +import socket +import string +import sys MAXFTPCACHE = 10 # Trim the ftp cache beyond this size + class CDURLopener(urllib.URLopener): def __init__(self, proxies=None): - urllib.URLopener.__init__(self,proxies) + urllib.URLopener.__init__(self, proxies) self._userObject = None # Attach an object to be returned with callbacks @@ -17,12 +22,15 @@ def setUserObject(self, userObject): # Use FTP protocol def open_ftp(self, url): host, path = urllib.splithost(url) - if not host: raise IOError, ('ftp error', 'no host given') + if not host: + raise IOError, ('ftp error', 'no host given') host, port = urllib.splitport(host) user, host = urllib.splituser(host) # if user: user, passwd = splitpasswd(user) - if user: passwd = getpass.getpass() - else: passwd = None + if user: + passwd = getpass.getpass() + else: + passwd = None host = urllib.unquote(host) user = urllib.unquote(user or '') passwd = urllib.unquote(passwd or '') @@ -36,7 +44,8 @@ def open_ftp(self, url): path = urllib.unquote(path) dirs = string.splitfields(path, '/') dirs, file = dirs[:-1], dirs[-1] - if dirs and not dirs[0]: dirs = dirs[1:] + if dirs and not dirs[0]: + dirs = dirs[1:] key = (user, host, port, string.joinfields(dirs, '/')) # XXX thread unsafe! if len(self.ftpcache) > MAXFTPCACHE: @@ -47,12 +56,14 @@ def open_ftp(self, url): del self.ftpcache[k] v.close() try: - if not self.ftpcache.has_key(key): - print 'Creating ftpwrapper: ',user,host,port,dirs + if key not in self.ftpcache: + print 'Creating ftpwrapper: ', user, host, port, dirs self.ftpcache[key] = \ urllib.ftpwrapper(user, passwd, host, port, dirs) - if not file: type = 'D' - else: type = 'I' + if not file: + type = 'D' + else: + type = 'I' for attr in attrs: attr, value = urllib.splitvalue(attr) if string.lower(attr) == 'type' and \ @@ -60,18 +71,19 @@ def open_ftp(self, url): type = string.upper(value) (fp, retrlen) = self.ftpcache[key].retrfile(file, type) if retrlen is not None and retrlen >= 0: - import mimetools, StringIO + import mimetools + import StringIO headers = mimetools.Message(StringIO.StringIO( 'Content-Length: %d\n' % retrlen)) else: headers = noheaders() return urllib.addinfourl(fp, headers, "ftp:" + url) - except urllib.ftperrors(), msg: + except urllib.ftperrors() as msg: raise IOError, ('ftp error', msg), sys.exc_info()[2] def retrieve(self, url, filename=None, reporthook=None, blocksize=262144): url = urllib.unwrap(url) - if self.tempcache and self.tempcache.has_key(url): + if self.tempcache and url in self.tempcache: return self.tempcache[url] type, url1 = urllib.splittype(url) if not filename and (not type or type == 'file'): @@ -80,7 +92,7 @@ def retrieve(self, url, filename=None, reporthook=None, blocksize=262144): hdrs = fp.info() del fp return url2pathname(urllib.splithost(url1)[1]), hdrs - except IOError, msg: + except IOError as msg: pass fp = self.open(url) headers = fp.info() @@ -101,28 +113,28 @@ def retrieve(self, url, filename=None, reporthook=None, blocksize=262144): size = -1 blocknum = 1 if reporthook: - if headers.has_key("content-length"): + if "content-length" in headers: size = int(headers["Content-Length"]) stayopen = reporthook(0, bs, size, self._userObject) - if stayopen==0: + if stayopen == 0: raise KeyboardInterrupt bytesread = 0 block = fp.read(bs) if reporthook: stayopen = reporthook(1, bs, size, self._userObject) - if stayopen==0: + if stayopen == 0: raise KeyboardInterrupt while block: tfp.write(block) bytesread = bytesread + len(block) -## print blocknum, bytesread, size, -## if blocknum*blocksize!=bytesread: -## print ' (*)' -## else: -## print +# print blocknum, bytesread, size, +# if blocknum*blocksize!=bytesread: +# print ' (*)' +# else: +# print if block and reporthook: stayopen = reporthook(blocknum, bs, size, self._userObject) - if stayopen==0: + if stayopen == 0: raise KeyboardInterrupt blocknum = blocknum + 1 block = fp.read(bs) @@ -132,23 +144,25 @@ def retrieve(self, url, filename=None, reporthook=None, blocksize=262144): del tfp return result + def sampleReportHook(blocknum, blocksize, size, userObj): - sizekb = size/1024 - percent = min(100,int(100.0*float(blocknum*blocksize)/float(size))) - print "Read: %3d%% of %dK"%(percent,sizekb) + sizekb = size / 1024 + percent = min(100, int(100.0 * float(blocknum * blocksize) / float(size))) + print "Read: %3d%% of %dK" % (percent, sizekb) return 1 if __name__ == '__main__': import sys - if len(sys.argv)!=4: + if len(sys.argv) != 4: print 'Usage: cdurllib.py URL filename blocksize' sys.exit(1) url = sys.argv[1] filename = sys.argv[2] blocksize = int(sys.argv[3]) - + urlopener = CDURLopener() - fname, headers = urlopener.retrieve(url, filename, sampleReportHook, blocksize) + fname, headers = urlopener.retrieve( + url, filename, sampleReportHook, blocksize) print fname, 'written' diff --git a/Packages/cdms2/Lib/cdurlparse.py b/Packages/cdms2/Lib/cdurlparse.py index e743ebbc89..d88dd4087e 100644 --- a/Packages/cdms2/Lib/cdurlparse.py +++ b/Packages/cdms2/Lib/cdurlparse.py @@ -9,25 +9,26 @@ # A classification of schemes ('' means apply by default) uses_relative = ['ftp', 'http', 'ldap', 'gopher', 'nntp', 'wais', 'file', - 'https', 'shttp', - 'prospero', ''] + 'https', 'shttp', + 'prospero', ''] uses_netloc = ['ftp', 'http', 'ldap', 'gopher', 'nntp', 'telnet', 'wais', - 'file', - 'https', 'shttp', 'snews', - 'prospero', ''] + 'file', + 'https', 'shttp', 'snews', + 'prospero', ''] non_hierarchical = ['gopher', 'hdl', 'mailto', 'news', 'telnet', 'wais', - 'snews', - ] + 'snews', + ] uses_params = ['ftp', 'hdl', 'prospero', 'http', 'ldap', - 'https', 'shttp', - ''] + 'https', 'shttp', + ''] uses_query = ['http', 'ldap', 'wais', - 'https', 'shttp', - 'gopher', - ''] -uses_fragment = ['ftp', 'hdl', 'http', 'ldap', 'gopher', 'news', 'nntp', 'wais', - 'https', 'shttp', 'snews', - 'file', 'prospero', ''] + 'https', 'shttp', + 'gopher', + ''] +uses_fragment = [ + 'ftp', 'hdl', 'http', 'ldap', 'gopher', 'news', 'nntp', 'wais', + 'https', 'shttp', 'snews', + 'file', 'prospero', ''] # Characters valid in scheme names scheme_chars = string.letters + string.digits + '+-.' @@ -35,6 +36,7 @@ MAX_CACHE_SIZE = 20 _parse_cache = {} + def clear_cache(): """Clear the parse cache.""" global _parse_cache @@ -46,134 +48,141 @@ def clear_cache(): # Return a 6-tuple: (scheme, netloc, path, params, query, fragment). # Note that we don't break the components up in smaller bits # (e.g. netloc is a single string) and we don't expand % escapes. -def urlparse(url, scheme = '', allow_fragments = 1): - key = url, scheme, allow_fragments - cached = _parse_cache.get(key, None) - if cached: - return cached - if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth - clear_cache() - find = string.find - netloc = path = params = query = fragment = '' - i = find(url, ':') - if i > 0: - if url[:i] in ['http','ldap']: # optimize the common case - scheme = string.lower(url[:i]) - url = url[i+1:] - if url[:2] == '//': - i = find(url, '/', 2) - if i < 0: - i = len(url) - netloc = url[2:i] - url = url[i:] - if allow_fragments: - i = string.rfind(url, '#') - if i >= 0: - fragment = url[i+1:] - url = url[:i] - i = find(url, '?') - if i >= 0: - query = url[i+1:] - url = url[:i] - i = find(url, ';') - if i >= 0: - params = url[i+1:] - url = url[:i] - tuple = scheme, netloc, url, params, query, fragment - _parse_cache[key] = tuple - return tuple - for c in url[:i]: - if c not in scheme_chars: - break - else: - scheme, url = string.lower(url[:i]), url[i+1:] - if scheme in uses_netloc: - if url[:2] == '//': - i = find(url, '/', 2) - if i < 0: - i = len(url) - netloc, url = url[2:i], url[i:] - if allow_fragments and scheme in uses_fragment: - i = string.rfind(url, '#') - if i >= 0: - url, fragment = url[:i], url[i+1:] - if scheme in uses_query: - i = find(url, '?') - if i >= 0: - url, query = url[:i], url[i+1:] - if scheme in uses_params: - i = find(url, ';') - if i >= 0: - url, params = url[:i], url[i+1:] - tuple = scheme, netloc, url, params, query, fragment - _parse_cache[key] = tuple - return tuple +def urlparse(url, scheme='', allow_fragments=1): + key = url, scheme, allow_fragments + cached = _parse_cache.get(key, None) + if cached: + return cached + if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth + clear_cache() + find = string.find + netloc = path = params = query = fragment = '' + i = find(url, ':') + if i > 0: + if url[:i] in ['http', 'ldap']: # optimize the common case + scheme = string.lower(url[:i]) + url = url[i + 1:] + if url[:2] == '//': + i = find(url, '/', 2) + if i < 0: + i = len(url) + netloc = url[2:i] + url = url[i:] + if allow_fragments: + i = string.rfind(url, '#') + if i >= 0: + fragment = url[i + 1:] + url = url[:i] + i = find(url, '?') + if i >= 0: + query = url[i + 1:] + url = url[:i] + i = find(url, ';') + if i >= 0: + params = url[i + 1:] + url = url[:i] + tuple = scheme, netloc, url, params, query, fragment + _parse_cache[key] = tuple + return tuple + for c in url[:i]: + if c not in scheme_chars: + break + else: + scheme, url = string.lower(url[:i]), url[i + 1:] + if scheme in uses_netloc: + if url[:2] == '//': + i = find(url, '/', 2) + if i < 0: + i = len(url) + netloc, url = url[2:i], url[i:] + if allow_fragments and scheme in uses_fragment: + i = string.rfind(url, '#') + if i >= 0: + url, fragment = url[:i], url[i + 1:] + if scheme in uses_query: + i = find(url, '?') + if i >= 0: + url, query = url[:i], url[i + 1:] + if scheme in uses_params: + i = find(url, ';') + if i >= 0: + url, params = url[:i], url[i + 1:] + tuple = scheme, netloc, url, params, query, fragment + _parse_cache[key] = tuple + return tuple # Put a parsed URL back together again. This may result in a slightly # different, but equivalent URL, if the URL that was parsed originally # had redundant delimiters, e.g. a ? with an empty query (the draft # states that these are equivalent). -def urlunparse((scheme, netloc, url, params, query, fragment)): - if netloc or (scheme in uses_netloc and url[:2] == '//'): - if url[:1] != '/': url = '/' + url - url = '//' + (netloc or '') + url - if scheme: - url = scheme + ':' + url - if params: - url = url + ';' + params - if query: - url = url + '?' + query - if fragment: - url = url + '#' + fragment - return url + + +def urlunparse(xxx_todo_changeme): + (scheme, netloc, url, params, query, fragment) = xxx_todo_changeme + if netloc or (scheme in uses_netloc and url[:2] == '//'): + if url[:1] != '/': + url = '/' + url + url = '//' + (netloc or '') + url + if scheme: + url = scheme + ':' + url + if params: + url = url + ';' + params + if query: + url = url + '?' + query + if fragment: + url = url + '#' + fragment + return url # Join a base URL and a possibly relative URL to form an absolute # interpretation of the latter. -def urljoin(base, url, allow_fragments = 1): - if not base: - return url - bscheme, bnetloc, bpath, bparams, bquery, bfragment = \ - urlparse(base, '', allow_fragments) - scheme, netloc, path, params, query, fragment = \ - urlparse(url, bscheme, allow_fragments) - if scheme != bscheme or scheme not in uses_relative: - return urlunparse((scheme, netloc, path, - params, query, fragment)) - if scheme in uses_netloc: - if netloc: - return urlunparse((scheme, netloc, path, - params, query, fragment)) - netloc = bnetloc - if path[:1] == '/': - return urlunparse((scheme, netloc, path, - params, query, fragment)) - if not path: - return urlunparse((scheme, netloc, bpath, - params, query or bquery, fragment)) - i = rfind(bpath, '/') - if i >= 0: - path = bpath[:i] + '/' + path - segments = splitfields(path, '/') - if segments[-1] == '.': - segments[-1] = '' - while '.' in segments: - segments.remove('.') - while 1: - i = 1 - n = len(segments) - 1 - while i < n: - if segments[i] == '..' and segments[i-1]: - del segments[i-1:i+1] - break - i = i+1 - else: - break - if len(segments) == 2 and segments[1] == '..' and segments[0] == '': - segments[-1] = '' - elif len(segments) >= 2 and segments[-1] == '..': - segments[-2:] = [''] - return urlunparse((scheme, netloc, joinfields(segments, '/'), - params, query, fragment)) + + +def urljoin(base, url, allow_fragments=1): + if not base: + return url + bscheme, bnetloc, bpath, bparams, bquery, bfragment = \ + urlparse(base, '', allow_fragments) + scheme, netloc, path, params, query, fragment = \ + urlparse(url, bscheme, allow_fragments) + if scheme != bscheme or scheme not in uses_relative: + return urlunparse((scheme, netloc, path, + params, query, fragment)) + if scheme in uses_netloc: + if netloc: + return urlunparse((scheme, netloc, path, + params, query, fragment)) + netloc = bnetloc + if path[:1] == '/': + return urlunparse((scheme, netloc, path, + params, query, fragment)) + if not path: + return urlunparse((scheme, netloc, bpath, + params, query or bquery, fragment)) + i = rfind(bpath, '/') + if i >= 0: + path = bpath[:i] + '/' + path + segments = splitfields(path, '/') + if segments[-1] == '.': + segments[-1] = '' + while '.' in segments: + segments.remove('.') + while True: + i = 1 + n = len(segments) - 1 + while i < n: + if segments[i] == '..' and segments[i - 1]: + del segments[i - 1:i + 1] + break + i = i + 1 + else: + break + if len(segments) == 2 and segments[1] == '..' and segments[0] == '': + segments[-1] = '' + elif len(segments) >= 2 and segments[-1] == '..': + segments[-2:] = [''] + return urlunparse((scheme, netloc, joinfields(segments, '/'), + params, query, fragment)) + def urldefrag(url): """Removes any existing fragment from URL. @@ -222,35 +231,37 @@ def urldefrag(url): """ # XXX The result for //g is actually http://g/; is this a problem? + def test(): - import sys - base = '' - if sys.argv[1:]: - fn = sys.argv[1] - if fn == '-': - fp = sys.stdin - else: - fp = open(fn) - else: - import StringIO - fp = StringIO.StringIO(test_input) - while 1: - line = fp.readline() - if not line: break - words = string.split(line) - if not words: - continue - url = words[0] - parts = urlparse(url) - print '%-10s : %s' % (url, parts) - abs = urljoin(base, url) - if not base: - base = abs - wrapped = '' % abs - print '%-10s = %s' % (url, wrapped) - if len(words) == 3 and words[1] == '=': - if wrapped != words[2]: - print 'EXPECTED', words[2], '!!!!!!!!!!' + import sys + base = '' + if sys.argv[1:]: + fn = sys.argv[1] + if fn == '-': + fp = sys.stdin + else: + fp = open(fn) + else: + import StringIO + fp = StringIO.StringIO(test_input) + while True: + line = fp.readline() + if not line: + break + words = string.split(line) + if not words: + continue + url = words[0] + parts = urlparse(url) + print '%-10s : %s' % (url, parts) + abs = urljoin(base, url) + if not base: + base = abs + wrapped = '' % abs + print '%-10s = %s' % (url, wrapped) + if len(words) == 3 and words[1] == '=': + if wrapped != words[2]: + print 'EXPECTED', words[2], '!!!!!!!!!!' if __name__ == '__main__': - test() + test() diff --git a/Packages/cdms2/Lib/cdxmllib.py b/Packages/cdms2/Lib/cdxmllib.py index 7c1aff3082..70b91d9ba1 100644 --- a/Packages/cdms2/Lib/cdxmllib.py +++ b/Packages/cdms2/Lib/cdxmllib.py @@ -11,6 +11,7 @@ version = '0.3' + class Error(RuntimeError): pass @@ -20,7 +21,7 @@ class Error(RuntimeError): _opS = '[ \t\r\n]*' # optional white space _Name = '[a-zA-Z_:][-a-zA-Z0-9._:]*' # valid XML name _QStr = "(?:'[^']*'|\"[^\"]*\")" # quoted XML string -illegal = re.compile('[^\t\r\n -\176\240-\377]') # illegal chars in content +illegal = re.compile('[^\t\r\n -\176\240-\377]') # illegal chars in content interesting = re.compile('[]&<]') amp = re.compile('&') @@ -33,37 +34,37 @@ class Error(RuntimeError): attrfind = re.compile( _S + '(?P' + _Name + ')' '(' + _opS + '=' + _opS + - '(?P'+_QStr+'|[-a-zA-Z0-9.:+*%?!\(\)_#=~]+))?') + '(?P' + _QStr + '|[-a-zA-Z0-9.:+*%?!\(\)_#=~]+))?') starttagopen = re.compile('<' + _Name) starttagend = re.compile(_opS + '(?P/?)>') -starttagmatch = re.compile('<(?P'+_Name+')' - '(?P(?:'+attrfind.pattern+')*)'+ - starttagend.pattern) +starttagmatch = re.compile('<(?P' + _Name + ')' + '(?P(?:' + attrfind.pattern + ')*)' + + starttagend.pattern) endtagopen = re.compile('') -endbracketfind = re.compile('(?:[^>\'"]|'+_QStr+')*>') +endbracketfind = re.compile('(?:[^>\'"]|' + _QStr + ')*>') tagfind = re.compile(_Name) cdataopen = re.compile(r'') # this matches one of the following: # SYSTEM SystemLiteral # PUBLIC PubidLiteral SystemLiteral -_SystemLiteral = '(?P<%s>'+_QStr+')' +_SystemLiteral = '(?P<%s>' + _QStr + ')' _PublicLiteral = '(?P<%s>"[-\'\(\)+,./:=?;!*#@$_%% \n\ra-zA-Z0-9]*"|' \ - "'[-\(\)+,./:=?;!*#@$_%% \n\ra-zA-Z0-9]*')" + "'[-\(\)+,./:=?;!*#@$_%% \n\ra-zA-Z0-9]*')" _ExternalId = '(?:SYSTEM|' \ - 'PUBLIC'+_S+_PublicLiteral%'pubid'+ \ - ')'+_S+_SystemLiteral%'syslit' -doctype = re.compile(''+_Name+')' - '(?:'+_S+_ExternalId+')?'+_opS) -xmldecl = re.compile('<\?xml'+_S+ - 'version'+_opS+'='+_opS+'(?P'+_QStr+')'+ - '(?:'+_S+'encoding'+_opS+'='+_opS+ - "(?P'[A-Za-z][-A-Za-z0-9._]*'|" - '"[A-Za-z][-A-Za-z0-9._]*"))?' - '(?:'+_S+'standalone'+_opS+'='+_opS+ - '(?P\'(?:yes|no)\'|"(?:yes|no)"))?'+ - _opS+'\?>') + 'PUBLIC' + _S + _PublicLiteral % 'pubid' + \ + ')' + _S + _SystemLiteral % 'syslit' +doctype = re.compile('' + _Name + ')' + '(?:' + _S + _ExternalId + ')?' + _opS) +xmldecl = re.compile('<\?xml' + _S + + 'version' + _opS + '=' + _opS + '(?P' + _QStr + ')' + + '(?:' + _S + 'encoding' + _opS + '=' + _opS + + "(?P'[A-Za-z][-A-Za-z0-9._]*'|" + '"[A-Za-z][-A-Za-z0-9._]*"))?' + '(?:' + _S + 'standalone' + _opS + '=' + _opS + + '(?P\'(?:yes|no)\'|"(?:yes|no)"))?' + + _opS + '\?>') procopen = re.compile(r'<\?(?P' + _Name + ')' + _opS) procclose = re.compile(_opS + r'\?>') commentopen = re.compile(' tm1.shape = (1, :, :) tm1 = reshape(cvar, tuple([1] + list(cvar.shape))) # Attach needed items var = cdms2.createVariable(tm1, - axes = axes, - grid = grid, - attributes = atts, - id = cvar.standard_name) + axes=axes, + grid=grid, + attributes=atts, + id=cvar.standard_name) # Create cdms2 transient variable if firsttime: @@ -253,26 +268,31 @@ def createTransientVariableFromIndices(self, fileIndices, timeIndices): # insert the new time axis. taA = new.getTime() newTime = axisConcatenate((taA, timeAxis), - attributes = axisTime.attributes, - id = axisTime.id) - axes = self.buildAxes(newTime, self.fvs[file].getAxisList()) + attributes=axisTime.attributes, + id=axisTime.id) + axes = self.buildAxes( + newTime, + self.fvs[file].getAxisList()) tmp = MV2concatenate((new, var)) new = cdms2.createVariable(tmp, - axes = axes, - grid = grid, - attributes = atts, - id = cvar.standard_name) + axes=axes, + grid=grid, + attributes=atts, + id=cvar.standard_name) else: new = self.fvs[fileIndices][timeIndices] return new + class TimeFileVariable: + """ Construct an aggregated time dependant variable. """ + def __init__(self, hostObj, varName): """ Create a list of file variable with grid attached @@ -297,8 +317,12 @@ def __init__(self, hostObj, varName): # Open the files fn = hostObj.timeVars[varName][gridIndex][timeFileIndex] - f = cdms2.open(fn, mode) # Need f and u because they serve slightly different purposes - u = CdunifFile(fn, mode) # f.axes exists while axes is not a part of u + f = cdms2.open( + fn, + mode) # Need f and u because they serve slightly different purposes + u = CdunifFile( + fn, + mode) # f.axes exists while axes is not a part of u # u.variables[varName].gridIndex = gridIndex # Turn the coordinates into a list @@ -310,12 +334,14 @@ def __init__(self, hostObj, varName): coordsaux = f._convention_.getAxisAuxIds(u.variables, coords1d) # Convert the variable into a FileVariable - f.variables[varName] = FileVariable(f, varName, u.variables[varName]) + f.variables[varName] = FileVariable( + f, varName, u.variables[varName]) # Add the coordinates to the file for coord in coords: f.variables[coord] = g.variables[coord] - f.variables[coord] = FileAxis2D(f, coord, g.variables[coord]) + f.variables[coord] = FileAxis2D( + f, coord, g.variables[coord]) # Build the axes for key in f.axes.keys(): @@ -323,7 +349,8 @@ def __init__(self, hostObj, varName): # Set the boundaries for coord in coords: - bounds = f._convention_.getVariableBounds(f, f.variables[coord]) + bounds = f._convention_.getVariableBounds( + f, f.variables[coord]) f.variables[coord].setBounds(bounds) # Initialize the domain @@ -331,10 +358,16 @@ def __init__(self, hostObj, varName): var.initDomain(f.axes) # Add the grid - gridkey, lat, lon = f.variables[varName].generateGridkey(f._convention_, f.variables) + gridkey, lat, lon = f.variables[ + varName].generateGridkey(f._convention_, f.variables) gridname = ("grid%d_" % gridIndex) + "%dx%d" % lat.shape -# grid = FileGenericGrid(lat, lon, gridname, parent = f, maskvar = None) - grid = FileCurveGrid(lat, lon, gridname, parent = f, maskvar = None) +# grid = FileGenericGrid(lat, lon, gridname, parent = f, maskvar = None) + grid = FileCurveGrid( + lat, + lon, + gridname, + parent=f, + maskvar=None) f.variables[varName]._grid_ = grid vars.append(f.variables[varName]) @@ -343,23 +376,23 @@ def __init__(self, hostObj, varName): self._repr_string = "TimeFileVariable" - def listall(self, all = None): + def listall(self, all=None): """ Gain access to cdms2 listall method. Requires a TimeFileVariable @param all @returns list """ - return self[0][0].listall(all = all) + return self[0][0].listall(all=all) - def showall(self, all = None, device = None): + def showall(self, all=None, device=None): """ Gain access to cdms2 showall method Requires a TimeFileVariable @param all - @param device + @param device @returns list """ - return self[0][0][:].showall(all = all, device = device) + return self[0][0][:].showall(all=all, device=device) def __getitem__(self, gridIndex): """ @@ -367,11 +400,13 @@ def __getitem__(self, gridIndex): """ return self.vars[gridIndex] -############################################################################### -############## DEPRECIATED - Testing required to fully remove ################# -############################################################################### +# +# DEPRECIATED - Testing required to fully remove ################# +# + class TimeTransientVariable: + def __init__(self, hostObj, varName, **slicekwargs): """ Constructor @@ -402,15 +437,18 @@ def __init__(self, hostObj, varName, **slicekwargs): for timeFileIndex in range(hostObj.nTimeDataFiles): - fName = hostObj.timeDepVars[varName][gridIndex][timeFileIndex] + fName = hostObj.timeDepVars[ + varName][ + gridIndex][ + timeFileIndex] fh = cdms2.open(fName, hostObj=hostObj) # TransientVariable var = fh(varName, **slicekwargs) # Attach the grid to the variable - grid = cdms2.gsStaticVariable.createTransientGrid(gFName, \ - var.attributes['coordinates']) + grid = cdms2.gsStaticVariable.createTransientGrid(gFName, + var.attributes['coordinates']) axis0 = var.getAxis(0) gridaxes = grid.getAxisList() axes = [axis0] + list(gridaxes) @@ -420,21 +458,21 @@ def __init__(self, hostObj, varName, **slicekwargs): # Create cdms2 transient variable if timeFileIndex == 0: new = cdms2.createVariable(var, - axes = axes, - grid = grid, - attributes = atts, - id = var.standard_name) + axes=axes, + grid=grid, + attributes=atts, + id=var.standard_name) else: - tmp =MV2concatenate((new, var)) + tmp = MV2concatenate((new, var)) axis0 = tmp.getAxis(0) gridaxes = grid.getAxisList() axes = [axis0, gridaxes[0], gridaxes[1]] # new.append(tmp) new = cdms2.createVariable(tmp, - axes = axes, - grid = grid, - attributes = atts, - id = var.standard_name) + axes=axes, + grid=grid, + attributes=atts, + id=var.standard_name) fh.close() # Add the variable to the index @@ -443,10 +481,10 @@ def __init__(self, hostObj, varName, **slicekwargs): self._repr_string = "TimeTransientVariable" -################################################################### +# def test(): pass -if __name__ == '__main__': test() - +if __name__ == '__main__': + test() diff --git a/Packages/cdms2/Lib/gui.py b/Packages/cdms2/Lib/gui.py index 65e13cd8c3..44fab4cb69 100644 --- a/Packages/cdms2/Lib/gui.py +++ b/Packages/cdms2/Lib/gui.py @@ -5,6 +5,7 @@ _progressParent = None # Parent frame of progress gui + def setProgressParent(parent): """ Enable the FTP progress GUI, and set the parent frame. @@ -12,7 +13,9 @@ def setProgressParent(parent): """ global _progressParent _progressParent = parent - cache.useWindow() # Notify cache module that window dialogs should be used. + cache.useWindow() + # Notify cache module that window dialogs should be used. + def getProgressParent(): """ @@ -21,6 +24,7 @@ def getProgressParent(): """ return _progressParent + def updateProgressGui(blocknum, blocksize, size, prog): """ Callback function for the FTP progress dialog. @@ -31,25 +35,28 @@ def updateProgressGui(blocknum, blocksize, size, prog): Return: 0 to signal that a cancel has been received, 1 to continue reading. """ - sizekb = size/1024L - percent = min(100,int(100.0*float(blocknum*blocksize)/float(size))) - if percent<100: - noInterrupt = prog.Update(percent,"Read: %3d%% of %dK"%(percent,sizekb)) + sizekb = size / 1024 + percent = min(100, int(100.0 * float(blocknum * blocksize) / float(size))) + if percent < 100: + noInterrupt = prog.Update( + percent, "Read: %3d%% of %dK" % + (percent, sizekb)) else: noInterrupt = 1 # Don't interrupt - finish up cleanly prog.Destroy() - if noInterrupt==0: + if noInterrupt == 0: prog.Destroy() return noInterrupt + class CdProgressDialog(wxProgressDialog): # is the parent frame. # filename is the file being read. - def __init__(self, frame, filename): - wxProgressDialog.__init__(self,"FTP: %s"%filename, - "Connecting ...", - 100, - frame, - wxPD_CAN_ABORT | wxPD_APP_MODAL | wxPD_REMAINING_TIME) + def __init__(self, frame, filename): + wxProgressDialog.__init__(self, "FTP: %s" % filename, + "Connecting ...", + 100, + frame, + wxPD_CAN_ABORT | wxPD_APP_MODAL | wxPD_REMAINING_TIME) diff --git a/Packages/cdms2/Lib/hgrid.py b/Packages/cdms2/Lib/hgrid.py index 1f53d37026..aa09ca8c5b 100644 --- a/Packages/cdms2/Lib/hgrid.py +++ b/Packages/cdms2/Lib/hgrid.py @@ -1,5 +1,5 @@ -## Automatically adapted for numpy.oldnumeric Aug 01, 2007 by -## Further modified to be pure new numpy June 24th 2008 +# Automatically adapted for numpy.oldnumeric Aug 01, 2007 by +# Further modified to be pure new numpy June 24th 2008 """CDMS HorizontalGrid objects""" @@ -7,19 +7,20 @@ import cdms2 import os import os.path -## import PropertiedClasses -from error import CDMSError -from grid import AbstractGrid, LongitudeType, LatitudeType, VerticalType, TimeType, CoordTypeToLoc -from coord import TransientVirtualAxis -from axis import getAutoBounds, allclose -import bindex,_bindex +# import PropertiedClasses +from .error import CDMSError +from .grid import AbstractGrid, LongitudeType, LatitudeType, VerticalType, TimeType, CoordTypeToLoc +from .coord import TransientVirtualAxis +from .axis import getAutoBounds, allclose +import bindex, _bindex +from functools import reduce MethodNotImplemented = "Method not yet implemented" def _flatten(boundsar): boundsshape = boundsar.shape - if len(boundsshape)>2: - newshape = (reduce((lambda x,y: x*y), boundsshape[:-1], 1), boundsshape[-1]) + if len(boundsshape) > 2: + newshape = (reduce((lambda x, y: x*y), boundsshape[:-1], 1), boundsshape[-1]) boundsar.shape = newshape return boundsar @@ -40,17 +41,17 @@ def __init__(self, latAxis, lonAxis, id=None, maskvar=None, tempmask=None, node= # Generate default bounds def genBounds(self): - raise CDMSError, MethodNotImplemented + raise CDMSError(MethodNotImplemented) # Get the n-th axis. naxis is 0 or 1. def getAxis(self, naxis): - raise CDMSError, MethodNotImplemented + raise CDMSError(MethodNotImplemented) def getBounds(self): """Get the grid cell boundaries, as a tuple (latitudeBounds, longitudeBounds) """ latbnds, lonbnds = (self._lataxis_.getExplicitBounds(), self._lonaxis_.getExplicitBounds()) - if (latbnds is None or lonbnds is None) and getAutoBounds() in [1,2]: + if (latbnds is None or lonbnds is None) and getAutoBounds() in [1, 2]: nlatbnds, nlonbnds = self.genBounds() if latbnds is None: latbnds = nlatbnds @@ -76,27 +77,27 @@ def getMask(self): def getMesh(self): """Get the mesh array used by the meshfill plot.""" - raise CDMSError, MethodNotImplemented + raise CDMSError(MethodNotImplemented) def getWeightsArray(self): """Return normalized area weights, as an array of the same shape as the grid. """ - raise CDMSError, MethodNotImplemented + raise CDMSError(MethodNotImplemented) def listall (self, all=None): - result=[] + result = [] result.append('Grid has Python id %s.' % hex(id(self))) return result - def setMask(self,mask,permanent=0): + def setMask(self, mask, permanent=0): self._maskVar_ = mask def subGridRegion(self, latRegion, lonRegion): - raise CDMSError, MethodNotImplemented + raise CDMSError(MethodNotImplemented) def hasCoordType(self, coordType): - return ((coordType==LatitudeType) or (coordType==LongitudeType)) + return ((coordType == LatitudeType) or (coordType == LongitudeType)) def checkConvex(self): """Check that each cell of the grid is convex in lon-lat space, with nodes defined counter-clockwise. @@ -116,10 +117,10 @@ def checkConvex(self): for n0 in range(nnode): n1 = (n0+1)%nnode n2 = (n1+1)%nnode - vec0lon = lonb[:,n1] - lonb[:,n0] - vec0lat = latb[:,n1] - latb[:,n0] - vec1lon = lonb[:,n2] - lonb[:,n1] - vec1lat = latb[:,n2] - latb[:,n1] + vec0lon = lonb[:, n1] - lonb[:, n0] + vec0lat = latb[:, n1] - latb[:, n0] + vec1lon = lonb[:, n2] - lonb[:, n1] + vec1lat = latb[:, n2] - latb[:, n1] cross = vec0lon*vec1lat - vec0lat*vec1lon mask = where(less(cross, 0.0), 1, 0) @@ -170,24 +171,24 @@ def fixCutCells(self, nonConvexCells, threshold=270.0): for node in range(2*nnode): n0 = node%nnode n1 = (n0+1)%nnode - vec0lon = lonb2[k,n1]-lonb2[k,n0] - if vec0lon>threshold: - lonb2[k,n1] -= 360.0 - elif vec0lon<-threshold: - lonb2[k,n1] += 360.0 + vec0lon = lonb2[k, n1]-lonb2[k, n0] + if vec0lon > threshold: + lonb2[k, n1] -= 360.0 + elif vec0lon < -threshold: + lonb2[k, n1] += 360.0 # If the cross-product test still fails, restore # the original values and add to the nonConvexCells list for n0 in range(nnode): n1 = (n0+1)%nnode n2 = (n1+1)%nnode - vec0lon = lonb2[k,n1] - lonb2[k,n0] - vec0lat = latb2[k,n1] - latb2[k,n0] - vec1lon = lonb2[k,n2] - lonb2[k,n1] - vec1lat = latb2[k,n2] - latb2[k,n1] + vec0lon = lonb2[k, n1] - lonb2[k, n0] + vec0lat = latb2[k, n1] - latb2[k, n0] + vec1lon = lonb2[k, n2] - lonb2[k, n1] + vec1lat = latb2[k, n2] - latb2[k, n1] cross = vec0lon*vec1lat - vec0lat*vec1lon - if cross<0: + if cross < 0: lonb2[k] = savelons newbadcells.append(nonConvexCells[k]) break @@ -207,7 +208,7 @@ def __init__(self, latAxis, lonAxis, id=None, maskvar=None, tempmask=None, node= """Create a curvilinear grid. """ if latAxis.shape != lonAxis.shape: - raise CDMSError, 'Latitude and longitude axes must have the same shape.' + raise CDMSError('Latitude and longitude axes must have the same shape.') AbstractHorizontalGrid.__init__(self, latAxis, lonAxis, id, maskvar, tempmask, node) self._index_ = None @@ -217,7 +218,7 @@ def clone(self, copyData=1): return TransientCurveGrid(newlat, newlon, id=self.id) def __repr__(self): - return ""%(self.id, `self.shape`) + return ""%(self.id, repr(self.shape)) __str__ = __repr__ def getMesh(self, transpose=None): @@ -226,20 +227,20 @@ def getMesh(self, transpose=None): latbounds and lonbounds according to the tuple, (1,0,2) in this case. """ if self._mesh_ is None: - LAT=0 - LON=1 + LAT = 0 + LON = 1 latbounds, lonbounds = self.getBounds() -## ## following work aronud a numpy.ma bug -## latbounds=latbounds.filled() -## lonbounds=lonbounds.filled() +# following work aronud a numpy.ma bug +# latbounds=latbounds.filled() +# lonbounds=lonbounds.filled() if latbounds is None or lonbounds is None: - raise CDMSError, 'No boundary data is available for grid %s'%self.id - if (transpose is not None) and (transpose[1]==0): - latbounds = numpy.transpose(latbounds, (1,0,2)) - lonbounds = numpy.transpose(lonbounds, (1,0,2)) - mesh = numpy.zeros((self.size(),2,latbounds.shape[-1]),latbounds.dtype.char) - mesh[:,LAT,:] = numpy.reshape(latbounds,(self.size(),latbounds.shape[-1])) - mesh[:,LON,:] = numpy.reshape(lonbounds,(self.size(),latbounds.shape[-1])) + raise CDMSError('No boundary data is available for grid %s'%self.id) + if (transpose is not None) and (transpose[1] == 0): + latbounds = numpy.transpose(latbounds, (1, 0, 2)) + lonbounds = numpy.transpose(lonbounds, (1, 0, 2)) + mesh = numpy.zeros((self.size(), 2, latbounds.shape[-1]), latbounds.dtype.char) + mesh[:, LAT,:] = numpy.reshape(latbounds, (self.size(), latbounds.shape[-1])) + mesh[:, LON,:] = numpy.reshape(lonbounds, (self.size(), latbounds.shape[-1])) self._mesh_ = mesh return self._mesh_ @@ -291,9 +292,9 @@ def writeScrip(self, cufile, gridTitle=None): mask.shape = (ngrid,) clat = numpy.ma.filled(copy.copy(blat)) - clat.shape = (ngrid,4) + clat.shape = (ngrid, 4) clon = numpy.ma.filled(copy.copy(blon)) - clon.shape = (ngrid,4) + clon.shape = (ngrid, 4) # Write the file if gridTitle is None: @@ -309,12 +310,12 @@ def writeScrip(self, cufile, gridTitle=None): gridcenterlon.units = "degrees" gridimask = cufile.createVariable("grid_imask", 'i', ("grid_size",)) gridimask.units = "unitless" - gridcornerlat = cufile.createVariable("grid_corner_lat", 'd', ("grid_size","grid_corners")) + gridcornerlat = cufile.createVariable("grid_corner_lat", 'd', ("grid_size", "grid_corners")) gridcornerlat.units = "degrees" - gridcornerlon = cufile.createVariable("grid_corner_lon", 'd', ("grid_size","grid_corners")) + gridcornerlon = cufile.createVariable("grid_corner_lon", 'd', ("grid_size", "grid_corners")) gridcornerlon.units = "degrees" - griddims[:] = numpy.array([nj,ni], numpy.int32) + griddims[:] = numpy.array([nj, ni], numpy.int32) gridcenterlat[:] = centerLat gridcenterlon[:] = centerLon gridimask[:] = mask @@ -324,9 +325,9 @@ def writeScrip(self, cufile, gridTitle=None): def toGenericGrid(self, gridid=None): import copy - from auxcoord import TransientAuxAxis1D - from coord import TransientVirtualAxis - from gengrid import TransientGenericGrid + from .auxcoord import TransientAuxAxis1D + from .coord import TransientVirtualAxis + from .gengrid import TransientGenericGrid lat = numpy.ma.filled(self._lataxis_) latunits = self._lataxis_.units @@ -345,11 +346,11 @@ def toGenericGrid(self, gridid=None): mask.shape = (ngrid,) cornerLat = numpy.ma.filled(copy.copy(blat)) - cornerLat.shape = (ngrid,4) + cornerLat.shape = (ngrid, 4) cornerLon = numpy.ma.filled(copy.copy(blon)) - cornerLon.shape = (ngrid,4) + cornerLon.shape = (ngrid, 4) - iaxis = TransientVirtualAxis("cell",ngrid) + iaxis = TransientVirtualAxis("cell", ngrid) lataxis = TransientAuxAxis1D(centerLat, axes=(iaxis,), bounds=cornerLat, attributes={'units':latunits}, id="latitude") @@ -381,21 +382,21 @@ def writeg( self, file ): The file, normally a CdmsFile, should already be open for writing and will be closed.""" import time - from tvariable import TransientVariable + from .tvariable import TransientVariable # Set attributes - if ( hasattr(file,'Conventions') ): - if ( file.Conventions.find('Gridspec')<0 ): + if ( hasattr(file, 'Conventions') ): + if ( file.Conventions.find('Gridspec') < 0 ): file.Conventions = file.Conventions + ' Gridspec-0.0' else: file.Conventions = 'Gridspec-0.0' - if ( hasattr(file,'gs_filetypes') ): - if ( file.gs_filetypes.find('Curvilinear_Tile')<0 ): + if ( hasattr(file, 'gs_filetypes') ): + if ( file.gs_filetypes.find('Curvilinear_Tile') < 0 ): file.gs_filetypes = file.gs_filetypes + ' Curvilinear_Tile' else: file.gs_filetypes = 'Curvilinear_Tile' - t=time.time() - id=int((t-int(t))*1.0e9) + t = time.time() + id = int((t-int(t))*1.0e9) file.gs_id = id file.gs_originalfilename = os.path.basename( file.id ) @@ -411,19 +412,19 @@ def writeg( self, file ): getattr( file, 'history', '' ) + newhistory # former tile variable and attributes - if ( hasattr(self,'long_name') and self.long_name!=None ): + if ( hasattr(self, 'long_name') and self.long_name != None ): file.long_name = self.long_name else: file.long_name = 'gridspec_tile' # gs_geometryType is no longer required of Gridspec files, but as yet # there is no other proposal for describing the geometry (July 2010) - if ( hasattr(self,'gs_geometryType') and self.gs_geometryType!=None): + if ( hasattr(self, 'gs_geometryType') and self.gs_geometryType != None): file.gs_geometryType = self.gs_geometryType else: file.gs_geometryType = 'spherical' # gs_discretizationType is no longer required of Gridspec files, but it's # harmless and may come in useful - if ( hasattr(self,'gs_discretizationType') and self.gs_discretizationType!=None ): + if ( hasattr(self, 'gs_discretizationType') and self.gs_discretizationType != None ): file.gs_discretizationType = self.gs_discretizationType else: file.gs_discretizationType = 'logically_rectangular' @@ -434,11 +435,11 @@ def writeg( self, file ): # Set up and write variables. When written, cdms writes not only the arrays # but also their coordinates, e.g. gs_nip. - x=self._lonaxis_ - if ( not hasattr(x,'units') ): + x = self._lonaxis_ + if ( not hasattr(x, 'units') ): print "Warning, no units found for longitude" x.units = 'degree_east' - if ( not hasattr(x,'standard_name') ): + if ( not hasattr(x, 'standard_name') ): print "Warning, no standard_name found for longitude axis" x.standard_name = 'longitude' if ( x.standard_name == 'geographic_longitude'): @@ -448,11 +449,11 @@ def writeg( self, file ): # _lonaxis_ is a TransientAxis2D, hence a TransientVariable # But I don't know where the attribute _TransientVariable__domain comes from - y=self._lataxis_ - if ( not hasattr(y,'units') ): + y = self._lataxis_ + if ( not hasattr(y, 'units') ): print "Warning, no units found for latitude" y.units = 'degree_north' - if ( not hasattr(y,'standard_name') ): + if ( not hasattr(y, 'standard_name') ): print "Warning, no standard_name found for latitude axis" y.standard_name = 'latitude' if ( y.standard_name == 'geographic_latitude'): @@ -460,18 +461,18 @@ def writeg( self, file ): y.standard_name = 'latitude' y.id = file.gs_latv - if( not hasattr(x,'_TransientVariable__domain') ): + if( not hasattr(x, '_TransientVariable__domain') ): # There probably doesn't exist enough information to write a correct # grid, but this will help. x._TransientVariable__domain = [ (x,), (y,) ] - x._TransientVariable__domain[0][0].id='gs_njp' - x._TransientVariable__domain[1][0].id='gs_nip' - if ( not hasattr(y,'_TransientVariable__domain') ) : + x._TransientVariable__domain[0][0].id = 'gs_njp' + x._TransientVariable__domain[1][0].id = 'gs_nip' + if ( not hasattr(y, '_TransientVariable__domain') ) : # There probably doesn't exist enough information to write a correct # grid, but this will help. y._TransientVariable__domain = [ (x,), (y,) ] - y._TransientVariable__domain[0][0].id='gs_njp' - y._TransientVariable__domain[1][0].id='gs_nip' + y._TransientVariable__domain[0][0].id = 'gs_njp' + y._TransientVariable__domain[1][0].id = 'gs_nip' file.write(x) file.write(y) @@ -486,12 +487,12 @@ def write_gridspec( self, filename ): # The functionality (other than checking gsfile) is now done by the writeg # method above. if ( not hasattr( self, "gsfile" ) ): - self.gsfile=None - self.gspath=None - if ( self.gsfile!=None ): + self.gsfile = None + self.gspath = None + if ( self.gsfile != None ): return ( tcg.gsfile, tcg.gspath ) else: - raise RuntimeError, 'The libCF/Gridspec API does not provide for writing CurveGrids<<<' + raise RuntimeError('The libCF/Gridspec API does not provide for writing CurveGrids<<<') def init_from_gridspec( self, filename ): """reads to grid from a Gridspec-compliant file. The filename should be a @@ -575,8 +576,8 @@ def getGridSlices(self, domainlist, newaxislist, slicelist): j = k k += 1 - if i==-1 or j==-1: - raise RuntimeError, 'Grid lat/lon domains do not match variable domain' + if i == -1 or j == -1: + raise RuntimeError('Grid lat/lon domains do not match variable domain') return ((islice, jslice), (inewaxis, jnewaxis)) @@ -584,14 +585,14 @@ def getIndex(self): """Get the grid index""" if self._index_ is None: # Trying to stick in Stephane Raynaud's patch for autodetection - nj,ni = self._lataxis_.shape + nj, ni = self._lataxis_.shape dlon = numpy.max(self._lonaxis_)-numpy.min(self._lonaxis_) - dx = max(dlon/ni,dlon/nj) + dx = max(dlon/ni, dlon/nj) dlat = numpy.max(self._lataxis_)-numpy.min(self._lataxis_) - dy = max(dlat/ni,dlat/nj) + dy = max(dlat/ni, dlat/nj) latlin = numpy.ravel(numpy.ma.filled(self._lataxis_)) lonlin = numpy.ravel(numpy.ma.filled(self._lonaxis_)) - _bindex.setDeltas(dx,dy) + _bindex.setDeltas(dx, dy) self._index_ = bindex.bindexHorizontalGrid(latlin, lonlin) return self._index_ @@ -614,12 +615,12 @@ def intersect(self, spec): latlin = numpy.ravel(numpy.ma.filled(self._lataxis_)) lonlin = numpy.ravel(numpy.ma.filled(self._lonaxis_)) points = bindex.intersectHorizontalGrid(latspec, lonspec, latlin, lonlin, index) - if len(points)==0: - raise CDMSError, 'No data in the specified region, longitude=%s, latitude=%s'%(`lonspec`, `latspec`) + if len(points) == 0: + raise CDMSError('No data in the specified region, longitude=%s, latitude=%s'%(repr(lonspec), repr(latspec))) fullmask = numpy.ones(ni*nj) numpy.put(fullmask, points, 0) - fullmask = numpy.reshape(fullmask, (ni,nj)) + fullmask = numpy.reshape(fullmask, (ni, nj)) iind = points/nj jind = points - iind*nj @@ -628,7 +629,7 @@ def intersect(self, spec): yid = self.getAxis(0).id xid = self.getAxis(1).id - indexspecs = {yid:slice(imin,imax), xid:slice(jmin,jmax)} + indexspecs = {yid:slice(imin, imax), xid:slice(jmin, jmax)} return submask, indexspecs @@ -677,9 +678,9 @@ def reconcile(self, axes): used = [] # axes already matched for i in missing: for item in axes: - if (item not in used) and len(selfaxes[i])==len(item) and allclose(selfaxes[i], item): - result._lataxis_.setAxis(i,item) - result._lonaxis_.setAxis(i,item) + if (item not in used) and len(selfaxes[i]) == len(item) and allclose(selfaxes[i], item): + result._lataxis_.setAxis(i, item) + result._lonaxis_.setAxis(i, item) used.append(item) break else: @@ -693,18 +694,18 @@ def flatAxes(self): having the same length as the number of cells in the grid, similarly for flatlon.""" if self._flataxes_ is None: - import MV2 as MV + from . import MV2 as MV alat = MV.filled(self.getLatitude()) alon = MV.filled(self.getLongitude()) alatflat = numpy.ravel(alat) alonflat = numpy.ravel(alon) self._flataxes_ = (alatflat, alonflat) return self._flataxes_ - shape = property(_getShape,None) + shape = property(_getShape, None) -## PropertiedClasses.set_property (AbstractCurveGrid, 'shape', -## AbstractCurveGrid._getShape, nowrite=1, -## nodelete=1) +# PropertiedClasses.set_property (AbstractCurveGrid, 'shape', +# AbstractCurveGrid._getShape, nowrite=1, +# nodelete=1) class DatasetCurveGrid(AbstractCurveGrid): @@ -715,7 +716,7 @@ def __init__(self, latAxis, lonAxis, id, parent=None, maskvar=None, tempmask=Non self.parent = parent def __repr__(self): - return ""%(self.id, `self.shape`) + return ""%(self.id, repr(self.shape)) class FileCurveGrid(AbstractCurveGrid): @@ -726,7 +727,7 @@ def __init__(self, latAxis, lonAxis, id, parent=None, maskvar=None, tempmask=Non self.parent = parent def __repr__(self): - return ""%(self.id, `self.shape`) + return ""%(self.id, repr(self.shape)) class TransientCurveGrid(AbstractCurveGrid): @@ -741,7 +742,7 @@ def __init__(self, latAxis, lonAxis, id=None, maskvar=None, tempmask=None): AbstractCurveGrid.__init__(self, latAxis, lonAxis, id, maskvar, tempmask) def __repr__(self): - return ""%(self.id, `self.shape`) + return ""%(self.id, repr(self.shape)) def toCurveGrid(self, gridid=None): if gridid is None: @@ -758,17 +759,17 @@ def readScripCurveGrid(fileobj, dims, whichType, whichGrid): whichType is the type of file, either "grid" or "mapping" if whichType is "mapping", whichGrid is the choice of grid, either "source" or "destination" """ - from coord import TransientAxis2D + from .coord import TransientAxis2D if 'S' in fileobj.variables.keys(): - if whichType=="grid": + if whichType == "grid": gridCornerLatName = 'grid_corner_lat' gridCornerLonName = 'grid_corner_lon' gridMaskName = 'grid_imask' gridCenterLatName = 'grid_center_lat' gridCenterLonName = 'grid_center_lon' titleName = 'title' - elif whichGrid=="destination": + elif whichGrid == "destination": gridCornerLatName = 'yv_b' gridCornerLonName = 'xv_b' gridMaskName = 'mask_b' @@ -783,14 +784,14 @@ def readScripCurveGrid(fileobj, dims, whichType, whichGrid): gridCenterLonName = 'xc_a' titleName = 'source_grid' else: - if whichType=="grid": + if whichType == "grid": gridCornerLatName = 'grid_corner_lat' gridCornerLonName = 'grid_corner_lon' gridMaskName = 'grid_imask' gridCenterLatName = 'grid_center_lat' gridCenterLonName = 'grid_center_lon' titleName = 'title' - elif whichGrid=="destination": + elif whichGrid == "destination": gridCornerLatName = 'dst_grid_corner_lat' gridCornerLonName = 'dst_grid_corner_lon' gridMaskName = 'dst_grid_imask' @@ -813,17 +814,17 @@ def readScripCurveGrid(fileobj, dims, whichType, whichGrid): nj = dims[0] gridshape = (ni, nj) boundsshape = (ni, nj, ncorners) - if hasattr(cornerLat, 'units') and cornerLat.units.lower()[0:6]=='radian': + if hasattr(cornerLat, 'units') and cornerLat.units.lower()[0:6] == 'radian': cornerLat = (cornerLat*(180.0/numpy.pi)).reshape(boundsshape) cornerLon = (cornerLon*(180.0/numpy.pi)).reshape(boundsshape) else: cornerLat = cornerLat.reshape(boundsshape) cornerLon = cornerLon.reshape(boundsshape) - iaxis = TransientVirtualAxis("i",ni) - jaxis = TransientVirtualAxis("j",nj) + iaxis = TransientVirtualAxis("i", ni) + jaxis = TransientVirtualAxis("j", nj) - if vardict.has_key(gridMaskName): + if gridMaskName in vardict: # SCRIP convention: 0 for invalid data # numpy.ma convention: 1 for invalid data mask = 1 - fileobj(gridMaskName) @@ -831,27 +832,27 @@ def readScripCurveGrid(fileobj, dims, whichType, whichGrid): else: mask = None - if vardict.has_key(gridCenterLatName): + if gridCenterLatName in vardict: centerLat = fileobj(gridCenterLatName).reshape(gridshape) gclat = fileobj[gridCenterLatName] - if hasattr(gclat, "units") and gclat.units.lower()=='radians': + if hasattr(gclat, "units") and gclat.units.lower() == 'radians': centerLat *= (180.0/numpy.pi) else: - centerLat = cornerLat[:,:,0] + centerLat = cornerLat[:,:, 0] - if vardict.has_key(gridCenterLonName): + if gridCenterLonName in vardict: centerLon = fileobj(gridCenterLonName).reshape(gridshape) gclon = fileobj[gridCenterLonName] - if hasattr(gclon, "units") and gclon.units.lower()=='radians': + if hasattr(gclon, "units") and gclon.units.lower() == 'radians': centerLon *= (180.0/numpy.pi) else: - centerLon = cornerLon[:,:,0] + centerLon = cornerLon[:,:, 0] - if hasattr(fileobj,titleName): + if hasattr(fileobj, titleName): gridid = getattr(fileobj, titleName) - gridid = gridid.strip().replace(' ','_') + gridid = gridid.strip().replace(' ', '_') else: - gridid="" + gridid = "" lataxis = TransientAxis2D(centerLat, axes=(iaxis, jaxis), bounds=cornerLat, attributes={'units':'degrees_north'}, id="latitude") diff --git a/Packages/cdms2/Lib/mvBaseWriter.py b/Packages/cdms2/Lib/mvBaseWriter.py index 5f099c704b..db80b843e3 100644 --- a/Packages/cdms2/Lib/mvBaseWriter.py +++ b/Packages/cdms2/Lib/mvBaseWriter.py @@ -3,13 +3,14 @@ """ Abstract class for writing data into file -This code is provided with the hope that it will be useful. +This code is provided with the hope that it will be useful. No guarantee is provided whatsoever. Use at your own risk. Alex Pletzer, Tech-X Corp. (2011) """ -import mvSphereMesh +from . import mvSphereMesh + class BaseWriter: @@ -17,7 +18,7 @@ def __init__(self, var, sphereRadius=1.0, maxElev=0.1): """ Constructor @param var a cdms2 variable - @param sphereRadius radius of the sphere upon which the grid will + @param sphereRadius radius of the sphere upon which the grid will be projected @param maxElev max elevation/depth normalized to the sphere radius """ @@ -26,12 +27,12 @@ def __init__(self, var, sphereRadius=1.0, maxElev=0.1): self.shape = sphere_mesh.shape - # there is currently a bug in vizSchema which causes - # visit to crash if the leading index is 1, this is + # there is currently a bug in vizSchema which causes + # visit to crash if the leading index is 1, this is # a workaround the problem if self.shape[0] == 1: - self.shape = list(sphere_mesh.shape[1:]) + [1,] - + self.shape = list(sphere_mesh.shape[1:]) + [1, ] + self.mesh = sphere_mesh.getXYZCoords(sphereRadius) def write(self, filename): @@ -39,5 +40,5 @@ def write(self, filename): Write data to file. This method is overloaded. @param filename file name """ - raise NotImplementedError, \ - 'write method not implemented in derived class' + raise NotImplementedError( + 'write method not implemented in derived class') diff --git a/Packages/cdms2/Lib/mvCdmsRegrid.py b/Packages/cdms2/Lib/mvCdmsRegrid.py index 5a1f34361e..24c169acfe 100644 --- a/Packages/cdms2/Lib/mvCdmsRegrid.py +++ b/Packages/cdms2/Lib/mvCdmsRegrid.py @@ -9,8 +9,9 @@ import re import numpy import cdms2 -from error import CDMSError +from .error import CDMSError import regrid2 +from functools import reduce def _areCellsOk(cornerCoords, mask=None): """ @@ -43,8 +44,8 @@ def projectToSphere(the, lam): # compute area elements in Cartesian space lat0 = numpy.array(cornerCoords[0][ :-1, :-1], numpy.float64) lat1 = numpy.array(cornerCoords[0][ :-1, 1: ], numpy.float64) - lat2 = numpy.array(cornerCoords[0][1: , 1: ], numpy.float64) - lat3 = numpy.array(cornerCoords[0][1: , :-1], numpy.float64) + lat2 = numpy.array(cornerCoords[0][1:, 1: ], numpy.float64) + lat3 = numpy.array(cornerCoords[0][1:, :-1], numpy.float64) the0 = lat0*numpy.pi/180. the1 = lat1*numpy.pi/180. @@ -52,8 +53,8 @@ def projectToSphere(the, lam): the3 = lat3*numpy.pi/180. lam0 = numpy.array(cornerCoords[1][ :-1, :-1], numpy.float64)*numpy.pi/180. lam1 = numpy.array(cornerCoords[1][ :-1, 1: ], numpy.float64)*numpy.pi/180. - lam2 = numpy.array(cornerCoords[1][1: , 1: ], numpy.float64)*numpy.pi/180. - lam3 = numpy.array(cornerCoords[1][1: , :-1], numpy.float64)*numpy.pi/180. + lam2 = numpy.array(cornerCoords[1][1:, 1: ], numpy.float64)*numpy.pi/180. + lam3 = numpy.array(cornerCoords[1][1:, :-1], numpy.float64)*numpy.pi/180. x0, y0, z0 = projectToSphere(the0, lam0) x1, y1, z1 = projectToSphere(the1, lam1) @@ -128,7 +129,7 @@ def projectToSphere(the, lam): if len(inds[0]) > 0: # package the result badCellIndices = [(inds[0][i], inds[1][i]) for i in range(len(inds[0]))] - bcis1 = [(inds[0][i] , inds[1][i]+1) for i in range(len(inds[0]))] + bcis1 = [(inds[0][i], inds[1][i]+1) for i in range(len(inds[0]))] bcis2 = [(inds[0][i]+1, inds[1][i]+1) for i in range(len(inds[0]))] bcis3 = [(inds[0][i]+1, inds[1][i] ) for i in range(len(inds[0]))] badCellCoords = [[(cornerCoords[0][badCellIndices[i]], cornerCoords[1][badCellIndices[i]]), @@ -159,10 +160,10 @@ def _buildBounds(bounds): bnd[:-1] = bounds[..., 0] bnd[ -1] = bounds[ -1, 1] elif len(bndShape) > 1: - bnd[:-1, :-1] = bounds[ :, :, 0] - bnd[:-1, -1] = bounds[ :, -1, 1] + bnd[:-1, :-1] = bounds[:,:, 0] + bnd[:-1, -1] = bounds[:, -1, 1] bnd[ -1, -1] = bounds[ -1, -1, 2] - bnd[ -1, :-1] = bounds[ -1, :, 3] + bnd[ -1, :-1] = bounds[ -1,:, 3] return bnd @@ -381,8 +382,9 @@ def __init__(self, srcGrid, dstGrid, dtype, # If LibCF handleCut is True, the bounds are needed to extend the grid # close the cut at the top - if re.search('LibCF', regridTool, re.I) and args.has_key('handleCut'): - if args['handleCut']: srcBounds = getBoundList(srcCoords) + if re.search('LibCF', regridTool, re.I) and 'handleCut' in args: + if args['handleCut']: + srcBounds = getBoundList(srcCoords) srcCoordsArrays = [numpy.array(sc) for sc in srcCoords] dstCoordsArrays = [numpy.array(dc) for dc in dstCoords] @@ -439,7 +441,7 @@ def __call__(self, srcVar, **args): **args) # fill in diagnostic data - if args.has_key('diag'): + if 'diag' in args: self.regridObj.fillInDiagnosticData(diag = args['diag'], rootPe = 0) # construct the axis list for dstVar diff --git a/Packages/cdms2/Lib/mvSphereMesh.py b/Packages/cdms2/Lib/mvSphereMesh.py index 7b95922e29..1a5469c9cc 100644 --- a/Packages/cdms2/Lib/mvSphereMesh.py +++ b/Packages/cdms2/Lib/mvSphereMesh.py @@ -3,22 +3,24 @@ """ Class for representing grids on the sphere Alex Pletzer, Tech-X Corp. (2011) -This code is provided with the hope that it will be useful. +This code is provided with the hope that it will be useful. No guarantee is provided whatsoever. Use at your own risk. """ import numpy +from functools import reduce + class SphereMesh: - + def __init__(self, var, sphereThickness=0.1): """ Constructor @param var cdms2 variable - @param sphereThickness thickness of the shell in normalized + @param sphereThickness thickness of the shell in normalized sphere radius """ - + self.isRectilinear = True self.ndims = 0 self.elvPositiveDown = False @@ -40,7 +42,7 @@ def __init__(self, var, sphereThickness=0.1): if getattr(elvs, 'positive') == 'down': self.elvPositiveDown = True - # determine the dimensionality and + # determine the dimensionality and # whether the grid is rectilinear for axis in lons, lats, elvs: if axis is not None: @@ -53,25 +55,31 @@ def __init__(self, var, sphereThickness=0.1): self.shape = [] for axis in lons, lats, elvs: if axis is not None: - self.shape.append( len(axis) ) + self.shape.append(len(axis)) self.shape.reverse() while len(self.shape) < 3: - self.shape = [1,] + list(self.shape) + self.shape = [1, ] + list(self.shape) # store lon, lat, elv as a curvilinear grid if self.isRectilinear: # apply tensore product of axes to generat curvilinear coordinates if elvs is not None: - self.elvs = numpy.outer(numpy.outer( numpy.ones(self.shape[:0], numpy.float32), elvs), - numpy.ones(self.shape[0+1:], numpy.float32)).reshape(self.shape) + self.elvs = numpy.outer( + numpy.outer( + numpy.ones(self.shape[:0], + numpy.float32), + elvs), + numpy.ones(self.shape[0 + 1:], numpy.float32)).reshape(self.shape) else: - self.elvs = numpy.zeros( self.shape, numpy.float32 ) - self.lats = numpy.outer(numpy.outer( numpy.ones(self.shape[:1], numpy.float32), lats), - numpy.ones(self.shape[1+1:], numpy.float32)).reshape(self.shape) - self.lons = numpy.outer(numpy.outer( numpy.ones(self.shape[:2], numpy.float32), lons), - numpy.ones(self.shape[2+1:], numpy.float32)).reshape(self.shape) - + self.elvs = numpy.zeros(self.shape, numpy.float32) + self.lats = numpy.outer( + numpy.outer(numpy.ones(self.shape[:1], numpy.float32), lats), + numpy.ones(self.shape[1 + 1:], numpy.float32)).reshape(self.shape) + self.lons = numpy.outer( + numpy.outer(numpy.ones(self.shape[:2], numpy.float32), lons), + numpy.ones(self.shape[2 + 1:], numpy.float32)).reshape(self.shape) + else: # already in curvilinear form self.lons = lons[:] @@ -79,10 +87,10 @@ def __init__(self, var, sphereThickness=0.1): if elvs is not None: self.elvs = elvs[:] else: - self.elvs = numpy.zeros( self.shape, numpy.float32 ) + self.elvs = numpy.zeros(self.shape, numpy.float32) # reshape as flat arrays - sz = reduce(lambda x, y: x*y, self.shape) + sz = reduce(lambda x, y: x * y, self.shape) self.lons = numpy.reshape(self.lons, (sz,)) self.lats = numpy.reshape(self.lats, (sz,)) self.elvs = numpy.reshape(self.elvs, (sz,)) @@ -90,32 +98,33 @@ def __init__(self, var, sphereThickness=0.1): def getXYZCoords(self, sphereRadius=1.0): """ Get the curvilinear cartesian coordinates - @param sphereRadius radius of sphere + @param sphereRadius radius of sphere @return mesh """ - sz = reduce(lambda x, y: x*y, self.shape) - rr = sphereRadius*(1.0 + self.elvs) + sz = reduce(lambda x, y: x * y, self.shape) + rr = sphereRadius * (1.0 + self.elvs) diffElv = self.maxElv - self.minElv - rr = sphereRadius*numpy.ones(self.lons.shape, numpy.float32 ) + rr = sphereRadius * numpy.ones(self.lons.shape, numpy.float32) if diffElv != 0: - coeff = sphereRadius*self.sphereThickness/diffElv + coeff = sphereRadius * self.sphereThickness / diffElv if self.elvPositiveDown: # depth - rr += coeff*(self.maxElv - self.elvs) + rr += coeff * (self.maxElv - self.elvs) else: # height - rr += coeff*(self.elvs - self.minElv) + rr += coeff * (self.elvs - self.minElv) - mesh = numpy.zeros( (sz, 3), numpy.float32 ) - cosLats = numpy.cos( self.lats*numpy.pi/180. ) - mesh[:, 0] = rr*numpy.cos(self.lons*numpy.pi/180.)*cosLats - mesh[:, 1] = rr*numpy.sin(self.lons*numpy.pi/180.)*cosLats - mesh[:, 2] = rr*numpy.sin(self.lats*numpy.pi/180.) + mesh = numpy.zeros((sz, 3), numpy.float32) + cosLats = numpy.cos(self.lats * numpy.pi / 180.) + mesh[:, 0] = rr * numpy.cos(self.lons * numpy.pi / 180.) * cosLats + mesh[:, 1] = rr * numpy.sin(self.lons * numpy.pi / 180.) * cosLats + mesh[:, 2] = rr * numpy.sin(self.lats * numpy.pi / 180.) return mesh -##################################################################### +# # Tests + def test2DRect(): """ Test data on 2D rectilinear grid @@ -123,17 +132,18 @@ def test2DRect(): import cdms2 from numpy import pi, cos, sin nlat, nlon = 12, 15 - grid = cdms2.createUniformGrid(-0.0, nlat, 60./(nlat-1), - 0., nlon, 30./nlon) + grid = cdms2.createUniformGrid(-0.0, nlat, 60. / (nlat - 1), + 0., nlon, 30. / nlon) lons = grid.getLongitude() lats = grid.getLatitude() - data = numpy.outer(cos(3*pi*lats[:]/180.0), - sin(5*pi*lons[:]/180.0)) - var = cdms2.createVariable(data, id='fake_data_2d_rect', + data = numpy.outer(cos(3 * pi * lats[:] / 180.0), + sin(5 * pi * lons[:] / 180.0)) + var = cdms2.createVariable(data, id='fake_data_2d_rect', axes=(lats, lons)) sphere_mesh = SphereMesh(var, 0.1) print sphere_mesh.getXYZCoords() + def test2D(): """ Test data on 2D curvilinear grid @@ -143,33 +153,34 @@ def test2D(): from cdms2.hgrid import TransientCurveGrid from numpy import pi, cos, sin nlat, nlon = 3, 4 - dlon, dlat = 60.0/float(nlon - 1), 30.0/float(nlat - 1) - lons1D = numpy.array([0.0 + i*dlon for i in range(nlon)]) - lats1D = numpy.array([0.0 + j*dlat for j in range(nlat)]) + dlon, dlat = 60.0 / float(nlon - 1), 30.0 / float(nlat - 1) + lons1D = numpy.array([0.0 + i * dlon for i in range(nlon)]) + lats1D = numpy.array([0.0 + j * dlat for j in range(nlat)]) lons = numpy.outer(numpy.ones((nlat,)), lons1D) lats = numpy.outer(lats1D, numpy.ones((nlon,))) - data = cos(3*pi*lats/180.0) * sin(5*pi*lons/180.0) + data = cos(3 * pi * lats / 180.0) * sin(5 * pi * lons / 180.0) # create grid iaxis = TransientVirtualAxis("i", nlon) jaxis = TransientVirtualAxis("j", nlat) - lataxis = TransientAxis2D(lats, - axes=(jaxis, iaxis), - attributes={'units': 'degree_north'}, - id='lats') - lonaxis = TransientAxis2D(lons, - axes=(jaxis, iaxis), - attributes={'units': 'degree_east'}, - id='lons') - grid = TransientCurveGrid(lataxis, lonaxis, id='lats_lons') - - var = cdms2.createVariable(data, id='fake_data_2d', - axes = grid.getAxisList(), - grid = grid, - attributes = {'coordinates': 'lats lons'}, + lataxis = TransientAxis2D(lats, + axes=(jaxis, iaxis), + attributes={'units': 'degree_north'}, + id='lats') + lonaxis = TransientAxis2D(lons, + axes=(jaxis, iaxis), + attributes={'units': 'degree_east'}, + id='lons') + grid = TransientCurveGrid(lataxis, lonaxis, id='lats_lons') + + var = cdms2.createVariable(data, id='fake_data_2d', + axes=grid.getAxisList(), + grid=grid, + attributes={'coordinates': 'lats lons'}, ) sphere_mesh = SphereMesh(var) print sphere_mesh.getXYZCoords() + def test3DRect(): """ Test data on 3d rectilinear grid @@ -177,29 +188,30 @@ def test3DRect(): import cdms2 from numpy import pi, cos, sin, exp nelv, nlat, nlon = 3, 4, 5 - delv, dlon, dlat = 90000./float(nelv-1), \ - 60.0/float(nlon-1), 30.0/float(nlat-1) - elvs1D = numpy.array([100000 - i*delv for i in range(nelv)]) - lons1D = numpy.array([0.0 + i*dlon for i in range(nlon)]) - lats1D = numpy.array([0.0 + i*dlat for i in range(nlat)]) + delv, dlon, dlat = 90000. / float(nelv - 1), \ + 60.0 / float(nlon - 1), 30.0 / float(nlat - 1) + elvs1D = numpy.array([100000 - i * delv for i in range(nelv)]) + lons1D = numpy.array([0.0 + i * dlon for i in range(nlon)]) + lats1D = numpy.array([0.0 + i * dlat for i in range(nlat)]) # any order should work - lons = numpy.zeros( (nlon, nlat, nelv), numpy.float32 ) - lats = numpy.zeros( (nlon, nlat, nelv), numpy.float32 ) - elvs = numpy.zeros( (nlon, nlat, nelv), numpy.float32 ) - data = numpy.zeros( (nlon, nlat, nelv), numpy.float32 ) + lons = numpy.zeros((nlon, nlat, nelv), numpy.float32) + lats = numpy.zeros((nlon, nlat, nelv), numpy.float32) + elvs = numpy.zeros((nlon, nlat, nelv), numpy.float32) + data = numpy.zeros((nlon, nlat, nelv), numpy.float32) for i in range(nlon): for j in range(nlat): for k in range(nelv): elvs[i, j, k] = elvs1D[k] lats[i, j, k] = lats1D[j] lons[i, j, k] = lons1D[i] - data[i, j, k] = cos(3*pi*lats[i, j, k]/180.) * \ - sin(5*pi*lons[i, j, k]/180.) * exp(-elvs[i, j, k]) - var = cdms2.createVariable(data, id='fake_data_3d_rect', + data[i, j, k] = cos(3 * pi * lats[i, j, k] / 180.) * \ + sin(5 * pi * lons[i, j, k] / 180.) * exp(-elvs[i, j, k]) + var = cdms2.createVariable(data, id='fake_data_3d_rect', axes=(elvs, lats, lons)) sphere_mesh = SphereMesh(var) print sphereMesh.getXYZCoords() + def test3DposDown(): """ Test 3d data with elev positive down. Need to work with 1D axes. @@ -208,31 +220,33 @@ def test3DposDown(): import cdms2 import numpy nlev, nlat, nlon = 4, 5, 6 - dlev, dlat, dlon = 5000./float(nlev-1), 180./float(nlat-1), 360./float(nlon-1) + dlev, dlat, dlon = 5000. / \ + float(nlev - 1), 180. / float(nlat - 1), 360. / float(nlon - 1) levs1d = numpy.arange(0., 5001., dlev) - lats1d = numpy.array([0. - i*dlat for i in range(nlat)]) - lons1d = numpy.array([0. - i*dlon for i in range(nlon)]) + lats1d = numpy.array([0. - i * dlat for i in range(nlat)]) + lons1d = numpy.array([0. - i * dlon for i in range(nlon)]) data = numpy.zeros((nlev, nlat, nlon), numpy.float32) for k in range(nlev): for j in range(nlat): for i in range(nlon): - data[k, j, i] = numpy.cos(3*numpy.pi*lats1d[j]/180.) * \ - numpy.sin(5*numpy.pi*lons1d[i]/180.) * \ + data[k, j, i] = numpy.cos(3 * numpy.pi * lats1d[j] / 180.) * \ + numpy.sin(5 * numpy.pi * lons1d[i] / 180.) * \ numpy.exp(-levs1d[k]) - a1 = cdms2.axis.TransientAxis(levs1d, id = 'levels', - attributes = {'positive':'down'}) - a2 = cdms2.axis.TransientAxis(lats1d, id = 'latitude') - a3 = cdms2.axis.TransientAxis(lons1d, id = 'longitude') - var = cdms2.createVariable(data, id = 'pos_down_3d_data', - axes = (a1, a2, a3)) + a1 = cdms2.axis.TransientAxis(levs1d, id='levels', + attributes={'positive': 'down'}) + a2 = cdms2.axis.TransientAxis(lats1d, id='latitude') + a3 = cdms2.axis.TransientAxis(lons1d, id='longitude') + var = cdms2.createVariable(data, id='pos_down_3d_data', + axes=(a1, a2, a3)) sphereMesh = SphereMesh(var) aa = sphereMesh.getXYZCoords() bb = aa.reshape((4, 5, 6, 3)) - for i in range(nlev): print levs1d[i], bb[i, 0, 0, :] + for i in range(nlev): + print levs1d[i], bb[i, 0, 0, :] -if __name__ == '__main__': +if __name__ == '__main__': # test2DRect() # test2D() # test3DRect() diff --git a/Packages/cdms2/Lib/mvVTKSGWriter.py b/Packages/cdms2/Lib/mvVTKSGWriter.py index 353d8b19bb..fd23272459 100644 --- a/Packages/cdms2/Lib/mvVTKSGWriter.py +++ b/Packages/cdms2/Lib/mvVTKSGWriter.py @@ -3,13 +3,14 @@ """ Write data to VTK file format using the structured grid format Alex Pletzer, Tech-X Corp. (2011) -This code is provided with the hope that it will be useful. +This code is provided with the hope that it will be useful. No guarantee is provided whatsoever. Use at your own risk. """ import numpy import time -import mvBaseWriter +from . import mvBaseWriter + class VTKSGWriter(mvBaseWriter.BaseWriter): @@ -29,10 +30,10 @@ def write(self, filename): npts = self.mesh.shape[0] print >> f, 'POINTS %d float' % npts for i in range(npts): - print >> f, '%f %f %f' % tuple(self.mesh[i,:]) + print >> f, '%f %f %f' % tuple(self.mesh[i, :]) n0, n1, n2 = self.shape # nodal data - print >> f, 'POINT_DATA %d' % (n0*n1*n2) + print >> f, 'POINT_DATA %d' % (n0 * n1 * n2) print >> f, 'SCALARS %s float' % (self.var.id) print >> f, 'LOOKUP_TABLE default' if n0 > 1: @@ -43,34 +44,34 @@ def write(self, filename): else: for j in range(n1): for i in range(n2): - print >> f, '%f' % self.var[j, i] + print >> f, '%f' % self.var[j, i] f.close() -###################################################################### +# def test2DRect(): import cdms2 from numpy import pi, cos, sin nlat, nlon = 6, 10 - grid = cdms2.createUniformGrid(-0.0, nlat, 60./(nlat-1), - 0., nlon, 30./nlon) + grid = cdms2.createUniformGrid(-0.0, nlat, 60. / (nlat - 1), + 0., nlon, 30. / nlon) lons = grid.getLongitude() lats = grid.getLatitude() - data = numpy.outer(cos(3*pi*lats[:]/180.0), - sin(5*pi*lons[:]/180.0)) - var = cdms2.createVariable(data, id='fake_data_2d_rect', + data = numpy.outer(cos(3 * pi * lats[:] / 180.0), + sin(5 * pi * lons[:] / 180.0)) + var = cdms2.createVariable(data, id='fake_data_2d_rect', axes=(lats, lons)) vw = VTKSGWriter(var) vw.write('test2DRect_SG.vtk') + def test3D(): import cdms2 var = cdms2.open('sample_data/ta_ncep_87-6-88-4.nc', 'r')('ta') - vw = VTKSGWriter(var[0,0:10,0:20,0:30]) + vw = VTKSGWriter(var[0, 0:10, 0:20, 0:30]) vw.write('test3D_SG.vtk') -if __name__ == '__main__': +if __name__ == '__main__': test2DRect() test3D() - diff --git a/Packages/cdms2/Lib/mvVTKUGWriter.py b/Packages/cdms2/Lib/mvVTKUGWriter.py index 50dcb11952..e712efaf7e 100644 --- a/Packages/cdms2/Lib/mvVTKUGWriter.py +++ b/Packages/cdms2/Lib/mvVTKUGWriter.py @@ -3,13 +3,14 @@ """ Write data to VTK file format using the unstructured grid format Alex Pletzer, Tech-X Corp. (2011) -This code is provided with the hope that it will be useful. +This code is provided with the hope that it will be useful. No guarantee is provided whatsoever. Use at your own risk. """ import numpy import time -import mvBaseWriter +from . import mvBaseWriter + class VTKUGWriter(mvBaseWriter.BaseWriter): @@ -26,9 +27,9 @@ def write(self, filename): npts = self.mesh.shape[0] print >> f, 'POINTS %d float' % npts for i in range(npts): - print >> f, '%f %f %f' % tuple(self.mesh[i,:]) + print >> f, '%f %f %f' % tuple(self.mesh[i, :]) n0, n1, n2 = self.shape - ncells = (n0 - 1)*(n1 - 1)*(n2 - 1) + ncells = (n0 - 1) * (n1 - 1) * (n2 - 1) if ncells != 0: # 3d ntot = ncells * (8 + 1) @@ -36,17 +37,17 @@ def write(self, filename): for k in range(n0 - 1): for j in range(n1 - 1): for i in range(n2 - 1): - index = i + n2*(j + n1*k) + index = i + n2 * (j + n1 * k) print >> f, '8 %d %d %d %d %d %d %d %d' % \ - (index, index+1, index+1+n2, index+n2, - index+n1*n2, index+n1*n2+1, - index+n1*n2+1+n2, index+n1*n2+n2) + (index, index + 1, index + 1 + n2, index + n2, + index + n1 * n2, index + n1 * n2 + 1, + index + n1 * n2 + 1 + n2, index + n1 * n2 + n2) print >> f, 'CELL_TYPES %d' % ncells for i in range(ncells): # hexahedron print >> f, 12 # nodal data - print >> f, 'POINT_DATA %d' % (n0*n1*n2) + print >> f, 'POINT_DATA %d' % (n0 * n1 * n2) print >> f, 'SCALARS %s float' % (self.var.id) print >> f, 'LOOKUP_TABLE default' for k in range(n0): @@ -55,52 +56,52 @@ def write(self, filename): print >> f, '%f' % self.var[k, j, i] else: # 2d - ncells = (n1 - 1)*(n2 - 1) + ncells = (n1 - 1) * (n2 - 1) ntot = ncells * (4 + 1) print >> f, 'CELLS %d %d' % (ncells, ntot) for j in range(n1 - 1): for i in range(n2 - 1): - index = i + n2*j + index = i + n2 * j print >> f, '4 %d %d %d %d' % \ - (index, index+1, index+1+n2, index+n2) + (index, index + 1, index + 1 + n2, index + n2) print >> f, 'CELL_TYPES %d' % ncells for i in range(ncells): # quad print >> f, 9 # nodal data - print >> f, 'POINT_DATA %d' % (n0*n1*n2) + print >> f, 'POINT_DATA %d' % (n0 * n1 * n2) print >> f, 'SCALARS %s float' % (self.var.id) print >> f, 'LOOKUP_TABLE default' for j in range(n1): for i in range(n2): - print >> f, '%f' % self.var[j, i] + print >> f, '%f' % self.var[j, i] f.close() -###################################################################### +# def test2DRect(): import cdms2 from numpy import pi, cos, sin nlat, nlon = 6, 10 - grid = cdms2.createUniformGrid(-0.0, nlat, 60./(nlat-1), - 0., nlon, 30./nlon) + grid = cdms2.createUniformGrid(-0.0, nlat, 60. / (nlat - 1), + 0., nlon, 30. / nlon) lons = grid.getLongitude() lats = grid.getLatitude() - data = numpy.outer(cos(3*pi*lats[:]/180.0), - sin(5*pi*lons[:]/180.0)) - var = cdms2.createVariable(data, id='fake_data_2d_rect', + data = numpy.outer(cos(3 * pi * lats[:] / 180.0), + sin(5 * pi * lons[:] / 180.0)) + var = cdms2.createVariable(data, id='fake_data_2d_rect', axes=(lats, lons)) vw = VTKUGWriter(var) vw.write('test2DRect.vtk') + def test3D(): import cdms2 var = cdms2.open('sample_data/ta_ncep_87-6-88-4.nc', 'r')('ta') - vw = VTKUGWriter(var[0,0:10,0:20,0:30]) + vw = VTKUGWriter(var[0, 0:10, 0:20, 0:30]) vw.write('test3D.vtk') -if __name__ == '__main__': +if __name__ == '__main__': test2DRect() test3D() - diff --git a/Packages/cdms2/Lib/mvVsWriter.py b/Packages/cdms2/Lib/mvVsWriter.py index aed81a79ff..72bb315264 100644 --- a/Packages/cdms2/Lib/mvVsWriter.py +++ b/Packages/cdms2/Lib/mvVsWriter.py @@ -3,14 +3,15 @@ """ Write data to VizSchema compliant file Alex Pletzer, Tech-X Corp. (2011) -This code is provided with the hope that it will be useful. +This code is provided with the hope that it will be useful. No guarantee is provided whatsoever. Use at your own risk. """ import numpy -import mvBaseWriter +from . import mvBaseWriter import re + class VsWriter(mvBaseWriter.BaseWriter): def write(self, filename): @@ -21,58 +22,59 @@ def write(self, filename): try: import tables except: - raise ImportError, 'You must have pytables installed' - + raise ImportError('You must have pytables installed') + if filename.find('.vsh5') < 0 and filename.find('.h5') < 0: - filename += '.vsh5' # VizSchema hdf5 format + filename += '.vsh5' # VizSchema hdf5 format # open file h5file = tables.openFile(filename, 'w') # put mesh meshid = 'mesh_' + self.var.id - mdata = numpy.reshape(self.mesh, self.shape + [3,]) + mdata = numpy.reshape(self.mesh, self.shape + [3, ]) mset = h5file.createArray("/", meshid, mdata) mset.attrs.vsType = "mesh" mset.attrs.vsKind = "structured" mset.attrs.vsIndexOrder = "compMinorC" # data - dset = h5file.createArray("/", self.var.id, + dset = h5file.createArray("/", self.var.id, numpy.reshape(self.var, self.shape)) dset.attrs.vsType = "variable" dset.attrs.vsMesh = meshid # additional attributes for a in self.var.attributes: # Skip mpi objects - if re.match('mpi',a.lower()): + if re.match('mpi', a.lower()): continue setattr(dset.attrs, a, getattr(self.var, a)) # close file h5file.close() -###################################################################### +# + def test2DRect(): import cdms2 from numpy import pi, cos, sin nlat, nlon = 3, 4 - grid = cdms2.createUniformGrid(-0.0, nlat, 60./(nlat-1), - 0., nlon, 30./nlon) + grid = cdms2.createUniformGrid(-0.0, nlat, 60. / (nlat - 1), + 0., nlon, 30. / nlon) lons = grid.getLongitude() lats = grid.getLatitude() - data = numpy.outer(cos(3*pi*lats[:]/180.0), - sin(5*pi*lons[:]/180.0)) - var = cdms2.createVariable(data, id='fake_data_2d_rect', + data = numpy.outer(cos(3 * pi * lats[:] / 180.0), + sin(5 * pi * lons[:] / 180.0)) + var = cdms2.createVariable(data, id='fake_data_2d_rect', axes=(lats, lons)) vw = VsWriter(var) vw.write('test2DRect.vsh5') + def test3D(): import cdms2 var = cdms2.open('sample_data/ta_ncep_87-6-88-4.nc', 'r')('ta') - vw = VsWriter(var[0,0:10,0:20,0:30]) + vw = VsWriter(var[0, 0:10, 0:20, 0:30]) vw.write('test3D.vsh5') -if __name__ == '__main__': +if __name__ == '__main__': test2DRect() test3D() - diff --git a/Packages/cdms2/Lib/restApi.py b/Packages/cdms2/Lib/restApi.py index 2b5ef10595..0ee17424da 100644 --- a/Packages/cdms2/Lib/restApi.py +++ b/Packages/cdms2/Lib/restApi.py @@ -6,116 +6,136 @@ except: pass import os -#import bz2 +# import bz2 + class esgfConnectionException(Exception): pass + + class esgfDatasetException(Exception): pass + + class esgfFilesException(Exception): pass - ## def __init__(self,value): - ## self.value=value - ## def __repr__(self): - ## msg = "rest API error: %s" % repr(value) - ## print msg - ## return msg + # def __init__(self,value): + # self.value=value + # def __repr__(self): + # msg = "rest API error: %s" % repr(value) + # print msg + # return msg + class FacetConnection(object): - def __init__(self,host='pcmdi9.llnl.gov'): - self.rqst="http://%s/esg-search/search?facets=*&type=Dataset&limit=1&latest=true" % host - self.rqst_count="http://%s/esg-search/search?facets=*&type=File&limit=0&latest=true" % host + + def __init__(self, host='pcmdi9.llnl.gov'): + self.rqst = "http://%s/esg-search/search?facets=*&type=Dataset&limit=1&latest=true" % host + self.rqst_count = "http://%s/esg-search/search?facets=*&type=File&limit=0&latest=true" % host self.EsgfObjectException = esgfConnectionException - def get_xmlelement(self,facet_param=None): + + def get_xmlelement(self, facet_param=None): try: - rqst=self.rqst + rqst = self.rqst if facet_param: - rqst=rqst+'&%s'%facet_param - #print rqst + rqst = rqst + '&%s' % facet_param + # print rqst url = urllib2.urlopen(rqst) - except Exception,msg: - raise self.EsgfObjectException(msg) + except Exception as msg: + raise self.EsgfObjectException(msg) r = url.read() try: e = xml.etree.ElementTree.fromstring(r) return e - except Exception,err: - raise self.EsgfObjectException("Could not interpret server's results: %s" % err) - def make_facet_dict(self,xmlelement): - facet_dict={} + except Exception as err: + raise self.EsgfObjectException( + "Could not interpret server's results: %s" % + err) + + def make_facet_dict(self, xmlelement): + facet_dict = {} for lst in xmlelement.findall('lst'): - if lst.get('name')=='facet_counts': - myelement=lst + if lst.get('name') == 'facet_counts': + myelement = lst for node in myelement.findall('lst'): - if node.get('name')=='facet_fields': + if node.get('name') == 'facet_fields': for child in node.getchildren(): - facet_name=child.get('name') - facet_dict[facet_name]=[] + facet_name = child.get('name') + facet_dict[facet_name] = [] for grandchild in child.getchildren(): - facet_dict[facet_name].append("%s (%s)"%(str(grandchild.get('name')),str(grandchild.text))) + facet_dict[facet_name].append( + "%s (%s)" % + (str(grandchild.get('name')), str(grandchild.text))) return facet_dict - def get_xmlelement_count(self,facet_param=None): + + def get_xmlelement_count(self, facet_param=None): try: - rqst=self.rqst_count + rqst = self.rqst_count if facet_param: - rqst=rqst+'&%s'%facet_param - #print rqst + rqst = rqst + '&%s' % facet_param + # print rqst url = urllib2.urlopen(rqst) - except Exception,msg: - raise self.EsgfObjectException(msg) + except Exception as msg: + raise self.EsgfObjectException(msg) r = url.read() try: e = xml.etree.ElementTree.fromstring(r) return e - except Exception,err: - raise self.EsgfObjectException("Could not interpret server's results: %s" % err) + except Exception as err: + raise self.EsgfObjectException( + "Could not interpret server's results: %s" % + err) - def make_facet_dict_count(self,xmlelement): - myelementlist=xmlelement.findall('result') - count=None + def make_facet_dict_count(self, xmlelement): + myelementlist = xmlelement.findall('result') + count = None if len(myelementlist) > 0: - myelement=myelementlist[0] - count=int(myelement.get('numFound')) + myelement = myelementlist[0] + count = int(myelement.get('numFound')) return count -validSearchTypes = ["Dataset","File"]#"ById","ByTimeStamp"] +validSearchTypes = ["Dataset", "File"] # "ById","ByTimeStamp"] + + class esgfConnection(object): - def __init__(self,host,port=80,timeout=15,limit=None,offset=0,mapping=None,datasetids=None,fileids=None,restPath=None): - self.port=port - url=str(host).replace("://","^^^---^^^") - sp= url.split("/") - host = sp[0].replace("^^^---^^^","://") + + def __init__(self, host, port=80, timeout=15, limit=None, offset=0, + mapping=None, datasetids=None, fileids=None, restPath=None): + self.port = port + url = str(host).replace("://", "^^^---^^^") + sp = url.split("/") + host = sp[0].replace("^^^---^^^", "://") if restPath is None: restPath = "/".join(sp[1:]) - if len(restPath)==0: - self.restPath="/esg-search/search" + if len(restPath) == 0: + self.restPath = "/esg-search/search" else: - self.restPath=restPath + self.restPath = restPath else: - self.restPath=restPath - self.host=host - #self.host="esg-datanode.jpl.nasa.gov" + self.restPath = restPath + self.host = host + # self.host="esg-datanode.jpl.nasa.gov" self.defaultSearchType = "Dataset" self.EsgfObjectException = esgfConnectionException - self.validSearchTypes=validSearchTypes - self.validSearchTypes=["Dataset",] - all = self._search("facets=*",searchType=None) - ## Now figure out the facet fields + self.validSearchTypes = validSearchTypes + self.validSearchTypes = ["Dataset", ] + all = self._search("facets=*", searchType=None) + # Now figure out the facet fields self.serverOrder = [] for e in all: - if e.tag=="lst" and "name" in e.keys() and e.get("name")=="responseHeader": - ## ok found the Header + if e.tag == "lst" and "name" in e.keys() and e.get("name") == "responseHeader": + # ok found the Header for s in e: - if s.get("name")=="params": - params=s + if s.get("name") == "params": + params = s break - self.params={"text":None,"limit":limit,"offset":offset} - self.searchableKeys=set(["text","limit","offset"]) + self.params = {"text": None, "limit": limit, "offset": offset} + self.searchableKeys = set(["text", "limit", "offset"]) for p in params: - if p.get("name")=="facet.field": + if p.get("name") == "facet.field": for f in p: self.serverOrder.append(f.text) - self.params[f.text]=None + self.params[f.text] = None self.searchableKeys.add(f.text) self.keys = self.params.keys @@ -123,61 +143,68 @@ def __init__(self,host,port=80,timeout=15,limit=None,offset=0,mapping=None,datas self.values = self.params.values if datasetids is not None: - self.datasetids=genutil.StringConstructor(datasetids) + self.datasetids = genutil.StringConstructor(datasetids) else: - self.datasetids=None + self.datasetids = None if fileids is not None: - self.fileids=genutil.StringConstructor(fileids) + self.fileids = genutil.StringConstructor(fileids) if datasetids is not None: - self.fileids.template=self.fileids.template.replace("%(datasetid)",self.datasetids.template) + self.fileids.template = self.fileids.template.replace( + "%(datasetid)", self.datasetids.template) elif self.datasetids is not None: - self.fileids=genutil.StringConstructor("%s.%%(filename)" % self.datasetids.template) + self.fileids = genutil.StringConstructor( + "%s.%%(filename)" % + self.datasetids.template) else: - self.fileids=None - #self.setMapping(mapping) - self.mapping=mapping - - ## def setUserOrder(self,value): - ## self.userOrder=value - ## def getUserOrder(self): - ## return self.userOrder - ## order=property(getUserOrder,setUserOrder) - def __getitem__(self,key): + self.fileids = None + # self.setMapping(mapping) + self.mapping = mapping + + # def setUserOrder(self,value): + # self.userOrder=value + # def getUserOrder(self): + # return self.userOrder + # order=property(getUserOrder,setUserOrder) + def __getitem__(self, key): try: val = self.params[key] except: raise self.EsgfObjectException("Invalid key: %s" % repr(key)) return val - def __setitem__(self,key,value): + + def __setitem__(self, key, value): if not key in self.params.keys(): - raise self.EsgfObjectException("Invalid key: %s, valid keys are: %s" % (repr(key),repr(self.params.keys()))) - self.params[key]=value + raise self.EsgfObjectException( + "Invalid key: %s, valid keys are: %s" % + (repr(key), repr(self.params.keys()))) + self.params[key] = value return - - def _search(self,search="",searchType=None,stringType=False): + def _search(self, search="", searchType=None, stringType=False): if searchType is None: - searchType=self.defaultSearchType + searchType = self.defaultSearchType if not searchType in self.validSearchTypes: - raise self.EsgfObjectException("Valid Search types are: %s" % repr(self.validSearchTypes)) - while search[0]=="&": - search=search[1:] - rqst = "%s/?type=%s&%s" % (self.restPath,searchType,search) - #print "REQUEST: %s%s" % (self.host,rqst) - myhost=str(self.host) - myport=str(self.port) - if myhost.find("://")>-1: - urltype="" + raise self.EsgfObjectException( + "Valid Search types are: %s" % + repr(self.validSearchTypes)) + while search[0] == "&": + search = search[1:] + rqst = "%s/?type=%s&%s" % (self.restPath, searchType, search) + # print "REQUEST: %s%s" % (self.host,rqst) + myhost = str(self.host) + myport = str(self.port) + if myhost.find("://") > -1: + urltype = "" else: - urltype="http://" + urltype = "http://" try: - rqst="%s%s:%s/%s" % (urltype,myhost,myport,rqst) - tmp=rqst[6:].replace("//","/") - rqst=rqst[:6]+tmp - #print "Request:%s"%rqst + rqst = "%s%s:%s/%s" % (urltype, myhost, myport, rqst) + tmp = rqst[6:].replace("//", "/") + rqst = rqst[:6] + tmp + # print "Request:%s"%rqst url = urllib2.urlopen(rqst) - except Exception,msg: - raise self.EsgfObjectException(msg) + except Exception as msg: + raise self.EsgfObjectException(msg) r = url.read() if stringType: return r @@ -185,539 +212,591 @@ def _search(self,search="",searchType=None,stringType=False): try: e = xml.etree.ElementTree.fromstring(r) return e - except Exception,err: - raise self.EsgfObjectException("Could not interpret server's results: %s" % err) + except Exception as err: + raise self.EsgfObjectException( + "Could not interpret server's results: %s" % + err) - def generateRequest(self,stringType=False,**keys): + def generateRequest(self, stringType=False, **keys): search = "" - params={"limit":self["limit"],"offset":self["offset"]} + params = {"limit": self["limit"], "offset": self["offset"]} - ## for k in self.keys(): - ## if self[k] is not None and k in self.searchableKeys and k!="type": - ## params[k]=self[k] + # for k in self.keys(): + # if self[k] is not None and k in self.searchableKeys and k!="type": + # params[k]=self[k] - for k in keys.keys(): if k == "stringType": - stringType=keys[k] + stringType = keys[k] continue elif k == "type": continue - ## elif not k in self.searchableKeys: - ## raise self.EsgfObjectException("Invalid key: %s, valid keys are: %s" % (repr(k),repr(self.params.keys()))) + # elif not k in self.searchableKeys: + # raise self.EsgfObjectException("Invalid key: %s, valid keys are: + # %s" % (repr(k),repr(self.params.keys()))) if keys[k] is not None: - params[k]=keys[k] + params[k] = keys[k] search = "" for k in params.keys(): - if isinstance(params[k],list): + if isinstance(params[k], list): for v in params[k]: - if isinstance(v,str): - v=v.strip() - search+="&%s=%s" % (k,v) + if isinstance(v, str): + v = v.strip() + search += "&%s=%s" % (k, v) else: v = params[k] - if isinstance(v,str): - v=v.strip() - search+="&%s=%s" % (k,v) + if isinstance(v, str): + v = v.strip() + search += "&%s=%s" % (k, v) -# search = "&".join(map(lambda x : "%s=%s" % (x[0],x[1]), params.items())) - search=search.replace(" ","%20") +# search = "&".join(map(lambda x : "%s=%s" % (x[0],x[1]), params.items())) + search = search.replace(" ", "%20") return search - - def request(self,**keys): - numFound=0 + + def request(self, **keys): + numFound = 0 cont = True - r=[] + r = [] limit = self["limit"] while cont: - #print "Continuing",limit - self["offset"]=numFound - if limit is None or limit>1000: - self["limit"]=1000 + # print "Continuing",limit + self["offset"] = numFound + if limit is None or limit > 1000: + self["limit"] = 1000 search = self.generateRequest(**keys) - stringType=keys.get("stringType",False) - r.append(self._search(search,stringType=stringType)) - if numFound==0: + stringType = keys.get("stringType", False) + r.append(self._search(search, stringType=stringType)) + if numFound == 0: for s in r[0][:]: - if s.tag=="result": + if s.tag == "result": n = int(s.get("numFound")) - numFound+=self["limit"] + numFound += self["limit"] if limit is None: - if numFound>=n: + if numFound >= n: cont = False else: - if numFound>=limit: - cont=False - #print "N is:",numFound,n - self["limit"]=limit - self["offset"]=0 + if numFound >= limit: + cont = False + # print "N is:",numFound,n + self["limit"] = limit + self["offset"] = 0 return r - - def extractTag(self,f): - out=None - if f.tag=="str": - out=f.text - elif f.tag=="arr": - out=[] + + def extractTag(self, f): + out = None + if f.tag == "str": + out = f.text + elif f.tag == "arr": + out = [] for sub in f[:]: out.append(self.extractTag(sub)) - - elif f.tag=="float": + + elif f.tag == "float": out = float(f.text) - elif f.tag=="int": + elif f.tag == "int": out = int(f.text) - elif f.tag=="date": - ## Convert to cdtime? - out =f.text + elif f.tag == "date": + # Convert to cdtime? + out = f.text else: - out=f - if isinstance(out,list) and len(out)==1: - out=out[0] + out = f + if isinstance(out, list) and len(out) == 1: + out = out[0] return out - - def searchDatasets(self,**keys): + + def searchDatasets(self, **keys): resps = self.request(**keys) - stringType=keys.get("stringType",False) + stringType = keys.get("stringType", False) if stringType: return resp datasets = [] for resp in resps: for r in resp[:]: - if r.tag=="result": - ##Ok let's go thru these datasets + if r.tag == "result": + # Ok let's go thru these datasets for d in r[:]: - #print "************************************************" - tmpkeys={} + # print + # "************************************************" + tmpkeys = {} for f in d[:]: k = f.get("name") - tmpkeys[k]=self.extractTag(f) - if tmpkeys["type"]=="Dataset": + tmpkeys[k] = self.extractTag(f) + if tmpkeys["type"] == "Dataset": datasetid = tmpkeys["id"] - #print datasetid,self.restPath - #print "KEYS FOR DATASET",keys.keys() - datasets.append(esgfDataset(host=self.host,port=self.port,limit=1000,offset=0,mapping=self.mapping,datasetids=self.datasetids,fileids=self.fileids,keys=tmpkeys,originalKeys=keys,restPath=self.restPath)) + # print datasetid,self.restPath + # print "KEYS FOR DATASET",keys.keys() + datasets.append( + esgfDataset(host=self.host, + port=self.port, + limit=1000, + offset=0, + mapping=self.mapping, + datasetids=self.datasetids, + fileids=self.fileids, + keys=tmpkeys, + originalKeys=keys, + restPath=self.restPath)) return datasets + class esgfDataset(esgfConnection): - def __init__(self,host=None,port=80,limit=1000,offset=0,mapping=None,datasetids=None,fileids=None,_http=None,restPath=None,keys={},originalKeys={}): + + def __init__(self, host=None, port=80, limit=1000, offset=0, mapping=None, + datasetids=None, fileids=None, _http=None, restPath=None, keys={}, originalKeys={}): if host is None: raise esgfDatasetException("You need to pass url") - self.host=host - #self.host="esg-datanode.jpl.nasa.gov" - self.port=port - self.defaultSearchType="File" + self.host = host + # self.host="esg-datanode.jpl.nasa.gov" + self.port = port + self.defaultSearchType = "File" if restPath is None: - self.restPath="/esg-search/search" + self.restPath = "/esg-search/search" else: - self.restPath=restPath + self.restPath = restPath if datasetids is None: if "dataset_id_template_" in keys: - tmp=keys["dataset_id_template_"] - if tmp[:5]=="cmip5": - tmp = tmp.replace("valid_institute","institute") - tmp="%(project)"+tmp[5:] - self.datasetids = genutil.StringConstructor(tmp.replace(")s",")")) - elif "project" in keys and keys["project"]=="cmip5": - self.datasetids = genutil.StringConstructor("%(project).%(product).%(institute).%(model).%(experiment).%(time_frequency).%(realm).%(cmor_table).%(ensemble)") + tmp = keys["dataset_id_template_"] + if tmp[:5] == "cmip5": + tmp = tmp.replace("valid_institute", "institute") + tmp = "%(project)" + tmp[5:] + self.datasetids = genutil.StringConstructor( + tmp.replace(")s", ")")) + elif "project" in keys and keys["project"] == "cmip5": + self.datasetids = genutil.StringConstructor( + "%(project).%(product).%(institute).%(model).%(experiment).%(time_frequency).%(realm).%(cmor_table).%(ensemble)") else: - self.datasetids=None - if isinstance(datasetids,genutil.StringConstructor): - self.datasetids=datasetids - elif isinstance(datasetids,str): - self.datasetids=genutil.StringConstructor(datasetids) + self.datasetids = None + if isinstance(datasetids, genutil.StringConstructor): + self.datasetids = datasetids + elif isinstance(datasetids, str): + self.datasetids = genutil.StringConstructor(datasetids) if fileids is not None: - if isinstance(fileids,genutil.StringConstructor): - self.fileids=fileids + if isinstance(fileids, genutil.StringConstructor): + self.fileids = fileids else: - self.fileids=genutil.StringConstructor(fileids) + self.fileids = genutil.StringConstructor(fileids) if self.datasetids is not None: - self.fileids.template=self.fileids.template.replace("%(datasetid)",self.datasetids.template) + self.fileids.template = self.fileids.template.replace( + "%(datasetid)", self.datasetids.template) elif self.datasetids is not None: - self.fileids=genutil.StringConstructor("%s.%%(filename)" % self.datasetids.template) + self.fileids = genutil.StringConstructor( + "%s.%%(filename)" % + self.datasetids.template) else: - self.fileids=None - self.originalKeys=originalKeys - self.validSearchTypes=validSearchTypes - self.validSearchTypes=["File",] + self.fileids = None + self.originalKeys = originalKeys + self.validSearchTypes = validSearchTypes + self.validSearchTypes = ["File", ] self.EsgfObjectException = esgfDatasetException - self.params=keys + self.params = keys self.keys = self.params.keys self.items = self.params.items self.values = self.params.values - #self.id=self["id"] - self.params["limit"]=limit - self.params["offset"]=offset - self.mapping=mapping - #print "SEARCHING DS:",originalKeys - self.resp=None + # self.id=self["id"] + self.params["limit"] = limit + self.params["offset"] = offset + self.mapping = mapping + # print "SEARCHING DS:",originalKeys + self.resp = None self.cacheTime = None # self.search() # self.remap() - - ## Ok now we need to "map" this according to the user wishes - - - - ## def mappedItems(): - ## mapped=[] - ## mapppoint=self.mapped - ## for k in self.mapping.keys(): - ## keys=[] - ## level=[k,mappoint.keys()] - ## mappoint - def _extractFiles(self,resp,**inKeys): - ## We need to stick in there the bit from Luca to fill in the matching key from facet for now it's empty - files=[] - skipped = ["type","title","timestamp","service","id","score","file_url","service_type"] + + # Ok now we need to "map" this according to the user wishes + + # def mappedItems(): + # mapped=[] + # mapppoint=self.mapped + # for k in self.mapping.keys(): + # keys=[] + # level=[k,mappoint.keys()] + # mappoint + def _extractFiles(self, resp, **inKeys): + # We need to stick in there the bit from Luca to fill in the matching + # key from facet for now it's empty + files = [] + skipped = [ + "type", + "title", + "timestamp", + "service", + "id", + "score", + "file_url", + "service_type"] for r in resp[:]: - if r.tag=="result": + if r.tag == "result": for d in r[:][:]: - keys={} + keys = {} for f in d[:]: k = f.get("name") - keys[k]=self.extractTag(f) - if keys["type"]=="File": - ## if self["id"]=="obs4MIPs.NASA-JPL.AIRS.mon": - ## verbose=True - ## else: - ## verbose=False - ## #verbose=True - ## if verbose: print "OK",keys["variable"],keys["file_id"],self["id"] - ## if verbose: print "FILEIDS:",self.fileids - ## if verbose: print "Fileids:",self.fileids.template - ## if verbose: print "keys:",keys - ## if self.fileids is not None: - ## try: - ## if verbose: print "file:",keys["file_id"],self.fileids.template - ## k2 = self.fileids.reverse(keys["file_id"]) - ## if verbose: print "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@",k2 - ## for k in k2.keys(): - ## keys[k]=k2[k] - ## except: - ## if verbose: print "Failed:",ids[i].text,self.fileids.template - ## pass - ## if verbose: print "KEYS FOR FILE:",keys.keys() - ## if verbose: print "INKEYS:",inKeys.keys() - ## matched = True - ## matchWithKeys = {} - ## for k in self.keys(): - ## if k in self.originalKeys.keys(): - ## matchWithKeys[k]=self.originalKeys[k] - ## else: - ## matchWithKeys[k]=self[k] - ## for s in skipped: - ## try: - ## matchWithKeys.pop(s) - ## except: - ## pass - ## for k in inKeys.keys(): - ## matchWithKeys[k]=inKeys[k] - ## if verbose: print "matching:",matchWithKeys.keys() - ## for k in keys.keys(): - ## if k in matchWithKeys.keys(): - ## if verbose: print "Testing:",k,keys[k] - ## v = matchWithKeys[k] - ## if isinstance(v,(str,int,float)): - ## if verbose: print "\tComparing with:",v - ## if v != keys[k]: - ## matched = False - ## if verbose: print "\t\tNOPE" - ## break - ## elif isinstance(v,list): - ## if verbose: print "\tComparing with (and %i more):%s"%(len(v),v[0]),v - ## if not keys[k] in v: - ## matched = False - ## if verbose: print "\t\tNOPE" - ## break - ## else: - ## print "\twould compare %s with type: %s if I knew how to" % (str(v),type(v)) - ## if verbose: print keys["file_id"],matched - ## if matched : - ## for k in self.keys(): - ## if not k in keys.keys(): - ## keys[k]=self[k] - ## print "KEYS:",keys - files.append(esgfFile(**keys)) + keys[k] = self.extractTag(f) + if keys["type"] == "File": + # if self["id"]=="obs4MIPs.NASA-JPL.AIRS.mon": + # verbose=True + # else: + # verbose=False + # verbose=True + # if verbose: print "OK",keys["variable"],keys["file_id"],self["id"] + # if verbose: print "FILEIDS:",self.fileids + # if verbose: print "Fileids:",self.fileids.template + # if verbose: print "keys:",keys + # if self.fileids is not None: + # try: + # if verbose: print "file:",keys["file_id"],self.fileids.template + # k2 = self.fileids.reverse(keys["file_id"]) + # if verbose: print "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@",k2 + # for k in k2.keys(): + # keys[k]=k2[k] + # except: + # if verbose: print "Failed:",ids[i].text,self.fileids.template + # pass + # if verbose: print "KEYS FOR FILE:",keys.keys() + # if verbose: print "INKEYS:",inKeys.keys() + # matched = True + # matchWithKeys = {} + # for k in self.keys(): + # if k in self.originalKeys.keys(): + # matchWithKeys[k]=self.originalKeys[k] + # else: + # matchWithKeys[k]=self[k] + # for s in skipped: + # try: + # matchWithKeys.pop(s) + # except: + # pass + # for k in inKeys.keys(): + # matchWithKeys[k]=inKeys[k] + # if verbose: print "matching:",matchWithKeys.keys() + # for k in keys.keys(): + # if k in matchWithKeys.keys(): + # if verbose: print "Testing:",k,keys[k] + # v = matchWithKeys[k] + # if isinstance(v,(str,int,float)): + # if verbose: print "\tComparing with:",v + # if v != keys[k]: + # matched = False + # if verbose: print "\t\tNOPE" + # break + # elif isinstance(v,list): + # if verbose: print "\tComparing with (and %i more):%s"%(len(v),v[0]),v + # if not keys[k] in v: + # matched = False + # if verbose: print "\t\tNOPE" + # break + # else: + # print "\twould compare %s with type: %s if I knew how to" % (str(v),type(v)) + # if verbose: print keys["file_id"],matched + # if matched : + # for k in self.keys(): + # if not k in keys.keys(): + # keys[k]=self[k] + # print "KEYS:",keys + files.append(esgfFile(**keys)) return files - + def info(self): print self def __str__(self): st = "Dataset Information\nid: %s\nKeys:\n" % self.id for k in self.keys(): - st+="\t%s : %s\n" % (k,self[k]) + st += "\t%s : %s\n" % (k, self[k]) return st - + def clearWebCache(self): self.resp = None - def saveCache(self,target="."): + def saveCache(self, target="."): if self.resp is None: return if os.path.isdir(target): - target = os.path.join(target,"esgfDatasetsCache.pckl") + target = os.path.join(target, "esgfDatasetsCache.pckl") if os.path.exists(target): - f=open(source) - #dict=eval(bz2.decompress(f.read())) - dict=eval(f.read()) + f = open(source) + # dict=eval(bz2.decompress(f.read())) + dict = eval(f.read()) f.close() else: - dict={} - dict[self.id]=[self["timestamp"],xml.etree.ElementTree.tostring(self.resp),self.originalKeys] - f=open(target,"w") - #f.write(bz2.compress(repr(self.cache))) + dict = {} + dict[self.id] = [self["timestamp"], + xml.etree.ElementTree.tostring( + self.resp), + self.originalKeys] + f = open(target, "w") + # f.write(bz2.compress(repr(self.cache))) f.write(repr(self.cache)) f.close() - - def loadCache(self,source): - if isinstance(source,dict): - dict=source + + def loadCache(self, source): + if isinstance(source, dict): + dict = source else: if os.path.isdir(source): - source = os.path.join(source,"esgfDatasetsCache.pckl") + source = os.path.join(source, "esgfDatasetsCache.pckl") if os.path.exists(source): - f=open(source) - #dict=eval(bz2.decompress(f.read())) - dict=eval(f.read()) + f = open(source) + # dict=eval(bz2.decompress(f.read())) + dict = eval(f.read()) f.close() else: - dict={} - vals = dict.get(self.id,["",None,{}]) + dict = {} + vals = dict.get(self.id, ["", None, {}]) if vals[1] is not None: - self.cacheTime=vals[0] - self.resp=xml.etree.ElementTree.fromstring(vals[0]) - self.originalKeys=vals[1] - + self.cacheTime = vals[0] + self.resp = xml.etree.ElementTree.fromstring(vals[0]) + self.originalKeys = vals[1] + def clearOriginalQueryCache(self): - self.originalKeys={} + self.originalKeys = {} def clear(self): self.clearWebCache() self.clearOriginalQueryCache() - - def search(self,**keys): - #search = self.generateRequest(**keys) - stringType=keys.get("stringType",False) + + def search(self, **keys): + # search = self.generateRequest(**keys) + stringType = keys.get("stringType", False) keys.update(self.originalKeys) - st="" + st = "" if not "limit" in keys: - keys["limit"]=[self["limit"]] + keys["limit"] = [self["limit"]] if not "offset" in keys: - keys["offset"]=[self["offset"]] + keys["offset"] = [self["offset"]] for k in keys: - if k in ["searchString","stringType",]: + if k in ["searchString", "stringType", ]: continue for v in keys[k]: - st+="&%s=%s"%(k,v) - #st+="&%s=%s" % (k,keys[k]) - #if self.resp is None: - #self.resp = self._search("dataset_id=%s%s" % (self["id"],st),stringType=stringType) - self.resp = self._search(st,stringType=stringType) + st += "&%s=%s" % (k, v) + # st+="&%s=%s" % (k,keys[k]) + # if self.resp is None: + # self.resp = self._search("dataset_id=%s%s" % + # (self["id"],st),stringType=stringType) + self.resp = self._search(st, stringType=stringType) if stringType: return self.resp - return esgfFiles(self._extractFiles(self.resp,**keys),self) + return esgfFiles(self._extractFiles(self.resp, **keys), self) class esgfFiles(object): - def __init__(self,files,parent,mapping=None,datasetids=None,fileids=None): - self._files=files - if not isinstance(parent,esgfDataset): + + def __init__(self, files, parent, + mapping=None, datasetids=None, fileids=None): + self._files = files + if not isinstance(parent, esgfDataset): raise esgfFilesException("parent must be an esgfDataset instance") - self.parent=parent + self.parent = parent self.EsgfObjectException = esgfFilesException if datasetids is None: - datasetids=parent.datasetids - if isinstance(datasetids,genutil.StringConstructor): - self.datasetids=datasetids - elif isinstance(datasetids,str): - self.datasetids=genutil.StringConstructor(datasetids) + datasetids = parent.datasetids + if isinstance(datasetids, genutil.StringConstructor): + self.datasetids = datasetids + elif isinstance(datasetids, str): + self.datasetids = genutil.StringConstructor(datasetids) else: - self.datasetids=None + self.datasetids = None if fileids is not None: - if isinstance(fileids,genutil.StringConstructor): - self.fileids=fileids + if isinstance(fileids, genutil.StringConstructor): + self.fileids = fileids else: - self.fileids=genutil.StringConstructor(fileids) + self.fileids = genutil.StringConstructor(fileids) if self.datasetids is not None: - self.fileids.template=self.fileids.template.replace("%(datasetid)",self.datasetids.template) + self.fileids.template = self.fileids.template.replace( + "%(datasetid)", self.datasetids.template) elif self.datasetids is not None: - self.fileids=genutil.StringConstructor("%s.%%(filename)" % self.datasetids.template) + self.fileids = genutil.StringConstructor( + "%s.%%(filename)" % + self.datasetids.template) else: - self.fileids=parent.fileids + self.fileids = parent.fileids if mapping is None: - mapping=parent.mapping + mapping = parent.mapping self.setMapping(mapping) self.remap() - self.projects_dict = {"CMIP5": "%(project).%(product).%(institute).%(model).%(experiment).%(time_frequency).%(realm).%(cmor_table).%(ensemble)" } - - def __getitem__(self,item): - if isinstance(item,int): + self.projects_dict = { + "CMIP5": "%(project).%(product).%(institute).%(model).%(experiment).%(time_frequency).%(realm).%(cmor_table).%(ensemble)"} + + def __getitem__(self, item): + if isinstance(item, int): return self._files[item] - elif isinstance(item,str): + elif isinstance(item, str): for f in self._files: - if f["id"]==item: + if f["id"] == item: return f - elif isinstance(item,slice): + elif isinstance(item, slice): return self._files[item] else: raise esgfFilesException("unknown item type: %s" % type(item)) - def __setitem__(self,item): + + def __setitem__(self, item): raise esgfFilesException("You cannot set items") + def __len__(self): return len(self._files) + def getMapping(self): - if isinstance(self.mapping,genutil.StringConstructor): + if isinstance(self.mapping, genutil.StringConstructor): return self.mapping.template else: return self.mapping + def getMappingKeys(self): - if isinstance(self.mapping,genutil.StringConstructor): + if isinstance(self.mapping, genutil.StringConstructor): return self.mapping.keys() else: return None - def setMapping(self,mapping): + + def setMapping(self, mapping): if mapping is None: - self.mapping="" + self.mapping = "" if self.datasetids is not None: - self.mapping=self.datasetids + self.mapping = self.datasetids else: for k in self.parent.keys(): - if not k in ["limit","offset","text"]: - self.mapping+="%%(%s)" % k + if not k in ["limit", "offset", "text"]: + self.mapping += "%%(%s)" % k else: - self.mapping=mapping - #print "Stage 1 mapping:",self.mapping - if not isinstance(self.mapping,genutil.StringConstructor): + self.mapping = mapping + # print "Stage 1 mapping:",self.mapping + if not isinstance(self.mapping, genutil.StringConstructor): if self.datasetids is not None: - self.mapping=self.mapping.replace("%(datasetid)",self.datasetids.template) + self.mapping = self.mapping.replace( + "%(datasetid)", + self.datasetids.template) self.mapping = genutil.StringConstructor(self.mapping) - #print "Stage 2:",self.mapping.template - - ## vk = self.parent.keys() - ## for k in self.mapping.keys(): - ## ok = False - ## if self.datasetids is not None: - ## vk += self.datasetids.keys() - ## if k in self.datasetids.keys(): - ## ok = True - ## if self.fileids is not None: - ## vk+=self.fileids.keys() - ## if k in self.fileids.keys(): - ## ok = True - ## if k in self.parent.keys(): - ## ok=True - ## ## Ok second to last hope... Matching to datasetids - ## if isinstance(self.datasetids,genutil.StringConstructor) and ok is False: - ## try: - ## mapid = self.datasetids.reverse(self.parent.id) - ## vk+=mapid.keys() - ## if k in mapid.keys(): - ## ok = True - - ## except: - ## #print "Couldn't map: %s to %s" % (self.parent.id,self.datasetids.template) - ## pass - ## if ok is False: - ## vk = set(vk) - ## raise self.EsgfObjectException("Invalid mapping key: %s, valid keys are: %s" % (k,sorted(vk))) - - def remap(self,mapping=None,verbose=False): + # print "Stage 2:",self.mapping.template + + # vk = self.parent.keys() + # for k in self.mapping.keys(): + # ok = False + # if self.datasetids is not None: + # vk += self.datasetids.keys() + # if k in self.datasetids.keys(): + # ok = True + # if self.fileids is not None: + # vk+=self.fileids.keys() + # if k in self.fileids.keys(): + # ok = True + # if k in self.parent.keys(): + # ok=True + # Ok second to last hope... Matching to datasetids + # if isinstance(self.datasetids,genutil.StringConstructor) and ok is False: + # try: + # mapid = self.datasetids.reverse(self.parent.id) + # vk+=mapid.keys() + # if k in mapid.keys(): + # ok = True + + # except: + # print "Couldn't map: %s to %s" % (self.parent.id,self.datasetids.template) + # pass + # if ok is False: + # vk = set(vk) + # raise self.EsgfObjectException("Invalid mapping key: %s, valid keys + # are: %s" % (k,sorted(vk))) + + def remap(self, mapping=None, verbose=False): if mapping is None: thismapping = self.mapping else: thismapping = mapping - self.mapped={} + self.mapped = {} savedmapping = thismapping - #print "Remap:",self.mapping.template - ## if verbose: print "################ REMAPPING: %s: %s #############################" % (thismapping.template,repr(thismapping.keys())) + # print "Remap:",self.mapping.template + # if verbose: print "################ REMAPPING: %s: %s + # #############################" % + # (thismapping.template,repr(thismapping.keys())) for f in self._files: - mappoint=self.mapped - tabs="" - nok=0 + mappoint = self.mapped + tabs = "" + nok = 0 nlevels = len(thismapping.keys()) - #print "This mapping",thismapping.template,nlevels + # print "This mapping",thismapping.template,nlevels if nlevels == 0: - ## ok no mapping, let's try to figure this one out + # ok no mapping, let's try to figure this one out if 'dataset_id_template_' in f.keys(): - #print "We are good to go" - ds = f['dataset_id_template_'].replace(")s",")") + # print "We are good to go" + ds = f['dataset_id_template_'].replace(")s", ")") thismapping = genutil.StringConstructor(ds) for k in thismapping.keys(): - ## if verbose: print tabs,"keys:",k,"File keys:",f.keys() - ## if k == self.mapping.keys()[0]: - ## f.matched.keys() - ## else: - ## ## if verbose: print + # if verbose: print tabs,"keys:",k,"File keys:",f.keys() + # if k == self.mapping.keys()[0]: + # f.matched.keys() + # else: + # if verbose: print if k in f.keys(): - ## if verbose: print tabs,k,f[k] - nok+=1 + # if verbose: print tabs,k,f[k] + nok += 1 cont = f[k] - if not isinstance(cont,(str,int,float)): + if not isinstance(cont, (str, int, float)): break if not cont in mappoint.keys(): - mappoint[cont]={} + mappoint[cont] = {} elif k in self.parent.keys(): - ## if verbose: print tabs,k,f[k] - nok+=1 + # if verbose: print tabs,k,f[k] + nok += 1 cont = self[k] if not cont in mappoint.keys(): - mappoint[cont]={} - elif isinstance(self.fileids,genutil.StringConstructor): + mappoint[cont] = {} + elif isinstance(self.fileids, genutil.StringConstructor): try: mapid = self.fileids.reverse(self.parent.id) - ## if verbose: - ## print "MAPID:",k,mapid + # if verbose: + # print "MAPID:",k,mapid if k in mapid.keys(): - ## if verbose: print tabs,k,mapid[k] - nok+=1 + # if verbose: print tabs,k,mapid[k] + nok += 1 cont = mapid[k] if not cont in mappoint.keys(): - mappoint[cont]={} + mappoint[cont] = {} except: break else: break - mappoint=mappoint[cont] - tabs+="\t" - tmp = mappoint.get("files",[]) + mappoint = mappoint[cont] + tabs += "\t" + tmp = mappoint.get("files", []) tmp.append(f) mappoint["files"] = tmp thismapping = savedmapping - ## if verbose: print "################ REMAPPED: %s #############################" % (thismapping,) + # if verbose: print "################ REMAPPED: %s + # #############################" % (thismapping,) + class esgfFile(object): - def __init__(self,**keys): - self.__items__=keys + + def __init__(self, **keys): + self.__items__ = keys self.keys = self.__items__.keys self.items = self.__items__.items self.values = self.__items__.values - services=[] - #print "Keys:",self.keys() - #print self["url"] - S=self["url"] - if isinstance(S,str): - S=[S,] + services = [] + # print "Keys:",self.keys() + # print self["url"] + S = self["url"] + if isinstance(S, str): + S = [S, ] for service in S: - url,s2,s1 = service.split("|") - setattr(self,s1,url) + url, s2, s1 = service.split("|") + setattr(self, s1, url) services.append(s1) - self.services=services - self.id=self["id"] + self.services = services + self.id = self["id"] - def __getitem__(self,key): + def __getitem__(self, key): val = self.__items__[key] return val - - def __setitem__(self,key,value): - self.__items__[key]=value + + def __setitem__(self, key, value): + self.__items__[key] = value return def __str__(self): - st = "File Information\nid: %s\nParent Dataset: %s" % (self["id"],self["dataset_id"]) - st+="Matched keys: %s\n" % (repr(self.__items__)) + st = "File Information\nid: %s\nParent Dataset: %s" % ( + self["id"], self["dataset_id"]) + st += "Matched keys: %s\n" % (repr(self.__items__)) for service in self.services: - st+="service: %s @ %s\n" % (service,getattr(self,service)) + st += "service: %s @ %s\n" % (service, getattr(self, service)) return st[:-1] diff --git a/Packages/cdms2/Lib/selectors.py b/Packages/cdms2/Lib/selectors.py index f224439762..2a35705d1d 100644 --- a/Packages/cdms2/Lib/selectors.py +++ b/Packages/cdms2/Lib/selectors.py @@ -1,42 +1,48 @@ """Classes to support easy selection of climate data""" import cdtime -from axis import axisMatches -from error import CDMSError -from grid import AbstractRectGrid, defaultRegion, setRegionSpecs, LongitudeType, LatitudeType, TimeType, VerticalType +from .axis import axisMatches +from .error import CDMSError +from .grid import AbstractRectGrid, defaultRegion, setRegionSpecs, LongitudeType, LatitudeType, TimeType, VerticalType _debug = 0 + + class SelectorError (CDMSError): + "The exception type for errors in the selector packages" - def __init__ (self, args): + + def __init__(self, args): self.args = args + class Selector: + """Selector class""" - def __init__ (self, *args, **kwargs): + + def __init__(self, *args, **kwargs): """Positional args are SelectorComponents or Selectors Keyword args and their value are passed to kwselect to create - selectors. All the selector components are put into the + selectors. All the selector components are put into the components list of this Selector, along with all the components of any Selector arguments. """ self.__components = [] self.refine(*args, **kwargs) for a in args: - if isinstance(a,SelectorComponent): + if isinstance(a, SelectorComponent): try: - self.__str__=a.__str__ + self.__str__ = a.__str__ except: pass - - def components (self): + def components(self): "List of selector components, each an instance of SelectorComponent." return self.__components[:] - - def refine (self, *args, **kwargs): - """Add components to this selector using the same syntax as the - constructor. Ignores non-keyword arguments that are not + + def refine(self, *args, **kwargs): + """Add components to this selector using the same syntax as the + constructor. Ignores non-keyword arguments that are not SelectorComponents or Selectors. """ for a in args: @@ -46,27 +52,27 @@ def refine (self, *args, **kwargs): self.__components.append(a) elif isinstance(a, Selector): for x in a.components(): - self.refine(x) + self.refine(x) else: self.refine(positionalComponent(a)) for k, v in kwargs.items(): - self.refine(kwselect(k, v)) - - def __repr__ (self): + self.refine(kwselect(k, v)) + + def __repr__(self): s = 'Selector(' sep = '' for c in self.__components: s = s + sep + repr(c) sep = ', ' return s + ')' - - def __and__ (self, other): - """Implements the & operator, which returns + + def __and__(self, other): + """Implements the & operator, which returns self.clone() refined by other """ if not isinstance(other, Selector): - raise SelectorError, 'Cannot combine Selector with non-selector' + raise SelectorError('Cannot combine Selector with non-selector') s = self.clone() s.refine(other) return s @@ -75,7 +81,7 @@ def clone(self): "Makes a copy of this Selector." return Selector(*self.__components) - def __call__ (self, *args, **kwargs): + def __call__(self, *args, **kwargs): """Return a new selector consisting of this one refined by the given arguments. Arguments are as per the constructor or method refine. """ @@ -92,31 +98,31 @@ def select(self, variable, *args, **kwargs): Options modify the result of the selection. The options and their default values are: -- raw = 0: if 1, return an numpy.ma only - -- squeeze = 0: If 1, eliminate any dimensions of length 1 + -- squeeze = 0: If 1, eliminate any dimensions of length 1 from the result. - -- order = None: If given, is a string such as + -- order = None: If given, is a string such as variable.getOrder() returns. Result is permuted into this order. - -- grid = None: If given, is a grid object; result is + -- grid = None: If given, is a grid object; result is regridded onto this grid. Each of the components contributes arguments suitable for the - subRegion call in class cdms.AbstractVariable. If a component + subRegion call in class cdms.AbstractVariable. If a component is to modify the same axis as a previous component, its application is postponed. subRegion is called and the result is then fed - to each of the components' "post" method. This returns a + to each of the components' "post" method. This returns a possibly modified result, which becomes the input to the next - component's post method. + component's post method. This procedure is repeated until no more components are postponed. Then the options are applied to the result in the order - listed above, and the result is returned. + listed above, and the result is returned. Execption SelectorError is thrown if the selection is impossible. The result is a TransientVariable and id(variable) <> id(result) even if there are no components. - """ + """ d = kwargs.copy() raw = d.setdefault('raw', 0) squeeze = d.setdefault('squeeze', 0) @@ -125,13 +131,14 @@ def select(self, variable, *args, **kwargs): del d['squeeze'], d['grid'], d['order'], d['raw'] # make the selector s = self(*args, **d) - return s.unmodified_select(variable, - squeeze=squeeze, - order=order, - grid=grid, + return s.unmodified_select(variable, + squeeze=squeeze, + order=order, + grid=grid, raw=raw) - def unmodified_select(self, variable, raw=0, squeeze=0, order=None, grid=None): + def unmodified_select( + self, variable, raw=0, squeeze=0, order=None, grid=None): "Select using this selector without further modification" result = variable components = self.components() @@ -143,71 +150,80 @@ def unmodified_select(self, variable, raw=0, squeeze=0, order=None, grid=None): newcomponents = [] specs = defaultRegion() for c in components: - if c.specifyGrid(variable, vargrid, specs): # specs is modified + if c.specifyGrid(variable, vargrid, specs): # specs is modified newcomponents.append(c) components = newcomponents if specs != defaultRegion(): vgindices = result.getGridIndices() mask, indexspecs = vargrid.intersect(specs) result = result(**indexspecs) - result = result.setMaskFromGridMask(mask, vgindices) # Propagate the grid mask to result + result = result.setMaskFromGridMask( + mask, + vgindices) # Propagate the grid mask to result # Now select on non-coordinate components. while(components): axes = result.getAxisList() - if _debug: print "Axes:", axes - specifications = [':']*len(axes) - confined_by = [None]*len(axes) - aux = {} # for extra state + if _debug: + print "Axes:", axes + specifications = [':'] * len(axes) + confined_by = [None] * len(axes) + aux = {} # for extra state overflow = [] - if _debug: print "Component list:", components + if _debug: + print "Component list:", components for c in components: if c.specify(result, axes, specifications, confined_by, aux): - if _debug: print 'Defer ' + repr(c) + if _debug: + print 'Defer ' + repr(c) overflow.append(c) elif _debug: print "After applying", c, ":" - print "specifications=", specifications + print "specifications=", specifications print "Confined_by", confined_by print "aux", aux print "-----------------" - if _debug: + if _debug: print 'About to call subRegion:', specifications fetched = result.subRegion(*specifications) axismap = range(len(axes)) for c in components: - if c in overflow: continue - fetched = c.post(fetched, result, axes, specifications, + if c in overflow: + continue + fetched = c.post(fetched, result, axes, specifications, confined_by, aux, axismap) if not len(overflow) < len(components): - raise SelectorError, \ - 'Internal selector error, infinite loop detected.' + raise SelectorError( + 'Internal selector error, infinite loop detected.') components = overflow result = fetched if squeeze != 0 or \ order is not None or \ grid is not None or \ - raw !=0 or \ - result is variable: + raw != 0 or \ + result is variable: # result is variable when there are no components, for example. return result.subRegion(squeeze=squeeze, order=order, grid=grid, raw=raw) else: return result - + + class SelectorComponent: + """Base class representing selection for a given set of axes. """ - def specify (self, slab, axes, specifications, confined_by, aux): - """Refine the specification suitable for slab.subRegion + + def specify(self, slab, axes, specifications, confined_by, aux): + """Refine the specification suitable for slab.subRegion Set confined_by to yourself for each axis you confine. - If you would normally confine an axis to ':', don't, + If you would normally confine an axis to ':', don't, unless you *require* that axis not be confined by other components. - + Returning: - Return 1 if you wish to skip your turn. You'll be called + Return 1 if you wish to skip your turn. You'll be called later with the results of the other selectors. Raise a SelectorError exception if you can't do your job. @@ -218,10 +234,10 @@ def specify (self, slab, axes, specifications, confined_by, aux): Store any info you want in dictionary aux[id(self)] """ return 0 - + def specifyGrid(self, var, grid, specs): """Refine the specification suitable for grid.intersect(). - + 'var' is a variable. 'grid' is the grid associated with the variable. 'specs' is the result set of specifications, of the form defined in the grid module. @@ -235,47 +251,53 @@ def specifyGrid(self, var, grid, specs): """ return 1 - def post (self, fetched, slab, axes, specifications, confined_by, aux, axismap): + def post(self, fetched, slab, axes, + specifications, confined_by, aux, axismap): """Post-process fetched if desired, return new value. - Arguments slab, axes, specifications, confined_by, and aux are - pre-subRegion call. - - axismap gives the indices of fetched's axes in axes and should + Arguments slab, axes, specifications, confined_by, and aux are + pre-subRegion call. + + axismap gives the indices of fetched's axes in axes and should be modified as required by this method. Set axismap[i] to None to indicate that you have eliminated an axis. """ return fetched - + + class axisComponent (SelectorComponent): + "A SelectorComponent that confines exactly one axis or coordinate dimension (e.g. latitude)." - def __init__ (self, id, spec): + + def __init__(self, id, spec): self.id = id self.spec = spec - def specify (self, slab, axes, specifications, confined_by, aux): + def specify(self, slab, axes, specifications, confined_by, aux): "Do specification for axis self.id; skip if axis not present." for i in range(len(axes)): if axisMatches(axes[i], self.id): - if confined_by[i] is None: - specifications[i] = self.spec - confined_by[i] = self - return 0 - else: - return 1 + if confined_by[i] is None: + specifications[i] = self.spec + confined_by[i] = self + return 0 + else: + return 1 return 0 - def __repr__ (self): - s = repr(self.__class__)+'("'+self.id+'", '+repr(self.spec) + ')' + def __repr__(self): + s = repr(self.__class__) + \ + '("' + self.id + '", ' + repr(self.spec) + ')' return s - + + class coordinateComponent(axisComponent): + "A SelectorComponent that confines exactly one coordinate dimension (e.g., latitude)" def __init__(self, id, spec): axisComponent.__init__(self, id, spec) def specifyGrid(self, var, grid, specs): - "Determine if this component confines the grid, and if so set the specs and return 1" if grid.hasCoordType(self.id): setRegionSpecs(grid, self.spec, self.id, specs) @@ -283,42 +305,52 @@ def specifyGrid(self, var, grid, specs): else: return 1 + class requiredComponent (SelectorComponent): + """Checks to see that a specific id axis must be present.""" - def __init__ (self, ids): + + def __init__(self, ids): """Checks to see that a specific axis or axes must be present. Initialize with a sequence of ids. """ self.ids = ids - - def specify (self, slab, axes, specifications, confined_by, aux): + + def specify(self, slab, axes, specifications, confined_by, aux): """Doesn't confine but checks for existance.""" for id in self.ids: for i in range(len(axes)): if axisMatches(axes[i], id): break else: - raise SelectorError, \ - 'Required axis %s not present in this variable.' % (id,) + raise SelectorError( + 'Required axis %s not present in this variable.' % + (id,)) return 0 + class indexComponent (axisComponent): - """An axisComponent that confines exactly one axis by - specifying indices. + + """An axisComponent that confines exactly one axis by + specifying indices. """ - def __init__ (self, id, start=None, stop=None, stride=None): + + def __init__(self, id, start=None, stop=None, stride=None): self.id = id - self.spec = slice(start,stop, stride) + self.spec = slice(start, stop, stride) + class indexedComponent (SelectorComponent): - """A SelectorComponent that confines exactly one axis - whose index is given. + + """A SelectorComponent that confines exactly one axis + whose index is given. """ - def __init__ (self, index, value): + + def __init__(self, index, value): self.index = index self.spec = value - def specify (self, slab, axes, specifications, confined_by, aux): + def specify(self, slab, axes, specifications, confined_by, aux): "Do the specification for axis whose index is self.index." i = self.index if confined_by[i] is None: @@ -328,13 +360,16 @@ def specify (self, slab, axes, specifications, confined_by, aux): else: return 1 + class positionalComponent (SelectorComponent): + """A SelectorComponent that confines the next axis available. """ - def __init__ (self, v): + + def __init__(self, v): self.v = v - def specify (self, slab, axes, specifications, confined_by, aux): + def specify(self, slab, axes, specifications, confined_by, aux): "Find the next unconfined axis and confine it." n = 0 for i in range(len(axes)): @@ -344,51 +379,59 @@ def specify (self, slab, axes, specifications, confined_by, aux): aux[id(self)] = i return 0 else: - raise SelectorError, \ - 'positional component cannot be applied, insufficent rank:' +\ - repr(self) + raise SelectorError('positional component cannot be applied, insufficent rank:' + + repr(self)) - def __repr__ (self): + def __repr__(self): s = repr(self.__class__) + '(' + repr(self.v) + ')' return s - -def longitude (*value): + + +def longitude(*value): "Creates default selector corresponding to keyword longitude = value" if not value: return all if len(value) == 1: value = value[0] - if value == ':': return all + if value == ':': + return all return Selector(coordinateComponent(LongitudeType, value)) - -def latitude (*value): + + +def latitude(*value): "Creates default selector corresponding to keyword latitude = value" if not value: return all if len(value) == 1: value = value[0] - if value == ':': return all + if value == ':': + return all return Selector(coordinateComponent(LatitudeType, value)) - -def time (*value): + + +def time(*value): """Creates a default selector corresponding to keyword time=value """ if not value: return all if len(value) == 1: value = value[0] - if value == ':': return all + if value == ':': + return all return Selector(coordinateComponent(TimeType, value)) -def level (*value): + +def level(*value): "Creates default selector corresponding to keyword level = value" if not value: return all if len(value) == 1: value = value[0] - if value == ':': return all + if value == ':': + return all return Selector(coordinateComponent(VerticalType, value)) + def required(values): """Creates a selector that requires a certain axis to be present.""" if values is None: @@ -397,12 +440,13 @@ def required(values): values = (values,) return Selector(requiredComponent(values)) -def kwselect (k, value): + +def kwselect(k, value): """Turn a keyword/value pair into a SelectorComponent The words latitude, longitude, time, and level are used to pass value to the routine of the same name. Otherise, axis is called using k as the id. - """ + """ kx = k[0:3].lower() if kx == 'lat': return latitude(value) @@ -416,17 +460,25 @@ def kwselect (k, value): return required(value) else: return Selector(requiredComponent((k,)), axisComponent(k, value)) - + all = Selector() -def timeslice (start=None,stop=None,stride=None): + +def timeslice(start=None, stop=None, stride=None): return Selector(indexComponent('time', start, stop, stride)) -def latitudeslice (start=None,stop=None,stride=None): + + +def latitudeslice(start=None, stop=None, stride=None): return Selector(indexComponent('latitude', start, stop, stride)) -def longitudeslice (start=None,stop=None,stride=None): + + +def longitudeslice(start=None, stop=None, stride=None): return Selector(indexComponent('longitude', start, stop, stride)) -def levelslice (start=None,stop=None,stride=None): + + +def levelslice(start=None, stop=None, stride=None): return Selector(indexComponent('level', start, stop, stride)) -def setslice (id, start=None,stop=None,stride=None): - return Selector(indexComponent(id, start, stop, stride)) + +def setslice(id, start=None, stop=None, stride=None): + return Selector(indexComponent(id, start, stop, stride)) diff --git a/Packages/cdms2/Lib/slabinterface.py b/Packages/cdms2/Lib/slabinterface.py index 642aea1312..f12738ee7d 100644 --- a/Packages/cdms2/Lib/slabinterface.py +++ b/Packages/cdms2/Lib/slabinterface.py @@ -1,64 +1,67 @@ -## Automatically adapted for numpy.oldnumeric Aug 01, 2007 by -## Further modified to be pure new numpy June 24th 2008 +# Automatically adapted for numpy.oldnumeric Aug 01, 2007 by +# Further modified to be pure new numpy June 24th 2008 "Read part of the old cu slab interface implemented over CDMS" import numpy import sys -from error import CDMSError -from axis import std_axis_attributes +from .error import CDMSError +from .axis import std_axis_attributes import cdms2 as cdms + class Slab: + """Slab is the cu api This is an abstract class to inherit in AbstractVariable About axes: weight and bounds attributes always set but may be None - if bounds are None, getdimattribute returns result of querying the + if bounds are None, getdimattribute returns result of querying the axis. """ std_slab_atts = ['filename', - 'missing_value', - 'comments', - 'grid_name', - 'grid_type', - 'time_statistic', - 'long_name', - 'units'] - def __init__ (self): + 'missing_value', + 'comments', + 'grid_name', + 'grid_type', + 'time_statistic', + 'long_name', + 'units'] + + def __init__(self): pass - def getattribute (self, name): + def getattribute(self, name): "Get the attribute name." - defaultdict = {'filename':'N/A', - 'comments':'', - 'grid_name':'N/A', - 'grid_type':'N/A', - 'time_statistic':'', - 'long_name':'', - 'units':''} + defaultdict = {'filename': 'N/A', + 'comments': '', + 'grid_name': 'N/A', + 'grid_type': 'N/A', + 'time_statistic': '', + 'long_name': '', + 'units': ''} result = None - if name in defaultdict.keys() and not hasattr(self,name): - if name=='filename': - if (not hasattr(self,'parent')) or self.parent is None: + if name in defaultdict.keys() and not hasattr(self, name): + if name == 'filename': + if (not hasattr(self, 'parent')) or self.parent is None: result = '' else: result = self.parent.id - elif name=='grid_name': + elif name == 'grid_name': grid = self.getGrid() if grid is None: result = defaultdict[name] else: result = grid.id - elif name=='grid_type': + elif name == 'grid_type': grid = self.getGrid() if grid is None: result = defaultdict[name] - elif isinstance(grid,cdms.grid.TransientRectGrid): + elif isinstance(grid, cdms.grid.TransientRectGrid): result = grid.getType() - elif isinstance(grid,cdms.gengrid.AbstractGenericGrid): + elif isinstance(grid, cdms.gengrid.AbstractGenericGrid): result = 'GenericGrid' - elif isinstance(grid,cdms.hgrid.AbstractCurveGrid): + elif isinstance(grid, cdms.hgrid.AbstractCurveGrid): result = 'CurvilinearGrid' else: result = defaultdict[name] @@ -67,23 +70,23 @@ def getattribute (self, name): result = getattr(self, name) except AttributeError: result = None - + return result - def setattribute (self, name, value): + def setattribute(self, name, value): "Set the attribute name to value." setattr(self, name, value) - def createattribute (self, name, value): + def createattribute(self, name, value): "Create an attribute and set its name to value." setattr(self, name, value) - def deleteattribute (self, name): + def deleteattribute(self, name): "Delete the named attribute." if hasattr(self, name): delattr(self, name) - def listattributes (self): + def listattributes(self): "Return a list of attribute names." return self.attributes.keys() @@ -92,10 +95,11 @@ def listdimattributes(self, dim): a = self.getAxis(dim) result = [] for x in std_axis_attributes + a.attributes.keys(): - if not x in result: result.append(x) + if not x in result: + result.append(x) return result - def getdimattribute (self, dim, field): + def getdimattribute(self, dim, field): """Get the attribute named field from the dim'th dimension. For bounds returns the old cu one-dimensional version. """ @@ -121,38 +125,37 @@ def getdimattribute (self, dim, field): return g.getWeights()[0] elif d.isLongitude(): return g.getWeights()[1] - else: #should be impossible, actually + else: # should be impossible, actually return numpy.ones(len(d)) elif field == "bounds": b = d.getBounds() n = b.shape[0] - result = numpy.zeros(n+1, b.dtype.char) - result[0:-1] = b[:,0] - result[-1] = b[-1,1] + result = numpy.zeros(n + 1, b.dtype.char) + result[0:-1] = b[:, 0] + result[-1] = b[-1, 1] return result - elif d.attributes.has_key(field): + elif field in d.attributes: return d.attributes[field] else: - raise CDMSError, "No %s attribute on given axis." % field - - + raise CDMSError("No %s attribute on given axis." % field) + def showdim(self): - "Show the dimension attributes and values." + "Show the dimension attributes and values." result = [] for nd in range(self.rank()): - result.append('** Dimension ' + str(nd+1) + ' **') + result.append('** Dimension ' + str(nd + 1) + ' **') result = result + self.getAxis(nd).listall(1) print '\n'.join(result) def listdimnames(self): "Return a list of the names of the dimensions." - result=[] + result = [] for nd in range(self.rank()): result.append(self.getdimattribute(nd, 'name')) return result - def listall (self, all=None): + def listall(self, all=None): "Get list of info about this slab." vname = self.id result = [] @@ -162,8 +165,10 @@ def listall (self, all=None): for x in Slab.std_slab_atts: result.append(x + ": " + str(self.getattribute(x))) for x in self.attributes.keys(): - if x in Slab.std_slab_atts: continue - if x == 'name': continue + if x in Slab.std_slab_atts: + continue + if x == 'name': + continue result.append(x + ": " + str(self.attributes[x])) g = self.getGrid() if g is None: @@ -171,29 +176,30 @@ def listall (self, all=None): else: result = result + g.listall(all) for nd in range(self.rank()): - result.append('** Dimension ' + str(nd+1) + ' **') + result.append('** Dimension ' + str(nd + 1) + ' **') result = result + self.getAxis(nd).listall(all) result.append('*** End of description for %s ***' % vname) return result def info(self, flag=None, device=None): "Write info about slab; include dimension values and weights if flag" - if device is None: device = sys.stdout + if device is None: + device = sys.stdout device.write('\n'.join(self.listall(all=flag))) device.write("\n") -def cdms_bounds2cu_bounds (b): + +def cdms_bounds2cu_bounds(b): "Bounds are len(v) by 2 in cdms but len(v)+1 in cu" - cub = numpy.ma.zeros(len(b)+1, numpy.float32) + cub = numpy.ma.zeros(len(b) + 1, numpy.float32) b1 = b.astype(numpy.float32) - if len(b)>1: - if (b[0,0] 1: + if (b[0, 0] < b[0, 1]) == (b[0, 0] < b[-1, 0]): + cub[0] = b[0, 0] + cub[1:] = b[:, 1] else: - cub[0] = b[0,1] - cub[1:] = b[:,0] + cub[0] = b[0, 1] + cub[1:] = b[:, 0] else: cub[:] = b[0] - return numpy.array( cub ) - + return numpy.array(cub) diff --git a/Packages/cdms2/Lib/sliceut.py b/Packages/cdms2/Lib/sliceut.py index e69e832626..4b8b55821a 100644 --- a/Packages/cdms2/Lib/sliceut.py +++ b/Packages/cdms2/Lib/sliceut.py @@ -6,41 +6,41 @@ # Returns a slice, or None if the intersection is empty. -def sliceIntersect(aSlice,interval): - p0,p1 = interval +def sliceIntersect(aSlice, interval): + p0, p1 = interval i = aSlice.start j = aSlice.stop k = aSlice.step if k is None: - k=1 + k = 1 # If the slice has a negative step, generate the # equivalent slice with positive step - irev=0 - if k<0: + irev = 0 + if k < 0: k = -k - pk = ((j-i+k)/k)*k+i - j = i+1 + pk = ((j - i + k) / k) * k + i + j = i + 1 i = pk irev = 1 # Calculate the intersection for an increasing slice - px = ((p0-i+k-1)/k)*k+i - a = max(px,i) - b = min(j,p1) - if a0: + if step > 0: start = aSlice.start stop = aSlice.stop else: @@ -72,121 +74,131 @@ def lenSlice(aSlice): stop = aSlice.start step = -step - return ((stop-start-1)/step + 1) + return ((stop - start - 1) / step + 1) + -def reverseSlice(s,size): +def reverseSlice(s, size): """For 'reversed' slices (slices with negative stride), return an equivalent slice with positive step. For positive strides, just return the slice unchanged. """ - if s.step>0 or s.step is None: + if s.step > 0 or s.step is None: return s i = s.start j = s.stop k = s.step if i is None: - i=size-1 - elif i<0: - i = i%size + i = size - 1 + elif i < 0: + i = i % size if j is None: - j=-1 - elif -size-10: - wrap1 = slice(i,size,k) - wrap2 = slice((i-size)%k, j-size, k) + i, j, k = s.start, s.stop, s.step + if k > 0: + wrap1 = slice(i, size, k) + wrap2 = slice((i - size) % k, j - size, k) else: - wrap1 = slice(i-size, None, k) - wrap2 = slice(size+(i-size)%k, j, k) - return (wrap1,wrap2) + wrap1 = slice(i - size, None, k) + wrap2 = slice(size + (i - size) % k, j, k) + return (wrap1, wrap2) -def splitSliceExt(s,size): +def splitSliceExt(s, size): """ mf 20010330 -- For a 'wraparound' slice, return N equivalent slices within the range 0...(N*size) N = anything""" - i,j,k = s.start,s.stop,s.step + i, j, k = s.start, s.stop, s.step # slice of form [i:] sets j to large int - if j>2000000000L: + if j > 2000000000: j = size - _debug=0 - if(_debug): print "SSSS0: ",i,j,k - - wrap=[] - - if k>0: - - iter=0 - if(_debug): print "SSSS1: iter ",iter,j,size,k - while(j>0): - if(_debug): print " " - if(_debug): print "SSSS2: iter",iter,j,size,k - jo=size - if(iter>0): jo=size+1 - if(_debug): print "SSSS3: iter",iter,j,jo - if(j 0: + + iter = 0 + if(_debug): + print "SSSS1: iter ", iter, j, size, k + while(j > 0): + if(_debug): + print " " + if(_debug): + print "SSSS2: iter", iter, j, size, k + jo = size + if(iter > 0): + jo = size + 1 + if(_debug): + print "SSSS3: iter", iter, j, jo + if(j < size): + jo = j + if(_debug): + print "SSSS4: iter", iter, j, jo + wrap.append(slice(i, jo, k)) + j = j - size + i = 0 + iter = iter + 1 + else: - wraprev=[] - iter=0 - if(_debug): print "SSSS1 neg: iter ",iter,i,j,size,k - while(i>=0): - if(_debug): print " " - if(_debug): print "SSSS2 neg: iter",iter,i,j,size,k - io=size-1 - if(_debug): print "SSSS3 neg: iter",iter,i,j,io - if(i= 0): + if(_debug): + print " " + if(_debug): + print "SSSS2 neg: iter", iter, i, j, size, k + io = size - 1 + if(_debug): + print "SSSS3 neg: iter", iter, i, j, io + if(i < size): + io = i + if(_debug): + print "SSSS4 neg: iter", iter, i, j, io + + # mf 20010405 python does not return nothing for + # slice(size-1,size-1,-1); force it + if(not (io == size - 1 and j == size - 1)): + wraprev.append(slice(io, j, k)) + + i = i - size + j = None + iter = iter + 1 # # reverse # - for k in range(0,len(wraprev)): - kk=len(wraprev)-k-1 + for k in range(0, len(wraprev)): + kk = len(wraprev) - k - 1 wrap.append(wraprev[kk]) - if(_debug): print "SSSS5 neg: ",kk,wraprev[kk] + if(_debug): + print "SSSS5 neg: ", kk, wraprev[kk] return (wrap) - - - - - - - diff --git a/Packages/cdms2/Lib/tvariable.py b/Packages/cdms2/Lib/tvariable.py index 7d9db8b9db..9ebcca9c6d 100644 --- a/Packages/cdms2/Lib/tvariable.py +++ b/Packages/cdms2/Lib/tvariable.py @@ -1,4 +1,4 @@ -# Automatically adapted for numpy.oldnumeric Aug 01, 2007 by +# Automatically adapted for numpy.oldnumeric Aug 01, 2007 by # Further modified to be pure new numpy June 24th 2008 """ @@ -8,18 +8,18 @@ """ import json import re -import typeconv +from . import typeconv import numpy from numpy import sctype2char -from error import CDMSError -from avariable import AbstractVariable +from .error import CDMSError +from .avariable import AbstractVariable -from axis import createAxis, AbstractAxis -from grid import createRectGrid, AbstractRectGrid -from hgrid import AbstractCurveGrid -from gengrid import AbstractGenericGrid +from .axis import createAxis, AbstractAxis +from .grid import createRectGrid, AbstractRectGrid +from .hgrid import AbstractCurveGrid +from .gengrid import AbstractGenericGrid -# dist array support +# dist array support HAVE_MPI = False try: from mpi4py import MPI @@ -30,38 +30,43 @@ id_builtin = id # built_in gets clobbered by keyword + def fromJSON(jsn): """ Recreate a TV from a dumped jsn object""" D = json.loads(jsn) - ## First recreates the axes - axes=[] + # First recreates the axes + axes = [] for a in D["_axes"]: - ax = createAxis(numpy.array(a["_values"],dtype=a["_dtype"]),id=a["id"]) - for k,v in a.iteritems(): - if not k in ["_values","id","_dtype"]: - setattr(ax,k,v) + ax = createAxis( + numpy.array(a["_values"], + dtype=a["_dtype"]), + id=a["id"]) + for k, v in a.iteritems(): + if not k in ["_values", "id", "_dtype"]: + setattr(ax, k, v) axes.append(ax) - ## Now prep the variable - V= createVariable(D["_values"],id=D["id"],typecode=D["_dtype"]) + # Now prep the variable + V = createVariable(D["_values"], id=D["id"], typecode=D["_dtype"]) V.setAxisList(axes) - for k,v in D.iteritems(): - if not k in ["id","_values","_axes","_grid","_fill_value","_dtype",]: - setattr(V,k,v) + for k, v in D.iteritems(): + if not k in ["id", "_values", "_axes", "_grid", "_fill_value", "_dtype", ]: + setattr(V, k, v) V.set_fill_value(D["_fill_value"]) return V -class TransientVariable(AbstractVariable,numpy.ma.MaskedArray): +class TransientVariable(AbstractVariable, numpy.ma.MaskedArray): + "An in-memory variable." variable_count = 0 _missing = numpy.ma.MaskedArray.fill_value - def _getShape(self): return self._data.shape - shape = property(_getShape,None) + shape = property(_getShape, None) + def iscontiguous(self): return self.flags['CONTIGUOUS'] @@ -70,79 +75,84 @@ def ascontiguousarray(self): out = numpy.ascontiguousarray(d) m = numpy.ma.getmask(self) if m is not numpy.ma.nomask: - m= numpy.ascontiguousarray(m) - out = TransientVariable(out,mask=m,attributes=self.attributes) + m = numpy.ascontiguousarray(m) + out = TransientVariable(out, mask=m, attributes=self.attributes) out.setAxisList(self.getAxisList()) out.setMissing(self.getMissing()) return out - + ascontiguous = ascontiguousarray - + def asma(self): - return numpy.ma.array(self._data,mask=self._mask) - - def _update_from(self,obj): - numpy.ma.MaskedArray._update_from(self,obj) - if not hasattr(self,'___cdms_internals__'): - self.__dict__['___cdms_internals__']=['__cdms_internals__','___cdms_internals__','_node_','parent','attributes','shape'] - if not hasattr(self,'attributes'): - self.attributes={} - self._grid_ = getattr(obj,'_grid_',None) + return numpy.ma.array(self._data, mask=self._mask) + + def _update_from(self, obj): + numpy.ma.MaskedArray._update_from(self, obj) + if not hasattr(self, '___cdms_internals__'): + self.__dict__[ + '___cdms_internals__'] = [ + '__cdms_internals__', + '___cdms_internals__', + '_node_', + 'parent', + 'attributes', + 'shape'] + if not hasattr(self, 'attributes'): + self.attributes = {} + self._grid_ = getattr(obj, '_grid_', None) try: - for nm,val in obj.__dict__.items(): - if nm[0]=='_': -## print nm + for nm, val in obj.__dict__.items(): + if nm[0] == '_': +# print nm pass -## self.__dict__[nm]=val +# self.__dict__[nm]=val else: - setattr(self,nm,val) - except Exception,err: + setattr(self, nm, val) + except Exception as err: pass - id = getattr(self,'id',None) + id = getattr(self, 'id', None) if id is None: - TransientVariable.variable_count+=1 - id = 'variable_'+str(TransientVariable.variable_count) - self.id=id - self.name = getattr(obj,'name',id) - if not hasattr(self,'__domain'): + TransientVariable.variable_count += 1 + id = 'variable_' + str(TransientVariable.variable_count) + self.id = id + self.name = getattr(obj, 'name', id) + if not hasattr(self, '__domain'): self.initDomain(axes=None) - - def __array_finalize__(self,obj): - numpy.ma.MaskedArray.__array_finalize__(self,obj) + def __array_finalize__(self, obj): + numpy.ma.MaskedArray.__array_finalize__(self, obj) return - - __mul__ = AbstractVariable.__mul__ - __rmul__ = AbstractVariable.__rmul__ - __imul__ = AbstractVariable.__imul__ - __abs__ = AbstractVariable.__abs__ - __neg__ = AbstractVariable.__neg__ - __add__ = AbstractVariable.__add__ - __iadd__ = AbstractVariable.__iadd__ - __radd__ = AbstractVariable.__radd__ + __mul__ = AbstractVariable.__mul__ + __rmul__ = AbstractVariable.__rmul__ + __imul__ = AbstractVariable.__imul__ + __abs__ = AbstractVariable.__abs__ + __neg__ = AbstractVariable.__neg__ + __add__ = AbstractVariable.__add__ + __iadd__ = AbstractVariable.__iadd__ + __radd__ = AbstractVariable.__radd__ __lshift__ = AbstractVariable.__lshift__ __rshift__ = AbstractVariable.__rshift__ - __sub__ = AbstractVariable.__sub__ - __rsub__ = AbstractVariable.__rsub__ - __isub__ = AbstractVariable.__isub__ - __div__ = AbstractVariable.__div__ - __rdiv__ = AbstractVariable.__rdiv__ - __idiv__ = AbstractVariable.__idiv__ - __pow__ = AbstractVariable.__pow__ - __eq__ = AbstractVariable.__eq__ - __ne__ = AbstractVariable.__ne__ - __lt__ = AbstractVariable.__lt__ - __le__ = AbstractVariable.__le__ - __gt__ = AbstractVariable.__gt__ - __ge__ = AbstractVariable.__ge__ - __sqrt__ = AbstractVariable.__sqrt__ - - def __init__(self,data, typecode=None, copy=1, savespace=0, + __sub__ = AbstractVariable.__sub__ + __rsub__ = AbstractVariable.__rsub__ + __isub__ = AbstractVariable.__isub__ + __div__ = AbstractVariable.__div__ + __rdiv__ = AbstractVariable.__rdiv__ + __idiv__ = AbstractVariable.__idiv__ + __pow__ = AbstractVariable.__pow__ + __eq__ = AbstractVariable.__eq__ + __ne__ = AbstractVariable.__ne__ + __lt__ = AbstractVariable.__lt__ + __le__ = AbstractVariable.__le__ + __gt__ = AbstractVariable.__gt__ + __ge__ = AbstractVariable.__ge__ + __sqrt__ = AbstractVariable.__sqrt__ + + def __init__(self, data, typecode=None, copy=1, savespace=0, mask=numpy.ma.nomask, fill_value=None, grid=None, - axes=None, attributes=None, id=None, copyaxes=1, dtype=None, - order=False, no_update_from=False,**kargs): - """createVariable (self, data, typecode=None, copy=0, savespace=0, + axes=None, attributes=None, id=None, copyaxes=1, dtype=None, + order=False, no_update_from=False, **kargs): + """createVariable (self, data, typecode=None, copy=0, savespace=0, mask=None, fill_value=None, grid=None, axes=None, attributes=None, id=None, dtype=None, order=False) The savespace argument is ignored, for backward compatibility only. @@ -150,69 +160,70 @@ def __init__(self,data, typecode=None, copy=1, savespace=0, try: if data.fill_value is not None: self._setmissing(data.fill_value) - fill_value=data.fill_value + fill_value = data.fill_value except: pass if fill_value is not None: - self._setmissing(fill_value) - if attributes is not None and "_FillValue" in attributes.keys(): - self._setmissing(attributes["_FillValue"]) + self._setmissing(fill_value) + if attributes is not None and "_FillValue" in attributes.keys(): + self._setmissing(attributes["_FillValue"]) - # tile index, None means no mosaic + # tile index, None means no mosaic self.tileIndex = None - + # Compatibility: assuming old typecode, map to new if dtype is None and typecode is not None: dtype = typeconv.convtypecode2(typecode) typecode = sctype2char(dtype) if isinstance(data, tuple): data = list(data) - - AbstractVariable.__init__ (self) + + AbstractVariable.__init__(self) if isinstance(data, AbstractVariable): if not isinstance(data, TransientVariable): data = data.subSlice() -## if attributes is None: attributes = data.attributes +# if attributes is None: attributes = data.attributes if axes is None and not no_update_from: axes = map(lambda x: x[0], data.getDomain()) if grid is None and not no_update_from: grid = data.getGrid() if (grid is not None) and (not isinstance(grid, AbstractRectGrid)) \ - and (not grid.checkAxes(axes)): - grid = grid.reconcile(axes) # Make sure grid and axes are consistent - - ncopy = (copy!=0) + and (not grid.checkAxes(axes)): + grid = grid.reconcile( + axes) # Make sure grid and axes are consistent + ncopy = (copy != 0) # Initialize the geometry if grid is not None: - copyaxes=0 # Otherwise grid axes won't match domain. + copyaxes = 0 # Otherwise grid axes won't match domain. if axes is not None: - self.initDomain(axes, copyaxes=copyaxes) # Note: clobbers the grid, so set the grid after. + self.initDomain(axes, copyaxes=copyaxes) + # Note: clobbers the grid, so set the grid after. if grid is not None: self.setGrid(grid) - + # Initialize the attributes if attributes is not None: for key, value in attributes.items(): - if (key in ['shape','flat','imaginary','real'] or key[0]=='_') and key not in ['_FillValue']: - raise CDMSError, 'Bad key in attributes: ' + key + if (key in ['shape', 'flat', 'imaginary', 'real'] or key[0] == '_') and key not in ['_FillValue']: + raise CDMSError('Bad key in attributes: ' + key) elif key == 'missing_value': - #ignore if fill value given explicitly + # ignore if fill value given explicitly if fill_value is None: self._setmissing(value) - elif key not in ['scale_factor','add_offset']: + elif key not in ['scale_factor', 'add_offset']: setattr(self, key, value) # Sync up missing_value attribute and the fill value. self.missing_value = self._getmissing() self._FillValue = self._getmissing() if id is not None: - if not isinstance(id,(unicode,str)): - raise CDMSError, 'id must be a string' + if not isinstance(id, (unicode, str)): + raise CDMSError('id must be a string') self.id = id - elif hasattr(data,'id'): + elif hasattr(data, 'id'): self.id = data.id if self.id is None: @@ -227,22 +238,21 @@ def __init__(self,data, typecode=None, copy=1, savespace=0, self.__mpiWindows = {} self.__mpiType = self.__getMPIType() - def _getmissing(self): return self._missing - def _setmissing(self,value): - self._missing=numpy.array(value).astype(self.dtype) + def _setmissing(self, value): + self._missing = numpy.array(value).astype(self.dtype) - missing = property(_getmissing,_setmissing) - fill_value = property(_getmissing,_setmissing) - _FillValue = property(_getmissing,_setmissing) - missing_value = property(_getmissing,_setmissing) + missing = property(_getmissing, _setmissing) + fill_value = property(_getmissing, _setmissing) + _FillValue = property(_getmissing, _setmissing) + missing_value = property(_getmissing, _setmissing) - def __new__(cls, data, typecode=None, copy=0, savespace=0, - mask=numpy.ma.nomask, fill_value=None, grid=None, - axes=None, attributes=None, id=None, copyaxes=1, dtype=None, order=False,**kargs): - """createVariable (self, data, typecode=None, copy=0, savespace=0, + def __new__(cls, data, typecode=None, copy=0, savespace=0, + mask=numpy.ma.nomask, fill_value=None, grid=None, + axes=None, attributes=None, id=None, copyaxes=1, dtype=None, order=False, **kargs): + """createVariable (self, data, typecode=None, copy=0, savespace=0, mask=None, fill_value=None, grid=None, axes=None, attributes=None, id=None, dtype=None, order=False) The savespace argument is ignored, for backward compatibility only. @@ -258,61 +268,60 @@ def __new__(cls, data, typecode=None, copy=0, savespace=0, data = data.subSlice() if isinstance(data, numpy.ma.MaskedArray): try: - if fill_value is None: fill_value = data.fill_value + if fill_value is None: + fill_value = data.fill_value except: pass - ncopy = (copy!=0) + ncopy = (copy != 0) if mask is None: try: mask = data.mask - except Exception,err: + except Exception as err: mask = numpy.ma.nomask # Handle the case where ar[i:j] returns a single masked value if data is numpy.ma.masked: - #shape = tuple(len(axes)*[1]) + # shape = tuple(len(axes)*[1]) data = numpy.ma.masked.data - #data.shape = shape + # data.shape = shape mask = numpy.ma.masked.mask - #mask.shape = shape -## if data.getattr('mask',None) is not numpy.ma.nomask: -## mask = data.mask -## print 'passing:',mask.shape,data.shape,numpy.shape(cls) + # mask.shape = shape +# if data.getattr('mask',None) is not numpy.ma.nomask: +# mask = data.mask +# print 'passing:',mask.shape,data.shape,numpy.shape(cls) if fill_value is not None: fill_value = numpy.array(fill_value).astype(dtype) else: fill_value = numpy.ma.MaskedArray(1).astype(dtype).item() + self = numpy.ma.MaskedArray.__new__(cls, data, dtype=dtype, + copy=ncopy, + mask=mask, + fill_value=fill_value, + subok=False, + order=order) - self = numpy.ma.MaskedArray.__new__(cls, data, dtype = dtype, - copy = ncopy, - mask = mask, - fill_value = fill_value, - subok = False, - order = order) - - - return self # typecode = numpy.ma.array.typecode def typecode(self): return self.dtype.char - def assignValue(self,data): + def assignValue(self, data): self[...] = data def getValue(self, squeeze=1): return self.filled() - def expertSlice (self, slicelist): + def expertSlice(self, slicelist): return numpy.ma.MaskedArray.__getitem__(self, slicelist) - def initDomain (self, axes, copyaxes=1): - # lazy evaluation via getAxis to avoid creating axes that aren't ever used. + def initDomain(self, axes, copyaxes=1): + # lazy evaluation via getAxis to avoid creating axes that aren't ever + # used. newgrid = None - self.__domain = [None]*self.rank() + self.__domain = [None] * self.rank() if axes is not None: flataxes = [] try: @@ -325,22 +334,25 @@ def initDomain (self, axes, copyaxes=1): elif isinstance(item, AbstractRectGrid) or isinstance(item, AbstractCurveGrid): flataxes.append(item.getAxis(0)) flataxes.append(item.getAxis(1)) - copyaxes=0 + copyaxes = 0 newgrid = item elif isinstance(item, AbstractGenericGrid): flataxes.append(item.getAxis(0)) - copyaxes=0 + copyaxes = 0 newgrid = item else: - raise CDMSError, "Invalid item in axis list:\n"+`item` + raise CDMSError( + "Invalid item in axis list:\n" + repr(item)) if len(flataxes) != self.rank(): - raise CDMSError, "Wrong number of axes to initialize domain." + raise CDMSError("Wrong number of axes to initialize domain.") for i in range(len(flataxes)): if flataxes[i] is not None: - if (not flataxes[i].isVirtual()) and copyaxes==1: + if (not flataxes[i].isVirtual()) and copyaxes == 1: self.copyAxis(i, flataxes[i]) else: - self.setAxis(i, flataxes[i]) # No sense copying a virtual axis. + self.setAxis( + i, + flataxes[i]) # No sense copying a virtual axis. if newgrid is not None: # Do this after setting the axes, so the grid is consistent self.setGrid(newgrid) @@ -350,25 +362,31 @@ def getDomain(self): junk = self.getAxis(i) # will force a fill in return self.__domain - def getAxis (self, n): - if n < 0: n = n + self.rank() + def getAxis(self, n): + if n < 0: + n = n + self.rank() if self.__domain[n] is None: length = numpy.ma.size(self, n) - # axis = createAxis(numpy.ma.arange(numpy.ma.size(self, n), typecode=numpy.Float)) - axis = createAxis(numpy.ma.arange(numpy.ma.size(self, n), dtype=numpy.float_)) + # axis = createAxis(numpy.ma.arange(numpy.ma.size(self, n), + # typecode=numpy.Float)) + axis = createAxis( + numpy.ma.arange(numpy.ma.size(self, n), dtype=numpy.float_)) axis.id = "axis_" + str(n) self.__domain[n] = (axis, 0, length, length) return self.__domain[n][0] - - def setAxis (self, n, axis, savegrid=0): + + def setAxis(self, n, axis, savegrid=0): """Set n axis of self to a copy of axis. (0-based index) """ - if n < 0: n = n + self.rank() + if n < 0: + n = n + self.rank() axislen = self.shape[n] - if len(axis)!=axislen: - raise CDMSError,"axis length %d does not match corresponding dimension %d"%(len(axis),axislen) + if len(axis) != axislen: + raise CDMSError( + "axis length %d does not match corresponding dimension %d" % + (len(axis), axislen)) if not isinstance(axis, AbstractAxis): - raise CDMSError,"copydimension, other not a slab." + raise CDMSError("copydimension, other not a slab.") self.__domain[n] = (axis, 0, len(axis), len(axis)) def setAxisList(self, axislist): @@ -376,26 +394,27 @@ def setAxisList(self, axislist): for i in range(len(axislist)): self.setAxis(i, axislist[i]) - def copyAxis (self, n, axis): + def copyAxis(self, n, axis): """Set n axis of self to a copy of axis. (0-based index) Invalidates grid. """ - if n < 0: n = n + self.rank() + if n < 0: + n = n + self.rank() if not isinstance(axis, AbstractAxis): - raise CDMSError,"copydimension, other not an axis." + raise CDMSError("copydimension, other not an axis.") b = axis.getBounds() mycopy = createAxis(axis[:], b) mycopy.id = axis.id for k, v in axis.attributes.items(): - setattr(mycopy, k, v) - self.setAxis (n, mycopy) - - def copyDomain (self, other): + setattr(mycopy, k, v) + self.setAxis(n, mycopy) + + def copyDomain(self, other): "Set the axes and grid by copying variable other." if not isinstance(other, AbstractVariable): - raise CDMSError,"copyDomain, other not a variable." + raise CDMSError("copyDomain, other not a variable.") if self.rank() != other.rank(): - raise CDMSError, "copyDomain, ranks do not match." + raise CDMSError("copyDomain, ranks do not match.") for i in range(self.rank()): self.copyAxis(i, other.getAxis(i)) self.setGrid(other.getGrid()) @@ -406,21 +425,23 @@ def getGrid(self): for i in range(self.rank()): ax = self.getAxis(i) if ax.isLatitude(): - order = order+'y' + order = order + 'y' lat = ax elif ax.isLongitude(): - order = order+'x' + order = order + 'x' lon = ax - if len(order)==2: break + if len(order) == 2: + break - if order in ['yx','xy']: - self._grid_ = createRectGrid(lat,lon,order) + if order in ['yx', 'xy']: + self._grid_ = createRectGrid(lat, lon, order) return self._grid_ - def astype (self, tc): + def astype(self, tc): "return self as array of given type." - maresult = numpy.ma.MaskedArray.astype(self,tc) - return TransientVariable(maresult, copy=0, axes=self.getAxisList(), fill_value=self.fill_value, + maresult = numpy.ma.MaskedArray.astype(self, tc) + return TransientVariable( + maresult, copy=0, axes=self.getAxisList(), fill_value=self.fill_value, attributes=self.attributes, id=self.id, grid=self.getGrid()) def setMaskFromGridMask(self, mask, gridindices): @@ -437,18 +458,18 @@ def setMaskFromGridMask(self, mask, gridindices): shapeprep.append(self.shape[i]) # Broadcast mask - if tprep!=[]: + if tprep != []: newshape = tuple(shapeprep + list(mask.shape)) bigmask = numpy.resize(mask, newshape) # Generate the tranpose vector t = tuple(tprep + list(gridindices)) - tinv = [0]*len(t) + tinv = [0] * len(t) for i in range(len(t)): tinv[t[i]] = i # And reshape to fit the variable - if tinv!=range(len(tinv)): + if tinv != range(len(tinv)): bigmask = numpy.transpose(bigmask, tuple(tinv)) else: @@ -463,25 +484,25 @@ def setMaskFromGridMask(self, mask, gridindices): return result # Old cu interface - def copydimension (self, idim, other, jdim): - """Set idim dimension of self to variable other's jdim'th + def copydimension(self, idim, other, jdim): + """Set idim dimension of self to variable other's jdim'th This is for old cu compatibility. Use copyAxis for new code. """ if not isinstance(other, AbstractVariable): - raise CDMSError,"copydimension, other not a variable." + raise CDMSError("copydimension, other not a variable.") a = other.getAxis(jdim) self.copyAxis(idim, a) def setdimattribute(self, dim, field, value): "Set the attribute named field from the dim'th dimension." if dim < 0 or dim >= self.rank(): - raise CDMSError, "setdimattribute, dim out of bounds." + raise CDMSError("setdimattribute, dim out of bounds.") d = self.getAxis(dim) if field == "name": if not isinstance(value, basestring): - raise CDMSError, "setdimattribute: name not a string" + raise CDMSError("setdimattribute: name not a string") d.id = value - + elif field == "values": # note -- invalidates grid, may break old code. a = createAxis(numpy.ma.filled(value[:])) @@ -492,28 +513,29 @@ def setdimattribute(self, dim, field, value): elif field == "units": if not isinstance(value, basestring): - raise CDMSError, "setdimattribute: units not a string" + raise CDMSError("setdimattribute: units not a string") d.units = value elif field == "weights": # Well, you can't really do this without modifying the grid - raise CDMSError, "setdimattribute weights not implemented." + raise CDMSError("setdimattribute weights not implemented.") elif field == "bounds": if value is None: - d.setBounds(None) + d.setBounds(None) else: - b = numpy.ma.filled(value) - if numpy.ma.rank(b) == 2: - d.setBounds(b) - elif numpy.ma.rank(b) == 1: - b1 = numpy.zeros((len(b)-1,2), b.dtype.char) - b1[:,0] = b[:-1] - b1[:,1] = b[1:] - d.setBounds(b1) - else: - raise CDMSError, \ - "setdimattribute, bounds improper shape: " + b.shape + b = numpy.ma.filled(value) + if numpy.ma.rank(b) == 2: + d.setBounds(b) + elif numpy.ma.rank(b) == 1: + b1 = numpy.zeros((len(b) - 1, 2), b.dtype.char) + b1[:, 0] = b[:-1] + b1[:, 1] = b[1:] + d.setBounds(b1) + else: + raise CDMSError( + "setdimattribute, bounds improper shape: " + + b.shape) else: setattr(d, field, value) @@ -524,57 +546,57 @@ def clone(self, copyData=1): result = createVariable(self, copy=copyData) return result - def dumps(self,*args,**kargs): - ## Probably need something for curv/gen grids + def dumps(self, *args, **kargs): + # Probably need something for curv/gen grids """ Dumps Variable to a jason object, args are passed directly to json.dump""" - J={} - for k,v in self.attributes.iteritems(): - if k=="autoApiInfo": + J = {} + for k, v in self.attributes.iteritems(): + if k == "autoApiInfo": continue - J[k]=v - J['id']=self.id - axes=[] + J[k] = v + J['id'] = self.id + axes = [] for a in self.getAxisList(): - ax={} - for A,v in a.attributes.iteritems(): - ax[A]=v - ax['id']=a.id - ax["_values"]=a[:].tolist() - ax["_dtype"]=a[:].dtype.char + ax = {} + for A, v in a.attributes.iteritems(): + ax[A] = v + ax['id'] = a.id + ax["_values"] = a[:].tolist() + ax["_dtype"] = a[:].dtype.char axes.append(ax) - J["_axes"]=axes - J["_values"]=self[:].filled(self.fill_value).tolist() - J["_fill_value"]=float(self.fill_value) - J["_dtype"]=self.typecode() - J["_grid"]=None #self.getGrid() - return json.dumps(J,*args,**kargs) + J["_axes"] = axes + J["_values"] = self[:].filled(self.fill_value).tolist() + J["_fill_value"] = float(self.fill_value) + J["_dtype"] = self.typecode() + J["_grid"] = None # self.getGrid() + return json.dumps(J, *args, **kargs) def isEncoded(self): "Transient variables are not encoded" return 0 - def __len__ (self): + def __len__(self): "Length of first dimension" - if self.rank()>0: - (axis,start,length,true_length) = self.getDomain()[0] + if self.rank() > 0: + (axis, start, length, true_length) = self.getDomain()[0] else: length = 0 return length - def __str__ (self): + def __str__(self): return numpy.ma.MaskedArray.__str__(self) - def __repr__ (self): + def __repr__(self): return self.id + '\n' + numpy.ma.MaskedArray.__repr__(self) + '\n' def set_fill_value(self, value): "Set missing value attribute and fill value" AbstractVariable.setMissing(self, value) - #self.__dict__['_fill_value'] = self.missing_value - ## Fix submitted by Ghislain Picard, this was broken with numpy 1.5 - numpy.ma.MaskedArray.set_fill_value(self,value) + # self.__dict__['_fill_value'] = self.missing_value + # Fix submitted by Ghislain Picard, this was broken with numpy 1.5 + numpy.ma.MaskedArray.set_fill_value(self, value) - def setMissing (self, value): + def setMissing(self, value): "Set missing value attribute and fill value" self.set_fill_value(value) @@ -595,7 +617,7 @@ def getTileIndex(self): """ return self.tileIndex - def toVisit(self, filename, format='Vs', sphereRadius=1.0, + def toVisit(self, filename, format='Vs', sphereRadius=1.0, maxElev=0.1): """ Save data to file for postprocessing by the VisIt visualization tool @@ -604,21 +626,21 @@ def toVisit(self, filename, format='Vs', sphereRadius=1.0, sphereRadius: radius of the earth maxElev: maximum elevation for representation on the sphere """ - import mvSphereMesh - import mvVTKSGWriter - import mvVsWriter + from . import mvSphereMesh + from . import mvVTKSGWriter + from . import mvVsWriter try: # required by mvVsWriter import tables except: - # fall back + # fall back format = 'VTK' def generateTimeFileName(filename, tIndex, tIndexMax, suffix): - ndigits = len('%d'%tIndexMax) - itdigits = len('%d'%tIndex) - tiStr = '0'*(ndigits-itdigits) + ('%d'%tIndex) - return re.sub(r'\.' + suffix, '_%s.%s' % (tiStr, suffix), + ndigits = len('%d' % tIndexMax) + itdigits = len('%d' % tIndex) + tiStr = '0' * (ndigits - itdigits) + ('%d' % tIndex) + return re.sub(r'\.' + suffix, '_%s.%s' % (tiStr, suffix), filename) # determine whether data are time dependent @@ -632,46 +654,48 @@ def generateTimeFileName(filename, tIndex, tIndexMax, suffix): counter += 1 if axis == 'time': timeIndex = counter - - if timeAxis == None or timeIndex == -1: + + if timeAxis is None or timeIndex == -1: # static data if format == 'VTK': vw = mvVTKSGWriter.VTKSGWriter(self, maxElev) - if filename.find('.vtk') == -1: + if filename.find('.vtk') == -1: filename += '.vtk' vw.write(filename) else: vw = mvVsWriter.VsWriter(self, maxElev) - if filename.find('.vsh5') == -1: + if filename.find('.vsh5') == -1: filename += '.vsh5' vw.write(filename) else: # time dependent data tIndexMax = len(timeAxis) for tIndex in range(tIndexMax): - sliceOp = 'self[' + (':,'*timeIndex) + ('%d,'%tIndex) + '...]' + sliceOp = 'self[' + ( + ':,' * timeIndex) + ('%d,' % + tIndex) + '...]' var = eval(sliceOp) if format == 'VTK': if filename.find('.vtk') == -1: filename += '.vtk' - tFilename = generateTimeFileName(filename, + tFilename = generateTimeFileName(filename, tIndex, tIndexMax, 'vtk') vw = mvVTKSGWriter.VTKSGWriter(var, maxElev) vw.write(tFilename) else: if filename.find('.h5') == -1: filename += '.h5' - tFilename = generateTimeFileName(filename, + tFilename = generateTimeFileName(filename, tIndex, tIndexMax, 'h5') vw = mvVsWriter.VsWriter(var, maxElev) vw.write(tFilename) - - # Following are distributed array methods, they require mpi4py + + # Following are distributed array methods, they require mpi4py # to be installed def setMPIComm(self, comm): """ - Set the MPI communicator. This is a no-op if MPI + Set the MPI communicator. This is a no-op if MPI is not available. """ if HAVE_MPI: @@ -698,7 +722,7 @@ def getMPISize(self): def exposeHalo(self, ghostWidth=1): """ Expose the halo to other processors. The halo is the region - within the local MPI data domain that is accessible to other + within the local MPI data domain that is accessible to other processors. The halo encompasses the edge of the data region and has thickness ghostWidth. @@ -715,9 +739,9 @@ def exposeHalo(self, ghostWidth=1): # given direction, a 1 represents a layer of # thickness ghostWidth on the high index side, # -1 on the low index side. - winId = tuple( [0 for i in range(dim) ] \ - + [drect] + \ - [0 for i in range(dim+1, ndims) ] ) + winId = tuple([0 for i in range(dim)] + + [drect] + + [0 for i in range(dim + 1, ndims)]) slce = slice(0, ghostWidth) if drect == 1: @@ -726,19 +750,19 @@ def exposeHalo(self, ghostWidth=1): slab = self.__getSlab(dim, slce) # create the MPI window - dataSrc = numpy.zeros(self[slab].shape, self.dtype) - dataDst = numpy.zeros(self[slab].shape, self.dtype) + dataSrc = numpy.zeros(self[slab].shape, self.dtype) + dataDst = numpy.zeros(self[slab].shape, self.dtype) self.__mpiWindows[winId] = { 'slab': slab, 'dataSrc': dataSrc, 'dataDst': dataDst, 'window': MPI.Win.Create(dataSrc, comm=self.__mpiComm), - } - + } + def getHaloEllipsis(self, side): """ - Get the ellipsis for a given halo side. - + Get the ellipsis for a given halo side. + side - a tuple of zeros and one +1 or -1. To access the "north" side for instance, set side=(1, 0), (-1, 0) to access the south side, (0, 1) the east @@ -746,7 +770,7 @@ def getHaloEllipsis(self, side): Return none if halo was not exposed (see exposeHalo) """ - if HAVE_MPI and self.__mpiWindows.has_key(side): + if HAVE_MPI and side in self.__mpiWindows: return self.__mpiWindows[side]['slab'] else: return None @@ -754,20 +778,20 @@ def getHaloEllipsis(self, side): def fetchHaloData(self, pe, side): """ Fetch the halo data from another processor. The halo side - is a subdomain of the halo that is exposed to other + is a subdomain of the halo that is exposed to other processors. It is an error to call this method when MPI is not enabled. This is a collective method (must be called by all processes), which involves synchronization of data among all processors. pe - processor owning the halo data. This is a no - operation when pe is None. + operation when pe is None. side - a tuple of zeros and one +1 or -1. To access the "north" side for instance, set side=(1, 0), (-1, 0) to access the south side, (0, 1) the east - side, etc. + side, etc. - Note: collective, all procs must invoke this method. If some + Note: collective, all procs must invoke this method. If some processors should not fetch then pass None for pe. """ if HAVE_MPI: @@ -780,37 +804,37 @@ def fetchHaloData(self, pe, side): dataSrc[...] = self[slab] win = iw['window'] - win.Fence() # get the data ready + win.Fence() # get the data ready if pe is not None: - win.Get( [dataDst, self.__mpiType], pe ) - win.Fence() # make sure the communication completed + win.Get([dataDst, self.__mpiType], pe) + win.Fence() # make sure the communication completed return dataDst else: - raise CDMSError, 'Must have MPI to invoke fetchHaloData' + raise CDMSError('Must have MPI to invoke fetchHaloData') def freeHalo(self): """ - Free the MPI windows attached to the halo. This must be + Free the MPI windows attached to the halo. This must be called before MPI_Finalize. """ for iw in self.__mpiWindows: - self.__mpiWindows[iw]['window'].Free() + self.__mpiWindows[iw]['window'].Free() def __getSlab(self, dim, slce): """ Get slab. A slab is a multi-dimensional slice extending in all directions except along dim where slce applies - + dim - dimension (0=first index, 1=2nd index...) slce - python slice object along dimension dim - + return slab """ ndims = len(self.shape) - + slab = [ slice(0, None) for i in range(dim) ] \ - + [slce] + \ - [ slice(0, None) for i in range(dim+1, ndims) ] + + [slce] + \ + [slice(0, None) for i in range(dim + 1, ndims)] return tuple(slab) def __getMPIType(self): @@ -834,50 +858,54 @@ def __getMPIType(self): elif dtyp == numpy.int8: typ = MPI.INT8_T else: - return None + return None else: return typ -## PropertiedClasses.set_property(TransientVariable, 'shape', -## nowrite=1, nodelete=1) +# PropertiedClasses.set_property(TransientVariable, 'shape', +# nowrite=1, nodelete=1) + -def createVariable(*args,**kargs): - if kargs.get("fromJSON",False): +def createVariable(*args, **kargs): + if kargs.get("fromJSON", False): return fromJSON(*args) else: - return TransientVariable(*args,**kargs) + return TransientVariable(*args, **kargs) -def isVariable (s): + +def isVariable(s): "Is s a variable?" return isinstance(s, AbstractVariable) + def asVariable(s, writeable=1): - """Returns s if s is a Variable; if writeable is 1, return - s if s is a TransientVariable. If s is not a variable of + """Returns s if s is a Variable; if writeable is 1, return + s if s is a TransientVariable. If s is not a variable of the desired type, attempt to make it so and return that. If we fail raise CDMSError """ target_class = AbstractVariable - if writeable: target_class = TransientVariable + if writeable: + target_class = TransientVariable if isinstance(s, target_class): return s elif isinstance(s, AbstractVariable): return s.subSlice() - + try: result = createVariable(s) except CDMSError: - result = None - + result = None + # if result.dtype.char == numpy.ma.PyObject: if issubclass(result.dtype.type, numpy.object_): result = None if result is None: - raise CDMSError, "asVariable could not make a Variable from the input." + raise CDMSError("asVariable could not make a Variable from the input.") return result if __name__ == '__main__': - for s in [(20,), (4,5)]: + for s in [(20,), (4, 5)]: x = numpy.arange(20) x.shape = s t = createVariable(x) @@ -886,21 +914,24 @@ def asVariable(s, writeable=1): assert numpy.ma.allclose(x, t) assert t.dtype.char == numpy.int assert numpy.ma.size(t) == numpy.ma.size(x) - assert numpy.ma.size(t,0) == len(t) - assert numpy.ma.allclose(t.getAxis(0)[:], numpy.ma.arange(numpy.ma.size(t,0))) + assert numpy.ma.size(t, 0) == len(t) + assert numpy.ma.allclose( + t.getAxis(0)[:], numpy.ma.arange(numpy.ma.size(t, 0))) t.missing_value = -99 assert t.missing_value == -99 assert t.fill_value == -99 - t = createVariable(numpy.ma.arange(5), mask=[0,0,0,1,0]) - t.set_fill_value (1000) + t = createVariable(numpy.ma.arange(5), mask=[0, 0, 0, 1, 0]) + t.set_fill_value(1000) assert t.fill_value == 1000 assert t.missing_value == 1000 t.missing_value = -99 assert t[2] == 2 t[3] = numpy.ma.masked assert t[3] is numpy.ma.masked - f = createVariable(numpy.ma.arange(5, typecode=numpy.float32), mask=[0,0,0,1,0]) - f2 = createVariable(numpy.ma.arange(5, typecode=numpy.float32), mask=[0,0,0,1,0]) + f = createVariable( + numpy.ma.arange(5, typecode=numpy.float32), mask=[0, 0, 0, 1, 0]) + f2 = createVariable( + numpy.ma.arange(5, typecode=numpy.float32), mask=[0, 0, 0, 1, 0]) f[3] = numpy.ma.masked assert f[3] is numpy.ma.masked assert numpy.ma.allclose(2.0, f[2]) @@ -910,6 +941,6 @@ def asVariable(s, writeable=1): assert t.getdimattribute(0, 'name') == 'fudge' f2b = f2.getdimattribute(0, 'bounds') t.setdimattribute(0, 'bounds', f2b) - assert numpy.ma.allclose(f.getdimattribute(0,'bounds'), f2.getdimattribute(0,'bounds')) + assert numpy.ma.allclose( + f.getdimattribute(0, 'bounds'), f2.getdimattribute(0, 'bounds')) print "Transient Variable test passed ok." - diff --git a/Packages/cdms2/Lib/typeconv.py b/Packages/cdms2/Lib/typeconv.py index 0f2bd2c182..541785877c 100644 --- a/Packages/cdms2/Lib/typeconv.py +++ b/Packages/cdms2/Lib/typeconv.py @@ -4,21 +4,23 @@ oldtype2dtype = {'1': np.dtype(np.byte), 's': np.dtype(np.short), -# 'i': np.dtype(np.intc), -# 'l': np.dtype(int), -# 'b': np.dtype(np.ubyte), + # 'i': np.dtype(np.intc), + # 'l': np.dtype(int), + # 'b': np.dtype(np.ubyte), 'w': np.dtype(np.ushort), 'u': np.dtype(np.uintc), -# 'f': np.dtype(np.single), -# 'd': np.dtype(float), -# 'F': np.dtype(np.csingle), -# 'D': np.dtype(complex), -# 'O': np.dtype(object), -# 'c': np.dtype('c'), + # 'f': np.dtype(np.single), + # 'd': np.dtype(float), + # 'F': np.dtype(np.csingle), + # 'D': np.dtype(complex), + # 'O': np.dtype(object), + # 'c': np.dtype('c'), None: np.dtype(int) - } + } # converts typecode=None to int + + def convtypecode(typecode, dtype=None): if dtype is None: try: @@ -28,8 +30,10 @@ def convtypecode(typecode, dtype=None): else: return dtype -#if both typecode and dtype are None +# if both typecode and dtype are None # return None + + def convtypecode2(typecode, dtype=None): if dtype is None: if typecode is None: @@ -45,10 +49,12 @@ def convtypecode2(typecode, dtype=None): _changedtypes = {'B': 'b', 'b': '1', 'h': 's', -## 'H': 'w', + # 'H': 'w', 'I': 'u'} + class _oldtypecodes(dict): + def __getitem__(self, obj): char = np.dtype(obj).char try: diff --git a/Packages/cdms2/Lib/variable.py b/Packages/cdms2/Lib/variable.py index 23650e3d84..f271acc1dc 100644 --- a/Packages/cdms2/Lib/variable.py +++ b/Packages/cdms2/Lib/variable.py @@ -1,20 +1,20 @@ -## Automatically adapted for numpy.oldnumeric Aug 01, 2007 by +# Automatically adapted for numpy.oldnumeric Aug 01, 2007 by """ DatasetVariable: Dataset-based variables """ from cdms2 import Cdunif import numpy -import cdmsNode +from . import cdmsNode import cdtime import copy import os import sys -import cdmsobj -from cdmsobj import CdmsObj, getPathFromTemplate, Max32int -from avariable import AbstractVariable -from sliceut import * -from error import CDMSError +from . import cdmsobj +from .cdmsobj import CdmsObj, getPathFromTemplate, Max32int +from .avariable import AbstractVariable +from .sliceut import * +from .error import CDMSError InvalidGridElement = "Grid domain elements are not yet implemented: " InvalidRegion = "Invalid region: " @@ -24,44 +24,47 @@ WriteNotImplemented = "Dataset write operation not implemented" FileClosed = "Cannot read from closed file or dataset, variable: " + def timeindex(value, units, basetime, delta, delunits, calendar): """ Calculate (t - basetime)/delu where t = reltime(value, units) and delu is the time interval (delta, delunits) (e.g., 1 month). """ tval = cdtime.reltime(value, units) - tounits = "%s since %s"%(delunits, basetime) + tounits = "%s since %s" % (delunits, basetime) newval = tval.torel(tounits, calendar) - return int(newval.value/delta) + return int(newval.value / delta) + class DatasetVariable(AbstractVariable): - def __init__(self,parent,id, variableNode=None): + def __init__(self, parent, id, variableNode=None): """ "Variable (parent, variableNode=None)" variableNode is the variable tree node, if any. parent is the containing dataset instance. """ - AbstractVariable.__init__ (self, parent, variableNode) - val = self.__cdms_internals__ + ['domain','name_in_file'] + AbstractVariable.__init__(self, parent, variableNode) + val = self.__cdms_internals__ + ['domain', 'name_in_file'] self.___cdms_internals__ = val self.id = id self.domain = [] # Get self.name_in_file from the .xml file if present if not hasattr(self, 'name_in_file'): self.name_in_file = id - + # if self.attributes.has_key('name_in_file'): # self.name_in_file = self.attributes['name_in_file'] - if variableNode is not None: - self._numericType_ = cdmsNode.CdToNumericType.get(variableNode.datatype) + if variableNode is not None: + self._numericType_ = cdmsNode.CdToNumericType.get( + variableNode.datatype) else: self._numericType_ = numpy.float assert self.id is not None - - def __len__ (self): + + def __len__(self): "Length of first dimension" - if len(self.domain)>0: - (axis,start,length,true_length) = self.domain[0] + if len(self.domain) > 0: + (axis, start, length, true_length) = self.domain[0] else: length = 0 @@ -72,33 +75,34 @@ def __len__ (self): # parentid = self.parent.id # else: # parentid = "**CLOSED**" -# return ""%(self.id, parentid, `self.shape`) +# return ""%(self.id, parentid, +# `self.shape`) def __getitem__(self, key): if self.parent is None: - raise CDMSError, FileClosed+str(self.id) + raise CDMSError(FileClosed + str(self.id)) return AbstractVariable.__getitem__(self, key) - + def getValue(self, squeeze=1): """Return the entire set of values.""" if self.parent is None: - raise CDMSError, FileClosed+self.id + raise CDMSError(FileClosed + self.id) return self.getSlice(Ellipsis, squeeze=squeeze) - + def __getslice__(self, low, high): if self.parent is None: - raise CDMSError, FileClosed+self.id + raise CDMSError(FileClosed + self.id) # Hack to prevent netCDF overflow error on 64-bit architectures high = min(Max32int, high) - + return AbstractVariable.__getslice__(self, low, high) def __setitem__(self, index, value): - raise CDMSError, WriteNotImplemented + raise CDMSError(WriteNotImplemented) def __setslice__(self, low, high, value): - raise CDMSError, WriteNotImplemented + raise CDMSError(WriteNotImplemented) def _getShape(self): return self.getShape() @@ -108,19 +112,19 @@ def _getdtype(self): return numpy.dtype(tc) def getShape(self): - shape=[] - for (axis,start,length,true_length) in self.domain: + shape = [] + for (axis, start, length, true_length) in self.domain: shape.append(length) return tuple(shape) - def typecode (self): - return numpy.dtype(self._numericType_).char + def typecode(self): + return numpy.dtype(self._numericType_).char def size(self): "Number of elements." n = 1 for k in self.shape: - n = k*n + n = k * n return n def initDomain(self, axisdict, griddict): @@ -133,9 +137,9 @@ def initDomain(self, axisdict, griddict): if domelem is None: domelem = griddict.get(dename) if grid is None: - raise CDMSError, NoSuchAxisOrGrid + dename + raise CDMSError(NoSuchAxisOrGrid + dename) else: - raise CDMSError, InvalidGridElement + dename + raise CDMSError(InvalidGridElement + dename) partlenstr = denode.getExternalAttr('partition_length') if partlenstr is not None: truelen = int(partlenstr) @@ -145,26 +149,27 @@ def initDomain(self, axisdict, griddict): # Get the template def getTemplate(self): - if hasattr(self,'template'): + if hasattr(self, 'template'): template = self.template - elif hasattr(self.parent,'template'): + elif hasattr(self.parent, 'template'): template = self.parent.template else: template = None return template - def getAxis (self, n): - if n < 0: n = n + self.rank() + def getAxis(self, n): + if n < 0: + n = n + self.rank() return self.domain[n][0] - def getDomain (self): + def getDomain(self): return self.domain # Get the paths associated with the interval region specified # by 'intervals'. This incorporates most of the logic of __getitem__, # without actually reading the data. - # - # 'specs' is a list of interval range specifications as defined + # + # 'specs' is a list of interval range specifications as defined # for getSlice. # # The function returns a list of tuples of the form (path,slicelist), @@ -176,11 +181,11 @@ def getDomain (self): # f = Cdunif.CdunifFile(path,'r') # var = f.variables[self.name_in_file] # data = apply(var.getitem,slicelist) - # + # def getPaths(self, *specs, **keys): # Create an equivalent list of slices - speclist = self._process_specs (specs, keys) + speclist = self._process_specs(specs, keys) slicelist = self.specs2slices(speclist) # Generate the filelist @@ -190,20 +195,20 @@ def getPaths(self, *specs, **keys): result = [] if partitionSlices is None: pass - elif npart==0: + elif npart == 0: filename, slicelist = partitionSlices if filename is not None: result.append((filename, tuple(slicelist))) - elif npart==1: + elif npart == 1: for filename, slicelist in partitionSlices: if filename is not None: result.append((filename, tuple(slicelist))) - elif npart==2: + elif npart == 2: for filelist in partitionSlices: for filename, slicelist in filelist: if filename is not None: result.append((filename, tuple(slicelist))) - + return result def genMatch(self, axis, interval, matchnames): @@ -217,34 +222,34 @@ def genMatch(self, axis, interval, matchnames): returns the modified matchnames tuple. """ if axis.isTime(): - if hasattr(self.parent,'cdms_filemap'): + if hasattr(self.parent, 'cdms_filemap'): start = interval[0] end = interval[1] else: # Use template method time0 = axis[interval[0]] - time1 = axis[interval[1]-1] - isabs = (axis.units.find(" as ")!=-1) + time1 = axis[interval[1] - 1] + isabs = (axis.units.find(" as ") != -1) if isabs: - start = cdtime.abstime(time0,axis.units) - end = cdtime.abstime(time1,axis.units) + start = cdtime.abstime(time0, axis.units) + end = cdtime.abstime(time1, axis.units) else: cal = axis.getCalendar() - start = cdtime.reltime(time0,axis.units).tocomp(cal) - end = cdtime.reltime(time1,axis.units).tocomp(cal) + start = cdtime.reltime(time0, axis.units).tocomp(cal) + end = cdtime.reltime(time1, axis.units).tocomp(cal) matchnames[1] = start matchnames[2] = end elif axis.isForecast(): start = axis.getValue()[interval[0]] - end = axis.getValue()[interval[1]-1] + end = axis.getValue()[interval[1] - 1] matchnames[5] = start matchnames[6] = end else: - if hasattr(self.parent,'cdms_filemap'): + if hasattr(self.parent, 'cdms_filemap'): start = interval[0] end = interval[1] else: start = int(axis[interval[0]]) - end = int(axis[interval[1]-1]) + end = int(axis[interval[1] - 1]) matchnames[3] = start matchnames[4] = end @@ -254,12 +259,13 @@ def getFilePath(self, matchnames, template): """Lookup or generate the file path, depending on whether a filemap or template is present. """ - if hasattr(self.parent,'cdms_filemap'): + if hasattr(self.parent, 'cdms_filemap'): id, tstart, tend, levstart, levend, fcstart, fcend = matchnames - filename = self.parent._filemap_[(self.id, tstart, levstart, fcstart)] + filename = self.parent._filemap_[ + (self.id, tstart, levstart, fcstart)] # ... filemap uses dataset IDs else: - filename = getPathFromTemplate(template,matchnames) + filename = getPathFromTemplate(template, matchnames) return filename def getPartition(self, axis): @@ -268,7 +274,7 @@ def getPartition(self, axis): get the partition from the _varpart_ attribute, otherwise (for templating) use axis.partition. """ - if hasattr(self.parent,'cdms_filemap'): + if hasattr(self.parent, 'cdms_filemap'): if axis.isTime(): partition = self._varpart_[0] elif axis.isForecast(): @@ -279,7 +285,7 @@ def getPartition(self, axis): partition = axis.partition return partition - def expertPaths (self, slist): + def expertPaths(self, slist): """ expertPaths(self, slicelist) takes a list of slices, returns a 3-tuple: (npart, dimensionlist, partitionSlices) where: @@ -316,7 +322,7 @@ def expertPaths (self, slist): # Handle rank-0 variables separately if self.rank() == 0: - matchnames = [realid,None,None,None,None,None,None] + matchnames = [realid, None, None, None, None, None, None] filename = self.getFilePath(matchnames, template) result = (0, (), (filename, [])) @@ -325,162 +331,164 @@ def expertPaths (self, slist): # Find the number of partitioned axes npart = 0 ndim = 0 - for (axis,start,length,true_length) in self.domain: - if hasattr(axis,'partition'): - npart = npart+1 - if npart==1: + for (axis, start, length, true_length) in self.domain: + if hasattr(axis, 'partition'): + npart = npart + 1 + if npart == 1: part1 = axis npart1 = ndim - elif npart==2: + elif npart == 2: part2 = axis npart2 = ndim else: - raise CDMSError, TooManyPartitions + variable.id - ndim = ndim+1 + raise CDMSError(TooManyPartitions + variable.id) + ndim = ndim + 1 # If no partitioned axes, just read the data - if npart==0: - matchnames = [realid,None,None,None,None,None,None] + if npart == 0: + matchnames = [realid, None, None, None, None, None, None] filename = self.getFilePath(matchnames, template) result = (0, (), (filename, slicelist)) # If one partitioned axes: - elif npart==1: + elif npart == 1: # intersect the slice and partition for that axis slice1 = slicelist[npart1] - (axis,startelem,length,true_length) = self.domain[npart1] + (axis, startelem, length, true_length) = self.domain[npart1] partition = slicePartition(slice1, self.getPartition(axis)) - if partition==[]: + if partition == []: return (1, (npart1,), None) # For each (interval, partslice) in the partition: resultlist = [] (firstinterval, firstslice) = partition[0] prevhigh = firstinterval[0] - for (interval,partslice) in partition: + for (interval, partslice) in partition: # If the previous interval high is less than # the current interval low value, interpose # missing data. low = interval[0] - if prevhigh1, it might not intersect, # so don't interpose missing data in this case. if missing_slice is not None: slicelist[npart1] = missing_slice - resultlist.append((None,copy.copy(slicelist))) + resultlist.append((None, copy.copy(slicelist))) prevhigh = interval[1] # generate the filename - matchnames = [realid, None, None, None, None,None,None] + matchnames = [realid, None, None, None, None, None, None] matchnames = self.genMatch(axis, interval, matchnames) filename = self.getFilePath(matchnames, template) # adjust the partslice for the interval offset # and replace in the slice list - filestart = partslice.start-interval[0] - filestop = partslice.stop-interval[0] - fileslice = slice(filestart,filestop,partslice.step) + filestart = partslice.start - interval[0] + filestop = partslice.stop - interval[0] + fileslice = slice(filestart, filestop, partslice.step) slicelist[npart1] = fileslice - resultlist.append((filename,copy.copy(slicelist))) + resultlist.append((filename, copy.copy(slicelist))) - result = (1,(npart1,),resultlist) + result = (1, (npart1,), resultlist) # If two partitioned axes, 2-D version of previous case - if npart==2: + if npart == 2: slice1 = slicelist[npart1] slice2 = slicelist[npart2] - (axis1,startelem1,length1,true_length1) = self.domain[npart1] - (axis2,startelem2,length2,true_length2) = self.domain[npart2] + (axis1, startelem1, length1, true_length1) = self.domain[npart1] + (axis2, startelem2, length2, true_length2) = self.domain[npart2] partition1 = slicePartition(slice1, self.getPartition(axis1)) partition2 = slicePartition(slice2, self.getPartition(axis2)) - if partition1==[] or partition2==[]: - return (2, (npart1,npart2), None) + if partition1 == [] or partition2 == []: + return (2, (npart1, npart2), None) # For each (interval, partslice) in the partition: resultlist = [] (firstinterval1, firstslice1) = partition1[0] prevhigh1 = firstinterval1[0] - for (interval1,partslice1) in partition1: + for (interval1, partslice1) in partition1: # If the previous interval high is less than # the current interval low value, interpose # missing data. low = interval1[0] - if prevhigh1 Date: Wed, 27 Apr 2016 09:36:48 -0700 Subject: [PATCH 11/11] begining of adressing flake8 but giving up --- Packages/cdms2/Lib/restApi.py | 126 ++++------------------------ Packages/cdms2/Lib/selectors.py | 4 +- Packages/cdms2/Lib/slabinterface.py | 3 +- Packages/cdms2/Lib/tvariable.py | 26 +++--- Packages/cdms2/Lib/typeconv.py | 2 +- Packages/cdms2/Lib/variable.py | 15 +--- Packages/cdms2/Lib/xlink.py | 11 --- 7 files changed, 32 insertions(+), 155 deletions(-) delete mode 100644 Packages/cdms2/Lib/xlink.py diff --git a/Packages/cdms2/Lib/restApi.py b/Packages/cdms2/Lib/restApi.py index 0ee17424da..159d50112a 100644 --- a/Packages/cdms2/Lib/restApi.py +++ b/Packages/cdms2/Lib/restApi.py @@ -1,4 +1,3 @@ -import cdms2 import urllib2 import xml.etree.ElementTree try: @@ -173,7 +172,7 @@ def __getitem__(self, key): return val def __setitem__(self, key, value): - if not key in self.params.keys(): + if key not in self.params.keys(): raise self.EsgfObjectException( "Invalid key: %s, valid keys are: %s" % (repr(key), repr(self.params.keys()))) @@ -183,7 +182,7 @@ def __setitem__(self, key, value): def _search(self, search="", searchType=None, stringType=False): if searchType is None: searchType = self.defaultSearchType - if not searchType in self.validSearchTypes: + if searchType not in self.validSearchTypes: raise self.EsgfObjectException( "Valid Search types are: %s" % repr(self.validSearchTypes)) @@ -226,14 +225,8 @@ def generateRequest(self, stringType=False, **keys): # params[k]=self[k] for k in keys.keys(): - if k == "stringType": - stringType = keys[k] + if k in ["stringType", "type"]: continue - elif k == "type": - continue - # elif not k in self.searchableKeys: - # raise self.EsgfObjectException("Invalid key: %s, valid keys are: - # %s" % (repr(k),repr(self.params.keys()))) if keys[k] is not None: params[k] = keys[k] @@ -309,23 +302,18 @@ def searchDatasets(self, **keys): resps = self.request(**keys) stringType = keys.get("stringType", False) if stringType: - return resp + return resps datasets = [] for resp in resps: for r in resp[:]: if r.tag == "result": # Ok let's go thru these datasets for d in r[:]: - # print - # "************************************************" tmpkeys = {} for f in d[:]: k = f.get("name") tmpkeys[k] = self.extractTag(f) if tmpkeys["type"] == "Dataset": - datasetid = tmpkeys["id"] - # print datasetid,self.restPath - # print "KEYS FOR DATASET",keys.keys() datasets.append( esgfDataset(host=self.host, port=self.port, @@ -364,7 +352,7 @@ def __init__(self, host=None, port=80, limit=1000, offset=0, mapping=None, tmp.replace(")s", ")")) elif "project" in keys and keys["project"] == "cmip5": self.datasetids = genutil.StringConstructor( - "%(project).%(product).%(institute).%(model).%(experiment).%(time_frequency).%(realm).%(cmor_table).%(ensemble)") + "%(project).%(product).%(institute).%(model).%(experiment).%(time_frequency).%(realm).%(cmor_table).%(ensemble)") # noqa else: self.datasetids = None if isinstance(datasetids, genutil.StringConstructor): @@ -393,38 +381,16 @@ def __init__(self, host=None, port=80, limit=1000, offset=0, mapping=None, self.keys = self.params.keys self.items = self.params.items self.values = self.params.values - # self.id=self["id"] self.params["limit"] = limit self.params["offset"] = offset self.mapping = mapping - # print "SEARCHING DS:",originalKeys self.resp = None self.cacheTime = None -# self.search() -# self.remap() - - # Ok now we need to "map" this according to the user wishes - - # def mappedItems(): - # mapped=[] - # mapppoint=self.mapped - # for k in self.mapping.keys(): - # keys=[] - # level=[k,mappoint.keys()] - # mappoint + def _extractFiles(self, resp, **inKeys): # We need to stick in there the bit from Luca to fill in the matching # key from facet for now it's empty files = [] - skipped = [ - "type", - "title", - "timestamp", - "service", - "id", - "score", - "file_url", - "service_type"] for r in resp[:]: if r.tag == "result": for d in r[:][:]: @@ -433,66 +399,6 @@ def _extractFiles(self, resp, **inKeys): k = f.get("name") keys[k] = self.extractTag(f) if keys["type"] == "File": - # if self["id"]=="obs4MIPs.NASA-JPL.AIRS.mon": - # verbose=True - # else: - # verbose=False - # verbose=True - # if verbose: print "OK",keys["variable"],keys["file_id"],self["id"] - # if verbose: print "FILEIDS:",self.fileids - # if verbose: print "Fileids:",self.fileids.template - # if verbose: print "keys:",keys - # if self.fileids is not None: - # try: - # if verbose: print "file:",keys["file_id"],self.fileids.template - # k2 = self.fileids.reverse(keys["file_id"]) - # if verbose: print "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@",k2 - # for k in k2.keys(): - # keys[k]=k2[k] - # except: - # if verbose: print "Failed:",ids[i].text,self.fileids.template - # pass - # if verbose: print "KEYS FOR FILE:",keys.keys() - # if verbose: print "INKEYS:",inKeys.keys() - # matched = True - # matchWithKeys = {} - # for k in self.keys(): - # if k in self.originalKeys.keys(): - # matchWithKeys[k]=self.originalKeys[k] - # else: - # matchWithKeys[k]=self[k] - # for s in skipped: - # try: - # matchWithKeys.pop(s) - # except: - # pass - # for k in inKeys.keys(): - # matchWithKeys[k]=inKeys[k] - # if verbose: print "matching:",matchWithKeys.keys() - # for k in keys.keys(): - # if k in matchWithKeys.keys(): - # if verbose: print "Testing:",k,keys[k] - # v = matchWithKeys[k] - # if isinstance(v,(str,int,float)): - # if verbose: print "\tComparing with:",v - # if v != keys[k]: - # matched = False - # if verbose: print "\t\tNOPE" - # break - # elif isinstance(v,list): - # if verbose: print "\tComparing with (and %i more):%s"%(len(v),v[0]),v - # if not keys[k] in v: - # matched = False - # if verbose: print "\t\tNOPE" - # break - # else: - # print "\twould compare %s with type: %s if I knew how to" % (str(v),type(v)) - # if verbose: print keys["file_id"],matched - # if matched : - # for k in self.keys(): - # if not k in keys.keys(): - # keys[k]=self[k] - # print "KEYS:",keys files.append(esgfFile(**keys)) return files @@ -514,7 +420,7 @@ def saveCache(self, target="."): if os.path.isdir(target): target = os.path.join(target, "esgfDatasetsCache.pckl") if os.path.exists(target): - f = open(source) + f = open(target) # dict=eval(bz2.decompress(f.read())) dict = eval(f.read()) f.close() @@ -560,9 +466,9 @@ def search(self, **keys): stringType = keys.get("stringType", False) keys.update(self.originalKeys) st = "" - if not "limit" in keys: + if "limit" not in keys: keys["limit"] = [self["limit"]] - if not "offset" in keys: + if "offset" not in keys: keys["offset"] = [self["offset"]] for k in keys: if k in ["searchString", "stringType", ]: @@ -615,7 +521,8 @@ def __init__(self, files, parent, self.setMapping(mapping) self.remap() self.projects_dict = { - "CMIP5": "%(project).%(product).%(institute).%(model).%(experiment).%(time_frequency).%(realm).%(cmor_table).%(ensemble)"} + "CMIP5": "%(project).%(product).%(institute).%(model).%(experiment).%(time_frequency).%(realm).%(cmor_table).%(ensemble)" # noqa + } def __getitem__(self, item): if isinstance(item, int): @@ -654,7 +561,7 @@ def setMapping(self, mapping): self.mapping = self.datasetids else: for k in self.parent.keys(): - if not k in ["limit", "offset", "text"]: + if k not in ["limit", "offset", "text"]: self.mapping += "%%(%s)" % k else: self.mapping = mapping @@ -731,24 +638,21 @@ def remap(self, mapping=None, verbose=False): cont = f[k] if not isinstance(cont, (str, int, float)): break - if not cont in mappoint.keys(): + if cont not in mappoint.keys(): mappoint[cont] = {} elif k in self.parent.keys(): # if verbose: print tabs,k,f[k] nok += 1 cont = self[k] - if not cont in mappoint.keys(): + if cont not in mappoint.keys(): mappoint[cont] = {} elif isinstance(self.fileids, genutil.StringConstructor): try: mapid = self.fileids.reverse(self.parent.id) - # if verbose: - # print "MAPID:",k,mapid if k in mapid.keys(): - # if verbose: print tabs,k,mapid[k] nok += 1 cont = mapid[k] - if not cont in mappoint.keys(): + if cont not in mappoint.keys(): mappoint[cont] = {} except: break diff --git a/Packages/cdms2/Lib/selectors.py b/Packages/cdms2/Lib/selectors.py index 2a35705d1d..9ec6c6721a 100644 --- a/Packages/cdms2/Lib/selectors.py +++ b/Packages/cdms2/Lib/selectors.py @@ -1,6 +1,5 @@ """Classes to support easy selection of climate data""" -import cdtime from .axis import axisMatches from .error import CDMSError from .grid import AbstractRectGrid, defaultRegion, setRegionSpecs, LongitudeType, LatitudeType, TimeType, VerticalType @@ -203,7 +202,7 @@ def unmodified_select( grid is not None or \ raw != 0 or \ result is variable: - # result is variable when there are no components, for example. + # result is variable when there are no components, for example. return result.subRegion(squeeze=squeeze, order=order, grid=grid, raw=raw) else: @@ -371,7 +370,6 @@ def __init__(self, v): def specify(self, slab, axes, specifications, confined_by, aux): "Find the next unconfined axis and confine it." - n = 0 for i in range(len(axes)): if confined_by[i] is None: specifications[i] = self.v diff --git a/Packages/cdms2/Lib/slabinterface.py b/Packages/cdms2/Lib/slabinterface.py index f12738ee7d..a0fc8910e1 100644 --- a/Packages/cdms2/Lib/slabinterface.py +++ b/Packages/cdms2/Lib/slabinterface.py @@ -95,7 +95,7 @@ def listdimattributes(self, dim): a = self.getAxis(dim) result = [] for x in std_axis_attributes + a.attributes.keys(): - if not x in result: + if x not in result: result.append(x) return result @@ -192,7 +192,6 @@ def info(self, flag=None, device=None): def cdms_bounds2cu_bounds(b): "Bounds are len(v) by 2 in cdms but len(v)+1 in cu" cub = numpy.ma.zeros(len(b) + 1, numpy.float32) - b1 = b.astype(numpy.float32) if len(b) > 1: if (b[0, 0] < b[0, 1]) == (b[0, 0] < b[-1, 0]): cub[0] = b[0, 0] diff --git a/Packages/cdms2/Lib/tvariable.py b/Packages/cdms2/Lib/tvariable.py index 9ebcca9c6d..0a2b3cdd55 100644 --- a/Packages/cdms2/Lib/tvariable.py +++ b/Packages/cdms2/Lib/tvariable.py @@ -43,14 +43,14 @@ def fromJSON(jsn): dtype=a["_dtype"]), id=a["id"]) for k, v in a.iteritems(): - if not k in ["_values", "id", "_dtype"]: + if k not in ["_values", "id", "_dtype"]: setattr(ax, k, v) axes.append(ax) # Now prep the variable V = createVariable(D["_values"], id=D["id"], typecode=D["_dtype"]) V.setAxisList(axes) for k, v in D.iteritems(): - if not k in ["id", "_values", "_axes", "_grid", "_fill_value", "_dtype", ]: + if k not in ["id", "_values", "_axes", "_grid", "_fill_value", "_dtype", ]: setattr(V, k, v) V.set_fill_value(D["_fill_value"]) return V @@ -103,12 +103,10 @@ def _update_from(self, obj): try: for nm, val in obj.__dict__.items(): if nm[0] == '_': -# print nm pass -# self.__dict__[nm]=val else: setattr(self, nm, val) - except Exception as err: + except Exception: pass id = getattr(self, 'id', None) if id is None: @@ -193,14 +191,12 @@ def __init__(self, data, typecode=None, copy=1, savespace=0, grid = grid.reconcile( axes) # Make sure grid and axes are consistent - ncopy = (copy != 0) - # Initialize the geometry if grid is not None: copyaxes = 0 # Otherwise grid axes won't match domain. if axes is not None: self.initDomain(axes, copyaxes=copyaxes) - # Note: clobbers the grid, so set the grid after. + # Note: clobbers the grid, so set the grid after. if grid is not None: self.setGrid(grid) @@ -277,7 +273,7 @@ def __new__(cls, data, typecode=None, copy=0, savespace=0, if mask is None: try: mask = data.mask - except Exception as err: + except Exception: mask = numpy.ma.nomask # Handle the case where ar[i:j] returns a single masked value @@ -359,7 +355,7 @@ def initDomain(self, axes, copyaxes=1): def getDomain(self): for i in range(self.rank()): if self.__domain[i] is None: - junk = self.getAxis(i) # will force a fill in + self.getAxis(i) # will force a fill in return self.__domain def getAxis(self, n): @@ -442,7 +438,7 @@ def astype(self, tc): maresult = numpy.ma.MaskedArray.astype(self, tc) return TransientVariable( maresult, copy=0, axes=self.getAxisList(), fill_value=self.fill_value, - attributes=self.attributes, id=self.id, grid=self.getGrid()) + attributes=self.attributes, id=self.id, grid=self.getGrid()) def setMaskFromGridMask(self, mask, gridindices): """Set the mask for self, given a grid mask and the variable domain @@ -626,12 +622,11 @@ def toVisit(self, filename, format='Vs', sphereRadius=1.0, sphereRadius: radius of the earth maxElev: maximum elevation for representation on the sphere """ - from . import mvSphereMesh from . import mvVTKSGWriter from . import mvVsWriter try: # required by mvVsWriter - import tables + import tables # noqa except: # fall back format = 'VTK' @@ -739,8 +734,7 @@ def exposeHalo(self, ghostWidth=1): # given direction, a 1 represents a layer of # thickness ghostWidth on the high index side, # -1 on the low index side. - winId = tuple([0 for i in range(dim)] - + [drect] + + winId = tuple([0 for i in range(dim)] + [drect] + [0 for i in range(dim + 1, ndims)]) slce = slice(0, ghostWidth) @@ -832,7 +826,7 @@ def __getSlab(self, dim, slce): """ ndims = len(self.shape) - slab = [ slice(0, None) for i in range(dim) ] \ + slab = [slice(0, None) for i in range(dim)] \ + [slce] + \ [slice(0, None) for i in range(dim + 1, ndims)] return tuple(slab) diff --git a/Packages/cdms2/Lib/typeconv.py b/Packages/cdms2/Lib/typeconv.py index 541785877c..e63ad64d34 100644 --- a/Packages/cdms2/Lib/typeconv.py +++ b/Packages/cdms2/Lib/typeconv.py @@ -1,6 +1,6 @@ +import numpy as np __all__ = ['oldtype2dtype', 'convtypecode', 'convtypecode2', 'oldtypecodes'] -import numpy as np oldtype2dtype = {'1': np.dtype(np.byte), 's': np.dtype(np.short), diff --git a/Packages/cdms2/Lib/variable.py b/Packages/cdms2/Lib/variable.py index f271acc1dc..c96dd5e215 100644 --- a/Packages/cdms2/Lib/variable.py +++ b/Packages/cdms2/Lib/variable.py @@ -3,17 +3,13 @@ """ DatasetVariable: Dataset-based variables """ -from cdms2 import Cdunif import numpy from . import cdmsNode import cdtime import copy -import os -import sys -from . import cdmsobj -from .cdmsobj import CdmsObj, getPathFromTemplate, Max32int +from .cdmsobj import getPathFromTemplate, Max32int from .avariable import AbstractVariable -from .sliceut import * +from .sliceut import * # noqa from .error import CDMSError InvalidGridElement = "Grid domain elements are not yet implemented: " @@ -335,10 +331,8 @@ def expertPaths(self, slist): if hasattr(axis, 'partition'): npart = npart + 1 if npart == 1: - part1 = axis npart1 = ndim elif npart == 2: - part2 = axis npart2 = ndim else: raise CDMSError(TooManyPartitions + variable.id) @@ -476,7 +470,7 @@ def expertSlice(self, initslist): # Handle negative slices revlist = [] - # Slices to apply to result if reversals needed + # Slices to apply to result if reversals needed slist = [] # Slices with positive strides haveReversals = 0 # True iff result array needs reversing i = 0 @@ -500,7 +494,6 @@ def expertSlice(self, initslist): for i in range(len(self.domain)): if self.domain[i][0].isForecast(): fci = i - fcv = initslist[i].start break # If no intersection, return an 'empty' array. @@ -536,7 +529,7 @@ def expertSlice(self, initslist): result = self._returnArray( var.getitem( *tuple(slicelist[0:fci] + slicelist[fci + 1:])), - 0) + 0) # But the result still needs an index in the forecast direction, # which is simple to do because there is only one forecast # per file: diff --git a/Packages/cdms2/Lib/xlink.py b/Packages/cdms2/Lib/xlink.py deleted file mode 100644 index 6270a75098..0000000000 --- a/Packages/cdms2/Lib/xlink.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -CDMS Xlink objects - pointers to other objects -""" - - -class Xlink(CdmsObj): - - def __init__(self, xlinkNode=None): - assert xlinkNode is None or xlinkNode.tag == 'xlink',\ - 'Node is not a link node' - CdmsObj.__init__(self, xlinkNode)