More early resolve issues

cime_config/cesm/config_files.xml

@@ -66,7 +66,7 @@
 
   <entry id="CONFIG_TESTS_FILE">
     <type>char</type>
-    <default_value>$CIMEROOT/scripts/Testing/Testcases/config_tests.xml</default_value>
+    <default_value>$CIMEROOT/cime_config/config_tests.xml</default_value>
     <group>test</group>
     <file>env_test.xml</file>
     <desc>file containing system test descriptions </desc>

cime_config/cesm/machines/config_batch.xml

@@ -424,7 +424,8 @@
 
   <batch_system MACH="yellowstone" type="lsf" version="9.1">
     <queues>
-      <queue walltimemax="24:00" jobmin="1" jobmax="8">caldera</queue>
+      <queue walltimemax="24:00" jobmin="1" jobmax="1" jobname="case.lt_archive">hpss</queue>
+      <queue walltimemax="24:00" jobmin="1" jobmax="8" >caldera</queue>
       <queue walltimemax="12:00" jobmin="9" jobmax="16384" default="true">regular</queue>
       <queue walltimemax="12:00" jobmin="16385" jobmax="65536">capability</queue>
     </queues>

components/data_comps/dice/bld/build-namelist

@@ -306,6 +306,9 @@ if (defined $opts{'infile'}) {
 
 my %xmlvars = ();
 SetupTools::getxmlvars(${CASEROOT},\%xmlvars);
+# need to expand DIN_LOC_ROOT first
+$xmlvars{DIN_LOC_ROOT}=SetupTools::expand_xml_var($xmlvars{DIN_LOC_ROOT}, \%xmlvars);
+
 foreach my $attr (keys %xmlvars) {
     $xmlvars{$attr} = SetupTools::expand_xml_var($xmlvars{$attr}, \%xmlvars);
 }

driver_cpl/cime_config/config_component.xml

@@ -1146,14 +1146,6 @@
     <desc>number of wav cells in j direction - DO NOT EDIT (for experts only)</desc>
   </entry>
 
-  <entry id="MASK_GRID">
-    <type>char</type>
-    <default_value>UNSET</default_value>
-    <group>build_grid</group>
-    <file>env_build.xml</file>
-    <desc>grid mask - DO NOT EDIT (for experts only)</desc>
-  </entry>
-
   <entry id="PTS_MODE">
     <type>logical</type>
     <valid_values>TRUE,FALSE</valid_values>
@@ -1766,14 +1758,6 @@
     <desc>Machine name</desc>
   </entry>
 
-  <entry id="MACHINES_FILE">
-    <type>char</type>
-    <default_value></default_value>
-    <group>case_def</group>
-    <file>env_case.xml</file>
-    <desc>full pathname of file specifying supported machines location</desc>
-  </entry>
-
   <entry id="MACHDIR">
     <type>char</type>
     <default_value></default_value>

scripts/create_newcase

@@ -154,9 +154,9 @@ def _main_func():
 
     with Case(caseroot, read_only=False) as case:
         # Set values for env_case.xml
-        case.set_value("CASE", os.path.basename(caseroot))
-        case.set_value("CASEROOT", caseroot)
-        case.set_value("SRCROOT", srcroot)
+        case.set_lookup_value("CASE", os.path.basename(caseroot))
+        case.set_lookup_value("CASEROOT", caseroot)
+        case.set_lookup_value("SRCROOT", srcroot)
 
         # Configure the Case
         case.configure(compset, grid, machine_name=machine, project=project,

utils/perl5lib/Streams/TemplateGeneric.pm

@@ -425,7 +425,9 @@ sub GetDataFilenames {
   #
   # Get path
   #
+
   my $filepath = $self->GetDataFilepath( $type );
+
   my $key = "fileNames";
   my $info;
   if (      $type eq "data" ) {
@@ -463,8 +465,8 @@ sub GetDataFilepath {
   if ( ! defined($self->{'template'}) ) {
      die "${nm}:: a template has NOT been read in yet -- abort.\n";
   }
-  my $defaults_ref = $self->{'defaults'};
-  my %defaults     = %$defaults_ref;
+  my %defaults = %{$self->{'defaults'}};
+
   my $key;
   if (      $type eq "data" ) {
      $key = "fieldInfo";
@@ -473,13 +475,14 @@ sub GetDataFilepath {
   } else {
      die "${nm}:: bad input type to method: $type should be data or domain\n";
   }
+
   my $Info_ref = $defaults{$key};
+
   if ( ref($Info_ref) ne "HASH" ) {
      die "${nm}:: $key is NOT a hash -- something must have went wrong in the Read\n";
   }
-  my %Info = %$Info_ref;
-  my $filepath = $self->__Sub__( $Info_ref, 'filePath');
 
+  my $filepath = $self->__Sub__( $Info_ref, 'filePath');
   return( $filepath );
 }
 
@@ -490,7 +493,6 @@ sub expandXMLVar {
     my $value        = shift;
     my $varhash_ref  = shift;
     my $nm = "expandXMLVar";
-
     if ( ! defined($value) ) {
        die "${nm}:: a value was NOT input\n";
     }
@@ -559,6 +561,7 @@ sub __Sub__ {
    my $lastmonth = $opts{'lastmonth'};
 
    my $value = $$Info_ref{$name};
+
    $value =~  s/^[ \n]+//;          # remove leading spaces
    $value =~  s/[ \n]+$//;          # remove ending spaces
 

utils/python/CIME/XML/env_batch.py

@@ -276,7 +276,7 @@ def set_job_defaults(self, bjobs, pesize=None, walltime=None, force_queue=None):
             else:
                 task_count = int(task_count)
 
-            queue = force_queue if force_queue is not None else self.select_best_queue(task_count)
+            queue = force_queue if force_queue is not None else self.select_best_queue(task_count, job)
             self.set_value("JOB_QUEUE", queue, subgroup=job)
 
             walltime = self.get_max_walltime(queue) if walltime is None else walltime
@@ -330,6 +330,8 @@ def get_submit_args(self, case, job):
             if name is None:
                 submitargs+=" %s"%flag
             else:
+                if name.startswith("$"):
+                    name = name[1:]
                 val = case.get_value(name,subgroup=job)
                 if val is None:
                     val = case.get_resolved_value(name)
@@ -464,7 +466,7 @@ def get_job_id(self, output):
         jobid = re.search(jobid_pattern, output).group(1)
         return jobid
 
-    def select_best_queue(self, num_pes):
+    def select_best_queue(self, num_pes, job=None):
         # Make sure to check default queue first.
         all_queues = []
         all_queues.append( self.get_default_queue())
@@ -473,8 +475,12 @@ def select_best_queue(self, num_pes):
             if queue is not None:
                 jobmin = queue.get("jobmin")
                 jobmax = queue.get("jobmax")
+                jobname = queue.get("jobname")
+                if jobname is not None:
+                    if job == jobname:
+                        return queue.text
                 # if the fullsum is between the min and max # jobs, then use this queue.
-                if jobmin is not None and jobmax is not None and num_pes >= int(jobmin) and num_pes <= int(jobmax):
+                elif jobmin is not None and jobmax is not None and num_pes >= int(jobmin) and num_pes <= int(jobmax):
                     return queue.text
         return None
 

utils/python/CIME/XML/machines.py

@@ -27,9 +27,9 @@ def __init__(self, infile=None, files=None, machine=None):
         if infile is None:
             if files is None:
                 files = Files()
-            infile = files.get_value("MACHINES_SPEC_FILE")
+            infile = files.get_value("MACHINES_SPEC_FILE", resolved=False)
             self.machines_dir = os.path.dirname(infile)
-
+            infile = files.get_resolved_value(infile)
         GenericXML.__init__(self, infile)
 
         # Append the contents of $HOME/.cime/config_machines.xml if it exists
@@ -340,7 +340,7 @@ def get_full_mpirun(self, check_members, case, job):
         batch_system = self.get_value("BATCH_SYSTEM")
         if batch_system == "cobalt":
             mpi_arg_string += " : "
-        
+
         return "%s %s %s" % (executable if executable is not None else "", mpi_arg_string, default_run_suffix)
 
     def print_values(self):

utils/python/CIME/buildnml.py

@@ -154,7 +154,9 @@ def _build_data_nml(case, caseroot, compclass):
 
         rc, out, err = run_cmd(cmd, from_dir=confdir)
         expect(rc==0,"Command %s failed rc=%d\nout=%s\nerr=%s"%(cmd,rc,out,err))
-
+        if out is not None:
+            logger.debug("cmd=%s"%cmd)
+            logger.info("out = %s"%out)
         # copy namelist files and stream text files, to rundir
         if os.path.isdir(rundir):
             filename = compname + "_in"

utils/python/CIME/case.py

@@ -80,18 +80,16 @@ def __init__(self, case_root=None, read_only=True):
         # for xml files that haven't been created yet. We need a place
         # to store them until we are ready to create the file. At file
         # creation we get the values for those fields from this lookup
-        # table and then remove the entry. This was what I came up
-        # with in the perl anyway and I think that we still need it here.
+        # table and then remove the entry.
         self.lookups = {}
-        self.lookups['CIMEROOT'] = os.path.abspath(get_cime_root())
+        self.set_lookup_value('CIMEROOT',os.path.abspath(get_cime_root()))
 
         self._compsetname = None
         self._gridname = None
         self._compsetsfile = None
         self._pesfile = None
         self._gridfile = None
         self._components = []
-        self._component_config_files = []
         self._component_classes = []
 
     # Define __enter__ and __exit__ so that we can use this as a context manager
@@ -286,11 +284,12 @@ def set_value(self, item, value, subgroup=None, ignore_type=False):
                 logger.debug("Will rewrite file %s %s",env_file.filename, item)
                 self._env_files_that_need_rewrite.add(env_file)
                 return result
-        if result is None:
-            if item in self.lookups.keys() and self.lookups[item] is not None:
-                logger.warn("Item %s already in lookups with value %s"%(item,self.lookups[item]))
-            else:
-                self.lookups[item] = value
+
+    def set_lookup_value(self, item, value):
+        if item in self.lookups.keys() and self.lookups[item] is not None:
+            logger.warn("Item %s already in lookups with value %s"%(item,self.lookups[item]))
+        else:
+            self.lookups[item] = value
 
 
     def _set_compset_and_pesfile(self, compset_name, user_compset=False, pesfile=None):
@@ -384,6 +383,10 @@ def _get_component_config_data(self):
         # Determine list of component classes that this coupler/driver knows how
         # to deal with. This list follows the same order as compset longnames follow.
         files = Files()
+        # Add the group and elements for the config_files.xml
+        for env_file in self._env_entryid_files:
+            env_file.add_elements_by_group(files, attlist)
+
         drv_config_file = files.get_value("CONFIG_DRV_FILE")
         drv_comp = Component(drv_config_file)
         for env_file in self._env_entryid_files:
@@ -399,16 +402,15 @@ def _get_component_config_data(self):
             comp_class = self._component_classes[i]
             comp_name  = self._components[i-1]
             node_name = 'CONFIG_' + comp_class + '_FILE'
-            comp_config_file = files.get_value(node_name, {"component":comp_name}, resolved=True)
+            # Add the group and elements for the config_files.xml
+            comp_config_file = files.get_value(node_name, {"component":comp_name}, resolved=False)
+            self.set_value(node_name, comp_config_file)
+            comp_config_file = self.get_resolved_value(comp_config_file)
             expect(comp_config_file is not None,"No config file for component %s"%comp_name)
             compobj = Component(comp_config_file)
             for env_file in self._env_entryid_files:
                 env_file.add_elements_by_group(compobj, attributes=attlist)
-            self._component_config_files.append((node_name,comp_config_file))
 
-        # Add the group and elements for the config_files.xml
-        for env_file in self._env_entryid_files:
-            env_file.add_elements_by_group(files, attlist)
 
         for key,value in self.lookups.items():
             result = self.set_value(key,value)
@@ -462,10 +464,11 @@ def configure(self, compset_name, grid_name, machine_name=None,
         grids = Grids(gridfile)
 
         gridinfo = grids.get_grid_info(name=grid_name, compset=self._compsetname)
+
         self._gridname = gridinfo["GRID"]
         for key,value in gridinfo.items():
             logger.debug("Set grid %s %s"%(key,value))
-            self.set_value(key,value)
+            self.set_lookup_value(key,value)
 
         #--------------------------------------------
         # component config data
@@ -474,10 +477,6 @@ def configure(self, compset_name, grid_name, machine_name=None,
 
         self.get_compset_var_settings()
 
-        # Add the group and elements for the config_files.xml
-        for config_file in self._component_config_files:
-            self.set_value(config_file[0],config_file[1])
-
         #--------------------------------------------
         # machine
         #--------------------------------------------
@@ -831,11 +830,13 @@ def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None):
         if newcase_cimeroot != clone_cimeroot:
             logger.warning(" case  CIMEROOT is %s " %newcase_cimeroot)
             logger.warning(" clone CIMEROOT is %s " %clone_cimeroot)
-            logger.warning(" It is NOT recommended to clone cases from different versions of CIMEROOT")
+            logger.warning(" It is NOT recommended to clone cases from different versions of CIME.")
+
 
         # *** create case object as deepcopy of clone object ***
         srcroot = os.path.join(newcase_cimeroot,"..")
         newcase = self.copy(newcasename, newcaseroot, newsrcroot=srcroot)
+        newcase.set_value("CIMEROOT", newcase_cimeroot)
 
         # determine if will use clone executable or not
         if keepexe: