Merge pull request #2003 from ESMCI/jgfouca/py3_str_unicode_confusion

Fix critical bug with py3 str handling
There were many places hardcoded to check for the 'str' type.
With the python3 conversion, some of our strings have become
type unicode instead. This caused CIME to mistakenly think
these values were not strings in some cases.

Test suite: scripts_regression_tests
Test baseline:
Test namelist changes:
Test status: bit for bit

Fixes [CIME Github issue #]

User interface changes?: N

Update gh-pages html (Y/N)?: N

Code review: @jedwards4b

scripts/lib/CIME/XML/entry_id.py

@@ -8,6 +8,7 @@
 from CIME.XML.generic_xml import GenericXML
 
 from copy import deepcopy
+import six
 
 logger = logging.getLogger(__name__)
 
@@ -229,7 +230,7 @@ def _set_value(self, node, value, vid=None, subgroup=None, ignore_type=False):
     def get_valid_value_string(self, node, value,vid=None,  ignore_type=False):
         valid_values = self._get_valid_values(node)
         if ignore_type:
-            expect(type(value) is str, "Value must be type string if ignore_type is true")
+            expect(isinstance(value, six.string_types), "Value must be type string if ignore_type is true")
             str_value = value
             return str_value
         type_str = self._get_type_info(node)

scripts/lib/CIME/XML/generic_xml.py

@@ -209,7 +209,7 @@ def get_resolved_value(self, raw_value):
         if item_data is None:
             return None
 
-        if type(item_data) is not str:
+        if not isinstance(item_data, six.string_types):
             return item_data
 
         for m in env_ref_re.finditer(item_data):

scripts/lib/CIME/case.py

@@ -5,7 +5,7 @@
 through the Case module.
 """
 from copy import deepcopy
-import glob, os, shutil, math
+import glob, os, shutil, math, six
 from CIME.XML.standard_module_setup import *
 #pylint: disable=import-error,redefined-builtin
 from six.moves import input
@@ -250,7 +250,7 @@ def get_values(self, item, attribute=None, resolved=True, subgroup=None):
                 vtype = env_file.get_type_info(item)
                 if resolved:
                     for result in results:
-                        if type(result) is str:
+                        if isinstance(result, six.string_types):
                             result = self.get_resolved_value(result)
                             new_results.append(convert_to_type(result, vtype, item))
                         else:
@@ -264,7 +264,7 @@ def get_values(self, item, attribute=None, resolved=True, subgroup=None):
             if len(results) > 0:
                 if resolved:
                     for result in results:
-                        if type(result) is str:
+                        if isinstance(result, six.string_types):
                             new_results.append(self.get_resolved_value(result))
                         else:
                             new_results.append(result)
@@ -281,7 +281,7 @@ def get_value(self, item, attribute=None, resolved=True, subgroup=None):
             result = env_file.get_value(item, attribute, resolved=False, subgroup=subgroup)
 
             if result is not None:
-                if resolved and type(result) is str:
+                if resolved and isinstance(result, six.string_types):
                     result = self.get_resolved_value(result)
                     vtype = env_file.get_type_info(item)
                     if vtype is not None:
@@ -293,7 +293,7 @@ def get_value(self, item, attribute=None, resolved=True, subgroup=None):
             result = env_file.get_value(item, attribute, resolved=False, subgroup=subgroup)
 
             if result is not None:
-                if resolved and type(result) is str:
+                if resolved and isinstance(result, six.string_types):
                     return self.get_resolved_value(result)
                 return result
 
@@ -576,7 +576,7 @@ def get_compset_components(self):
     def __iter__(self):
         for entryid_file in self._env_entryid_files:
             for key, val in entryid_file:
-                if type(val) is str and '$' in val:
+                if isinstance(val, six.string_types) and '$' in val:
                     yield key, self.get_resolved_value(val)
                 else:
                     yield key, val

scripts/lib/CIME/case_setup.py

@@ -12,7 +12,7 @@
 from CIME.utils             import get_cime_root, run_and_log_case_status, get_model
 from CIME.test_status       import *
 
-import shutil
+import shutil, six
 
 logger = logging.getLogger(__name__)
 
@@ -80,7 +80,7 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False):
 
     # Check that userdefine settings are specified before expanding variable
     for vid, value in case:
-        expect(not (type(value) is str and "USERDEFINED_required_build" in value),
+        expect(not (isinstance(value, six.string_types) and "USERDEFINED_required_build" in value),
                "Parameter '{}' must be defined".format(vid))
 
     # Remove batch scripts

scripts/lib/CIME/compare_namelists.py

@@ -1,4 +1,4 @@
-import os, re, logging
+import os, re, logging, six
 
 from collections import OrderedDict
 from CIME.utils  import expect
@@ -365,7 +365,7 @@ def _compare_values(name, gold_value, comp_value, case):
                 comments += "  dict variable '{}' has extra key {} with value {}\n".format(name, key, comp_value[key])
 
     else:
-        expect(type(gold_value) is str, "Unexpected type found: '{}'".format(type(gold_value)))
+        expect(isinstance(gold_value, six.string_types), "Unexpected type found: '{}'".format(type(gold_value)))
         norm_gold_value = _normalize_string_value(name, gold_value, case)
         norm_comp_value = _normalize_string_value(name, comp_value, case)