Merge pull request #3038 from jedwards4b/chksum_fixes_maint-5.6

Chksum fixes maint 5.6
Fix chksum issues - only download chksum file(s) if requested, look on all servers in config_inputdata.xml for chksum files and merge those found.

Test suite: scripts_regression_tests.py
Test baseline:
Test namelist changes:
Test status: bit for bit

Fixes #3033
Fixes #3034
Fixes #3036

User interface changes?:

Update gh-pages html (Y/N)?:

Code review:sacks

scripts/lib/CIME/Servers/ftp.py

@@ -5,6 +5,7 @@
 from CIME.XML.standard_module_setup import *
 from CIME.Servers.generic_server import GenericServer
 from ftplib import FTP as FTPpy
+from ftplib import all_errors as all_ftp_errors
 
 logger = logging.getLogger(__name__)
 # I think that multiple inheritence would be useful here, but I couldnt make it work
@@ -19,6 +20,8 @@ def __init__(self, address, user='', passwd=''):
         if not passwd:
             passwd = ''
 
+        self._ftp_server = address
+
         stat = self.ftp.login(user, passwd)
         logger.debug("login stat {}".format(stat))
         if "Login successful" not in stat:
@@ -29,14 +32,14 @@ def __init__(self, address, user='', passwd=''):
         if "Directory successfully changed" not in stat:
             logging.warning("FAIL: Could not cd to server root directory {}\n error {}".format(root_address, stat))
             return None
-        self._ftp_server = address
 
     def fileexists(self, rel_path):
         try:
             stat = self.ftp.nlst(rel_path)
-        except:
+        except all_ftp_errors:
             logger.warning("ERROR from ftp server, trying next server")
             return False
+
         if rel_path not in stat:
             if not stat or not stat[0].startswith(rel_path):
                 logging.warning("FAIL: File {} not found.\nerror {}".format(rel_path, stat))
@@ -46,7 +49,7 @@ def fileexists(self, rel_path):
     def getfile(self, rel_path, full_path):
         try:
             stat = self.ftp.retrbinary('RETR {}'.format(rel_path), open(full_path, "wb").write)
-        except:
+        except all_ftp_errors:
             logger.warning("ERROR from ftp server, trying next server")
             return False
 
@@ -59,7 +62,7 @@ def getfile(self, rel_path, full_path):
     def getdirectory(self, rel_path, full_path):
         try:
             stat = self.ftp.nlst(rel_path)
-        except:
+        except all_ftp_errors:
             logger.warning("ERROR from ftp server, trying next server")
             return False
 

scripts/lib/CIME/Servers/gftp.py

@@ -11,8 +11,6 @@
 class GridFTP(GenericServer):
     def __init__(self, address, user='', passwd=''):
         self._root_address = address
-
-
 
     def fileexists(self, rel_path):
         stat,out,err = run_cmd("globus-url-copy -list {}".format(os.path.join(self._root_address, os.path.dirname(rel_path))+os.sep))

scripts/lib/CIME/Servers/svn.py

@@ -15,6 +15,8 @@ def __init__(self, address, user='', passwd=''):
         if passwd:
             self._args += "--password {}".format(passwd)
 
+        self._svn_loc = address
+
         err = run_cmd("svn --non-interactive --trust-server-cert {} ls {}".format(self._args, address))[0]
         if err != 0:
             logging.warning(
@@ -23,7 +25,6 @@ def __init__(self, address, user='', passwd=''):
 This is most likely either a credential, proxy, or network issue .
 To check connection and store your credential run 'svn ls {0}' and permanently store your password""".format(address))
             return None
-        self._svn_loc = address
 
     def fileexists(self, rel_path):
         full_url = os.path.join(self._svn_loc, rel_path)
@@ -34,9 +35,11 @@ def fileexists(self, rel_path):
         return True
 
     def getfile(self, rel_path, full_path):
+        if not rel_path:
+            return False
         full_url = os.path.join(self._svn_loc, rel_path)
         stat, output, errput = \
-            run_cmd("svn --non-interactive --trust-server-cert {} export {} {}".format(self._args, full_url, full_path))
+                               run_cmd("svn --non-interactive --trust-server-cert {} export {} {}".format(self._args, full_url, full_path))
         if (stat != 0):
             logging.warning("svn export failed with output: {} and errput {}\n".format(output, errput))
             return False
@@ -54,7 +57,3 @@ def getdirectory(self, rel_path, full_path):
         else:
             logging.info("SUCCESS\n")
             return True
-
-
-
-

scripts/lib/CIME/Servers/wget.py

@@ -13,9 +13,11 @@ def __init__(self, address, user='', passwd=''):
             self._args += "--user {} ".format(user)
         if passwd:
             self._args += "--password {} ".format(passwd)
+        self._server_loc = address
+
         err = run_cmd("wget {} --spider {}".format(self._args, address))[0]
         expect(err == 0,"Could not connect to repo '{0}'\nThis is most likely either a proxy, or network issue .")
-        self._server_loc = address
+
 
     def fileexists(self, rel_path):
         full_url = os.path.join(self._server_loc, rel_path)
@@ -44,7 +46,7 @@ def getfile(self, rel_path, full_path):
     def getdirectory(self, rel_path, full_path):
         full_url = os.path.join(self._server_loc, rel_path)
         stat, output, errput = \
-                run_cmd("wget  {} {} -r -N --no-directories ".format(self._args, full_url+os.sep), from_dir=full_path)
+            run_cmd("wget  {} {} -r -N --no-directories ".format(self._args, full_url+os.sep), from_dir=full_path)
         logger.debug(output)
         logger.debug(errput)
         if (stat != 0):

scripts/lib/CIME/case/check_input_data.py

@@ -13,42 +13,59 @@
 chksum_hash = dict()
 local_chksum_file = 'inputdata_checksum.dat'
 
-def _download_checksum_file(server, rundir, chksum_file):
+def _download_checksum_file(rundir):
     """
-    Return True if successfully downloaded
-    server is an object handle of type CIME.Servers
-    rundir is where the file will be downloaded to
-    chksum_file is the file to be downloaded (specified in config_inputdata.xml)
-    user is the user name of the person running the script
+    Download the checksum files from each server and merge them into rundir.
     """
-    success = False
-    rel_path = chksum_file
-    full_path = os.path.join(rundir, local_chksum_file)
-    new_file = full_path + '.raw'
-    protocol = type(server).__name__
-    logging.info("Trying to download file: '{}' to path '{}' using {} protocol.".format(rel_path, new_file, protocol))
-    tmpfile = None
-    if os.path.isfile(full_path):
-        tmpfile = full_path+".tmp"
-        os.rename(full_path, tmpfile)
-    # Use umask to make sure files are group read/writable. As long as parent directories
-    # have +s, then everything should work.
-    with SharedArea():
-        success = server.getfile(rel_path, new_file)
-        if success:
-            _reformat_chksum_file(full_path, new_file)
-            if tmpfile:
-                _merge_chksum_files(full_path, tmpfile)
-            chksum_hash.clear()
+    inputdata = Inputdata()
+    protocol = "svn"
+    # download and merge all available chksum files.
+    while protocol is not None:
+        protocol, address, user, passwd, chksum_file = inputdata.get_next_server()
+        if protocol not in vars(CIME.Servers):
+            logger.warning("Client protocol {} not enabled".format(protocol))
+            continue
+        logger.info("Using protocol {} with user {} and passwd {}".format(protocol, user, passwd))
+        if protocol == "svn":
+            server = CIME.Servers.SVN(address, user, passwd)
+        elif protocol == "gftp":
+            server = CIME.Servers.GridFTP(address, user, passwd)
+        elif protocol == "ftp":
+            server = CIME.Servers.FTP(address, user, passwd)
+        elif protocol == "wget":
+            server = CIME.Servers.WGET(address, user, passwd)
         else:
-            if tmpfile and os.path.isfile(tmpfile):
-                os.rename(tmpfile, full_path)
-                logger.warning("Could not automatically download file "+full_path+
-                           " Restoring existing version.")
+            expect(False, "Unsupported inputdata protocol: {}".format(protocol))
+
+
+        success = False
+        rel_path = chksum_file
+        full_path = os.path.join(rundir, local_chksum_file)
+        new_file = full_path + '.raw'
+        protocol = type(server).__name__
+        logging.info("Trying to download file: '{}' to path '{}' using {} protocol.".format(rel_path, new_file, protocol))
+        tmpfile = None
+        if os.path.isfile(full_path):
+            tmpfile = full_path+".tmp"
+            os.rename(full_path, tmpfile)
+        # Use umask to make sure files are group read/writable. As long as parent directories
+        # have +s, then everything should work.
+        with SharedArea():
+            success = server.getfile(rel_path, new_file)
+            if success:
+                _reformat_chksum_file(full_path, new_file)
+                if tmpfile:
+                    _merge_chksum_files(full_path, tmpfile)
+                chksum_hash.clear()
             else:
-                logger.warning("Could not automatically download file {}".
-                               format(full_path))
-    return success
+                if tmpfile and os.path.isfile(tmpfile):
+                    os.rename(tmpfile, full_path)
+                    logger.warning("Could not automatically download file "+full_path+
+                                   " Restoring existing version.")
+                else:
+                    logger.warning("Could not automatically download file {}".
+                                   format(full_path))
+
 
 def _reformat_chksum_file(chksum_file, server_file):
     """
@@ -93,7 +110,7 @@ def _download_if_in_repo(server, input_data_root, rel_path, isdirectory=False):
     user is the user name of the person running the script
     isdirectory indicates that this is a directory download rather than a single file
     """
-    if not server.fileexists(rel_path):
+    if not (rel_path or server.fileexists(rel_path)):
         return False
 
     full_path = os.path.join(input_data_root, rel_path)
@@ -133,34 +150,14 @@ def check_all_input_data(self, protocol=None, address=None, input_data_root=None
         success = self.check_input_data(protocol=protocol, address=address, download=download,
                                         input_data_root=input_data_root, data_list_dir=data_list_dir, chksum=chksum)
     else:
-        if chksum or download:
-            inputdata = Inputdata()
-            protocol = "svn"
-            success = False
-            while not success and protocol is not None:
-                protocol, address, user, passwd, chksum_file = inputdata.get_next_server()
-                if protocol not in vars(CIME.Servers):
-                    logger.warning("Client protocol {} not enabled".format(protocol))
-                    continue
-                logger.info("Using protocol {} with user {} and passwd {}".format(protocol, user, passwd))
-                if protocol == "svn":
-                    server = CIME.Servers.SVN(address, user, passwd)
-                elif protocol == "gftp":
-                    server = CIME.Servers.GridFTP(address, user, passwd)
-                elif protocol == "ftp":
-                    server = CIME.Servers.FTP(address, user, passwd)
-                elif protocol == "wget":
-                    server = CIME.Servers.WGET(address, user, passwd)
-                else:
-                    expect(False, "Unsupported inputdata protocol: {}".format(protocol))
-
-
-                success = _download_checksum_file(server, self.get_value("RUNDIR"),
-                                                  chksum_file)
+        if chksum:
+            _download_checksum_file(self.get_value("RUNDIR"))
 
         success = self.check_input_data(protocol=protocol, address=address, download=False,
                                         input_data_root=input_data_root, data_list_dir=data_list_dir, chksum=chksum)
         if download and not success:
+            if not chksum:
+                _download_checksum_file(self.get_value("RUNDIR"))
             success = _downloadfromserver(self, input_data_root, data_list_dir)
 
     expect(not download or (download and success), "Could not find all inputdata on any server")
@@ -320,7 +317,6 @@ def check_input_data(case, protocol="svn", address=None, input_data_root=None, d
                             no_files_missing = False
                         else:
                             logging.debug("  Found input file: '{}'".format(full_path))
-
                     else:
                         # There are some special values of rel_path that
                         # we need to ignore - some of the component models