From 190495f3bd1aebade57373b529d721a066fab0d1 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 11 Mar 2019 11:10:50 -0600 Subject: [PATCH 1/5] fix checksum issues --- scripts/lib/CIME/Servers/ftp.py | 11 +++++++---- scripts/lib/CIME/Servers/gftp.py | 2 -- scripts/lib/CIME/Servers/svn.py | 7 ++----- scripts/lib/CIME/Servers/wget.py | 6 ++++-- scripts/lib/CIME/case/check_input_data.py | 1 - 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/scripts/lib/CIME/Servers/ftp.py b/scripts/lib/CIME/Servers/ftp.py index 7a179bc06d7..21aa5207692 100644 --- a/scripts/lib/CIME/Servers/ftp.py +++ b/scripts/lib/CIME/Servers/ftp.py @@ -5,6 +5,7 @@ from CIME.XML.standard_module_setup import * from CIME.Servers.generic_server import GenericServer from ftplib import FTP as FTPpy +from ftplib import all_errors as all_ftp_errors logger = logging.getLogger(__name__) # I think that multiple inheritence would be useful here, but I couldnt make it work @@ -19,6 +20,8 @@ def __init__(self, address, user='', passwd=''): if not passwd: passwd = '' + self._ftp_server = address + stat = self.ftp.login(user, passwd) logger.debug("login stat {}".format(stat)) if "Login successful" not in stat: @@ -29,14 +32,14 @@ def __init__(self, address, user='', passwd=''): if "Directory successfully changed" not in stat: logging.warning("FAIL: Could not cd to server root directory {}\n error {}".format(root_address, stat)) return None - self._ftp_server = address def fileexists(self, rel_path): try: stat = self.ftp.nlst(rel_path) - except: + except all_ftp_errors: logger.warning("ERROR from ftp server, trying next server") return False + if rel_path not in stat: if not stat or not stat[0].startswith(rel_path): logging.warning("FAIL: File {} not found.\nerror {}".format(rel_path, stat)) @@ -46,7 +49,7 @@ def fileexists(self, rel_path): def getfile(self, rel_path, full_path): try: stat = self.ftp.retrbinary('RETR {}'.format(rel_path), open(full_path, "wb").write) - except: + except all_ftp_errors: logger.warning("ERROR from ftp server, trying next server") return False @@ -59,7 +62,7 @@ def getfile(self, rel_path, full_path): def getdirectory(self, rel_path, full_path): try: stat = self.ftp.nlst(rel_path) - except: + except all_ftp_errors: logger.warning("ERROR from ftp server, trying next server") return False diff --git a/scripts/lib/CIME/Servers/gftp.py b/scripts/lib/CIME/Servers/gftp.py index ef19ad34791..b99c9f3049f 100644 --- a/scripts/lib/CIME/Servers/gftp.py +++ b/scripts/lib/CIME/Servers/gftp.py @@ -11,8 +11,6 @@ class GridFTP(GenericServer): def __init__(self, address, user='', passwd=''): self._root_address = address - - def fileexists(self, rel_path): stat,out,err = run_cmd("globus-url-copy -list {}".format(os.path.join(self._root_address, os.path.dirname(rel_path))+os.sep)) diff --git a/scripts/lib/CIME/Servers/svn.py b/scripts/lib/CIME/Servers/svn.py index 623e5558c3d..cbde52e8f1a 100644 --- a/scripts/lib/CIME/Servers/svn.py +++ b/scripts/lib/CIME/Servers/svn.py @@ -15,6 +15,8 @@ def __init__(self, address, user='', passwd=''): if passwd: self._args += "--password {}".format(passwd) + self._svn_loc = address + err = run_cmd("svn --non-interactive --trust-server-cert {} ls {}".format(self._args, address))[0] if err != 0: logging.warning( @@ -23,7 +25,6 @@ def __init__(self, address, user='', passwd=''): This is most likely either a credential, proxy, or network issue . To check connection and store your credential run 'svn ls {0}' and permanently store your password""".format(address)) return None - self._svn_loc = address def fileexists(self, rel_path): full_url = os.path.join(self._svn_loc, rel_path) @@ -54,7 +55,3 @@ def getdirectory(self, rel_path, full_path): else: logging.info("SUCCESS\n") return True - - - - diff --git a/scripts/lib/CIME/Servers/wget.py b/scripts/lib/CIME/Servers/wget.py index 77ceafcacf7..eb685791884 100644 --- a/scripts/lib/CIME/Servers/wget.py +++ b/scripts/lib/CIME/Servers/wget.py @@ -13,9 +13,11 @@ def __init__(self, address, user='', passwd=''): self._args += "--user {} ".format(user) if passwd: self._args += "--password {} ".format(passwd) + self._server_loc = address + err = run_cmd("wget {} --spider {}".format(self._args, address))[0] expect(err == 0,"Could not connect to repo '{0}'\nThis is most likely either a proxy, or network issue .") - self._server_loc = address + def fileexists(self, rel_path): full_url = os.path.join(self._server_loc, rel_path) @@ -44,7 +46,7 @@ def getfile(self, rel_path, full_path): def getdirectory(self, rel_path, full_path): full_url = os.path.join(self._server_loc, rel_path) stat, output, errput = \ - run_cmd("wget {} {} -r -N --no-directories ".format(self._args, full_url+os.sep), from_dir=full_path) + run_cmd("wget {} {} -r -N --no-directories ".format(self._args, full_url+os.sep), from_dir=full_path) logger.debug(output) logger.debug(errput) if (stat != 0): diff --git a/scripts/lib/CIME/case/check_input_data.py b/scripts/lib/CIME/case/check_input_data.py index 914e7dba404..47eb1e11b4e 100644 --- a/scripts/lib/CIME/case/check_input_data.py +++ b/scripts/lib/CIME/case/check_input_data.py @@ -320,7 +320,6 @@ def check_input_data(case, protocol="svn", address=None, input_data_root=None, d no_files_missing = False else: logging.debug(" Found input file: '{}'".format(full_path)) - else: # There are some special values of rel_path that # we need to ignore - some of the component models From ae29df5ce9d1fa4d72ee9a3def8c4220657881cf Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 11 Mar 2019 11:23:43 -0600 Subject: [PATCH 2/5] copy correct files --- scripts/lib/CIME/Servers/svn.py | 4 +++- scripts/lib/CIME/case/check_input_data.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/lib/CIME/Servers/svn.py b/scripts/lib/CIME/Servers/svn.py index cbde52e8f1a..3b8dd18ea83 100644 --- a/scripts/lib/CIME/Servers/svn.py +++ b/scripts/lib/CIME/Servers/svn.py @@ -35,9 +35,11 @@ def fileexists(self, rel_path): return True def getfile(self, rel_path, full_path): + if not rel_path: + return False full_url = os.path.join(self._svn_loc, rel_path) stat, output, errput = \ - run_cmd("svn --non-interactive --trust-server-cert {} export {} {}".format(self._args, full_url, full_path)) + run_cmd("svn --non-interactive --trust-server-cert {} export {} {}".format(self._args, full_url, full_path)) if (stat != 0): logging.warning("svn export failed with output: {} and errput {}\n".format(output, errput)) return False diff --git a/scripts/lib/CIME/case/check_input_data.py b/scripts/lib/CIME/case/check_input_data.py index 47eb1e11b4e..04072fa2017 100644 --- a/scripts/lib/CIME/case/check_input_data.py +++ b/scripts/lib/CIME/case/check_input_data.py @@ -93,7 +93,7 @@ def _download_if_in_repo(server, input_data_root, rel_path, isdirectory=False): user is the user name of the person running the script isdirectory indicates that this is a directory download rather than a single file """ - if not server.fileexists(rel_path): + if not (rel_path or server.fileexists(rel_path)): return False full_path = os.path.join(input_data_root, rel_path) @@ -137,7 +137,8 @@ def check_all_input_data(self, protocol=None, address=None, input_data_root=None inputdata = Inputdata() protocol = "svn" success = False - while not success and protocol is not None: + # download and merge all available chksum files. + while protocol is not None: protocol, address, user, passwd, chksum_file = inputdata.get_next_server() if protocol not in vars(CIME.Servers): logger.warning("Client protocol {} not enabled".format(protocol)) From 5ee9d905825762f14d319bb998f6960a9971e296 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 11 Mar 2019 13:27:35 -0600 Subject: [PATCH 3/5] remove unused variable --- scripts/lib/CIME/case/check_input_data.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/lib/CIME/case/check_input_data.py b/scripts/lib/CIME/case/check_input_data.py index 04072fa2017..7ad5581e828 100644 --- a/scripts/lib/CIME/case/check_input_data.py +++ b/scripts/lib/CIME/case/check_input_data.py @@ -136,7 +136,6 @@ def check_all_input_data(self, protocol=None, address=None, input_data_root=None if chksum or download: inputdata = Inputdata() protocol = "svn" - success = False # download and merge all available chksum files. while protocol is not None: protocol, address, user, passwd, chksum_file = inputdata.get_next_server() @@ -156,8 +155,8 @@ def check_all_input_data(self, protocol=None, address=None, input_data_root=None expect(False, "Unsupported inputdata protocol: {}".format(protocol)) - success = _download_checksum_file(server, self.get_value("RUNDIR"), - chksum_file) + _download_checksum_file(server, self.get_value("RUNDIR"), + chksum_file) success = self.check_input_data(protocol=protocol, address=address, download=False, input_data_root=input_data_root, data_list_dir=data_list_dir, chksum=chksum) From 67c62b7c23799cabec7e5dea93fa370f0675eb4f Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 11 Mar 2019 14:40:39 -0600 Subject: [PATCH 4/5] only get chksum if needed --- scripts/lib/CIME/case/check_input_data.py | 101 +++++++++++----------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/scripts/lib/CIME/case/check_input_data.py b/scripts/lib/CIME/case/check_input_data.py index 7ad5581e828..d529743a261 100644 --- a/scripts/lib/CIME/case/check_input_data.py +++ b/scripts/lib/CIME/case/check_input_data.py @@ -13,7 +13,7 @@ chksum_hash = dict() local_chksum_file = 'inputdata_checksum.dat' -def _download_checksum_file(server, rundir, chksum_file): +def _download_checksum_file(rundir): """ Return True if successfully downloaded server is an object handle of type CIME.Servers @@ -21,33 +21,54 @@ def _download_checksum_file(server, rundir, chksum_file): chksum_file is the file to be downloaded (specified in config_inputdata.xml) user is the user name of the person running the script """ - success = False - rel_path = chksum_file - full_path = os.path.join(rundir, local_chksum_file) - new_file = full_path + '.raw' - protocol = type(server).__name__ - logging.info("Trying to download file: '{}' to path '{}' using {} protocol.".format(rel_path, new_file, protocol)) - tmpfile = None - if os.path.isfile(full_path): - tmpfile = full_path+".tmp" - os.rename(full_path, tmpfile) - # Use umask to make sure files are group read/writable. As long as parent directories - # have +s, then everything should work. - with SharedArea(): - success = server.getfile(rel_path, new_file) - if success: - _reformat_chksum_file(full_path, new_file) - if tmpfile: - _merge_chksum_files(full_path, tmpfile) - chksum_hash.clear() + inputdata = Inputdata() + protocol = "svn" + # download and merge all available chksum files. + while protocol is not None: + protocol, address, user, passwd, chksum_file = inputdata.get_next_server() + if protocol not in vars(CIME.Servers): + logger.warning("Client protocol {} not enabled".format(protocol)) + continue + logger.info("Using protocol {} with user {} and passwd {}".format(protocol, user, passwd)) + if protocol == "svn": + server = CIME.Servers.SVN(address, user, passwd) + elif protocol == "gftp": + server = CIME.Servers.GridFTP(address, user, passwd) + elif protocol == "ftp": + server = CIME.Servers.FTP(address, user, passwd) + elif protocol == "wget": + server = CIME.Servers.WGET(address, user, passwd) else: - if tmpfile and os.path.isfile(tmpfile): - os.rename(tmpfile, full_path) - logger.warning("Could not automatically download file "+full_path+ - " Restoring existing version.") + expect(False, "Unsupported inputdata protocol: {}".format(protocol)) + + + success = False + rel_path = chksum_file + full_path = os.path.join(rundir, local_chksum_file) + new_file = full_path + '.raw' + protocol = type(server).__name__ + logging.info("Trying to download file: '{}' to path '{}' using {} protocol.".format(rel_path, new_file, protocol)) + tmpfile = None + if os.path.isfile(full_path): + tmpfile = full_path+".tmp" + os.rename(full_path, tmpfile) + # Use umask to make sure files are group read/writable. As long as parent directories + # have +s, then everything should work. + with SharedArea(): + success = server.getfile(rel_path, new_file) + if success: + _reformat_chksum_file(full_path, new_file) + if tmpfile: + _merge_chksum_files(full_path, tmpfile) + chksum_hash.clear() else: - logger.warning("Could not automatically download file {}". - format(full_path)) + if tmpfile and os.path.isfile(tmpfile): + os.rename(tmpfile, full_path) + logger.warning("Could not automatically download file "+full_path+ + " Restoring existing version.") + else: + logger.warning("Could not automatically download file {}". + format(full_path)) return success def _reformat_chksum_file(chksum_file, server_file): @@ -133,34 +154,14 @@ def check_all_input_data(self, protocol=None, address=None, input_data_root=None success = self.check_input_data(protocol=protocol, address=address, download=download, input_data_root=input_data_root, data_list_dir=data_list_dir, chksum=chksum) else: - if chksum or download: - inputdata = Inputdata() - protocol = "svn" - # download and merge all available chksum files. - while protocol is not None: - protocol, address, user, passwd, chksum_file = inputdata.get_next_server() - if protocol not in vars(CIME.Servers): - logger.warning("Client protocol {} not enabled".format(protocol)) - continue - logger.info("Using protocol {} with user {} and passwd {}".format(protocol, user, passwd)) - if protocol == "svn": - server = CIME.Servers.SVN(address, user, passwd) - elif protocol == "gftp": - server = CIME.Servers.GridFTP(address, user, passwd) - elif protocol == "ftp": - server = CIME.Servers.FTP(address, user, passwd) - elif protocol == "wget": - server = CIME.Servers.WGET(address, user, passwd) - else: - expect(False, "Unsupported inputdata protocol: {}".format(protocol)) - - - _download_checksum_file(server, self.get_value("RUNDIR"), - chksum_file) + if chksum: + _download_checksum_file(self.get_value("RUNDIR")) success = self.check_input_data(protocol=protocol, address=address, download=False, input_data_root=input_data_root, data_list_dir=data_list_dir, chksum=chksum) if download and not success: + if not chksum: + _download_checksum_file(self.get_value("RUNDIR")) success = _downloadfromserver(self, input_data_root, data_list_dir) expect(not download or (download and success), "Could not find all inputdata on any server") From e90d56256ad09594979d3a3c3e632d94c0585334 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 11 Mar 2019 14:56:16 -0600 Subject: [PATCH 5/5] more cleanup --- scripts/lib/CIME/case/check_input_data.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/lib/CIME/case/check_input_data.py b/scripts/lib/CIME/case/check_input_data.py index d529743a261..ef431c7fb5c 100644 --- a/scripts/lib/CIME/case/check_input_data.py +++ b/scripts/lib/CIME/case/check_input_data.py @@ -15,11 +15,7 @@ def _download_checksum_file(rundir): """ - Return True if successfully downloaded - server is an object handle of type CIME.Servers - rundir is where the file will be downloaded to - chksum_file is the file to be downloaded (specified in config_inputdata.xml) - user is the user name of the person running the script + Download the checksum files from each server and merge them into rundir. """ inputdata = Inputdata() protocol = "svn" @@ -69,7 +65,7 @@ def _download_checksum_file(rundir): else: logger.warning("Could not automatically download file {}". format(full_path)) - return success + def _reformat_chksum_file(chksum_file, server_file): """