From 4cb474b9669c59300fedcb23d6ea5d6157beba88 Mon Sep 17 00:00:00 2001 From: Marco Rovere Date: Wed, 7 Jun 2023 16:54:07 +0200 Subject: [PATCH] Fix dqm-* scripts for python3. --- DQMServices/Components/python/HTTP.py | 3 +- DQMServices/Components/python/ROOTData.py | 4 +- DQMServices/Components/scripts/dqm-access | 69 ++++++++++++----------- DQMServices/Components/scripts/dqm-grep | 53 +++++++++-------- DQMServices/Components/scripts/dqm-ls | 37 ++++++------ 5 files changed, 87 insertions(+), 79 deletions(-) diff --git a/DQMServices/Components/python/HTTP.py b/DQMServices/Components/python/HTTP.py index e5e54c6402d2b..ab83978132177 100644 --- a/DQMServices/Components/python/HTTP.py +++ b/DQMServices/Components/python/HTTP.py @@ -1,5 +1,6 @@ from builtins import range from io import StringIO +from io import BytesIO from pycurl import * class RequestManager: @@ -103,7 +104,7 @@ def process(self): while self.queue and self.free: c = self.free.pop() c.task = self.queue.pop(0) - c.buffer = b = StringIO() + c.buffer = b = BytesIO() c.setopt(WRITEFUNCTION, b.write) self.request_init(c, *c.task) self.cm.add_handle(c) diff --git a/DQMServices/Components/python/ROOTData.py b/DQMServices/Components/python/ROOTData.py index 230d66c38c261..29d8cad30fc55 100644 --- a/DQMServices/Components/python/ROOTData.py +++ b/DQMServices/Components/python/ROOTData.py @@ -30,7 +30,7 @@ def loadStreamerInfo(literal, debug): the one contained in the DQM GUI source code.""" bitsarray = array('B') - bitsarray.fromstring(literal.decode('hex')) + bitsarray.frombytes(bytes.fromhex(literal)) tbuffer = TBufferFile(TBufferFile.kRead) tbuffer.Reset(); @@ -63,7 +63,7 @@ def literal2root (literal, rootType, debug=False): return None bitsarray = array('B') - bitsarray.fromstring(literal.decode('hex')) + bitsarray.frombytes(bytes.fromhex(literal)) tbuffer = TBufferFile(TBufferFile.kRead) tbuffer.SetBuffer(bitsarray,len(bitsarray),False) diff --git a/DQMServices/Components/scripts/dqm-access b/DQMServices/Components/scripts/dqm-access index 38a1c8970958c..2199da5700e9c 100755 --- a/DQMServices/Components/scripts/dqm-access +++ b/DQMServices/Components/scripts/dqm-access @@ -4,7 +4,8 @@ from DQMServices.Components.HTTP import RequestManager from DQMServices.Components.X509 import SSLOptions from optparse import OptionParser from time import strptime, time -import sys, re, cjson, pycurl, urllib +import sys, re, json, pycurl +from urllib.parse import quote DIR = 0 FILE = 1 @@ -89,8 +90,6 @@ def pattern_to_filter(pattern): if last.type == FILE and not last.recurse and last.pattern == "^$": filters.pop() - print filters - return filters #------------------------------------------------------------------------------- @@ -113,23 +112,25 @@ def find_matching_samples(options): all_samples = {} def req_error(c, url, errmsg, errno): - print >> sys.stderr, "%s: failed to retrieve samples: %s (%d)" \ - % (options.server, errmsg, errno) + print("%s: failed to retrieve samples: %s (%d)" \ + % (options.server, errmsg, errno), file=sys.stderr) sys.exit(1) def req_done(c): - all_samples['result'] = cjson.decode(c.buffer.getvalue()) + json_decoder = json.decoder.JSONDecoder() + samples = c.buffer.getvalue().decode('utf-8') + all_samples['result'] = json_decoder.decode(samples) reqman = RequestManager(ssl_opts = ssl_opts, user_agent = ident, request_respond = req_done, request_error = req_error) - print options.server + "/samples" + print(options.server + "/samples") reqman.put((options.server + "/samples",)) reqman.process() if not all_samples: - print >> sys.stderr, "%s: no samples" % options.server + print("%s: no samples" % options.server, file=sys.stderr) sys.exit(1) for sample_type in all_samples['result']['samples']: @@ -142,12 +143,13 @@ def fetch_tstreamerinfo(options, dataset): topdir = {} def req_error(c, url, errmsg, errno): - print >> sys.stderr, "%s: failed to retrieve TStreamerInfo: %s (%d)" \ - % (options.server, errmsg, errno) + print("%s: failed to retrieve TStreamerInfo: %s (%d)" \ + % (options.server, errmsg, errno), file=sys.stderr) sys.exit(1) def req_done(c): - topdir["contents"] = cjson.decode(c.buffer.getvalue())['contents'] + json_decoder = json.decoder.JSONDecoder() + topdir["contents"] = json_decoder.decode(c.buffer.getvalue().decode('utf-8'))['contents'] reqman = RequestManager(ssl_opts = ssl_opts, user_agent = ident, @@ -162,16 +164,16 @@ def fetch_tstreamerinfo(options, dataset): #------------------------------------------------------------------------------- def request_init(c, options, sample, path, filterspec): sample.update(path = path) - c.url = options.server + urllib.quote(url_content % sample) + c.url = options.server + quote(url_content % sample) if options.fetch_root: c.url+="?rootcontent=1" c.setopt(pycurl.URL, c.url) if False and options.verbose: - print c.url + print(c.url) #------------------------------------------------------------------------------- def report_error(c, task, errmsg, errno): - print >> sys.stderr, "FAILED to retrieve %s: %s (%d)" % (task, errmsg, errno) + print("FAILED to retrieve %s: %s (%d)" % (task, errmsg, errno), file=sys.stderr) #------------------------------------------------------------------------------- def match_filters(item, filters, poslist): @@ -209,20 +211,21 @@ def match_filters(item, filters, poslist): def process(c): global found, nreq options, sample, path, filterspec = c.task + json_decoder = json.decoder.JSONDecoder() nreq += 1 if options.verbose and nreq % 10 == 0: sys.stdout.write(".") sys.stdout.flush() if nreq % 750 == 0: - print + print() - reply = c.buffer.getvalue() + reply = c.buffer.getvalue().decode('utf-8') reply = re.sub(r'("value": ")"([A-Za-z0-9_]+")"', r'\1\2', reply) reply = re.sub(r'("(?:mean|rms|min|max)":) nan,', r'\1 "NaN",', reply) reply = re.sub(r'("(?:mean|rms|min|max|nentries)":) inf,', r'\1 "float(\'inf\')",', reply) reply = re.sub(r'("(?:mean|rms|min|max|nentries)":) -inf,', r'\1 "-float(\'inf\')",', reply) - reply = cjson.decode(reply) + reply = json_decoder.decode(reply) newreq = {} for item in reply['contents']: @@ -236,7 +239,7 @@ def process(c): newreq[newpath] = [] newreq[newpath].append((filters, newpos)) - for path, filterspec in newreq.iteritems(): + for path, filterspec in iter(newreq.items()): reqman.put((options, sample, path, filterspec)) #------------------------------------------------------------------------------- @@ -277,16 +280,16 @@ op.add_option("-f", "--filter", dest = "glob", help = "Filter monitor elements matching GLOB pattern") options, args = op.parse_args() if args: - print >> sys.stderr, "Too many arguments" + print("Too many arguments", file=sys.stderr) sys.exit(1) if not options.sample_expr: - print >> sys.stderr, "Sample predicate expression required" + print("Sample predicate expression required", file=sys.stderr) sys.exit(1) if not options.glob: - print >> sys.stderr, "Monitor element filter expression(s) required" + print("Monitor element filter expression(s) required", file=sys.stderr) sys.exit(1) if not options.server: - print >> sys.stderr, "Server contact string required" + print("Server contact string required", file=sys.stderr) sys.exit(1) # Adjust options @@ -300,9 +303,9 @@ if options.write: ssl_opts = SSLOptions() if options.verbose: - print "Using SSL cert dir", ssl_opts.ca_path - print "Using SSL private key", ssl_opts.key_file - print "Using SSL public key", ssl_opts.cert_file + print("Using SSL cert dir", ssl_opts.ca_path) + print("Using SSL private key", ssl_opts.key_file) + print("Using SSL public key", ssl_opts.cert_file) # Convert each glob pattern into a filter expression. filter_sets = map(lambda glob: pattern_to_filter(glob), options.glob) @@ -329,13 +332,13 @@ for sample in find_matching_samples(options): found = [] sample['section'] = 'archive' if options.verbose: - print "Scanning %s" % sample - reqman.put((options, sample, "/", map(lambda f: (f, [0]), filter_sets))) + print("Scanning %s" % sample) + reqman.put((options, sample, "/", list(map(lambda f: (f, [0]), filter_sets)))) reqman.process() if options.verbose: - print + print() if found: - print "%(section)s/%(run)d%(dataset)s:" % sample + print("%(section)s/%(run)d%(dataset)s:" % sample) found.sort() cwd = None @@ -360,13 +363,13 @@ for sample in find_matching_samples(options): # We are treating a directory if 'subdir' in item: - print " %s/%s" % (path, (options.long_listing and " = directory") or "") + print(" %s/%s" % (path, (options.long_listing and " = directory") or "")) if options.write: tfile_cd(path,ofile) # We are treating an int/double/string elif 'value' in item: - print " %s%s" % (path, (options.long_listing and " = %s" % item['value']) or "") + print(" %s%s" % (path, (options.long_listing and " = %s" % item['value']) or "")) # We have a TObject: histo, graph, profile...; maybe write to a file. else: @@ -375,7 +378,7 @@ for sample in find_matching_samples(options): message += " = [%s # %d]" % (item['properties']['type'], item['nentries']) if options.debug: message += " %s" % item['rootobj'] - print message + print(message) if options.write: indir = path.rsplit("/", 1)[0] if cwd != indir: @@ -392,4 +395,4 @@ for sample in find_matching_samples(options): end = time() if options.verbose: - print "\nFound %d objects in %d directories in %.3f seconds" % (nfound, ntotreq, end - start) + print("\nFound %d objects in %d directories in %.3f seconds" % (nfound, ntotreq, end - start)) diff --git a/DQMServices/Components/scripts/dqm-grep b/DQMServices/Components/scripts/dqm-grep index 5ef00a7cc53af..a02b6cdba4c65 100755 --- a/DQMServices/Components/scripts/dqm-grep +++ b/DQMServices/Components/scripts/dqm-grep @@ -51,7 +51,8 @@ key rather than a proxy, it will prompt for the key password. from DQMServices.Components.HTTP import RequestManager from DQMServices.Components.X509 import SSLOptions -import sys, re, cjson, pycurl, urllib +import sys, re, json, pycurl +from urllib.parse import quote from optparse import OptionParser from time import time @@ -170,12 +171,13 @@ match the requested predicate expression.""" all_samples = {} def req_error(c, url, errmsg, errno): - print >> sys.stderr, "%s: failed to retrieve samples: %s (%d)" \ - % (options.server, errmsg, errno) + print("%s: failed to retrieve samples: %s (%d)" \ + % (options.server, errmsg, errno), file=sys.stderr) sys.exit(1) def req_done(c): - all_samples['result'] = cjson.decode(c.buffer.getvalue()) + json_decoder = json.decoder.JSONDecoder() + all_samples['result'] = json_decoder.decode(c.buffer.getvalue().decode('utf-8')) reqman = RequestManager(ssl_opts = ssl_opts, user_agent = ident, @@ -185,7 +187,7 @@ match the requested predicate expression.""" reqman.process() if not all_samples: - print >> sys.stderr, "%s: no samples" % options.server + print("%s: no samples" % options.server, file=sys.stderr) sys.exit(1) for sample_type in all_samples['result']['samples']: @@ -196,14 +198,14 @@ match the requested predicate expression.""" def request_init(c, options, sample, filters, pos, path): """`RequestManager` callback to initialise JSON contents request.""" sample.update(path = path) - c.url = options.server + urllib.quote(url_content % sample) + c.url = options.server + quote(url_content % sample) c.setopt(pycurl.URL, c.url) if False and options.verbose: - print c.url + print(c.url) def report_error(c, task, errmsg, errno): """`RequestManager` callback to report JSON contents request errors.""" - print >> sys.stderr, "FAILED to retrieve %s: %s (%d)" % (task, errmsg, errno) + print("FAILED to retrieve %s: %s (%d)" % (task, errmsg, errno), file=sys.stderr) def match_filters(item, filters, poslist): """Match filter list created by `pattern_to_filter` against an object. @@ -272,18 +274,19 @@ If verbosity has been requested, also shows simple progress bar on the search progress, one dot for every ten directories retrieved.""" global found, nreq options, sample, filters, pos, path = c.task + json_decoder = json.decoder.JSONDecoder() nreq += 1 if options.verbose and nreq % 10 == 0: sys.stdout.write(".") sys.stdout.flush() if nreq % 750 == 0: - print + print() - reply = c.buffer.getvalue() + reply = c.buffer.getvalue().decode('utf-8') reply = re.sub(r'("value": ")"([A-Za-z0-9_]+")"', r'\1\2', reply) reply = re.sub(r'("(?:mean|rms|min|max)":) nan,', r'\1 "NaN",', reply) - reply = cjson.decode(reply) + reply = json_decoder.decode(reply) seen = set() for item in reply['contents']: @@ -314,24 +317,24 @@ op.add_option("-v", "--verbose", dest = "verbose", help = "Show verbose scan information") options, args = op.parse_args() if args: - print >> sys.stderr, "Too many arguments" + print("Too many arguments", file=sys.stderr) sys.exit(1) if not options.sample_expr: - print >> sys.stderr, "Sample predicate expression required" + print("Sample predicate expression required", file=sys.stderr) sys.exit(1) if not options.glob: - print >> sys.stderr, "Monitor element filter expression required" + print("Monitor element filter expression required", file=sys.stderr) sys.exit(1) if not options.server: - print >> sys.stderr, "Server contact string required" + print("Server contact string required", file=sys.stderr) sys.exit(1) # Get SSL X509 parametres. ssl_opts = SSLOptions() if options.verbose: - print "Using SSL cert dir", ssl_opts.ca_path - print "Using SSL private key", ssl_opts.key_file - print "Using SSL public key", ssl_opts.cert_file + print("Using SSL cert dir", ssl_opts.ca_path) + print("Using SSL private key", ssl_opts.key_file) + print("Using SSL public key", ssl_opts.cert_file) # Convert glob pattern into a filter expression. filters = pattern_to_filter(options.glob) @@ -353,26 +356,26 @@ for sample in find_matching_samples(options): found = [] sample['section'] = 'archive' if options.verbose: - print "Scanning %s" % sample + print("Scanning %s" % sample) reqman.put((options, sample, filters, [0], "/")) reqman.process() if options.verbose: - print + print() if found: - print "%(section)s/%(run)d%(dataset)s:" % sample + print("%(section)s/%(run)d%(dataset)s:" % sample) found.sort() for path, item in found: if 'subdir' in item: - print " %s/" % path + print(" %s/" % path) elif 'value' in item: - print " %s = %s" % (path, item['value']) + print(" %s = %s" % (path, item['value'])) else: - print " %s = [%s # %d]" % (path, item['properties']['type'], item['nentries']) + print(" %s = [%s # %d]" % (path, item['properties']['type'], item['nentries'])) nfound += len(found) ntotreq += nreq end = time() # Provide final summary. if options.verbose: - print "\nFound %d objects in %d directories in %.3f seconds" % (nfound, ntotreq, end - start) + print("\nFound %d objects in %d directories in %.3f seconds" % (nfound, ntotreq, end - start)) diff --git a/DQMServices/Components/scripts/dqm-ls b/DQMServices/Components/scripts/dqm-ls index a80b32fc9ba6f..c8cc0d28d3742 100755 --- a/DQMServices/Components/scripts/dqm-ls +++ b/DQMServices/Components/scripts/dqm-ls @@ -22,9 +22,10 @@ from optparse import OptionParser from time import time, strptime, strftime, gmtime from calendar import timegm from datetime import datetime -from urlparse import urlparse +from urllib.parse import urlparse, quote from tempfile import mkstemp from traceback import print_exc +from functools import cmp_to_key # Object types. DIR = 0 @@ -48,7 +49,7 @@ found = [] def logme(msg, *args): """Generate agent log message.""" procid = "[%s/%d]" % (__file__.rsplit("/", 1)[-1], os.getpid()) - print datetime.now(), procid, msg % args + print(datetime.now(), procid, msg % args) def myumask(): """Get the current process umask.""" @@ -65,7 +66,7 @@ def handle_init(c): def request_init(c, options, path, kind): """`RequestManager` callback to initialise directory contents request.""" - c.setopt(pycurl.URL, options.server + urllib.quote(path) + ((path != "/" and "/") or "")) + c.setopt(pycurl.URL, options.server + quote(path) + ((path != "/" and "/") or "")) assert c.temp_file == None assert c.temp_path == None assert c.local_path == None @@ -81,7 +82,7 @@ def request_init(c, options, path, kind): c.temp_path = tmp c.local_path = path.strip('/') c.buffer = None - except Exception, e: + except Exception as e: logme("ERROR: %s: %s", path, str(e)) print_exc() @@ -103,7 +104,7 @@ def cleanup(c): def report_error(c, task, errmsg, errno): """`RequestManager` callback to report directory contents request errors.""" - print >> sys.stderr, "FAILED to retrieve %s: %s (%d)" % (task, errmsg, errno) + print("FAILED to retrieve %s: %s (%d)" % (task, errmsg, errno), file=sys.stderr) global nfetched; nfetched += 1 def parse_dir_and_files(c): @@ -126,12 +127,12 @@ search progress, one dot for every ten directories retrieved.""" c.temp_file.close() c.temp_file = None - os.chmod(c.temp_path, 0666 & ~UMASK) + os.chmod(c.temp_path, 0o0666 & ~UMASK) os.system('mv %s %s' % (c.temp_path, c.local_path)) c.local_path = None logme("INFO: downloaded %s", path) - except Exception, e: + except Exception as e: logme("ERROR: downloading %s into %s failed: %s", path, c.local_path, str(e)) print_exc() @@ -141,7 +142,7 @@ search progress, one dot for every ten directories retrieved.""" elif kind == DIR: items = re.findall(r"(.*?)(\d+| |-)" r"( |\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d UTC)", - c.buffer.getvalue()) + c.buffer.getvalue().decode('utf-8')) for path, name, size, date in items: assert path.startswith(root_url) @@ -193,7 +194,7 @@ search progress, one dot for every ten directories retrieved.""" sys.stdout.write(".") sys.stdout.flush() if nfetched % 750 == 0: - print + print() def objcmp(a, b): diff = 0 @@ -228,10 +229,10 @@ op.add_option("-m", "--match_file", dest = "match_file", help = "Filter results based on exact file name matching.") options, args = op.parse_args() if args: - print >> sys.stderr, "Too many arguments" + print("Too many arguments", file=sys.stderr) sys.exit(1) if not options.server: - print >> sys.stderr, "Server contact string required" + print("Server contact string required", sys.stderr) sys.exit(1) UMASK = myumask() @@ -246,9 +247,9 @@ if gr: # Get SSL X509 parametres. ssl_opts = SSLOptions() if options.verbose: - print "Using SSL cert dir", ssl_opts.ca_path - print "Using SSL private key", ssl_opts.key_file - print "Using SSL public key", ssl_opts.cert_file + print("Using SSL cert dir", ssl_opts.ca_path) + print("Using SSL private key", ssl_opts.key_file) + print("Using SSL public key", ssl_opts.cert_file) # Start a request manager for contents. reqman = RequestManager(num_connections = options.connections, @@ -265,10 +266,10 @@ reqman.put((options, "/", DIR)) reqman.process() end = time() -for x in sorted(found, cmp=objcmp): - print "%20s\t%s\t%s" % (x['size'], +for x in sorted(found, key=cmp_to_key(objcmp)): + print("%20s\t%s\t%s" % (x['size'], strftime("%Y-%m-%d %H:%M:%S %Z", gmtime(x['date'])), - ((options.recursive and x['path']) or x['name'])) + ((options.recursive and x['path']) or x['name']))) if options.verbose: - print "\nFound %d directories, %d objects in %.3f seconds" % (nfetched, len(found), end - start) + print("\nFound %d directories, %d objects in %.3f seconds" % (nfetched, len(found), end - start))