From d400a57371ee62c2dcb9f3ebdd21a1808ca6b67b Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Mon, 22 Apr 2024 11:53:09 -0700 Subject: [PATCH 01/23] ENH: Added script for reconstructing PV aliases --- scripts/getPVAliases.py | 508 ++++++++++++++++++++++++++++++++++++++++ scripts/getPVAliases.sh | 45 ++++ 2 files changed, 553 insertions(+) create mode 100644 scripts/getPVAliases.py create mode 100755 scripts/getPVAliases.sh diff --git a/scripts/getPVAliases.py b/scripts/getPVAliases.py new file mode 100644 index 00000000..70407417 --- /dev/null +++ b/scripts/getPVAliases.py @@ -0,0 +1,508 @@ +# -*- coding: utf-8 -*- +""" +A script for gathering alias associations from an IOC +user: aberges +""" +############################################################################### +#%% Imports +############################################################################### + +import sys +import re +import os.path +import argparse +import json +import copy +import glob as gb +from colorama import Fore, Style +from prettytable import PrettyTable + +############################################################################### +#%% Functions +############################################################################### + +def simple_prompt(prompt: str, default: str = 'N'): + """ + Simple yes/no prompt which defaults to 'N' = False, but can be changed. + + """ + while True: + p = input(prompt).strip().lower() + if p in ['']: + p = default.lower() + if p[0] == 'y': + result = True + break + if p.lower()[0] == 'n': + result = False + break + print('Invalid Entry. Please choose again.') + return result + + +def request_file_dest(prompt: str, + default: str=os.getcwd(),): + """ + Requests the user for a destination to save a file. + Tests if the resultant path exists and makes the directory if necessary. + + Parameters + ---------- + prompt : str + Prompt to show the user + default: str + File destination to default to. + Default is '{current_dir}/tempt.txt' + Returns + ------- + str + Path to file destination. + """ + while True: + p = input(prompt+f'(default = {default}): ') + # Check for default + if p in ['']: + result = default + else: + result = p + confirm = simple_prompt('Is ' + + Fore.LIGHTYELLOW_EX + f'{result} ' + + Style.RESET_ALL + + 'correct? (y/N): ') + if confirm is True: + break + return result + + +def flatten_list(input_list: list): + """ + Flatten a list of lists and find its unique values. + Note: loses order due to set() call. + """ + _result = [e for l in input_list for e in l] + return list(set(_result)) + + +def search_file(file:str,output:list=None,patt:str=None,prefix:str='', + color_wrap:Fore=None): + """ + Searches file for regex match and appends result to list + + Parameters + ---------- + file : str + The file to read and search. Encoding must be utf-8 + output : list, optional + A list to appead your results to. The default is None. + patt : str, optional + The regex pattern to search for. The default is None. + prefix : str, optional + A str prefix to add to each line. The default is ''. + color_wrap : Fore, optional + Color wrapping using Colorama.Fore. The default is None. + + Returns + ------- + list + A list of the search results with the prefix prepended. + """ + if output is None: + output = [] + color = '' + reset = '' + if color_wrap is not None: + color = color_wrap + reset = Style.RESET_ALL + if os.path.isfile(file) is False: + print(f'{file} does not exist') + return '' + with open(file,'r',encoding='utf-8') as _f: + for line in _f.readlines(): + if re.search(patt,line): + output.append(re.sub(patt,color + r'\g<0>' + reset,line)) + return prefix + prefix.join(output) + + +def clean_ansi(text: str = None): + """ + Removes ANSI escape sequences from a str, including fg/bg formatting. + """ + ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') + return ansi_escape.sub('',text) + + +def fix_json(raw_data: str): + """ + Fixes JSON format of find_ioc output. + """ + # clean empty rows and white space + _temp = raw_data.replace(' ','').strip() + # capture and fix the keys not properly formatted to str + _temp = re.sub(r"(?'",raw_data) + # capture boolean tokens and fix them for json format + _temp = re.sub("True","true",_temp) + _temp = re.sub("False","false",_temp) + # then capture and fix digits not formatted to str + _temp = re.sub(r"(?<=:)\d+",r"'\g<0>'",_temp) + # then properly convert to list of json obj + result = (_temp + .replace('\'','\"') + .replace('},','}') + .replace(' {','{') + .strip() + .split('\n')) + return result + + +def find_ioc(hutch: str=None, patt: str=None): + """ + A pythonic grep_ioc for gathering IOC details from the cfg file + + Parameters + ---------- + hutch : str, optional + 3 letter lowercase hutch code. May also include 'all'. + The default is None. + patt : str, optional + Regex pattern to search for. The default is None. + + Raises + ------ + ValueError + Hutch code is invalid or regex pattern is missing. + + Returns + ------- + str + Returns the output as a list of dict after json formatting + + """ + hutch_list = ['xpp', 'xcs', 'cxi', 'mfx', 'mec', 'tmo', 'rix', 'xrt', + 'aux', 'det', 'fee', 'hpl', 'icl', 'las', 'lfe', 'kfe', 'tst', + 'thz', 'all'] + # check hutches + if (hutch is None)|(hutch not in tuple(hutch_list)): + print('Invalid entry. Please choose a valid hutch:\n' + +','.join(hutch_list)) + raise ValueError + # create file paths + if hutch in tuple(hutch_list): + if hutch == 'all': + path = gb.glob('/cds/group/pcds/pyps/config/*/iocmanager.cfg') + else: + path = [f'/cds/group/pcds/pyps/config/{hutch}/iocmanager.cfg'] + # check patt and generate the regex pattern + if patt is None: + print('No regex pattern supplied') + raise ValueError + patt = r'{.*'+patt+r'.*}' + # initialize output list + result=[] + # iterate and capture results. + for _file in path: + prefix = '' + if len(path) != 1: + prefix = _file+':' + output = search_file(file=_file,patt=patt,prefix=prefix) + if output != prefix: + result.append(output) + # reconstruct the list of str + _temp = ''.join(result) + if len(_temp) == 0: + print(f'{Fore.RED}No results found for {Style.RESET_ALL}{patt}' + + f'{Fore.RED} in{Style.RESET_ALL}' + + f'{hutch}') + return None + # capture the hutch from the cfg path if hutch = all + if hutch == 'all': + hutch_cfgs = re.findall(r'/.*cfg\:',_temp) + hutch_cfgs = [''.join(re.findall(r'(?<=/)\w+(?=/ioc)',s)) \ + for s in hutch_cfgs] + # strip the file information + _temp = re.sub(r'.*cfg\:','',_temp) + # now convert back to json and load + output = [json.loads(s) for s in fix_json(_temp)] + # and add the hutches back into the dicts if searching across all cfgs + if hutch == 'all': + for _i,_d in enumerate(output): + _d['hutch'] = hutch_cfgs[_i] + return output + + +def fix_dir(dir_path: str=None): + """ + Simple function for repairing the child release IOC path based on + the ioc_cfg output. Returns the proper dir for the child IOC.cfg file. + """ + # catches the short form path + if dir_path.startswith('ioc/'): + output_dir = '/cds/group/pcds/epics/'+dir_path + # for the rare, old child IOCs that only exist in their parent's release + elif 'common' in dir_path: + output_dir = dir_path + '/children' + # Otherwise fine! + else: + output_dir = dir_path + # Make sure the end of the path is a folder! + if output_dir[-1] != '/': + output_dir += '/' + return output_dir + + +def find_parent_ioc(file: str=None, path: str=None): + """ + Searches the child IOC for the parent's release pointer + Returns the parent's IOC as a str. + """ + file_dir = fix_dir(path) + if os.path.exists(f'{file_dir}{file}.cfg') is False: + return 'Invalid. Child does not exist.' + parent_ioc_release=search_file(file=f'{file_dir}{file}.cfg', + patt='^RELEASE').strip() + return parent_ioc_release.rsplit('=',maxsplit=1)[-1] + + +def build_table(input_data: list,columns: list=None,**kwargs): + """ + Build a prettytable from a list of dicts/JSON. + input_data must be a list(dict) + """ + if columns is None: + col_list = [] + # First get all unique key values from the dict + for _d in input_data: + col_list.extend(list(_d.keys())) + cols = sorted(list(set(col_list))) + else: + cols = columns + # initialize the table + _tbl = PrettyTable() + # add the headers + for c in cols: + _tbl.add_column(c,[],**kwargs) + # add the data, strip ANSI color from color headers if any + for _d in input_data: + _tbl.add_row([_d.get(clean_ansi(c),'') for c in cols]) + return _tbl + + +def acquire_aliases(dir_path:str,ioc:str): + """ + Scans the st.cmd of the child IOC for the main PV aliases. + Returns a list of dicts for the associations. + """ + _d = fix_dir(dir_path) + _f = f'{_d}build/iocBoot/{ioc}/st.cmd' + if os.path.exists(_f) is False: + print(f'{_f} does not exist') + return '' + search_result = search_file(file=_f,patt=r'db/alias.db') + _temp = re.findall(r'"RECORD=.*"',search_result) + output = [re.sub(r'\"|\)|RECORD\=|ALIAS\=','',s).split(',') for s in _temp] + return [{'record':s[0],'alias':s[-1]} for s in output] + + +def process_alias_template(parent_release: str,record: str, alias: str): + """ + Opens the parent db/alias.db file and preps it for processing. + Returns a list of the substituted associations + """ + _target_file = f'{parent_release}/db/alias.db' + if os.path.exists(_target_file): + with open(_target_file,encoding='utf-8') as _f: + _temp = _f.read() + else: + print(f'{parent_release} does not exist') + return None + # remove the 'alias' prefix from the tuple + _temp = re.sub(r'alias\(| +','',_temp) + _temp = re.sub(r'\)\s*\n','\n',_temp) + # then make the substitutions + _temp = re.sub(r'\$\(RECORD\)',record,_temp) + _temp = re.sub(r'\$\(ALIAS\)',alias,_temp) + return [s.replace('"','').split(',') for s in _temp.split()] + + +def show_temp_table(input_data : list, col_list : list): + """ + Formats the 'disable' column in the find_ioc json output for clarity + and prints the pretty table to the terminal. + """ + # color code the disable state for easier comprehensions + temp = copy.deepcopy(input_data) + for _d in temp: + if _d.get('disable') is not None: + if _d.get('disable') is True: + _d['disable'] = f'{Fore.LIGHTGREEN_EX}True{Style.RESET_ALL}' + else: + _d['disable'] = f'{Fore.RED}False{Style.RESET_ALL}' + + # prompt user for initial confirmation + print(f'{Fore.LIGHTGREEN_EX}Found the following:{Style.RESET_ALL}') + print(build_table(temp,col_list)) + + +############################################################################### +#%% Argparser +############################################################################### +# parser obj configuration +parser = argparse.ArgumentParser( + prog='gatherPVAliases', + description="""gathers all record <-> alias associations from a child's + ioc.cfg, st.cmd, and parent ioc.cfg.""", + epilog='') +# main command arguments +parser.add_argument('patt', type=str) +parser.add_argument('hutch', type=str) +parser.add_argument('-d','--dry_run', action='store_true', + default=False, + help='''Forces a dry run for the script. + No files are saved.''') + +############################################################################### +#%% Main +############################################################################### + +def main(): + """ + Main function entry point + """ + # parse args + args = parser.parse_args() + # search ioc_cfg and build the dataset + data = find_ioc(args.hutch,args.patt) + if data is None: + print(f'{Fore.RED}No results found for {Style.RESET_ALL}{args.patt}' + + f'{Fore.RED} in{Style.RESET_ALL}' + + f'{args.hutch}') + sys.exit() + + # find the parent directories + for _d in data: + _d['parent_ioc'] = find_parent_ioc(_d['id'],_d['dir']) + + # Hard code the column order for the find_ioc output + column_list = ['id','dir', + Fore.LIGHTYELLOW_EX + 'parent_ioc' + Style.RESET_ALL, + 'host','port', + Fore.LIGHTBLUE_EX + 'alias' + Style.RESET_ALL, + Fore.RED + 'disable' + Style.RESET_ALL] + if args.hutch == 'all': + column_list = ['hutch'] + column_list + show_temp_table(data,column_list) + + ans = simple_prompt('Proceed? (Y/n): ',default='Y') + # Abort if user gets cold feet + if ans is False: + sys.exit() + print(f'{Fore.RED}Skipping disabled child IOCs{Style.RESET_ALL}') + + # iterate through all the child ioc dictionaries + for _ioc in data: + if _ioc.get('disable') is not True: + # first acquire the base alias dictionary + alias_dicts = acquire_aliases(_ioc['dir'],_ioc['id']) + # show the record aliases to the user + print(Fore.LIGHTGREEN_EX + + 'The following substitutions were found in the st.cmd:' + + Style.RESET_ALL) + print(build_table(alias_dicts,['record','alias'],align='l')) + # optional skip for all resulting PV aliases + save_all = (simple_prompt( + 'Do you want to save save_all resulting PV aliases? ' + + 'This will generate ' + + Fore.LIGHTYELLOW_EX + + f'{len(alias_dicts)}' + + Style.RESET_ALL + + ' files (y/N): ')) + + # initialize a flag for skipping annoying prompts + skip_all = None + show_pvs = None + + # initialize a default file directory for dumping aliases + default_dest = os.getcwd() + '/' + f"{_ioc['id']}_alias" + + # now iterate through the alias dicts for PV alias substitutions + for i,a in enumerate(alias_dicts): + # then iterate through all the PVs from root PV + alias_list = process_alias_template(_ioc['parent_ioc'], + a['record'], a['alias']) + # capture output based on 61 char max record name + _output = [f"{l[0]:<61}{l[-1]:<61}" for l in alias_list] + # Demonstrate PV aliases on first iteration + if (i==0)|((show_pvs is True)&(skip_all is False)): + # show output to user, building a temp list of dict first + _temp = [{'PV' : l[0],'Alias' : l[-1]} for l in alias_list] + print(Fore.LIGHTGREEN_EX + + 'The following PV aliases are built:' + + Style.RESET_ALL) + print(build_table(_temp,['PV','Alias'],align='l')) + del _temp + + # If doing a dry run, skip this block + if args.dry_run is False: + # Respect the skip flag + if skip_all is True: + continue + # ask user for input + if save_all is False: + ans = (simple_prompt( + 'Would you like to save this PV set? (y/N): ')) + if ans is True: + # give the user an option to be lazy again + save_all = (simple_prompt( + 'Would you like to apply this for' + + ' all subsequent sets? (y/N): ')) + # Avoid some terminal spam using these flags + show_pvs = not save_all + skip_all = False + if ans is False: + skip_all = (simple_prompt( + 'Skip all further substitutions? (Y/n): ', + default='Y')) + save_data = False + # Avoid some terminal spam using this flag + show_pvs = not skip_all + continue + + else: + # Set flags to surpress prompts during dry run + ans = False + save_data = False + + if save_all is True: + # only ask once + dest = (request_file_dest( + 'Choose destination for data dump',default_dest)) + save_data = True + save_all = None + + # else pester the user to approve for every single dataset + elif (save_all is not None)&(ans is True): + dest = request_file_dest('Choose base file destination', + default_dest) + save_data = True + + # write to file, else do nothing + if (save_data is True)&(args.dry_run is False): + # make sure the destination exists and mkdir if it doesn't + if os.path.exists(dest) is False: + print(Fore.LIGHTBLUE_EX + + f'Making directory: {dest}' + Style.RESET_ALL) + os.mkdir(dest) + # pad leading 0 for file sorting + j = i + if i < 10: + j = '0'+f'{i}' + file_dest = dest+'/'+f"record_alias_{j}.txt" + with open(file_dest,'w',encoding='utf-8') as f: + f.write('\n'.join(_output)) + default_dest = dest + del _output + + sys.exit() + +if __name__ == '__main__': + main() + \ No newline at end of file diff --git a/scripts/getPVAliases.sh b/scripts/getPVAliases.sh new file mode 100755 index 00000000..c285a0c7 --- /dev/null +++ b/scripts/getPVAliases.sh @@ -0,0 +1,45 @@ +#!/usr/bin/bash + +# define usage +usage() +{ +cat << EOF +usage: gatherPVAliases [-h] [-d] patt hutch + +gathers all record <-> alias associations from a child's ioc.cfg, st.cmd, and +parent ioc.cfg. + +positional arguments: + patt + hutch + +options: + -h, --help show this help message and exit + -d, --dry_run Forces a dry run for the script. No files are saved. + +EOF +} + +# catch improper number of args or asking for help + +if [[ ("$#" -lt 2) || ("$1" == "-h") || ("$1" == "--help") ]]; then + usage + exit 0 +fi + +# source into pcds_conda if not currently active + +if [[ ($CONDA_SHLVL == 1) ]];then + if [[ "$(echo $CONDA_PREFIX | grep 'pcds-')" ]];then + : + else + source pcds_conda + fi +else + source pcds_conda +fi + +# execute python script +#python /cds/group/pcds/.../getPVAliases.py $@ + +exit From 05ab06e75ee6824fb44629dd73988da3ca6fcb0e Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Mon, 22 Apr 2024 12:22:25 -0700 Subject: [PATCH 02/23] MNT: Fixed relative path reference on getPVAliases wrapper --- scripts/getPVAliases.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/getPVAliases.sh b/scripts/getPVAliases.sh index c285a0c7..4c8d23dc 100755 --- a/scripts/getPVAliases.sh +++ b/scripts/getPVAliases.sh @@ -40,6 +40,6 @@ else fi # execute python script -#python /cds/group/pcds/.../getPVAliases.py $@ +python "$ENG_TOOLS_SCRIPTS/getPVAliases.py" $@ exit From c9df9014f3eae54aa664ac7f0a3e5c6006f8245d Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Tue, 23 Apr 2024 18:36:30 -0700 Subject: [PATCH 03/23] STY: conform to flake8 DOC: type hint returns ENH: consolidate bash script --- scripts/getPVAliases.py | 314 ++++++++++++++++++++++++++-------------- scripts/getPVAliases.sh | 43 +----- 2 files changed, 209 insertions(+), 148 deletions(-) diff --git a/scripts/getPVAliases.py b/scripts/getPVAliases.py index 70407417..a49e6ce5 100644 --- a/scripts/getPVAliases.py +++ b/scripts/getPVAliases.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- """ A script for gathering alias associations from an IOC -user: aberges +@author: aberges """ ############################################################################### -#%% Imports +# %% Imports ############################################################################### import sys @@ -18,13 +18,19 @@ from prettytable import PrettyTable ############################################################################### -#%% Functions +# %% Functions ############################################################################### -def simple_prompt(prompt: str, default: str = 'N'): + +def simple_prompt(prompt: str, default: str = 'N') -> bool: """ Simple yes/no prompt which defaults to 'N' = False, but can be changed. + Returns + ------- + bool + Prompted yes/no response + """ while True: p = input(prompt).strip().lower() @@ -41,18 +47,18 @@ def simple_prompt(prompt: str, default: str = 'N'): def request_file_dest(prompt: str, - default: str=os.getcwd(),): + default: str = os.getcwd(),) -> str: """ - Requests the user for a destination to save a file. + Requests the user for a destination to save a file. Tests if the resultant path exists and makes the directory if necessary. - + Parameters ---------- - prompt : str + prompt: str Prompt to show the user default: str File destination to default to. - Default is '{current_dir}/tempt.txt' + Default is '{current_dir}' Returns ------- str @@ -74,40 +80,49 @@ def request_file_dest(prompt: str, return result -def flatten_list(input_list: list): +def flatten_list(input_list: list[list]) -> list[str]: """ - Flatten a list of lists and find its unique values. + Flatten a 2D lists and find its unique values. Note: loses order due to set() call. + + Parameters + ---------- + input_list: list[list] + The 2D list to flatten. + + Returns + ------- + list + The list of unique elements from the 2D input_list """ - _result = [e for l in input_list for e in l] + _result = [e for lst in input_list for e in lst] return list(set(_result)) -def search_file(file:str,output:list=None,patt:str=None,prefix:str='', - color_wrap:Fore=None): +def search_file(*, file: str, patt: str = None, prefix: str = '', + color_wrap: Fore = None) -> list[str]: """ Searches file for regex match and appends result to list Parameters ---------- - file : str + file: str The file to read and search. Encoding must be utf-8 - output : list, optional + output: list, optional A list to appead your results to. The default is None. - patt : str, optional + patt: str, optional The regex pattern to search for. The default is None. - prefix : str, optional + prefix: str, optional A str prefix to add to each line. The default is ''. - color_wrap : Fore, optional + color_wrap: Fore, optional Color wrapping using Colorama.Fore. The default is None. Returns ------- - list + list[str] A list of the search results with the prefix prepended. """ - if output is None: - output = [] + output = [] color = '' reset = '' if color_wrap is not None: @@ -116,54 +131,63 @@ def search_file(file:str,output:list=None,patt:str=None,prefix:str='', if os.path.isfile(file) is False: print(f'{file} does not exist') return '' - with open(file,'r',encoding='utf-8') as _f: + with open(file, 'r', encoding='utf-8') as _f: for line in _f.readlines(): - if re.search(patt,line): - output.append(re.sub(patt,color + r'\g<0>' + reset,line)) + if re.search(patt, line): + output.append(re.sub(patt, color + r'\g<0>' + reset, line)) return prefix + prefix.join(output) -def clean_ansi(text: str = None): +def clean_ansi(text: str = None) -> str: """ Removes ANSI escape sequences from a str, including fg/bg formatting. """ ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') - return ansi_escape.sub('',text) + return ansi_escape.sub('', text) -def fix_json(raw_data: str): +def fix_json(raw_data: str) -> list[str]: """ - Fixes JSON format of find_ioc output. + Fixes JSON format of find_ioc/grep_ioc output. + + Parameters + ---------- + raw_data: str + Str output generated by find_ioc/grep_ioc, which is pseudo-JSON. + Returns + ------- + list[str] + The list of str ready for JSON loading """ # clean empty rows and white space - _temp = raw_data.replace(' ','').strip() + _temp = raw_data.replace(' ', '').strip() # capture and fix the keys not properly formatted to str - _temp = re.sub(r"(?'",raw_data) + _temp = re.sub(r"(?'", raw_data) # capture boolean tokens and fix them for json format - _temp = re.sub("True","true",_temp) - _temp = re.sub("False","false",_temp) + _temp = re.sub("True", "true", _temp) + _temp = re.sub("False", "false", _temp) # then capture and fix digits not formatted to str - _temp = re.sub(r"(?<=:)\d+",r"'\g<0>'",_temp) + _temp = re.sub(r"(?<=:)\d+", r"'\g<0>'", _temp) # then properly convert to list of json obj result = (_temp - .replace('\'','\"') - .replace('},','}') - .replace(' {','{') + .replace('\'', '\"') + .replace('},', '}') + .replace(' {', '{') .strip() .split('\n')) return result -def find_ioc(hutch: str=None, patt: str=None): +def find_ioc(hutch: str = None, patt: str = None) -> list[dict]: """ A pythonic grep_ioc for gathering IOC details from the cfg file Parameters ---------- - hutch : str, optional + hutch: str, optional 3 letter lowercase hutch code. May also include 'all'. The default is None. - patt : str, optional + patt: str, optional Regex pattern to search for. The default is None. Raises @@ -173,17 +197,17 @@ def find_ioc(hutch: str=None, patt: str=None): Returns ------- - str - Returns the output as a list of dict after json formatting + list[dict] + List of dictionaries generated by the JSON loading """ hutch_list = ['xpp', 'xcs', 'cxi', 'mfx', 'mec', 'tmo', 'rix', 'xrt', - 'aux', 'det', 'fee', 'hpl', 'icl', 'las', 'lfe', 'kfe', 'tst', - 'thz', 'all'] + 'aux', 'det', 'fee', 'hpl', 'icl', 'las', 'lfe', 'kfe', + 'tst', 'txi', 'thz', 'all'] # check hutches - if (hutch is None)|(hutch not in tuple(hutch_list)): + if (hutch is None) | (hutch not in tuple(hutch_list)): print('Invalid entry. Please choose a valid hutch:\n' - +','.join(hutch_list)) + + ','.join(hutch_list)) raise ValueError # create file paths if hutch in tuple(hutch_list): @@ -195,15 +219,15 @@ def find_ioc(hutch: str=None, patt: str=None): if patt is None: print('No regex pattern supplied') raise ValueError - patt = r'{.*'+patt+r'.*}' + patt = r'{.*' + patt + r'.*}' # initialize output list - result=[] + result = [] # iterate and capture results. for _file in path: prefix = '' if len(path) != 1: prefix = _file+':' - output = search_file(file=_file,patt=patt,prefix=prefix) + output = search_file(file=_file, patt=patt, prefix=prefix) if output != prefix: result.append(output) # reconstruct the list of str @@ -215,25 +239,37 @@ def find_ioc(hutch: str=None, patt: str=None): return None # capture the hutch from the cfg path if hutch = all if hutch == 'all': - hutch_cfgs = re.findall(r'/.*cfg\:',_temp) - hutch_cfgs = [''.join(re.findall(r'(?<=/)\w+(?=/ioc)',s)) \ + hutch_cfgs = re.findall(r'/.*cfg\:', _temp) + hutch_cfgs = [''.join(re.findall(r'(?<=/)\w+(?=/ioc)', s)) for s in hutch_cfgs] # strip the file information - _temp = re.sub(r'.*cfg\:','',_temp) + _temp = re.sub(r'.*cfg\:', '', _temp) # now convert back to json and load output = [json.loads(s) for s in fix_json(_temp)] # and add the hutches back into the dicts if searching across all cfgs if hutch == 'all': - for _i,_d in enumerate(output): + for _i, _d in enumerate(output): _d['hutch'] = hutch_cfgs[_i] return output -def fix_dir(dir_path: str=None): +def fix_dir(dir_path: str) -> str: """ Simple function for repairing the child release IOC path based on the ioc_cfg output. Returns the proper dir for the child IOC.cfg file. + + Parameters + ---------- + dir_path: str + The path to the child IOC's directory as a str. + + Returns + ------- + str + The typo-corrected path as a str. + """ + # catches the short form path if dir_path.startswith('ioc/'): output_dir = '/cds/group/pcds/epics/'+dir_path @@ -249,24 +285,53 @@ def fix_dir(dir_path: str=None): return output_dir -def find_parent_ioc(file: str=None, path: str=None): +def find_parent_ioc(file: str, path: str) -> str: """ Searches the child IOC for the parent's release pointer Returns the parent's IOC as a str. + + Parameters + ---------- + file : str, optional + DESCRIPTION. The default is None. + path : str, optional + DESCRIPTION. The default is None. + + Returns + ------- + str + Path to the parent IOC's release. + """ file_dir = fix_dir(path) if os.path.exists(f'{file_dir}{file}.cfg') is False: return 'Invalid. Child does not exist.' - parent_ioc_release=search_file(file=f'{file_dir}{file}.cfg', - patt='^RELEASE').strip() - return parent_ioc_release.rsplit('=',maxsplit=1)[-1] + parent_ioc_release = search_file(file=f'{file_dir}{file}.cfg', + patt='^RELEASE').strip() + return parent_ioc_release.rsplit('=', maxsplit=1)[-1] -def build_table(input_data: list,columns: list=None,**kwargs): +def build_table(input_data: list[dict], columns: list[str] = None, + **kwargs) -> PrettyTable: """ Build a prettytable from a list of dicts/JSON. input_data must be a list(dict) + Parameters + ---------- + input_data : list[dict] + The data to generate a PrettyTable from. + columns : list, optional + Columns for the PrettyTable headers. The default is None. + **kwargs: + kwargs to pass tohe PrettyTable() for extra customization. + + Returns + ------- + PrettyTable + PrettyTable object ready for terminal printing. + """ + if columns is None: col_list = [] # First get all unique key values from the dict @@ -279,51 +344,86 @@ def build_table(input_data: list,columns: list=None,**kwargs): _tbl = PrettyTable() # add the headers for c in cols: - _tbl.add_column(c,[],**kwargs) + _tbl.add_column(c, [], **kwargs) # add the data, strip ANSI color from color headers if any for _d in input_data: - _tbl.add_row([_d.get(clean_ansi(c),'') for c in cols]) + _tbl.add_row([_d.get(clean_ansi(c), '') for c in cols]) return _tbl -def acquire_aliases(dir_path:str,ioc:str): +def acquire_aliases(dir_path: str, ioc: str) -> list[dict]: """ Scans the st.cmd of the child IOC for the main PV aliases. - Returns a list of dicts for the associations. + Returns a list of dicts for the associations. This is the + top level PV name. + E.g. LM1K2:MCS2:01:m1 <--> LM2K2:INJ_MP1_MR1 + + Parameters + ---------- + dir_path : str + Path to the child IOC's release. + ioc : str + Child IOC's cfg file name. + + Returns + ------- + list[dict] + List of dicts for record <--> alias associations. + """ _d = fix_dir(dir_path) _f = f'{_d}build/iocBoot/{ioc}/st.cmd' if os.path.exists(_f) is False: print(f'{_f} does not exist') return '' - search_result = search_file(file=_f,patt=r'db/alias.db') - _temp = re.findall(r'"RECORD=.*"',search_result) - output = [re.sub(r'\"|\)|RECORD\=|ALIAS\=','',s).split(',') for s in _temp] - return [{'record':s[0],'alias':s[-1]} for s in output] + search_result = search_file(file=_f, patt=r'db/alias.db') + _temp = re.findall(r'"RECORD=.*"', search_result) + output = [re.sub(r'\"|\)|RECORD\=|ALIAS\=', '', s).split(',') + for s in _temp] + return [{'record': s[0], 'alias': s[-1]} for s in output] -def process_alias_template(parent_release: str,record: str, alias: str): +def process_alias_template(parent_release: str, record: str, + alias: str) -> list[str]: """ - Opens the parent db/alias.db file and preps it for processing. - Returns a list of the substituted associations + Opens the parent db/alias.db file and processes the + substitutions. + This is the second level of PV names (like in motor records). + E.g. LM1K2:MCS2:01:m1 <--> LM1K2:INJ_MP1_MR1.RBV + + Parameters + ---------- + parent_release : str + Path to the parent IOC's release. + record : str + The EPICS record for substitution to generate PV names. + alias : str + The alias for the EPICS record. + + Returns + ------- + list[str] + DESCRIPTION. + """ + _target_file = f'{parent_release}/db/alias.db' if os.path.exists(_target_file): - with open(_target_file,encoding='utf-8') as _f: + with open(_target_file, encoding='utf-8') as _f: _temp = _f.read() else: print(f'{parent_release} does not exist') return None # remove the 'alias' prefix from the tuple - _temp = re.sub(r'alias\(| +','',_temp) - _temp = re.sub(r'\)\s*\n','\n',_temp) + _temp = re.sub(r'alias\(| +', '', _temp) + _temp = re.sub(r'\)\s*\n', '\n', _temp) # then make the substitutions - _temp = re.sub(r'\$\(RECORD\)',record,_temp) - _temp = re.sub(r'\$\(ALIAS\)',alias,_temp) - return [s.replace('"','').split(',') for s in _temp.split()] + _temp = re.sub(r'\$\(RECORD\)', record, _temp) + _temp = re.sub(r'\$\(ALIAS\)', alias, _temp) + return [s.replace('"', '').split(',') for s in _temp.split()] -def show_temp_table(input_data : list, col_list : list): +def show_temp_table(input_data: list, col_list: list): """ Formats the 'disable' column in the find_ioc json output for clarity and prints the pretty table to the terminal. @@ -339,30 +439,31 @@ def show_temp_table(input_data : list, col_list : list): # prompt user for initial confirmation print(f'{Fore.LIGHTGREEN_EX}Found the following:{Style.RESET_ALL}') - print(build_table(temp,col_list)) + print(build_table(temp, col_list)) ############################################################################### -#%% Argparser +# %% Argparser ############################################################################### # parser obj configuration parser = argparse.ArgumentParser( - prog='gatherPVAliases', - description="""gathers all record <-> alias associations from a child's - ioc.cfg, st.cmd, and parent ioc.cfg.""", - epilog='') + prog='gatherPVAliases', + description="""gathers all record <-> alias associations from a child's + ioc.cfg, st.cmd, and parent ioc.cfg.""", + epilog='') # main command arguments parser.add_argument('patt', type=str) parser.add_argument('hutch', type=str) -parser.add_argument('-d','--dry_run', action='store_true', +parser.add_argument('-d', '--dry_run', action='store_true', default=False, help='''Forces a dry run for the script. No files are saved.''') ############################################################################### -#%% Main +# %% Main ############################################################################### + def main(): """ Main function entry point @@ -370,7 +471,7 @@ def main(): # parse args args = parser.parse_args() # search ioc_cfg and build the dataset - data = find_ioc(args.hutch,args.patt) + data = find_ioc(args.hutch, args.patt) if data is None: print(f'{Fore.RED}No results found for {Style.RESET_ALL}{args.patt}' + f'{Fore.RED} in{Style.RESET_ALL}' @@ -379,19 +480,19 @@ def main(): # find the parent directories for _d in data: - _d['parent_ioc'] = find_parent_ioc(_d['id'],_d['dir']) + _d['parent_ioc'] = find_parent_ioc(_d['id'], _d['dir']) # Hard code the column order for the find_ioc output - column_list = ['id','dir', + column_list = ['id', 'dir', Fore.LIGHTYELLOW_EX + 'parent_ioc' + Style.RESET_ALL, - 'host','port', + 'host', 'port', Fore.LIGHTBLUE_EX + 'alias' + Style.RESET_ALL, Fore.RED + 'disable' + Style.RESET_ALL] if args.hutch == 'all': column_list = ['hutch'] + column_list - show_temp_table(data,column_list) + show_temp_table(data, column_list) - ans = simple_prompt('Proceed? (Y/n): ',default='Y') + ans = simple_prompt('Proceed? (Y/n): ', default='Y') # Abort if user gets cold feet if ans is False: sys.exit() @@ -401,12 +502,12 @@ def main(): for _ioc in data: if _ioc.get('disable') is not True: # first acquire the base alias dictionary - alias_dicts = acquire_aliases(_ioc['dir'],_ioc['id']) + alias_dicts = acquire_aliases(_ioc['dir'], _ioc['id']) # show the record aliases to the user print(Fore.LIGHTGREEN_EX + 'The following substitutions were found in the st.cmd:' + Style.RESET_ALL) - print(build_table(alias_dicts,['record','alias'],align='l')) + print(build_table(alias_dicts, ['record', 'alias'], align='l')) # optional skip for all resulting PV aliases save_all = (simple_prompt( 'Do you want to save save_all resulting PV aliases? ' @@ -424,20 +525,21 @@ def main(): default_dest = os.getcwd() + '/' + f"{_ioc['id']}_alias" # now iterate through the alias dicts for PV alias substitutions - for i,a in enumerate(alias_dicts): + for i, a in enumerate(alias_dicts): # then iterate through all the PVs from root PV alias_list = process_alias_template(_ioc['parent_ioc'], a['record'], a['alias']) # capture output based on 61 char max record name - _output = [f"{l[0]:<61}{l[-1]:<61}" for l in alias_list] + _output = [f"{al[0]:<61}{al[-1]:<61}" for al in alias_list] # Demonstrate PV aliases on first iteration - if (i==0)|((show_pvs is True)&(skip_all is False)): + if (i == 0) | ((show_pvs is True) & (skip_all is False)): # show output to user, building a temp list of dict first - _temp = [{'PV' : l[0],'Alias' : l[-1]} for l in alias_list] + _temp = [{'PV': al[0], 'Alias': al[-1]} + for al in alias_list] print(Fore.LIGHTGREEN_EX + 'The following PV aliases are built:' + Style.RESET_ALL) - print(build_table(_temp,['PV','Alias'],align='l')) + print(build_table(_temp, ['PV', 'Alias'], align='l')) del _temp # If doing a dry run, skip this block @@ -472,20 +574,20 @@ def main(): save_data = False if save_all is True: - # only ask once + # only ask once dest = (request_file_dest( - 'Choose destination for data dump',default_dest)) + 'Choose destination for data dump', default_dest)) save_data = True save_all = None # else pester the user to approve for every single dataset - elif (save_all is not None)&(ans is True): + elif (save_all is not None) & (ans is True): dest = request_file_dest('Choose base file destination', - default_dest) + default_dest) save_data = True # write to file, else do nothing - if (save_data is True)&(args.dry_run is False): + if (save_data is True) & (args.dry_run is False): # make sure the destination exists and mkdir if it doesn't if os.path.exists(dest) is False: print(Fore.LIGHTBLUE_EX @@ -495,14 +597,14 @@ def main(): j = i if i < 10: j = '0'+f'{i}' - file_dest = dest+'/'+f"record_alias_{j}.txt" - with open(file_dest,'w',encoding='utf-8') as f: + file_dest = dest + '/' + f"record_alias_{j}.txt" + with open(file_dest, 'w', encoding='utf-8') as f: f.write('\n'.join(_output)) default_dest = dest del _output sys.exit() + if __name__ == '__main__': main() - \ No newline at end of file diff --git a/scripts/getPVAliases.sh b/scripts/getPVAliases.sh index 4c8d23dc..4d0859bd 100755 --- a/scripts/getPVAliases.sh +++ b/scripts/getPVAliases.sh @@ -1,45 +1,4 @@ #!/usr/bin/bash -# define usage -usage() -{ -cat << EOF -usage: gatherPVAliases [-h] [-d] patt hutch - -gathers all record <-> alias associations from a child's ioc.cfg, st.cmd, and -parent ioc.cfg. - -positional arguments: - patt - hutch - -options: - -h, --help show this help message and exit - -d, --dry_run Forces a dry run for the script. No files are saved. - -EOF -} - -# catch improper number of args or asking for help - -if [[ ("$#" -lt 2) || ("$1" == "-h") || ("$1" == "--help") ]]; then - usage - exit 0 -fi - -# source into pcds_conda if not currently active - -if [[ ($CONDA_SHLVL == 1) ]];then - if [[ "$(echo $CONDA_PREFIX | grep 'pcds-')" ]];then - : - else - source pcds_conda - fi -else - source pcds_conda -fi - # execute python script -python "$ENG_TOOLS_SCRIPTS/getPVAliases.py" $@ - -exit +/cds/group/pcds/pyps/conda/py39/envs/pcds-5.8.2/bin/python "$ENG_TOOLS_SCRIPTS/getPVAliases.py" $@ From e2e7300585988b190ea9f52acb19a0beda18ff3e Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Tue, 23 Apr 2024 18:44:17 -0700 Subject: [PATCH 04/23] MNT: read hutch codes from '...pyps/config' --- scripts/getPVAliases.py | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/scripts/getPVAliases.py b/scripts/getPVAliases.py index a49e6ce5..1c5511b7 100644 --- a/scripts/getPVAliases.py +++ b/scripts/getPVAliases.py @@ -17,6 +17,14 @@ from colorama import Fore, Style from prettytable import PrettyTable +############################################################################### +# %% Constants +############################################################################### + +VALID_HUTCH = sorted([s for s + in next(os.walk('/cds/group/pcds/pyps/config'))[1] + if '.' not in s]) + ############################################################################### # %% Functions ############################################################################### @@ -99,7 +107,8 @@ def flatten_list(input_list: list[list]) -> list[str]: return list(set(_result)) -def search_file(*, file: str, patt: str = None, prefix: str = '', +def search_file(*, file: str, output: list = None, + patt: str = None, prefix: str = '', color_wrap: Fore = None) -> list[str]: """ Searches file for regex match and appends result to list @@ -122,7 +131,8 @@ def search_file(*, file: str, patt: str = None, prefix: str = '', list[str] A list of the search results with the prefix prepended. """ - output = [] + if output is None: + output = [] color = '' reset = '' if color_wrap is not None: @@ -157,7 +167,7 @@ def fix_json(raw_data: str) -> list[str]: Returns ------- list[str] - The list of str ready for JSON loading + The list of str ready for JSON loading """ # clean empty rows and white space _temp = raw_data.replace(' ', '').strip() @@ -178,7 +188,8 @@ def fix_json(raw_data: str) -> list[str]: return result -def find_ioc(hutch: str = None, patt: str = None) -> list[dict]: +def find_ioc(hutch: str = None, patt: str = None, + valid_hutch: list[str] = VALID_HUTCH) -> list[dict]: """ A pythonic grep_ioc for gathering IOC details from the cfg file @@ -189,6 +200,9 @@ def find_ioc(hutch: str = None, patt: str = None) -> list[dict]: The default is None. patt: str, optional Regex pattern to search for. The default is None. + valid_hutch: list[str], optional + List of valid hutch codes to use. The default is taken + from the directories in '/cds/group/pcds/pyps/config' Raises ------ @@ -201,16 +215,13 @@ def find_ioc(hutch: str = None, patt: str = None) -> list[dict]: List of dictionaries generated by the JSON loading """ - hutch_list = ['xpp', 'xcs', 'cxi', 'mfx', 'mec', 'tmo', 'rix', 'xrt', - 'aux', 'det', 'fee', 'hpl', 'icl', 'las', 'lfe', 'kfe', - 'tst', 'txi', 'thz', 'all'] # check hutches - if (hutch is None) | (hutch not in tuple(hutch_list)): + if (hutch is None) | (hutch not in tuple(valid_hutch)): print('Invalid entry. Please choose a valid hutch:\n' - + ','.join(hutch_list)) + + ','.join(valid_hutch)) raise ValueError # create file paths - if hutch in tuple(hutch_list): + if hutch in tuple(valid_hutch): if hutch == 'all': path = gb.glob('/cds/group/pcds/pyps/config/*/iocmanager.cfg') else: @@ -260,7 +271,7 @@ def fix_dir(dir_path: str) -> str: Parameters ---------- - dir_path: str + dir_path : str The path to the child IOC's directory as a str. Returns @@ -318,9 +329,9 @@ def build_table(input_data: list[dict], columns: list[str] = None, input_data must be a list(dict) Parameters ---------- - input_data : list[dict] + input_data: list[dict] The data to generate a PrettyTable from. - columns : list, optional + columns: list, optional Columns for the PrettyTable headers. The default is None. **kwargs: kwargs to pass tohe PrettyTable() for extra customization. From 35474cb5e68ab929d76cdc83badffe90bff7347d Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Mon, 13 May 2024 18:50:13 -0700 Subject: [PATCH 05/23] ENH: Added grep_more_ioc.py and associated shell wrapper, simiplified getPVAliases.py as a result --- scripts/getPVAliases.py | 331 ++++---------------------- scripts/grep_more_ioc.py | 494 +++++++++++++++++++++++++++++++++++++++ scripts/grep_more_ioc.sh | 4 + 3 files changed, 541 insertions(+), 288 deletions(-) create mode 100644 scripts/grep_more_ioc.py create mode 100755 scripts/grep_more_ioc.sh diff --git a/scripts/getPVAliases.py b/scripts/getPVAliases.py index 1c5511b7..d108793f 100644 --- a/scripts/getPVAliases.py +++ b/scripts/getPVAliases.py @@ -11,49 +11,31 @@ import re import os.path import argparse -import json import copy -import glob as gb from colorama import Fore, Style from prettytable import PrettyTable +from grep_more_ioc import (find_ioc, simple_prompt, search_file, + find_parent_ioc, fix_dir, clean_ansi) + ############################################################################### # %% Constants ############################################################################### -VALID_HUTCH = sorted([s for s - in next(os.walk('/cds/group/pcds/pyps/config'))[1] - if '.' not in s]) +VALID_HUTCH = sorted([s for s + in next(os.walk('/cds/group/pcds/pyps/config'))[1] + if '.' not in s]+['all']) +# Keys from iocmanager. Found in /cds/group/pcds/config/*/iocmanager/utils.py +# Update this as needed +DEF_IMGR_KEYS = ['procmgr_config', 'hosts', 'dir', 'id', 'cmd', + 'flags', 'port', 'host', 'disable', 'history', + 'delay', 'alias', 'hard'] ############################################################################### # %% Functions ############################################################################### -def simple_prompt(prompt: str, default: str = 'N') -> bool: - """ - Simple yes/no prompt which defaults to 'N' = False, but can be changed. - - Returns - ------- - bool - Prompted yes/no response - - """ - while True: - p = input(prompt).strip().lower() - if p in ['']: - p = default.lower() - if p[0] == 'y': - result = True - break - if p.lower()[0] == 'n': - result = False - break - print('Invalid Entry. Please choose again.') - return result - - def request_file_dest(prompt: str, default: str = os.getcwd(),) -> str: """ @@ -91,7 +73,7 @@ def request_file_dest(prompt: str, def flatten_list(input_list: list[list]) -> list[str]: """ Flatten a 2D lists and find its unique values. - Note: loses order due to set() call. + Note: loses order due to set() call. sear Parameters ---------- @@ -107,221 +89,6 @@ def flatten_list(input_list: list[list]) -> list[str]: return list(set(_result)) -def search_file(*, file: str, output: list = None, - patt: str = None, prefix: str = '', - color_wrap: Fore = None) -> list[str]: - """ - Searches file for regex match and appends result to list - - Parameters - ---------- - file: str - The file to read and search. Encoding must be utf-8 - output: list, optional - A list to appead your results to. The default is None. - patt: str, optional - The regex pattern to search for. The default is None. - prefix: str, optional - A str prefix to add to each line. The default is ''. - color_wrap: Fore, optional - Color wrapping using Colorama.Fore. The default is None. - - Returns - ------- - list[str] - A list of the search results with the prefix prepended. - """ - if output is None: - output = [] - color = '' - reset = '' - if color_wrap is not None: - color = color_wrap - reset = Style.RESET_ALL - if os.path.isfile(file) is False: - print(f'{file} does not exist') - return '' - with open(file, 'r', encoding='utf-8') as _f: - for line in _f.readlines(): - if re.search(patt, line): - output.append(re.sub(patt, color + r'\g<0>' + reset, line)) - return prefix + prefix.join(output) - - -def clean_ansi(text: str = None) -> str: - """ - Removes ANSI escape sequences from a str, including fg/bg formatting. - """ - ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') - return ansi_escape.sub('', text) - - -def fix_json(raw_data: str) -> list[str]: - """ - Fixes JSON format of find_ioc/grep_ioc output. - - Parameters - ---------- - raw_data: str - Str output generated by find_ioc/grep_ioc, which is pseudo-JSON. - Returns - ------- - list[str] - The list of str ready for JSON loading - """ - # clean empty rows and white space - _temp = raw_data.replace(' ', '').strip() - # capture and fix the keys not properly formatted to str - _temp = re.sub(r"(?'", raw_data) - # capture boolean tokens and fix them for json format - _temp = re.sub("True", "true", _temp) - _temp = re.sub("False", "false", _temp) - # then capture and fix digits not formatted to str - _temp = re.sub(r"(?<=:)\d+", r"'\g<0>'", _temp) - # then properly convert to list of json obj - result = (_temp - .replace('\'', '\"') - .replace('},', '}') - .replace(' {', '{') - .strip() - .split('\n')) - return result - - -def find_ioc(hutch: str = None, patt: str = None, - valid_hutch: list[str] = VALID_HUTCH) -> list[dict]: - """ - A pythonic grep_ioc for gathering IOC details from the cfg file - - Parameters - ---------- - hutch: str, optional - 3 letter lowercase hutch code. May also include 'all'. - The default is None. - patt: str, optional - Regex pattern to search for. The default is None. - valid_hutch: list[str], optional - List of valid hutch codes to use. The default is taken - from the directories in '/cds/group/pcds/pyps/config' - - Raises - ------ - ValueError - Hutch code is invalid or regex pattern is missing. - - Returns - ------- - list[dict] - List of dictionaries generated by the JSON loading - - """ - # check hutches - if (hutch is None) | (hutch not in tuple(valid_hutch)): - print('Invalid entry. Please choose a valid hutch:\n' - + ','.join(valid_hutch)) - raise ValueError - # create file paths - if hutch in tuple(valid_hutch): - if hutch == 'all': - path = gb.glob('/cds/group/pcds/pyps/config/*/iocmanager.cfg') - else: - path = [f'/cds/group/pcds/pyps/config/{hutch}/iocmanager.cfg'] - # check patt and generate the regex pattern - if patt is None: - print('No regex pattern supplied') - raise ValueError - patt = r'{.*' + patt + r'.*}' - # initialize output list - result = [] - # iterate and capture results. - for _file in path: - prefix = '' - if len(path) != 1: - prefix = _file+':' - output = search_file(file=_file, patt=patt, prefix=prefix) - if output != prefix: - result.append(output) - # reconstruct the list of str - _temp = ''.join(result) - if len(_temp) == 0: - print(f'{Fore.RED}No results found for {Style.RESET_ALL}{patt}' - + f'{Fore.RED} in{Style.RESET_ALL}' - + f'{hutch}') - return None - # capture the hutch from the cfg path if hutch = all - if hutch == 'all': - hutch_cfgs = re.findall(r'/.*cfg\:', _temp) - hutch_cfgs = [''.join(re.findall(r'(?<=/)\w+(?=/ioc)', s)) - for s in hutch_cfgs] - # strip the file information - _temp = re.sub(r'.*cfg\:', '', _temp) - # now convert back to json and load - output = [json.loads(s) for s in fix_json(_temp)] - # and add the hutches back into the dicts if searching across all cfgs - if hutch == 'all': - for _i, _d in enumerate(output): - _d['hutch'] = hutch_cfgs[_i] - return output - - -def fix_dir(dir_path: str) -> str: - """ - Simple function for repairing the child release IOC path based on - the ioc_cfg output. Returns the proper dir for the child IOC.cfg file. - - Parameters - ---------- - dir_path : str - The path to the child IOC's directory as a str. - - Returns - ------- - str - The typo-corrected path as a str. - - """ - - # catches the short form path - if dir_path.startswith('ioc/'): - output_dir = '/cds/group/pcds/epics/'+dir_path - # for the rare, old child IOCs that only exist in their parent's release - elif 'common' in dir_path: - output_dir = dir_path + '/children' - # Otherwise fine! - else: - output_dir = dir_path - # Make sure the end of the path is a folder! - if output_dir[-1] != '/': - output_dir += '/' - return output_dir - - -def find_parent_ioc(file: str, path: str) -> str: - """ - Searches the child IOC for the parent's release pointer - Returns the parent's IOC as a str. - - Parameters - ---------- - file : str, optional - DESCRIPTION. The default is None. - path : str, optional - DESCRIPTION. The default is None. - - Returns - ------- - str - Path to the parent IOC's release. - - """ - file_dir = fix_dir(path) - if os.path.exists(f'{file_dir}{file}.cfg') is False: - return 'Invalid. Child does not exist.' - parent_ioc_release = search_file(file=f'{file_dir}{file}.cfg', - patt='^RELEASE').strip() - return parent_ioc_release.rsplit('=', maxsplit=1)[-1] - - def build_table(input_data: list[dict], columns: list[str] = None, **kwargs) -> PrettyTable: """ @@ -509,6 +276,9 @@ def main(): sys.exit() print(f'{Fore.RED}Skipping disabled child IOCs{Style.RESET_ALL}') + # initialize the final output to write to file + final_output = [] + # iterate through all the child ioc dictionaries for _ioc in data: if _ioc.get('disable') is not True: @@ -521,27 +291,27 @@ def main(): print(build_table(alias_dicts, ['record', 'alias'], align='l')) # optional skip for all resulting PV aliases save_all = (simple_prompt( - 'Do you want to save save_all resulting PV aliases? ' - + 'This will generate ' + 'Do you want to save all resulting PV aliases? ' + + 'This will append ' + Fore.LIGHTYELLOW_EX + f'{len(alias_dicts)}' + Style.RESET_ALL - + ' files (y/N): ')) + + ' record sets (y/N): ')) - # initialize a flag for skipping annoying prompts + # initialize flags skip_all = None show_pvs = None + save_data = None # initialize a default file directory for dumping aliases default_dest = os.getcwd() + '/' + f"{_ioc['id']}_alias" - # now iterate through the alias dicts for PV alias substitutions for i, a in enumerate(alias_dicts): # then iterate through all the PVs from root PV alias_list = process_alias_template(_ioc['parent_ioc'], a['record'], a['alias']) # capture output based on 61 char max record name - _output = [f"{al[0]:<61}{al[-1]:<61}" for al in alias_list] + _chunk = [f"{al[0]:<61}{al[-1]:<61}" for al in alias_list] # Demonstrate PV aliases on first iteration if (i == 0) | ((show_pvs is True) & (skip_all is False)): # show output to user, building a temp list of dict first @@ -560,59 +330,44 @@ def main(): continue # ask user for input if save_all is False: - ans = (simple_prompt( + save_data = (simple_prompt( 'Would you like to save this PV set? (y/N): ')) - if ans is True: + if save_data is True: # give the user an option to be lazy again save_all = (simple_prompt( 'Would you like to apply this for' - + ' all subsequent sets? (y/N): ')) + + ' ALL remaining sets? (y/N): ')) # Avoid some terminal spam using these flags show_pvs = not save_all skip_all = False - if ans is False: + if save_data is False: skip_all = (simple_prompt( 'Skip all further substitutions? (Y/n): ', default='Y')) - save_data = False # Avoid some terminal spam using this flag show_pvs = not skip_all continue - else: # Set flags to surpress prompts during dry run - ans = False save_data = False - - if save_all is True: - # only ask once - dest = (request_file_dest( - 'Choose destination for data dump', default_dest)) - save_data = True - save_all = None - - # else pester the user to approve for every single dataset - elif (save_all is not None) & (ans is True): - dest = request_file_dest('Choose base file destination', - default_dest) - save_data = True - - # write to file, else do nothing - if (save_data is True) & (args.dry_run is False): - # make sure the destination exists and mkdir if it doesn't - if os.path.exists(dest) is False: - print(Fore.LIGHTBLUE_EX - + f'Making directory: {dest}' + Style.RESET_ALL) - os.mkdir(dest) - # pad leading 0 for file sorting - j = i - if i < 10: - j = '0'+f'{i}' - file_dest = dest + '/' + f"record_alias_{j}.txt" - with open(file_dest, 'w', encoding='utf-8') as f: - f.write('\n'.join(_output)) - default_dest = dest - del _output + if (save_data or save_all) and (args.dry_run is False): + final_output.append('\n'.join(_chunk)) + del _chunk + + # write to file, else do nothing + if (len(final_output) > 0) & (args.dry_run is False): + dest = request_file_dest('Choose base file destination', + default_dest) + # make sure the destination exists and mkdir if it doesn't + if os.path.exists(dest) is False: + print(Fore.LIGHTBLUE_EX + + f'Making directory: {dest}' + Style.RESET_ALL) + os.mkdir(dest) + file_dest = dest + "/record_alias_dump.txt" + with open(file_dest, 'w', encoding='utf-8') as f: + f.write('\n'.join(final_output)) + default_dest = dest + del final_output sys.exit() diff --git a/scripts/grep_more_ioc.py b/scripts/grep_more_ioc.py new file mode 100644 index 00000000..27636c44 --- /dev/null +++ b/scripts/grep_more_ioc.py @@ -0,0 +1,494 @@ +"""A tool for enhancing the grep_ioc CLI tool for ECS-Delivery team +enginer = aberges +version = 1.0.6 +""" +############################################################################### +# %% Imports +############################################################################### + +import sys +import re +import os.path +import argparse +import json +import glob as gb +from shutil import get_terminal_size +from colorama import Fore, Style +import pandas as pd + +############################################################################### +# %% Global settings +############################################################################### +# Change max rows displayed to prevent truncating the dataframe +# We'll assume 1000 rows as an upper limit + +pd.set_option("display.max_rows",1000) + +############################################################################### +# %% Constants +############################################################################### + +VALID_HUTCH = sorted([s for s + in next(os.walk('/cds/group/pcds/pyps/config'))[1] + if '.' not in s]+['all']) +# Keys from iocmanager. Found in /cds/group/pcds/config/*/iocmanager/utils.py +# Update this as needed +DEF_IMGR_KEYS = ['procmgr_config', 'hosts', 'dir', 'id', 'cmd', + 'flags', 'port', 'host', 'disable', 'history', + 'delay', 'alias', 'hard'] + +############################################################################### +# %% Functions +############################################################################### + +def search_file(*, file: str, output: list = None, + patt: str = None, prefix: str = '', + quiet: bool=False, color_wrap: Fore = None) -> list[str]: + """ + Searches file for regex match and appends result to list + + Parameters + ---------- + file: str + The file to read and search. Encoding must be utf-8 + output: list, optional + A list to appead your results to. The default is None. + patt: str, optional + The regex pattern to search for. The default is None. + prefix: str, optional + A str prefix to add to each line. The default is ''. + color_wrap: Fore, optional + Color wrapping using Colorama.Fore. The default is None. + quiet: bool, optional + Whether to surpress the warning printed to terminal when "file" + does not exist. The default is False. + + Returns + ------- + list[str] + A list of the search results with the prefix prepended. + """ + if output is None: + output = [] + color = '' + reset = '' + if color_wrap is not None: + color = color_wrap + reset = Style.RESET_ALL + if os.path.isfile(file) is False: + if not quiet: + print(f'{file} does not exist') + return '' + with open(file, 'r', encoding='utf-8') as _f: + for line in _f.readlines(): + if re.search(patt, line): + output.append(re.sub(patt, color + r'\g<0>' + reset, line)) + return prefix + prefix.join(output) + + +def print_skip_comments(file:str): + """Prints contents of a file while ignoring comments""" + with open(file,'r',encoding='utf_8') as _f: + for line in _f.readlines(): + if not line.strip().startswith('#'): + print(line.strip()) + + +def simple_prompt(prompt: str, default: str = 'N'): + """Simple yes/no prompt which defaults to No""" + while True: + p = input(prompt).strip().lower() + if p in ['']: + p = default.lower() + if p[0] == 'y': + result = True + break + if p.lower()[0] == 'n': + result = False + break + print('Invalid Entry. Please choose again.') + return result + +def clean_ansi(text: str = None) -> str: + """ + Removes ANSI escape sequences from a str, including fg/bg formatting. + """ + ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') + return ansi_escape.sub('', text) + + +def fix_json(raw_data: str, keys: list[str] = None) -> list[str]: + """ + Fixes JSON format of find_ioc/grep_ioc output. + + Parameters + ---------- + raw_data: str + Str output generated by find_ioc/grep_ioc, which is pseudo-JSON. + keys: list[str] + A list of valid keys to use for scraping the IOC.cfg file. + Returns + ------- + list[str] + The list of str ready for JSON loading + """ + if keys is None: + valid_keys = re.compile(r'(?=\s?:\s?)|'.join(DEF_IMGR_KEYS)) + # clean empty rows and white space + _temp = raw_data.replace(' ', '').strip() + # capture and fix the keys not properly formatted to str + #_temp = re.sub(r"(?'", raw_data) + _temp = re.sub(valid_keys,r"'\g<0>'", raw_data) + # capture boolean tokens and fix them for json format + _temp = re.sub("True", "true", _temp) + _temp = re.sub("False", "false", _temp) + # then capture and fix digits not formatted to str + _temp = re.sub(r"(?<=:)\d+", r"'\g<0>'", _temp) + # then properly convert to list of json obj + result = (_temp + .replace('\'', '\"') + .replace('},', '}') + .replace(' {', '{') + .strip() + .split('\n')) + return result + + +def find_ioc(hutch: str = None, patt: str = None, + valid_hutch: list[str] = VALID_HUTCH) -> list[dict]: + """ + A pythonic grep_ioc for gathering IOC details from the cfg file + + Parameters + ---------- + hutch: str, optional + 3 letter lowercase hutch code. May also include 'all'. + The default is None. + patt: str, optional + Regex pattern to search for. The default is None. + valid_hutch: list[str], optional + List of valid hutch codes to use. The default is taken + from the directories in '/cds/group/pcds/pyps/config' + + Raises + ------ + ValueError + Hutch code is invalid or regex pattern is missing. + + Returns + ------- + list[dict] + List of dictionaries generated by the JSON loading + + """ + # check hutches + if (hutch is None) | (hutch not in tuple(valid_hutch)): + print('Invalid entry. Please choose a valid hutch:\n' + + ','.join(valid_hutch)) + raise ValueError + # create file paths + if hutch in tuple(valid_hutch): + if hutch == 'all': + path = gb.glob('/cds/group/pcds/pyps/config/*/iocmanager.cfg') + else: + path = [f'/cds/group/pcds/pyps/config/{hutch}/iocmanager.cfg'] + # check patt and generate the regex pattern + if patt is None: + print('No regex pattern supplied') + raise ValueError + _patt = r'{.*' + patt + r'.*}' + # initialize output list + result = [] + # iterate and capture results. + for _file in path: + prefix = '' + if len(path) != 1: + prefix = _file+':' + output = search_file(file=_file, patt=_patt, prefix=prefix) + if output != prefix: + result.append(output) + # reconstruct the list of str + _temp = ''.join(result) + if len(_temp) == 0: + print(f'{Fore.RED}No results found for {Style.RESET_ALL}{patt}' + + f'{Fore.RED} in{Style.RESET_ALL} ' + + f'{hutch}') + return None + # capture the hutch from the cfg path if hutch = all + if hutch == 'all': + hutch_cfgs = re.findall(r'/.*cfg\:', _temp) + hutch_cfgs = [''.join(re.findall(r'(?<=/)\w+(?=/ioc)', s)) + for s in hutch_cfgs] + # strip the file information + _temp = re.sub(r'.*cfg\:', '', _temp) + # now convert back to json and load + output = [json.loads(s) for s in fix_json(_temp)] + # and add the hutches back into the dicts if searching across all cfgs + if hutch == 'all': + for _i, _d in enumerate(output): + _d['hutch'] = hutch_cfgs[_i] + return output + + +def fix_dir(dir_path: str) -> str: + """ + Simple function for repairing the child release IOC path based on + the ioc_cfg output. Returns the proper dir for the child IOC.cfg file. + + Parameters + ---------- + dir_path : str + The path to the child IOC's directory as a str. + + Returns + ------- + str + The typo-corrected path as a str. + + """ + + # catches the short form path + if dir_path.startswith('ioc/'): + output_dir = '/cds/group/pcds/epics/'+dir_path + # for the rare, old child IOCs that only exist in their parent's release + elif 'common' in dir_path: + output_dir = dir_path + '/children' + # Otherwise fine! + else: + output_dir = dir_path + # Make sure the end of the path is a folder! + if output_dir[-1] != '/': + output_dir += '/' + return output_dir + + +def find_parent_ioc(file: str, path: str) -> str: + """ + Searches the child IOC for the parent's release pointer + Returns the parent's IOC as a str. + + Parameters + ---------- + file : str, optional + DESCRIPTION. The default is None. + path : str, optional + DESCRIPTION. The default is None. + + Returns + ------- + str + Path to the parent IOC's release. + + """ + file_dir = fix_dir(path) + if os.path.exists(f'{file_dir}{file}.cfg') is False: + return 'Invalid. Child does not exist.' + parent_ioc_release = search_file(file=f'{file_dir}{file}.cfg', + patt='^RELEASE').strip() + return parent_ioc_release.rsplit('=', maxsplit=1)[-1] + + +def print_frame2term(dataframe: pd.DataFrame=None,): + """Wrapper for displaying the dataframe to proper terminal size""" + with pd.option_context('display.max_rows', None, + 'display.max_columns', None, + 'display.width', + get_terminal_size(fallback=(120,50))[0], + ): + print(dataframe) + +############################################################################### +# %% Arg Parser +############################################################################### + +def build_parser(): + """ + Builds the parser & subparsers for the main function + """ + # parser obj configuration + parser = argparse.ArgumentParser( + prog='grep_more_ioc', + description='transforms grep_ioc output to json object' + + ' and prints in pandas.DataFrame', + epilog='A work in progress') + # main command arguments + parser.add_argument('patt', type=str) + parser.add_argument('hutch', type=str) + parser.add_argument('-d','--ignore_disabled', + action='store_true', + default=False, + help='Flag for excluding based' + + ' on the "disabled" state.') + # subparsers + subparsers = (parser + .add_subparsers(help='Follow-up commands after grep_ioc ' + + 'executes')) +#-----------------------------------------------------------------------------# +# print subarguments +#-----------------------------------------------------------------------------# + print_frame = (subparsers + .add_parser('print', + help='Just a simple print of the dataframe' + + ' to the terminal.')) + + print_frame.add_argument('print', + action='store_true',default=False) + + print_frame.add_argument('-c','--skip_comments', action='store_true', + default=False, + help='Prints the IOC.cfg' + + ' file with comments skipped') + + print_frame.add_argument('-r','--release', action='store_true', + default=False, + help="Adds the parent IOC's" + + " release to the dataframe") + + print_frame.add_argument('-s','--print_dirs', action='store_true', + default=False, + help='Prints the child & parent IOC' + + 'directories as the final output') +#-----------------------------------------------------------------------------# +# search subarguments +#-----------------------------------------------------------------------------# + search = subparsers.add_parser('search', + help='For searching regex patts in the' + + ' IOC *.cfg captured by grep_ioc.' + + ' Useful for checking the release pointers') + + search.add_argument('search', + type=str, help='PATT to use for regex search in file', + metavar='PATT') + search.add_argument('-q','--quiet', action='store_true', default=False, + help='Surpresses file warning for paths that do not' + + 'exist.') + search.add_argument('-o','--only_search', action='store_true', + default=False, + help="Don't print the dataframe, just search results.") + return parser + +############################################################################### +#%% Main +############################################################################### +def main(): + """ + Main entry point of the program. For using with CLI tools. + """ + parser = build_parser() + args = parser.parse_args() + # read grep_ioc output + data = find_ioc(args.hutch,args.patt) + + # exit if grep_ioc finds nothing + if (data is None or len(data) == 0): + print(f'{Fore.RED}No IOCs were found.\nExiting . . .{Style.RESET_ALL}') + sys.exit() + + # create the dataframe after fixing the json format + df = pd.json_normalize(data) + + # reorder the dataframe if searching all hutches + if args.hutch == 'all': + df.insert(0,'hutch',df.pop('hutch')) + + # pad the disable column based on the grep_ioc output + if 'disable' not in df.columns: + df['disable'] = df.index.size*[False] + if 'disable' in df.columns: + df.disable.fillna(False,inplace=True) + + # Fill the NaN with empty strings for rarely used keys + for _col in df.columns: + if _col not in ['delay']: + df[_col].fillna('',inplace=True) + else: + df[_col].fillna(0,inplace=True) + + # check for the ignore_disabled flag + if args.ignore_disabled is True: + df = df[~df.disable].reset_index(drop=True) + +#-----------------------------------------------------------------------------# +# %%% print +#-----------------------------------------------------------------------------# + # print the dataframe + if hasattr(args,'print'): + if args.release is True: + # intialize list for adding a new column + output_list=[] + # iterate through ioc and directory pairs + for f,d in df.loc[:,['id','dir']].values: + search_result = find_parent_ioc(f,d) + # catch parent IOCs running out of dev + if 'epics-dev' in search_result: + output_str = search_result + # abbreviate path for standard IOC releases + elif 'common' in search_result: + output_str = '/'.join( + search_result.rsplit('common/',maxsplit=1)[-1]) + # add it to the list + output_list.append(output_str) + # Then, finally, add the column to the dataframe + df['Release Version'] = output_list + # put it next to the child dirs + df.insert(df.columns.tolist().index('dir')+1, + 'Release Version', + df.pop('Release Version')) + + print_frame2term(df) + + if args.skip_comments is True: + for ioc,d in df.loc[:,['id','dir']].values: + # fixes dirs if grep_ioc truncates the path due to common ioc dir + target_dir = fix_dir(d) + print(f'{Fore.LIGHTBLUE_EX}Now in: {target_dir}' + +Style.RESET_ALL) + print(f'{Fore.LIGHTYELLOW_EX}{ioc}:{Style.RESET_ALL}') + # prints the contents of the file while ignoring comments + print_skip_comments(file=f'{target_dir}{ioc}.cfg') + + if args.print_dirs is True: + print(f'{Fore.LIGHTRED_EX}\nDumping directories:\n' + +Style.RESET_ALL) + for f,d in df.loc[:,['id','dir']].values: + search_result=find_parent_ioc(f,d) + d = fix_dir(d) + # Print this for easier cd / pushd shenanigans + print(f'{d}{Fore.LIGHTYELLOW_EX}{f}.cfg{Style.RESET_ALL}' + +'\n\t\t|-->' + + f'{Fore.LIGHTGREEN_EX}RELEASE={Style.RESET_ALL}' + + f'{Fore.LIGHTBLUE_EX}{search_result}{Style.RESET_ALL}' + ) + +#-----------------------------------------------------------------------------# +# %%% search +#-----------------------------------------------------------------------------# + # do a local grep on each file in their corresponding directory + if hasattr(args,'search'): + # optionally print the dataframe + if not args.only_search: + print_frame2term(df) + check_search = [] + for ioc,d in df.loc[:,['id','dir']].values: + target_dir = fix_dir(d) + # Search for pattern after moving into the directory + if args.search is not None: + search_result = (search_file(file=f'{target_dir}{ioc}.cfg', + patt=args.search, + color_wrap=Fore.LIGHTRED_EX, + quiet=args.quiet) + .strip() + ) + if len(search_result) > 0: + print(f'{Fore.LIGHTYELLOW_EX}{ioc}:{Style.RESET_ALL}') + print(''.join(search_result.strip())) + check_search.append(len(search_result)) + if len(check_search) == 0: + print(Fore.RED + 'No search results found' + Style.RESET_ALL) +#-----------------------------------------------------------------------------# +# %%% Exit +#-----------------------------------------------------------------------------# + sys.exit() + +if __name__ == '__main__': + main() diff --git a/scripts/grep_more_ioc.sh b/scripts/grep_more_ioc.sh new file mode 100755 index 00000000..4b4e93f9 --- /dev/null +++ b/scripts/grep_more_ioc.sh @@ -0,0 +1,4 @@ +#!/usr/bin/bash + +# execute python script +/cds/group/pcds/pyps/conda/py39/envs/pcds-5.8.4/bin/python /cds/home/a/aberges/bin/py_tools_wip/grep_more_ioc.py "$@" From 0c1933966e2a11e02b94b52999a9564f07166eac Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Mon, 13 May 2024 19:29:53 -0700 Subject: [PATCH 06/23] BUG: Fixed bugs on parent release prints and fixed rel. path in shell command --- scripts/grep_more_ioc.py | 4 ++-- scripts/grep_more_ioc.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/grep_more_ioc.py b/scripts/grep_more_ioc.py index 27636c44..72486abe 100644 --- a/scripts/grep_more_ioc.py +++ b/scripts/grep_more_ioc.py @@ -424,8 +424,8 @@ def main(): output_str = search_result # abbreviate path for standard IOC releases elif 'common' in search_result: - output_str = '/'.join( - search_result.rsplit('common/',maxsplit=1)[-1]) + output_str = (search_result + .rsplit('common/',maxsplit=1)[-1]) # add it to the list output_list.append(output_str) # Then, finally, add the column to the dataframe diff --git a/scripts/grep_more_ioc.sh b/scripts/grep_more_ioc.sh index 4b4e93f9..14bff9a4 100755 --- a/scripts/grep_more_ioc.sh +++ b/scripts/grep_more_ioc.sh @@ -1,4 +1,4 @@ #!/usr/bin/bash # execute python script -/cds/group/pcds/pyps/conda/py39/envs/pcds-5.8.4/bin/python /cds/home/a/aberges/bin/py_tools_wip/grep_more_ioc.py "$@" +/cds/group/pcds/pyps/conda/py39/envs/pcds-5.8.4/bin/python $ENG_TOOLS_SCRIPTS/grep_more_ioc.py "$@" From 9d028c338db1155c010c23ff198a3033f440c98e Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Tue, 14 May 2024 11:31:24 -0700 Subject: [PATCH 07/23] MNT: flake8, general style, shellcheck --- scripts/getPVAliases.py | 2 +- scripts/getPVAliases.sh | 2 +- scripts/grep_more_ioc.py | 122 +++++++++++++++++++++------------------ 3 files changed, 68 insertions(+), 58 deletions(-) diff --git a/scripts/getPVAliases.py b/scripts/getPVAliases.py index d108793f..b40599c0 100644 --- a/scripts/getPVAliases.py +++ b/scripts/getPVAliases.py @@ -357,7 +357,7 @@ def main(): # write to file, else do nothing if (len(final_output) > 0) & (args.dry_run is False): dest = request_file_dest('Choose base file destination', - default_dest) + default_dest) # make sure the destination exists and mkdir if it doesn't if os.path.exists(dest) is False: print(Fore.LIGHTBLUE_EX diff --git a/scripts/getPVAliases.sh b/scripts/getPVAliases.sh index 4d0859bd..d62b6b94 100755 --- a/scripts/getPVAliases.sh +++ b/scripts/getPVAliases.sh @@ -1,4 +1,4 @@ #!/usr/bin/bash # execute python script -/cds/group/pcds/pyps/conda/py39/envs/pcds-5.8.2/bin/python "$ENG_TOOLS_SCRIPTS/getPVAliases.py" $@ +/cds/group/pcds/pyps/conda/py39/envs/pcds-5.8.2/bin/python "$ENG_TOOLS_SCRIPTS/getPVAliases.py" "$@" diff --git a/scripts/grep_more_ioc.py b/scripts/grep_more_ioc.py index 72486abe..3da26b5c 100644 --- a/scripts/grep_more_ioc.py +++ b/scripts/grep_more_ioc.py @@ -22,7 +22,7 @@ # Change max rows displayed to prevent truncating the dataframe # We'll assume 1000 rows as an upper limit -pd.set_option("display.max_rows",1000) +pd.set_option("display.max_rows", 1000) ############################################################################### # %% Constants @@ -41,9 +41,10 @@ # %% Functions ############################################################################### + def search_file(*, file: str, output: list = None, patt: str = None, prefix: str = '', - quiet: bool=False, color_wrap: Fore = None) -> list[str]: + quiet: bool = False, color_wrap: Fore = None) -> list[str]: """ Searches file for regex match and appends result to list @@ -86,9 +87,9 @@ def search_file(*, file: str, output: list = None, return prefix + prefix.join(output) -def print_skip_comments(file:str): +def print_skip_comments(file: str): """Prints contents of a file while ignoring comments""" - with open(file,'r',encoding='utf_8') as _f: + with open(file, 'r', encoding='utf_8') as _f: for line in _f.readlines(): if not line.strip().startswith('#'): print(line.strip()) @@ -109,6 +110,7 @@ def simple_prompt(prompt: str, default: str = 'N'): print('Invalid Entry. Please choose again.') return result + def clean_ansi(text: str = None) -> str: """ Removes ANSI escape sequences from a str, including fg/bg formatting. @@ -126,19 +128,18 @@ def fix_json(raw_data: str, keys: list[str] = None) -> list[str]: raw_data: str Str output generated by find_ioc/grep_ioc, which is pseudo-JSON. keys: list[str] - A list of valid keys to use for scraping the IOC.cfg file. + A list of valid keys to use for scraping the IOC.cfg file. Returns ------- list[str] - The list of str ready for JSON loading + The list of str ready for JSON loading """ if keys is None: valid_keys = re.compile(r'(?=\s?:\s?)|'.join(DEF_IMGR_KEYS)) # clean empty rows and white space _temp = raw_data.replace(' ', '').strip() # capture and fix the keys not properly formatted to str - #_temp = re.sub(r"(?'", raw_data) - _temp = re.sub(valid_keys,r"'\g<0>'", raw_data) + _temp = re.sub(valid_keys, r"'\g<0>'", raw_data) # capture boolean tokens and fix them for json format _temp = re.sub("True", "true", _temp) _temp = re.sub("False", "false", _temp) @@ -288,12 +289,12 @@ def find_parent_ioc(file: str, path: str) -> str: return parent_ioc_release.rsplit('=', maxsplit=1)[-1] -def print_frame2term(dataframe: pd.DataFrame=None,): +def print_frame2term(dataframe: pd.DataFrame = None,): """Wrapper for displaying the dataframe to proper terminal size""" with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', - get_terminal_size(fallback=(120,50))[0], + get_terminal_size(fallback=(120, 50))[0], ): print(dataframe) @@ -301,20 +302,21 @@ def print_frame2term(dataframe: pd.DataFrame=None,): # %% Arg Parser ############################################################################### + def build_parser(): """ Builds the parser & subparsers for the main function """ # parser obj configuration parser = argparse.ArgumentParser( - prog='grep_more_ioc', - description='transforms grep_ioc output to json object' - + ' and prints in pandas.DataFrame', - epilog='A work in progress') + prog='grep_more_ioc', + description='transforms grep_ioc output to json object' + + ' and prints in pandas.DataFrame', + epilog='With extra utilities for daily ECS') # main command arguments parser.add_argument('patt', type=str) parser.add_argument('hutch', type=str) - parser.add_argument('-d','--ignore_disabled', + parser.add_argument('-d', '--ignore_disabled', action='store_true', default=False, help='Flag for excluding based' @@ -323,53 +325,56 @@ def build_parser(): subparsers = (parser .add_subparsers(help='Follow-up commands after grep_ioc ' + 'executes')) -#-----------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # # print subarguments -#-----------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # print_frame = (subparsers .add_parser('print', help='Just a simple print of the dataframe' + ' to the terminal.')) print_frame.add_argument('print', - action='store_true',default=False) + action='store_true', default=False) - print_frame.add_argument('-c','--skip_comments', action='store_true', + print_frame.add_argument('-c', '--skip_comments', action='store_true', default=False, help='Prints the IOC.cfg' + ' file with comments skipped') - print_frame.add_argument('-r','--release', action='store_true', + print_frame.add_argument('-r', '--release', action='store_true', default=False, help="Adds the parent IOC's" + " release to the dataframe") - print_frame.add_argument('-s','--print_dirs', action='store_true', + print_frame.add_argument('-s', '--print_dirs', action='store_true', default=False, help='Prints the child & parent IOC' + 'directories as the final output') -#-----------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # # search subarguments -#-----------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # search = subparsers.add_parser('search', - help='For searching regex patts in the' - + ' IOC *.cfg captured by grep_ioc.' - + ' Useful for checking the release pointers') + help='For searching regex patts in the' + + ' IOC *.cfg captured by grep_ioc.' + + ' Useful for checking the pointers' + + "to the parent's release") search.add_argument('search', - type=str, help='PATT to use for regex search in file', - metavar='PATT') - search.add_argument('-q','--quiet', action='store_true', default=False, + type=str, help='PATT to use for regex search in file', + metavar='PATT') + search.add_argument('-q', '--quiet', action='store_true', default=False, help='Surpresses file warning for paths that do not' + 'exist.') - search.add_argument('-o','--only_search', action='store_true', + search.add_argument('-o', '--only_search', action='store_true', default=False, help="Don't print the dataframe, just search results.") return parser ############################################################################### -#%% Main +# %% Main ############################################################################### + + def main(): """ Main entry point of the program. For using with CLI tools. @@ -377,7 +382,7 @@ def main(): parser = build_parser() args = parser.parse_args() # read grep_ioc output - data = find_ioc(args.hutch,args.patt) + data = find_ioc(args.hutch, args.patt) # exit if grep_ioc finds nothing if (data is None or len(data) == 0): @@ -389,43 +394,43 @@ def main(): # reorder the dataframe if searching all hutches if args.hutch == 'all': - df.insert(0,'hutch',df.pop('hutch')) + df.insert(0, 'hutch', df.pop('hutch')) # pad the disable column based on the grep_ioc output if 'disable' not in df.columns: df['disable'] = df.index.size*[False] if 'disable' in df.columns: - df.disable.fillna(False,inplace=True) + df.disable.fillna(False, inplace=True) # Fill the NaN with empty strings for rarely used keys for _col in df.columns: if _col not in ['delay']: - df[_col].fillna('',inplace=True) + df[_col].fillna('', inplace=True) else: - df[_col].fillna(0,inplace=True) + df[_col].fillna(0, inplace=True) # check for the ignore_disabled flag if args.ignore_disabled is True: df = df[~df.disable].reset_index(drop=True) -#-----------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # # %%% print -#-----------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # # print the dataframe - if hasattr(args,'print'): + if hasattr(args, 'print'): if args.release is True: # intialize list for adding a new column - output_list=[] + output_list = [] # iterate through ioc and directory pairs - for f,d in df.loc[:,['id','dir']].values: - search_result = find_parent_ioc(f,d) + for f, d in df.loc[:, ['id', 'dir']].values: + search_result = find_parent_ioc(f, d) # catch parent IOCs running out of dev if 'epics-dev' in search_result: output_str = search_result # abbreviate path for standard IOC releases elif 'common' in search_result: output_str = (search_result - .rsplit('common/',maxsplit=1)[-1]) + .rsplit('common/', maxsplit=1)[-1]) # add it to the list output_list.append(output_str) # Then, finally, add the column to the dataframe @@ -438,38 +443,39 @@ def main(): print_frame2term(df) if args.skip_comments is True: - for ioc,d in df.loc[:,['id','dir']].values: - # fixes dirs if grep_ioc truncates the path due to common ioc dir + for ioc, d in df.loc[:, ['id', 'dir']].values: + # fixes dirs if ioc_manager truncates the path due to + # common ioc dir path target_dir = fix_dir(d) print(f'{Fore.LIGHTBLUE_EX}Now in: {target_dir}' - +Style.RESET_ALL) + + Style.RESET_ALL) print(f'{Fore.LIGHTYELLOW_EX}{ioc}:{Style.RESET_ALL}') # prints the contents of the file while ignoring comments print_skip_comments(file=f'{target_dir}{ioc}.cfg') if args.print_dirs is True: print(f'{Fore.LIGHTRED_EX}\nDumping directories:\n' - +Style.RESET_ALL) - for f,d in df.loc[:,['id','dir']].values: - search_result=find_parent_ioc(f,d) + + Style.RESET_ALL) + for f, d in df.loc[:, ['id', 'dir']].values: + search_result = find_parent_ioc(f, d) d = fix_dir(d) # Print this for easier cd / pushd shenanigans print(f'{d}{Fore.LIGHTYELLOW_EX}{f}.cfg{Style.RESET_ALL}' - +'\n\t\t|-->' + + '\n\t\t|-->' + f'{Fore.LIGHTGREEN_EX}RELEASE={Style.RESET_ALL}' + f'{Fore.LIGHTBLUE_EX}{search_result}{Style.RESET_ALL}' ) -#-----------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # # %%% search -#-----------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # # do a local grep on each file in their corresponding directory - if hasattr(args,'search'): + if hasattr(args, 'search'): # optionally print the dataframe if not args.only_search: print_frame2term(df) check_search = [] - for ioc,d in df.loc[:,['id','dir']].values: + for ioc, d in df.loc[:, ['id', 'dir']].values: target_dir = fix_dir(d) # Search for pattern after moving into the directory if args.search is not None: @@ -485,10 +491,14 @@ def main(): check_search.append(len(search_result)) if len(check_search) == 0: print(Fore.RED + 'No search results found' + Style.RESET_ALL) -#-----------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # # %%% Exit -#-----------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # sys.exit() + +# --------------------------------------------------------------------------- # +# %%% Entry point +# --------------------------------------------------------------------------- # if __name__ == '__main__': main() From 7eb1c33971765769b0ba9f64654f4344653cd475 Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Tue, 14 May 2024 11:43:37 -0700 Subject: [PATCH 08/23] MNT: Fixed help doctext for grep_more_ioc argparser --- scripts/grep_more_ioc.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/grep_more_ioc.py b/scripts/grep_more_ioc.py index 3da26b5c..471f024d 100644 --- a/scripts/grep_more_ioc.py +++ b/scripts/grep_more_ioc.py @@ -310,9 +310,9 @@ def build_parser(): # parser obj configuration parser = argparse.ArgumentParser( prog='grep_more_ioc', - description='transforms grep_ioc output to json object' + description='Transforms grep_ioc output to json object' + ' and prints in pandas.DataFrame', - epilog='With extra utilities for daily ECS') + epilog='With extra utilities for daily ECS work.') # main command arguments parser.add_argument('patt', type=str) parser.add_argument('hutch', type=str) @@ -354,10 +354,10 @@ def build_parser(): # search subarguments # --------------------------------------------------------------------------- # search = subparsers.add_parser('search', - help='For searching regex patts in the' - + ' IOC *.cfg captured by grep_ioc.' - + ' Useful for checking the pointers' - + "to the parent's release") + help='For using regex-like searches in the' + + ' child IOC.cfg captured by grep_ioc.' + + ' Useful for quickly gathering instance' + + ' information, IP addr, etc.') search.add_argument('search', type=str, help='PATT to use for regex search in file', From 655c7b98828f8cd92906f5a99b8cbff8294e3cf2 Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Tue, 14 May 2024 11:46:46 -0700 Subject: [PATCH 09/23] MNT: Proper quoting for in grep_more_ioc.sh --- scripts/grep_more_ioc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/grep_more_ioc.sh b/scripts/grep_more_ioc.sh index 14bff9a4..c9451236 100755 --- a/scripts/grep_more_ioc.sh +++ b/scripts/grep_more_ioc.sh @@ -1,4 +1,4 @@ #!/usr/bin/bash # execute python script -/cds/group/pcds/pyps/conda/py39/envs/pcds-5.8.4/bin/python $ENG_TOOLS_SCRIPTS/grep_more_ioc.py "$@" +/cds/group/pcds/pyps/conda/py39/envs/pcds-5.8.4/bin/python "$ENG_TOOLS_SCRIPTS/grep_more_ioc.py" "$@" From 607a9727c2e03a23d3349703676142ac28987aa3 Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Tue, 14 May 2024 11:52:52 -0700 Subject: [PATCH 10/23] MNT: sort imports for style enforcement --- scripts/getPVAliases.py | 7 ++++--- scripts/grep_more_ioc.py | 11 ++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/getPVAliases.py b/scripts/getPVAliases.py index b40599c0..7d983523 100644 --- a/scripts/getPVAliases.py +++ b/scripts/getPVAliases.py @@ -7,15 +7,16 @@ # %% Imports ############################################################################### +import argparse +import copy import sys import re import os.path -import argparse -import copy + from colorama import Fore, Style -from prettytable import PrettyTable from grep_more_ioc import (find_ioc, simple_prompt, search_file, find_parent_ioc, fix_dir, clean_ansi) +from prettytable import PrettyTable ############################################################################### diff --git a/scripts/grep_more_ioc.py b/scripts/grep_more_ioc.py index 471f024d..620f54dc 100644 --- a/scripts/grep_more_ioc.py +++ b/scripts/grep_more_ioc.py @@ -6,15 +6,16 @@ # %% Imports ############################################################################### -import sys -import re -import os.path import argparse -import json import glob as gb +import json +import os.path +import re +import sys from shutil import get_terminal_size -from colorama import Fore, Style + import pandas as pd +from colorama import Fore, Style ############################################################################### # %% Global settings From f31039fbdca076e961c27694b48815bf3d5aad57 Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Tue, 14 May 2024 11:56:29 -0700 Subject: [PATCH 11/23] MNT: isort, once more with feeling --- scripts/getPVAliases.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/getPVAliases.py b/scripts/getPVAliases.py index 7d983523..d33585ef 100644 --- a/scripts/getPVAliases.py +++ b/scripts/getPVAliases.py @@ -9,16 +9,15 @@ import argparse import copy -import sys -import re import os.path +import re +import sys from colorama import Fore, Style from grep_more_ioc import (find_ioc, simple_prompt, search_file, find_parent_ioc, fix_dir, clean_ansi) from prettytable import PrettyTable - ############################################################################### # %% Constants ############################################################################### From 04d62b31a7076c5080ab1e2b78cba93a96914b21 Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Tue, 14 May 2024 11:59:16 -0700 Subject: [PATCH 12/23] MNT: isort is not my friend --- scripts/getPVAliases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/getPVAliases.py b/scripts/getPVAliases.py index d33585ef..dbf32112 100644 --- a/scripts/getPVAliases.py +++ b/scripts/getPVAliases.py @@ -14,8 +14,8 @@ import sys from colorama import Fore, Style -from grep_more_ioc import (find_ioc, simple_prompt, search_file, - find_parent_ioc, fix_dir, clean_ansi) +from grep_more_ioc import (clean_ansi, find_ioc, find_parent_ioc, fix_dir, + search_file, simple_prompt) from prettytable import PrettyTable ############################################################################### From 07eb6f7c5754672f8651a8aa8cd42e6f42127304 Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Tue, 14 May 2024 12:50:44 -0700 Subject: [PATCH 13/23] BUG: Fixed bad catches on find_parent_ioc in grep_more_ioc.py --- scripts/grep_more_ioc.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/grep_more_ioc.py b/scripts/grep_more_ioc.py index 620f54dc..a86432fc 100644 --- a/scripts/grep_more_ioc.py +++ b/scripts/grep_more_ioc.py @@ -431,7 +431,13 @@ def main(): # abbreviate path for standard IOC releases elif 'common' in search_result: output_str = (search_result - .rsplit('common/', maxsplit=1)[-1]) + .rsplit(r'common/', maxsplit=1)[-1]) + # check for children living in parent's dir + elif '$$UP(PATH)' in search_result: + output_str = d.rsplit(r'/children', maxsplit=1)[0] + # else use the full path that's found + else: + output_str = search_result # add it to the list output_list.append(output_str) # Then, finally, add the column to the dataframe From fa196003dffdf624238072e855681fdbb3af5005 Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Thu, 16 May 2024 19:41:38 -0700 Subject: [PATCH 14/23] MAINT: Various bugfixing, style fixing, incorporating review changes --- README.md | 22 +++++++++++++++++ scripts/constants.py | 27 +++++++++++++++++++++ scripts/getPVAliases.py | 52 +++++++--------------------------------- scripts/getPVAliases.sh | 4 +++- scripts/grep_more_ioc.py | 42 +++++++++++++------------------- scripts/grep_more_ioc.sh | 4 +++- 6 files changed, 81 insertions(+), 70 deletions(-) create mode 100644 scripts/constants.py diff --git a/README.md b/README.md index a6ae291f..362a514f 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,28 @@ usage: grep_ioc KEYWORD [hutch]
+ + grep_more_ioc + +usage: grep_more_ioc [-h] [-d] patt hutch {print,search}
+ positional arguments:
+ patt Regex str to search through iocmanager.cfg
+ hutch hutch to search
+ -h, --help Show help message and exit
+ -d, --ignore_disabled Exclude IOCs based on disabled state
+ {print, search}
+ print Prints all the matching IOCs in a dataframe
+ -h, --help Show help message and exit
+ -c, --skip_comments Prints IOC.cfg file with comments skipped
+ -r, --release Includes the parent IOC release in the dataframe
+ -s, --print_dirs Dump child & parent directors to the terminal + search Regex-like search of child IOCs
+ PATT The regex str to use in the search
+ -h, --help Show help message and exit
+ -q, --quiet Surpresses file warning for paths that do not exist
+ -o, --only_search Skip printing dataframe, only print search results
+ + grep_pv diff --git a/scripts/constants.py b/scripts/constants.py new file mode 100644 index 00000000..d2ca8e11 --- /dev/null +++ b/scripts/constants.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +""" +Constants used in grep_more_ioc and getPVAliases +""" +############################################################################### +# %% Imports +############################################################################### + +import glob as gb + +############################################################################### +# %% Constants +############################################################################### + +# Check the directories for the iocmanager config file +VALID_HUTCH = sorted([d for d in gb.glob('/cds/group/pcds/pyps/config' + + '/*/') + if gb.glob(d + 'iocmanager.cfg')]) +# Trim to 3 letter hutch code, include 'all' = '*' +VALID_HUTCH = ['all'] + [s.rsplit(r'/', maxsplit=2)[-2] for s in VALID_HUTCH] + +# Keys from iocmanager. Found in /cds/group/pcds/config/*/iocmanager/utils.py +# Update this as needed +DEF_IMGR_KEYS = ['procmgr_config', 'hosts', 'dir', 'id', 'cmd', + 'flags', 'port', 'host', 'disable', 'history', + 'delay', 'alias', 'hard'] + \ No newline at end of file diff --git a/scripts/getPVAliases.py b/scripts/getPVAliases.py index dbf32112..7474238f 100644 --- a/scripts/getPVAliases.py +++ b/scripts/getPVAliases.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """ A script for gathering alias associations from an IOC -@author: aberges """ ############################################################################### # %% Imports @@ -18,29 +17,15 @@ search_file, simple_prompt) from prettytable import PrettyTable -############################################################################### -# %% Constants -############################################################################### - -VALID_HUTCH = sorted([s for s - in next(os.walk('/cds/group/pcds/pyps/config'))[1] - if '.' not in s]+['all']) -# Keys from iocmanager. Found in /cds/group/pcds/config/*/iocmanager/utils.py -# Update this as needed -DEF_IMGR_KEYS = ['procmgr_config', 'hosts', 'dir', 'id', 'cmd', - 'flags', 'port', 'host', 'disable', 'history', - 'delay', 'alias', 'hard'] - ############################################################################### # %% Functions ############################################################################### -def request_file_dest(prompt: str, - default: str = os.getcwd(),) -> str: +def request_dir(prompt: str, + default: str = os.getcwd(),) -> str: """ Requests the user for a destination to save a file. - Tests if the resultant path exists and makes the directory if necessary. Parameters ---------- @@ -70,25 +55,6 @@ def request_file_dest(prompt: str, return result -def flatten_list(input_list: list[list]) -> list[str]: - """ - Flatten a 2D lists and find its unique values. - Note: loses order due to set() call. sear - - Parameters - ---------- - input_list: list[list] - The 2D list to flatten. - - Returns - ------- - list - The list of unique elements from the 2D input_list - """ - _result = [e for lst in input_list for e in lst] - return list(set(_result)) - - def build_table(input_data: list[dict], columns: list[str] = None, **kwargs) -> PrettyTable: """ @@ -226,16 +192,16 @@ def show_temp_table(input_data: list, col_list: list): # parser obj configuration parser = argparse.ArgumentParser( prog='gatherPVAliases', - description="""gathers all record <-> alias associations from a child's - ioc.cfg, st.cmd, and parent ioc.cfg.""", - epilog='') + description="gathers all record <-> alias associations from a child's " + "ioc.cfg, st.cmd, and parent ioc.cfg.", + epilog='') # main command arguments parser.add_argument('patt', type=str) parser.add_argument('hutch', type=str) parser.add_argument('-d', '--dry_run', action='store_true', default=False, - help='''Forces a dry run for the script. - No files are saved.''') + help="Forces a dry run for the script. " + "No files are saved.") ############################################################################### # %% Main @@ -356,8 +322,8 @@ def main(): # write to file, else do nothing if (len(final_output) > 0) & (args.dry_run is False): - dest = request_file_dest('Choose base file destination', - default_dest) + dest = request_dir('Choose base file destination', + default_dest) # make sure the destination exists and mkdir if it doesn't if os.path.exists(dest) is False: print(Fore.LIGHTBLUE_EX diff --git a/scripts/getPVAliases.sh b/scripts/getPVAliases.sh index d62b6b94..1f5bb1f4 100755 --- a/scripts/getPVAliases.sh +++ b/scripts/getPVAliases.sh @@ -1,4 +1,6 @@ #!/usr/bin/bash # execute python script -/cds/group/pcds/pyps/conda/py39/envs/pcds-5.8.2/bin/python "$ENG_TOOLS_SCRIPTS/getPVAliases.py" "$@" +THIS_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" + +/cds/group/pcds/pyps/conda/py39/envs/pcds-5.8.2/bin/python "${THIS_DIR}/getPVAliases.py" "$@" diff --git a/scripts/grep_more_ioc.py b/scripts/grep_more_ioc.py index a86432fc..126ab42d 100644 --- a/scripts/grep_more_ioc.py +++ b/scripts/grep_more_ioc.py @@ -1,6 +1,6 @@ -"""A tool for enhancing the grep_ioc CLI tool for ECS-Delivery team -enginer = aberges -version = 1.0.6 +# -*- coding: utf-8 -*- +""" +A tool for enhancing the grep_ioc CLI tool for ECS-Delivery team """ ############################################################################### # %% Imports @@ -15,6 +15,7 @@ from shutil import get_terminal_size import pandas as pd +from constants import (DEF_IMGR_KEYS, VALID_HUTCH) from colorama import Fore, Style ############################################################################### @@ -25,19 +26,6 @@ pd.set_option("display.max_rows", 1000) -############################################################################### -# %% Constants -############################################################################### - -VALID_HUTCH = sorted([s for s - in next(os.walk('/cds/group/pcds/pyps/config'))[1] - if '.' not in s]+['all']) -# Keys from iocmanager. Found in /cds/group/pcds/config/*/iocmanager/utils.py -# Update this as needed -DEF_IMGR_KEYS = ['procmgr_config', 'hosts', 'dir', 'id', 'cmd', - 'flags', 'port', 'host', 'disable', 'history', - 'delay', 'alias', 'hard'] - ############################################################################### # %% Functions ############################################################################### @@ -90,10 +78,14 @@ def search_file(*, file: str, output: list = None, def print_skip_comments(file: str): """Prints contents of a file while ignoring comments""" - with open(file, 'r', encoding='utf_8') as _f: - for line in _f.readlines(): - if not line.strip().startswith('#'): - print(line.strip()) + try: + with open(file, 'r', encoding='utf_8') as _f: + for line in _f.readlines(): + if not line.strip().startswith('#'): + print(line.strip()) + except FileNotFoundError: + print(f'{Fore.RED}Could not open {Style.RESET_ALL}{file}' + + f' {Fore.RED}does not exist{Style.RESET_ALL}') def simple_prompt(prompt: str, default: str = 'N'): @@ -323,9 +315,9 @@ def build_parser(): help='Flag for excluding based' + ' on the "disabled" state.') # subparsers - subparsers = (parser - .add_subparsers(help='Follow-up commands after grep_ioc ' - + 'executes')) + subparsers = parser.add_subparsers( + help='Follow-up commands after grep_ioc executes' + ) # --------------------------------------------------------------------------- # # print subarguments # --------------------------------------------------------------------------- # @@ -350,7 +342,7 @@ def build_parser(): print_frame.add_argument('-s', '--print_dirs', action='store_true', default=False, help='Prints the child & parent IOC' - + 'directories as the final output') + + ' directories as the final output') # --------------------------------------------------------------------------- # # search subarguments # --------------------------------------------------------------------------- # @@ -365,7 +357,7 @@ def build_parser(): metavar='PATT') search.add_argument('-q', '--quiet', action='store_true', default=False, help='Surpresses file warning for paths that do not' - + 'exist.') + + ' exist.') search.add_argument('-o', '--only_search', action='store_true', default=False, help="Don't print the dataframe, just search results.") diff --git a/scripts/grep_more_ioc.sh b/scripts/grep_more_ioc.sh index c9451236..c20c434e 100755 --- a/scripts/grep_more_ioc.sh +++ b/scripts/grep_more_ioc.sh @@ -1,4 +1,6 @@ #!/usr/bin/bash # execute python script -/cds/group/pcds/pyps/conda/py39/envs/pcds-5.8.4/bin/python "$ENG_TOOLS_SCRIPTS/grep_more_ioc.py" "$@" +THIS_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" + +/cds/group/pcds/pyps/conda/py39/envs/pcds-5.8.4/bin/python "${THIS_DIR}/grep_more_ioc.py" "$@" From d1832942f4b2ab2ed253f1c56aef902c431c6ce5 Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Thu, 16 May 2024 19:49:19 -0700 Subject: [PATCH 15/23] STY: flake8 and isort compliance --- scripts/constants.py | 3 +-- scripts/getPVAliases.py | 7 ++++--- scripts/grep_more_ioc.py | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/constants.py b/scripts/constants.py index d2ca8e11..ee0b34f7 100644 --- a/scripts/constants.py +++ b/scripts/constants.py @@ -14,7 +14,7 @@ # Check the directories for the iocmanager config file VALID_HUTCH = sorted([d for d in gb.glob('/cds/group/pcds/pyps/config' - + '/*/') + + '/*/') if gb.glob(d + 'iocmanager.cfg')]) # Trim to 3 letter hutch code, include 'all' = '*' VALID_HUTCH = ['all'] + [s.rsplit(r'/', maxsplit=2)[-2] for s in VALID_HUTCH] @@ -24,4 +24,3 @@ DEF_IMGR_KEYS = ['procmgr_config', 'hosts', 'dir', 'id', 'cmd', 'flags', 'port', 'host', 'disable', 'history', 'delay', 'alias', 'hard'] - \ No newline at end of file diff --git a/scripts/getPVAliases.py b/scripts/getPVAliases.py index 7474238f..1ea5d8a6 100644 --- a/scripts/getPVAliases.py +++ b/scripts/getPVAliases.py @@ -13,9 +13,10 @@ import sys from colorama import Fore, Style +from prettytable import PrettyTable + from grep_more_ioc import (clean_ansi, find_ioc, find_parent_ioc, fix_dir, search_file, simple_prompt) -from prettytable import PrettyTable ############################################################################### # %% Functions @@ -193,8 +194,8 @@ def show_temp_table(input_data: list, col_list: list): parser = argparse.ArgumentParser( prog='gatherPVAliases', description="gathers all record <-> alias associations from a child's " - "ioc.cfg, st.cmd, and parent ioc.cfg.", - epilog='') + "ioc.cfg, st.cmd, and parent ioc.cfg.", + epilog='') # main command arguments parser.add_argument('patt', type=str) parser.add_argument('hutch', type=str) diff --git a/scripts/grep_more_ioc.py b/scripts/grep_more_ioc.py index 126ab42d..5733714d 100644 --- a/scripts/grep_more_ioc.py +++ b/scripts/grep_more_ioc.py @@ -15,9 +15,10 @@ from shutil import get_terminal_size import pandas as pd -from constants import (DEF_IMGR_KEYS, VALID_HUTCH) from colorama import Fore, Style +from constants import DEF_IMGR_KEYS, VALID_HUTCH + ############################################################################### # %% Global settings ############################################################################### From 43d391f05b3f8cfcd8fcbef0aeafabd86c68df4a Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Fri, 17 May 2024 11:36:47 -0700 Subject: [PATCH 16/23] MAINT: One last bug fix and style fix --- scripts/grep_more_ioc.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/scripts/grep_more_ioc.py b/scripts/grep_more_ioc.py index 5733714d..610f4bbf 100644 --- a/scripts/grep_more_ioc.py +++ b/scripts/grep_more_ioc.py @@ -85,8 +85,9 @@ def print_skip_comments(file: str): if not line.strip().startswith('#'): print(line.strip()) except FileNotFoundError: - print(f'{Fore.RED}Could not open {Style.RESET_ALL}{file}' - + f' {Fore.RED}does not exist{Style.RESET_ALL}') + print(f'{Fore.RED}Could not open {Style.RESET_ALL}' + + f'{file}.' + + f'{Fore.RED} Does not exist{Style.RESET_ALL}') def simple_prompt(prompt: str, default: str = 'N'): @@ -454,16 +455,23 @@ def main(): print_skip_comments(file=f'{target_dir}{ioc}.cfg') if args.print_dirs is True: - print(f'{Fore.LIGHTRED_EX}\nDumping directories:\n' + print(f'{Fore.LIGHTBLUE_EX}\nDumping directories:\n' + Style.RESET_ALL) for f, d in df.loc[:, ['id', 'dir']].values: search_result = find_parent_ioc(f, d) d = fix_dir(d) + # check for cases where child IOC.cfg DNE + if not os.path.exists(f'{d}{f}.cfg'): + child_ioc = '' + color_prefix = Fore.LIGHTRED_EX + else: + child_ioc = f'{f}.cfg' + color_prefix = Fore.LIGHTBLUE_EX # Print this for easier cd / pushd shenanigans - print(f'{d}{Fore.LIGHTYELLOW_EX}{f}.cfg{Style.RESET_ALL}' + print(f'{d}{Fore.LIGHTYELLOW_EX}{child_ioc}{Style.RESET_ALL}' + '\n\t\t|-->' + f'{Fore.LIGHTGREEN_EX}RELEASE={Style.RESET_ALL}' - + f'{Fore.LIGHTBLUE_EX}{search_result}{Style.RESET_ALL}' + + f'{color_prefix}{search_result}{Style.RESET_ALL}' ) # --------------------------------------------------------------------------- # From ad07b6f13772248bf83817f344f0ec6701588b49 Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Fri, 17 May 2024 11:54:16 -0700 Subject: [PATCH 17/23] MAINT: more style fixing --- README.md | 7 ++++--- scripts/getPVAliases.py | 3 +-- scripts/grep_more_ioc.py | 4 +--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 362a514f..8a721e69 100644 --- a/README.md +++ b/README.md @@ -330,7 +330,8 @@ usage: ioctool <ioc>|<pv> [option]
telnet : starts a telnet session with the ioc
pvs : opens the IOC.pvlist file in less
- + + ipmConfigEpics @@ -662,9 +663,9 @@ Optional arguments:
set_gem_timing - + Usage: set_gem_timing [SC or NC] - \ No newline at end of file + diff --git a/scripts/getPVAliases.py b/scripts/getPVAliases.py index 1ea5d8a6..9dddc1c0 100644 --- a/scripts/getPVAliases.py +++ b/scripts/getPVAliases.py @@ -13,10 +13,9 @@ import sys from colorama import Fore, Style -from prettytable import PrettyTable - from grep_more_ioc import (clean_ansi, find_ioc, find_parent_ioc, fix_dir, search_file, simple_prompt) +from prettytable import PrettyTable ############################################################################### # %% Functions diff --git a/scripts/grep_more_ioc.py b/scripts/grep_more_ioc.py index 610f4bbf..c46e4fc3 100644 --- a/scripts/grep_more_ioc.py +++ b/scripts/grep_more_ioc.py @@ -16,7 +16,6 @@ import pandas as pd from colorama import Fore, Style - from constants import DEF_IMGR_KEYS, VALID_HUTCH ############################################################################### @@ -318,8 +317,7 @@ def build_parser(): + ' on the "disabled" state.') # subparsers subparsers = parser.add_subparsers( - help='Follow-up commands after grep_ioc executes' - ) + help='Follow-up commands after grep_ioc executes') # --------------------------------------------------------------------------- # # print subarguments # --------------------------------------------------------------------------- # From 122d4ff0b263676d52066f6bf5c3bc25e7f6d3e9 Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Mon, 20 May 2024 15:37:35 -0700 Subject: [PATCH 18/23] BUG/ENH: Fixed missing IOCs due to inline breaks. Added search_procmgr function to grep_more_ioc to circumvent this --- scripts/grep_more_ioc.py | 71 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/scripts/grep_more_ioc.py b/scripts/grep_more_ioc.py index c46e4fc3..8551ef1d 100644 --- a/scripts/grep_more_ioc.py +++ b/scripts/grep_more_ioc.py @@ -33,9 +33,10 @@ def search_file(*, file: str, output: list = None, patt: str = None, prefix: str = '', - quiet: bool = False, color_wrap: Fore = None) -> list[str]: + quiet: bool = False, color_wrap: Fore = None) -> str: """ - Searches file for regex match and appends result to list + Searches file for regex match and appends the result to a list, + then formats back into a str with the prefix prepended. Parameters ---------- @@ -76,6 +77,54 @@ def search_file(*, file: str, output: list = None, return prefix + prefix.join(output) +def search_procmgr(*, file: str, patt: str = None, output: list = None, + prefix: str = '') -> str: + """ + Very similar to search_file, except it is to be used exclusively with + iocmanager.cfg files. Grabs the procmgr_cfg lists and searches the + regex 'patt' there. Can prepend text with 'prefix' and/or append + the results in 'output'. + + Parameters + ---------- + file : str + The iocmanager.cfg file to search. + patt : str, optional + Regex pattern to search with. The default is None. + output : list, optional + A list to append the results to. The default is None. + prefix : str, optional + A prefix to add to the start of each result. The default is ''. + + Returns + ------- + str + A list[str] that is flattened back into a single body with the prefix + prepended. Each result is separated by 'prefix' and '\n'. + + """ + # Some initialization + if output is None: + output = [] + _patt = r'{.*' + patt + r'.*}' + # First open the iocmanager.cfg, if it exists + if not (os.path.exists(file) and ('iocmanager.cfg' in file)): + print(f'{file} does not exist or is otherwise invalid.') + return '' + with open(file, 'r', encoding='utf-8') as _f: + raw_text = _f.read() + # then only grab the procmgr_cfg for the search + pmgr_key = r'procmgr_config = [\n ' + pmgr = raw_text[(raw_text.find(pmgr_key)+len(pmgr_key)):-3] + # get rid of those pesty inline breaks within the JSOB obj + pmgr = pmgr.replace(',\n ', ',').replace('},{', '},\n{') + # now let we'll finally search through the IOCs and insert into output + output.extend(re.findall(_patt, pmgr)) + # now return the searches with the prefix prepended and the necessary + # line break for later JSONification + return prefix + prefix.join([s + '\n' for s in output]) + + def print_skip_comments(file: str): """Prints contents of a file while ignoring comments""" try: @@ -191,7 +240,6 @@ def find_ioc(hutch: str = None, patt: str = None, if patt is None: print('No regex pattern supplied') raise ValueError - _patt = r'{.*' + patt + r'.*}' # initialize output list result = [] # iterate and capture results. @@ -199,7 +247,7 @@ def find_ioc(hutch: str = None, patt: str = None, prefix = '' if len(path) != 1: prefix = _file+':' - output = search_file(file=_file, patt=_patt, prefix=prefix) + output = search_procmgr(file=_file, patt=patt, prefix=prefix) if output != prefix: result.append(output) # reconstruct the list of str @@ -343,6 +391,9 @@ def build_parser(): default=False, help='Prints the child & parent IOC' + ' directories as the final output') + print_frame.add_argument('-y', '--print_history', action='store_true', + default=False, + help="Prints the child IOC's history to terminal") # --------------------------------------------------------------------------- # # search subarguments # --------------------------------------------------------------------------- # @@ -472,6 +523,18 @@ def main(): + f'{color_prefix}{search_result}{Style.RESET_ALL}' ) + if args.print_history is True: + print(f'{Fore.LIGHTMAGENTA_EX}\nDumping histories:\n' + + Style.RESET_ALL) + if 'history' in df.columns: + for f, h in df.loc[:, ['id', 'history']].values: + print(f'{Fore.LIGHTYELLOW_EX}{f}{Style.RESET_ALL}' + + '\nhistory:\n\t' + + '\n\t'.join(h)) + else: + print(f'{Fore.LIGHTRED_EX}No histories found in captured IOCs.' + + Style.RESET_ALL) + # --------------------------------------------------------------------------- # # %%% search # --------------------------------------------------------------------------- # From 06a21bbb179aa17aef14b3bf8c93c0cf1820a0d6 Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Tue, 21 May 2024 14:14:36 -0700 Subject: [PATCH 19/23] DOC: Improved help documentation for grep_more_ioc and getPVAliases --- scripts/getPVAliases.py | 20 +++++++++++++++----- scripts/grep_more_ioc.py | 18 +++++++++++++----- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/scripts/getPVAliases.py b/scripts/getPVAliases.py index 9dddc1c0..1e27c1ea 100644 --- a/scripts/getPVAliases.py +++ b/scripts/getPVAliases.py @@ -13,6 +13,7 @@ import sys from colorama import Fore, Style +from constants import VALID_HUTCH from grep_more_ioc import (clean_ansi, find_ioc, find_parent_ioc, fix_dir, search_file, simple_prompt) from prettytable import PrettyTable @@ -192,12 +193,20 @@ def show_temp_table(input_data: list, col_list: list): # parser obj configuration parser = argparse.ArgumentParser( prog='gatherPVAliases', + formatter_class=argparse.RawTextHelpFormatter, description="gathers all record <-> alias associations from a child's " - "ioc.cfg, st.cmd, and parent ioc.cfg.", + "ioc.cfg, st.cmd, and parent ioc.cfg and then optionally " + "saves it to a record_alias_dump.txt file.", epilog='') # main command arguments -parser.add_argument('patt', type=str) -parser.add_argument('hutch', type=str) +parser.add_argument('patt', type=str, + help='Regex pattern to match IOCs with. ' + '\nCan match anything in the IOC procmanager object. ' + 'e.g. "lm2k2" or "mcs2" or "ek9000"') +parser.add_argument('hutch', type=str, + help='3 letter hutch code. Use "all" to search through ' + 'all hutches.\n' + f'Valid arguments: {", ".join(VALID_HUTCH)}') parser.add_argument('-d', '--dry_run', action='store_true', default=False, help="Forces a dry run for the script. " @@ -257,12 +266,13 @@ def main(): print(build_table(alias_dicts, ['record', 'alias'], align='l')) # optional skip for all resulting PV aliases save_all = (simple_prompt( - 'Do you want to save all resulting PV aliases? ' + 'Do you want to save all resulting PV <--> alias ' + + 'associations found in this st.cmd?\n' + 'This will append ' + Fore.LIGHTYELLOW_EX + f'{len(alias_dicts)}' + Style.RESET_ALL - + ' record sets (y/N): ')) + + ' record <--> alias sets to your final output (y/N): ')) # initialize flags skip_all = None diff --git a/scripts/grep_more_ioc.py b/scripts/grep_more_ioc.py index 8551ef1d..a683d01b 100644 --- a/scripts/grep_more_ioc.py +++ b/scripts/grep_more_ioc.py @@ -352,12 +352,20 @@ def build_parser(): # parser obj configuration parser = argparse.ArgumentParser( prog='grep_more_ioc', + formatter_class=argparse.RawTextHelpFormatter, description='Transforms grep_ioc output to json object' + ' and prints in pandas.DataFrame', - epilog='With extra utilities for daily ECS work.') + epilog='For more information on subcommands, use: ' + 'grep_more_ioc . all [subcommand] --help') # main command arguments - parser.add_argument('patt', type=str) - parser.add_argument('hutch', type=str) + parser.add_argument('patt', type=str, + help='Regex pattern to match IOCs with. ' + '\nCan match anything in the IOC procmanager object. ' + 'e.g. "lm2k2" or "mcs2" or "gige"') + parser.add_argument('hutch', type=str, + help='3 letter hutch code. Use "all" to search through' + ' all hutches.\n' + f'Valid arguments: {", ".join(VALID_HUTCH)}') parser.add_argument('-d', '--ignore_disabled', action='store_true', default=False, @@ -365,7 +373,7 @@ def build_parser(): + ' on the "disabled" state.') # subparsers subparsers = parser.add_subparsers( - help='Follow-up commands after grep_ioc executes') + help='Required subcommands after capturing IOC information:') # --------------------------------------------------------------------------- # # print subarguments # --------------------------------------------------------------------------- # @@ -400,7 +408,7 @@ def build_parser(): search = subparsers.add_parser('search', help='For using regex-like searches in the' + ' child IOC.cfg captured by grep_ioc.' - + ' Useful for quickly gathering instance' + + '\nUseful for quickly gathering instance' + ' information, IP addr, etc.') search.add_argument('search', From b2037ed13f26f11da032fdacac0ff3c5df959a0c Mon Sep 17 00:00:00 2001 From: aberges-SLAC Date: Tue, 21 May 2024 15:06:13 -0700 Subject: [PATCH 20/23] DOC: Updated README.md to include new scripts --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8a721e69..c1e736c5 100644 --- a/README.md +++ b/README.md @@ -251,15 +251,20 @@ usage: grep_ioc KEYWORD [hutch]
usage: grep_more_ioc [-h] [-d] patt hutch {print,search}
positional arguments:
patt Regex str to search through iocmanager.cfg
- hutch hutch to search
+ e.g. 'mcs2', 'lm2k2-atm.*', 'ek9000', 'gige.*'
+ hutch 3 letter hutch code to search through.
+ Use 'all' to search through all hutches. -h, --help Show help message and exit
-d, --ignore_disabled Exclude IOCs based on disabled state
+ Necessary subcommands.
+ Use: grep_more_ioc . all [subcommand] --help for more information {print, search}
print Prints all the matching IOCs in a dataframe
-h, --help Show help message and exit
-c, --skip_comments Prints IOC.cfg file with comments skipped
-r, --release Includes the parent IOC release in the dataframe
- -s, --print_dirs Dump child & parent directors to the terminal + -s, --print_dirs Dump child & parent directors to the terminal
+ -y, --print_history Dump child IOC's history to terminal, if it exists
search Regex-like search of child IOCs
PATT The regex str to use in the search
-h, --help Show help message and exit
From e3bfae40210d2bf5b2a44209633b788d1a4ab876 Mon Sep 17 00:00:00 2001 From: Adam Berges <149725219+aberges-SLAC@users.noreply.github.com> Date: Wed, 22 May 2024 12:18:00 -0700 Subject: [PATCH 21/23] DOC: Update README.md Added getPVAliases to README and fixed formatting --- README.md | 47 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index c1e736c5..d20a100d 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,23 @@ usage: get_lastRun options
+ + getPVAliases + +usage: gatherPVAliases [-h] [-d] patt hutch
+positional arguments:
+ patt | Regex pattern to match IOCs with.
+ -->Can match anything in the IOC procmanager object. e.g. "lm2k2" or "mcs2" or "ek9000"
+ hutch | 3 letter hutch code. Use "all" to search through all hutches.
+ -->Valid arguments: all, aux, cxi, det, hpl, icl, kfe, las, lfe, mec,
+ mfx, rix, rrl, thz, tmo, tst, txi, ued, xcs, xpp, xrt
+ +optional arguments:
+ -h, --help | show this help message and exit
+ -d, --dry_run | Forces a dry run for the script. No files are saved.
+ + + grep_ioc @@ -253,23 +270,29 @@ usage: grep_more_ioc [-h] [-d] patt hutch {print,search}
patt Regex str to search through iocmanager.cfg
e.g. 'mcs2', 'lm2k2-atm.*', 'ek9000', 'gige.*'
hutch 3 letter hutch code to search through.
- Use 'all' to search through all hutches. + Use 'all' to search through all hutches.
+ Valid arguments: all, aux, cxi, det, hpl, icl, kfe,
+ las, lfe, mec, mfx, rix, rrl, thz, tmo, tst, txi, ued,
+ xcs, xpp, xrt
-h, --help Show help message and exit
-d, --ignore_disabled Exclude IOCs based on disabled state
Necessary subcommands.
Use: grep_more_ioc . all [subcommand] --help for more information {print, search}
- print Prints all the matching IOCs in a dataframe
- -h, --help Show help message and exit
- -c, --skip_comments Prints IOC.cfg file with comments skipped
- -r, --release Includes the parent IOC release in the dataframe
- -s, --print_dirs Dump child & parent directors to the terminal
- -y, --print_history Dump child IOC's history to terminal, if it exists
- search Regex-like search of child IOCs
- PATT The regex str to use in the search
- -h, --help Show help message and exit
- -q, --quiet Surpresses file warning for paths that do not exist
- -o, --only_search Skip printing dataframe, only print search results
+ print | Prints all the matching IOCs in a dataframe
+ usage: grep_more_ioc patt hutch print [-h] [-c] [-r] [-s] [-y]
+ -h, --help | Show help message and exit
+ -c, --skip_comments | Prints IOC.cfg file with comments skipped
+ -r, --release | Includes the parent IOC release in the dataframe
+ -s, --print_dirs | Dump child & parent directors to the terminal
+ -y, --print_history | Dump child IOC's history to terminal, if it exists
+ search | Regex-like search of child IOCs
+ usage: grep_more_ioc patt hutch search [-h] [-q] [-o] PATT
+ PATT | The regex str to use in the search
+ -h, --help | Show help message and exit
+ -q, --quiet | Surpresses file warning for paths that do not exist
+ -o, --only_search | Skip printing dataframe, only print search results
+ From 09fd47e6749db99dfa7a9391f7079c3cdbc4375a Mon Sep 17 00:00:00 2001 From: Adam Berges <149725219+aberges-SLAC@users.noreply.github.com> Date: Wed, 22 May 2024 12:18:30 -0700 Subject: [PATCH 22/23] MAINT: Rename getPVAliases.sh to getPVAliases --- scripts/{getPVAliases.sh => getPVAliases} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{getPVAliases.sh => getPVAliases} (100%) diff --git a/scripts/getPVAliases.sh b/scripts/getPVAliases similarity index 100% rename from scripts/getPVAliases.sh rename to scripts/getPVAliases From be4e07e5bbd48bcab8063b79c094b9f123e2a448 Mon Sep 17 00:00:00 2001 From: Adam Berges <149725219+aberges-SLAC@users.noreply.github.com> Date: Wed, 22 May 2024 12:18:50 -0700 Subject: [PATCH 23/23] MAINT: Rename grep_more_ioc.sh to grep_more_ioc --- scripts/{grep_more_ioc.sh => grep_more_ioc} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{grep_more_ioc.sh => grep_more_ioc} (100%) diff --git a/scripts/grep_more_ioc.sh b/scripts/grep_more_ioc similarity index 100% rename from scripts/grep_more_ioc.sh rename to scripts/grep_more_ioc