From 14afa027b713ce6ae373160b8a80b7005b4325aa Mon Sep 17 00:00:00 2001 From: Thomas Lavocat Date: Tue, 30 Aug 2022 14:54:21 +0200 Subject: [PATCH] tool/update_tool: new tool to explore db updates Db updates are tricky to review. This tools brings some utility functions to try to simplify the process. It gives you access to 3 commands: Ls, Diff and Report List the changed DB entry and know if the manifest or the image info was updated easily. The tool also ignores by default the volatiles UUID so you are not polluted by that during the review process. Diff the changed files, either all of them one after the other or a specific set given by the user. It will open a difftool (vimdiff by default) to diff either the manifest or the image-info (or both if they both changed). Report the change as an ascii table with color codes (by default) or as a github TODO list to be integrated within the github PR. --- tools/update_tool | 424 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 424 insertions(+) create mode 100755 tools/update_tool diff --git a/tools/update_tool b/tools/update_tool new file mode 100755 index 00000000..12813bbd --- /dev/null +++ b/tools/update_tool @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import sys +import tempfile +import subprocess +from abc import ABC, abstractmethod +import columnify + +RESET = "\033[0m" +GREEN = "\033[32m" +BOLD = "\033[1m" +RED = "\033[31m" +YELLOW = "\033[33m" +STRIKE = "\033[9m" + + +class Report(ABC): + + def __init__(self): + self.header() + + @abstractmethod + def header(self): + pass + + @abstractmethod + def row(self, name, mstatus, istatus): + pass + + +class CliReport(Report): + + status_map = {"added": f"{BOLD}{GREEN}", + "updated": f"{GREEN}", + "removed": f"{STRIKE}{RED}", + "unchanged": f"{YELLOW}"} + + def __init__(self): + self.terminal_width = os.get_terminal_size().columns + section1 = 0.7 + section2 = 0.15 + section3 = 0.15 + self.nwidth = int(self.terminal_width*section1)-5 + self.mwidth = int(self.terminal_width*section2)-2 + self.iwidth = int(self.terminal_width*section3)-1 + super().__init__() + + @classmethod + def truncate(cls, s, l): + if len(s) > l: + s = f"{s[0:l-3]}..." + return s + + @classmethod + def adjust(cls, s, l): + return cls.truncate(s, l).ljust(l) + + def header(self): + print("| ", "DB entry".ljust(self.nwidth), end="| ") + print(self.adjust("manifest", self.mwidth), end="| ") + print(self.adjust("image-info", self.iwidth), end="|\n") + + print(f"|-{'-'*(self.nwidth+1)}", end="|-") + print(f"{'-'*self.mwidth}", end="|-") + print(f"{'-'*self.iwidth}", end="|\n") + + def row(self, name, mstatus, istatus): + name = self.truncate(name, self.nwidth) + ms = CliReport.status_map[mstatus] + ims = CliReport.status_map[istatus] + print("| ", f"{name}".ljust(self.nwidth), end="| ") + print(f"{ms}{self.adjust(mstatus,self.mwidth)}{RESET}", end="| ") + print(f"{ims}{self.adjust(istatus,self.iwidth)}{RESET}", end="|\n") + + print(f"|-{'-'*(self.nwidth+1)}", end="|-") + print(f"{'-'*self.mwidth}", end="|-") + print(f"{'-'*self.iwidth}", end="|\n") + + +class GHReport(Report): + + status_map = {"added": "🆕", + "updated": "🔁", + "removed": "🗑️", + "unchanged": "💾"} + + def header(self): + print("Please review each DB entry") + + def row(self, name, mstatus, istatus): + print(name, end=": ") + print(f"manifest: {GHReport.status_map[mstatus]}{mstatus}", end=", ") + print(f"image-info:{GHReport.status_map[istatus]}{istatus}.") + + +def compare(obj1, obj2): + """ + Compare two json objects and return as a string the conclusion of the diff. + Obj1 is the version of Obj2 back in time. So when obj2 is not and obj1 is, + we can say that something was removed and vice versa. + """ + if obj1 and not obj2: + return "removed" + if not obj1 and obj2: + return "added" + if obj1 != obj2: + return "updated" + return "unchanged" + + +def list_changed_files(): + """ + Go through the diff between this commit and the last one and list all the + files in there. An update is only valid if it contains only the DB part that + is updates anyway. + """ + try: + command = subprocess.run( + ["git", "diff-tree", "--no-commit-id", "--name-only", "HEAD", "-r"], + capture_output=True, + check=True) + return command.stdout.decode("utf-8").split("\n") + except subprocess.CalledProcessError: + return None + + +def get_db_entry_at_previous_version(file): + """ + Retrieves a version of a DB file from the previous commit state. + """ + if not file: + return None + try: + command = subprocess.run( + ["git", "show", f"HEAD^:{file}"], + capture_output=True, + check=True) + return json.loads(command.stdout) + except subprocess.CalledProcessError: + return None + return None + + +def get_db_entry_at_current_version(file): + """ + Retrieves a version of a DB file at the current state of the DB + """ + if not file: + return None + with open(file, "r", encoding="utf-8") as f: + return json.load(f) + + +def filter_fn(imi): + """ + Filter specific fields in the image-info that can't be compared + together. + """ + def lvm2(imi): + """ + LVM2 partitions have a UUID that is not fixed. Replace the value + upon comparison time + """ + partitions = imi.get("partitions") + if partitions: + for partition in partitions: + if partition.get("fstype") == "LVM2_member": + partition["uuid"] = "2022-07-01-fixed-uuid" + return imi + + def iso(imi): + """ + For isos, the partition UUID is the date of the build. Replace + that with a fixed one for the comparison. + """ + if "image-format" in imi and "type" in imi["image-format"] and imi["image-format"]["type"] == "raw": + if "partitions" in imi: + for partition in imi["partitions"]: + if "fstype" in partition: + if partition["fstype"] == "iso9660": + partition["uuid"] = "2022-07-01-fixed-uuid" + return imi + return iso(lvm2(imi)) + + +def diff(what, json1, json2, tool="vimdiff"): + """ + Starts a diff between two json objects with a specific tool to do so. Writes + each json to disk in a temporary directory and then invokes the difftool on + these two files. + """ + with tempfile.TemporaryDirectory() as tempdir: + a = os.path.join(tempdir, "1") + b = os.path.join(tempdir, "2") + with open(a, encoding="utf-8", mode="w") as f: + json.dump(json1, f, indent=4) + with open(b, encoding="utf-8", mode="w") as f: + json.dump(json2, f, indent=4) + opendiff = input(f"- diff {what}? [Y, n]") + if opendiff in ("y", "Y", "", None): + subprocess.call([tool, a, b]) + + +def extract(db_entry, do_filter=True): + """ + Returns two json objects from a db entry. One manifest and one image info. + Can apply (and will by default) filters to the image-info to get rid of some + volatile UUIds that aren't important. + """ + manifest = None + imi = None + if not db_entry: + return None, None + if db_entry["image-info"]: + if do_filter: + imi = filter_fn(db_entry["image-info"]) + else: + imi = db_entry["image-info"] + if db_entry["manifest"]: + manifest = db_entry["manifest"] + return manifest, imi + + +def progress(count, total, prefix=''): + """ + Prints a nice progress bar that takes up to the all width of the terminal + """ + bar_len = os.get_terminal_size().columns - 10 - len(prefix) + filled_len = int(round(bar_len * count / float(total))) + + percents = round(100.0 * count / float(total), 1) + barr = '-' * (filled_len-1) + ">" + ' ' * (bar_len - filled_len) + + sys.stdout.write(f'{prefix}{percents}%[{barr}]\r') + sys.stdout.flush() + + +def erase_bar(): + """ + Erases the progress bar + """ + sys.stdout.write(f'{" "*os.get_terminal_size().columns}\r') + sys.stdout.flush() + + +def ls_command(args): + """ + Lists all the changed files between this and the previous commit and puts + some icons for the user to know what changed and what not + """ + files = [] + lst = list_changed_files() + for i, file in enumerate(lst): + filename = os.path.splitext(os.path.basename(file))[0] + prev = get_db_entry_at_previous_version(file) + curr = get_db_entry_at_current_version(file) + + if not (prev or curr): + continue + + prev_manifest, prev_imi = extract(prev, do_filter=not args.no_filter) + curr_manifest, curr_imi = extract(curr, do_filter=not args.no_filter) + + manifest_status = compare(prev_manifest, curr_manifest) + image_info_status = compare(prev_imi, curr_imi) + + if manifest_status != "unchanged" or image_info_status != "unchanged": + ms = GHReport.status_map[manifest_status] + ims = GHReport.status_map[image_info_status] + files.append(f"{ms}{ims} {filename}") + progress(i+1, len(lst), "listing changed files: ") + erase_bar() + print("(manifest)(image-info) file:") + print("🆕: added 🔁: updated 🗑️: removed 💾: unchanged\n") + + if args.l: + for f in files: + print(f) + else: + print(columnify.columnify( + items=files, + line_width=os.get_terminal_size().columns)) + + +def diff_command(args): + """ + Diffs either some files or all the files with the difftool provided by the + user. + """ + for file in list_changed_files(): + filename = os.path.splitext(os.path.basename(file))[0] + + if args.file[0] and filename not in args.file[0]: + continue + + prev = get_db_entry_at_previous_version(file) + curr = get_db_entry_at_current_version(file) + + if not (prev or curr): + continue + + prev_manifest, prev_imi = extract(prev, do_filter=not args.no_filter) + curr_manifest, curr_imi = extract(curr, do_filter=not args.no_filter) + + manifest_status = compare(prev_manifest, curr_manifest) + image_info_status = compare(prev_imi, curr_imi) + + # Since changes are filtered because we encounter UUID volatility on + # some entries, do not print the ones that after this filtering have in + # fact no changes at all. + if manifest_status == "unchanged" and image_info_status == "unchanged": + continue + + print(f"{BOLD}{filename}{RESET}:") + + if manifest_status != "unchanged": + diff("manifest", prev_manifest, curr_manifest, tool=args.manifest_diff_tool) + + if image_info_status != "unchanged": + diff("image-info", prev_imi, curr_imi, tool=args.image_info_diff_tool) + + +def report_command(args): + """ + Prints a summary of the DB update as an ascii table. Or create a github TODO + list to be included in the update PR + """ + reporter = None + if args.github_markdown: + reporter = GHReport() + else: + reporter = CliReport() + for file in list_changed_files(): + filename = os.path.splitext(os.path.basename(file))[0] + + prev = get_db_entry_at_previous_version(file) + curr = get_db_entry_at_current_version(file) + + if not (prev or curr): + continue + + prev_manifest, prev_imi = extract(prev, do_filter=not args.no_filter) + curr_manifest, curr_imi = extract(curr, do_filter=not args.no_filter) + + manifest_status = compare(prev_manifest, curr_manifest) + image_info_status = compare(prev_imi, curr_imi) + + # Since changes are filtered because we encounter UUID volatility on + # some entries, do not print the ones that after this filtering have in + # fact no changes at all. + if manifest_status == "unchanged" and image_info_status == "unchanged": + continue + + reporter.row(filename, manifest_status, image_info_status) + + +def main(): + parser = argparse.ArgumentParser(description="""A tool to explore a DB + update commit.""") + + parser.add_argument( + "--no-filter", + action="store_true", + default=False, + help="Disable UUID masking" + ) + subparsers = parser.add_subparsers( + title="command", + required=True, + dest='command', + help='Command to execute') + + # Report command + parser_report = subparsers.add_parser('report', help="Generate a report in CLI or github format") + parser_report.add_argument( + "--github-markdown", + action="store_true", + default=False, + help="Create a TODO list for github to ask a reviewer to check everything" + ) + + # Diff command + parser_diff = subparsers.add_parser('diff', help="Diff all or a" + "select set of distributions to their previous state in the DB") + parser_diff.add_argument( + "--manifest-diff-tool", + default="vimdiff", + help="tool to diff the manifests" + ) + parser_diff.add_argument( + "--image-info-diff-tool", + default="vimdiff", + help="tool to diff the image info" + ) + parser_diff.add_argument( + dest="file", + action='append', + nargs="*", + help="file(s) to diff, use several time to specify several files" + ) + + # Ls command + parser_ls = subparsers.add_parser('ls', help="List the changes since previous version") + parser_ls.add_argument( + "-l", + action="store_true", + default=False, + ) + + args = parser.parse_args() + + if args.command == "report": + report_command(args) + if args.command == "diff": + diff_command(args) + if args.command == "ls": + ls_command(args) + + +if __name__ == "__main__": + sys.exit(main())