From 77fda8f8e079aeeea1c2aa4bc043f5a56a27928f Mon Sep 17 00:00:00 2001 From: Dean Roehrich Date: Fri, 15 Nov 2024 12:52:43 -0600 Subject: [PATCH] Allow vendoring into nnf-integration-test repo (#231) Adjust vendor-new-api.py so it can vendor into the nnf-integration-test repo. This relies on a new crd-bumper.yaml file in that repo to give some hints. Refactor code that checks whether an API is a hub or a spoke. Refactor code that references the crd-bumper.yaml file. This simplifies the use of the object that holds the content of that file. Reformat code with 'black'. Signed-off-by: Dean Roehrich --- tools/crd-bumper/crd-bumper.py | 13 ++++--- tools/crd-bumper/pkg/conversion_gen.py | 13 ------- tools/crd-bumper/pkg/create_apis.py | 10 +---- tools/crd-bumper/pkg/hub_spoke_util.py | 52 ++++++++++++++++++++++++++ tools/crd-bumper/pkg/vendoring.py | 19 ++++++++-- tools/crd-bumper/unserve.py | 3 +- tools/crd-bumper/vendor-new-api.py | 51 +++++++++++++++++++------ 7 files changed, 117 insertions(+), 44 deletions(-) create mode 100644 tools/crd-bumper/pkg/hub_spoke_util.py diff --git a/tools/crd-bumper/crd-bumper.py b/tools/crd-bumper/crd-bumper.py index 0ff27272..c73d7144 100755 --- a/tools/crd-bumper/crd-bumper.py +++ b/tools/crd-bumper/crd-bumper.py @@ -31,6 +31,7 @@ from pkg.git_cli import GitCLI from pkg.conversion_gen import ConversionGen from pkg.make_cmd import MakeCmd +from pkg.hub_spoke_util import HubSpokeUtil WORKING_DIR = "workingspace" @@ -148,7 +149,7 @@ def main(): project = Project(args.dryrun) # Load any repo-specific local config. - bumper_cfg = None + bumper_cfg = {} if os.path.isfile("crd-bumper.yaml"): with open("crd-bumper.yaml", "r", encoding="utf-8") as fi: bumper_cfg = yaml.safe_load(fi) @@ -158,7 +159,7 @@ def main(): ) makecmd = MakeCmd(args.dryrun, project, args.prev_ver, args.new_ver) - validate_args(args, cgen) + validate_args(args) if args.branch is None: args.branch = f"api-{args.new_ver}" @@ -335,11 +336,11 @@ def bump_controllers(cgen, makecmd, git, stage, project, args, bumper_cfg): controllers.edit_util_conversion_test() # Bump any other, non-controller, directories of code. - if bumper_cfg is not None and "extra_go_dirs" in bumper_cfg: + if "extra_go_dirs" in bumper_cfg: for extra_dir in bumper_cfg["extra_go_dirs"].split(","): controllers.update_extras(extra_dir) # Bump any necessary references in the config/ dir. - if bumper_cfg is not None and "extra_config_dirs" in bumper_cfg: + if "extra_config_dirs" in bumper_cfg: for extra_dir in bumper_cfg["extra_config_dirs"].split(","): controllers.update_extra_config(extra_dir) @@ -423,11 +424,11 @@ def prologue(git, cmd_elem, operation_order): git.expect_previous(cmd_elem[0], op_cmd_list) -def validate_args(args, cgen): +def validate_args(args): """Validate the commandline args""" if args.most_recent_spoke: - if not cgen.is_spoke(args.most_recent_spoke): + if not HubSpokeUtil.is_spoke(args.most_recent_spoke): print(f"API --most-recent-spoke {args.most_recent_spoke} is not a spoke.") sys.exit(1) if args.most_recent_spoke in [args.prev_ver, args.new_ver]: diff --git a/tools/crd-bumper/pkg/conversion_gen.py b/tools/crd-bumper/pkg/conversion_gen.py index 877fbc1f..25b7b10f 100644 --- a/tools/crd-bumper/pkg/conversion_gen.py +++ b/tools/crd-bumper/pkg/conversion_gen.py @@ -38,19 +38,6 @@ def __init__(self, dryrun, project, prev_ver, new_ver, most_recent_spoke): self.module() self._preferred_alias = self.set_preferred_api_alias() - def is_spoke(self, ver): - """ - Determine whether or not the 'ver' API is a spoke. - """ - path = f"api/{ver}/conversion.go" - if not os.path.isfile(path): - return False - fu = FileUtil(self._dryrun, path) - line = fu.find_in_file(" ConvertTo(dstRaw conversion.Hub) ") - if line is None: - return False - return True - def preferred_api_alias(self): """Return the preferred alias.""" return self._preferred_alias diff --git a/tools/crd-bumper/pkg/create_apis.py b/tools/crd-bumper/pkg/create_apis.py index 65ee03c7..d2b441dc 100644 --- a/tools/crd-bumper/pkg/create_apis.py +++ b/tools/crd-bumper/pkg/create_apis.py @@ -22,6 +22,7 @@ from .project import Project from .fileutil import FileUtil +from .hub_spoke_util import HubSpokeUtil class CreateApis: @@ -45,14 +46,7 @@ def prev_is_hub(self): for _, dir_names, _ in os.walk("api", followlinks=False): if len(dir_names) > 1: - path = f"api/{self._prev_ver}/conversion.go" - if not os.path.isfile(path): - return False - fu = FileUtil(self._dryrun, path) - line = fu.find_in_file(" Hub() ") - if line is None: - return False - break + return HubSpokeUtil.is_hub(self._prev_ver) return True def create(self): diff --git a/tools/crd-bumper/pkg/hub_spoke_util.py b/tools/crd-bumper/pkg/hub_spoke_util.py new file mode 100644 index 00000000..d0552a56 --- /dev/null +++ b/tools/crd-bumper/pkg/hub_spoke_util.py @@ -0,0 +1,52 @@ +# Copyright 2024 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from .fileutil import FileUtil + + +class HubSpokeUtil: + """Simple utilities related to hub and spokes.""" + + @staticmethod + def is_spoke(ver): + """ + Determine whether or not the 'ver' API is a spoke. + """ + path = f"api/{ver}/conversion.go" + if not os.path.isfile(path): + return False + fu = FileUtil(False, path) + line = fu.find_in_file(" ConvertTo(dstRaw conversion.Hub) ") + if line is None: + return False + return True + + @staticmethod + def is_hub(ver): + """ + Determine whether or not the 'ver' API is a hub. + """ + path = f"api/{ver}/conversion.go" + if not os.path.isfile(path): + return False + fu = FileUtil(False, path) + line = fu.find_in_file(" Hub() ") + if line is None: + return False + return True diff --git a/tools/crd-bumper/pkg/vendoring.py b/tools/crd-bumper/pkg/vendoring.py index ffced04f..473c7beb 100644 --- a/tools/crd-bumper/pkg/vendoring.py +++ b/tools/crd-bumper/pkg/vendoring.py @@ -68,7 +68,7 @@ def verify_one_api_version(self): self._current_ver = None self.set_current_api_version() - def set_preferred_api_alias(self): + def set_preferred_api_alias(self, main_file=None): """ What is this repo using as the alias for this module's API? @@ -81,7 +81,10 @@ def set_preferred_api_alias(self): We'll look at cmd/main.go to get an answer. """ - fname = "cmd/main.go" + if main_file is not None: + fname = main_file + else: + fname = "cmd/main.go" fu = FileUtil(self._dryrun, fname) # Find the import. line = fu.find_in_file( @@ -95,6 +98,10 @@ def set_preferred_api_alias(self): if self._preferred_alias is None: raise ValueError(f"Expected to find the module's alias in {fname}.") + def update_go_file(self, full_path): + """Bump the given Go file to point at the new hub.""" + self._point_go_files_at_new_hub(full_path) + def update_go_files(self, top=None): """Walk over Go files, bumping them to point at the new hub If top=None then this walks over the cmd/, internal/, and api/ directories @@ -106,7 +113,9 @@ def update_go_files(self, top=None): raise NotADirectoryError(f"{top} is not a directory.") top = [top] else: - top = ["cmd", "api/" + self._hub_ver, "internal/controller", "controllers"] + top = ["cmd", "internal/controller", "controllers"] + if self._hub_ver is not None: + top.append("api/" + self._hub_ver) for dname in top: if os.path.isdir(dname): @@ -140,7 +149,9 @@ def _point_go_files_at_new_hub(self, path): line2 = line.replace(self._current_ver, self._vendor_hub_ver) fu.replace_in_file(line, line2) # This matches: dwsv1alpha1. (yes, dot) - fu.replace_in_file(f"{group}{self._current_ver}.", f"{group}{self._vendor_hub_ver}.") + fu.replace_in_file( + f"{group}{self._current_ver}.", f"{group}{self._vendor_hub_ver}." + ) fu.store() def update_config_files(self, top): diff --git a/tools/crd-bumper/unserve.py b/tools/crd-bumper/unserve.py index 1d321f22..4794b7c6 100755 --- a/tools/crd-bumper/unserve.py +++ b/tools/crd-bumper/unserve.py @@ -27,6 +27,7 @@ from pkg.make_cmd import MakeCmd from pkg.project import Project from pkg.unserve import Unserve +from pkg.hub_spoke_util import HubSpokeUtil WORKING_DIR = "workingspace" BRANCH_SUFFIX = "unserve" @@ -109,7 +110,7 @@ def main(): ) sys.exit(1) - if not cgen.is_spoke(args.spoke_ver): + if not HubSpokeUtil.is_spoke(args.spoke_ver): print(f"API --spoke-ver {args.spoke_ver} is not a spoke.") sys.exit(1) diff --git a/tools/crd-bumper/vendor-new-api.py b/tools/crd-bumper/vendor-new-api.py index b471aa97..7ac1cf63 100755 --- a/tools/crd-bumper/vendor-new-api.py +++ b/tools/crd-bumper/vendor-new-api.py @@ -24,10 +24,12 @@ import sys import yaml +from pkg.conversion_gen import ConversionGen from pkg.git_cli import GitCLI from pkg.make_cmd import MakeCmd from pkg.vendoring import Vendor from pkg.go_cli import GoCLI +from pkg.hub_spoke_util import HubSpokeUtil WORKING_DIR = "workingspace" @@ -35,8 +37,8 @@ PARSER.add_argument( "--hub-ver", type=str, - required=True, - help="Version of the hub API.", + required=False, + help="Version of the hub API of the target repo.", ) PARSER.add_argument( "--vendor-hub-ver", @@ -77,7 +79,7 @@ "--this-branch", action="store_true", dest="this_branch", - help="Continue working in the current branch. Use when stepping through with 'step'.", + help="Continue working in the current branch.", ) PARSER.add_argument( "--dry-run", @@ -112,8 +114,13 @@ def main(): gocli = GoCLI(args.dryrun) + if args.hub_ver: + if not HubSpokeUtil.is_hub(args.hub_ver): + print(f"API --hub-ver {args.hub_ver} is not a hub.") + sys.exit(1) + # Load any repo-specific local config. - bumper_cfg = None + bumper_cfg = {} if os.path.isfile("crd-bumper.yaml"): with open("crd-bumper.yaml", "r", encoding="utf-8") as fi: bumper_cfg = yaml.safe_load(fi) @@ -150,23 +157,36 @@ def vendor_new_api(args, makecmd, git, gocli, bumper_cfg): ) sys.exit(0) try: + if not os.path.isdir("vendor"): + # This repo doesn't normally vendor, so go get it. + # The .gitignore file should already be covering it. + gocli.tidy() + gocli.vendor() vendor.set_current_api_version() - vendor.set_preferred_api_alias() + main_file = None + if "alternate_main" in bumper_cfg: + main_file = bumper_cfg["alternate_main"] + vendor.set_preferred_api_alias(main_file) except ValueError as ex: print(str(ex)) sys.exit(1) - print(f"Updating files from {vendor.current_api_version()} to {args.hub_ver}") + print( + f"Updating files from {vendor.current_api_version()} to {args.vendor_hub_ver}" + ) # Update the Go files that are in the usual kubebuilder locations. vendor.update_go_files() # Bump any other, non-controller, directories of code. - if bumper_cfg is not None and "extra_go_dirs" in bumper_cfg: + if "extra_go_dirs" in bumper_cfg: for extra_dir in bumper_cfg["extra_go_dirs"].split(","): vendor.update_go_files(extra_dir) + if "extra_go_files" in bumper_cfg: + for full_path in bumper_cfg["extra_go_files"].split(","): + vendor.update_go_file(full_path) # Bump any necessary references in the config/ dir. - if bumper_cfg is not None and "extra_config_dirs" in bumper_cfg: + if "extra_config_dirs" in bumper_cfg: for extra_dir in bumper_cfg["extra_config_dirs"].split(","): vendor.update_config_files(extra_dir) @@ -174,11 +194,18 @@ def vendor_new_api(args, makecmd, git, gocli, bumper_cfg): gocli.tidy() gocli.vendor() - makecmd.manifests() - makecmd.generate() - makecmd.generate_go_conversions() + have_controller_gen = True + if "skip_controller_gen" in bumper_cfg: + if bumper_cfg["skip_controller_gen"] is True: + have_controller_gen = False + + if have_controller_gen: + makecmd.manifests() + makecmd.generate() + makecmd.generate_go_conversions() makecmd.fmt() - makecmd.clean_bin() + if have_controller_gen: + makecmd.clean_bin() vendor.commit(git, "vendor-new-api")