-
Notifications
You must be signed in to change notification settings - Fork 2.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Load and calculate Destination of files for
kustomize localize (#4652)
* Implement file localization for localize Implement file localization for new command kustomize localize * Patch file localization Fix lint errors, address feedback, begin transition to file loader from localizer * Add Repo() to Loader * Implement locLoader factory + cleanup * Add domain to RepoSpec Fix ssh relative url host parsing for non-github domain on the side * Implement Load(), New(), Root(), Dst() for LocLoader * Address repospec code review comments * Address 1st round of code review feedback * Address feedback #2 Removed localized path calculations to cover edge case and improve readability. * Remove ldr Cleanup logging * Address code review round #3 * Address code review feedback #4
- Loading branch information
1 parent
a8010ac
commit bf17fe1
Showing
27 changed files
with
792 additions
and
100 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -11,6 +11,7 @@ import ( | |
| "strings" | ||
| "time" | ||
|
|
||
| "sigs.k8s.io/kustomize/kyaml/errors" | ||
| "sigs.k8s.io/kustomize/kyaml/filesys" | ||
| ) | ||
|
|
||
|
|
@@ -21,13 +22,18 @@ import ( | |
| // with a unique name that isn't created until clone time. | ||
| const notCloned = filesys.ConfirmedDir("/notCloned") | ||
|
|
||
| // Schemes returns the schemes that repo urls may have | ||
| func Schemes() []string { | ||
| return []string{"https://", "http://", "ssh://"} | ||
| } | ||
|
|
||
| // RepoSpec specifies a git repository and a branch and path therein. | ||
| type RepoSpec struct { | ||
| // Raw, original spec, used to look for cycles. | ||
| // TODO(monopole): Drop raw, use processed fields instead. | ||
| raw string | ||
|
|
||
| // Host, e.g. github.com | ||
| // Host, e.g. https://github.com/ | ||
| Host string | ||
|
|
||
| // orgRepo name (organization/repoName), | ||
|
|
@@ -86,17 +92,14 @@ func NewRepoSpecFromURL(n string) (*RepoSpec, error) { | |
| if filepath.IsAbs(n) { | ||
| return nil, fmt.Errorf("uri looks like abs path: %s", n) | ||
| } | ||
| host, orgRepo, path, gitRef, gitSubmodules, suffix, gitTimeout := parseGitURL(n) | ||
| if orgRepo == "" { | ||
| return nil, fmt.Errorf("url lacks orgRepo: %s", n) | ||
| repoSpec := parseGitURL(n) | ||
| if repoSpec.Host == "" { | ||
| return nil, errors.Errorf("url lacks host: %s", n) | ||
| } | ||
| if host == "" { | ||
| return nil, fmt.Errorf("url lacks host: %s", n) | ||
| if repoSpec.OrgRepo == "" { | ||
| return nil, errors.Errorf("url lacks orgRepo: %s", n) | ||
| } | ||
| return &RepoSpec{ | ||
| raw: n, Host: host, OrgRepo: orgRepo, | ||
| Dir: notCloned, Path: path, Ref: gitRef, GitSuffix: suffix, | ||
| Submodules: gitSubmodules, Timeout: gitTimeout}, nil | ||
| return repoSpec, nil | ||
| } | ||
|
|
||
| const ( | ||
|
|
@@ -108,44 +111,47 @@ const ( | |
| // From strings like [email protected]:someOrg/someRepo.git or | ||
| // https://github.com/someOrg/someRepo?ref=someHash, extract | ||
| // the parts. | ||
| func parseGitURL(n string) ( | ||
| host string, orgRepo string, path string, gitRef string, gitSubmodules bool, gitSuff string, gitTimeout time.Duration) { | ||
| func parseGitURL(n string) *RepoSpec { | ||
| repoSpec := &RepoSpec{raw: n, Dir: notCloned} | ||
|
|
||
| if strings.Contains(n, gitDelimiter) { | ||
| index := strings.Index(n, gitDelimiter) | ||
| // Adding _git/ to host | ||
| host = normalizeGitHostSpec(n[:index+len(gitDelimiter)]) | ||
| orgRepo = strings.Split(strings.Split(n[index+len(gitDelimiter):], "/")[0], "?")[0] | ||
| path, gitRef, gitTimeout, gitSubmodules = peelQuery(n[index+len(gitDelimiter)+len(orgRepo):]) | ||
| return | ||
| repoSpec.Host = normalizeGitHostSpec(n[:index+len(gitDelimiter)]) | ||
| // url before next /, though make sure before ? query string delimiter | ||
| repoSpec.OrgRepo = strings.Split(strings.Split(n[index+len(gitDelimiter):], "/")[0], "?")[0] | ||
| repoSpec.Path, repoSpec.Ref, repoSpec.Timeout, repoSpec.Submodules = | ||
| peelQuery(n[index+len(gitDelimiter)+len(repoSpec.OrgRepo):]) | ||
| return repoSpec | ||
| } | ||
| host, n = parseHostSpec(n) | ||
| gitSuff = gitSuffix | ||
|
|
||
| repoSpec.Host, n = parseHostSpec(n) | ||
|
|
||
| repoSpec.GitSuffix = gitSuffix | ||
| if strings.Contains(n, gitSuffix) { | ||
| index := strings.Index(n, gitSuffix) | ||
| orgRepo = n[0:index] | ||
| repoSpec.OrgRepo = n[0:index] | ||
| n = n[index+len(gitSuffix):] | ||
| // always try to include / in previous component | ||
| if len(n) > 0 && n[0] == '/' { | ||
| n = n[1:] | ||
| } | ||
| path, gitRef, gitTimeout, gitSubmodules = peelQuery(n) | ||
| return | ||
| } | ||
|
|
||
| i := strings.Index(n, "/") | ||
| if i < 1 { | ||
| path, gitRef, gitTimeout, gitSubmodules = peelQuery(n) | ||
| return | ||
| } | ||
| j := strings.Index(n[i+1:], "/") | ||
| if j >= 0 { | ||
| } else if strings.Contains(n, "/") { | ||
| // there exist at least 2 components that we can interpret as org + repo | ||
| i := strings.Index(n, "/") | ||
| j := strings.Index(n[i+1:], "/") | ||
| // only 2 components, so can use peelQuery to group everything before ? query string delimiter as OrgRepo | ||
| if j == -1 { | ||
| repoSpec.OrgRepo, repoSpec.Ref, repoSpec.Timeout, repoSpec.Submodules = peelQuery(n) | ||
| return repoSpec | ||
| } | ||
| // extract first 2 components | ||
| j += i + 1 | ||
| orgRepo = n[:j] | ||
| path, gitRef, gitTimeout, gitSubmodules = peelQuery(n[j+1:]) | ||
| return | ||
| repoSpec.OrgRepo = n[:j] | ||
| n = n[j+1:] | ||
| } | ||
| path = "" | ||
| orgRepo, gitRef, gitTimeout, gitSubmodules = peelQuery(n) | ||
| return host, orgRepo, path, gitRef, gitSubmodules, gitSuff, gitTimeout | ||
| repoSpec.Path, repoSpec.Ref, repoSpec.Timeout, repoSpec.Submodules = peelQuery(n) | ||
| return repoSpec | ||
| } | ||
|
|
||
| // Clone git submodules by default. | ||
|
|
@@ -197,6 +203,7 @@ func peelQuery(arg string) (string, string, time.Duration, bool) { | |
|
|
||
| func parseHostSpec(n string) (string, string) { | ||
| var host string | ||
| // TODO(annasong): handle ports, non-github ssh with userinfo, github.com sub-domains | ||
| // Start accumulating the host part. | ||
| for _, p := range []string{ | ||
| // Order matters here. | ||
|
|
@@ -207,24 +214,29 @@ func parseHostSpec(n string) (string, string) { | |
| host += p | ||
| } | ||
| } | ||
| // ssh relative path | ||
| if host == "git@" { | ||
| i := strings.Index(n, "/") | ||
| if i > -1 { | ||
| var i int | ||
| // git orgRepo delimiter for ssh relative path | ||
| j := strings.Index(n, ":") | ||
| // orgRepo delimiter only valid if not preceded by / | ||
| k := strings.Index(n, "/") | ||
| // orgRepo delimiter exists, so extend host | ||
| if j > -1 && (k == -1 || k > j) { | ||
| i = j | ||
| // should try to group / with previous component | ||
| // this is possible if git url username expansion follows | ||
| if k == i+1 { | ||
| i = k | ||
| } | ||
| host += n[:i+1] | ||
| n = n[i+1:] | ||
| } else { | ||
| i = strings.Index(n, ":") | ||
| if i > -1 { | ||
| host += n[:i+1] | ||
| n = n[i+1:] | ||
| } | ||
| } | ||
| return host, n | ||
| } | ||
|
|
||
| // If host is a http(s) or ssh URL, grab the domain part. | ||
| for _, p := range []string{ | ||
| "ssh://", "https://", "http://"} { | ||
| for _, p := range Schemes() { | ||
| if strings.HasSuffix(host, p) { | ||
| i := strings.Index(n, "/") | ||
| if i > -1 { | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,22 +19,25 @@ var pathNames = []string{"README.md", "foo/krusty.txt", ""} | |
|
|
||
| var hrefArgs = []string{"someBranch", "master", "v0.1.0", ""} | ||
|
|
||
| var hostNamesRawAndNormalized = [][]string{ | ||
| {"gh:", "gh:"}, | ||
| {"GH:", "gh:"}, | ||
| {"gitHub.com/", "https://github.com/"}, | ||
| {"github.com:", "https://github.com/"}, | ||
| {"http://github.com/", "https://github.com/"}, | ||
| {"https://github.com/", "https://github.com/"}, | ||
| {"hTTps://github.com/", "https://github.com/"}, | ||
| {"https://git-codecommit.us-east-2.amazonaws.com/", "https://git-codecommit.us-east-2.amazonaws.com/"}, | ||
| {"https://fabrikops2.visualstudio.com/", "https://fabrikops2.visualstudio.com/"}, | ||
| {"ssh://git.example.com:7999/", "ssh://git.example.com:7999/"}, | ||
| {"git::https://gitlab.com/", "https://gitlab.com/"}, | ||
| {"git::http://git.example.com/", "http://git.example.com/"}, | ||
| {"git::https://git.example.com/", "https://git.example.com/"}, | ||
| {"[email protected]:", "[email protected]:"}, | ||
| {"[email protected]/", "[email protected]:"}, | ||
| func hostsRawNormalizedAndDomain() [][]string { | ||
| return [][]string{ | ||
| {"gh:", "gh:"}, | ||
| {"GH:", "gh:"}, | ||
| {"gitHub.com/", "https://github.com/"}, | ||
| {"github.com:", "https://github.com/"}, | ||
| {"http://github.com/", "https://github.com/"}, | ||
| {"https://github.com/", "https://github.com/"}, | ||
| {"hTTps://github.com/", "https://github.com/"}, | ||
| {"https://git-codecommit.us-east-2.amazonaws.com/", "https://git-codecommit.us-east-2.amazonaws.com/"}, | ||
| {"https://fabrikops2.visualstudio.com/", "https://fabrikops2.visualstudio.com/"}, | ||
| {"ssh://git.example.com:7999/", "ssh://git.example.com:7999/"}, | ||
| {"git::https://gitlab.com/", "https://gitlab.com/"}, | ||
| {"git::http://git.example.com/", "http://git.example.com/"}, | ||
| {"git::https://git.example.com/", "https://git.example.com/"}, | ||
| {"[email protected]:", "[email protected]:"}, | ||
| {"[email protected]/", "[email protected]:"}, | ||
| {"[email protected]:", "[email protected]:"}, | ||
| } | ||
| } | ||
|
|
||
| func makeURL(hostFmt, orgRepo, path, href string) string { | ||
|
|
@@ -50,7 +53,7 @@ func makeURL(hostFmt, orgRepo, path, href string) string { | |
|
|
||
| func TestNewRepoSpecFromUrl(t *testing.T) { | ||
| var bad [][]string | ||
| for _, tuple := range hostNamesRawAndNormalized { | ||
| for _, tuple := range hostsRawNormalizedAndDomain() { | ||
| hostRaw := tuple[0] | ||
| hostSpec := tuple[1] | ||
| for _, orgRepo := range orgRepos { | ||
|
|
@@ -91,10 +94,10 @@ func TestNewRepoSpecFromUrl(t *testing.T) { | |
|
|
||
| var badData = [][]string{ | ||
| {"/tmp", "uri looks like abs path"}, | ||
| {"iauhsdiuashduas", "url lacks orgRepo"}, | ||
| {"iauhsdiuashduas", "url lacks host"}, | ||
| {"htxxxtp://github.com/", "url lacks host"}, | ||
| {"ssh://git.example.com", "url lacks orgRepo"}, | ||
| {"git::___", "url lacks orgRepo"}, | ||
| {"git::___", "url lacks host"}, | ||
| } | ||
|
|
||
| func TestNewRepoSpecFromUrlErrors(t *testing.T) { | ||
|
|
@@ -147,6 +150,7 @@ func TestNewRepoSpecFromUrl_CloneSpecs(t *testing.T) { | |
| ref: "", | ||
| }, | ||
| "t6": { | ||
| // TODO(annasong): replace illegal test case with legal variations | ||
| input: "[email protected]:10022/infra/kubernetes/thanos-base.git?ref=v0.1.0", | ||
| cloneSpec: "[email protected]:10022/infra/kubernetes/thanos-base.git", | ||
| absPath: notCloned.String(), | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| // Copyright 2022 The Kubernetes Authors. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // This package contains utilities for the command kustomize localize, which is | ||
| // documented under proposals/localize-command or at | ||
| // https://github.com/kubernetes-sigs/kustomize/blob/master/proposals/22-04-localize-command.md | ||
| package localizer |
Oops, something went wrong.