Skip to content

Commit

Permalink
fix: performance issues due to http $refs
Browse files Browse the repository at this point in the history 
When processing OpenAPI specs that contain a large number of http $refs,
vervet performance becomes unbearably slow. This is caused by overly
simplistic URI resolution in kin-openapi. Solution is a fix upstream in
getkin/kin-openapi#469. Until that lands, vervet
may use a go.mod replace directive to obtain the fix.

Drive-by: simplify implementation of Document.LoadReference, which was
changing the current working directory unnecessarily in order to process
relative paths.
  • Loading branch information
@cmars
cmars committed Feb 17, 2022
1 parent db163f8 commit 99e4588
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 38 deletions.
40 changes: 14 additions & 26 deletions document.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package vervet
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/url"
"os"
Expand Down Expand Up @@ -112,43 +111,32 @@ func (d *Document) ResolveRefs() error {
// target. The relative path of the reference is returned, so that references
// may be chain-loaded with successive calls.
func (d *Document) LoadReference(relPath, refPath string, target interface{}) (_ string, returnErr error) {
cwd, err := os.Getwd()
refUrl, err := url.Parse(refPath)
if err != nil {
return "", err
}
defer func() {
err := os.Chdir(cwd)
if refUrl.Scheme == "" || refUrl.Scheme == "file" {
refPath, err = filepath.Abs(filepath.Join(relPath, refUrl.Path))
if err != nil {
log.Println("warning: failed to restore working directory: %w", err)
if returnErr == nil {
returnErr = err
}
return "", err
}
}()
err = os.Chdir(relPath)
if err != nil {
return "", err
refUrl.Path = refPath
}

// Parse and load the contents of the referenced document.
u, err := url.Parse(refPath)
if err != nil {
return "", fmt.Errorf("failed to parse %q: %w", refPath, err)
}
if u.Scheme != "" || u.Host != "" {
return "", fmt.Errorf("URL %q not supported", refPath)
}
contents, err := ioutil.ReadFile(u.Path)
l := openapi3.NewLoader()
l.IsExternalRefsAllowed = true
contents, err := openapi3.DefaultReadFromURI(l, refUrl)
if err != nil {
return "", err
return "", fmt.Errorf("failed to read %q: %w", refUrl, err)
}
// If the reference is to an element in the referenced document, further resolve that.
if u.Fragment != "" {
parts := strings.Split(u.Fragment, "/")
if refUrl.Fragment != "" {
parts := strings.Split(refUrl.Fragment, "/")
// TODO: support actual jsonpaths if/when needed. For now only
// top-level properties are supported.
if parts[0] != "" || len(parts) > 2 {
return "", fmt.Errorf("URL %q not supported", u.String())
return "", fmt.Errorf("URL %q not supported", refUrl)
}
elements := map[string]interface{}{}
err := yaml.Unmarshal(contents, &elements)
Expand All @@ -157,7 +145,7 @@ func (d *Document) LoadReference(relPath, refPath string, target interface{}) (_
}
elementDoc, ok := elements[parts[1]]
if !ok {
return "", fmt.Errorf("element %q not found in %q", parts[1], u.Path)
return "", fmt.Errorf("element %q not found in %q", parts[1], refUrl.Path)
}
contents, err = json.Marshal(elementDoc)
if err != nil {
Expand All @@ -171,5 +159,5 @@ func (d *Document) LoadReference(relPath, refPath string, target interface{}) (_
return "", err
}

return filepath.Abs(filepath.Dir(u.Path))
return filepath.Abs(filepath.Dir(refUrl.Path))
}
3 changes: 3 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ require (
go.uber.org/multierr v1.7.0
)

// TODO: remove once https://github.com/getkin/kin-openapi/pull/469 lands
replace github.com/getkin/kin-openapi => github.com/cmars/kin-openapi v0.0.0-20220216164516-8ffc85653bfb

require (
github.com/Microsoft/go-winio v0.5.1 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20220113124808-70ae35bab23f // indirect
Expand Down
14 changes: 2 additions & 12 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@ github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpz
github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3 h1:XcF0cTDJeiuZ5NU8w7WUDge0HRwwNRmxj/GGk6KSA6g=
github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
github.com/ProtonMail/go-crypto v0.0.0-20220113124808-70ae35bab23f h1:J2FzIrXN82q5uyUraeJpLIm7U6PffRwje2ORho5yIik=
github.com/ProtonMail/go-crypto v0.0.0-20220113124808-70ae35bab23f/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
Expand All @@ -23,6 +21,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5O
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/cmars/kin-openapi v0.0.0-20220216164516-8ffc85653bfb h1:2Gx5x0oQWCtSfFwYV9I0RECvyc2x9N8aC2OUT0qDOnw=
github.com/cmars/kin-openapi v0.0.0-20220216164516-8ffc85653bfb/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU=
github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
Expand All @@ -35,10 +35,6 @@ github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk=
github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
github.com/getkin/kin-openapi v0.87.0 h1:eeb0WBIgRiXra7ZY0Vo+jWloqvaF2kNEaxAyb+39N+E=
github.com/getkin/kin-openapi v0.87.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
github.com/getkin/kin-openapi v0.88.0 h1:BjJ2JERWJbYE1o1RGEj/5LmR5qw7ecfl3O3su4ImR+0=
github.com/getkin/kin-openapi v0.88.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
Expand Down Expand Up @@ -140,15 +136,11 @@ golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 h1:/pEO3GD/ABYAjuakUS6xSEmmlyVS4kxBNkeA9tLJiTI=
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce h1:Roh6XWxHFKrPgC/EQhVubSAGQ6Ozk6IdxHSzt1mR0EI=
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d h1:1n1fc535VhN8SYtD4cDUyNlfpAF2ROMM9+11equK3hs=
golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand All @@ -165,8 +157,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
Expand Down

0 comments on commit 99e4588

Please sign in to comment.