From 77f34c8a0a434a87f3e557a0fb2f95853ca16edd Mon Sep 17 00:00:00 2001 From: Casey Marshall Date: Mon, 20 Dec 2021 19:46:28 -0600 Subject: [PATCH] feat: cache resolved refs, improve URI reader extensibility This improves extensibility of how URI references are read when loading OpenAPI 3 documents, and introduces simple URI-based caching. One could, for example, integate new URI readers, such as one backed by embed.FS. A basic map-based cache implementation is provided, which may cover many simple use cases. The cache interface introduced here may be implemented with a third-party backend such as an LRU cache or Redis for more advanced use cases. RFC 7234 HTTP caching may also be implemented by customizing the HTTP client used to read remote URI references. Customizing the HTTP client also allows further customization of the transport, timeouts, etc. --- openapi3/loader.go | 21 +------ openapi3/loader_uri_reader.go | 115 ++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 19 deletions(-) create mode 100644 openapi3/loader_uri_reader.go diff --git a/openapi3/loader.go b/openapi3/loader.go index 0b8d0e1cc..8af733c3b 100644 --- a/openapi3/loader.go +++ b/openapi3/loader.go @@ -5,8 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" - "net/http" "net/url" "path" "path/filepath" @@ -31,7 +29,7 @@ type Loader struct { IsExternalRefsAllowed bool // ReadFromURIFunc allows overriding the any file/URL reading func - ReadFromURIFunc func(loader *Loader, url *url.URL) ([]byte, error) + ReadFromURIFunc ReadFromURIFunc Context context.Context @@ -121,22 +119,7 @@ func (loader *Loader) readURL(location *url.URL) ([]byte, error) { if f := loader.ReadFromURIFunc; f != nil { return f(loader, location) } - - if location.Scheme != "" && location.Host != "" { - resp, err := http.Get(location.String()) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode > 399 { - return nil, fmt.Errorf("error loading %q: request returned status code %d", location.String(), resp.StatusCode) - } - return ioutil.ReadAll(resp.Body) - } - if location.Scheme != "" || location.Host != "" || location.RawQuery != "" { - return nil, fmt.Errorf("unsupported URI: %q", location.String()) - } - return ioutil.ReadFile(location.Path) + return DefaultReadFromURI(loader, location) } // LoadFromData loads a spec from a byte array diff --git a/openapi3/loader_uri_reader.go b/openapi3/loader_uri_reader.go new file mode 100644 index 000000000..20cc2d2bb --- /dev/null +++ b/openapi3/loader_uri_reader.go @@ -0,0 +1,115 @@ +package openapi3 + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/url" +) + +// ReadFromURIFunc defines a function which reads the contents of a resource +// located at a URI. +type ReadFromURIFunc func(loader *Loader, url *url.URL) ([]byte, error) + +// ErrURINotSupported indicates the ReadFromURIFunc does not know how to handle a +// given URI. +var ErrURINotSupported = fmt.Errorf("unsupported URI") + +// ReadFromURIs returns a ReadFromURIFunc which tries to read a URI using the +// given reader functions, in the same order. If a reader function does not +// support the URI and returns ErrURINotSupported, the next function is checked +// until a match is found, or the URI is not supported by any. +func ReadFromURIs(readers ...ReadFromURIFunc) ReadFromURIFunc { + return func(loader *Loader, url *url.URL) ([]byte, error) { + for i := range readers { + buf, err := readers[i](loader, url) + if err == ErrURINotSupported { + continue + } else if err != nil { + return nil, err + } + return buf, nil + } + return nil, ErrURINotSupported + } +} + +// DefaultReadFromURI returns a ReadFromURIFunc which can read remote HTTP URIs and +// local file URIs. +var DefaultReadFromURI = ReadFromURIs(ReadFromHTTP(http.DefaultClient), ReadFromFile) + +// ReadFromHTTP returns a ReadFromURIFunc which uses the given http.Client to +// read the contents from a remote HTTP URI. This client may be customized to +// implement timeouts, RFC 7234 caching, etc. +func ReadFromHTTP(cl *http.Client) ReadFromURIFunc { + return func(loader *Loader, location *url.URL) ([]byte, error) { + if location.Scheme == "" || location.Host == "" { + return nil, ErrURINotSupported + } + req, err := http.NewRequest("GET", location.String(), nil) + if err != nil { + return nil, err + } + resp, err := cl.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode > 399 { + return nil, fmt.Errorf("error loading %q: request returned status code %d", location.String(), resp.StatusCode) + } + return ioutil.ReadAll(resp.Body) + } +} + +// ReadFromFile is a ReadFromURIFunc which reads local file URIs. +func ReadFromFile(loader *Loader, location *url.URL) ([]byte, error) { + if location.Host != "" { + return nil, ErrURINotSupported + } + if location.Scheme != "" && location.Scheme != "file" { + return nil, ErrURINotSupported + } + return ioutil.ReadFile(location.Path) +} + +// URIReadCache defines a cache for the contents read from URI references. +type URIReadCache interface { + Get(location *url.URL) ([]byte, bool) + Set(location *url.URL, contents []byte) +} + +// MapURIReadCache implements URIReadCache with a simple map. +type MapURIReadCache map[string][]byte + +// Get implements URIReadCache. +func (m MapURIReadCache) Get(location *url.URL) ([]byte, bool) { + contents, ok := m[location.String()] + return contents, ok +} + +// Set implements URIReadCache. +func (m MapURIReadCache) Set(location *url.URL, contents []byte) { + m[location.String()] = contents +} + +// ReadFromCache returns a cached ReadFromURIFunc. If cache is nil, a new +// internal cache is allocated, scoped to the given reader. +func ReadFromCache(cache URIReadCache, r ReadFromURIFunc) ReadFromURIFunc { + if cache == nil { + cache = MapURIReadCache{} + } + return func(loader *Loader, location *url.URL) ([]byte, error) { + var err error + cached, ok := cache.Get(location) + if ok { + return cached, nil + } + cached, err = r(loader, location) + if err != nil { + return nil, err + } + cache.Set(location, cached) + return cached, nil + } +}