From 7ffff72429b33d0b2b4c4c56019ed1d4fb54d6e9 Mon Sep 17 00:00:00 2001 From: Michael Kedar Date: Fri, 2 Feb 2024 13:34:05 +1100 Subject: [PATCH] Guided Remediation: add npm registry clients & `.npmrc` parsing (#778) The datasource and `DependencyClient` for querying the npm registry API directly, instead of relying on deps.dev. Also, parses `.npmrc` configs to allow resolution of requirements from private registries. Practically unchanged from what we had internally, besides a bunch of linting complaints. --- go.mod | 1 + go.sum | 2 + internal/remediation/relaxer/npm.go | 4 +- internal/remediation/relaxer/relaxer.go | 2 +- .../resolution/client/npm_registry_client.go | 286 ++++++++++++++++++ .../resolution/datasource/npm_registry.go | 175 +++++++++++ .../datasource/npm_registry_cache.go | 68 +++++ internal/resolution/datasource/npmrc.go | 276 +++++++++++++++++ 8 files changed, 811 insertions(+), 3 deletions(-) create mode 100644 internal/resolution/client/npm_registry_client.go create mode 100644 internal/resolution/datasource/npm_registry.go create mode 100644 internal/resolution/datasource/npm_registry_cache.go create mode 100644 internal/resolution/datasource/npmrc.go diff --git a/go.mod b/go.mod index 07ae59a5b3..e2e1e8eb96 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( golang.org/x/vuln v1.0.1 google.golang.org/grpc v1.60.1 google.golang.org/protobuf v1.31.0 + gopkg.in/ini.v1 v1.67.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index b9ade1b329..e3b8eca456 100644 --- a/go.sum +++ b/go.sum @@ -246,6 +246,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/remediation/relaxer/npm.go b/internal/remediation/relaxer/npm.go index e7976b49b5..a5925632ca 100644 --- a/internal/remediation/relaxer/npm.go +++ b/internal/remediation/relaxer/npm.go @@ -8,9 +8,9 @@ import ( "deps.dev/util/semver" ) -type NPMRelaxer struct{} +type NpmRelaxer struct{} -func (r NPMRelaxer) Relax(ctx context.Context, cl resolve.Client, req resolve.RequirementVersion, allowMajor bool) (resolve.RequirementVersion, bool) { +func (r NpmRelaxer) Relax(ctx context.Context, cl resolve.Client, req resolve.RequirementVersion, allowMajor bool) (resolve.RequirementVersion, bool) { c, err := semver.NPM.ParseConstraint(req.Version) if err != nil { // The specified version is not a valid semver constraint diff --git a/internal/remediation/relaxer/relaxer.go b/internal/remediation/relaxer/relaxer.go index 4d4e110da7..053f5e7bdc 100644 --- a/internal/remediation/relaxer/relaxer.go +++ b/internal/remediation/relaxer/relaxer.go @@ -27,7 +27,7 @@ func GetRelaxer(ecosystem resolve.System) (RequirementRelaxer, error) { // TODO: is using ecosystem fine, or should this be per manifest? switch ecosystem { //nolint:exhaustive case resolve.NPM: - return NPMRelaxer{}, nil + return NpmRelaxer{}, nil default: return nil, errors.New("unsupported ecosystem") } diff --git a/internal/resolution/client/npm_registry_client.go b/internal/resolution/client/npm_registry_client.go new file mode 100644 index 0000000000..2324264d85 --- /dev/null +++ b/internal/resolution/client/npm_registry_client.go @@ -0,0 +1,286 @@ +package client + +import ( + "context" + "crypto/x509" + "encoding/gob" + "fmt" + "os" + "slices" + "strings" + + pb "deps.dev/api/v3alpha" + "deps.dev/util/resolve" + "deps.dev/util/resolve/dep" + "deps.dev/util/semver" + "github.com/google/osv-scanner/internal/resolution/datasource" + "github.com/google/osv-scanner/pkg/depsdev" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +const npmRegistryCacheExt = ".resolve.npm" + +type NpmRegistryClient struct { + api *datasource.NpmRegistryAPIClient + + // Fallback client for dealing with bundleDependencies. + ic pb.InsightsClient + fallback *resolve.APIClient +} + +func NewNpmRegistryClient(workdir string) (*NpmRegistryClient, error) { + api, err := datasource.NewNpmRegistryAPIClient(workdir) + if err != nil { + return nil, err + } + + certPool, err := x509.SystemCertPool() + if err != nil { + return nil, fmt.Errorf("getting system cert pool: %w", err) + } + creds := credentials.NewClientTLSFromCert(certPool, "") + conn, err := grpc.Dial(depsdev.DepsdevAPI, grpc.WithTransportCredentials(creds)) + if err != nil { + return nil, fmt.Errorf("dialling %q: %w", depsdev.DepsdevAPI, err) + } + ic := pb.NewInsightsClient(conn) + + return &NpmRegistryClient{ + api: api, + ic: ic, + fallback: resolve.NewAPIClient(ic), + }, nil +} + +func (c *NpmRegistryClient) Version(ctx context.Context, vk resolve.VersionKey) (resolve.Version, error) { + if isNpmBundle(vk.PackageKey) { // bundled dependencies, fallback to deps.dev client + return c.fallback.Version(ctx, vk) + } + + return resolve.Version{VersionKey: vk}, nil +} + +func (c *NpmRegistryClient) Versions(ctx context.Context, pk resolve.PackageKey) ([]resolve.Version, error) { + if isNpmBundle(pk) { // bundled dependencies, fallback to deps.dev client + return c.fallback.Versions(ctx, pk) + } + + vers, err := c.api.Versions(ctx, pk.Name) + if err != nil { + return nil, err + } + + vks := make([]resolve.Version, len(vers.Versions)) + for i, v := range vers.Versions { + vks[i] = resolve.Version{ + VersionKey: resolve.VersionKey{ + PackageKey: pk, + Version: v, + VersionType: resolve.Concrete, + }} + } + + slices.SortFunc(vks, func(a, b resolve.Version) int { return semver.NPM.Compare(a.Version, b.Version) }) + + return vks, nil +} + +func (c *NpmRegistryClient) Requirements(ctx context.Context, vk resolve.VersionKey) ([]resolve.RequirementVersion, error) { + if vk.System != resolve.NPM { + return nil, fmt.Errorf("unsupported system: %v", vk.System) + } + + if isNpmBundle(vk.PackageKey) { // bundled dependencies, fallback to deps.dev client + return c.fallback.Requirements(ctx, vk) + } + dependencies, err := c.api.Dependencies(ctx, vk.Name, vk.Version) + if err != nil { + return nil, err + } + + // Preallocate the dependency slice, which will hold all the dependencies of each type. + // The npm resolver expects bundled dependencies included twice in different forms: + // {foo@*|Scope="bundle"} and {mangled-name-of>0.1.2>foo@1.2.3}, hence the 2*len(bundled) + depCount := len(dependencies.Dependencies) + len(dependencies.DevDependencies) + + len(dependencies.OptionalDependencies) + len(dependencies.PeerDependencies) + + 2*len(dependencies.BundleDependencies) + deps := make([]resolve.RequirementVersion, 0, depCount) + addDeps := func(ds map[string]string, t dep.Type) { + for name, req := range ds { + typ := t.Clone() + if r, ok := strings.CutPrefix(req, "npm:"); ok { + // This dependency is aliased, add it as a + // dependency on the actual name, with the + // KnownAs attribute set to the alias. + typ.AddAttr(dep.KnownAs, name) + name = r + req = "" + if i := strings.LastIndex(r, "@"); i > 0 { + name = r[:i] + req = r[i+1:] + } + } + deps = append(deps, resolve.RequirementVersion{ + Type: typ, + VersionKey: resolve.VersionKey{ + PackageKey: resolve.PackageKey{ + System: resolve.NPM, + Name: name, + }, + VersionType: resolve.Requirement, + Version: req, + }, + }) + } + } + addDeps(dependencies.Dependencies, dep.NewType()) + addDeps(dependencies.DevDependencies, dep.NewType(dep.Dev)) + addDeps(dependencies.OptionalDependencies, dep.NewType(dep.Opt)) + + peerType := dep.NewType() + peerType.AddAttr(dep.Scope, "peer") + addDeps(dependencies.PeerDependencies, peerType) + + // The resolver expects bundleDependencies to be present as regular + // dependencies with a "*" version specifier, even if they were already + // in the regular dependencies. + bundleType := dep.NewType() + bundleType.AddAttr(dep.Scope, "bundle") + for _, name := range dependencies.BundleDependencies { + deps = append(deps, resolve.RequirementVersion{ + Type: bundleType, + VersionKey: resolve.VersionKey{ + PackageKey: resolve.PackageKey{ + System: resolve.NPM, + Name: name, + }, + VersionType: resolve.Requirement, + Version: "*", + }, + }) + } + + // Correctly resolving the bundled dependencies would require downloading the package. + // Instead, call the fallback deps.dev client to get the bundled dependencies with mangled names. + if len(dependencies.BundleDependencies) > 0 { + fallbackReqs, err := c.fallback.Requirements(ctx, vk) + if err != nil { + // TODO: make some placeholder if the package doesn't exist on deps.dev + return nil, err + } + for _, req := range fallbackReqs { + if isNpmBundle(req.PackageKey) { + deps = append(deps, req) + } + } + } + + resolve.SortDependencies(deps) + + return deps, nil +} + +func (c *NpmRegistryClient) MatchingVersions(ctx context.Context, vk resolve.VersionKey) ([]resolve.Version, error) { + if isNpmBundle(vk.PackageKey) { // bundled dependencies, fallback to deps.dev client + return c.fallback.MatchingVersions(ctx, vk) + } + + versions, err := c.api.Versions(ctx, vk.Name) + if err != nil { + return nil, err + } + + if concVer, ok := versions.Tags[vk.Version]; ok { + // matched a tag, return just the concrete version of the tag + return []resolve.Version{{ + VersionKey: resolve.VersionKey{ + PackageKey: vk.PackageKey, + Version: concVer, + VersionType: resolve.Concrete, + }, + }}, nil + } + + resVersions := make([]resolve.Version, len(versions.Versions)) + for i, v := range versions.Versions { + resVersions[i] = resolve.Version{ + VersionKey: resolve.VersionKey{ + PackageKey: vk.PackageKey, + Version: v, + VersionType: resolve.Concrete, + }, + } + } + + return resolve.MatchRequirement(vk, resVersions), nil +} + +func isNpmBundle(pk resolve.PackageKey) bool { + // Bundles are represented in resolution with a 'mangled' name containing its origin e.g. "root-pkg>1.0.0>bundled-package" + // '>' is not a valid character for a npm package, so it'll only be found here. + return strings.Contains(pk.Name, ">") +} + +func (c *NpmRegistryClient) PreFetch(ctx context.Context, imports []resolve.RequirementVersion, manifestPath string) { + // It doesn't matter if loading the cache fails + _ = c.LoadCache(manifestPath) + + // Use the deps.dev client to fetch complete dependency graphs of our direct imports + for _, im := range imports { + // Get the preferred version of the import requirement + vks, err := c.MatchingVersions(ctx, im.VersionKey) + if err != nil || len(vks) == 0 { + continue + } + + vk := vks[len(vks)-1] + + // Make a request for the precomputed dependency tree + resp, err := c.ic.GetDependencies(ctx, &pb.GetDependenciesRequest{ + VersionKey: &pb.VersionKey{ + System: pb.System(vk.System), + Name: vk.Name, + Version: vk.Version, + }, + }) + if err != nil { + continue + } + + // Send off queries to cache the packages in the dependency tree + for _, node := range resp.GetNodes() { + pbvk := node.GetVersionKey() + vk := resolve.VersionKey{ + PackageKey: resolve.PackageKey{ + System: resolve.System(pbvk.GetSystem()), + Name: pbvk.GetName(), + }, + Version: pbvk.GetVersion(), + VersionType: resolve.Concrete, + } + go c.Requirements(ctx, vk) //nolint:errcheck + } + } + // don't bother waiting for goroutines to finish. +} + +func (c *NpmRegistryClient) WriteCache(path string) error { + f, err := os.Create(path + npmRegistryCacheExt) + if err != nil { + return err + } + defer f.Close() + + return gob.NewEncoder(f).Encode(c.api) +} + +func (c *NpmRegistryClient) LoadCache(path string) error { + f, err := os.Open(path + npmRegistryCacheExt) + if err != nil { + return err + } + defer f.Close() + + return gob.NewDecoder(f).Decode(&c.api) +} diff --git a/internal/resolution/datasource/npm_registry.go b/internal/resolution/datasource/npm_registry.go new file mode 100644 index 0000000000..ed7f31e4a4 --- /dev/null +++ b/internal/resolution/datasource/npm_registry.go @@ -0,0 +1,175 @@ +package datasource + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "sync" + "time" + + "github.com/tidwall/gjson" + "golang.org/x/exp/maps" +) + +type NpmRegistryAPIClient struct { + // Registries from the npmrc config + // This should only be written to when the client is first being created. + // Other functions should not modify it & it is not covered by the mutex. + registries npmRegistries + + // cache fields + mu sync.Mutex + cacheTimestamp *time.Time // If set, this means we loaded from a cache + details map[string]npmRegistryPackageDetails +} + +type npmRegistryPackageDetails struct { + // Only cache the info needed for the DependencyClient + Versions map[string]npmRegistryDependencies + Tags map[string]string +} + +func NewNpmRegistryAPIClient(workdir string) (*NpmRegistryAPIClient, error) { + npmrc, err := loadNpmrc(workdir) + if err != nil { + return nil, err + } + + return &NpmRegistryAPIClient{ + registries: parseRegistryInfo(npmrc), + details: make(map[string]npmRegistryPackageDetails), + }, nil +} + +type npmRegistryVersions struct { + Versions []string + Tags map[string]string +} + +func (c *NpmRegistryAPIClient) Versions(ctx context.Context, pkg string) (npmRegistryVersions, error) { + pkgDetails, err := c.getPackageDetails(ctx, pkg) + if err != nil { + return npmRegistryVersions{}, err + } + + return npmRegistryVersions{ + Versions: maps.Keys(pkgDetails.Versions), + Tags: pkgDetails.Tags, + }, nil +} + +type npmRegistryDependencies struct { + // TODO: These maps should preserve ordering from JSON response + Dependencies map[string]string + DevDependencies map[string]string + PeerDependencies map[string]string + OptionalDependencies map[string]string + BundleDependencies []string +} + +func (c *NpmRegistryAPIClient) Dependencies(ctx context.Context, pkg, version string) (npmRegistryDependencies, error) { + pkgDetails, err := c.getPackageDetails(ctx, pkg) + if err != nil { + return npmRegistryDependencies{}, err + } + + if deps, ok := pkgDetails.Versions[version]; ok { + return deps, nil + } + + return npmRegistryDependencies{}, fmt.Errorf("no version %s for package %s", version, pkg) +} + +func (c *NpmRegistryAPIClient) FullJSON(ctx context.Context, pkg, version string) (gjson.Result, error) { + return c.get(ctx, pkg, version) +} + +func (c *NpmRegistryAPIClient) get(ctx context.Context, urlComponents ...string) (gjson.Result, error) { + req, err := c.registries.buildRequest(ctx, urlComponents...) + if err != nil { + return gjson.Result{}, err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return gjson.Result{}, err + } + + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return gjson.Result{}, errors.New(resp.Status) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return gjson.Result{}, err + } + + res := gjson.ParseBytes(body) + + return res, nil +} + +func (c *NpmRegistryAPIClient) getPackageDetails(ctx context.Context, pkg string) (npmRegistryPackageDetails, error) { + c.mu.Lock() + pkgData, ok := c.details[pkg] + c.mu.Unlock() + if ok { + return pkgData, nil + } + + // Not cached, make the network request + jsonData, err := c.get(ctx, pkg) + if err != nil { + return npmRegistryPackageDetails{}, err + } + + versions := make(map[string]npmRegistryDependencies) + for v, data := range jsonData.Get("versions").Map() { + versions[v] = npmRegistryDependencies{ + Dependencies: jsonToStringMap(data.Get("dependencies")), + DevDependencies: jsonToStringMap(data.Get("devDependencies")), + PeerDependencies: jsonToStringMap(data.Get("peerDependencies")), + OptionalDependencies: jsonToStringMap(data.Get("optionalDependencies")), + BundleDependencies: jsonToStringSlice(data.Get("bundleDependencies")), + } + } + pkgData = npmRegistryPackageDetails{ + Versions: versions, + Tags: jsonToStringMap(jsonData.Get("dist-tags")), + } + + c.mu.Lock() + c.details[pkg] = pkgData + c.mu.Unlock() + + return pkgData, nil +} + +func jsonToStringSlice(v gjson.Result) []string { + arr := v.Array() + if len(arr) == 0 { + return nil + } + strs := make([]string, len(arr)) + for i, s := range arr { + strs[i] = s.String() + } + + return strs +} + +func jsonToStringMap(v gjson.Result) map[string]string { + mp := v.Map() + if len(mp) == 0 { + return nil + } + strs := make(map[string]string) + for k, s := range mp { + strs[k] = s.String() + } + + return strs +} diff --git a/internal/resolution/datasource/npm_registry_cache.go b/internal/resolution/datasource/npm_registry_cache.go new file mode 100644 index 0000000000..88473696bd --- /dev/null +++ b/internal/resolution/datasource/npm_registry_cache.go @@ -0,0 +1,68 @@ +package datasource + +import ( + "strings" + "time" + + "golang.org/x/exp/maps" +) + +type npmRegistryCache struct { + Timestamp *time.Time + Details map[string]npmRegistryPackageDetails + RegistryURLs map[string]string +} + +func (c *NpmRegistryAPIClient) GobEncode() ([]byte, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.cacheTimestamp == nil { + now := time.Now().UTC() + c.cacheTimestamp = &now + } + + cache := npmRegistryCache{ + Timestamp: c.cacheTimestamp, + Details: c.details, + RegistryURLs: make(map[string]string), + } + + // store the registry URL for each scope (but not the auth info) + for scope, reg := range c.registries { + cache.RegistryURLs[scope] = reg.URL + } + + return gobMarshal(&cache) +} + +func (c *NpmRegistryAPIClient) GobDecode(b []byte) error { + // decode the cached data + var cache npmRegistryCache + if err := gobUnmarshal(b, &cache); err != nil { + return err + } + + if cache.Timestamp != nil && time.Since(*cache.Timestamp) >= cacheExpiry { + // Cache expired + return nil + } + + c.mu.Lock() + defer c.mu.Unlock() + + // remove any cache entries whose registry has changed + maps.DeleteFunc(cache.Details, func(pkg string, _ npmRegistryPackageDetails) bool { + scope := "" + if strings.HasPrefix(pkg, "@") { + scope, _, _ = strings.Cut(pkg, "/") + } + + return cache.RegistryURLs[scope] != c.registries[scope].URL + }) + + c.cacheTimestamp = cache.Timestamp + c.details = cache.Details + + return nil +} diff --git a/internal/resolution/datasource/npmrc.go b/internal/resolution/datasource/npmrc.go new file mode 100644 index 0000000000..3dd8fb30c7 --- /dev/null +++ b/internal/resolution/datasource/npmrc.go @@ -0,0 +1,276 @@ +package datasource + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/google/osv-scanner/internal/cachedregexp" + "gopkg.in/ini.v1" +) + +type npmrcConfig struct { + *ini.Section +} + +func loadNpmrc(workdir string) (npmrcConfig, error) { + // Find & parse the 4 npmrc files (builtin, global, user, project) + values set in environment variables + // https://docs.npmjs.com/cli/v10/configuring-npm/npmrc + // https://docs.npmjs.com/cli/v10/using-npm/config + + // project npmrc is always in ./.npmrc + projectFile, _ := filepath.Abs(filepath.Join(workdir, ".npmrc")) + builtinFile := builtinNpmrc() + envVarOpts, _ := envVarNpmrc() + + opts := ini.LoadOptions{ + Loose: true, // ignore missing files + KeyValueDelimiters: "=", // default delimiters are "=:", but npmrc uses : in some keys + } + // Make use of data overwriting to load the correct values + fullNpmrc, err := ini.LoadSources(opts, builtinFile, projectFile, envVarOpts) + if err != nil { + return npmrcConfig{}, err + } + + // user npmrc is either set as userconfig, or ${HOME}/.npmrc + // though userconfig cannot be set in the user or global npmrcs + var userFile string + switch { + case fullNpmrc.Section("").HasKey("userconfig"): + userFile = os.ExpandEnv(fullNpmrc.Section("").Key("userconfig").String()) + // TODO: npm config replaces only ${VAR}, not $VAR + // and if VAR is unset, it will leave the string as "${VAR}" + default: + homeDir, err := os.UserHomeDir() + if err == nil { // only set userFile if homeDir exists + userFile = filepath.Join(homeDir, ".npmrc") + } + } + + // reload the npmrc files with the user file included + fullNpmrc, err = ini.LoadSources(opts, builtinFile, userFile, projectFile, envVarOpts) + if err != nil { + return npmrcConfig{}, err + } + + var globalFile string + // global npmrc is either set as globalconfig, prefix/etc/npmrc, ${PREFIX}/etc/npmrc + // cannot be set within the global npmrc itself + switch { + case fullNpmrc.Section("").HasKey("globalconfig"): + globalFile = os.ExpandEnv(fullNpmrc.Section("").Key("globalconfig").String()) + case fullNpmrc.Section("").HasKey("prefix"): + prefix := os.ExpandEnv(fullNpmrc.Section("").Key("prefix").String()) + globalFile, _ = filepath.Abs(filepath.Join(prefix, "etc", "npmrc")) + case os.Getenv("PREFIX") != "": + globalFile, _ = filepath.Abs(filepath.Join(os.Getenv("PREFIX"), "etc", "npmrc")) + default: + globalFile = filepath.Join("/etc", "npmrc") // TODO: what should this be actually? + } + + // return final joined config, with correct overriding order + fullNpmrc, err = ini.LoadSources(opts, builtinFile, globalFile, userFile, projectFile, envVarOpts) + if err != nil { + return npmrcConfig{}, err + } + + return npmrcConfig{fullNpmrc.Section("")}, nil +} + +func envVarNpmrc() ([]byte, error) { + // parse npm config settings that were set in environment variables, + // returns a ini.Load()-able byte array of the values + + iniFile := ini.Empty() + // npm config environment variables seem to be case-insensitive, interpreted in lowercase + // get all the matching environment variables and their values + const envPrefix = "npm_config_" + for _, env := range os.Environ() { + split := strings.SplitN(env, "=", 2) + k := strings.ToLower(split[0]) + v := split[1] + if s, ok := strings.CutPrefix(k, envPrefix); ok { + if _, err := iniFile.Section("").NewKey(s, v); err != nil { + return nil, err + } + } + } + var buf bytes.Buffer + _, err := iniFile.WriteTo(&buf) + + return buf.Bytes(), err +} + +func builtinNpmrc() string { + // builtin is always at /path/to/npm/npmrc + npmExec, err := exec.LookPath("npm") + if err != nil { + return "" + } + npmExec, err = filepath.EvalSymlinks(npmExec) + if err != nil { + return "" + } + npmrc := filepath.Join(filepath.Dir(npmExec), "..", "npmrc") + npmrc, err = filepath.Abs(npmrc) + if err != nil { + return "" + } + + return npmrc +} + +type npmRegistryAuthInfo struct { + authToken string + auth string + username string + password string +} + +func (authInfo npmRegistryAuthInfo) addAuth(header http.Header) { + switch { + case authInfo.authToken != "": + header.Set("Authorization", "Bearer "+authInfo.authToken) + case authInfo.auth != "": + header.Set("Authorization", "Basic "+authInfo.auth) + case authInfo.username != "" && authInfo.password != "": + // auth is base64-encoded "username:password" + // password is stored already base64-encoded + authBytes := []byte(authInfo.username + ":") + b, err := base64.StdEncoding.DecodeString(authInfo.password) + if err != nil { + // TODO: npm seems to actually be quite lenient with invalid encodings + panic(fmt.Sprintf("Unable to decode registry password: %v", err)) + } + authBytes = append(authBytes, b...) + auth := base64.StdEncoding.EncodeToString(authBytes) + header.Set("Authorization", "Basic "+auth) + } +} + +type npmRegistryInfo struct { + URL string + authInfo *npmRegistryAuthInfo +} + +// create the http request to the registry api +// urlComponents should be (package) or (package, version) +func (info npmRegistryInfo) buildRequest(ctx context.Context, urlComponents ...string) (*http.Request, error) { + for i := range urlComponents { + urlComponents[i] = url.PathEscape(urlComponents[i]) + } + reqURL, err := url.JoinPath(info.URL, urlComponents...) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + if err != nil { + return nil, err + } + info.authInfo.addAuth(req.Header) + + return req, nil +} + +type npmRegistries map[string]npmRegistryInfo + +// create the http request to the corresponding npm registry api +// urlComponents should be (package) or (package, version) +func (r npmRegistries) buildRequest(ctx context.Context, urlComponents ...string) (*http.Request, error) { + if len(urlComponents) == 0 { + return nil, errors.New("no package specified in npm request") + } + // find the corresponding registryInfo for the package's scope + pkg := urlComponents[0] + scope := "" + if strings.HasPrefix(pkg, "@") { + scope, _, _ = strings.Cut(pkg, "/") + } + info, ok := r[scope] + if !ok { + // no specific rules for this scope, use the default scope + info = r[""] + } + + return info.buildRequest(ctx, urlComponents...) +} + +func parseRegistryInfo(npmrc npmrcConfig) npmRegistries { + infos := make(npmRegistries) // map of @scope to info + auths := make(map[string]*npmRegistryAuthInfo) // map of host url to auth + + getOrCreateAuth := func(host string) *npmRegistryAuthInfo { + host, _ = strings.CutSuffix(host, "/") + if authInfo, ok := auths[host]; ok { + return authInfo + } + auths[host] = &npmRegistryAuthInfo{} + + return auths[host] + } + + makeRegistryInfo := func(fullURL string) npmRegistryInfo { + u, err := url.Parse(fullURL) + if err != nil { + panic(fmt.Sprintf("Error parsing url %s: %v", fullURL, err)) + } + + return npmRegistryInfo{ + URL: fullURL, + authInfo: getOrCreateAuth(u.Host), + } + } + + // set the default registry + infos[""] = makeRegistryInfo("https://registry.npmjs.org") + // Regexes for matching the scope/host in npmrc keys + var ( + urlRegex = cachedregexp.MustCompile(`^(@.*):registry$`) + authTokenRegex = cachedregexp.MustCompile(`^//(.*):_authToken$`) + authRegex = cachedregexp.MustCompile(`^//(.*):_auth$`) + usernameRegex = cachedregexp.MustCompile(`^//(.*):username$`) + passwordRegex = cachedregexp.MustCompile(`^//(.*):_password$`) + ) + + for _, k := range npmrc.Keys() { + name := k.Name() + value := os.ExpandEnv(k.String()) + // TODO: npm config replaces only ${VAR}, not $VAR + // and if VAR is unset, it will leave the string as "${VAR}" + switch { + case name == "registry": + infos[""] = makeRegistryInfo(value) + case urlRegex.MatchString(name): + scope := urlRegex.FindStringSubmatch(name)[1] + infos[scope] = makeRegistryInfo(value) + case authTokenRegex.MatchString(name): + u := authTokenRegex.FindStringSubmatch(name)[1] + info := getOrCreateAuth(u) + info.authToken = value + case authRegex.MatchString(name): + u := authRegex.FindStringSubmatch(name)[1] + info := getOrCreateAuth(u) + info.auth = value + case usernameRegex.MatchString(name): + u := usernameRegex.FindStringSubmatch(name)[1] + info := getOrCreateAuth(u) + info.username = value + case passwordRegex.MatchString(name): + u := passwordRegex.FindStringSubmatch(name)[1] + info := getOrCreateAuth(u) + info.password = value + } + } + + return infos +}