From 33820719e2c049ff93e86d29251b96d496f2b6e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Spie=C3=9F?= Date: Fri, 12 May 2017 21:37:15 +0200 Subject: [PATCH] [Replace #2507] Support VSTS and other custom registries with unusual feeds (#3231) * Fix issue 2505 * Fix flow check * Fix test errors * Load custom package host suffix from .yarnrc * Update tests with new customHostPrefix parameter * Use customHostSuffix parameter to determine if request is to registry --- .../.npmrc | 1 + .../.npmrc | 1 + .../registries/is-request-to-registry.js | 111 +++++++++++------- src/fetchers/base-fetcher.js | 2 + src/fetchers/tarball-fetcher.js | 2 +- src/registries/is-request-to-registry.js | 13 +- src/registries/npm-registry.js | 76 +++++++----- src/resolvers/registries/npm-resolver.js | 1 + src/types.js | 1 + 9 files changed, 129 insertions(+), 79 deletions(-) diff --git a/__tests__/fixtures/install/install-from-authed-private-registry-no-slash/.npmrc b/__tests__/fixtures/install/install-from-authed-private-registry-no-slash/.npmrc index d64920ea68..729940e72b 100644 --- a/__tests__/fixtures/install/install-from-authed-private-registry-no-slash/.npmrc +++ b/__tests__/fixtures/install/install-from-authed-private-registry-no-slash/.npmrc @@ -1,2 +1,3 @@ //registry.yarnpkg.com/:_authToken=abc123 +//registry.yarnpkg.com/:always-auth=true @types:registry=https://registry.yarnpkg.com diff --git a/__tests__/fixtures/install/install-from-authed-private-registry/.npmrc b/__tests__/fixtures/install/install-from-authed-private-registry/.npmrc index c86299f566..e2d18acf7c 100644 --- a/__tests__/fixtures/install/install-from-authed-private-registry/.npmrc +++ b/__tests__/fixtures/install/install-from-authed-private-registry/.npmrc @@ -1,2 +1,3 @@ //registry.yarnpkg.com/:_authToken=abc123 +//registry.yarnpkg.com/:always-auth=true @types:registry=https://registry.yarnpkg.com/ diff --git a/__tests__/registries/is-request-to-registry.js b/__tests__/registries/is-request-to-registry.js index 2509888430..34047fb5cd 100644 --- a/__tests__/registries/is-request-to-registry.js +++ b/__tests__/registries/is-request-to-registry.js @@ -3,49 +3,70 @@ import isRequestToRegistry from '../../src/registries/is-request-to-registry.js'; -test('isRequestToRegistry functional test', () => { - expect(isRequestToRegistry( - 'http://foo.bar:80/foo/bar/baz', - 'http://foo.bar/foo/', - )).toBe(true); - - expect(isRequestToRegistry( - 'http://foo.bar/foo/bar/baz', - 'http://foo.bar/foo/', - )).toBe(true); - - expect(isRequestToRegistry( - 'https://foo.bar:443/foo/bar/baz', - 'https://foo.bar/foo/', - )).toBe(true); - - expect(isRequestToRegistry( - 'https://foo.bar/foo/bar/baz', - 'https://foo.bar:443/foo/', - )).toBe(true); - - expect(isRequestToRegistry( - 'http://foo.bar:80/foo/bar/baz', - 'https://foo.bar/foo/', - )).toBe(true); - - expect(isRequestToRegistry( - 'http://foo.bar/blah/whatever/something', - 'http://foo.bar/foo/', - )).toBe(false); - - expect(isRequestToRegistry( - 'https://wrong.thing/foo/bar/baz', - 'https://foo.bar/foo/', - )).toBe(false); - - expect(isRequestToRegistry( - 'https://foo.bar:1337/foo/bar/baz', - 'https://foo.bar/foo/', - )).toBe(false); - - expect(isRequestToRegistry( - 'http://foo.bar/foo/bar/baz', - 'https://foo.bar/foo/bar/baz', - )).toBe(true); +describe('isRequestToRegistry functional test', () => { + test('request to registry url matching', () => { + expect(isRequestToRegistry( + 'http://foo.bar:80/foo/bar/baz', + 'http://foo.bar/foo/', + )).toBe(true); + + expect(isRequestToRegistry( + 'http://foo.bar/foo/bar/baz', + 'http://foo.bar/foo/', + )).toBe(true); + + expect(isRequestToRegistry( + 'http://foo.bar/foo/00000000-1111-4444-8888-000000000000/baz', + 'http://foo.bar/foo/', + )).toBe(true); + + expect(isRequestToRegistry( + 'https://foo.bar:443/foo/bar/baz', + 'https://foo.bar/foo/', + )).toBe(true); + + expect(isRequestToRegistry( + 'https://foo.bar/foo/bar/baz', + 'https://foo.bar:443/foo/', + )).toBe(true); + + expect(isRequestToRegistry( + 'https://foo.bar/foo/bar/baz', + 'https://foo.bar:443/foo/', + )).toBe(true); + + expect(isRequestToRegistry( + 'http://foo.bar:80/foo/bar/baz', + 'https://foo.bar/foo/', + )).toBe(true); + + expect(isRequestToRegistry( + 'https://wrong.thing/foo/bar/baz', + 'https://foo.bar/foo/', + )).toBe(false); + + expect(isRequestToRegistry( + 'https://foo.bar:1337/foo/bar/baz', + 'https://foo.bar/foo/', + )).toBe(false); + }); + + test('isRequestToRegistry with custom host prefix', () => { + expect(isRequestToRegistry( + 'http://pkgs.host.com:80/foo/bar/baz', + 'http://pkgs.host.com/bar/baz', + 'some.host.org', + )).toBe(false); + + expect(isRequestToRegistry( + 'http://foo.bar/foo/bar/baz', + 'https://foo.bar/foo/bar/baz', + )).toBe(true); + + expect(isRequestToRegistry( + 'http://pkgs.host.com:80/foo/bar/baz', + 'http://pkgs.host.com/bar/baz', + 'pkgs.host.com', + )).toBe(true); + }); }); diff --git a/src/fetchers/base-fetcher.js b/src/fetchers/base-fetcher.js index 1e32b97367..1e7ea8124d 100644 --- a/src/fetchers/base-fetcher.js +++ b/src/fetchers/base-fetcher.js @@ -13,6 +13,7 @@ const path = require('path'); export default class BaseFetcher { constructor(dest: string, remote: PackageRemote, config: Config) { this.reporter = config.reporter; + this.packageName = remote.packageName; this.reference = remote.reference; this.registry = remote.registry; this.hash = remote.hash; @@ -24,6 +25,7 @@ export default class BaseFetcher { reporter: Reporter; remote: PackageRemote; registry: RegistryNames; + packageName: ?string; reference: string; config: Config; hash: ?string; diff --git a/src/fetchers/tarball-fetcher.js b/src/fetchers/tarball-fetcher.js index 915f45f363..58783bf3db 100644 --- a/src/fetchers/tarball-fetcher.js +++ b/src/fetchers/tarball-fetcher.js @@ -178,7 +178,7 @@ export default class TarballFetcher extends BaseFetcher { .pipe(extractorStream) .on('error', reject); }, - }); + }, this.packageName); } async _fetch(): Promise { diff --git a/src/registries/is-request-to-registry.js b/src/registries/is-request-to-registry.js index 0821dc3fd8..a46eda2ebb 100644 --- a/src/registries/is-request-to-registry.js +++ b/src/registries/is-request-to-registry.js @@ -2,17 +2,22 @@ import url from 'url'; -export default function isRequestToRegistry(requestUrl: string, registry: string): boolean { +export default function isRequestToRegistry(requestUrl: string, registry: string, customHostSuffix: ? any): boolean { const requestParsed = url.parse(requestUrl); const registryParsed = url.parse(registry); + const requestHost = requestParsed.hostname || ''; + const registryHost = registryParsed.hostname || ''; const requestPort = getPortOrDefaultPort(requestParsed.port, requestParsed.protocol); const registryPort = getPortOrDefaultPort(registryParsed.port, registryParsed.protocol); const requestPath = requestParsed.path || ''; const registryPath = registryParsed.path || ''; - return (requestParsed.hostname === registryParsed.hostname) && - (requestPort === registryPort) && - requestPath.startsWith(registryPath); + return (requestHost === registryHost) && + (requestPort === registryPort) && + (requestPath.startsWith(registryPath) || + // For some registries, the package path does not prefix with the registry path + (!!customHostSuffix && customHostSuffix.length > 0 && requestHost.endsWith(customHostSuffix)) + ); } function getPortOrDefaultPort(port: ?string, protocol: ?string): ?string { diff --git a/src/registries/npm-registry.js b/src/registries/npm-registry.js index 523053d969..38f35331d1 100644 --- a/src/registries/npm-registry.js +++ b/src/registries/npm-registry.js @@ -9,7 +9,7 @@ import * as fs from '../util/fs.js'; import NpmResolver from '../resolvers/registries/npm-resolver.js'; import envReplace from '../util/env-replace.js'; import Registry from './base-registry.js'; -import {addSuffix, removePrefix} from '../util/misc'; +import {addSuffix} from '../util/misc'; import isRequestToRegistry from './is-request-to-registry.js'; const userHome = require('../util/user-home-dir').default; @@ -18,6 +18,8 @@ const url = require('url'); const ini = require('ini'); const DEFAULT_REGISTRY = 'https://registry.npmjs.org/'; +const REGEX_REGISTRY_PREFIX = /^https?:/; +const REGEX_REGISTRY_SUFFIX = /registry\/?$/; function getGlobalPrefix(): string { if (process.env.PREFIX) { @@ -51,18 +53,17 @@ export default class NpmRegistry extends Registry { return name.replace('/', '%2f'); } - request(pathname: string, opts?: RegistryRequestOptions = {}): Promise<*> { - const registry = addSuffix(this.getRegistry(pathname), '/'); + request(pathname: string, opts?: RegistryRequestOptions = {}, packageName: ?string): Promise<*> { + const registry = this.getRegistry(packageName || pathname); const requestUrl = url.resolve(registry, pathname); - const alwaysAuth = this.getScopedOption(registry.replace(/^https?:/, ''), 'always-auth') - || this.getOption('always-auth') - || removePrefix(requestUrl, registry)[0] === '@'; + const alwaysAuth = this.getRegistryOrGlobalOption(registry, 'always-auth'); + const customHostSuffix = this.getRegistryOrGlobalOption(registry, 'custom-host-suffix'); const headers = Object.assign({ 'Accept': 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*', }, opts.headers); - if (this.token || (alwaysAuth && isRequestToRegistry(requestUrl, registry))) { - const authorization = this.getAuth(pathname); + if (this.token || (alwaysAuth && isRequestToRegistry(requestUrl, registry, customHostSuffix))) { + const authorization = this.getAuth(packageName || pathname); if (authorization) { headers.authorization = authorization; } @@ -160,7 +161,7 @@ export default class NpmRegistry extends Registry { const availableRegistries = this.getAvailableRegistries(); const registry = availableRegistries.find((registry) => packageName.startsWith(registry)); if (registry) { - return registry; + return addSuffix(registry, '/'); } } @@ -168,7 +169,7 @@ export default class NpmRegistry extends Registry { const registry = this.getScopedOption(scope, 'registry') || this.registries.yarn.getScopedOption(scope, 'registry'); if (registry) { - return String(registry); + return addSuffix(String(registry), '/'); } } @@ -180,28 +181,26 @@ export default class NpmRegistry extends Registry { return this.token; } - for (let registry of [this.getRegistry(packageName), '', DEFAULT_REGISTRY]) { - registry = registry.replace(/^https?:/, ''); + const registry = this.getRegistry(packageName); - // Check for bearer token. - let auth = this.getScopedOption(registry.replace(/\/?$/, '/'), '_authToken'); - if (auth) { - return `Bearer ${String(auth)}`; - } + // Check for bearer token. + const authToken = this.getRegistryOrGlobalOption(registry, '_authToken'); + if (authToken) { + return `Bearer ${String(authToken)}`; + } - // Check for basic auth token. - auth = this.getScopedOption(registry, '_auth'); - if (auth) { - return `Basic ${String(auth)}`; - } + // Check for basic auth token. + const auth = this.getRegistryOrGlobalOption(registry, '_auth'); + if (auth) { + return `Basic ${String(auth)}`; + } - // Check for basic username/password auth. - const username = this.getScopedOption(registry, 'username'); - const password = this.getScopedOption(registry, '_password'); - if (username && password) { - const pw = new Buffer(String(password), 'base64').toString(); - return 'Basic ' + new Buffer(String(username) + ':' + pw).toString('base64'); - } + // Check for basic username/password auth. + const username = this.getRegistryOrGlobalOption(registry, 'username'); + const password = this.getRegistryOrGlobalOption(registry, '_password'); + if (username && password) { + const pw = new Buffer(String(password), 'base64').toString(); + return 'Basic ' + new Buffer(String(username) + ':' + pw).toString('base64'); } return ''; @@ -210,4 +209,23 @@ export default class NpmRegistry extends Registry { getScopedOption(scope: string, option: string): mixed { return this.getOption(scope + (scope ? ':' : '') + option); } + + getRegistryOption(registry: string, option: string): mixed { + const pre = REGEX_REGISTRY_PREFIX; + const suf = REGEX_REGISTRY_SUFFIX; + + // When registry is used config scope, the trailing '/' is required + const reg = addSuffix(registry, '/'); + + // 1st attempt, try to get option for the given registry URL + // 2nd attempt, remove the 'https?:' prefix of the registry URL + // 3nd attempt, remove the 'registry/?' suffix of the registry URL + return this.getScopedOption(reg, option) + || reg.match(pre) && this.getRegistryOption(reg.replace(pre, ''), option) + || reg.match(suf) && this.getRegistryOption(reg.replace(suf, ''), option); + } + + getRegistryOrGlobalOption(registry: string, option: string): mixed { + return this.getRegistryOption(registry, option) || this.getOption(option); + } } diff --git a/src/resolvers/registries/npm-resolver.js b/src/resolvers/registries/npm-resolver.js index 4912280d8b..76b05c28f5 100644 --- a/src/resolvers/registries/npm-resolver.js +++ b/src/resolvers/registries/npm-resolver.js @@ -191,6 +191,7 @@ export default class NpmResolver extends RegistryResolver { reference: this.cleanRegistry(dist.tarball), hash: dist.shasum, registry: 'npm', + packageName: info.name, }; } diff --git a/src/types.js b/src/types.js index 67ca3ae6cb..e167f3285b 100644 --- a/src/types.js +++ b/src/types.js @@ -42,6 +42,7 @@ export type PackageRemote = { reference: string, resolved?: ?string, hash: ?string, + packageName?: string, }; // `dependencies` field in package info