update to referer-less npm-registry-fetch

.github/settings.yml

@@ -0,0 +1,2 @@
+---
+_extends: 'open-source-project-boilerplate'

.github/workflows/ci.yml

@@ -0,0 +1,39 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        node-version: [10.x, 12.x, 13.x]
+        # todo: add windows-latest to this list.
+        # Right now, it's just too flaky and hard to get PWD and HOME out
+        # of all the snapshots properly.
+        os: [ubuntu-latest, macOS-latest]
+      fail-fast: false
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/[email protected]
+
+      - name: Use Nodejs ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Install dependencies
+        run: npm install
+
+      # Run for all environments
+      - name: Run Tap Tests
+        run: npm test
+
+      # Push coverage for specific environment
+      - name: Run Tap Tests (push coverage results)
+        if: matrix.os == 'macOS-latest' && matrix.node-version == '12.x'
+        run: npm test
+        env:
+          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}

.gitignore

@@ -2,6 +2,7 @@
 /*
 /.*
 
+!.github/
 !bin/
 !lib/
 !docs/

README.md

@@ -113,6 +113,14 @@ See below for valid `opts` values.
 
 ### Options
 
+Options are passed to
+[`npm-registry-fetch`](http://npm.im/npm-registry-fetch) and
+[`cacache`](http://npm.im/cacache), so in addition to these, anything for
+those modules can be given to pacote as well.
+
+Options object is cloned, and mutated along the way to add integrity,
+resolved, and other properties, as they are determined.
+
 * `cache` Where to store cache entries and temp files.  Passed to
   [`cacache`](http://npm.im/cacache).  Defaults to the same cache directory
   that npm will use by default, based on platform and environment.
@@ -140,6 +148,10 @@ See below for valid `opts` values.
   packument.  Defaults to `latest`.
 * `registry` The npm registry to use by default.  Defaults to
   `https://registry.npmjs.org/`.
+* `fullMetadata` Fetch the full metadata from the registry for packuments,
+  including information not strictly required for installation (author,
+  description, etc.)  Defaults to `true` when `before` is set, since the
+  version publish time is part of the extended packument metadata.
 
 ## Extracted File Modes
 

lib/fetcher.js

@@ -79,13 +79,13 @@ class FetcherBase {
     this.umask = opts.umask || 0o022
     this.log = opts.log || procLog
 
-    this.preferOnline = !!opts.preferOnline || !!opts['prefer-online']
-    this.preferOffline = !!opts.preferOffline || !!opts['prefer-offline']
+    this.preferOnline = !!opts.preferOnline
+    this.preferOffline = !!opts.preferOffline
     this.offline = !!opts.offline
 
-    this.before = opts.before || opts['enjoy-by'] || opts.before
-    this.fullMetadata = this.before ? true
-      : (opts.fullMetadata || opts['full-metadata'])
+    this.before = opts.before
+    this.fullMetadata = this.before ? true : !!opts.fullMetadata
+
     this.defaultTag = opts.defaultTag || 'latest'
     this.registry = opts.registry || 'https://registry.npmjs.org'
 

lib/registry.js

@@ -20,9 +20,6 @@ class RegistryFetcher extends Fetcher {
   constructor (spec, opts) {
     super(spec, opts)
 
-    // try to use corgis if available
-    this.fullMetadata = !!opts.fullMetadata
-
     // handle case when npm-package-arg guesses wrong.
     if (this.spec.type === 'tag' &&
         this.spec.rawSpec === '' &&
@@ -57,7 +54,7 @@ class RegistryFetcher extends Fetcher {
   [_headers] () {
     return {
       // npm will override UA, but ensure that we always send *something*
-      'user-agent': this.opts['user-agent'] ||
+      'user-agent': this.opts.userAgent ||
         `pacote/${pacoteVersion} node/${process.version}`,
       ...(this.opts.headers || {}),
       'pacote-version': pacoteVersion,

lib/remote.js

@@ -43,14 +43,15 @@ class RemoteFetcher extends Fetcher {
   [_headers] () {
     return {
       // npm will override this, but ensure that we always send *something*
-      'user-agent': this.opts['user-agent'] ||
+      'user-agent': this.opts.userAgent ||
         `pacote/${pacoteVersion} node/${process.version}`,
       ...(this.opts.headers || {}),
       'pacote-version': pacoteVersion,
       'pacote-req-type': 'tarball',
       'pacote-pkg-id': this.pkgid,
       ...(this.integrity ? { 'pacote-integrity': String(this.integrity) }
         : {}),
+      ...(this.opts.headers || {}),
     }
   }
 

lib/util/git/spawn.js

@@ -26,9 +26,9 @@ module.exports = (gitArgs, gitOpts, opts = {}) => {
       })
       .then(({stdout}) => stdout)
   }, opts.retry !== null && opts.retry !== undefined ? opts.retry : {
-    retries: opts['fetch-retries'] || 2,
-    factor: opts['fetch-retry-factor'] || 10,
-    maxTimeout: opts['fetch-retry-maxtimeout'] || 60000,
-    minTimeout: opts['fetch-retry-mintimeout'] || 1000,
+    retries: opts.fetchRetries || 2,
+    factor: opts.fetchRetryFactor || 10,
+    maxTimeout: opts.fetchRetryMaxtimeout || 60000,
+    minTimeout: opts.fetchRetryMintimeout || 1000,
   })
 }

package.json

@@ -46,7 +46,7 @@
     "npm-package-arg": "^8.0.0",
     "npm-packlist": "^2.1.0",
     "npm-pick-manifest": "^6.0.0",
-    "npm-registry-fetch": "^7.0.1",
+    "npm-registry-fetch": "^8.0.0",
     "osenv": "^0.1.5",
     "promise-inflight": "^1.0.1",
     "promise-retry": "^1.1.1",

test/dir.js

@@ -73,6 +73,19 @@ t.test('make bins executable', async t => {
   const f = new DirFetcher(spec, {})
   const target = resolve(me, basename(file))
   const res = await f.extract(target)
-  t.matchSnapshot(res, 'results of unpack')
+  // node v13.8 swapped out their zlib implementation with chromium's
+  // This is slightly faster and results in better compression for most
+  // tarballs.  However, it does mean that the snapshotted integrity is
+  // not valid.  Check if it's the new one, and if so, use the old instead,
+  // so that the snapshot continues to be valid.  At some point, when we
+  // drop support for node versions prior to 14, we can just remove this
+  // and re-generate the snapshot.
+  const oldIntegrity = 'sha512-rlE32nBV7XgKCm0I7YqAewyVPbaRJWUQMZUFLlngGK3imG+som3Hin7d/zPTikWg64tHIxb8VXeeq6u0IRRfmQ=='
+  const newIntegrity = 'sha512-J9g/qC58EQ6h3xMyc1lPP2vlmjy6N5symUYih/l9M3A340A1OHPc88oMSAwVdLKj/lT3NbekLXVjU6ONnPbJYg=='
+  const resTest = {
+    ...res,
+    ...(res.integrity === newIntegrity ? { integrity: oldIntegrity }: {}),
+  }
+  t.matchSnapshot(resTest, 'results of unpack')
   t.equal(fs.statSync(target + '/script.js').mode & 0o111, 0o111)
 })

test/remote.js

@@ -52,14 +52,21 @@ t.test('start server', t => {
 t.test('packument', t => {
   //const url = 'https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz'
   const url = `${server}/abbrev.tgz`
-  const f = new RemoteFetcher(url, { cache })
+  const f = new RemoteFetcher(url, {
+    cache,
+    preferOffline: true,
+    headers: {
+      'not-referer': 'http://example.com',
+    },
+    projectScope: '@npmcli',
+    npmSession: 'foobarbaz',
+  })
   // run twice to pull from cache the second time
   return t.resolveMatchSnapshot(f.packument(), 'packument')
     .then(() => {
       const f2 = new RemoteFetcher(`abbrev@${url}`, {
-        cache,
         pkgid: `remote:abbrev@${url}`,
-        preferOffline: true,
+        cache,
       })
       return t.resolveMatchSnapshot(f2.packument(), 'packument 2')
     })
@@ -72,15 +79,16 @@ t.test('packument', t => {
           {
             connection: 'keep-alive',
             'user-agent': `pacote/${version} node/${process.version}`,
-            'npm-scope': 'undefined',
-            'npm-session': 'undefined',
-            referer: 'undefined',
             'pacote-version': version,
             'pacote-req-type': 'tarball',
             'pacote-pkg-id': `remote:${server}/abbrev.tgz`,
             accept: '*/*',
             'accept-encoding': 'gzip,deflate',
             host: require('url').parse(server).host,
+            'npm-session': 'foobarbaz',
+            'npm-scope': '@npmcli',
+            'npm-in-ci': String,
+            'not-referer': 'http://example.com',
           }
         ]
       ])
@@ -129,6 +137,7 @@ t.test('get a timeout error from the http fetch', t => {
   const f = new RemoteFetcher(url, { cache: cache + '/fresh', timeout: 1 })
   return t.rejects(f.extract(me + '/timeout'), {
     name: 'FetchError',
-    message: `network timeout at: ${server}/timeout`,
+    message: /timeout/,
+    code: /FETCH_ERROR|ERR_SOCKET_TIMEOUT/,
   })
 })

test/util/git/spawn.js

@@ -36,11 +36,11 @@ process.exit(1)
         minTimeout: 1,
       },
     },
-    'namespaced fetch-retry-* configs': {
-      'fetch-retries': 2,
-      'fetch-retry-factor': 1,
-      'fetch-retry-maxtimeout': 1000,
-      'fetch-retry-mintimeout': 1,
+    'namespaced fetchRetry* configs': {
+      fetchRetries: 2,
+      fetchRetryFactor: 1,
+      fetchRetryMaxtimeout: 1000,
+      fetchRetryMintimeout: 1,
     }
   }
   const er = {