feat: can override tgz download options

speed up download from the wall
cnpm · Jan 22, 2019 · 4b57c11 · 4b57c11
1 parent d0c3f1b
commit 4b57c11
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 9 deletions.
diff --git a/config/index.js b/config/index.js
@@ -212,6 +212,9 @@ var config = {
 
   // changes streaming sync
   syncChangesStream: false,
+  syncDownloadOptions: {
+    // formatRedirectUrl: function (url, location)
+  },
   handleSyncRegistry: 'http://127.0.0.1:7001',
 
   // default badge subject

diff --git a/controllers/sync_module_worker.js b/controllers/sync_module_worker.js
@@ -1305,15 +1305,18 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
   var filepath = common.getTarballFilepath(filename);
   var ws = fs.createWriteStream(filepath);
 
-  var options = {
+  var downloadOptions = {
     writeStream: ws,
     followRedirect: true,
     timeout: 600000, // 10 minutes download
     headers: {
-      'user-agent': USER_AGENT
+      'user-agent': USER_AGENT,
     },
     gzip: true,
   };
+  if (config.syncDownloadOptions) {
+    Object.assign(downloadOptions, config.syncDownloadOptions);
+  }
 
   var dependencies = Object.keys(sourcePackage.dependencies || {});
   var devDependencies = [];
@@ -1359,7 +1362,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
     logger.syncInfo('[sync_module_worker] downloading %j to %j', downurl, filepath);
     var r;
     try {
-      r = yield urllib.request(downurl, options);
+      r = yield urllib.request(downurl, downloadOptions);
     } catch (err) {
       logger.syncInfo('[sync_module_worker] download %j to %j error: %s', downurl, filepath, err);
       throw err;
@@ -1411,16 +1414,16 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
       throw err;
     }
 
-    options = {
+    var uploadOptions = {
       key: common.getCDNKey(sourcePackage.name, filename),
       size: dataSize,
       shasum: shasum
     };
     // upload to NFS
-    logger.syncInfo('[sync_module_worker] uploading %j to nfs', options);
+    logger.syncInfo('[sync_module_worker] uploading %j to nfs', uploadOptions);
     var result;
     try {
-      result = yield nfs.upload(filepath, options);
+      result = yield nfs.upload(filepath, uploadOptions);
     } catch (err) {
       logger.syncInfo('[sync_module_worker] upload %j to nfs error: %s', err);
       throw err;

diff --git a/test/services/npm.test.js b/test/services/npm.test.js
@@ -58,17 +58,17 @@ describe('test/services/npm.test.js', () => {
       pkg.time['0.0.1-alpha1'].should.equal('2012-03-02T21:46:14.725Z');
       pkg.versions['0.0.1-alpha1'].version.should.equal('0.0.1-alpha1');
       pkg.versions['0.0.1-alpha1'].dist.shasum.should.equal('cfa9394e29c3eb0fe58998f5bf5bda79aa7d3e2e');
-      pkg.versions['0.0.1-alpha1'].dist.tarball.should.equal('http://registry.npmjs.org/shelljs/-/shelljs-0.0.1alpha1.tgz');
+      pkg.versions['0.0.1-alpha1'].dist.tarball.should.equal('https://registry.npmjs.org/shelljs/-/shelljs-0.0.1alpha1.tgz');
 
       pkg.time['0.7.5'].should.equal('2016-10-27T05:50:21.479Z');
       pkg.versions['0.7.5'].version.should.equal('0.7.5');
       pkg.versions['0.7.5'].dist.shasum.should.equal('2eef7a50a21e1ccf37da00df767ec69e30ad0675');
-      pkg.versions['0.7.5'].dist.tarball.should.equal('http://registry.npmjs.org/shelljs/-/shelljs-0.7.5.tgz');
+      pkg.versions['0.7.5'].dist.tarball.should.equal('https://registry.npmjs.org/shelljs/-/shelljs-0.7.5.tgz');
 
       pkg.time['0.0.6-pre2'].should.equal('2012-05-25T18:14:25.441Z');
       pkg.versions['0.0.6-pre2'].version.should.equal('0.0.6-pre2');
       pkg.versions['0.0.6-pre2'].dist.shasum.should.equal('8c3eecaddba6f425bd5cae001f80a4d224750911');
-      pkg.versions['0.0.6-pre2'].dist.tarball.should.equal('http://registry.npmjs.org/shelljs/-/shelljs-0.0.6pre2.tgz');
+      pkg.versions['0.0.6-pre2'].dist.tarball.should.equal('https://registry.npmjs.org/shelljs/-/shelljs-0.0.6pre2.tgz');
     });
   });