From 4b57c118a0b044f41b1c98eaf92449221c984c15 Mon Sep 17 00:00:00 2001 From: fengmk2 <fengmk2@gmail.com> Date: Tue, 22 Jan 2019 21:35:26 +0800 Subject: [PATCH] feat: can override tgz download options speed up download from the wall --- config/index.js | 3 +++ controllers/sync_module_worker.js | 15 +++++++++------ test/services/npm.test.js | 6 +++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/config/index.js b/config/index.js index 845d2e348..82d9b6dd4 100644 --- a/config/index.js +++ b/config/index.js @@ -212,6 +212,9 @@ var config = { // changes streaming sync syncChangesStream: false, + syncDownloadOptions: { + // formatRedirectUrl: function (url, location) + }, handleSyncRegistry: 'http://127.0.0.1:7001', // default badge subject diff --git a/controllers/sync_module_worker.js b/controllers/sync_module_worker.js index c873eab96..0e24d7a61 100644 --- a/controllers/sync_module_worker.js +++ b/controllers/sync_module_worker.js @@ -1305,15 +1305,18 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack var filepath = common.getTarballFilepath(filename); var ws = fs.createWriteStream(filepath); - var options = { + var downloadOptions = { writeStream: ws, followRedirect: true, timeout: 600000, // 10 minutes download headers: { - 'user-agent': USER_AGENT + 'user-agent': USER_AGENT, }, gzip: true, }; + if (config.syncDownloadOptions) { + Object.assign(downloadOptions, config.syncDownloadOptions); + } var dependencies = Object.keys(sourcePackage.dependencies || {}); var devDependencies = []; @@ -1359,7 +1362,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack logger.syncInfo('[sync_module_worker] downloading %j to %j', downurl, filepath); var r; try { - r = yield urllib.request(downurl, options); + r = yield urllib.request(downurl, downloadOptions); } catch (err) { logger.syncInfo('[sync_module_worker] download %j to %j error: %s', downurl, filepath, err); throw err; @@ -1411,16 +1414,16 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack throw err; } - options = { + var uploadOptions = { key: common.getCDNKey(sourcePackage.name, filename), size: dataSize, shasum: shasum }; // upload to NFS - logger.syncInfo('[sync_module_worker] uploading %j to nfs', options); + logger.syncInfo('[sync_module_worker] uploading %j to nfs', uploadOptions); var result; try { - result = yield nfs.upload(filepath, options); + result = yield nfs.upload(filepath, uploadOptions); } catch (err) { logger.syncInfo('[sync_module_worker] upload %j to nfs error: %s', err); throw err; diff --git a/test/services/npm.test.js b/test/services/npm.test.js index eab44d2ed..1316210fe 100644 --- a/test/services/npm.test.js +++ b/test/services/npm.test.js @@ -58,17 +58,17 @@ describe('test/services/npm.test.js', () => { pkg.time['0.0.1-alpha1'].should.equal('2012-03-02T21:46:14.725Z'); pkg.versions['0.0.1-alpha1'].version.should.equal('0.0.1-alpha1'); pkg.versions['0.0.1-alpha1'].dist.shasum.should.equal('cfa9394e29c3eb0fe58998f5bf5bda79aa7d3e2e'); - pkg.versions['0.0.1-alpha1'].dist.tarball.should.equal('http://registry.npmjs.org/shelljs/-/shelljs-0.0.1alpha1.tgz'); + pkg.versions['0.0.1-alpha1'].dist.tarball.should.equal('https://registry.npmjs.org/shelljs/-/shelljs-0.0.1alpha1.tgz'); pkg.time['0.7.5'].should.equal('2016-10-27T05:50:21.479Z'); pkg.versions['0.7.5'].version.should.equal('0.7.5'); pkg.versions['0.7.5'].dist.shasum.should.equal('2eef7a50a21e1ccf37da00df767ec69e30ad0675'); - pkg.versions['0.7.5'].dist.tarball.should.equal('http://registry.npmjs.org/shelljs/-/shelljs-0.7.5.tgz'); + pkg.versions['0.7.5'].dist.tarball.should.equal('https://registry.npmjs.org/shelljs/-/shelljs-0.7.5.tgz'); pkg.time['0.0.6-pre2'].should.equal('2012-05-25T18:14:25.441Z'); pkg.versions['0.0.6-pre2'].version.should.equal('0.0.6-pre2'); pkg.versions['0.0.6-pre2'].dist.shasum.should.equal('8c3eecaddba6f425bd5cae001f80a4d224750911'); - pkg.versions['0.0.6-pre2'].dist.tarball.should.equal('http://registry.npmjs.org/shelljs/-/shelljs-0.0.6pre2.tgz'); + pkg.versions['0.0.6-pre2'].dist.tarball.should.equal('https://registry.npmjs.org/shelljs/-/shelljs-0.0.6pre2.tgz'); }); });