From ed368bf6ab65e8141406a8b66f54d01785144490 Mon Sep 17 00:00:00 2001 From: Arda TANRIKULU Date: Sun, 1 Sep 2024 14:12:50 +0300 Subject: [PATCH] fix(node-fetch/node-libcurl): SSL and async handling issues (#1617) --- .changeset/nine-bugs-shop.md | 11 ++ .npmrc | 2 +- package.json | 2 +- packages/node-fetch/src/fetchCurl.ts | 171 ++++++++++++++------------- packages/node-fetch/src/utils.ts | 24 ++++ yarn.lock | 90 +++++++------- 6 files changed, 179 insertions(+), 121 deletions(-) create mode 100644 .changeset/nine-bugs-shop.md diff --git a/.changeset/nine-bugs-shop.md b/.changeset/nine-bugs-shop.md new file mode 100644 index 00000000000..17cd65f891d --- /dev/null +++ b/.changeset/nine-bugs-shop.md @@ -0,0 +1,11 @@ +--- +'@whatwg-node/node-fetch': patch +--- + +# Fixes for usage of `node-libcurl` + +- Fix \`Error: SSL peer certificate or SSH remove key was not ok error\`, and use `tls.rootCertificates` as default certificates. + +[Learn more](https://github.com/JCMais/node-libcurl/blob/develop/COMMON_ISSUES.md) + +- Fix `API function called from within callback` by preventing the use of `curl_easy_perform` and `curl_multi_perform` inside callbacks. \ No newline at end of file diff --git a/.npmrc b/.npmrc index 268c392d3cb..16919e78669 100644 --- a/.npmrc +++ b/.npmrc @@ -1 +1 @@ -provenance=true +provenance=true \ No newline at end of file diff --git a/package.json b/package.json index 522cb47c43c..b1f9f6143fb 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ "ts:check": "tsc --noEmit" }, "optionalDependencies": { - "node-libcurl": "4.0.0", "uWebSockets.js": "uNetworking/uWebSockets.js#v20.48.0" }, "devDependencies": { @@ -56,6 +55,7 @@ "husky": "9.1.5", "jest": "29.7.0", "lint-staged": "15.2.9", + "node-libcurl": "npm:@ardatan/node-libcurl@4.0.2", "patch-package": "8.0.0", "prettier": "3.3.3", "rimraf": "6.0.1", diff --git a/packages/node-fetch/src/fetchCurl.ts b/packages/node-fetch/src/fetchCurl.ts index 87935e2b076..6b02ed09c84 100644 --- a/packages/node-fetch/src/fetchCurl.ts +++ b/packages/node-fetch/src/fetchCurl.ts @@ -1,7 +1,8 @@ import { PassThrough, Readable, promises as streamPromises } from 'stream'; +import { rootCertificates } from 'tls'; import { PonyfillRequest } from './Request.js'; import { PonyfillResponse } from './Response.js'; -import { defaultHeadersSerializer, isNodeReadable } from './utils.js'; +import { createDeferredPromise, defaultHeadersSerializer, isNodeReadable } from './utils.js'; export function fetchCurl( fetchRequest: PonyfillRequest, @@ -20,6 +21,8 @@ export function fetchCurl( if (process.env.NODE_EXTRA_CA_CERTS) { curlHandle.setOpt('CAINFO', process.env.NODE_EXTRA_CA_CERTS); + } else { + curlHandle.setOpt('CAINFO_BLOB', rootCertificates.join('\n')); } curlHandle.enable(CurlFeature.StreamResponse); @@ -76,89 +79,99 @@ export function fetchCurl( curlHandle.enable(CurlFeature.NoHeaderParsing); - return new Promise(function promiseResolver(resolve, reject) { - let streamResolved: Readable | undefined; - if (fetchRequest['_signal']) { - fetchRequest['_signal'].onabort = () => { - if (curlHandle.isOpen) { - try { - curlHandle.pause(CurlPause.Recv); - } catch (e) { - reject(e); - } - } - }; - } - curlHandle.once('end', function endListener() { - try { - curlHandle.close(); - } catch (e) { - reject(e); - } - }); - curlHandle.once('error', function errorListener(error: any) { - if (streamResolved && !streamResolved.closed && !streamResolved.destroyed) { - streamResolved.destroy(error); - } else { - if (error.message === 'Operation was aborted by an application callback') { - error.message = 'The operation was aborted.'; + const deferredPromise = createDeferredPromise>(); + let streamResolved: Readable | undefined; + if (fetchRequest['_signal']) { + fetchRequest['_signal'].onabort = () => { + if (curlHandle.isOpen) { + try { + curlHandle.pause(CurlPause.Recv); + } catch (e) { + deferredPromise.reject(e); } - reject(error); } - try { - curlHandle.close(); - } catch (e) { - reject(e); + }; + } + curlHandle.once('end', function endListener() { + try { + curlHandle.close(); + } catch (e) { + deferredPromise.reject(e); + } + }); + curlHandle.once('error', function errorListener(error: any) { + if (streamResolved && !streamResolved.closed && !streamResolved.destroyed) { + streamResolved.destroy(error); + } else { + if (error.message === 'Operation was aborted by an application callback') { + error.message = 'The operation was aborted.'; } - }); - curlHandle.once( - 'stream', - function streamListener(stream: Readable, status: number, headersBuf: Buffer) { - const outputStream = new PassThrough(); - - streamPromises - .pipeline(stream, outputStream, { - end: true, - signal: fetchRequest['_signal'] ?? undefined, - }) - .then(() => { - if (!stream.destroyed) { - stream.resume(); - } - }) - .catch(reject); - const headersFlat = headersBuf - .toString('utf8') - .split(/\r?\n|\r/g) - .filter(headerFilter => { - if (headerFilter && !headerFilter.startsWith('HTTP/')) { - if ( - fetchRequest.redirect === 'error' && - (headerFilter.includes('location') || headerFilter.includes('Location')) - ) { - if (!stream.destroyed) { - stream.resume(); - } - outputStream.destroy(); - reject(new Error('redirect is not allowed')); + deferredPromise.reject(error); + } + try { + curlHandle.close(); + } catch (e) { + deferredPromise.reject(e); + } + }); + curlHandle.once( + 'stream', + function streamListener(stream: Readable, status: number, headersBuf: Buffer) { + const outputStream = new PassThrough(); + + streamPromises + .pipeline(stream, outputStream, { + end: true, + signal: fetchRequest['_signal'] ?? undefined, + }) + .then(() => { + if (!stream.destroyed) { + stream.resume(); + } + }) + .catch(deferredPromise.reject); + const headersFlat = headersBuf + .toString('utf8') + .split(/\r?\n|\r/g) + .filter(headerFilter => { + if (headerFilter && !headerFilter.startsWith('HTTP/')) { + if ( + fetchRequest.redirect === 'error' && + (headerFilter.includes('location') || headerFilter.includes('Location')) + ) { + if (!stream.destroyed) { + stream.resume(); } - return true; + outputStream.destroy(); + deferredPromise.reject(new Error('redirect is not allowed')); } - return false; - }); - const headersInit = headersFlat.map( - headerFlat => headerFlat.split(/:\s(.+)/).slice(0, 2) as [string, string], - ); - const ponyfillResponse = new PonyfillResponse(outputStream, { - status, - headers: headersInit, - url: curlHandle.getInfo(Curl.info.REDIRECT_URL)?.toString() || fetchRequest.url, - redirected: Number(curlHandle.getInfo(Curl.info.REDIRECT_COUNT)) > 0, + return true; + } + return false; }); - resolve(ponyfillResponse); - streamResolved = outputStream; - }, - ); + const headersInit = headersFlat.map( + headerFlat => headerFlat.split(/:\s(.+)/).slice(0, 2) as [string, string], + ); + const ponyfillResponse = new PonyfillResponse(outputStream, { + status, + headers: headersInit, + url: curlHandle.getInfo(Curl.info.REDIRECT_URL)?.toString() || fetchRequest.url, + redirected: Number(curlHandle.getInfo(Curl.info.REDIRECT_COUNT)) > 0, + }); + deferredPromise.resolve(ponyfillResponse); + streamResolved = outputStream; + }, + ); + let count = 0; + try { + count = Curl.getCount(); + } catch {} + if (count > 0) { + setImmediate(() => { + curlHandle.perform(); + }); + } else { curlHandle.perform(); - }); + } + return deferredPromise.promise; } diff --git a/packages/node-fetch/src/utils.ts b/packages/node-fetch/src/utils.ts index fe0c2e2b8af..811f41bf0e2 100644 --- a/packages/node-fetch/src/utils.ts +++ b/packages/node-fetch/src/utils.ts @@ -74,3 +74,27 @@ export function isArrayBufferView(obj: any): obj is ArrayBufferView { export function isNodeReadable(obj: any): obj is Readable { return obj != null && obj.pipe != null; } + +export interface DeferredPromise { + promise: Promise; + resolve: (value: T) => void; + reject: (reason: any) => void; +} + +export function createDeferredPromise(): DeferredPromise { + let resolveFn: (value: T) => void; + let rejectFn: (reason: any) => void; + const promise = new Promise(function deferredPromiseExecutor(resolve, reject) { + resolveFn = resolve; + rejectFn = reject; + }); + return { + promise, + get resolve() { + return resolveFn; + }, + get reject() { + return rejectFn; + }, + }; +} diff --git a/yarn.lock b/yarn.lock index d0e8fa9f69d..ac159f17086 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1966,7 +1966,7 @@ globby "^11.0.0" read-yaml-file "^1.1.0" -"@mapbox/node-pre-gyp@1.0.11": +"@mapbox/node-pre-gyp@^1.0.11": version "1.0.11" resolved "https://registry.yarnpkg.com/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz#417db42b7f5323d79e93b34a6d7a2a12c0df43fa" integrity sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ== @@ -7281,10 +7281,10 @@ mvdan-sh@^0.10.1: resolved "https://registry.yarnpkg.com/mvdan-sh/-/mvdan-sh-0.10.1.tgz#5b3a4462a89cf20739b12d851589342c875f4d1f" integrity sha512-kMbrH0EObaKmK3nVRKUIIya1dpASHIEusM13S4V1ViHFuxuNxCo+arxoa6j/dbV22YBGjl7UKJm9QQKJ2Crzhg== -nan@2.18.0: - version "2.18.0" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.18.0.tgz#26a6faae7ffbeb293a39660e88a76b82e30b7554" - integrity sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w== +nan@^2.20.0: + version "2.20.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.20.0.tgz#08c5ea813dd54ed16e5bd6505bf42af4f7838ca3" + integrity sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw== nanoid@^3.3.3, nanoid@^3.3.6: version "3.3.7" @@ -7336,7 +7336,7 @@ node-forge@^1: resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA== -node-gyp@10.0.1, node-gyp@^10.0.0: +node-gyp@^10.0.0: version "10.0.1" resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-10.0.1.tgz#205514fc19e5830fa991e4a689f9e81af377a966" integrity sha512-gg3/bHehQfZivQVfqIyy8wTdSymF9yTyP4CJifK73imyNMU8AIGQE2pUa7dNWfmMeG9cDVF2eehiRMv0LC1iAg== @@ -7352,23 +7352,39 @@ node-gyp@10.0.1, node-gyp@^10.0.0: tar "^6.1.2" which "^4.0.0" +node-gyp@^10.2.0: + version "10.2.0" + resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-10.2.0.tgz#80101c4aa4f7ab225f13fcc8daaaac4eb1a8dd86" + integrity sha512-sp3FonBAaFe4aYTcFdZUn2NYkbP7xroPGYvQmP4Nl5PxamznItBnNCgjrVTKrEfQynInMsJvZrdmqUnysCJ8rw== + dependencies: + env-paths "^2.2.0" + exponential-backoff "^3.1.1" + glob "^10.3.10" + graceful-fs "^4.2.6" + make-fetch-happen "^13.0.0" + nopt "^7.0.0" + proc-log "^4.1.0" + semver "^7.3.5" + tar "^6.2.1" + which "^4.0.0" + node-int64@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" integrity sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw== -node-libcurl@4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/node-libcurl/-/node-libcurl-4.0.0.tgz#57568c3f7c08a76e822994d2b674d4dd56752127" - integrity sha512-v+u+OgSq6ldvf8MrdjieAy/mv8WeTN94nrTomh62zhItF2HH0Ckin/QEqs8+35DWyYrE5nBM2480UtWVXktzbQ== +"node-libcurl@npm:@ardatan/node-libcurl@4.0.2": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@ardatan/node-libcurl/-/node-libcurl-4.0.2.tgz#c7ee83254e73b277420869d4b59dd5e2fda7bd01" + integrity sha512-H1XB1bFiAxA5dc8vRnQEGC4hsbVVATAdhR8trUcG+MivG98aPB3BrkE2j3qEE3RMHoo3uWx5tzctTnoCXIM8fA== dependencies: - "@mapbox/node-pre-gyp" "1.0.11" + "@mapbox/node-pre-gyp" "^1.0.11" env-paths "2.2.0" - nan "2.18.0" - node-gyp "10.0.1" - npmlog "7.0.1" - rimraf "5.0.5" - tslib "2.6.2" + nan "^2.20.0" + node-gyp "^10.2.0" + npmlog "^7.0.1" + rimraf "^5.0.5" + tslib "^2.6.2" node-releases@^2.0.18: version "2.0.18" @@ -7482,16 +7498,6 @@ npm-run-path@^5.1.0: dependencies: path-key "^4.0.0" -npmlog@7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-7.0.1.tgz#7372151a01ccb095c47d8bf1d0771a4ff1f53ac8" - integrity sha512-uJ0YFk/mCQpLBt+bxN88AKd+gyqZvZDbtiNxk6Waqcj2aPRyfVx8ITawkyQynxUagInjdYT1+qj4NfA5KJJUxg== - dependencies: - are-we-there-yet "^4.0.0" - console-control-strings "^1.1.0" - gauge "^5.0.0" - set-blocking "^2.0.0" - npmlog@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-5.0.1.tgz#f06678e80e29419ad67ab964e0fa69959c1eb8b0" @@ -7502,6 +7508,16 @@ npmlog@^5.0.1: gauge "^3.0.0" set-blocking "^2.0.0" +npmlog@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-7.0.1.tgz#7372151a01ccb095c47d8bf1d0771a4ff1f53ac8" + integrity sha512-uJ0YFk/mCQpLBt+bxN88AKd+gyqZvZDbtiNxk6Waqcj2aPRyfVx8ITawkyQynxUagInjdYT1+qj4NfA5KJJUxg== + dependencies: + are-we-there-yet "^4.0.0" + console-control-strings "^1.1.0" + gauge "^5.0.0" + set-blocking "^2.0.0" + object-assign@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" @@ -8472,13 +8488,6 @@ rfdc@^1.2.0, rfdc@^1.3.0, rfdc@^1.4.1: resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.4.1.tgz#778f76c4fb731d93414e8f925fbecf64cce7f6ca" integrity sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA== -rimraf@5.0.5: - version "5.0.5" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-5.0.5.tgz#9be65d2d6e683447d2e9013da2bf451139a61ccf" - integrity sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A== - dependencies: - glob "^10.3.7" - rimraf@6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-6.0.1.tgz#ffb8ad8844dd60332ab15f52bc104bc3ed71ea4e" @@ -8501,6 +8510,13 @@ rimraf@^3.0.2: dependencies: glob "^7.1.3" +rimraf@^5.0.5: + version "5.0.10" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-5.0.10.tgz#23b9843d3dc92db71f96e1a2ce92e39fd2a8221c" + integrity sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ== + dependencies: + glob "^10.3.7" + rollup-plugin-inject@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/rollup-plugin-inject/-/rollup-plugin-inject-3.0.2.tgz#e4233855bfba6c0c12a312fd6649dff9a13ee9f4" @@ -9171,7 +9187,7 @@ tapable@^2.2.0: resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== -tar@^6.1.11, tar@^6.1.2: +tar@^6.1.11, tar@^6.1.2, tar@^6.2.1: version "6.2.1" resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.1.tgz#717549c541bc3c2af15751bea94b1dd068d4b03a" integrity sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A== @@ -9321,11 +9337,6 @@ tsconfig-paths@^3.15.0: minimist "^1.2.6" strip-bom "^3.0.0" -tslib@2.6.2: - version "2.6.2" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" - integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== - tslib@^2.0.0, tslib@^2.1.0, tslib@^2.2.0, tslib@^2.4.0, tslib@^2.6.2, tslib@^2.6.3: version "2.7.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.7.0.tgz#d9b40c5c40ab59e8738f297df3087bf1a2690c01" @@ -9426,7 +9437,6 @@ typescript@5.5.4: uWebSockets.js@uNetworking/uWebSockets.js#v20.48.0: version "20.48.0" - uid "51ae1d1fd92dff77cbbdc7c431021f85578da1a6" resolved "https://codeload.github.com/uNetworking/uWebSockets.js/tar.gz/51ae1d1fd92dff77cbbdc7c431021f85578da1a6" ufo@^1.5.4: