fix(node-fetch/node-libcurl): SSL and async handling issues (#1617)

.changeset/nine-bugs-shop.md

@@ -0,0 +1,11 @@
+---
+'@whatwg-node/node-fetch': patch
+---
+
+# Fixes for usage of `node-libcurl`
+
+- Fix \`Error: SSL peer certificate or SSH remove key was not ok error\`, and use `tls.rootCertificates` as default certificates.
+
+[Learn more](https://github.com/JCMais/node-libcurl/blob/develop/COMMON_ISSUES.md)
+
+- Fix `API function called from within callback` by preventing the use of `curl_easy_perform` and `curl_multi_perform` inside callbacks.

.npmrc

@@ -1 +1 @@
-provenance=true
+provenance=true

package.json

@@ -26,7 +26,6 @@
     "ts:check": "tsc --noEmit"
   },
   "optionalDependencies": {
-    "node-libcurl": "4.0.0",
     "uWebSockets.js": "uNetworking/uWebSockets.js#v20.48.0"
   },
   "devDependencies": {
@@ -56,6 +55,7 @@
     "husky": "9.1.5",
     "jest": "29.7.0",
     "lint-staged": "15.2.9",
+    "node-libcurl": "npm:@ardatan/[email protected]",
     "patch-package": "8.0.0",
     "prettier": "3.3.3",
     "rimraf": "6.0.1",

packages/node-fetch/src/fetchCurl.ts

@@ -1,7 +1,8 @@
 import { PassThrough, Readable, promises as streamPromises } from 'stream';
+import { rootCertificates } from 'tls';
 import { PonyfillRequest } from './Request.js';
 import { PonyfillResponse } from './Response.js';
-import { defaultHeadersSerializer, isNodeReadable } from './utils.js';
+import { createDeferredPromise, defaultHeadersSerializer, isNodeReadable } from './utils.js';
 
 export function fetchCurl<TResponseJSON = any, TRequestJSON = any>(
   fetchRequest: PonyfillRequest<TRequestJSON>,
@@ -20,6 +21,8 @@ export function fetchCurl<TResponseJSON = any, TRequestJSON = any>(
 
   if (process.env.NODE_EXTRA_CA_CERTS) {
     curlHandle.setOpt('CAINFO', process.env.NODE_EXTRA_CA_CERTS);
+  } else {
+    curlHandle.setOpt('CAINFO_BLOB', rootCertificates.join('\n'));
   }
 
   curlHandle.enable(CurlFeature.StreamResponse);
@@ -76,89 +79,99 @@ export function fetchCurl<TResponseJSON = any, TRequestJSON = any>(
 
   curlHandle.enable(CurlFeature.NoHeaderParsing);
 
-  return new Promise(function promiseResolver(resolve, reject) {
-    let streamResolved: Readable | undefined;
-    if (fetchRequest['_signal']) {
-      fetchRequest['_signal'].onabort = () => {
-        if (curlHandle.isOpen) {
-          try {
-            curlHandle.pause(CurlPause.Recv);
-          } catch (e) {
-            reject(e);
-          }
-        }
-      };
-    }
-    curlHandle.once('end', function endListener() {
-      try {
-        curlHandle.close();
-      } catch (e) {
-        reject(e);
-      }
-    });
-    curlHandle.once('error', function errorListener(error: any) {
-      if (streamResolved && !streamResolved.closed && !streamResolved.destroyed) {
-        streamResolved.destroy(error);
-      } else {
-        if (error.message === 'Operation was aborted by an application callback') {
-          error.message = 'The operation was aborted.';
+  const deferredPromise = createDeferredPromise<PonyfillResponse<TResponseJSON>>();
+  let streamResolved: Readable | undefined;
+  if (fetchRequest['_signal']) {
+    fetchRequest['_signal'].onabort = () => {
+      if (curlHandle.isOpen) {
+        try {
+          curlHandle.pause(CurlPause.Recv);
+        } catch (e) {
+          deferredPromise.reject(e);
         }
-        reject(error);
       }
-      try {
-        curlHandle.close();
-      } catch (e) {
-        reject(e);
+    };
+  }
+  curlHandle.once('end', function endListener() {
+    try {
+      curlHandle.close();
+    } catch (e) {
+      deferredPromise.reject(e);
+    }
+  });
+  curlHandle.once('error', function errorListener(error: any) {
+    if (streamResolved && !streamResolved.closed && !streamResolved.destroyed) {
+      streamResolved.destroy(error);
+    } else {
+      if (error.message === 'Operation was aborted by an application callback') {
+        error.message = 'The operation was aborted.';
       }
-    });
-    curlHandle.once(
-      'stream',
-      function streamListener(stream: Readable, status: number, headersBuf: Buffer) {
-        const outputStream = new PassThrough();
-
-        streamPromises
-          .pipeline(stream, outputStream, {
-            end: true,
-            signal: fetchRequest['_signal'] ?? undefined,
-          })
-          .then(() => {
-            if (!stream.destroyed) {
-              stream.resume();
-            }
-          })
-          .catch(reject);
-        const headersFlat = headersBuf
-          .toString('utf8')
-          .split(/\r?\n|\r/g)
-          .filter(headerFilter => {
-            if (headerFilter && !headerFilter.startsWith('HTTP/')) {
-              if (
-                fetchRequest.redirect === 'error' &&
-                (headerFilter.includes('location') || headerFilter.includes('Location'))
-              ) {
-                if (!stream.destroyed) {
-                  stream.resume();
-                }
-                outputStream.destroy();
-                reject(new Error('redirect is not allowed'));
+      deferredPromise.reject(error);
+    }
+    try {
+      curlHandle.close();
+    } catch (e) {
+      deferredPromise.reject(e);
+    }
+  });
+  curlHandle.once(
+    'stream',
+    function streamListener(stream: Readable, status: number, headersBuf: Buffer) {
+      const outputStream = new PassThrough();
+
+      streamPromises
+        .pipeline(stream, outputStream, {
+          end: true,
+          signal: fetchRequest['_signal'] ?? undefined,
+        })
+        .then(() => {
+          if (!stream.destroyed) {
+            stream.resume();
+          }
+        })
+        .catch(deferredPromise.reject);
+      const headersFlat = headersBuf
+        .toString('utf8')
+        .split(/\r?\n|\r/g)
+        .filter(headerFilter => {
+          if (headerFilter && !headerFilter.startsWith('HTTP/')) {
+            if (
+              fetchRequest.redirect === 'error' &&
+              (headerFilter.includes('location') || headerFilter.includes('Location'))
+            ) {
+              if (!stream.destroyed) {
+                stream.resume();
               }
-              return true;
+              outputStream.destroy();
+              deferredPromise.reject(new Error('redirect is not allowed'));
             }
-            return false;
-          });
-        const headersInit = headersFlat.map(
-          headerFlat => headerFlat.split(/:\s(.+)/).slice(0, 2) as [string, string],
-        );
-        const ponyfillResponse = new PonyfillResponse(outputStream, {
-          status,
-          headers: headersInit,
-          url: curlHandle.getInfo(Curl.info.REDIRECT_URL)?.toString() || fetchRequest.url,
-          redirected: Number(curlHandle.getInfo(Curl.info.REDIRECT_COUNT)) > 0,
+            return true;
+          }
+          return false;
         });
-        resolve(ponyfillResponse);
-        streamResolved = outputStream;
-      },
-    );
+      const headersInit = headersFlat.map(
+        headerFlat => headerFlat.split(/:\s(.+)/).slice(0, 2) as [string, string],
+      );
+      const ponyfillResponse = new PonyfillResponse(outputStream, {
+        status,
+        headers: headersInit,
+        url: curlHandle.getInfo(Curl.info.REDIRECT_URL)?.toString() || fetchRequest.url,
+        redirected: Number(curlHandle.getInfo(Curl.info.REDIRECT_COUNT)) > 0,
+      });
+      deferredPromise.resolve(ponyfillResponse);
+      streamResolved = outputStream;
+    },
+  );
+  let count = 0;
+  try {
+    count = Curl.getCount();
+  } catch {}
+  if (count > 0) {
+    setImmediate(() => {
+      curlHandle.perform();
+    });
+  } else {
     curlHandle.perform();
-  });
+  }
+  return deferredPromise.promise;
 }

packages/node-fetch/src/utils.ts

@@ -74,3 +74,27 @@ export function isArrayBufferView(obj: any): obj is ArrayBufferView {
 export function isNodeReadable(obj: any): obj is Readable {
   return obj != null && obj.pipe != null;
 }
+
+export interface DeferredPromise<T = void> {
+  promise: Promise<T>;
+  resolve: (value: T) => void;
+  reject: (reason: any) => void;
+}
+
+export function createDeferredPromise<T = void>(): DeferredPromise<T> {
+  let resolveFn: (value: T) => void;
+  let rejectFn: (reason: any) => void;
+  const promise = new Promise<T>(function deferredPromiseExecutor(resolve, reject) {
+    resolveFn = resolve;
+    rejectFn = reject;
+  });
+  return {
+    promise,
+    get resolve() {
+      return resolveFn;
+    },
+    get reject() {
+      return rejectFn;
+    },
+  };
+}