From 66176823305a0ebef1fba6c7ff04620a838b49f6 Mon Sep 17 00:00:00 2001
From: Divy Srivastava <dj.srivastava23@gmail.com>
Date: Fri, 11 Aug 2023 13:04:23 +0530
Subject: [PATCH] fix(node): object keys in publicEncrypt (#20128)

Fixes https://github.com/denoland/deno/issues/19935
---
 .../unit_node/crypto/crypto_cipher_test.ts    | 15 ++++++
 ext/node/polyfills/internal/crypto/cipher.ts  | 48 +++++++++++++++++--
 ext/node/polyfills/internal/crypto/keys.ts    |  2 +-
 3 files changed, 60 insertions(+), 5 deletions(-)

diff --git a/cli/tests/unit_node/crypto/crypto_cipher_test.ts b/cli/tests/unit_node/crypto/crypto_cipher_test.ts
index a8a5130cf15d57..417f5035cc1e90 100644
--- a/cli/tests/unit_node/crypto/crypto_cipher_test.ts
+++ b/cli/tests/unit_node/crypto/crypto_cipher_test.ts
@@ -33,6 +33,21 @@ Deno.test({
   },
 });
 
+Deno.test({
+  name: "rsa public encrypt (options) and private decrypt",
+  fn() {
+    const encrypted = crypto.publicEncrypt(
+      { key: Buffer.from(rsaPublicKey) },
+      input,
+    );
+    const decrypted = crypto.privateDecrypt(
+      Buffer.from(rsaPrivateKey),
+      Buffer.from(encrypted),
+    );
+    assertEquals(decrypted, input);
+  },
+});
+
 Deno.test({
   name: "rsa private encrypt and private decrypt",
   fn() {
diff --git a/ext/node/polyfills/internal/crypto/cipher.ts b/ext/node/polyfills/internal/crypto/cipher.ts
index e954562c58cc8e..3a8b41f0697aed 100644
--- a/ext/node/polyfills/internal/crypto/cipher.ts
+++ b/ext/node/polyfills/internal/crypto/cipher.ts
@@ -13,13 +13,26 @@ import { Buffer } from "node:buffer";
 import { notImplemented } from "ext:deno_node/_utils.ts";
 import type { TransformOptions } from "ext:deno_node/_stream.d.ts";
 import { Transform } from "ext:deno_node/_stream.mjs";
-import { KeyObject } from "./keys.ts";
+import {
+  getArrayBufferOrView,
+  KeyObject,
+} from "ext:deno_node/internal/crypto/keys.ts";
 import type { BufferEncoding } from "ext:deno_node/_global.d.ts";
 import type {
   BinaryLike,
   Encoding,
 } from "ext:deno_node/internal/crypto/types.ts";
 import { getDefaultEncoding } from "ext:deno_node/internal/crypto/util.ts";
+import {
+  isAnyArrayBuffer,
+  isArrayBufferView,
+} from "ext:deno_node/internal/util/types.ts";
+
+function isStringOrBuffer(val) {
+  return typeof val === "string" ||
+    isArrayBufferView(val) ||
+    isAnyArrayBuffer(val);
+}
 
 const { ops, encode } = globalThis.__bootstrap.core;
 
@@ -355,24 +368,51 @@ export function privateEncrypt(
   privateKey: ArrayBufferView | string | KeyObject,
   buffer: ArrayBufferView | string | KeyObject,
 ): Buffer {
+  const { data } = prepareKey(privateKey);
   const padding = privateKey.padding || 1;
-  return ops.op_node_private_encrypt(privateKey, buffer, padding);
+
+  buffer = getArrayBufferOrView(buffer, "buffer");
+  return ops.op_node_private_encrypt(data, buffer, padding);
 }
 
 export function privateDecrypt(
   privateKey: ArrayBufferView | string | KeyObject,
   buffer: ArrayBufferView | string | KeyObject,
 ): Buffer {
+  const { data } = prepareKey(privateKey);
   const padding = privateKey.padding || 1;
-  return ops.op_node_private_decrypt(privateKey, buffer, padding);
+
+  buffer = getArrayBufferOrView(buffer, "buffer");
+  return ops.op_node_private_decrypt(data, buffer, padding);
 }
 
 export function publicEncrypt(
   publicKey: ArrayBufferView | string | KeyObject,
   buffer: ArrayBufferView | string | KeyObject,
 ): Buffer {
+  const { data } = prepareKey(publicKey);
   const padding = publicKey.padding || 1;
-  return ops.op_node_public_encrypt(publicKey, buffer, padding);
+
+  buffer = getArrayBufferOrView(buffer, "buffer");
+  return ops.op_node_public_encrypt(data, buffer, padding);
+}
+
+function prepareKey(key) {
+  // TODO(@littledivy): handle these cases
+  // - node KeyObject
+  // - web CryptoKey
+  if (isStringOrBuffer(key)) {
+    return { data: getArrayBufferOrView(key, "key") };
+  } else if (typeof key == "object") {
+    const { key: data, encoding } = key;
+    if (!isStringOrBuffer(data)) {
+      throw new TypeError("Invalid key type");
+    }
+
+    return { data: getArrayBufferOrView(data, "key", encoding) };
+  }
+
+  throw new TypeError("Invalid key type");
 }
 
 export function publicDecrypt() {
diff --git a/ext/node/polyfills/internal/crypto/keys.ts b/ext/node/polyfills/internal/crypto/keys.ts
index f6e5cbb86c4911..be85b44a3e7454 100644
--- a/ext/node/polyfills/internal/crypto/keys.ts
+++ b/ext/node/polyfills/internal/crypto/keys.ts
@@ -39,7 +39,7 @@ import {
   forgivingBase64UrlEncode as encodeToBase64Url,
 } from "ext:deno_web/00_infra.js";
 
-const getArrayBufferOrView = hideStackFrames(
+export const getArrayBufferOrView = hideStackFrames(
   (
     buffer,
     name,