Bind out crc64 and fix various hash bidning issues (#583)

.github/workflows/ci.yml

@@ -167,7 +167,7 @@ jobs:
   check-docs:
     runs-on: ubuntu-20.04 # latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Check docs
@@ -179,7 +179,7 @@ jobs:
     runs-on: ubuntu-20.04 # latest
     steps:
     - name: Checkout Source
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0
@@ -191,7 +191,7 @@ jobs:
   check-lockfile-version:
     runs-on: ubuntu-20.04 # latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Check for edits to package-lock.json

lib/browser/crypto.ts

@@ -12,25 +12,35 @@
  */
 
 import * as Crypto from "crypto-js";
-import { fromUtf8 } from "@aws-sdk/util-utf8-browser";
 import { Hashable } from "../common/crypto";
 
 export { Hashable } from "../common/crypto";
 
 /**
- * Object that allows for continuous MD5 hashing of data.
- *
- * @category Crypto
+ * CryptoJS does not provide easy access to underlying bytes.
+ * As a workaround just dump it to a string and then reinterpret chars as individual bytes.
+ * Note: we are using Latin1 here because its a static sized 8 bit encoding so each char maps directly to a byte value.
+ * TODO: long term we would probably want to move to WebCrypto for SHA's and some other 3p for crc's and md5.
+ * @param hash 
+ * @returns 
  */
-export class Md5Hash {
-    private hash?: Crypto.WordArray;
+function wordArrayToUint8Array(hash: Crypto.WordArray) {
+    return Uint8Array.from(hash.toString(Crypto.enc.Latin1).split('').map(c => c.charCodeAt(0)));;
+}
+
+class BaseHash {
+    private hasher : any;
+
+    constructor(hasher: any) {
+        this.hasher = hasher;
+    }
 
     /**
      * Hashes additional data
      * @param data Additional data to hash
      */
     update(data: Hashable) {
-        this.hash = Crypto.MD5(data.toString(), this.hash ? this.hash.toString() : undefined);
+        this.hasher.update(data.toString());
     }
 
     /**
@@ -41,10 +51,20 @@ export class Md5Hash {
      * @returns the final hash digest
      */
     finalize(truncate_to?: number): DataView {
-        const digest = this.hash ? this.hash.toString() : '';
-        const truncated = digest.substring(0, truncate_to ? truncate_to : digest.length);
-        const bytes = fromUtf8(truncated);
-        return new DataView(bytes.buffer);
+        const hashBuffer = wordArrayToUint8Array(this.hasher.finalize()) ;
+        const truncated = hashBuffer.slice(0, truncate_to ? truncate_to : hashBuffer.length);
+        return new DataView(truncated.buffer);;
+    }
+}
+
+/**
+ * Object that allows for continuous MD5 hashing of data.
+ *
+ * @category Crypto
+ */
+export class Md5Hash extends BaseHash {
+    constructor() {
+        super(Crypto.algo.MD5.create());
     }
 }
 
@@ -71,29 +91,9 @@ export function hash_md5(data: Hashable, truncate_to?: number): DataView {
  *
  * @category Crypto
  */
-export class Sha256Hash {
-    private hash?: Crypto.WordArray;
-
-    /**
-     * Hashes additional data
-     * @param data Additional data to hash
-     */
-    update(data: Hashable) {
-        this.hash = Crypto.SHA256(data.toString(), this.hash ? this.hash.toString() : undefined);
-    }
-
-    /**
-     * Completes the hash computation and returns the final hash digest.
-     *
-     * @param truncate_to The maximum number of bytes to receive. Leave as undefined or 0 to receive the entire digest.
-     *
-     * @returns the final hash digest
-     */
-    finalize(truncate_to?: number): DataView {
-        const digest = this.hash ? this.hash.toString() : '';
-        const truncated = digest.substring(0, truncate_to ? truncate_to : digest.length);
-        const bytes = fromUtf8(truncated);
-        return new DataView(bytes.buffer);
+export class Sha256Hash extends BaseHash {
+    constructor() {
+        super(Crypto.algo.SHA256.create());
     }
 }
 
@@ -109,40 +109,19 @@ export class Sha256Hash {
  * @category Crypto
  */
 export function hash_sha256(data: Hashable, truncate_to?: number): DataView {
-    const digest = Crypto.SHA256(data.toString()).toString();
-    const truncated = digest.substring(0, truncate_to ? truncate_to : digest.length);
-    const bytes = fromUtf8(truncated);
-    return new DataView(bytes.buffer);
+    const sha256 = new Sha256Hash();
+    sha256.update(data);
+    return sha256.finalize(truncate_to);
 }
 
 /**
  * Object that allows for continuous SHA1 hashing of data.
  *
  * @category Crypto
  */
- export class Sha1Hash {
-    private hash?: Crypto.WordArray;
-
-    /**
-     * Hashes additional data
-     * @param data Additional data to hash
-     */
-    update(data: Hashable) {
-        this.hash = Crypto.SHA1(data.toString(), this.hash ? this.hash.toString() : undefined);
-    }
-
-    /**
-     * Completes the hash computation and returns the final hash digest.
-     *
-     * @param truncate_to The maximum number of bytes to receive. Leave as undefined or 0 to receive the entire digest.
-     *
-     * @returns the final hash digest
-     */
-    finalize(truncate_to?: number): DataView {
-        const digest = this.hash ? this.hash.toString() : '';
-        const truncated = digest.substring(0, truncate_to ? truncate_to : digest.length);
-        const bytes = fromUtf8(truncated);
-        return new DataView(bytes.buffer);
+ export class Sha1Hash extends BaseHash {
+    constructor() {
+        super(Crypto.algo.SHA1.create());
     }
 }
 
@@ -158,49 +137,24 @@ export function hash_sha256(data: Hashable, truncate_to?: number): DataView {
  * @category Crypto
  */
 export function hash_sha1(data: Hashable, truncate_to?: number): DataView {
-    const digest = Crypto.SHA1(data.toString()).toString();
-    const truncated = digest.substring(0, truncate_to ? truncate_to : digest.length);
-    const bytes = fromUtf8(truncated);
-    return new DataView(bytes.buffer);
+    const sha1 = new Sha1Hash();
+    sha1.update(data);
+    return sha1.finalize(truncate_to);
 }
 
 /**
  * Object that allows for continuous hashing of data with an hmac secret.
  *
  * @category Crypto
  */
-export class Sha256Hmac {
-    private hmac: any;
-
+export class Sha256Hmac extends BaseHash {
     /**
      * Constructor for the Sha256Hmac class type
      * @param secret secret key to seed the hmac process with
      */
     constructor(secret: Hashable) {
         // @ts-ignore types file doesn't have this signature of create()
-        this.hmac = Crypto.algo.HMAC.create(Crypto.algo.SHA256, secret);
-    }
-
-    /**
-     * Hashes additional data
-     * @param data Additional data to hash
-     */
-    update(data: Hashable) {
-        this.hmac.update(data.toString());
-    }
-
-    /**
-     * Completes the hash computation and returns the final hmac digest.
-     *
-     * @param truncate_to The maximum number of bytes to receive. Leave as undefined or 0 to receive the entire digest.
-     *
-     * @returns the final hmac digest
-     */
-    finalize(truncate_to?: number): DataView {
-        const digest = this.hmac.finalize();
-        const truncated = digest.toString().substring(0, truncate_to ? truncate_to : digest.length);
-        const bytes = fromUtf8(truncated);
-        return new DataView(bytes.buffer);
+        super(Crypto.algo.HMAC.create(Crypto.algo.SHA256, secret));
     }
 }
 

lib/native/binding.d.ts

@@ -142,6 +142,8 @@ export function hmac_sha256_compute(secret: StringLike, data: StringLike, trunca
 export function checksums_crc32(data: StringLike, previous?: number): number;
 /** @internal */
 export function checksums_crc32c(data: StringLike, previous?: number): number;
+/** @internal */
+export function checksums_crc64nvme(data: StringLike, previous?: DataView): DataView;
 
 /* MQTT5 Client */
 

lib/native/checksums.spec.ts

@@ -81,4 +81,21 @@ test('crc32c_large_buffer', () => {
     const output = checksums.crc32c(arr);
     const expected = 0xfb5b991d
     expect(output).toEqual(expected);
-});
+});
+
+test('crc64nvme_zeros_one_shot', () => {
+    const arr = new Uint8Array(32);
+    const output = checksums.crc64nvme(arr);
+    expect(output.getBigUint64(0)).toEqual(BigInt("0xCF3473434D4ECF3B"));
+});
+
+test('crc64nvme_zeros_iterated', () => {
+    const buffer = new ArrayBuffer(8);
+    let previous = new DataView(buffer);
+    previous.setBigUint64(0, BigInt(0));
+
+    for (let i = 0; i < 32; i++) {
+        previous = checksums.crc64nvme(new Uint8Array(1), previous);
+    }
+    expect(previous.getBigUint64(0)).toEqual(BigInt("0xCF3473434D4ECF3B"));
+});

lib/native/checksums.ts

@@ -38,4 +38,16 @@ export function crc32(data: Hashable, previous?: number): number {
  */
  export function crc32c(data: Hashable, previous?: number): number {
     return crt_native.checksums_crc32c(data, previous);
-}
+}
+
+/**
+ * Computes a crc64nvme checksum.
+ *
+ * @param data The data to checksum
+ * @param previous previous crc64nvme checksum result. Used if you are buffering large input.
+ *
+ * @category Crypto
+ */
+export function crc64nvme(data: Hashable, previous?: DataView): DataView {
+    return crt_native.checksums_crc64nvme(data, previous);
+}

lib/native/crypto.spec.ts

@@ -5,6 +5,7 @@
 
 import * as native from './crypto';
 import * as browser from '../browser/crypto';
+import '@test/custom_matchers'
 
 test('md5 multi-part matches', () => {
     const parts = ['ABC', '123', 'XYZ'];
@@ -17,15 +18,15 @@ test('md5 multi-part matches', () => {
     const native_hash = native_md5.finalize();
     const browser_hash = browser_md5.finalize();
 
-    expect(native_hash).toEqual(browser_hash);
+    expect(native_hash).toEqualDataView(browser_hash);
 });
 
 test('md5 one-shot matches', () => {
     const data = 'ABC123XYZ';
     const native_hash = native.hash_md5(data);
     const browser_hash = browser.hash_md5(data);
 
-    expect(native_hash).toEqual(browser_hash);
+    expect(native_hash).toEqualDataView(browser_hash);
 });
 
 test('SHA256 multi-part matches', () => {
@@ -39,15 +40,15 @@ test('SHA256 multi-part matches', () => {
     const native_hash = native_sha.finalize();
     const browser_hash = browser_sha.finalize();
 
-    expect(native_hash).toEqual(browser_hash);
+    expect(native_hash).toEqualDataView(browser_hash);
 });
 
 test('SHA256 one-shot matches', () => {
     const data = 'ABC123XYZ';
     const native_hash = native.hash_sha256(data);
     const browser_hash = browser.hash_sha256(data);
 
-    expect(native_hash).toEqual(browser_hash);
+    expect(native_hash).toEqualDataView(browser_hash);
 });
 
 test('SHA1 multi-part matches', () => {
@@ -61,15 +62,15 @@ test('SHA1 multi-part matches', () => {
     const native_hash = native_sha.finalize();
     const browser_hash = browser_sha.finalize();
 
-    expect(native_hash).toEqual(browser_hash);
+    expect(native_hash).toEqualDataView(browser_hash);
 });
 
 test('SHA1 one-shot matches', () => {
     const data = 'ABC123XYZ';
     const native_hash = native.hash_sha1(data);
     const browser_hash = browser.hash_sha1(data);
 
-    expect(native_hash).toEqual(browser_hash);
+    expect(native_hash).toEqualDataView(browser_hash);
 });
 
 test('hmac-256 multi-part matches', () => {
@@ -84,7 +85,7 @@ test('hmac-256 multi-part matches', () => {
     const native_hash = native_hmac.finalize();
     const browser_hash = browser_hmac.finalize();
 
-    expect(native_hash).toEqual(browser_hash);
+    expect(native_hash).toEqualDataView(browser_hash);
 });
 
 test('hmac-256 one-shot matches', () => {
@@ -93,5 +94,5 @@ test('hmac-256 one-shot matches', () => {
     const native_hash = native.hmac_sha256(secret, data);
     const browser_hash = browser.hmac_sha256(secret, data);
 
-    expect(native_hash).toEqual(browser_hash);
+    expect(native_hash).toEqualDataView(browser_hash);
 });

lib/native/crypto.ts

@@ -105,7 +105,6 @@ export function hash_sha256(data: Hashable, truncate_to?: number): DataView {
     }
 }
 
-
 /**
  * Computes an SHA1 hash. Use this if you don't need to stream the data you're hashing and can load the entire input
  * into memory.