Add varint reader/writer - Closes #5219 & #5221 & #5229 & #5230

elements/lisk-codec/.eslintignore

@@ -1,2 +1,3 @@
 dist-node
 jest.config.js
+benchmark

elements/lisk-codec/benchmark/varint.js

@@ -0,0 +1,61 @@
+/*
+ * Copyright © 2020 Lisk Foundation
+ *
+ * See the LICENSE file at the top-level directory of this distribution
+ * for licensing information.
+ *
+ * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
+ * no part of this software, including this file, may be copied, modified,
+ * propagated, or distributed except according to the terms contained in the
+ * LICENSE file.
+ *
+ * Removal or modification of this copyright notice is prohibited.
+ */
+
+const { Suite } = require('benchmark');
+const {
+    writeVarInt,
+    readVarInt,
+    writeSignedVarInt,
+    readSignedVarInt,
+} = require('../dist-node/varint');
+
+const suite = new Suite();
+
+suite
+    .add('writeVarInt#uint32', () => {
+        writeVarInt(2147483647, { dataType: 'uint32' });
+    })
+    .add('writeVarInt#uint64', () => {
+        writeVarInt(BigInt('8294967295'), { dataType: 'uint64' });
+    })
+    .add('writeSignedVarInt#sint32', () => {
+        writeSignedVarInt(2147483647, { dataType: 'sint32' });
+    })
+    .add('writeSignedVarInt#sint64', () => {
+        writeSignedVarInt(BigInt('9223372036854775807'), {
+            dataType: 'sint64',
+        });
+    })
+    .add('readVarInt#uint32', () => {
+        readVarInt(Buffer.from('ffffffff0f', 'hex'), { dataType: 'uint32' });
+    })
+    .add('readVarInt#uint64', () => {
+        readVarInt(Buffer.from('ffffffffffffffffff01', 'hex'), {
+            dataType: 'uint64',
+        });
+    })
+    .add('readSignedVarInt#sint32', () => {
+        readSignedVarInt(Buffer.from('feffffff0f', 'hex'), {
+            dataType: 'sint32',
+        });
+    })
+    .add('readSignedVarInt#sint64', () => {
+        readSignedVarInt(Buffer.from('ffffffffffffffffff01', 'hex'), {
+            dataType: 'sint64',
+        });
+    })
+    .on('cycle', function (event) {
+        console.log(String(event.target));
+    })
+    .run({ async: true });

elements/lisk-codec/package.json

@@ -35,11 +35,12 @@
 	},
 	"dependencies": {},
 	"devDependencies": {
-		"@types/node": "12.12.11",
 		"@types/jest": "25.1.3",
 		"@types/jest-when": "2.7.0",
+		"@types/node": "12.12.11",
 		"@typescript-eslint/eslint-plugin": "2.28.0",
 		"@typescript-eslint/parser": "2.28.0",
+		"benchmark": "2.1.4",
 		"eslint": "6.8.0",
 		"eslint-config-lisk-base": "1.2.2",
 		"eslint-config-prettier": "6.10.0",

elements/lisk-codec/src/varint.ts

@@ -0,0 +1,141 @@
+/*
+ * Copyright © 2020 Lisk Foundation
+ *
+ * See the LICENSE file at the top-level directory of this distribution
+ * for licensing information.
+ *
+ * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
+ * no part of this software, including this file, may be copied, modified,
+ * propagated, or distributed except according to the terms contained in the
+ * LICENSE file.
+ *
+ * Removal or modification of this copyright notice is prohibited.
+ */
+/* eslint-disable no-bitwise */
+/* eslint-disable no-param-reassign */
+
+const msg = 0x80;
+const rest = 0x7f;
+
+type int = bigint | number;
+
+const isNumber = (val: int): val is number => typeof val === 'number';
+
+interface SchemaProperty {
+	readonly dataType: string;
+}
+
+const writeVarIntNumber = (value: number): Buffer => {
+	const result: number[] = [];
+	let index = 0;
+	while (value > rest) {
+		result[index] = msg | ((value & rest) >>> 0);
+		value = (value >>> 7) >>> 0;
+		index += 1;
+	}
+
+	result[index] = value;
+
+	return Buffer.from(result);
+};
+
+const writeVarIntBigInt = (value: bigint): Buffer => {
+	const result: number[] = [];
+	let index = 0;
+	while (value > BigInt(rest)) {
+		result[index] = Number(BigInt(msg) | (value & BigInt(rest)));
+		value >>= BigInt(7);
+		index += 1;
+	}
+
+	result[Number(index)] = Number(value);
+
+	return Buffer.from(result);
+};
+
+export const writeVarInt = (value: int, _schema: SchemaProperty): Buffer =>
+	isNumber(value) ? writeVarIntNumber(value) : writeVarIntBigInt(value);
+
+export const writeSignedVarInt = (
+	value: int,
+	schema: SchemaProperty,
+): Buffer => {
+	if (schema.dataType === 'sint32') {
+		const number = Number(value);
+		if (number >= 0) {
+			return writeVarIntNumber(2 * number);
+		}
+		return writeVarIntNumber(-2 * number - 1);
+	}
+	const number = BigInt(value);
+	if (number >= BigInt(0)) {
+		return writeVarInt(BigInt(2) * number, schema);
+	}
+	return writeVarInt(BigInt(-2) * number - BigInt(1), schema);
+};
+
+const readVarIntNumber = (buffer: Buffer): number => {
+	let result = 0;
+	let index = 0;
+	for (let shift = 0; shift < 32; shift += 7) {
+		if (index >= buffer.length) {
+			throw new Error('Invalid buffer length');
+		}
+		const bit = buffer[index];
+		index += 1;
+		if (index === 5 && bit > 0x0f) {
+			throw new Error('Value out of range of uint32');
+		}
+		result = (result | ((bit & rest) << shift)) >>> 0;
+		if ((bit & msg) === 0) {
+			return result;
+		}
+	}
+	throw new Error('Terminating bit not found');
+};
+
+const readVarIntBigInt = (buffer: Buffer): bigint => {
+	let result = BigInt(0);
+	let index = 0;
+	for (let shift = BigInt(0); shift < BigInt(64); shift += BigInt(7)) {
+		if (index >= buffer.length) {
+			throw new Error('Invalid buffer length');
+		}
+		const bit = BigInt(buffer[index]);
+		index += 1;
+		if (index === 10 && bit > 0x01) {
+			throw new Error('Value out of range of uint64');
+		}
+		result |= (bit & BigInt(rest)) << shift;
+		if ((bit & BigInt(msg)) === BigInt(0)) {
+			return result;
+		}
+	}
+	throw new Error('Terminating bit not found');
+};
+
+export const readVarInt = (buffer: Buffer, schema: SchemaProperty): int =>
+	schema.dataType === 'uint32' || schema.dataType === 'sint32'
+		? readVarIntNumber(buffer)
+		: readVarIntBigInt(buffer);
+
+const readSignedVarIntNumber = (buffer: Buffer): number => {
+	const varInt = readVarIntNumber(buffer);
+	if (varInt % 2 === 0) {
+		return varInt / 2;
+	}
+	return -(varInt + 1) / 2;
+};
+
+const readSignedVarIntBigInt = (buffer: Buffer): bigint => {
+	const varInt = readVarIntBigInt(buffer);
+	if (varInt % BigInt(2) === BigInt(0)) {
+		return varInt / BigInt(2);
+	}
+	return -(varInt + BigInt(1)) / BigInt(2);
+};
+
+export const readSignedVarInt = (buffer: Buffer, schema: SchemaProperty): int =>
+	schema.dataType === 'sint32'
+		? readSignedVarIntNumber(buffer)
+		: readSignedVarIntBigInt(buffer);