Add varint reader/writer - Closes #5219 & #5221 & #5229 & #5230

elements/lisk-codec/.eslintignore

@@ -1,2 +1,3 @@
 dist-node
 jest.config.js
+benchmark

elements/lisk-codec/benchmark/varint.js

@@ -0,0 +1,61 @@
+/*
+ * Copyright © 2020 Lisk Foundation
+ *
+ * See the LICENSE file at the top-level directory of this distribution
+ * for licensing information.
+ *
+ * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
+ * no part of this software, including this file, may be copied, modified,
+ * propagated, or distributed except according to the terms contained in the
+ * LICENSE file.
+ *
+ * Removal or modification of this copyright notice is prohibited.
+ */
+
+const { Suite } = require('benchmark');
+const {
+    writeVarInt,
+    readVarInt,
+    writeSignedVarInt,
+    readSignedVarInt,
+} = require('../dist-node/varint');
+
+const suite = new Suite();
+
+suite
+    .add('writeVarInt#uint32', () => {
+        writeVarInt(2147483647, { dataType: 'uint32' });
+    })
+    .add('writeVarInt#uint64', () => {
+        writeVarInt(BigInt('8294967295'), { dataType: 'uint64' });
+    })
+    .add('writeSignedVarInt#sint32', () => {
+        writeSignedVarInt(2147483647, { dataType: 'sint32' });
+    })
+    .add('writeSignedVarInt#sint64', () => {
+        writeSignedVarInt(BigInt('9223372036854775807'), {
+            dataType: 'sint64',
+        });
+    })
+    .add('readVarInt#uint32', () => {
+        readVarInt(Buffer.from('ffffffff0f', 'hex'), { dataType: 'uint32' });
+    })
+    .add('readVarInt#uint64', () => {
+        readVarInt(Buffer.from('ffffffffffffffffff01', 'hex'), {
+            dataType: 'uint64',
+        });
+    })
+    .add('readSignedVarInt#sint32', () => {
+        readSignedVarInt(Buffer.from('feffffff0f', 'hex'), {
+            dataType: 'sint32',
+        });
+    })
+    .add('readSignedVarInt#sint64', () => {
+        readSignedVarInt(Buffer.from('ffffffffffffffffff01', 'hex'), {
+            dataType: 'sint64',
+        });
+    })
+    .on('cycle', function (event) {
+        console.log(String(event.target));
+    })
+    .run({ async: true });

elements/lisk-codec/package.json

@@ -35,11 +35,12 @@
 	},
 	"dependencies": {},
 	"devDependencies": {
-		"@types/node": "12.12.11",
 		"@types/jest": "25.1.3",
 		"@types/jest-when": "2.7.0",
+		"@types/node": "12.12.11",
 		"@typescript-eslint/eslint-plugin": "2.28.0",
 		"@typescript-eslint/parser": "2.28.0",
+		"benchmark": "2.1.4",
 		"eslint": "6.8.0",
 		"eslint-config-lisk-base": "1.2.2",
 		"eslint-config-prettier": "6.10.0",

elements/lisk-codec/src/varint.ts

@@ -0,0 +1,129 @@
+/*
+ * Copyright © 2020 Lisk Foundation
+ *
+ * See the LICENSE file at the top-level directory of this distribution
+ * for licensing information.
+ *
+ * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
+ * no part of this software, including this file, may be copied, modified,
+ * propagated, or distributed except according to the terms contained in the
+ * LICENSE file.
+ *
+ * Removal or modification of this copyright notice is prohibited.
+ */
+/* eslint-disable no-bitwise */
+/* eslint-disable no-param-reassign */
+
+const msg = 0x80;
+const rest = 0x7f;
+
+type int = bigint | number;
+
+const isNumber = (val: int): val is number => typeof val === 'number';
+
+interface SchemaProperty {
+	readonly dataType: string;
+}
+
+const writeVarIntNumber = (value: number): Buffer => {
+	const result: number[] = [];
+	let index = 0;
+	while (value > rest) {
+		result[index] = msg | ((value & rest) >>> 0);
+		value = (value >>> 7) >>> 0;
+		index += 1;
+	}
+
+	result[index] = value;
+
+	return Buffer.from(result);
+};
+
+const writeVarIntBigInt = (value: bigint): Buffer => {
+	const result: number[] = [];
+	let index = 0;
+	while (value > BigInt(rest)) {
+		result[index] = Number(BigInt(msg) | (value & BigInt(rest)));
+		value >>= BigInt(7);
+		index += 1;
+	}
+
+	result[Number(index)] = Number(value);
+
+	return Buffer.from(result);
+};
+
+export const writeVarInt = (value: int): Buffer =>
+	isNumber(value) ? writeVarIntNumber(value) : writeVarIntBigInt(value);
+
+export const writeSignedVarInt = (
+	value: int,
+	schema: SchemaProperty,
+): Buffer => {
+	if (schema.dataType === 'sint32') {
+		const number = Number(value);
+		return writeVarIntNumber(((number << 1) ^ (number >> 31)) >>> 0);
+	}
+	const number = BigInt(value);
+	return writeVarInt((number << BigInt(1)) ^ (number >> BigInt(63)));
+};
+
+const readVarIntNumber = (buffer: Buffer): number => {
+	let result = 0;
+	let index = 0;
+	for (let shift = 0; shift < 32; shift += 7) {
+		if (index >= buffer.length) {
+			throw new Error('Invalid buffer length');
+		}
+		const bit = buffer[index];
+		index += 1;
+		result = (result | ((bit & rest) << shift)) >>> 0;
+		if ((bit & msg) === 0) {
+			return result;
+		}
+	}
+	throw new Error('Out of range');
+};
+
+const readVarIntBigInt = (buffer: Buffer): bigint => {
+	let result = BigInt(0);
+	let index = 0;
+	for (let shift = BigInt(0); shift < BigInt(64); shift += BigInt(7)) {
+		if (index >= buffer.length) {
+			throw new Error('Invalid buffer length');
+		}
+		const bit = BigInt(buffer[index]);
+		index += 1;
+		result |= (bit & BigInt(rest)) << shift;
+		if ((bit & BigInt(msg)) === BigInt(0)) {
+			return result;
+		}
+	}
+	throw new Error('Out of range');
+};
+
+export const readVarInt = (buffer: Buffer, schema: SchemaProperty): int =>
+	schema.dataType === 'uint32' || schema.dataType === 'sint32'
+		? readVarIntNumber(buffer)
+		: readVarIntBigInt(buffer);
+
+const readSignedVarIntNumber = (buffer: Buffer): number => {
+	const varInt = readVarIntNumber(buffer);
+	if (varInt % 2 === 0) {
+		return varInt / 2;
+	}
+	return -(varInt + 1) / 2;
+};
+
+const readSignedVarIntBigInt = (buffer: Buffer): bigint => {
+	const varInt = readVarIntBigInt(buffer);
+	if (varInt % BigInt(2) === BigInt(0)) {
+		return varInt / BigInt(2);
+	}
+	return -(varInt + BigInt(1)) / BigInt(2);
+};
+
+export const readSignedVarInt = (buffer: Buffer, schema: SchemaProperty): int =>
+	schema.dataType === 'sint32'
+		? readSignedVarIntNumber(buffer)
+		: readSignedVarIntBigInt(buffer);

elements/lisk-codec/test/varint.spec.ts

@@ -0,0 +1,187 @@
+/*
+ * Copyright © 2020 Lisk Foundation
+ *
+ * See the LICENSE file at the top-level directory of this distribution
+ * for licensing information.
+ *
+ * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
+ * no part of this software, including this file, may be copied, modified,
+ * propagated, or distributed except according to the terms contained in the
+ * LICENSE file.
+ *
+ * Removal or modification of this copyright notice is prohibited.
+ */
+import {
+	writeVarInt,
+	readVarInt,
+	readSignedVarInt,
+	writeSignedVarInt,
+} from '../src/varint';
+
+describe('varint', () => {
+	describe('writer', () => {
+		it('should encode uint32', () => {
+			expect(writeVarInt(0)).toEqual(Buffer.from('00', 'hex'));
+			expect(writeVarInt(300)).toEqual(Buffer.from('ac02', 'hex'));
+			expect(writeVarInt(2147483647)).toEqual(Buffer.from('ffffffff07', 'hex'));
+			expect(writeVarInt(4294967295)).toEqual(Buffer.from('ffffffff0f', 'hex'));
+		});
+
+		it('should encode uint64', () => {
+			expect(writeVarInt(BigInt(0))).toEqual(Buffer.from('00', 'hex'));
+			expect(writeVarInt(BigInt(300))).toEqual(Buffer.from('ac02', 'hex'));
+			expect(writeVarInt(BigInt(2147483647))).toEqual(
+				Buffer.from('ffffffff07', 'hex'),
+			);
+			expect(writeVarInt(BigInt(4294967295))).toEqual(
+				Buffer.from('ffffffff0f', 'hex'),
+			);
+			expect(writeVarInt(BigInt('8294967295'))).toEqual(
+				Buffer.from('ffcfacf31e', 'hex'),
+			);
+			expect(writeVarInt(BigInt('18446744073709551615'))).toEqual(
+				Buffer.from('ffffffffffffffffff01', 'hex'),
+			);
+		});
+
+		it('should encode sint32', () => {
+			expect(writeSignedVarInt(0, { dataType: 'sint32' })).toEqual(
+				Buffer.from('00', 'hex'),
+			);
+			expect(writeSignedVarInt(1300, { dataType: 'sint32' })).toEqual(
+				Buffer.from('a814', 'hex'),
+			);
+			expect(writeSignedVarInt(-1300, { dataType: 'sint32' })).toEqual(
+				Buffer.from('a714', 'hex'),
+			);
+			expect(writeSignedVarInt(2147483647, { dataType: 'sint32' })).toEqual(
+				Buffer.from('feffffff0f', 'hex'),
+			);
+			expect(writeSignedVarInt(-2147483648, { dataType: 'sint32' })).toEqual(
+				Buffer.from('ffffffff0f', 'hex'),
+			);
+		});
+
+		it('should encode sint64', () => {
+			expect(writeSignedVarInt(BigInt(0), { dataType: 'sint64' })).toEqual(
+				Buffer.from('00', 'hex'),
+			);
+			expect(writeSignedVarInt(BigInt(1300), { dataType: 'sint64' })).toEqual(
+				Buffer.from('a814', 'hex'),
+			);
+			expect(writeSignedVarInt(BigInt(-1300), { dataType: 'sint64' })).toEqual(
+				Buffer.from('a714', 'hex'),
+			);
+			expect(
+				writeSignedVarInt(BigInt(2147483647), { dataType: 'sint64' }),
+			).toEqual(Buffer.from('feffffff0f', 'hex'));
+			expect(
+				writeSignedVarInt(BigInt(-2147483648), { dataType: 'sint64' }),
+			).toEqual(Buffer.from('ffffffff0f', 'hex'));
+			expect(
+				writeSignedVarInt(BigInt('9223372036854775807'), {
+					dataType: 'sint64',
+				}),
+			).toEqual(Buffer.from('feffffffffffffffff01', 'hex'));
+			expect(
+				writeSignedVarInt(BigInt('-9223372036854775808'), {
+					dataType: 'sint64',
+				}),
+			).toEqual(Buffer.from('ffffffffffffffffff01', 'hex'));
+		});
+	});
+
+	describe('reader', () => {
+		it('should decode uint32', () => {
+			expect(
+				readVarInt(Buffer.from('00', 'hex'), { dataType: 'uint32' }),
+			).toEqual(0);
+			expect(
+				readVarInt(Buffer.from('ac02', 'hex'), { dataType: 'uint32' }),
+			).toEqual(300);
+			expect(
+				readVarInt(Buffer.from('ffffffff07', 'hex'), { dataType: 'uint32' }),
+			).toEqual(2147483647);
+			expect(
+				readVarInt(Buffer.from('ffffffff0f', 'hex'), { dataType: 'uint32' }),
+			).toEqual(4294967295);
+		});
+
+		it('should decode uint64', () => {
+			expect(
+				readVarInt(Buffer.from('00', 'hex'), { dataType: 'uint64' }),
+			).toEqual(BigInt(0));
+			expect(
+				readVarInt(Buffer.from('ac02', 'hex'), { dataType: 'uint64' }),
+			).toEqual(BigInt(300));
+			expect(
+				readVarInt(Buffer.from('ffffffff07', 'hex'), { dataType: 'uint64' }),
+			).toEqual(BigInt(2147483647));
+			expect(
+				readVarInt(Buffer.from('ffffffff0f', 'hex'), { dataType: 'uint64' }),
+			).toEqual(BigInt(4294967295));
+			expect(
+				readVarInt(Buffer.from('ffcfacf31e', 'hex'), { dataType: 'uint64' }),
+			).toEqual(BigInt(8294967295));
+			expect(
+				readVarInt(Buffer.from('ffffffffffffffffff01', 'hex'), {
+					dataType: 'uint64',
+				}),
+			).toEqual(BigInt('18446744073709551615'));
+		});
+
+		it('should decode sint32', () => {
+			expect(
+				readSignedVarInt(Buffer.from('00', 'hex'), { dataType: 'sint32' }),
+			).toEqual(0);
+			expect(
+				readSignedVarInt(Buffer.from('a814', 'hex'), { dataType: 'sint32' }),
+			).toEqual(1300);
+			expect(
+				readSignedVarInt(Buffer.from('a714', 'hex'), { dataType: 'sint32' }),
+			).toEqual(-1300);
+			expect(
+				readSignedVarInt(Buffer.from('feffffff0f', 'hex'), {
+					dataType: 'sint32',
+				}),
+			).toEqual(2147483647);
+			expect(
+				readSignedVarInt(Buffer.from('ffffffff0f', 'hex'), {
+					dataType: 'sint32',
+				}),
+			).toEqual(-2147483648);
+		});
+
+		it('should decode sint64', () => {
+			expect(
+				readSignedVarInt(Buffer.from('00', 'hex'), { dataType: 'sint64' }),
+			).toEqual(BigInt(0));
+			expect(
+				readSignedVarInt(Buffer.from('a814', 'hex'), { dataType: 'sint64' }),
+			).toEqual(BigInt(1300));
+			expect(
+				readSignedVarInt(Buffer.from('a714', 'hex'), { dataType: 'sint64' }),
+			).toEqual(BigInt(-1300));
+			expect(
+				readSignedVarInt(Buffer.from('feffffff0f', 'hex'), {
+					dataType: 'sint64',
+				}),
+			).toEqual(BigInt(2147483647));
+			expect(
+				readSignedVarInt(Buffer.from('ffffffff0f', 'hex'), {
+					dataType: 'sint64',
+				}),
+			).toEqual(BigInt(-2147483648));
+			expect(
+				readSignedVarInt(Buffer.from('feffffffffffffffff01', 'hex'), {
+					dataType: 'sint64',
+				}),
+			).toEqual(BigInt('9223372036854775807'));
+			expect(
+				readSignedVarInt(Buffer.from('ffffffffffffffffff01', 'hex'), {
+					dataType: 'sint64',
+				}),
+			).toEqual(BigInt('-9223372036854775808'));
+		});
+	});
+});