Skip to content
This repository has been archived by the owner on Jun 11, 2024. It is now read-only.

Add varint reader/writer - Closes #5219 & #5221 & #5229 & #5230 #5319

Merged
merged 8 commits into from
May 13, 2020
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions elements/lisk-codec/.eslintignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
dist-node
jest.config.js
benchmark
61 changes: 61 additions & 0 deletions elements/lisk-codec/benchmark/varint.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Copyright © 2020 Lisk Foundation
*
* See the LICENSE file at the top-level directory of this distribution
* for licensing information.
*
* Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
* no part of this software, including this file, may be copied, modified,
* propagated, or distributed except according to the terms contained in the
* LICENSE file.
*
* Removal or modification of this copyright notice is prohibited.
*/

const { Suite } = require('benchmark');
const {
writeVarInt,
readVarInt,
writeSignedVarInt,
readSignedVarInt,
} = require('../dist-node/varint');

const suite = new Suite();

suite
.add('writeVarInt#uint32', () => {
writeVarInt(2147483647, { dataType: 'uint32' });
})
.add('writeVarInt#uint64', () => {
writeVarInt(BigInt('8294967295'), { dataType: 'uint64' });
})
.add('writeSignedVarInt#sint32', () => {
writeSignedVarInt(2147483647, { dataType: 'sint32' });
})
.add('writeSignedVarInt#sint64', () => {
writeSignedVarInt(BigInt('9223372036854775807'), {
dataType: 'sint64',
});
})
.add('readVarInt#uint32', () => {
readVarInt(Buffer.from('ffffffff0f', 'hex'), { dataType: 'uint32' });
})
.add('readVarInt#uint64', () => {
readVarInt(Buffer.from('ffffffffffffffffff01', 'hex'), {
dataType: 'uint64',
});
})
.add('readSignedVarInt#sint32', () => {
readSignedVarInt(Buffer.from('feffffff0f', 'hex'), {
dataType: 'sint32',
});
})
.add('readSignedVarInt#sint64', () => {
readSignedVarInt(Buffer.from('ffffffffffffffffff01', 'hex'), {
dataType: 'sint64',
});
})
.on('cycle', function (event) {
console.log(String(event.target));
})
.run({ async: true });
3 changes: 2 additions & 1 deletion elements/lisk-codec/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,12 @@
},
"dependencies": {},
"devDependencies": {
"@types/node": "12.12.11",
"@types/jest": "25.1.3",
"@types/jest-when": "2.7.0",
"@types/node": "12.12.11",
"@typescript-eslint/eslint-plugin": "2.28.0",
"@typescript-eslint/parser": "2.28.0",
"benchmark": "2.1.4",
"eslint": "6.8.0",
"eslint-config-lisk-base": "1.2.2",
"eslint-config-prettier": "6.10.0",
Expand Down
138 changes: 138 additions & 0 deletions elements/lisk-codec/src/varint.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
/*
* Copyright © 2020 Lisk Foundation
*
* See the LICENSE file at the top-level directory of this distribution
* for licensing information.
*
* Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
* no part of this software, including this file, may be copied, modified,
* propagated, or distributed except according to the terms contained in the
* LICENSE file.
*
* Removal or modification of this copyright notice is prohibited.
*/
/* eslint-disable no-bitwise */
/* eslint-disable no-param-reassign */

const msg = 0x80;
const rest = 0x7f;

type int = bigint | number;

const isNumber = (val: int): val is number => typeof val === 'number';

interface SchemaProperty {
readonly dataType: string;
}

const writeVarIntNumber = (value: number): Buffer => {
const result: number[] = [];
let index = 0;
while (value > rest) {
result[index] = msg | ((value & rest) >>> 0);
value = (value >>> 7) >>> 0;
index += 1;
}

result[index] = value;

return Buffer.from(result);
};

const writeVarIntBigInt = (value: bigint): Buffer => {
const result: number[] = [];
let index = 0;
while (value > BigInt(rest)) {
result[index] = Number(BigInt(msg) | (value & BigInt(rest)));
value >>= BigInt(7);
index += 1;
}

result[Number(index)] = Number(value);

return Buffer.from(result);
};

export const writeVarInt = (value: int, _schema: SchemaProperty): Buffer =>
isNumber(value) ? writeVarIntNumber(value) : writeVarIntBigInt(value);

export const writeSignedVarInt = (
value: int,
schema: SchemaProperty,
): Buffer => {
if (schema.dataType === 'sint32') {
const number = Number(value);
if (number >= 0) {
return writeVarIntNumber(2 * number);
}
return writeVarIntNumber(-2 * number - 1);
}
const number = BigInt(value);
if (number >= BigInt(0)) {
return writeVarInt(BigInt(2) * number, schema);
}
return writeVarInt(BigInt(-2) * number - BigInt(1), schema);
};

const readVarIntNumber = (buffer: Buffer): number => {
let result = 0;
let index = 0;
for (let shift = 0; shift < 32; shift += 7) {
if (index >= buffer.length) {
throw new Error('Invalid buffer length');
}
const bit = buffer[index];
index += 1;
if (index === 5 && bit > 0x0f) {
throw new Error('Value out of range of uint32');
}
result = (result | ((bit & rest) << shift)) >>> 0;
if ((bit & msg) === 0) {
return result;
}
}
throw new Error('Terminating bit not found');
};

const readVarIntBigInt = (buffer: Buffer): bigint => {
let result = BigInt(0);
let index = 0;
for (let shift = BigInt(0); shift < BigInt(64); shift += BigInt(7)) {
shuse2 marked this conversation as resolved.
Show resolved Hide resolved
if (index >= buffer.length) {
throw new Error('Invalid buffer length');
}
const bit = BigInt(buffer[index]);
index += 1;
result |= (bit & BigInt(rest)) << shift;
if ((bit & BigInt(msg)) === BigInt(0)) {
return result;
}
}
throw new Error('Terminating bit not found');
};

export const readVarInt = (buffer: Buffer, schema: SchemaProperty): int =>
schema.dataType === 'uint32' || schema.dataType === 'sint32'
? readVarIntNumber(buffer)
: readVarIntBigInt(buffer);

const readSignedVarIntNumber = (buffer: Buffer): number => {
const varInt = readVarIntNumber(buffer);
if (varInt % 2 === 0) {
return varInt / 2;
}
return -(varInt + 1) / 2;
};

const readSignedVarIntBigInt = (buffer: Buffer): bigint => {
const varInt = readVarIntBigInt(buffer);
if (varInt % BigInt(2) === BigInt(0)) {
return varInt / BigInt(2);
}
return -(varInt + BigInt(1)) / BigInt(2);
};

export const readSignedVarInt = (buffer: Buffer, schema: SchemaProperty): int =>
schema.dataType === 'sint32'
? readSignedVarIntNumber(buffer)
: readSignedVarIntBigInt(buffer);
Loading