From eca5c461c55f8cf30508a0905427ae1c30cc3797 Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Wed, 29 Jun 2022 17:59:04 +1000 Subject: [PATCH] feat: cli option to not encode/decode default values Resolves #43 --- packages/protons-runtime/src/codec.ts | 16 +++- packages/protons-runtime/src/codecs/bool.ts | 8 +- packages/protons-runtime/src/codecs/bytes.ts | 9 +- packages/protons-runtime/src/codecs/double.ts | 8 +- packages/protons-runtime/src/codecs/enum.ts | 12 ++- .../protons-runtime/src/codecs/fixed32.ts | 8 +- .../protons-runtime/src/codecs/fixed64.ts | 8 +- packages/protons-runtime/src/codecs/float.ts | 8 +- packages/protons-runtime/src/codecs/int32.ts | 8 +- packages/protons-runtime/src/codecs/int64.ts | 8 +- .../protons-runtime/src/codecs/message.ts | 55 +++++++++-- .../protons-runtime/src/codecs/sfixed32.ts | 8 +- .../protons-runtime/src/codecs/sfixed64.ts | 8 +- packages/protons-runtime/src/codecs/sint32.ts | 8 +- packages/protons-runtime/src/codecs/sint64.ts | 8 +- packages/protons-runtime/src/codecs/string.ts | 8 +- packages/protons-runtime/src/codecs/uint32.ts | 8 +- packages/protons-runtime/src/codecs/uint64.ts | 8 +- packages/protons/bin/protons.ts | 5 + packages/protons/src/index.ts | 13 +-- packages/protons/test/fixtures/basic.ts | 2 +- packages/protons/test/fixtures/circuit.ts | 4 +- packages/protons/test/fixtures/daemon.ts | 36 +++---- packages/protons/test/fixtures/dht.ts | 6 +- packages/protons/test/fixtures/noise.ts | 2 +- packages/protons/test/fixtures/peer.ts | 6 +- .../fixtures/test-no-default-on-wire.proto | 31 ++++++ .../test/fixtures/test-no-default-on-wire.ts | 94 +++++++++++++++++++ packages/protons/test/fixtures/test.ts | 4 +- packages/protons/test/index.spec.ts | 41 ++++++++ 30 files changed, 371 insertions(+), 77 deletions(-) create mode 100644 packages/protons/test/fixtures/test-no-default-on-wire.proto create mode 100644 packages/protons/test/fixtures/test-no-default-on-wire.ts diff --git a/packages/protons-runtime/src/codec.ts b/packages/protons-runtime/src/codec.ts index ab9a050..049893d 100644 --- a/packages/protons-runtime/src/codec.ts +++ b/packages/protons-runtime/src/codec.ts @@ -22,20 +22,32 @@ export interface EncodingLengthFunction { (value: T): number } +export interface DefaultValueFunction { + (): T +} + +export interface IsDefaultValueFunction { + (value: T): boolean +} + export interface Codec { name: string type: CODEC_TYPES encode: EncodeFunction decode: DecodeFunction encodingLength: EncodingLengthFunction + defaultValue: DefaultValueFunction + isDefaultValue: IsDefaultValueFunction } -export function createCodec (name: string, type: CODEC_TYPES, encode: EncodeFunction, decode: DecodeFunction, encodingLength: EncodingLengthFunction): Codec { +export function createCodec (name: string, type: CODEC_TYPES, encode: EncodeFunction, decode: DecodeFunction, encodingLength: EncodingLengthFunction, defaultValue: DefaultValueFunction, isDefaultValue: IsDefaultValueFunction): Codec { return { name, type, encode, decode, - encodingLength + encodingLength, + defaultValue, + isDefaultValue } } diff --git a/packages/protons-runtime/src/codecs/bool.ts b/packages/protons-runtime/src/codecs/bool.ts index ec9932b..22bfe30 100644 --- a/packages/protons-runtime/src/codecs/bool.ts +++ b/packages/protons-runtime/src/codecs/bool.ts @@ -1,4 +1,4 @@ -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction } from '../codec.js' const encodingLength: EncodingLengthFunction = function boolEncodingLength () { @@ -13,4 +13,8 @@ const decode: DecodeFunction = function boolDecode (buffer, offset) { return buffer.get(offset) > 0 } -export const bool = createCodec('bool', CODEC_TYPES.VARINT, encode, decode, encodingLength) +const defaultValue: DefaultValueFunction = () => false + +const isDefaultValue: IsDefaultValueFunction = (value: boolean) => !value + +export const bool = createCodec('bool', CODEC_TYPES.VARINT, encode, decode, encodingLength, defaultValue, isDefaultValue) diff --git a/packages/protons-runtime/src/codecs/bytes.ts b/packages/protons-runtime/src/codecs/bytes.ts index b8e826e..b8549d5 100644 --- a/packages/protons-runtime/src/codecs/bytes.ts +++ b/packages/protons-runtime/src/codecs/bytes.ts @@ -1,8 +1,9 @@ import { Uint8ArrayList } from 'uint8arraylist' import { unsigned } from '../utils/varint.js' -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction } from '../codec.js' +import { equals } from 'uint8arrays/equals' const encodingLength: EncodingLengthFunction = function bytesEncodingLength (val) { const len = val.byteLength @@ -24,4 +25,8 @@ const decode: DecodeFunction = function bytesDecode (buf, offset) { return buf.slice(offset, offset + byteLength) } -export const bytes = createCodec('bytes', CODEC_TYPES.LENGTH_DELIMITED, encode, decode, encodingLength) +const defaultValue: DefaultValueFunction = () => new Uint8Array() + +const isDefaultValue: IsDefaultValueFunction = (value: Uint8Array) => equals(value, defaultValue()) + +export const bytes = createCodec('bytes', CODEC_TYPES.LENGTH_DELIMITED, encode, decode, encodingLength, defaultValue, isDefaultValue) diff --git a/packages/protons-runtime/src/codecs/double.ts b/packages/protons-runtime/src/codecs/double.ts index 9aabc63..35f5e96 100644 --- a/packages/protons-runtime/src/codecs/double.ts +++ b/packages/protons-runtime/src/codecs/double.ts @@ -1,5 +1,5 @@ import { Uint8ArrayList } from 'uint8arraylist' -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction } from '../codec.js' const encodingLength: EncodingLengthFunction = function doubleEncodingLength () { @@ -17,4 +17,8 @@ const decode: DecodeFunction = function doubleDecode (buf, offset) { return buf.getFloat64(offset, true) } -export const double = createCodec('double', CODEC_TYPES.BIT64, encode, decode, encodingLength) +const defaultValue: DefaultValueFunction = () => 0 + +const isDefaultValue: IsDefaultValueFunction = (value) => value === defaultValue() + +export const double = createCodec('double', CODEC_TYPES.BIT64, encode, decode, encodingLength, defaultValue, isDefaultValue) diff --git a/packages/protons-runtime/src/codecs/enum.ts b/packages/protons-runtime/src/codecs/enum.ts index a8ef222..f2c2b72 100644 --- a/packages/protons-runtime/src/codecs/enum.ts +++ b/packages/protons-runtime/src/codecs/enum.ts @@ -1,6 +1,6 @@ import { unsigned } from '../utils/varint.js' -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction, Codec } from '../codec.js' export function enumeration (v: any): Codec { @@ -42,6 +42,14 @@ export function enumeration (v: any): Codec { return v[strValue] } + const defaultValue: DefaultValueFunction = function defaultValue () { + return Object.values(v)[0] as (string | number) + } + + const isDefaultValue: IsDefaultValueFunction = function isDefaultValue (val) { + return val === defaultValue() + } + // @ts-expect-error yeah yeah - return createCodec('enum', CODEC_TYPES.VARINT, encode, decode, encodingLength) + return createCodec('enum', CODEC_TYPES.VARINT, encode, decode, encodingLength, defaultValue, isDefaultValue) } diff --git a/packages/protons-runtime/src/codecs/fixed32.ts b/packages/protons-runtime/src/codecs/fixed32.ts index 727e547..88888b3 100644 --- a/packages/protons-runtime/src/codecs/fixed32.ts +++ b/packages/protons-runtime/src/codecs/fixed32.ts @@ -1,5 +1,5 @@ import { Uint8ArrayList } from 'uint8arraylist' -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction } from '../codec.js' const encodingLength: EncodingLengthFunction = function fixed32EncodingLength () { @@ -17,4 +17,8 @@ const decode: DecodeFunction = function fixed32Decode (buf, offset) { return buf.getInt32(offset, true) } -export const fixed32 = createCodec('fixed32', CODEC_TYPES.BIT32, encode, decode, encodingLength) +const defaultValue: DefaultValueFunction = () => 0 + +const isDefaultValue: IsDefaultValueFunction = (val) => val === defaultValue() + +export const fixed32 = createCodec('fixed32', CODEC_TYPES.BIT32, encode, decode, encodingLength, defaultValue, isDefaultValue) diff --git a/packages/protons-runtime/src/codecs/fixed64.ts b/packages/protons-runtime/src/codecs/fixed64.ts index cc3bc78..f86d13d 100644 --- a/packages/protons-runtime/src/codecs/fixed64.ts +++ b/packages/protons-runtime/src/codecs/fixed64.ts @@ -1,5 +1,5 @@ import { Uint8ArrayList } from 'uint8arraylist' -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction } from '../codec.js' const encodingLength: EncodingLengthFunction = function int64EncodingLength (val) { @@ -17,4 +17,8 @@ const decode: DecodeFunction = function int64Decode (buf, offset) { return buf.getBigInt64(offset, true) } -export const fixed64 = createCodec('fixed64', CODEC_TYPES.BIT64, encode, decode, encodingLength) +const defaultValue: DefaultValueFunction = () => 0n + +const isDefaultValue: IsDefaultValueFunction = (val) => val === defaultValue() + +export const fixed64 = createCodec('fixed64', CODEC_TYPES.BIT64, encode, decode, encodingLength, defaultValue, isDefaultValue) diff --git a/packages/protons-runtime/src/codecs/float.ts b/packages/protons-runtime/src/codecs/float.ts index 7ccdda2..77d3eee 100644 --- a/packages/protons-runtime/src/codecs/float.ts +++ b/packages/protons-runtime/src/codecs/float.ts @@ -1,5 +1,5 @@ import { Uint8ArrayList } from 'uint8arraylist' -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction } from '../codec.js' const encodingLength: EncodingLengthFunction = function floatEncodingLength () { @@ -17,4 +17,8 @@ const decode: DecodeFunction = function floatDecode (buf, offset) { return buf.getFloat32(offset, true) } -export const float = createCodec('float', CODEC_TYPES.BIT32, encode, decode, encodingLength) +const defaultValue: DefaultValueFunction = () => 0 + +const isDefaultValue: IsDefaultValueFunction = (val) => val === defaultValue() + +export const float = createCodec('float', CODEC_TYPES.BIT32, encode, decode, encodingLength, defaultValue, isDefaultValue) diff --git a/packages/protons-runtime/src/codecs/int32.ts b/packages/protons-runtime/src/codecs/int32.ts index ef4c636..c494a2f 100644 --- a/packages/protons-runtime/src/codecs/int32.ts +++ b/packages/protons-runtime/src/codecs/int32.ts @@ -1,5 +1,5 @@ import { signed } from '../utils/varint.js' -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction } from '../codec.js' const encodingLength: EncodingLengthFunction = function int32EncodingLength (val) { @@ -17,4 +17,8 @@ const decode: DecodeFunction = function int32Decode (buf, offset) { return signed.decode(buf, offset) } -export const int32 = createCodec('int32', CODEC_TYPES.VARINT, encode, decode, encodingLength) +const defaultValue: DefaultValueFunction = () => 0 + +const isDefaultValue: IsDefaultValueFunction = (val) => val === defaultValue() + +export const int32 = createCodec('int32', CODEC_TYPES.VARINT, encode, decode, encodingLength, defaultValue, isDefaultValue) diff --git a/packages/protons-runtime/src/codecs/int64.ts b/packages/protons-runtime/src/codecs/int64.ts index 47f571f..66f1c15 100644 --- a/packages/protons-runtime/src/codecs/int64.ts +++ b/packages/protons-runtime/src/codecs/int64.ts @@ -1,5 +1,5 @@ import { signed } from '../utils/big-varint.js' -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction } from '../codec.js' const encodingLength: EncodingLengthFunction = function int64EncodingLength (val) { @@ -17,4 +17,8 @@ const decode: DecodeFunction = function int64Decode (buf, offset) { return signed.decode(buf, offset) | 0n } -export const int64 = createCodec('int64', CODEC_TYPES.VARINT, encode, decode, encodingLength) +const defaultValue: DefaultValueFunction = () => 0n + +const isDefaultValue: IsDefaultValueFunction = (val) => val === defaultValue() + +export const int64 = createCodec('int64', CODEC_TYPES.VARINT, encode, decode, encodingLength, defaultValue, isDefaultValue) diff --git a/packages/protons-runtime/src/codecs/message.ts b/packages/protons-runtime/src/codecs/message.ts index 602a091..52ce46f 100644 --- a/packages/protons-runtime/src/codecs/message.ts +++ b/packages/protons-runtime/src/codecs/message.ts @@ -1,19 +1,28 @@ import { unsigned } from '../utils/varint.js' -import { createCodec, CODEC_TYPES } from '../codec.js' -import type { DecodeFunction, EncodeFunction, EncodingLengthFunction, Codec } from '../codec.js' +import type { + Codec, + DecodeFunction, + DefaultValueFunction, + EncodeFunction, + EncodingLengthFunction, + IsDefaultValueFunction +} from '../codec.js' +import { CODEC_TYPES, createCodec } from '../codec.js' import { Uint8ArrayList } from 'uint8arraylist' -import type { FieldDefs, FieldDef } from '../index.js' +import type { FieldDef, FieldDefs } from '../index.js' export interface Factory { new (obj: A): T } -export function message (fieldDefs: FieldDefs): Codec { +export function message (fieldDefs: FieldDefs, noDefaultOnWire: boolean): Codec { const encodingLength: EncodingLengthFunction = function messageEncodingLength (val: Record) { let length = 0 for (const fieldDef of Object.values(fieldDefs)) { - length += fieldDef.codec.encodingLength(val[fieldDef.name]) + if (!noDefaultOnWire || !fieldDef.codec.isDefaultValue(val[fieldDef.name])) { + length += fieldDef.codec.encodingLength(val[fieldDef.name]) + } } return unsigned.encodingLength(length) + length @@ -31,6 +40,10 @@ export function message (fieldDefs: FieldDefs): Codec { throw new Error(`Non optional field "${fieldDef.name}" was ${value === null ? 'null' : 'undefined'}`) } + if (noDefaultOnWire && fieldDef.codec.isDefaultValue(value)) { + return + } + const key = (fieldNumber << 3) | fieldDef.codec.type const prefix = new Uint8Array(unsigned.encodingLength(key)) unsigned.encode(key, prefix) @@ -116,15 +129,43 @@ export function message (fieldDefs: FieldDefs): Codec { offset += fieldLength } - // make sure repeated fields have an array if not set for (const fieldDef of Object.values(fieldDefs)) { + // make sure repeated fields have an array if not set if (fieldDef.repeats === true && fields[fieldDef.name] == null) { fields[fieldDef.name] = [] + } else if (noDefaultOnWire && fields[fieldDef.name] == null) { + // apply default values if not set + fields[fieldDef.name] = fieldDef.codec.defaultValue() } } return fields } - return createCodec('message', CODEC_TYPES.LENGTH_DELIMITED, encode, decode, encodingLength) + // Need to initialized sub messages as default value. + const defaultValue: DefaultValueFunction = function defaultValue () { + const defaultValue: any = {} + + for (const fieldDef of Object.values(fieldDefs)) { + if (fieldDef.codec.type === CODEC_TYPES.LENGTH_DELIMITED) { + defaultValue[fieldDef.name] = fieldDef.codec.defaultValue() + } + } + + return defaultValue + } + + const isDefaultValue: IsDefaultValueFunction = function isDefaultValue (val) { + for (const fieldDef of Object.values(fieldDefs)) { + // @ts-expect-error + const fieldValue = val[fieldDef.name] + + if (!fieldDef.codec.isDefaultValue(fieldValue)) { + return false + } + } + return true + } + + return createCodec('message', CODEC_TYPES.LENGTH_DELIMITED, encode, decode, encodingLength, defaultValue, isDefaultValue) } diff --git a/packages/protons-runtime/src/codecs/sfixed32.ts b/packages/protons-runtime/src/codecs/sfixed32.ts index 3958974..0608e45 100644 --- a/packages/protons-runtime/src/codecs/sfixed32.ts +++ b/packages/protons-runtime/src/codecs/sfixed32.ts @@ -1,5 +1,5 @@ import { Uint8ArrayList } from 'uint8arraylist' -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction } from '../codec.js' const encodingLength: EncodingLengthFunction = function sfixed32EncodingLength () { @@ -17,4 +17,8 @@ const decode: DecodeFunction = function sfixed32Decode (buf, offset) { return buf.getInt32(offset, true) } -export const sfixed32 = createCodec('sfixed32', CODEC_TYPES.BIT32, encode, decode, encodingLength) +const defaultValue: DefaultValueFunction = () => 0 + +const isDefaultValue: IsDefaultValueFunction = (val) => val === defaultValue() + +export const sfixed32 = createCodec('sfixed32', CODEC_TYPES.BIT32, encode, decode, encodingLength, defaultValue, isDefaultValue) diff --git a/packages/protons-runtime/src/codecs/sfixed64.ts b/packages/protons-runtime/src/codecs/sfixed64.ts index 9e9a0ff..41524bb 100644 --- a/packages/protons-runtime/src/codecs/sfixed64.ts +++ b/packages/protons-runtime/src/codecs/sfixed64.ts @@ -1,5 +1,5 @@ import { Uint8ArrayList } from 'uint8arraylist' -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction } from '../codec.js' const encodingLength: EncodingLengthFunction = function sfixed64EncodingLength () { @@ -17,4 +17,8 @@ const decode: DecodeFunction = function sfixed64Decode (buf, offset) { return buf.getBigInt64(offset, true) } -export const sfixed64 = createCodec('sfixed64', CODEC_TYPES.BIT64, encode, decode, encodingLength) +const defaultValue: DefaultValueFunction = () => 0n + +const isDefaultValue: IsDefaultValueFunction = (val) => val === defaultValue() + +export const sfixed64 = createCodec('sfixed64', CODEC_TYPES.BIT64, encode, decode, encodingLength, defaultValue, isDefaultValue) diff --git a/packages/protons-runtime/src/codecs/sint32.ts b/packages/protons-runtime/src/codecs/sint32.ts index 88bbb4a..52ba72b 100644 --- a/packages/protons-runtime/src/codecs/sint32.ts +++ b/packages/protons-runtime/src/codecs/sint32.ts @@ -1,5 +1,5 @@ import { zigzag } from '../utils/varint.js' -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction } from '../codec.js' const encodingLength: EncodingLengthFunction = function sint32EncodingLength (val) { @@ -18,4 +18,8 @@ const decode: DecodeFunction = function svarintDecode (buf, offset) { return zigzag.decode(buf, offset) } -export const sint32 = createCodec('sint32', CODEC_TYPES.VARINT, encode, decode, encodingLength) +const defaultValue: DefaultValueFunction = () => 0 + +const isDefaultValue: IsDefaultValueFunction = (val) => val === defaultValue() + +export const sint32 = createCodec('sint32', CODEC_TYPES.VARINT, encode, decode, encodingLength, defaultValue, isDefaultValue) diff --git a/packages/protons-runtime/src/codecs/sint64.ts b/packages/protons-runtime/src/codecs/sint64.ts index ca56ce8..568d197 100644 --- a/packages/protons-runtime/src/codecs/sint64.ts +++ b/packages/protons-runtime/src/codecs/sint64.ts @@ -1,5 +1,5 @@ import { zigzag } from '../utils/big-varint.js' -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction } from '../codec.js' const encodingLength: EncodingLengthFunction = function int64EncodingLength (val) { @@ -17,4 +17,8 @@ const decode: DecodeFunction = function int64Decode (buf, offset) { return zigzag.decode(buf, offset) } -export const sint64 = createCodec('sint64', CODEC_TYPES.VARINT, encode, decode, encodingLength) +const defaultValue: DefaultValueFunction = () => 0n + +const isDefaultValue: IsDefaultValueFunction = (val) => val === defaultValue() + +export const sint64 = createCodec('sint64', CODEC_TYPES.VARINT, encode, decode, encodingLength, defaultValue, isDefaultValue) diff --git a/packages/protons-runtime/src/codecs/string.ts b/packages/protons-runtime/src/codecs/string.ts index d544ade..9866f26 100644 --- a/packages/protons-runtime/src/codecs/string.ts +++ b/packages/protons-runtime/src/codecs/string.ts @@ -1,7 +1,7 @@ import { unsigned } from '../utils/varint.js' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction } from '../codec.js' import { Uint8ArrayList } from 'uint8arraylist' @@ -26,4 +26,8 @@ const decode: DecodeFunction = function stringDecode (buf, offset) { return uint8ArrayToString(buf.slice(offset, offset + strLen)) } -export const string = createCodec('string', CODEC_TYPES.LENGTH_DELIMITED, encode, decode, encodingLength) +const defaultValue: DefaultValueFunction = () => '' + +const isDefaultValue: IsDefaultValueFunction = (val) => val === defaultValue() + +export const string = createCodec('string', CODEC_TYPES.LENGTH_DELIMITED, encode, decode, encodingLength, defaultValue, isDefaultValue) diff --git a/packages/protons-runtime/src/codecs/uint32.ts b/packages/protons-runtime/src/codecs/uint32.ts index 42d4153..6676d82 100644 --- a/packages/protons-runtime/src/codecs/uint32.ts +++ b/packages/protons-runtime/src/codecs/uint32.ts @@ -1,5 +1,5 @@ import { unsigned } from '../utils/varint.js' -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction } from '../codec.js' const encodingLength: EncodingLengthFunction = function uint32EncodingLength (val) { @@ -22,4 +22,8 @@ const decode: DecodeFunction = function uint32Decode (buf, offset) { // return value > 2147483647 ? value - 4294967296 : value } -export const uint32 = createCodec('uint32', CODEC_TYPES.VARINT, encode, decode, encodingLength) +const defaultValue: DefaultValueFunction = () => 0 + +const isDefaultValue: IsDefaultValueFunction = (val) => val === defaultValue() + +export const uint32 = createCodec('uint32', CODEC_TYPES.VARINT, encode, decode, encodingLength, defaultValue, isDefaultValue) diff --git a/packages/protons-runtime/src/codecs/uint64.ts b/packages/protons-runtime/src/codecs/uint64.ts index f8c2925..ee9b6fc 100644 --- a/packages/protons-runtime/src/codecs/uint64.ts +++ b/packages/protons-runtime/src/codecs/uint64.ts @@ -1,5 +1,5 @@ import { unsigned } from '../utils/big-varint.js' -import { createCodec, CODEC_TYPES } from '../codec.js' +import { createCodec, CODEC_TYPES, DefaultValueFunction, IsDefaultValueFunction } from '../codec.js' import type { DecodeFunction, EncodeFunction, EncodingLengthFunction } from '../codec.js' const encodingLength: EncodingLengthFunction = function uint64EncodingLength (val) { @@ -18,4 +18,8 @@ const decode: DecodeFunction = function uint64Decode (buf, offset) { return unsigned.decode(buf, offset) } -export const uint64 = createCodec('uint64', CODEC_TYPES.VARINT, encode, decode, encodingLength) +const defaultValue: DefaultValueFunction = () => 0n + +const isDefaultValue: IsDefaultValueFunction = (val) => val === defaultValue() + +export const uint64 = createCodec('uint64', CODEC_TYPES.VARINT, encode, decode, encodingLength, defaultValue, isDefaultValue) diff --git a/packages/protons/bin/protons.ts b/packages/protons/bin/protons.ts index a2ce94c..272e0c4 100644 --- a/packages/protons/bin/protons.ts +++ b/packages/protons/bin/protons.ts @@ -10,6 +10,7 @@ async function main () { Options --output, -o Path to a directory to write transpiled typescript files into + --no-default-on-wire Do not encode default values over the wire, decode absent fields as default values Examples $ protons ./path/to/file.proto ./path/to/other/file.proto @@ -19,6 +20,10 @@ async function main () { output: { type: 'string', alias: 'o' + }, + defaultOnWire: { + type: 'boolean', + default: true } } }) diff --git a/packages/protons/src/index.ts b/packages/protons/src/index.ts index 023be59..06db059 100644 --- a/packages/protons/src/index.ts +++ b/packages/protons/src/index.ts @@ -125,7 +125,7 @@ function defineFields (fields: Record, messageDef: MessageDef, }) } -function compileMessage (messageDef: MessageDef, moduleDef: ModuleDef): string { +function compileMessage (messageDef: MessageDef, moduleDef: ModuleDef, noDefaultOnWire: boolean): string { if (isEnumDef(messageDef)) { moduleDef.imports.add('enumeration') @@ -158,7 +158,7 @@ export namespace ${messageDef.name} { if (messageDef.nested != null) { nested = '\n' nested += Object.values(messageDef.nested) - .map(def => compileMessage(def, moduleDef).trim()) + .map(def => compileMessage(def, moduleDef, noDefaultOnWire).trim()) .join('\n\n') .split('\n') .map(line => line.trim() === '' ? '' : ` ${line}`) @@ -213,7 +213,7 @@ export interface ${messageDef.name} { return `${fieldDef.id}: { name: '${name}', codec: ${codec}${fieldDef.options?.proto3_optional === true ? ', optional: true' : ''}${fieldDef.rule === 'repeated' ? ', repeats: true' : ''} }` }).join(',\n ')} - }) + }, ${noDefaultOnWire ? 'true' : 'false'}) } export const encode = (obj: ${messageDef.name}): Uint8Array => { @@ -242,7 +242,7 @@ interface ModuleDef { globals: Record } -function defineModule (def: ClassDef): ModuleDef { +function defineModule (def: ClassDef, noDefaultOnWire: boolean): ModuleDef { const moduleDef: ModuleDef = { imports: new Set(), importedTypes: new Set(), @@ -280,7 +280,7 @@ function defineModule (def: ClassDef): ModuleDef { for (const className of Object.keys(defs)) { const classDef = defs[className] - moduleDef.compiled.push(compileMessage(classDef, moduleDef)) + moduleDef.compiled.push(compileMessage(classDef, moduleDef, noDefaultOnWire)) } return moduleDef @@ -288,6 +288,7 @@ function defineModule (def: ClassDef): ModuleDef { interface Flags { output?: string + defaultOnWire: boolean } export async function generate (source: string, flags: Flags) { @@ -299,7 +300,7 @@ export async function generate (source: string, flags: Flags) { } const def = JSON.parse(json) - const moduleDef = defineModule(def) + const moduleDef = defineModule(def, !flags.defaultOnWire) let lines = [ '/* eslint-disable import/export */', diff --git a/packages/protons/test/fixtures/basic.ts b/packages/protons/test/fixtures/basic.ts index e340908..0df1eba 100644 --- a/packages/protons/test/fixtures/basic.ts +++ b/packages/protons/test/fixtures/basic.ts @@ -14,7 +14,7 @@ export namespace Basic { return message({ 1: { name: 'foo', codec: string }, 2: { name: 'num', codec: int32 } - }) + }, false) } export const encode = (obj: Basic): Uint8Array => { diff --git a/packages/protons/test/fixtures/circuit.ts b/packages/protons/test/fixtures/circuit.ts index 71ee1b9..e4363ed 100644 --- a/packages/protons/test/fixtures/circuit.ts +++ b/packages/protons/test/fixtures/circuit.ts @@ -86,7 +86,7 @@ export namespace CircuitRelay { return message({ 1: { name: 'id', codec: bytes }, 2: { name: 'addrs', codec: bytes, repeats: true } - }) + }, false) } export const encode = (obj: Peer): Uint8Array => { @@ -104,7 +104,7 @@ export namespace CircuitRelay { 2: { name: 'srcPeer', codec: CircuitRelay.Peer.codec(), optional: true }, 3: { name: 'dstPeer', codec: CircuitRelay.Peer.codec(), optional: true }, 4: { name: 'code', codec: CircuitRelay.Status.codec(), optional: true } - }) + }, false) } export const encode = (obj: CircuitRelay): Uint8Array => { diff --git a/packages/protons/test/fixtures/daemon.ts b/packages/protons/test/fixtures/daemon.ts index 7b4e7a1..8b29664 100644 --- a/packages/protons/test/fixtures/daemon.ts +++ b/packages/protons/test/fixtures/daemon.ts @@ -60,7 +60,7 @@ export namespace Request { 7: { name: 'disconnect', codec: DisconnectRequest.codec(), optional: true }, 8: { name: 'pubsub', codec: PSRequest.codec(), optional: true }, 9: { name: 'peerStore', codec: PeerstoreRequest.codec(), optional: true } - }) + }, false) } export const encode = (obj: Request): Uint8Array => { @@ -110,7 +110,7 @@ export namespace Response { 6: { name: 'peers', codec: PeerInfo.codec(), repeats: true }, 7: { name: 'pubsub', codec: PSResponse.codec(), optional: true }, 8: { name: 'peerStore', codec: PeerstoreResponse.codec(), optional: true } - }) + }, false) } export const encode = (obj: Response): Uint8Array => { @@ -132,7 +132,7 @@ export namespace IdentifyResponse { return message({ 1: { name: 'id', codec: bytes }, 2: { name: 'addrs', codec: bytes, repeats: true } - }) + }, false) } export const encode = (obj: IdentifyResponse): Uint8Array => { @@ -156,7 +156,7 @@ export namespace ConnectRequest { 1: { name: 'peer', codec: bytes }, 2: { name: 'addrs', codec: bytes, repeats: true }, 3: { name: 'timeout', codec: int64, optional: true } - }) + }, false) } export const encode = (obj: ConnectRequest): Uint8Array => { @@ -180,7 +180,7 @@ export namespace StreamOpenRequest { 1: { name: 'peer', codec: bytes }, 2: { name: 'proto', codec: string, repeats: true }, 3: { name: 'timeout', codec: int64, optional: true } - }) + }, false) } export const encode = (obj: StreamOpenRequest): Uint8Array => { @@ -202,7 +202,7 @@ export namespace StreamHandlerRequest { return message({ 1: { name: 'addr', codec: bytes }, 2: { name: 'proto', codec: string, repeats: true } - }) + }, false) } export const encode = (obj: StreamHandlerRequest): Uint8Array => { @@ -222,7 +222,7 @@ export namespace ErrorResponse { export const codec = (): Codec => { return message({ 1: { name: 'msg', codec: string } - }) + }, false) } export const encode = (obj: ErrorResponse): Uint8Array => { @@ -246,7 +246,7 @@ export namespace StreamInfo { 1: { name: 'peer', codec: bytes }, 2: { name: 'addr', codec: bytes }, 3: { name: 'proto', codec: string } - }) + }, false) } export const encode = (obj: StreamInfo): Uint8Array => { @@ -308,7 +308,7 @@ export namespace DHTRequest { 5: { name: 'value', codec: bytes, optional: true }, 6: { name: 'count', codec: int32, optional: true }, 7: { name: 'timeout', codec: int64, optional: true } - }) + }, false) } export const encode = (obj: DHTRequest): Uint8Array => { @@ -350,7 +350,7 @@ export namespace DHTResponse { 1: { name: 'type', codec: DHTResponse.Type.codec() }, 2: { name: 'peer', codec: PeerInfo.codec(), optional: true }, 3: { name: 'value', codec: bytes, optional: true } - }) + }, false) } export const encode = (obj: DHTResponse): Uint8Array => { @@ -372,7 +372,7 @@ export namespace PeerInfo { return message({ 1: { name: 'id', codec: bytes }, 2: { name: 'addrs', codec: bytes, repeats: true } - }) + }, false) } export const encode = (obj: PeerInfo): Uint8Array => { @@ -416,7 +416,7 @@ export namespace ConnManagerRequest { 2: { name: 'peer', codec: bytes, optional: true }, 3: { name: 'tag', codec: string, optional: true }, 4: { name: 'weight', codec: int64, optional: true } - }) + }, false) } export const encode = (obj: ConnManagerRequest): Uint8Array => { @@ -436,7 +436,7 @@ export namespace DisconnectRequest { export const codec = (): Codec => { return message({ 1: { name: 'peer', codec: bytes } - }) + }, false) } export const encode = (obj: DisconnectRequest): Uint8Array => { @@ -480,7 +480,7 @@ export namespace PSRequest { 1: { name: 'type', codec: PSRequest.Type.codec() }, 2: { name: 'topic', codec: string, optional: true }, 3: { name: 'data', codec: bytes, optional: true } - }) + }, false) } export const encode = (obj: PSRequest): Uint8Array => { @@ -510,7 +510,7 @@ export namespace PSMessage { 4: { name: 'topicIDs', codec: string, repeats: true }, 5: { name: 'signature', codec: bytes, optional: true }, 6: { name: 'key', codec: bytes, optional: true } - }) + }, false) } export const encode = (obj: PSMessage): Uint8Array => { @@ -532,7 +532,7 @@ export namespace PSResponse { return message({ 1: { name: 'topics', codec: string, repeats: true }, 2: { name: 'peerIDs', codec: bytes, repeats: true } - }) + }, false) } export const encode = (obj: PSResponse): Uint8Array => { @@ -572,7 +572,7 @@ export namespace PeerstoreRequest { 1: { name: 'type', codec: PeerstoreRequest.Type.codec() }, 2: { name: 'id', codec: bytes, optional: true }, 3: { name: 'protos', codec: string, repeats: true } - }) + }, false) } export const encode = (obj: PeerstoreRequest): Uint8Array => { @@ -594,7 +594,7 @@ export namespace PeerstoreResponse { return message({ 1: { name: 'peer', codec: PeerInfo.codec(), optional: true }, 2: { name: 'protos', codec: string, repeats: true } - }) + }, false) } export const encode = (obj: PeerstoreResponse): Uint8Array => { diff --git a/packages/protons/test/fixtures/dht.ts b/packages/protons/test/fixtures/dht.ts index f46af45..aa4bcf1 100644 --- a/packages/protons/test/fixtures/dht.ts +++ b/packages/protons/test/fixtures/dht.ts @@ -20,7 +20,7 @@ export namespace Record { 3: { name: 'author', codec: bytes, optional: true }, 4: { name: 'signature', codec: bytes, optional: true }, 5: { name: 'timeReceived', codec: string, optional: true } - }) + }, false) } export const encode = (obj: Record): Uint8Array => { @@ -98,7 +98,7 @@ export namespace Message { 1: { name: 'id', codec: bytes, optional: true }, 2: { name: 'addrs', codec: bytes, repeats: true }, 3: { name: 'connection', codec: Message.ConnectionType.codec(), optional: true } - }) + }, false) } export const encode = (obj: Peer): Uint8Array => { @@ -118,7 +118,7 @@ export namespace Message { 3: { name: 'record', codec: bytes, optional: true }, 8: { name: 'closerPeers', codec: Message.Peer.codec(), repeats: true }, 9: { name: 'providerPeers', codec: Message.Peer.codec(), repeats: true } - }) + }, false) } export const encode = (obj: Message): Uint8Array => { diff --git a/packages/protons/test/fixtures/noise.ts b/packages/protons/test/fixtures/noise.ts index 855a319..6497153 100644 --- a/packages/protons/test/fixtures/noise.ts +++ b/packages/protons/test/fixtures/noise.ts @@ -17,7 +17,7 @@ export namespace pb { 1: { name: 'identityKey', codec: bytes }, 2: { name: 'identitySig', codec: bytes }, 3: { name: 'data', codec: bytes } - }) + }, false) } export const encode = (obj: NoiseHandshakePayload): Uint8Array => { diff --git a/packages/protons/test/fixtures/peer.ts b/packages/protons/test/fixtures/peer.ts index 3d7c83c..da6a824 100644 --- a/packages/protons/test/fixtures/peer.ts +++ b/packages/protons/test/fixtures/peer.ts @@ -20,7 +20,7 @@ export namespace Peer { 3: { name: 'metadata', codec: Metadata.codec(), repeats: true }, 4: { name: 'pubKey', codec: bytes, optional: true }, 5: { name: 'peerRecordEnvelope', codec: bytes, optional: true } - }) + }, false) } export const encode = (obj: Peer): Uint8Array => { @@ -42,7 +42,7 @@ export namespace Address { return message
({ 1: { name: 'multiaddr', codec: bytes }, 2: { name: 'isCertified', codec: bool, optional: true } - }) + }, false) } export const encode = (obj: Address): Uint8Array => { @@ -64,7 +64,7 @@ export namespace Metadata { return message({ 1: { name: 'key', codec: string }, 2: { name: 'value', codec: bytes } - }) + }, false) } export const encode = (obj: Metadata): Uint8Array => { diff --git a/packages/protons/test/fixtures/test-no-default-on-wire.proto b/packages/protons/test/fixtures/test-no-default-on-wire.proto new file mode 100644 index 0000000..fc11bbf --- /dev/null +++ b/packages/protons/test/fixtures/test-no-default-on-wire.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +enum AnEnum { + HERP = 0; + DERP = 1; +} + +message SubMessage { + string foo = 1; +} + +message AllTheTypesNoDefaultOnWire { + optional bool field1 = 1; + optional int32 field2 = 2; + optional int64 field3 = 3; + optional uint32 field4 = 4; + optional uint64 field5 = 5; + optional sint32 field6 = 6; + optional sint64 field7 = 7; + optional double field8 = 8; + optional float field9 = 9; + optional string field10 = 10; + optional bytes field11 = 11; + optional AnEnum field12 = 12; + optional SubMessage field13 = 13; + repeated string field14 = 14; + optional fixed32 field15 = 15; + optional fixed64 field16 = 16; + optional sfixed32 field17 = 17; + optional sfixed64 field18 = 18; +} diff --git a/packages/protons/test/fixtures/test-no-default-on-wire.ts b/packages/protons/test/fixtures/test-no-default-on-wire.ts new file mode 100644 index 0000000..326c6ed --- /dev/null +++ b/packages/protons/test/fixtures/test-no-default-on-wire.ts @@ -0,0 +1,94 @@ +/* eslint-disable import/export */ +/* eslint-disable @typescript-eslint/no-namespace */ + +import { enumeration, encodeMessage, decodeMessage, message, string, bool, int32, int64, uint32, uint64, sint32, sint64, double, float, bytes, fixed32, fixed64, sfixed32, sfixed64 } from 'protons-runtime' +import type { Codec } from 'protons-runtime' + +export enum AnEnum { + HERP = 'HERP', + DERP = 'DERP' +} + +enum __AnEnumValues { + HERP = 0, + DERP = 1 +} + +export namespace AnEnum { + export const codec = () => { + return enumeration(__AnEnumValues) + } +} +export interface SubMessage { + foo: string +} + +export namespace SubMessage { + export const codec = (): Codec => { + return message({ + 1: { name: 'foo', codec: string } + }, true) + } + + export const encode = (obj: SubMessage): Uint8Array => { + return encodeMessage(obj, SubMessage.codec()) + } + + export const decode = (buf: Uint8Array): SubMessage => { + return decodeMessage(buf, SubMessage.codec()) + } +} + +export interface AllTheTypesNoDefaultOnWire { + field1?: boolean + field2?: number + field3?: bigint + field4?: number + field5?: bigint + field6?: number + field7?: bigint + field8?: number + field9?: number + field10?: string + field11?: Uint8Array + field12?: AnEnum + field13?: SubMessage + field14: string[] + field15?: number + field16?: bigint + field17?: number + field18?: bigint +} + +export namespace AllTheTypesNoDefaultOnWire { + export const codec = (): Codec => { + return message({ + 1: { name: 'field1', codec: bool, optional: true }, + 2: { name: 'field2', codec: int32, optional: true }, + 3: { name: 'field3', codec: int64, optional: true }, + 4: { name: 'field4', codec: uint32, optional: true }, + 5: { name: 'field5', codec: uint64, optional: true }, + 6: { name: 'field6', codec: sint32, optional: true }, + 7: { name: 'field7', codec: sint64, optional: true }, + 8: { name: 'field8', codec: double, optional: true }, + 9: { name: 'field9', codec: float, optional: true }, + 10: { name: 'field10', codec: string, optional: true }, + 11: { name: 'field11', codec: bytes, optional: true }, + 12: { name: 'field12', codec: AnEnum.codec(), optional: true }, + 13: { name: 'field13', codec: SubMessage.codec(), optional: true }, + 14: { name: 'field14', codec: string, repeats: true }, + 15: { name: 'field15', codec: fixed32, optional: true }, + 16: { name: 'field16', codec: fixed64, optional: true }, + 17: { name: 'field17', codec: sfixed32, optional: true }, + 18: { name: 'field18', codec: sfixed64, optional: true } + }, true) + } + + export const encode = (obj: AllTheTypesNoDefaultOnWire): Uint8Array => { + return encodeMessage(obj, AllTheTypesNoDefaultOnWire.codec()) + } + + export const decode = (buf: Uint8Array): AllTheTypesNoDefaultOnWire => { + return decodeMessage(buf, AllTheTypesNoDefaultOnWire.codec()) + } +} diff --git a/packages/protons/test/fixtures/test.ts b/packages/protons/test/fixtures/test.ts index 893be8b..c52f389 100644 --- a/packages/protons/test/fixtures/test.ts +++ b/packages/protons/test/fixtures/test.ts @@ -27,7 +27,7 @@ export namespace SubMessage { export const codec = (): Codec => { return message({ 1: { name: 'foo', codec: string } - }) + }, false) } export const encode = (obj: SubMessage): Uint8Array => { @@ -81,7 +81,7 @@ export namespace AllTheTypes { 16: { name: 'field16', codec: fixed64, optional: true }, 17: { name: 'field17', codec: sfixed32, optional: true }, 18: { name: 'field18', codec: sfixed64, optional: true } - }) + }, false) } export const encode = (obj: AllTheTypes): Uint8Array => { diff --git a/packages/protons/test/index.spec.ts b/packages/protons/test/index.spec.ts index 6927aca..2508b54 100644 --- a/packages/protons/test/index.spec.ts +++ b/packages/protons/test/index.spec.ts @@ -9,6 +9,7 @@ import fs from 'fs' import protobufjs, { Type as PBType } from 'protobufjs' import { Peer } from './fixtures/peer.js' import { CircuitRelay } from './fixtures/circuit.js' +import { AllTheTypesNoDefaultOnWire } from './fixtures/test-no-default-on-wire.js' const Long = protobufjs.util.Long @@ -139,6 +140,46 @@ describe('encode', () => { expect(AllTheTypes.decode(pbjsBuf)).to.deep.equal(allTheTypes) }) + it('should not encode any type with default value', () => { + const allDefaultValues: AllTheTypesNoDefaultOnWire = { + field1: false, + field2: 0, + field3: 0n, + field4: 0, + field5: 0n, + field6: 0, + field7: 0n, + field8: 0, + field9: 0, + field10: '', + field11: new Uint8Array(), + field12: AnEnum.HERP, // First value is the default one + field13: { + foo: '' + }, + field14: [], + field15: 0, + field16: 0n, + field17: 0, + field18: 0n + } + + // @ts-expect-error + const noValues: AllTheTypesNoDefaultOnWire = { + } + + const schema = pbjs.parseSchema(fs.readFileSync('./test/fixtures/test.proto', 'utf-8')).compile() + const pbjsBufAllDefault = schema.encodeAllTheTypes(longifyBigInts(allDefaultValues)) + const pbjsBufNoValues = schema.encodeAllTheTypes(longifyBigInts(noValues)) + + const encoded = AllTheTypesNoDefaultOnWire.encode(allDefaultValues) + + expect(encoded).to.equalBytes(pbjsBufNoValues) + + expect(AllTheTypesNoDefaultOnWire.decode(encoded)).to.deep.equal(allDefaultValues) + expect(AllTheTypesNoDefaultOnWire.decode(pbjsBufAllDefault)).to.deep.equal(allDefaultValues) + }) + it('decodes multiple sub messages', () => { const peer: Peer = { protocols: ['protocol1', 'protocol2'],