feat!: emit smallcaps-format data in all marshallers

This changes every `makeMarshal()` instance which performs object
graph serialization to add the `{serializeBodyFormat: 'smallcaps'}`
option. All emitted data will be in the "smallcaps" format, which is
more concise (less overhead). For example, `1n` (a BigInt) is encoded
as `#"+1"` instead of `{"@qclass":"bigint","digits":"1"}`.

The old format was JSON except with those special `@qclass` objects to
capture everything that JSON could not, including object
references ("slots"). The new format is a leading `#` prefix, plus
JSON, except that all the special cases are expressed as strings.

Most significantly, this changes the format of the data we publish to
`chainStorage`.  The old format encoded simple data as mostly JSON, so
bespoke parsers mostly worked, but even BigInts required special
handling. Off-chain followers who perform RPC queries to the chain are
highly encouraged to use the `@endo/marshal` library (or equivalent)
to parse the data. This library can understand old-format data too,
such as chain data published before this upgrade.

Notable changes:

* Many ad-hoc serializers were replaced with a real `makeMarshal`,
  which requires hardened inputs, so some new `harden()` calls were
  added
* Some marshallers which only perform unserialization (`fromCapData`)
  were not modified
* All unit tests which compared serialized data against
  manually-constructed examples had their examples updated
* Previously, the `makeMockChainStorageRoot()` utility parsed
  Far/Remotable objects into an object with `{ iface }`. Now, it is
  unserialized into actual Remotables. Both `t.deepEqual` and
  `t.snapshot` will correctly compare the `iface` values of two Far
  objects, so tests should simply construct a `Far(iface)` and include
  it in the specimen data. As a result, many `t.snapshot`-style tests
  had their snapshot data changed.

refs #6822

Co-authored-by: Chip Morningstar <[email protected]>

packages/agoric-cli/src/lib/wallet.js

@@ -37,7 +37,7 @@ export const getLastUpdate = (addr, { readLatestHead }) => {
  * @param {Pick<import('stream').Writable,'write'>} [stdout]
  */
 export const outputAction = (bridgeAction, stdout = process.stdout) => {
-  const capData = marshaller.serialize(bridgeAction);
+  const capData = marshaller.serialize(harden(bridgeAction));
   stdout.write(JSON.stringify(capData));
   stdout.write('\n');
 };

packages/agoric-cli/test/test-inter-cli.js

@@ -82,7 +82,7 @@ const makeNet = published => {
   const ctx = makeFromBoard();
   const m = boardSlottingMarshaller(ctx.convertSlotToVal);
   const fmt = obj => {
-    const capData = m.serialize(obj);
+    const capData = m.serialize(harden(obj));
     const values = [JSON.stringify(capData)];
     const specimen = { blockHeight: undefined, values };
     const txt = JSON.stringify({
@@ -583,10 +583,10 @@ test('README ex1: inter bid place by-price: printed offer is correct', async t =
 
   const txt = out.join('').trim();
   const obj = net.marshaller.unserialize(JSON.parse(txt));
-  obj.offer.result = 'Your bid has been accepted'; // pretend we processed it
+  const offer = { ...obj.offer, result: 'Your bid has been accepted' }; // pretend we processed it
 
   const assets = Object.values(agoricNames.vbankAsset);
-  const bidInfo = fmtBid(obj.offer, assets);
+  const bidInfo = fmtBid(offer, assets);
   t.deepEqual(bidInfo, expected);
 });
 

packages/cache/src/store.js

@@ -24,7 +24,9 @@ const makeKeyToString = (sanitize = obj => obj) => {
     return slot;
   };
 
-  const { serialize: keyToJsonable } = makeMarshal(valToSlot);
+  const { serialize: keyToJsonable } = makeMarshal(valToSlot, undefined, {
+    serializeBodyFormat: 'smallcaps',
+  });
   const keyToString = async keyP => {
     const key = await sanitize(keyP);
     const obj = keyToJsonable(key);

packages/cache/test/test-storage.js

@@ -19,6 +19,7 @@ const makeSimpleMarshaller = () => {
   };
   const toVal = slot => vals[slot];
   return makeMarshal(fromVal, toVal, {
+    serializeBodyFormat: 'smallcaps',
     marshalSaveError: err => {
       throw err;
     },
@@ -47,21 +48,21 @@ test('makeChainStorageCoordinator with non-remote values', async t => {
   t.is(await cache('brandName', 'barbosa'), 'barbosa');
   t.deepEqual(Object.keys(storageNodeState), ['cache']);
   t.deepEqual(JSON.parse(storageNodeState.cache), {
-    '{"body":"\\"brandName\\"","slots":[]}': {
-      body: '{"generation":{"@qclass":"bigint","digits":"1"},"value":"barbosa"}',
+    '{"body":"#\\"brandName\\"","slots":[]}': {
+      body: '#{"generation":"+1","value":"barbosa"}',
       slots: [],
     },
   });
 
   // One-time initialization (of 'frotz')
   t.is(await cache('frotz', 'default'), 'default');
   const afterFirstFrotz = {
-    '{"body":"\\"brandName\\"","slots":[]}': {
-      body: '{"generation":{"@qclass":"bigint","digits":"1"},"value":"barbosa"}',
+    '{"body":"#\\"brandName\\"","slots":[]}': {
+      body: '#{"generation":"+1","value":"barbosa"}',
       slots: [],
     },
-    '{"body":"\\"frotz\\"","slots":[]}': {
-      body: '{"generation":{"@qclass":"bigint","digits":"1"},"value":"default"}',
+    '{"body":"#\\"frotz\\"","slots":[]}': {
+      body: '#{"generation":"+1","value":"default"}',
       slots: [],
     },
   };
@@ -83,8 +84,8 @@ test('makeChainStorageCoordinator with non-remote values', async t => {
   );
   t.deepEqual(JSON.parse(storageNodeState.cache), {
     ...afterFirstFrotz,
-    '{"body":"[\\"complex\\",\\"passable\\"]","slots":[]}': {
-      body: '{"generation":{"@qclass":"bigint","digits":"1"},"value":{"arr":["hi","there"],"big":{"@qclass":"bigint","digits":"1"},"num":53,"str":"string"}}',
+    '{"body":"#[\\"complex\\",\\"passable\\"]","slots":[]}': {
+      body: '#{"generation":"+1","value":{"arr":["hi","there"],"big":"+1","num":53,"str":"string"}}',
       slots: [],
     },
   });
@@ -98,8 +99,8 @@ test('makeChainStorageCoordinator with remote values', async t => {
   t.is(await cache('brand', farThing), farThing);
   t.deepEqual(Object.keys(storageNodeState), ['cache']);
   t.deepEqual(JSON.parse(storageNodeState.cache), {
-    '{"body":"\\"brand\\"","slots":[]}': {
-      body: '{"generation":{"@qclass":"bigint","digits":"1"},"value":{"@qclass":"slot","iface":"Alleged: farThing","index":0}}',
+    '{"body":"#\\"brand\\"","slots":[]}': {
+      body: '#{"generation":"+1","value":"$0.Alleged: farThing"}',
       slots: [0],
     },
   });
@@ -114,8 +115,8 @@ test('makeChainStorageCoordinator with updater', async t => {
   t.is(await cache('counter', increment), 1);
   t.deepEqual(Object.keys(storageNodeState), ['cache']);
   t.deepEqual(JSON.parse(storageNodeState.cache), {
-    '{"body":"\\"counter\\"","slots":[]}': {
-      body: '{"generation":{"@qclass":"bigint","digits":"1"},"value":1}',
+    '{"body":"#\\"counter\\"","slots":[]}': {
+      body: '#{"generation":"+1","value":1}',
       slots: [],
     },
   });
@@ -124,8 +125,8 @@ test('makeChainStorageCoordinator with updater', async t => {
   t.is(await cache('counter', increment), 1);
   t.deepEqual(Object.keys(storageNodeState), ['cache']);
   t.deepEqual(JSON.parse(storageNodeState.cache), {
-    '{"body":"\\"counter\\"","slots":[]}': {
-      body: '{"generation":{"@qclass":"bigint","digits":"1"},"value":1}',
+    '{"body":"#\\"counter\\"","slots":[]}': {
+      body: '#{"generation":"+1","value":1}',
       slots: [],
     },
   });
@@ -134,8 +135,8 @@ test('makeChainStorageCoordinator with updater', async t => {
   t.is(await cache('counter', increment, M.any()), 2);
   t.deepEqual(Object.keys(storageNodeState), ['cache']);
   t.deepEqual(JSON.parse(storageNodeState.cache), {
-    '{"body":"\\"counter\\"","slots":[]}': {
-      body: '{"generation":{"@qclass":"bigint","digits":"2"},"value":2}',
+    '{"body":"#\\"counter\\"","slots":[]}': {
+      body: '#{"generation":"+2","value":2}',
       slots: [],
     },
   });
@@ -155,8 +156,8 @@ test('makeChainStorageCoordinator with remote updater', async t => {
   t.is(await cache('counter', counterObj.increment), 1);
   t.deepEqual(Object.keys(storageNodeState), ['cache']);
   t.deepEqual(JSON.parse(storageNodeState.cache), {
-    '{"body":"\\"counter\\"","slots":[]}': {
-      body: '{"generation":{"@qclass":"bigint","digits":"1"},"value":1}',
+    '{"body":"#\\"counter\\"","slots":[]}': {
+      body: '#{"generation":"+1","value":1}',
       slots: [],
     },
   });
@@ -166,8 +167,8 @@ test('makeChainStorageCoordinator with remote updater', async t => {
   t.is(counter, 1);
   t.deepEqual(Object.keys(storageNodeState), ['cache']);
   t.deepEqual(JSON.parse(storageNodeState.cache), {
-    '{"body":"\\"counter\\"","slots":[]}': {
-      body: '{"generation":{"@qclass":"bigint","digits":"1"},"value":1}',
+    '{"body":"#\\"counter\\"","slots":[]}': {
+      body: '#{"generation":"+1","value":1}',
       slots: [],
     },
   });
@@ -177,8 +178,8 @@ test('makeChainStorageCoordinator with remote updater', async t => {
   t.is(counter, 2);
   t.deepEqual(Object.keys(storageNodeState), ['cache']);
   t.deepEqual(JSON.parse(storageNodeState.cache), {
-    '{"body":"\\"counter\\"","slots":[]}': {
-      body: '{"generation":{"@qclass":"bigint","digits":"2"},"value":2}',
+    '{"body":"#\\"counter\\"","slots":[]}': {
+      body: '#{"generation":"+2","value":2}',
       slots: [],
     },
   });

packages/casting/src/defaults.js

@@ -114,6 +114,8 @@ export const MAKE_DEFAULT_UNSERIALIZER = () => {
     return obj;
   };
   return Far('marshal unserializer', {
-    unserialize: makeMarshal(undefined, slotToVal).unserialize,
+    unserialize: makeMarshal(undefined, slotToVal, {
+      serializeBodyFormat: 'smallcaps',
+    }).unserialize,
   });
 };

packages/cosmic-swingset/src/chain-main.js

@@ -341,7 +341,9 @@ export default async function main(progname, args, { env, homedir, agcc }) {
         STORAGE_PATH.BUNDLES,
         { sequence: true },
       );
-      const marshaller = makeMarshal();
+      const marshaller = makeMarshal(undefined, undefined, {
+        serializeBodyFormat: 'smallcaps',
+      });
       const publish = makeSerializeToStorage(
         installationStorageNode,
         marshaller,

packages/governance/test/unitTests/snapshots/test-committee.js.md

@@ -16,9 +16,7 @@ Generated by [AVA](https://avajs.dev).
         'published.committees.Economic_Committee.latestOutcome',
         {
           outcome: 'fail',
-          question: {
-            iface: 'Alleged: QuestionHandle',
-          },
+          question: Object @Alleged: QuestionHandle {},
           reason: 'No quorum',
         },
       ],
@@ -27,13 +25,9 @@ Generated by [AVA](https://avajs.dev).
         {
           closingRule: {
             deadline: 1n,
-            timer: {
-              iface: 'Alleged: ManualTimer',
-            },
-          },
-          counterInstance: {
-            iface: 'Alleged: InstanceHandle',
+            timer: Object @Alleged: ManualTimer {},
           },
+          counterInstance: Object @Alleged: InstanceHandle {},
           electionType: 'survey',
           issue: {
             text: 'why3',
@@ -49,9 +43,7 @@ Generated by [AVA](https://avajs.dev).
               text: 'why not?',
             },
           ],
-          questionHandle: {
-            iface: 'Alleged: QuestionHandle',
-          },
+          questionHandle: Object @Alleged: QuestionHandle {},
           quorumRule: 'majority',
           tieOutcome: {
             text: 'why not?',

packages/governance/test/unitTests/test-committee.js

@@ -4,6 +4,7 @@ import '@agoric/zoe/exported.js';
 
 import path from 'path';
 import { E } from '@endo/eventual-send';
+import { Far } from '@endo/far';
 import { makeZoeKit } from '@agoric/zoe';
 import fakeVatAdmin from '@agoric/zoe/tools/fakeVatAdmin.js';
 import bundleSource from '@endo/bundle-source';
@@ -81,7 +82,7 @@ test('committee-open question:one', async t => {
     maxChoices: 1,
     maxWinners: 1,
     closingRule: {
-      timer: buildManualTimer(t.log),
+      timer: harden(buildManualTimer(t.log)),
       deadline: 2n,
     },
     quorumRule: QuorumRule.MAJORITY,
@@ -102,13 +103,9 @@ test('committee-open question:one', async t => {
     {
       closingRule: {
         deadline: 2n,
-        timer: {
-          iface: 'Alleged: ManualTimer',
-        },
-      },
-      counterInstance: {
-        iface: 'Alleged: InstanceHandle',
+        timer: Far('ManualTimer'),
       },
+      counterInstance: Far('InstanceHandle'),
       electionType: 'survey',
       issue: {
         text: 'why',
@@ -124,9 +121,7 @@ test('committee-open question:one', async t => {
           text: 'why not?',
         },
       ],
-      questionHandle: {
-        iface: 'Alleged: QuestionHandle',
-      },
+      questionHandle: Far('QuestionHandle'),
       quorumRule: 'majority',
       tieOutcome: {
         text: 'why not?',