From 71c85ecb372c321d5a1f935952aa31b510007498 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Fri, 21 May 2021 14:33:00 -0700 Subject: [PATCH] fix(SwingSet): VOM tracks Presence vrefs in virtualized data If userspace puts a Presence into the `state` of a virtual object, or somewhere inside the value stored in vref-keyed `makeWeakStore()` entry, it gets serialized and stored as a vref, which doesn't (and should not) keep the Presence object alive. Allowing this Presence to leave RAM, remembering only the vref on disk, is a non-trivial part of the memory savings we obtain by using virtualized data. However, just because there is currently no Presence (for a given vref) does *not* mean that the vat cannot reach the vref. Liveslots will observe the Presence being collected (when the finalizer runs), but if the vref is still stored somewhere in virtualized data, liveslots must not emit a `syscall.dropImport` for it. This changes the virtual object manager to keep track of Presences used in virtualized data, and remember their vref in a Set. When liveslots' wants to `dropImport` a vref that no longer has a Presence, it will ask the VOM first. With this Set, the VOM can inhibit the `dropImport` call until later. At this stage, we simply add the vref to a Set and never remove it. This is safe but conservative. In the future, we'll need some form of refcounting to detect when the vref is no longer mentioned anywhere in virtualized data. At that point, the VOM will need to inform liveslots (or some sort of "reachability manager") that the VOM no longer needs the vref kept alive. The `syscall.dropImport` can be sent when neither the VOM nor a Presence is causing the vref to remain reachable. closes #3133 refs #3106 --- .../src/kernel/virtualObjectManager.js | 31 +++++++ .../virtualObjects/test-reachable-vrefs.js | 88 +++++++++++++++++++ .../tools/fakeVirtualObjectManager.js | 2 + 3 files changed, 121 insertions(+) create mode 100644 packages/SwingSet/test/virtualObjects/test-reachable-vrefs.js diff --git a/packages/SwingSet/src/kernel/virtualObjectManager.js b/packages/SwingSet/src/kernel/virtualObjectManager.js index 2aa203b4bb9..ceee417149e 100644 --- a/packages/SwingSet/src/kernel/virtualObjectManager.js +++ b/packages/SwingSet/src/kernel/virtualObjectManager.js @@ -196,6 +196,32 @@ export function makeVirtualObjectManager( */ const kindTable = new Map(); + /** + * Set of all import vrefs which are reachable from our virtualized data. + * These were Presences at one point. We add to this set whenever we store + * a Presence into the state of a virtual object, or the value of a + * makeWeakStore() instance. We currently never remove anything from the + * set, but eventually we'll use refcounts within virtual data to figure + * out when the vref becomes unreachable, allowing the vat do send a + * dropImport into the kernel and release the object. + * + */ + /** @type {Set} of vrefs */ + const reachableVrefs = new Set(); + + // We track imports, to preserve their vrefs against syscall.dropImport + // when the Presence goes away. + function addReachablePresence(vref) { + const { type, allocatedByVat } = parseVatSlot(vref); + if (type === 'object' && !allocatedByVat) { + reachableVrefs.add(vref); + } + } + + function isVrefReachable(vref) { + return reachableVrefs.has(vref); + } + /** * Set of all Remotables which are reachable by our virtualized data, e.g. * `makeWeakStore().set(key, remotable)` or `virtualObject.state.foo = @@ -305,6 +331,7 @@ export function makeVirtualObjectManager( X`${q(keyName)} already registered: ${key}`, ); const data = m.serialize(value); + data.slots.map(addReachablePresence); data.slots.map(addReachableRemotable); syscall.vatstoreSet(vkey, JSON.stringify(data)); } else { @@ -328,6 +355,7 @@ export function makeVirtualObjectManager( if (vkey) { assert(syscall.vatstoreGet(vkey), X`${q(keyName)} not found: ${key}`); const data = m.serialize(harden(value)); + data.slots.map(addReachablePresence); data.slots.map(addReachableRemotable); syscall.vatstoreSet(vkey, JSON.stringify(data)); } else { @@ -566,6 +594,7 @@ export function makeVirtualObjectManager( }, set: value => { const serializedValue = m.serialize(value); + serializedValue.slots.map(addReachablePresence); serializedValue.slots.map(addReachableRemotable); ensureState(); innerSelf.rawData[prop] = serializedValue; @@ -625,6 +654,7 @@ export function makeVirtualObjectManager( for (const prop of Object.getOwnPropertyNames(initialData)) { try { const data = m.serialize(initialData[prop]); + data.slots.map(addReachablePresence); data.slots.map(addReachableRemotable); rawData[prop] = data; } catch (e) { @@ -649,6 +679,7 @@ export function makeVirtualObjectManager( makeKind, VirtualObjectAwareWeakMap, VirtualObjectAwareWeakSet, + isVrefReachable, flushCache: cache.flush, makeVirtualObjectRepresentative, }); diff --git a/packages/SwingSet/test/virtualObjects/test-reachable-vrefs.js b/packages/SwingSet/test/virtualObjects/test-reachable-vrefs.js new file mode 100644 index 00000000000..64846115cce --- /dev/null +++ b/packages/SwingSet/test/virtualObjects/test-reachable-vrefs.js @@ -0,0 +1,88 @@ +import { test } from '../../tools/prepare-test-env-ava'; + +// eslint-disable-next-line import/order +import { Far, Remotable } from '@agoric/marshal'; + +import { makeVatSlot } from '../../src/parseVatSlots'; +import { makeFakeVirtualObjectManager } from '../../tools/fakeVirtualObjectManager'; + +// empty object, used as makeWeakStore() key +function makeKeyInstance(_state) { + return { + init() {}, + self: Far('key'), + }; +} + +function makeHolderInstance(state) { + return { + init(held) { + state.held = held; + }, + self: Far('holder', { + setHeld(held) { + state.held = held; + }, + getHeld() { + return state.held; + }, + }), + }; +} + +test('VOM tracks reachable vrefs', async t => { + const vomOptions = { cacheSize: 3 }; + const vom = makeFakeVirtualObjectManager(vomOptions); + const { makeWeakStore, makeKind } = vom; + const weakStore = makeWeakStore(); + const keyMaker = makeKind(makeKeyInstance); + const holderMaker = makeKind(makeHolderInstance); + + let count = 1001; + function makePresence() { + // Both Remotable() and the Far() convenience wrapper mark things as + // pass-by-reference. They are used when creating an (imported) Presence, + // not just an (exported) "Remotable". + const pres = Remotable(`Alleged: presence-${count}`, undefined, {}); + const vref = makeVatSlot('object', false, count); + vom.registerEntry(vref, pres); + count += 1; + return [vref, pres]; + } + + const [vref1, obj1] = makePresence(); + const key1 = keyMaker(); + t.falsy(vom.isVrefReachable(vref1)); + weakStore.init(key1, obj1); + t.truthy(vom.isVrefReachable(vref1)); + + const [vref2, obj2] = makePresence(); + const key2 = keyMaker(); + weakStore.init(key2, 'not yet'); + t.falsy(vom.isVrefReachable(vref2)); + weakStore.set(key2, obj2); + t.truthy(vom.isVrefReachable(vref2)); + + // storing Presences as the value for a non-virtual key just holds on to + // the Presence directly, and does not track the vref + + const [vref3, obj3] = makePresence(); + const key3 = {}; + weakStore.init(key3, obj3); + weakStore.set(key3, obj3); + t.falsy(vom.isVrefReachable(vref3)); + + // now check that Presences are tracked when in the state of a virtual + // object + const [vref4, obj4] = makePresence(); + t.falsy(vom.isVrefReachable(vref4)); + // eslint-disable-next-line no-unused-vars + const holder4 = holderMaker(obj4); + t.truthy(vom.isVrefReachable(vref4)); + + const [vref5, obj5] = makePresence(); + const holder5 = holderMaker('not yet'); + t.falsy(vom.isVrefReachable(vref5)); + holder5.setHeld(obj5); + t.truthy(vom.isVrefReachable(vref5)); +}); diff --git a/packages/SwingSet/tools/fakeVirtualObjectManager.js b/packages/SwingSet/tools/fakeVirtualObjectManager.js index 0826841ca83..0dc7109e83e 100644 --- a/packages/SwingSet/tools/fakeVirtualObjectManager.js +++ b/packages/SwingSet/tools/fakeVirtualObjectManager.js @@ -106,6 +106,7 @@ export function makeFakeVirtualObjectManager(options = {}) { makeKind, VirtualObjectAwareWeakMap, VirtualObjectAwareWeakSet, + isVrefReachable, flushCache, } = makeVirtualObjectManager( fakeSyscall, @@ -122,6 +123,7 @@ export function makeFakeVirtualObjectManager(options = {}) { makeWeakStore, VirtualObjectAwareWeakMap, VirtualObjectAwareWeakSet, + isVrefReachable, }; const debugTools = {