test(cosmic-swingset): Introduce a swingset instantiation discontinui…

…ty to the run policy test

An unrelated CI test experienced a failure when budgeted vat cleanup had
already removed a strict subset of a slowly-terminating vat's kvStore
entries that included the one with key "${vatID}.o.nextID"...
`provideVatKeeper` uses such keys as a "vat is alive" predicate that
must pass before it spins up any new keeper (whose primary purpose is
providing an interface for a *live* vat).

This is not an issue within a single swingset instantiation, in which
`provideVatKeeper` is memoized. But that cache starts empty with each
process, and the attempt to instantiate a new keeper for a slowly-
terminating vat in this state fails.

run-policy.test.js now covers this scenario.

packages/cosmic-swingset/test/run-policy.test.js

@@ -50,22 +50,23 @@ test('cleanup work must be limited by vat_cleanup_budget', async t => {
         Fail`port ${q(destPort)} not implemented for message ${msg}`;
     }
   };
+  const options = {
+    bundles: makeSourceDescriptors({
+      puppet: '@agoric/swingset-vat/tools/vat-puppet.js',
+    }),
+    configOverrides: {
+      // Ensure multiple spans and snapshots.
+      defaultManagerType: 'xsnap',
+      snapshotInitial: 2,
+      snapshotInterval: 4,
+    },
+    fixupBootMsg: () => ({
+      ...defaultBootstrapMessage,
+      params: makeCleanupBudgetParams({ Default: 0 }),
+    }),
+  };
   const { pushCoreEval, runNextBlock, shutdown, swingStore } =
-    await makeCosmicSwingsetTestKit(receiveBridgeSend, {
-      bundles: makeSourceDescriptors({
-        puppet: '@agoric/swingset-vat/tools/vat-puppet.js',
-      }),
-      configOverrides: {
-        // Ensure multiple spans and snapshots.
-        defaultManagerType: 'xsnap',
-        snapshotInitial: 2,
-        snapshotInterval: 4,
-      },
-      fixupBootMsg: () => ({
-        ...defaultBootstrapMessage,
-        params: makeCleanupBudgetParams({ Default: 0 }),
-      }),
-    });
+    await makeCosmicSwingsetTestKit(receiveBridgeSend, options);
   const mapStore = provideEnhancedKVStore(
     /** @type {KVStore<string>} */ (swingStore.kernelStorage.kvStore),
   );
@@ -89,6 +90,10 @@ test('cleanup work must be limited by vat_cleanup_budget', async t => {
     'v8',
     `time to update expected vatID to ${JSON.stringify(vatIDs)}.at(-1)?`,
   );
+  // This key is/was used as a predicate for vat liveness.
+  // https://github.com/Agoric/agoric-sdk/blob/7ae1f278fa8cbeb0cfc777b7cebf507b1f07c958/packages/SwingSet/src/kernel/state/kernelKeeper.js#L1706
+  const sentinelKey = `${vatID}.o.nextID`;
+  t.true(mapStore.has(sentinelKey));
   const getKV = () =>
     [...mapStore].filter(([key]) => key.startsWith(`${vatID}.`));
   // console.log(swingStore.kernelStorage);
@@ -182,15 +187,36 @@ test('cleanup work must be limited by vat_cleanup_budget', async t => {
     params: makeCleanupBudgetParams({ Default: 2 ** 32, Kv: 3 }),
   });
   t.is(getKV().length, onlyKV.length - 3, 'initial kvStore deletion');
-  await runNextBlock();
+  let lastBlockInfo = await runNextBlock();
   t.is(getKV().length, onlyKV.length - 6, 'further kvStore deletion');
 
-  // Allow remaining cleanup.
-  await runNextBlock({
-    params: makeCleanupBudgetParams({ Default: 2 ** 32 }),
-  });
-  await runNextBlock();
-  t.is(getKV().length, 0, 'cleanup complete');
+  // Wait for the sentinel key to be removed, then re-instantiate the swingset
+  // and allow remaining cleanup.
+  while (mapStore.has(sentinelKey)) {
+    lastBlockInfo = await runNextBlock();
+  }
+  await shutdown({ kernelOnly: true });
+  {
+    // Pick up where we left off with the same data and block,
+    // but with a new budget.
+    const newOptions = {
+      ...options,
+      swingStore,
+      fixupBootMsg: () => ({
+        ...defaultBootstrapMessage,
+        ...lastBlockInfo,
+        params: makeCleanupBudgetParams({ Default: 2 ** 32 }),
+      }),
+    };
+    // eslint-disable-next-line no-shadow
+    const { runNextBlock, shutdown } = await makeCosmicSwingsetTestKit(
+      receiveBridgeSend,
+      newOptions,
+    );
+
+    await runNextBlock();
+    t.is(getKV().length, 0, 'cleanup complete');
 
-  await shutdown();
+    await shutdown();
+  }
 });

packages/cosmic-swingset/tools/test-kit.js

@@ -134,6 +134,8 @@ const baseConfig = harden({
  *   to make any changes
  * @param {import('@agoric/telemetry').SlogSender} [options.slogSender]
  * @param {import('../src/chain-main.js').CosmosSwingsetConfig} [options.swingsetConfig]
+ * @param {import('@agoric/swing-store').SwingStore} [options.swingStore]
+ *   defaults to a new in-memory store
  * @param {SwingSetConfig['vats']} [options.vats] extra static vat configuration
  * @param {string} [options.baseBootstrapManifest] identifies the colletion of
  *   "behaviors" to run at bootstrap for creating and configuring the initial
@@ -162,6 +164,7 @@ export const makeCosmicSwingsetTestKit = async (
     fixupConfig,
     slogSender,
     swingsetConfig = {},
+    swingStore = initSwingStore(), // in-memory
     vats,
 
     // Options for vats (particularly the reflective bootstrap vat).
@@ -196,7 +199,6 @@ export const makeCosmicSwingsetTestKit = async (
 
   if (fixupConfig) config = fixupConfig(config);
 
-  const swingStore = initSwingStore(); // in-memory
   const { hostStorage } = swingStore;
 
   const actionQueueStorage = makeQueueStorageMock().storage;
@@ -233,7 +235,7 @@ export const makeCosmicSwingsetTestKit = async (
     slogSender = await makeSlogSender({ env });
   }
 
-  const { blockingSend, shutdown: shutdownKernel } = await launch({
+  const launchResult = await launch({
     swingStore,
     actionQueueStorage,
     highPriorityQueueStorage,
@@ -248,8 +250,12 @@ export const makeCosmicSwingsetTestKit = async (
     slogSender,
     swingsetConfig,
   });
-  const shutdown = async () => {
-    await Promise.all([shutdownKernel, hostStorage.close()]);
+  const { blockingSend, shutdown: shutdownKernel } = launchResult;
+  /** @type {(options?: { kernelOnly?: boolean }) => Promise<void>} */
+  const shutdown = async ({ kernelOnly = false } = {}) => {
+    await shutdownKernel();
+    if (kernelOnly) return;
+    await hostStorage.close();
   };
 
   /**
@@ -294,7 +300,6 @@ export const makeCosmicSwingsetTestKit = async (
     });
     return getLastBlockInfo();
   };
-  await runNextBlock();
 
   /** @type {InboundQueue} */
   const actionQueue = makeQueue(actionQueueStorage);