From 21c62b23adfae7118a37f5b25cb6a57fcf19558e Mon Sep 17 00:00:00 2001 From: hotpocket Date: Thu, 5 Dec 2024 04:34:30 -0800 Subject: [PATCH 01/18] move reusable grpc components into a base class --- src/extension/serializer.ts | 175 ++++++++++++++++++++---------------- 1 file changed, 97 insertions(+), 78 deletions(-) diff --git a/src/extension/serializer.ts b/src/extension/serializer.ts index 6c514bccc..0c54f4225 100644 --- a/src/extension/serializer.ts +++ b/src/extension/serializer.ts @@ -473,36 +473,23 @@ export class WasmSerializer extends SerializerBase { } } -export class GrpcSerializer extends SerializerBase { - private client?: ParserServiceClient - protected ready: ReadyPromise +// no common ancestor, any type used for protos +export abstract class GrpcBaseSerializer extends SerializerBase { + protected client: any + // todo(sebastian): naive cache for now, consider use lifecycle events for gc protected readonly plainCache = new Map>() protected readonly maskedCache = new Map>() protected readonly notebookDataCache = new Map() protected readonly cacheDocUriMapping: Map = new Map() - private serverReadyListener: Disposable | undefined - constructor( protected context: ExtensionContext, - protected server: IServer, kernel: Kernel, ) { super(context, kernel) - this.togglePreviewButton(GrpcSerializer.sessionOutputsEnabled()) - - this.ready = new Promise((resolve) => { - const disposable = server.onTransportReady(() => { - disposable.dispose() - resolve() - }) - }) - - this.serverReadyListener = server.onTransportReady(({ transport }) => - this.initParserClient(transport), - ) + this.togglePreviewButton(GrpcBaseSerializer.sessionOutputsEnabled()) this.disposables.push( // todo(sebastian): delete entries on session reset not notebook editor lifecycle @@ -512,23 +499,21 @@ export class GrpcSerializer extends SerializerBase { ) } - private async initParserClient(transport?: GrpcTransport) { - this.client = initParserClient(transport ?? (await this.server.transport())) - } + protected abstract cacheNotebookOutputs(notebook: any, cacheId: string | undefined): Promise public togglePreviewButton(state: boolean) { return commands.executeCommand('setContext', NOTEBOOK_HAS_OUTPUTS, state) } protected async handleOpenNotebook(doc: NotebookDocument) { - const cacheId = GrpcSerializer.getDocumentCacheId(doc.metadata) + const cacheId = GrpcBaseSerializer.getDocumentCacheId(doc.metadata) if (!cacheId) { this.togglePreviewButton(false) return } - if (GrpcSerializer.isDocumentSessionOutputs(doc.metadata)) { + if (GrpcBaseSerializer.isDocumentSessionOutputs(doc.metadata)) { this.togglePreviewButton(false) return } @@ -536,8 +521,8 @@ export class GrpcSerializer extends SerializerBase { this.cacheDocUriMapping.set(cacheId, doc.uri) } - protected async handleCloseNotebook(doc: NotebookDocument) { - const cacheId = GrpcSerializer.getDocumentCacheId(doc.metadata) + async handleCloseNotebook(doc: NotebookDocument) { + const cacheId = GrpcBaseSerializer.getDocumentCacheId(doc.metadata) /** * Remove cache */ @@ -547,8 +532,8 @@ export class GrpcSerializer extends SerializerBase { } } - protected async handleSaveNotebookOutputs(doc: NotebookDocument) { - const cacheId = GrpcSerializer.getDocumentCacheId(doc.metadata) + async handleSaveNotebookOutputs(doc: NotebookDocument) { + const cacheId = GrpcBaseSerializer.getDocumentCacheId(doc.metadata) if (!cacheId) { this.togglePreviewButton(false) @@ -590,7 +575,7 @@ export class GrpcSerializer extends SerializerBase { return bytes.length } - const sessionFile = GrpcSerializer.getOutputsUri(srcDocUri, sessionId) + const sessionFile = GrpcBaseSerializer.getOutputsUri(srcDocUri, sessionId) if (!sessionFile) { this.togglePreviewButton(false) return -1 @@ -605,7 +590,7 @@ export class GrpcSerializer extends SerializerBase { public async saveNotebookOutputs(uri: Uri): Promise { let cacheId: string | undefined this.cacheDocUriMapping.forEach((docUri, cid) => { - const src = GrpcSerializer.getSourceFileUri(uri) + const src = GrpcBaseSerializer.getSourceFileUri(uri) if (docUri.fsPath.toString() === src.fsPath.toString()) { cacheId = cid } @@ -627,7 +612,7 @@ export class GrpcSerializer extends SerializerBase { } public static getOutputsUri(docUri: Uri, sessionId: string): Uri { - return Uri.parse(GrpcSerializer.getOutputsFilePath(docUri.fsPath, sessionId)) + return Uri.parse(GrpcBaseSerializer.getOutputsFilePath(docUri.fsPath, sessionId)) } public static getSourceFilePath(outputsFile: string): string { @@ -645,28 +630,26 @@ export class GrpcSerializer extends SerializerBase { } public static getSourceFileUri(outputsUri: Uri): Uri { - return Uri.parse(GrpcSerializer.getSourceFilePath(outputsUri.fsPath)) + return Uri.parse(GrpcBaseSerializer.getSourceFilePath(outputsUri.fsPath)) } - protected applyIdentity(data: Notebook): Notebook { - const identity = this.lifecycleIdentity - switch (identity) { - case RunmeIdentity.UNSPECIFIED: - case RunmeIdentity.DOCUMENT: - break - default: { - data.cells.forEach((cell) => { - if (cell.kind !== CellKind.CODE) { - return - } - if (!cell.metadata?.['id'] && cell.metadata?.['runme.dev/id']) { - cell.metadata['id'] = cell.metadata['runme.dev/id'] - } - }) - } - } + public getMaskedCache(cacheId: string): Promise | undefined { + return this.maskedCache.get(cacheId) + } - return data + public getPlainCache(cacheId: string): Promise | undefined { + return this.plainCache.get(cacheId) + } + + public getNotebookDataCache(cacheId: string): NotebookData | undefined { + return this.notebookDataCache.get(cacheId) + } + + static sessionOutputsEnabled() { + const isAutoSaveOn = ContextState.getKey(NOTEBOOK_AUTOSAVE_ON) + const isSessionOutputs = getSessionOutputs() + + return isSessionOutputs && isAutoSaveOn } public static getDocumentCacheId( @@ -691,6 +674,60 @@ export class GrpcSerializer extends SerializerBase { return Boolean(sessionOutputId) } + // unable to implement marshal methods here, the underlying object structure is the same but + // the data types of the properties are different in some cases, presumably due to compile flags +} + +export class GrpcSerializer extends GrpcBaseSerializer { + protected client!: ParserServiceClient + protected ready: ReadyPromise + + private serverReadyListener: Disposable | undefined + + constructor( + protected context: ExtensionContext, + protected server: IServer, + kernel: Kernel, + ) { + super(context, kernel) + + this.ready = new Promise((resolve) => { + const disposable = server.onTransportReady(() => { + disposable.dispose() + resolve() + }) + }) + + this.serverReadyListener = server.onTransportReady(({ transport }) => + this.initParserClient(transport), + ) + } + + private async initParserClient(transport?: GrpcTransport) { + this.client = initParserClient(transport ?? (await this.server.transport())) + } + + protected applyIdentity(data: Notebook): Notebook { + const identity = this.lifecycleIdentity + switch (identity) { + case RunmeIdentity.UNSPECIFIED: + case RunmeIdentity.DOCUMENT: + break + default: { + data.cells.forEach((cell) => { + if (cell.kind !== CellKind.CODE) { + return + } + if (!cell.metadata?.['id'] && cell.metadata?.['runme.dev/id']) { + cell.metadata['id'] = cell.metadata['runme.dev/id'] + } + }) + } + } + + return data + } + public override async switchLifecycleIdentity( notebook: NotebookDocument, identity: RunmeIdentity, @@ -702,7 +739,7 @@ export class GrpcSerializer extends SerializerBase { await notebook.save() const source = await workspace.fs.readFile(notebook.uri) - const des = await this.client!.deserialize( + const des = await this.client.deserialize( DeserializeRequest.create({ source, options: { identity }, @@ -747,13 +784,13 @@ export class GrpcSerializer extends SerializerBase { data.metadata[RUNME_FRONTMATTER_PARSED] = notebook.frontmatter } - const cacheId = GrpcSerializer.getDocumentCacheId(data.metadata) + const cacheId = GrpcBaseSerializer.getDocumentCacheId(data.metadata) this.notebookDataCache.set(cacheId as string, data) const serialRequest = { notebook } const cacheOutputs = this.cacheNotebookOutputs(notebook, cacheId) - const request = this.client!.serialize(serialRequest) + const request = this.client.serialize(serialRequest) // run in parallel const [serialResult] = await Promise.all([request, cacheOutputs]) @@ -770,14 +807,8 @@ export class GrpcSerializer extends SerializerBase { return result } - static sessionOutputsEnabled() { - const isAutoSaveOn = ContextState.getKey(NOTEBOOK_AUTOSAVE_ON) - const isSessionOutputs = getSessionOutputs() - - return isSessionOutputs && isAutoSaveOn - } - - private async cacheNotebookOutputs( + // unable to abstract due to RunmeSession struct potential differences & notebook ts type validation issues + protected async cacheNotebookOutputs( notebook: Notebook, cacheId: string | undefined, ): Promise { @@ -812,10 +843,10 @@ export class GrpcSerializer extends SerializerBase { }) const plainReq = { notebook, options } - const plainRes = this.client!.serialize(plainReq) + const plainRes = this.client.serialize(plainReq) const maskedReq = { notebook: maskedNotebook, options } - const masked = this.client!.serialize(maskedReq).then((maskedRes) => { + const masked = this.client.serialize(maskedReq).then((maskedRes) => { if (maskedRes.response.result === undefined) { console.error('serialization of masked notebook failed') return Promise.resolve(new Uint8Array()) @@ -844,7 +875,7 @@ export class GrpcSerializer extends SerializerBase { await Promise.all([plain, masked]) } - // marshalNotebook converts VSCode's NotebookData to the Notebook proto. + // vscode/NotebookData to timostam-protobuf-ts/Notebook public static marshalNotebook( data: NotebookData, config?: { @@ -964,7 +995,7 @@ export class GrpcSerializer extends SerializerBase { return outputs } - private static marshalCellExecutionSummary( + protected static marshalCellExecutionSummary( executionSummary: NotebookCellExecutionSummary | undefined, ) { if (!executionSummary) { @@ -992,13 +1023,13 @@ export class GrpcSerializer extends SerializerBase { ): Promise { const identity = this.lifecycleIdentity const deserialRequest = DeserializeRequest.create({ source: content, options: { identity } }) - const request = await this.client!.deserialize(deserialRequest) + const request = await this.client.deserialize(deserialRequest) const { notebook } = request.response if (notebook === undefined) { throw new Error('deserialization failed to revive notebook') } - + // this new variable is not needed as notebook is modified in place const _notebook = this.applyIdentity(notebook) // we can remove ugly casting once we switch to GRPC @@ -1009,16 +1040,4 @@ export class GrpcSerializer extends SerializerBase { this.serverReadyListener?.dispose() super.dispose() } - - public getMaskedCache(cacheId: string): Promise | undefined { - return this.maskedCache.get(cacheId) - } - - public getPlainCache(cacheId: string): Promise | undefined { - return this.plainCache.get(cacheId) - } - - public getNotebookDataCache(cacheId: string): NotebookData | undefined { - return this.notebookDataCache.get(cacheId) - } } From 1000516849483057346fdbc8cd58b8cba8a5d77a Mon Sep 17 00:00:00 2001 From: hotpocket Date: Thu, 5 Dec 2024 07:29:24 -0800 Subject: [PATCH 02/18] Add ConnectSerializer and the code to select and run it. Refactor kernel slightly to use new base class. --- src/extension/extension.ts | 23 +- src/extension/kernel.ts | 10 +- src/extension/serializer.ts | 424 ++++++++++++++++++++++++++++++++++-- 3 files changed, 434 insertions(+), 23 deletions(-) diff --git a/src/extension/extension.ts b/src/extension/extension.ts index 4e2e4385e..b6cb026c3 100644 --- a/src/extension/extension.ts +++ b/src/extension/extension.ts @@ -73,7 +73,13 @@ import { runForkCommand, selectEnvironment, } from './commands' -import { WasmSerializer, GrpcSerializer, SerializerBase } from './serializer' +import { + WasmSerializer, + GrpcSerializer, + SerializerBase, + ConnectSerializer, + GrpcSerializerBase, +} from './serializer' import { RunmeLauncherProvider, RunmeTreeProvider } from './provider/launcher' import { RunmeLauncherProvider as RunmeLauncherProviderBeta } from './provider/launcherBeta' import { RunmeUriHandler } from './handler/uri' @@ -104,6 +110,8 @@ export class RunmeExtension { const grpcSerializer = kernel.hasExperimentEnabled('grpcSerializer') const grpcServer = kernel.hasExperimentEnabled('grpcServer') const grpcRunner = kernel.hasExperimentEnabled('grpcRunner') + const config = workspace.getConfiguration('runme') + const serializerAddress = config.get('serializerAddress') const server = new KernelServer( context.extensionUri, @@ -128,11 +136,16 @@ export class RunmeExtension { ) const reporter = new GrpcReporter(context, server) - const serializer = grpcSerializer - ? new GrpcSerializer(context, server, kernel) - : new WasmSerializer(context, kernel) + let serializer: SerializerBase + if (serializerAddress && serializerAddress.length > 0) { + serializer = new ConnectSerializer(context, serializerAddress, kernel) + } else if (grpcSerializer) { + serializer = new GrpcSerializer(context, server, kernel) + } else { + serializer = new WasmSerializer(context, kernel) + } this.serializer = serializer - kernel.setSerializer(serializer as GrpcSerializer) + kernel.setSerializer(serializer as GrpcSerializerBase) kernel.setReporter(reporter) let treeViewer: RunmeTreeProvider diff --git a/src/extension/kernel.ts b/src/extension/kernel.ts index 0dc107534..2b01b5660 100644 --- a/src/extension/kernel.ts +++ b/src/extension/kernel.ts @@ -101,7 +101,7 @@ import { handleCellOutputMessage } from './messages/cellOutput' import handleGitHubMessage, { handleGistMessage } from './messages/github' import { getNotebookCategories } from './utils' import PanelManager from './panels/panelManager' -import { GrpcSerializer, SerializerBase } from './serializer' +import { GrpcSerializerBase } from './serializer' import { askAlternativeOutputsAction, openSplitViewAsMarkdownText } from './commands' import { handlePlatformApiMessage } from './messages/platformRequest' import { handleGCPMessage } from './messages/gcp' @@ -148,7 +148,7 @@ export class Kernel implements Disposable { protected activeTerminals: ActiveTerminal[] = [] protected category?: string protected panelManager: PanelManager - protected serializer?: SerializerBase + protected serializer?: GrpcSerializerBase protected reporter?: GrpcReporter protected featuresState$? @@ -270,7 +270,7 @@ export class Kernel implements Disposable { this.category = category } - setSerializer(serializer: GrpcSerializer) { + setSerializer(serializer: GrpcSerializerBase) { this.serializer = serializer } @@ -315,7 +315,7 @@ export class Kernel implements Disposable { } async #setNotebookMode(notebookDocument: NotebookDocument): Promise { - const isSessionsOutput = GrpcSerializer.isDocumentSessionOutputs(notebookDocument.metadata) + const isSessionsOutput = GrpcSerializerBase.isDocumentSessionOutputs(notebookDocument.metadata) const notebookMode = isSessionsOutput ? NotebookMode.SessionOutputs : NotebookMode.Execution await ContextState.addKey(NOTEBOOK_MODE, notebookMode) } @@ -668,7 +668,7 @@ export class Kernel implements Disposable { private async _executeAll(cells: NotebookCell[]) { const sessionOutputsDoc = cells.find((c) => - GrpcSerializer.isDocumentSessionOutputs(c.notebook.metadata), + GrpcSerializerBase.isDocumentSessionOutputs(c.notebook.metadata), ) if (sessionOutputsDoc) { const { notebook } = sessionOutputsDoc diff --git a/src/extension/serializer.ts b/src/extension/serializer.ts index 0c54f4225..66158612b 100644 --- a/src/extension/serializer.ts +++ b/src/extension/serializer.ts @@ -1,4 +1,5 @@ import path from 'node:path' +import fs from 'node:fs' import { NotebookSerializer, @@ -23,6 +24,11 @@ import { GrpcTransport } from '@protobuf-ts/grpc-transport' import { ulid } from 'ulidx' import { maskString } from 'data-guardian' import YAML from 'yaml' +import { ParserService } from '@buf/stateful_runme.connectrpc_es/runme/parser/v1/parser_connect' +import { createGrpcTransport, GrpcTransportOptions } from '@connectrpc/connect-node' +import { createPromiseClient, PromiseClient } from '@connectrpc/connect' +// ts bindings generated by protoc-gen-es +import * as es_proto from '@buf/stateful_runme.bufbuild_es/runme/parser/v1/parser_pb' import { Serializer } from '../types' import { @@ -46,6 +52,7 @@ import { SerializeRequestOptions, RunmeSession, Frontmatter, + CellExecutionSummary, } from './grpc/serializerTypes' import { initParserClient, ParserServiceClient, type ReadyPromise } from './grpc/client' import Languages from './languages' @@ -474,7 +481,7 @@ export class WasmSerializer extends SerializerBase { } // no common ancestor, any type used for protos -export abstract class GrpcBaseSerializer extends SerializerBase { +export abstract class GrpcSerializerBase extends SerializerBase { protected client: any // todo(sebastian): naive cache for now, consider use lifecycle events for gc @@ -489,7 +496,7 @@ export abstract class GrpcBaseSerializer extends SerializerBase { ) { super(context, kernel) - this.togglePreviewButton(GrpcBaseSerializer.sessionOutputsEnabled()) + this.togglePreviewButton(GrpcSerializerBase.sessionOutputsEnabled()) this.disposables.push( // todo(sebastian): delete entries on session reset not notebook editor lifecycle @@ -506,14 +513,14 @@ export abstract class GrpcBaseSerializer extends SerializerBase { } protected async handleOpenNotebook(doc: NotebookDocument) { - const cacheId = GrpcBaseSerializer.getDocumentCacheId(doc.metadata) + const cacheId = GrpcSerializerBase.getDocumentCacheId(doc.metadata) if (!cacheId) { this.togglePreviewButton(false) return } - if (GrpcBaseSerializer.isDocumentSessionOutputs(doc.metadata)) { + if (GrpcSerializerBase.isDocumentSessionOutputs(doc.metadata)) { this.togglePreviewButton(false) return } @@ -522,7 +529,7 @@ export abstract class GrpcBaseSerializer extends SerializerBase { } async handleCloseNotebook(doc: NotebookDocument) { - const cacheId = GrpcBaseSerializer.getDocumentCacheId(doc.metadata) + const cacheId = GrpcSerializerBase.getDocumentCacheId(doc.metadata) /** * Remove cache */ @@ -533,7 +540,7 @@ export abstract class GrpcBaseSerializer extends SerializerBase { } async handleSaveNotebookOutputs(doc: NotebookDocument) { - const cacheId = GrpcBaseSerializer.getDocumentCacheId(doc.metadata) + const cacheId = GrpcSerializerBase.getDocumentCacheId(doc.metadata) if (!cacheId) { this.togglePreviewButton(false) @@ -575,7 +582,7 @@ export abstract class GrpcBaseSerializer extends SerializerBase { return bytes.length } - const sessionFile = GrpcBaseSerializer.getOutputsUri(srcDocUri, sessionId) + const sessionFile = GrpcSerializerBase.getOutputsUri(srcDocUri, sessionId) if (!sessionFile) { this.togglePreviewButton(false) return -1 @@ -590,7 +597,7 @@ export abstract class GrpcBaseSerializer extends SerializerBase { public async saveNotebookOutputs(uri: Uri): Promise { let cacheId: string | undefined this.cacheDocUriMapping.forEach((docUri, cid) => { - const src = GrpcBaseSerializer.getSourceFileUri(uri) + const src = GrpcSerializerBase.getSourceFileUri(uri) if (docUri.fsPath.toString() === src.fsPath.toString()) { cacheId = cid } @@ -612,7 +619,7 @@ export abstract class GrpcBaseSerializer extends SerializerBase { } public static getOutputsUri(docUri: Uri, sessionId: string): Uri { - return Uri.parse(GrpcBaseSerializer.getOutputsFilePath(docUri.fsPath, sessionId)) + return Uri.parse(GrpcSerializerBase.getOutputsFilePath(docUri.fsPath, sessionId)) } public static getSourceFilePath(outputsFile: string): string { @@ -630,7 +637,7 @@ export abstract class GrpcBaseSerializer extends SerializerBase { } public static getSourceFileUri(outputsUri: Uri): Uri { - return Uri.parse(GrpcBaseSerializer.getSourceFilePath(outputsUri.fsPath)) + return Uri.parse(GrpcSerializerBase.getSourceFilePath(outputsUri.fsPath)) } public getMaskedCache(cacheId: string): Promise | undefined { @@ -678,7 +685,7 @@ export abstract class GrpcBaseSerializer extends SerializerBase { // the data types of the properties are different in some cases, presumably due to compile flags } -export class GrpcSerializer extends GrpcBaseSerializer { +export class GrpcSerializer extends GrpcSerializerBase { protected client!: ParserServiceClient protected ready: ReadyPromise @@ -784,7 +791,7 @@ export class GrpcSerializer extends GrpcBaseSerializer { data.metadata[RUNME_FRONTMATTER_PARSED] = notebook.frontmatter } - const cacheId = GrpcBaseSerializer.getDocumentCacheId(data.metadata) + const cacheId = GrpcSerializerBase.getDocumentCacheId(data.metadata) this.notebookDataCache.set(cacheId as string, data) const serialRequest = { notebook } @@ -997,7 +1004,7 @@ export class GrpcSerializer extends GrpcBaseSerializer { protected static marshalCellExecutionSummary( executionSummary: NotebookCellExecutionSummary | undefined, - ) { + ): CellExecutionSummary | undefined { if (!executionSummary) { return undefined } @@ -1041,3 +1048,394 @@ export class GrpcSerializer extends GrpcBaseSerializer { super.dispose() } } + +export class ConnectSerializer extends GrpcSerializerBase { + log: ReturnType + client: PromiseClient + serializerServiceUrl: string + protected ready: ReadyPromise + + private serverReadyListener: Disposable | undefined + + constructor( + protected context: ExtensionContext, + serializerAddress: string, + kernel: Kernel, + ) { + super(context, kernel) + this.log = getLogger('ConnectSerializer') + this.serializerServiceUrl = serializerAddress + this.client = this.createSerializerClient() + this.ready = new Promise((resolve) => { + resolve() + }) + } + + createSerializerClient = () => { + this.log.info(`Serializer: Client pointed to: ${this.serializerServiceUrl}`) + // Server options: + // (1) pre-existing, started internally by this extension (assuming local execution so it has permissions to start) + // (2) pre-existing, started externally, presumably at the this.serializerServiceUrl address + // In both cases we need a key & cert (assuming tls) which may reside in: + // (a) this project: '/path/to/projectRoot/tls' + // (b) the runme binary server default: '~/.config/runme/tls' + // (c) other, manually configured + + // I will assume 2a here + const tlsPath = path.normalize(__dirname + '/../tls') + const keyPath = `${tlsPath}/key.pem` + const certPath = `${tlsPath}/cert.pem` + + let grpcOptions = { + baseUrl: this.serializerServiceUrl, + httpVersion: '2', + nodeOptions: { + rejectUnauthorized: false, // allow self-signed certificates + cert: fs.readFileSync(certPath), + key: fs.readFileSync(keyPath), + }, + } as GrpcTransportOptions + + return createPromiseClient(ParserService, createGrpcTransport(grpcOptions)) + + // return createConnectTransport({ + // baseUrl: this.serializerServiceUrl, + // httpVersion: '1.1', + // }) + } + + protected async saveNotebook( + data: NotebookData, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + token: CancellationToken, + ): Promise { + const marshalFrontmatter = this.lifecycleIdentity === RunmeIdentity.ALL + + const notebook = ConnectSerializer.marshalNotebook(data, { marshalFrontmatter }) + + if (marshalFrontmatter) { + data.metadata ??= {} + data.metadata[RUNME_FRONTMATTER_PARSED] = notebook.frontmatter + } + + const cacheId = GrpcSerializerBase.getDocumentCacheId(data.metadata) + this.notebookDataCache.set(cacheId as string, data) + + const serialRequest = new es_proto.SerializeRequest({ notebook }) + + const cacheOutputs = this.cacheNotebookOutputs(notebook, cacheId) + const request = this.client.serialize(serialRequest) + + // run in parallel + const [serialResult] = await Promise.all([request, cacheOutputs]) + + if (cacheId) { + await this.saveNotebookOutputsByCacheId(cacheId) + } + + if (serialResult.result === undefined) { + throw new Error('serialization of notebook failed') + } + + return serialResult.result + } + + protected async reviveNotebook( + content: Uint8Array, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + token: CancellationToken, + ): Promise { + const identity = this.lifecycleIdentity + // const markdown = Buffer.from(content).toString('utf8') + const deserializeRequest = new es_proto.DeserializeRequest({ + source: content, + options: { identity }, + }) + let res + try { + res = await this.client.deserialize(deserializeRequest) + } catch (e: any) { + if (e.name === 'ConnectError') { + e.message = `Unable to connect to the serializer service at ${this.serializerServiceUrl}` + } + log.error('Error in reviveNotebook ', e as any) + throw e + } + const notebook = res.notebook + + if (!notebook) { + return this.printCell('⚠️ __Error__: no cells found!') + } + return notebook as any as Serializer.Notebook // ugly cast :( + } + + protected applyIdentity(data: es_proto.Notebook): es_proto.Notebook { + const identity = this.lifecycleIdentity + switch (identity) { + case es_proto.RunmeIdentity.UNSPECIFIED: + case es_proto.RunmeIdentity.DOCUMENT: + break + default: { + data.cells.forEach((cell) => { + if (cell.kind !== es_proto.CellKind.CODE) { + return + } + if (!cell.metadata?.['id'] && cell.metadata?.['runme.dev/id']) { + cell.metadata['id'] = cell.metadata['runme.dev/id'] + } + }) + } + } + + return data + } + + public override async switchLifecycleIdentity( + notebook: NotebookDocument, + identity: es_proto.RunmeIdentity, + ): Promise { + // skip session outputs files + if (!!notebook.metadata['runme.dev/frontmatterParsed']?.runme?.session?.id) { + return false + } + + await notebook.save() + const source = await workspace.fs.readFile(notebook.uri) + const dr = new es_proto.DeserializeRequest({ + source, + options: { identity }, + }) + const deserialized = (await this.client.deserialize(dr)).notebook + + if (!deserialized) { + return false + } + + deserialized.metadata = { ...deserialized.metadata, ...notebook.metadata } + const notebookEdit = NotebookEdit.updateNotebookMetadata(deserialized.metadata) + const edits = [notebookEdit] + notebook.getCells().forEach((cell) => { + const descell = deserialized.cells[cell.index] + // skip if no IDs are present, means no cell identity required + if (!descell.metadata?.['id']) { + return + } + const metadata = { ...descell.metadata, ...cell.metadata } + metadata['id'] = metadata['runme.dev/id'] + edits.push(NotebookEdit.updateCellMetadata(cell.index, metadata)) + }) + + const edit = new WorkspaceEdit() + edit.set(notebook.uri, edits) + return await workspace.applyEdit(edit) + } + + // unable to abstract due to RunmeSession struct potential differences & notebook ts type validation issues + protected async cacheNotebookOutputs( + notebook: es_proto.Notebook, + cacheId: string | undefined, + ): Promise { + let session: es_proto.RunmeSession | undefined + const docUri = this.cacheDocUriMapping.get(cacheId ?? '') + const sid = this.kernel.getRunnerEnvironment()?.getSessionId() + if (sid && docUri) { + const relativePath = path.basename(docUri.fsPath) + session = new es_proto.RunmeSession({ + id: sid, + document: { relativePath }, + }) + } + + const outputs = { enabled: true, summary: true } + const options = SerializeRequestOptions.clone({ + outputs, + session, + }) + + const maskedNotebook = new es_proto.Notebook(notebook) + maskedNotebook.cells.forEach((cell) => { + cell.value = maskString(cell.value) + cell.outputs.forEach((out) => { + out.items.forEach((item) => { + if (item.mime === OutputType.stdout) { + const outDecoded = Buffer.from(item.data).toString('utf8') + item.data = Buffer.from(maskString(outDecoded)) + } + }) + }) + }) + + const plainReq = new es_proto.SerializeRequest({ notebook, options }) + const plainRes = this.client.serialize(plainReq) + + const maskedReq = new es_proto.SerializeRequest({ notebook: maskedNotebook, options }) + const masked = this.client.serialize(maskedReq).then((maskedRes) => { + if (maskedRes.result === undefined) { + console.error('serialization of masked notebook failed') + return Promise.resolve(new Uint8Array()) + } + return maskedRes.result + }) + + if (!cacheId) { + console.error('skip masked caching since no lifecycleId was found') + } else { + this.maskedCache.set(cacheId, masked) + } + + const plain = await plainRes + if (plain.result === undefined) { + throw new Error('serialization of notebook outputs failed') + } + + const bytes = plain.result + if (!cacheId) { + console.error('skip plain caching since no lifecycleId was found') + } else { + this.plainCache.set(cacheId, Promise.resolve(bytes)) + } + + await Promise.all([plain, masked]) + } + + // vscode/NotebookData to buf-es/Notebook + public static marshalNotebook( + data: NotebookData, + config?: { + marshalFrontmatter?: boolean + kernel?: Kernel + }, + ): es_proto.Notebook { + // the bulk copies cleanly except for what's below + const notebook = new es_proto.Notebook(data as any) + + // cannot gurantee it wasn't changed + if (notebook.metadata[RUNME_FRONTMATTER_PARSED]) { + delete notebook.metadata[RUNME_FRONTMATTER_PARSED] + } + + if (config?.marshalFrontmatter) { + const metadata = notebook.metadata as unknown as { + ['runme.dev/frontmatter']: string + } + notebook.frontmatter = this.marshalFrontmatter(metadata, config.kernel) + } + + notebook.cells.forEach(async (cell, cellIdx) => { + const dataExecSummary = data.cells[cellIdx].executionSummary + cell.executionSummary = this.marshalCellExecutionSummary(dataExecSummary) + const dataOutputs = data.cells[cellIdx].outputs + cell.outputs = this.marshalCellOutputs(cell.outputs, dataOutputs) + }) + + return notebook + } + + static marshalFrontmatter( + metadata: { ['runme.dev/frontmatter']?: string }, + kernel?: Kernel, + ): es_proto.Frontmatter { + if ( + !metadata.hasOwnProperty('runme.dev/frontmatter') || + typeof metadata['runme.dev/frontmatter'] !== 'string' + ) { + log.warn('no frontmatter found in metadata') + return new es_proto.Frontmatter({ + category: '', + // tag: '', //?? es_proto does not have tag, but timostam/prototype-ts does?? + cwd: '', + runme: { + id: '', + version: '', + }, + shell: '', + skipPrompts: false, + terminalRows: '', + }) + } + + const rawFrontmatter = metadata['runme.dev/frontmatter'] + let data: { + runme: { + id?: string + version?: string + } + } = { runme: {} } + + if (rawFrontmatter) { + try { + const yamlDocs = YAML.parseAllDocuments(metadata['runme.dev/frontmatter']) + data = (yamlDocs[0].toJS?.() || {}) as typeof data + } catch (error: any) { + log.warn('failed to parse frontmatter, reason: ', error.message) + } + } + + return new es_proto.Frontmatter({ + runme: { + id: data.runme?.id || '', + version: data.runme?.version || '', + session: { id: kernel?.getRunnerEnvironment()?.getSessionId() || '' }, + }, + category: '', + // tag: '', // es_proto does not have tag + cwd: '', + shell: '', + skipPrompts: false, + terminalRows: '', + }) + } + + private static marshalCellOutputs( + outputs: es_proto.CellOutput[], + dataOutputs: NotebookCellOutput[] | undefined, + ): es_proto.CellOutput[] { + if (!dataOutputs) { + return [] + } + + outputs.forEach((out, outIdx) => { + const dataOut: NotebookCellOutputWithProcessInfo = dataOutputs[outIdx] + // todo(sebastian): consider sending error state too + if (dataOut.processInfo?.exitReason?.type === 'exit') { + if (dataOut.processInfo.exitReason.code) { + out.processInfo!.exitReason!.code = dataOut.processInfo.exitReason.code + } else { + out.processInfo!.exitReason!.code = undefined + } + + if (dataOut.processInfo?.pid !== undefined) { + out.processInfo!.pid = BigInt(dataOut.processInfo.pid) + } else { + out.processInfo!.pid = undefined + } + } + out.items.forEach((item) => { + item.type = item.data.buffer ? 'Buffer' : typeof item.data + }) + }) + + return outputs + } + + protected static marshalCellExecutionSummary( + executionSummary: NotebookCellExecutionSummary | undefined, + ): es_proto.CellExecutionSummary | undefined { + if (!executionSummary) { + return undefined + } + + const { success, timing } = executionSummary + if (success === undefined || timing === undefined) { + return undefined + } + + return new es_proto.CellExecutionSummary({ + success: success, + timing: { + endTime: BigInt(timing!.endTime), + startTime: BigInt(timing!.startTime), + }, + }) + } +} From c616b3ce00570baddedbe44b8536e51534621aad Mon Sep 17 00:00:00 2001 From: hotpocket Date: Fri, 6 Dec 2024 03:18:45 -0800 Subject: [PATCH 03/18] handle error on missing tls similar to kernelServer. Add missing call to applyIdentity --- src/extension/serializer.ts | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/extension/serializer.ts b/src/extension/serializer.ts index 66158612b..28228f7b9 100644 --- a/src/extension/serializer.ts +++ b/src/extension/serializer.ts @@ -40,7 +40,7 @@ import { RUNME_FRONTMATTER_PARSED, VSCODE_LANGUAGEID_MAP, } from '../constants' -import { ServerLifecycleIdentity, getSessionOutputs } from '../utils/configuration' +import { ServerLifecycleIdentity, getSessionOutputs, getTLSDir } from '../utils/configuration' import { DeserializeRequest, @@ -1082,26 +1082,25 @@ export class ConnectSerializer extends GrpcSerializerBase { // (c) other, manually configured // I will assume 2a here - const tlsPath = path.normalize(__dirname + '/../tls') - const keyPath = `${tlsPath}/key.pem` - const certPath = `${tlsPath}/cert.pem` + const getTls = () => { + try { + const tlsPath = getTLSDir(Uri.parse(this.context.extensionPath)) + return { + key: fs.readFileSync(`${tlsPath}/key.pem`), + cert: fs.readFileSync(`${tlsPath}/cert.pem`), + } + } catch (e: any) { + throw new Error(`Failed to read TLS files: ${e instanceof Error ? e.message : String(e)}`) + } + } let grpcOptions = { baseUrl: this.serializerServiceUrl, httpVersion: '2', - nodeOptions: { - rejectUnauthorized: false, // allow self-signed certificates - cert: fs.readFileSync(certPath), - key: fs.readFileSync(keyPath), - }, + nodeOptions: { ...getTls(), rejectUnauthorized: false }, } as GrpcTransportOptions return createPromiseClient(ParserService, createGrpcTransport(grpcOptions)) - - // return createConnectTransport({ - // baseUrl: this.serializerServiceUrl, - // httpVersion: '1.1', - // }) } protected async saveNotebook( @@ -1163,6 +1162,12 @@ export class ConnectSerializer extends GrpcSerializerBase { } const notebook = res.notebook + if (notebook === undefined) { + throw new Error('deserialization failed to revive notebook') + } + + this.applyIdentity(notebook) + if (!notebook) { return this.printCell('⚠️ __Error__: no cells found!') } From ea26204d53a052d0e9e06bf2f4cd9b6c3b4ca3a6 Mon Sep 17 00:00:00 2001 From: hotpocket Date: Fri, 6 Dec 2024 03:19:32 -0800 Subject: [PATCH 04/18] add missing config block --- package.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/package.json b/package.json index 9adf49dfb..fce6d2c8c 100644 --- a/package.json +++ b/package.json @@ -1041,6 +1041,12 @@ "default": true, "markdownDescription": "If set to `true` enables Stateful Authentication Provider" }, + "runme.serializerAddress": { + "type": "string", + "xdefault": "", + "default": "https://localhost:9999", + "description": "The base URL for the connect protocol serialization service." + }, "runme.app.docsUrl": { "type": "string", "scope": "window", From a302049040e232cef3e9dbaf78a2539787523354 Mon Sep 17 00:00:00 2001 From: hotpocket Date: Thu, 12 Dec 2024 01:13:57 -0800 Subject: [PATCH 05/18] creating the maskedNotebook using `new` causes cells to be copied by reference, causing masked data to be written to the main markdown file which causes data loss --- src/extension/serializer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/extension/serializer.ts b/src/extension/serializer.ts index 28228f7b9..5297cf233 100644 --- a/src/extension/serializer.ts +++ b/src/extension/serializer.ts @@ -1252,12 +1252,12 @@ export class ConnectSerializer extends GrpcSerializerBase { } const outputs = { enabled: true, summary: true } - const options = SerializeRequestOptions.clone({ + const options = new es_proto.SerializeRequestOptions({ outputs, session, }) - const maskedNotebook = new es_proto.Notebook(notebook) + const maskedNotebook = notebook.clone() maskedNotebook.cells.forEach((cell) => { cell.value = maskString(cell.value) cell.outputs.forEach((out) => { From 68ea43773e729f5976dc61d1b6889153da227cd3 Mon Sep 17 00:00:00 2001 From: hotpocket Date: Thu, 12 Dec 2024 03:40:03 -0800 Subject: [PATCH 06/18] getOutputsUri no longer resides on GrpcSerializer and thus broke this test. --- tests/extension/serializer.test.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/extension/serializer.test.ts b/tests/extension/serializer.test.ts index 61de13e1a..2378ba243 100644 --- a/tests/extension/serializer.test.ts +++ b/tests/extension/serializer.test.ts @@ -9,7 +9,12 @@ import { import { expect, vi, it, describe, beforeEach } from 'vitest' import { isValid } from 'ulidx' -import { GrpcSerializer, SerializerBase, WasmSerializer } from '../../src/extension/serializer' +import { + GrpcSerializer, + SerializerBase, + WasmSerializer, + GrpcSerializerBase, +} from '../../src/extension/serializer' import { RunmeIdentity } from '../../src/extension/grpc/serializerTypes' import type { Kernel } from '../../src/extension/kernel' import { EventEmitter, Uri } from '../../__mocks__/vscode' @@ -517,7 +522,7 @@ describe('GrpcSerializer', () => { const serializer: any = new GrpcSerializer(context, new Server(), new Kernel()) - vi.spyOn(GrpcSerializer, 'getOutputsUri').mockReturnValue(fakeSrcDocUri) + vi.spyOn(GrpcSerializerBase, 'getOutputsUri').mockReturnValue(fakeSrcDocUri) await serializer.handleOpenNotebook({ uri: fakeSrcDocUri, @@ -592,7 +597,7 @@ describe('GrpcSerializer', () => { fixture.metadata['runme.dev/frontmatterParsed'].runme.id, fakeCachedBytes, ) - GrpcSerializer.getOutputsUri = vi.fn().mockImplementation(() => undefined) + GrpcSerializerBase.getOutputsUri = vi.fn().mockImplementation(() => undefined) await serializer.handleSaveNotebookOutputs({ uri: fakeSrcDocUri, metadata: fixture.metadata, @@ -631,7 +636,7 @@ describe('GrpcSerializer', () => { writeableSer.cacheDocUriMapping.set(fixture.metadata['runme.dev/cacheId'], fakeSrcDocUri) ContextState.getKey = vi.fn().mockImplementation(() => true) GrpcSerializer.sessionOutputsEnabled = vi.fn().mockReturnValue(true) - GrpcSerializer.getOutputsUri = vi.fn().mockImplementation(() => fakeSrcDocUri) + GrpcSerializerBase.getOutputsUri = vi.fn().mockImplementation(() => fakeSrcDocUri) const result = await writeableSer.serializeNotebook( { cells: [], metadata: fixture.metadata } as any, From 49fe3a7682727e38db0b7d373e7c9424d68428f8 Mon Sep 17 00:00:00 2001 From: hotpocket Date: Thu, 12 Dec 2024 06:32:29 -0800 Subject: [PATCH 07/18] replace GrpcSerializer with ConnectSerializer and fix types as needed. --- package.json | 6 - src/extension/ai/converters.ts | 5 +- src/extension/extension.ts | 22 +- src/extension/serializer.ts | 435 +++------------------------------ 4 files changed, 44 insertions(+), 424 deletions(-) diff --git a/package.json b/package.json index 8486f8021..875474b1e 100644 --- a/package.json +++ b/package.json @@ -1041,12 +1041,6 @@ "default": true, "markdownDescription": "If set to `true` enables Stateful Authentication Provider" }, - "runme.serializerAddress": { - "type": "string", - "xdefault": "", - "default": "https://localhost:9999", - "description": "The base URL for the connect protocol serialization service." - }, "runme.app.docsUrl": { "type": "string", "scope": "window", diff --git a/src/extension/ai/converters.ts b/src/extension/ai/converters.ts index 349e2e568..3b3079891 100644 --- a/src/extension/ai/converters.ts +++ b/src/extension/ai/converters.ts @@ -11,8 +11,6 @@ import * as serializerTypes from '../grpc/serializerTypes' import * as serializer from '../serializer' import { Kernel } from '../kernel' -import * as protos from './protos' - // Converter provides converstion routines from vscode data types to protocol buffer types. // It is a class because in order to handle the conversion we need to keep track of the kernel // because we need to add execution information to the cells before serializing. @@ -30,8 +28,7 @@ export class Converter { let notebookDataWithExec = new vscode.NotebookData(cellDataWithExec) // marshalNotebook returns a protocol buffer using the ts client library from buf we need to // convert it to es - let notebookProto = serializer.GrpcSerializer.marshalNotebook(notebookDataWithExec) - return protos.notebookTSToES(notebookProto) + return serializer.GrpcSerializer.marshalNotebook(notebookDataWithExec) } } diff --git a/src/extension/extension.ts b/src/extension/extension.ts index b6cb026c3..4b4cdde0a 100644 --- a/src/extension/extension.ts +++ b/src/extension/extension.ts @@ -73,13 +73,7 @@ import { runForkCommand, selectEnvironment, } from './commands' -import { - WasmSerializer, - GrpcSerializer, - SerializerBase, - ConnectSerializer, - GrpcSerializerBase, -} from './serializer' +import { WasmSerializer, SerializerBase, GrpcSerializer, GrpcSerializerBase } from './serializer' import { RunmeLauncherProvider, RunmeTreeProvider } from './provider/launcher' import { RunmeLauncherProvider as RunmeLauncherProviderBeta } from './provider/launcherBeta' import { RunmeUriHandler } from './handler/uri' @@ -110,8 +104,6 @@ export class RunmeExtension { const grpcSerializer = kernel.hasExperimentEnabled('grpcSerializer') const grpcServer = kernel.hasExperimentEnabled('grpcServer') const grpcRunner = kernel.hasExperimentEnabled('grpcRunner') - const config = workspace.getConfiguration('runme') - const serializerAddress = config.get('serializerAddress') const server = new KernelServer( context.extensionUri, @@ -136,14 +128,10 @@ export class RunmeExtension { ) const reporter = new GrpcReporter(context, server) - let serializer: SerializerBase - if (serializerAddress && serializerAddress.length > 0) { - serializer = new ConnectSerializer(context, serializerAddress, kernel) - } else if (grpcSerializer) { - serializer = new GrpcSerializer(context, server, kernel) - } else { - serializer = new WasmSerializer(context, kernel) - } + + const serializer = grpcSerializer + ? new GrpcSerializer(context, server, kernel) + : new WasmSerializer(context, kernel) this.serializer = serializer kernel.setSerializer(serializer as GrpcSerializerBase) kernel.setReporter(reporter) diff --git a/src/extension/serializer.ts b/src/extension/serializer.ts index 5297cf233..73e00bef8 100644 --- a/src/extension/serializer.ts +++ b/src/extension/serializer.ts @@ -20,13 +20,12 @@ import { NotebookCellExecutionSummary, commands, } from 'vscode' -import { GrpcTransport } from '@protobuf-ts/grpc-transport' import { ulid } from 'ulidx' import { maskString } from 'data-guardian' import YAML from 'yaml' import { ParserService } from '@buf/stateful_runme.connectrpc_es/runme/parser/v1/parser_connect' import { createGrpcTransport, GrpcTransportOptions } from '@connectrpc/connect-node' -import { createPromiseClient, PromiseClient } from '@connectrpc/connect' +import { createClient as createConnectClient, Client as ConnectClient } from '@connectrpc/connect' // ts bindings generated by protoc-gen-es import * as es_proto from '@buf/stateful_runme.bufbuild_es/runme/parser/v1/parser_pb' @@ -40,21 +39,15 @@ import { RUNME_FRONTMATTER_PARSED, VSCODE_LANGUAGEID_MAP, } from '../constants' -import { ServerLifecycleIdentity, getSessionOutputs, getTLSDir } from '../utils/configuration' - import { - DeserializeRequest, - SerializeRequest, - Notebook, - RunmeIdentity, - CellKind, - CellOutput, - SerializeRequestOptions, - RunmeSession, - Frontmatter, - CellExecutionSummary, -} from './grpc/serializerTypes' -import { initParserClient, ParserServiceClient, type ReadyPromise } from './grpc/client' + ServerLifecycleIdentity, + getSessionOutputs, + getTLSDir, + getTLSEnabled, +} from '../utils/configuration' + +import { RunmeIdentity } from './grpc/serializerTypes' +import { type ReadyPromise } from './grpc/client' import Languages from './languages' import { PLATFORM_OS } from './constants' import { initWasm } from './utils' @@ -686,10 +679,11 @@ export abstract class GrpcSerializerBase extends SerializerBase { } export class GrpcSerializer extends GrpcSerializerBase { - protected client!: ParserServiceClient + client!: ConnectClient protected ready: ReadyPromise + protected serverUrl!: string - private serverReadyListener: Disposable | undefined + private serverReadyListener?: Disposable constructor( protected context: ExtensionContext, @@ -697,382 +691,20 @@ export class GrpcSerializer extends GrpcSerializerBase { kernel: Kernel, ) { super(context, kernel) - + this.ready = new Promise((resolve) => { + resolve() + }) + this.serverReadyListener = server.onTransportReady(() => this.initClient()) + // cleanup listener when it's outlived its purpose this.ready = new Promise((resolve) => { const disposable = server.onTransportReady(() => { disposable.dispose() resolve() }) }) - - this.serverReadyListener = server.onTransportReady(({ transport }) => - this.initParserClient(transport), - ) - } - - private async initParserClient(transport?: GrpcTransport) { - this.client = initParserClient(transport ?? (await this.server.transport())) - } - - protected applyIdentity(data: Notebook): Notebook { - const identity = this.lifecycleIdentity - switch (identity) { - case RunmeIdentity.UNSPECIFIED: - case RunmeIdentity.DOCUMENT: - break - default: { - data.cells.forEach((cell) => { - if (cell.kind !== CellKind.CODE) { - return - } - if (!cell.metadata?.['id'] && cell.metadata?.['runme.dev/id']) { - cell.metadata['id'] = cell.metadata['runme.dev/id'] - } - }) - } - } - - return data - } - - public override async switchLifecycleIdentity( - notebook: NotebookDocument, - identity: RunmeIdentity, - ): Promise { - // skip session outputs files - if (!!notebook.metadata['runme.dev/frontmatterParsed']?.runme?.session?.id) { - return false - } - - await notebook.save() - const source = await workspace.fs.readFile(notebook.uri) - const des = await this.client.deserialize( - DeserializeRequest.create({ - source, - options: { identity }, - }), - ) - - const deserialized = des.response.notebook - if (!deserialized) { - return false - } - - deserialized.metadata = { ...deserialized.metadata, ...notebook.metadata } - const notebookEdit = NotebookEdit.updateNotebookMetadata(deserialized.metadata) - const edits = [notebookEdit] - notebook.getCells().forEach((cell) => { - const descell = deserialized.cells[cell.index] - // skip if no IDs are present, means no cell identity required - if (!descell.metadata?.['id']) { - return - } - const metadata = { ...descell.metadata, ...cell.metadata } - metadata['id'] = metadata['runme.dev/id'] - edits.push(NotebookEdit.updateCellMetadata(cell.index, metadata)) - }) - - const edit = new WorkspaceEdit() - edit.set(notebook.uri, edits) - return await workspace.applyEdit(edit) - } - - protected async saveNotebook( - data: NotebookData, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - token: CancellationToken, - ): Promise { - const marshalFrontmatter = this.lifecycleIdentity === RunmeIdentity.ALL - - const notebook = GrpcSerializer.marshalNotebook(data, { marshalFrontmatter }) - - if (marshalFrontmatter) { - data.metadata ??= {} - data.metadata[RUNME_FRONTMATTER_PARSED] = notebook.frontmatter - } - - const cacheId = GrpcSerializerBase.getDocumentCacheId(data.metadata) - this.notebookDataCache.set(cacheId as string, data) - - const serialRequest = { notebook } - - const cacheOutputs = this.cacheNotebookOutputs(notebook, cacheId) - const request = this.client.serialize(serialRequest) - - // run in parallel - const [serialResult] = await Promise.all([request, cacheOutputs]) - - if (cacheId) { - await this.saveNotebookOutputsByCacheId(cacheId) - } - - const { result } = serialResult.response - if (result === undefined) { - throw new Error('serialization of notebook failed') - } - - return result - } - - // unable to abstract due to RunmeSession struct potential differences & notebook ts type validation issues - protected async cacheNotebookOutputs( - notebook: Notebook, - cacheId: string | undefined, - ): Promise { - let session: RunmeSession | undefined - const docUri = this.cacheDocUriMapping.get(cacheId ?? '') - const sid = this.kernel.getRunnerEnvironment()?.getSessionId() - if (sid && docUri) { - const relativePath = path.basename(docUri.fsPath) - session = { - id: sid, - document: { relativePath }, - } - } - - const outputs = { enabled: true, summary: true } - const options = SerializeRequestOptions.clone({ - outputs, - session, - }) - - const maskedNotebook = Notebook.clone(notebook) - maskedNotebook.cells.forEach((cell) => { - cell.value = maskString(cell.value) - cell.outputs.forEach((out) => { - out.items.forEach((item) => { - if (item.mime === OutputType.stdout) { - const outDecoded = Buffer.from(item.data).toString('utf8') - item.data = Buffer.from(maskString(outDecoded)) - } - }) - }) - }) - - const plainReq = { notebook, options } - const plainRes = this.client.serialize(plainReq) - - const maskedReq = { notebook: maskedNotebook, options } - const masked = this.client.serialize(maskedReq).then((maskedRes) => { - if (maskedRes.response.result === undefined) { - console.error('serialization of masked notebook failed') - return Promise.resolve(new Uint8Array()) - } - return maskedRes.response.result - }) - - if (!cacheId) { - console.error('skip masked caching since no lifecycleId was found') - } else { - this.maskedCache.set(cacheId, masked) - } - - const plain = await plainRes - if (plain.response.result === undefined) { - throw new Error('serialization of notebook outputs failed') - } - - const bytes = plain.response.result - if (!cacheId) { - console.error('skip plain caching since no lifecycleId was found') - } else { - this.plainCache.set(cacheId, Promise.resolve(bytes)) - } - - await Promise.all([plain, masked]) - } - - // vscode/NotebookData to timostam-protobuf-ts/Notebook - public static marshalNotebook( - data: NotebookData, - config?: { - marshalFrontmatter?: boolean - kernel?: Kernel - }, - ): Notebook { - // the bulk copies cleanly except for what's below - const notebook = Notebook.clone(data as any) - - // cannot gurantee it wasn't changed - if (notebook.metadata[RUNME_FRONTMATTER_PARSED]) { - delete notebook.metadata[RUNME_FRONTMATTER_PARSED] - } - - if (config?.marshalFrontmatter) { - const metadata = notebook.metadata as unknown as { - ['runme.dev/frontmatter']: string - } - notebook.frontmatter = this.marshalFrontmatter(metadata, config.kernel) - } - - notebook.cells.forEach(async (cell, cellIdx) => { - const dataExecSummary = data.cells[cellIdx].executionSummary - cell.executionSummary = this.marshalCellExecutionSummary(dataExecSummary) - const dataOutputs = data.cells[cellIdx].outputs - cell.outputs = this.marshalCellOutputs(cell.outputs, dataOutputs) - }) - - return notebook - } - - static marshalFrontmatter( - metadata: { ['runme.dev/frontmatter']?: string }, - kernel?: Kernel, - ): Frontmatter { - if ( - !metadata.hasOwnProperty('runme.dev/frontmatter') || - typeof metadata['runme.dev/frontmatter'] !== 'string' - ) { - log.warn('no frontmatter found in metadata') - return { - category: '', - tag: '', - cwd: '', - runme: { - id: '', - version: '', - }, - shell: '', - skipPrompts: false, - terminalRows: '', - } - } - - const rawFrontmatter = metadata['runme.dev/frontmatter'] - let data: { - runme: { - id?: string - version?: string - } - } = { runme: {} } - - if (rawFrontmatter) { - try { - const yamlDocs = YAML.parseAllDocuments(metadata['runme.dev/frontmatter']) - data = (yamlDocs[0].toJS?.() || {}) as typeof data - } catch (error: any) { - log.warn('failed to parse frontmatter, reason: ', error.message) - } - } - - return { - runme: { - id: data.runme?.id || '', - version: data.runme?.version || '', - session: { id: kernel?.getRunnerEnvironment()?.getSessionId() || '' }, - }, - category: '', - tag: '', - cwd: '', - shell: '', - skipPrompts: false, - terminalRows: '', - } - } - - private static marshalCellOutputs( - outputs: CellOutput[], - dataOutputs: NotebookCellOutput[] | undefined, - ): CellOutput[] { - if (!dataOutputs) { - return [] - } - - outputs.forEach((out, outIdx) => { - const dataOut: NotebookCellOutputWithProcessInfo = dataOutputs[outIdx] - // todo(sebastian): consider sending error state too - if (dataOut.processInfo?.exitReason?.type === 'exit') { - if (dataOut.processInfo.exitReason.code) { - out.processInfo!.exitReason!.code!.value = dataOut.processInfo.exitReason.code - } else { - out.processInfo!.exitReason!.code = undefined - } - - if (dataOut.processInfo?.pid !== undefined) { - out.processInfo!.pid = { value: dataOut.processInfo.pid.toString() } - } else { - out.processInfo!.pid = undefined - } - } - out.items.forEach((item) => { - item.type = item.data.buffer ? 'Buffer' : typeof item.data - }) - }) - - return outputs - } - - protected static marshalCellExecutionSummary( - executionSummary: NotebookCellExecutionSummary | undefined, - ): CellExecutionSummary | undefined { - if (!executionSummary) { - return undefined - } - - const { success, timing } = executionSummary - if (success === undefined || timing === undefined) { - return undefined - } - - return { - success: { value: success }, - timing: { - endTime: { value: timing!.endTime.toString() }, - startTime: { value: timing!.startTime.toString() }, - }, - } - } - - protected async reviveNotebook( - content: Uint8Array, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - token: CancellationToken, - ): Promise { - const identity = this.lifecycleIdentity - const deserialRequest = DeserializeRequest.create({ source: content, options: { identity } }) - const request = await this.client.deserialize(deserialRequest) - - const { notebook } = request.response - if (notebook === undefined) { - throw new Error('deserialization failed to revive notebook') - } - // this new variable is not needed as notebook is modified in place - const _notebook = this.applyIdentity(notebook) - - // we can remove ugly casting once we switch to GRPC - return _notebook as unknown as Serializer.Notebook - } - - public dispose(): void { - this.serverReadyListener?.dispose() - super.dispose() - } -} - -export class ConnectSerializer extends GrpcSerializerBase { - log: ReturnType - client: PromiseClient - serializerServiceUrl: string - protected ready: ReadyPromise - - private serverReadyListener: Disposable | undefined - - constructor( - protected context: ExtensionContext, - serializerAddress: string, - kernel: Kernel, - ) { - super(context, kernel) - this.log = getLogger('ConnectSerializer') - this.serializerServiceUrl = serializerAddress - this.client = this.createSerializerClient() - this.ready = new Promise((resolve) => { - resolve() - }) } - createSerializerClient = () => { - this.log.info(`Serializer: Client pointed to: ${this.serializerServiceUrl}`) + private initClient(): void { // Server options: // (1) pre-existing, started internally by this extension (assuming local execution so it has permissions to start) // (2) pre-existing, started externally, presumably at the this.serializerServiceUrl address @@ -1082,25 +714,34 @@ export class ConnectSerializer extends GrpcSerializerBase { // (c) other, manually configured // I will assume 2a here - const getTls = () => { + const addTlsConfigIfEnabled = () => { try { + if (!getTLSEnabled()) { + return {} + } const tlsPath = getTLSDir(Uri.parse(this.context.extensionPath)) return { - key: fs.readFileSync(`${tlsPath}/key.pem`), - cert: fs.readFileSync(`${tlsPath}/cert.pem`), + nodeOptions: { + key: fs.readFileSync(`${tlsPath}/key.pem`), + cert: fs.readFileSync(`${tlsPath}/cert.pem`), + rejectUnauthorized: false, + }, } } catch (e: any) { throw new Error(`Failed to read TLS files: ${e instanceof Error ? e.message : String(e)}`) } } - let grpcOptions = { - baseUrl: this.serializerServiceUrl, + const s = getTLSEnabled() ? 's' : '' + this.serverUrl = `http${s}://${this.server.address()}` + + let grpcOptions = { + baseUrl: this.serverUrl, httpVersion: '2', - nodeOptions: { ...getTls(), rejectUnauthorized: false }, - } as GrpcTransportOptions + ...addTlsConfigIfEnabled(), + } - return createPromiseClient(ParserService, createGrpcTransport(grpcOptions)) + this.client = createConnectClient(ParserService, createGrpcTransport(grpcOptions)) } protected async saveNotebook( @@ -1110,7 +751,7 @@ export class ConnectSerializer extends GrpcSerializerBase { ): Promise { const marshalFrontmatter = this.lifecycleIdentity === RunmeIdentity.ALL - const notebook = ConnectSerializer.marshalNotebook(data, { marshalFrontmatter }) + const notebook = GrpcSerializer.marshalNotebook(data, { marshalFrontmatter }) if (marshalFrontmatter) { data.metadata ??= {} @@ -1155,7 +796,7 @@ export class ConnectSerializer extends GrpcSerializerBase { res = await this.client.deserialize(deserializeRequest) } catch (e: any) { if (e.name === 'ConnectError') { - e.message = `Unable to connect to the serializer service at ${this.serializerServiceUrl}` + e.message = `Unable to connect to the serializer service at ${this.serverUrl}` } log.error('Error in reviveNotebook ', e as any) throw e @@ -1423,7 +1064,7 @@ export class ConnectSerializer extends GrpcSerializerBase { return outputs } - protected static marshalCellExecutionSummary( + private static marshalCellExecutionSummary( executionSummary: NotebookCellExecutionSummary | undefined, ): es_proto.CellExecutionSummary | undefined { if (!executionSummary) { From 827354a9a361e81d2e5de6fbcbb080add1db21d9 Mon Sep 17 00:00:00 2001 From: hotpocket Date: Thu, 12 Dec 2024 06:35:34 -0800 Subject: [PATCH 08/18] client init in serializer.ts now calls `getTLSEnabled()` which needs`isWindows()` --- tests/extension/serializer.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/extension/serializer.test.ts b/tests/extension/serializer.test.ts index 2378ba243..73b099532 100644 --- a/tests/extension/serializer.test.ts +++ b/tests/extension/serializer.test.ts @@ -72,6 +72,7 @@ vi.mock('../../src/extension/languages', () => ({ vi.mock('../../src/extension/utils', () => ({ initWasm: vi.fn(), + isWindows: vi.fn().mockReturnValue(false), })) vi.mock('../../src/extension/features') @@ -467,11 +468,11 @@ describe('GrpcSerializer', () => { const summary = notebookData.cells[1].executionSummary expect(summary?.success).toBeDefined() - expect(summary?.success?.value).toStrictEqual(false) + expect(summary?.success).toStrictEqual(false) expect(summary?.timing).toBeDefined() - expect(summary?.timing?.startTime?.value).toStrictEqual('1701444499517') - expect(summary?.timing?.endTime?.value).toStrictEqual('1701444501696') + expect(summary?.timing?.startTime).toStrictEqual('1701444499517') + expect(summary?.timing?.endTime).toStrictEqual('1701444501696') }) }) @@ -492,9 +493,9 @@ describe('GrpcSerializer', () => { const { processInfo } = cells.outputs[0] expect(processInfo?.exitReason).toBeDefined() expect(processInfo?.exitReason?.type).toStrictEqual('exit') - expect(processInfo?.exitReason?.code?.value).toStrictEqual(16) + expect(processInfo?.exitReason?.code).toStrictEqual(16) expect(processInfo?.pid).toBeDefined() - expect(processInfo?.pid?.value).toStrictEqual('98354') + expect(processInfo?.pid).toStrictEqual('98354') }) }) From 052b70fd9a4edaf2427a443af7d1d1faf1f3287b Mon Sep 17 00:00:00 2001 From: hotpocket Date: Thu, 12 Dec 2024 06:36:01 -0800 Subject: [PATCH 09/18] i have no idea what's going on here but this makes it work. --- src/extension/messages/platformRequest/saveCellExecution.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/extension/messages/platformRequest/saveCellExecution.ts b/src/extension/messages/platformRequest/saveCellExecution.ts index dc3ff5506..fd1e65b04 100644 --- a/src/extension/messages/platformRequest/saveCellExecution.ts +++ b/src/extension/messages/platformRequest/saveCellExecution.ts @@ -4,7 +4,7 @@ import { Uri, env, workspace, commands } from 'vscode' import { TelemetryReporter } from 'vscode-telemetry' import getMAC from 'getmac' import YAML from 'yaml' -import { FetchResult } from '@apollo/client' +import { FetchResult, MutationOptions } from '@apollo/client' import { ClientMessages, NOTEBOOK_AUTOSAVE_ON, RUNME_FRONTMATTER_PARSED } from '../../../constants' import { ClientMessage, FeatureName, IApiMessage } from '../../../types' @@ -183,10 +183,10 @@ export default async function saveCellExecution( }, }, } - const result = await graphClient.mutate(mutation) + const result = await graphClient.mutate(mutation as MutationOptions) data = result } - // TODO: Remove the legacy createCellExecution mutation once the reporter is fully tested. + // TODO: Remove the legacy create`ecution mutation once the reporter is fully tested. else { const cell = await getCellById({ editor, id: message.output.id }) if (!cell) { From 70fc30d9d836d02182426a8a4fa6d010e0f9516f Mon Sep 17 00:00:00 2001 From: hotpocket Date: Fri, 13 Dec 2024 05:19:41 -0800 Subject: [PATCH 10/18] fix struct and data items that differ in new implementation. One test still fails whose solution is unclear "should backfill the output type for buffers". A comment has been added to start the discussion around this test. --- tests/extension/serializer.test.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/extension/serializer.test.ts b/tests/extension/serializer.test.ts index 73b099532..c540f9b18 100644 --- a/tests/extension/serializer.test.ts +++ b/tests/extension/serializer.test.ts @@ -8,6 +8,7 @@ import { } from 'vscode' import { expect, vi, it, describe, beforeEach } from 'vitest' import { isValid } from 'ulidx' +import { RunmeIdentity, Notebook } from '@buf/stateful_runme.bufbuild_es/runme/parser/v1/parser_pb' import { GrpcSerializer, @@ -15,7 +16,6 @@ import { WasmSerializer, GrpcSerializerBase, } from '../../src/extension/serializer' -import { RunmeIdentity } from '../../src/extension/grpc/serializerTypes' import type { Kernel } from '../../src/extension/kernel' import { EventEmitter, Uri } from '../../__mocks__/vscode' import { Serializer } from '../../src/types' @@ -373,7 +373,7 @@ describe('GrpcSerializer', () => { const serializer: any = new GrpcSerializer(context, new Server(), new Kernel()) serializer.client = { deserialize: vi.fn().mockResolvedValue({ - response: { notebook: { cells: descells, metadata } }, + notebook: { cells: descells, metadata }, }), } vi.mocked(workspace.applyEdit).mockResolvedValue(true) @@ -471,8 +471,8 @@ describe('GrpcSerializer', () => { expect(summary?.success).toStrictEqual(false) expect(summary?.timing).toBeDefined() - expect(summary?.timing?.startTime).toStrictEqual('1701444499517') - expect(summary?.timing?.endTime).toStrictEqual('1701444501696') + expect(summary?.timing?.startTime).toStrictEqual(1701444499517n) + expect(summary?.timing?.endTime).toStrictEqual(1701444501696n) }) }) @@ -487,6 +487,8 @@ describe('GrpcSerializer', () => { const items = cells.outputs[0].items expect(items.length).toBe(2) items.forEach((item) => { + // item.data is a Uint8Array in both the es and ts proto type + // is this a bug? if so where: in the fixture, the proto, the serializer? expect((item.data as any).type).toBe('Buffer') expect(item.mime).toBeDefined() }) @@ -495,7 +497,7 @@ describe('GrpcSerializer', () => { expect(processInfo?.exitReason?.type).toStrictEqual('exit') expect(processInfo?.exitReason?.code).toStrictEqual(16) expect(processInfo?.pid).toBeDefined() - expect(processInfo?.pid).toStrictEqual('98354') + expect(processInfo?.pid).toStrictEqual(98354n) }) }) @@ -632,7 +634,7 @@ describe('GrpcSerializer', () => { const fixture = deepCopyFixture() const writeableSer: any = new GrpcSerializer(context, new Server(), new Kernel()) writeableSer.client = { - serialize: vi.fn().mockResolvedValue({ response: { result: fakeCachedBytes } }), + serialize: vi.fn().mockResolvedValue({ result: fakeCachedBytes }), } writeableSer.cacheDocUriMapping.set(fixture.metadata['runme.dev/cacheId'], fakeSrcDocUri) ContextState.getKey = vi.fn().mockImplementation(() => true) @@ -681,20 +683,18 @@ describe('GrpcSerializer', () => { const context: any = { extensionUri: { fsPath: '/foo/bar' }, } - const fixture = { + const fixture = new Notebook({ cells: [], metadata: { 'runme.dev/finalLineBreaks': '1', 'runme.dev/frontmatter': '---\nrunme:\n id: 01HF7B0KJPF469EG9ZWDNKKACQ\n version: v2.0\n---', }, - } + }) const serialize = vi.fn().mockImplementation(() => Promise.resolve({ - response: { - result: new Uint8Array([4, 3, 2, 1]), - }, + result: new Uint8Array([4, 3, 2, 1]), }), ) const ser = new GrpcSerializer(context, new Server(), new Kernel()) From af02abc975894a9b7b323b2f1f6f2e57a5adf789 Mon Sep 17 00:00:00 2001 From: hotpocket Date: Fri, 13 Dec 2024 05:22:00 -0800 Subject: [PATCH 11/18] fix typo --- src/extension/messages/platformRequest/saveCellExecution.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extension/messages/platformRequest/saveCellExecution.ts b/src/extension/messages/platformRequest/saveCellExecution.ts index b0ed0c79b..5d084d592 100644 --- a/src/extension/messages/platformRequest/saveCellExecution.ts +++ b/src/extension/messages/platformRequest/saveCellExecution.ts @@ -190,7 +190,7 @@ export default async function saveCellExecution( const result = await graphClient.mutate(mutation as MutationOptions) data = result } - // TODO: Remove the legacy create`ecution mutation once the reporter is fully tested. + // TODO: Remove the legacy createCellExecution mutation once the reporter is fully tested. else { const cell = await getCellById({ editor, id: message.output.id }) if (!cell) { From 81c8c03f2815b123bd7a50ca27a5fd8530c73e79 Mon Sep 17 00:00:00 2001 From: hotpocket Date: Sat, 14 Dec 2024 23:13:24 -0800 Subject: [PATCH 12/18] fix invalid proto prop reference --- tests/extension/serializer.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/extension/serializer.test.ts b/tests/extension/serializer.test.ts index c540f9b18..614e517bd 100644 --- a/tests/extension/serializer.test.ts +++ b/tests/extension/serializer.test.ts @@ -487,9 +487,7 @@ describe('GrpcSerializer', () => { const items = cells.outputs[0].items expect(items.length).toBe(2) items.forEach((item) => { - // item.data is a Uint8Array in both the es and ts proto type - // is this a bug? if so where: in the fixture, the proto, the serializer? - expect((item.data as any).type).toBe('Buffer') + expect(item.type).toBe('Buffer') expect(item.mime).toBeDefined() }) const { processInfo } = cells.outputs[0] From 14b0f520b24730db89f0d1e03053d8c3c927bae6 Mon Sep 17 00:00:00 2001 From: hotpocket Date: Sun, 15 Dec 2024 00:13:43 -0800 Subject: [PATCH 13/18] combine grpc base and grpc serializer. reorder changes to minimize PR diff. --- src/extension/extension.ts | 4 +- src/extension/kernel.ts | 10 +- src/extension/serializer.ts | 364 ++++++++++++++--------------- tests/extension/serializer.test.ts | 13 +- 4 files changed, 186 insertions(+), 205 deletions(-) diff --git a/src/extension/extension.ts b/src/extension/extension.ts index 5bd19b069..fedfe59cf 100644 --- a/src/extension/extension.ts +++ b/src/extension/extension.ts @@ -70,7 +70,7 @@ import { runForkCommand, selectEnvironment, } from './commands' -import { WasmSerializer, SerializerBase, GrpcSerializer, GrpcSerializerBase } from './serializer' +import { WasmSerializer, SerializerBase, GrpcSerializer } from './serializer' import { RunmeLauncherProvider, RunmeTreeProvider } from './provider/launcher' import { RunmeLauncherProvider as RunmeLauncherProviderBeta } from './provider/launcherBeta' import { RunmeUriHandler } from './handler/uri' @@ -131,7 +131,7 @@ export class RunmeExtension { ? new GrpcSerializer(context, server, kernel) : new WasmSerializer(context, kernel) this.serializer = serializer - kernel.setSerializer(serializer as GrpcSerializerBase) + kernel.setSerializer(serializer as GrpcSerializer) kernel.setReporter(reporter) let treeViewer: RunmeTreeProvider diff --git a/src/extension/kernel.ts b/src/extension/kernel.ts index 08ba3df23..8a46a72bb 100644 --- a/src/extension/kernel.ts +++ b/src/extension/kernel.ts @@ -103,7 +103,7 @@ import { handleCellOutputMessage } from './messages/cellOutput' import handleGitHubMessage, { handleGistMessage } from './messages/github' import { getNotebookCategories } from './utils' import PanelManager from './panels/panelManager' -import { GrpcSerializerBase } from './serializer' +import { GrpcSerializer } from './serializer' import { askAlternativeOutputsAction, openSplitViewAsMarkdownText } from './commands' import { handlePlatformApiMessage } from './messages/platformRequest' import { handleGCPMessage } from './messages/gcp' @@ -151,7 +151,7 @@ export class Kernel implements Disposable { protected activeTerminals: ActiveTerminal[] = [] protected category?: string protected panelManager: PanelManager - protected serializer?: GrpcSerializerBase + protected serializer?: GrpcSerializer protected reporter?: GrpcReporter protected featuresState$?: FeatureObserver @@ -290,7 +290,7 @@ export class Kernel implements Disposable { this.category = category } - setSerializer(serializer: GrpcSerializerBase) { + setSerializer(serializer: GrpcSerializer) { this.serializer = serializer } @@ -335,7 +335,7 @@ export class Kernel implements Disposable { } async #setNotebookMode(notebookDocument: NotebookDocument): Promise { - const isSessionsOutput = GrpcSerializerBase.isDocumentSessionOutputs(notebookDocument.metadata) + const isSessionsOutput = GrpcSerializer.isDocumentSessionOutputs(notebookDocument.metadata) const notebookMode = isSessionsOutput ? NotebookMode.SessionOutputs : NotebookMode.Execution await ContextState.addKey(NOTEBOOK_MODE, notebookMode) } @@ -688,7 +688,7 @@ export class Kernel implements Disposable { private async _executeAll(cells: NotebookCell[]) { const sessionOutputsDoc = cells.find((c) => - GrpcSerializerBase.isDocumentSessionOutputs(c.notebook.metadata), + GrpcSerializer.isDocumentSessionOutputs(c.notebook.metadata), ) if (sessionOutputsDoc) { const { notebook } = sessionOutputsDoc diff --git a/src/extension/serializer.ts b/src/extension/serializer.ts index 73e00bef8..70a173b6f 100644 --- a/src/extension/serializer.ts +++ b/src/extension/serializer.ts @@ -473,9 +473,10 @@ export class WasmSerializer extends SerializerBase { } } -// no common ancestor, any type used for protos -export abstract class GrpcSerializerBase extends SerializerBase { - protected client: any +export class GrpcSerializer extends SerializerBase { + private client!: ConnectClient + protected ready: ReadyPromise + protected serverUrl!: string // todo(sebastian): naive cache for now, consider use lifecycle events for gc protected readonly plainCache = new Map>() @@ -483,14 +484,25 @@ export abstract class GrpcSerializerBase extends SerializerBase { protected readonly notebookDataCache = new Map() protected readonly cacheDocUriMapping: Map = new Map() + private serverReadyListener?: Disposable + constructor( protected context: ExtensionContext, + protected server: IServer, kernel: Kernel, ) { super(context, kernel) - this.togglePreviewButton(GrpcSerializerBase.sessionOutputsEnabled()) + this.togglePreviewButton(GrpcSerializer.sessionOutputsEnabled()) + // cleanup listener when it's outlived its purpose + this.ready = new Promise((resolve) => { + const disposable = server.onTransportReady(() => { + disposable.dispose() + resolve() + }) + }) + this.serverReadyListener = server.onTransportReady(() => this.initClient()) this.disposables.push( // todo(sebastian): delete entries on session reset not notebook editor lifecycle // workspace.onDidCloseNotebookDocument(this.handleCloseNotebook.bind(this)), @@ -499,21 +511,59 @@ export abstract class GrpcSerializerBase extends SerializerBase { ) } - protected abstract cacheNotebookOutputs(notebook: any, cacheId: string | undefined): Promise + private initClient(): void { + // Server options: + // (1) pre-existing, started internally by this extension (assuming local execution so it has permissions to start) + // (2) pre-existing, started externally, presumably at the this.serverUrl address + // In both cases we need a key & cert (assuming tls) which may reside in: + // (a) this project: '/path/to/projectRoot/tls' + // (b) the runme binary server default: '~/.config/runme/tls' + // (c) other, manually configured + + // assuming 2a, which is the default when starting the server as a result of loading this extension + const addTlsConfigIfEnabled = () => { + try { + if (!getTLSEnabled()) { + return {} + } + const tlsPath = getTLSDir(Uri.parse(this.context.extensionPath)) + return { + nodeOptions: { + key: fs.readFileSync(`${tlsPath}/key.pem`), + cert: fs.readFileSync(`${tlsPath}/cert.pem`), + rejectUnauthorized: false, + }, + } + } catch (e: any) { + throw new Error(`Failed to read TLS files: ${e instanceof Error ? e.message : String(e)}`) + } + } + + const s = getTLSEnabled() ? 's' : '' + this.serverUrl = `http${s}://${this.server.address()}` + + let grpcOptions = { + baseUrl: this.serverUrl, + httpVersion: '2', + ...addTlsConfigIfEnabled(), + } + + this.client = createConnectClient(ParserService, createGrpcTransport(grpcOptions)) + } public togglePreviewButton(state: boolean) { return commands.executeCommand('setContext', NOTEBOOK_HAS_OUTPUTS, state) } protected async handleOpenNotebook(doc: NotebookDocument) { - const cacheId = GrpcSerializerBase.getDocumentCacheId(doc.metadata) + const cacheId = GrpcSerializer.getDocumentCacheId(doc.metadata) if (!cacheId) { this.togglePreviewButton(false) return } - if (GrpcSerializerBase.isDocumentSessionOutputs(doc.metadata)) { + if (GrpcSerializer.isDocumentSessionOutputs(doc.metadata)) { this.togglePreviewButton(false) return } @@ -521,8 +571,8 @@ export abstract class GrpcSerializerBase extends SerializerBase { this.cacheDocUriMapping.set(cacheId, doc.uri) } - async handleCloseNotebook(doc: NotebookDocument) { - const cacheId = GrpcSerializerBase.getDocumentCacheId(doc.metadata) + protected async handleCloseNotebook(doc: NotebookDocument) { + const cacheId = GrpcSerializer.getDocumentCacheId(doc.metadata) /** * Remove cache */ @@ -532,8 +582,8 @@ export abstract class GrpcSerializerBase extends SerializerBase { } } - async handleSaveNotebookOutputs(doc: NotebookDocument) { - const cacheId = GrpcSerializerBase.getDocumentCacheId(doc.metadata) + protected async handleSaveNotebookOutputs(doc: NotebookDocument) { + const cacheId = GrpcSerializer.getDocumentCacheId(doc.metadata) if (!cacheId) { this.togglePreviewButton(false) @@ -575,7 +625,7 @@ export abstract class GrpcSerializerBase extends SerializerBase { return bytes.length } - const sessionFile = GrpcSerializerBase.getOutputsUri(srcDocUri, sessionId) + const sessionFile = GrpcSerializer.getOutputsUri(srcDocUri, sessionId) if (!sessionFile) { this.togglePreviewButton(false) return -1 @@ -590,7 +640,7 @@ export abstract class GrpcSerializerBase extends SerializerBase { public async saveNotebookOutputs(uri: Uri): Promise { let cacheId: string | undefined this.cacheDocUriMapping.forEach((docUri, cid) => { - const src = GrpcSerializerBase.getSourceFileUri(uri) + const src = GrpcSerializer.getSourceFileUri(uri) if (docUri.fsPath.toString() === src.fsPath.toString()) { cacheId = cid } @@ -612,7 +662,7 @@ export abstract class GrpcSerializerBase extends SerializerBase { } public static getOutputsUri(docUri: Uri, sessionId: string): Uri { - return Uri.parse(GrpcSerializerBase.getOutputsFilePath(docUri.fsPath, sessionId)) + return Uri.parse(GrpcSerializer.getOutputsFilePath(docUri.fsPath, sessionId)) } public static getSourceFilePath(outputsFile: string): string { @@ -630,26 +680,28 @@ export abstract class GrpcSerializerBase extends SerializerBase { } public static getSourceFileUri(outputsUri: Uri): Uri { - return Uri.parse(GrpcSerializerBase.getSourceFilePath(outputsUri.fsPath)) - } - - public getMaskedCache(cacheId: string): Promise | undefined { - return this.maskedCache.get(cacheId) - } - - public getPlainCache(cacheId: string): Promise | undefined { - return this.plainCache.get(cacheId) + return Uri.parse(GrpcSerializer.getSourceFilePath(outputsUri.fsPath)) } - public getNotebookDataCache(cacheId: string): NotebookData | undefined { - return this.notebookDataCache.get(cacheId) - } - - static sessionOutputsEnabled() { - const isAutoSaveOn = ContextState.getKey(NOTEBOOK_AUTOSAVE_ON) - const isSessionOutputs = getSessionOutputs() + protected applyIdentity(data: es_proto.Notebook): es_proto.Notebook { + const identity = this.lifecycleIdentity + switch (identity) { + case es_proto.RunmeIdentity.UNSPECIFIED: + case es_proto.RunmeIdentity.DOCUMENT: + break + default: { + data.cells.forEach((cell) => { + if (cell.kind !== es_proto.CellKind.CODE) { + return + } + if (!cell.metadata?.['id'] && cell.metadata?.['runme.dev/id']) { + cell.metadata['id'] = cell.metadata['runme.dev/id'] + } + }) + } + } - return isSessionOutputs && isAutoSaveOn + return data } public static getDocumentCacheId( @@ -674,74 +726,45 @@ export abstract class GrpcSerializerBase extends SerializerBase { return Boolean(sessionOutputId) } - // unable to implement marshal methods here, the underlying object structure is the same but - // the data types of the properties are different in some cases, presumably due to compile flags -} - -export class GrpcSerializer extends GrpcSerializerBase { - client!: ConnectClient - protected ready: ReadyPromise - protected serverUrl!: string - - private serverReadyListener?: Disposable + public override async switchLifecycleIdentity( + notebook: NotebookDocument, + identity: es_proto.RunmeIdentity, + ): Promise { + // skip session outputs files + if (!!notebook.metadata['runme.dev/frontmatterParsed']?.runme?.session?.id) { + return false + } - constructor( - protected context: ExtensionContext, - protected server: IServer, - kernel: Kernel, - ) { - super(context, kernel) - this.ready = new Promise((resolve) => { - resolve() - }) - this.serverReadyListener = server.onTransportReady(() => this.initClient()) - // cleanup listener when it's outlived its purpose - this.ready = new Promise((resolve) => { - const disposable = server.onTransportReady(() => { - disposable.dispose() - resolve() - }) + await notebook.save() + const source = await workspace.fs.readFile(notebook.uri) + const dreq = new es_proto.DeserializeRequest({ + source, + options: { identity }, }) - } - private initClient(): void { - // Server options: - // (1) pre-existing, started internally by this extension (assuming local execution so it has permissions to start) - // (2) pre-existing, started externally, presumably at the this.serializerServiceUrl address - // In both cases we need a key & cert (assuming tls) which may reside in: - // (a) this project: '/path/to/projectRoot/tls' - // (b) the runme binary server default: '~/.config/runme/tls' - // (c) other, manually configured + const deserialized = (await this.client.deserialize(dreq)).notebook - // I will assume 2a here - const addTlsConfigIfEnabled = () => { - try { - if (!getTLSEnabled()) { - return {} - } - const tlsPath = getTLSDir(Uri.parse(this.context.extensionPath)) - return { - nodeOptions: { - key: fs.readFileSync(`${tlsPath}/key.pem`), - cert: fs.readFileSync(`${tlsPath}/cert.pem`), - rejectUnauthorized: false, - }, - } - } catch (e: any) { - throw new Error(`Failed to read TLS files: ${e instanceof Error ? e.message : String(e)}`) - } + if (!deserialized) { + return false } - const s = getTLSEnabled() ? 's' : '' - this.serverUrl = `http${s}://${this.server.address()}` - - let grpcOptions = { - baseUrl: this.serverUrl, - httpVersion: '2', - ...addTlsConfigIfEnabled(), - } + deserialized.metadata = { ...deserialized.metadata, ...notebook.metadata } + const notebookEdit = NotebookEdit.updateNotebookMetadata(deserialized.metadata) + const edits = [notebookEdit] + notebook.getCells().forEach((cell) => { + const descell = deserialized.cells[cell.index] + // skip if no IDs are present, means no cell identity required + if (!descell.metadata?.['id']) { + return + } + const metadata = { ...descell.metadata, ...cell.metadata } + metadata['id'] = metadata['runme.dev/id'] + edits.push(NotebookEdit.updateCellMetadata(cell.index, metadata)) + }) - this.client = createConnectClient(ParserService, createGrpcTransport(grpcOptions)) + const edit = new WorkspaceEdit() + edit.set(notebook.uri, edits) + return await workspace.applyEdit(edit) } protected async saveNotebook( @@ -758,7 +781,7 @@ export class GrpcSerializer extends GrpcSerializerBase { data.metadata[RUNME_FRONTMATTER_PARSED] = notebook.frontmatter } - const cacheId = GrpcSerializerBase.getDocumentCacheId(data.metadata) + const cacheId = GrpcSerializer.getDocumentCacheId(data.metadata) this.notebookDataCache.set(cacheId as string, data) const serialRequest = new es_proto.SerializeRequest({ notebook }) @@ -780,104 +803,15 @@ export class GrpcSerializer extends GrpcSerializerBase { return serialResult.result } - protected async reviveNotebook( - content: Uint8Array, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - token: CancellationToken, - ): Promise { - const identity = this.lifecycleIdentity - // const markdown = Buffer.from(content).toString('utf8') - const deserializeRequest = new es_proto.DeserializeRequest({ - source: content, - options: { identity }, - }) - let res - try { - res = await this.client.deserialize(deserializeRequest) - } catch (e: any) { - if (e.name === 'ConnectError') { - e.message = `Unable to connect to the serializer service at ${this.serverUrl}` - } - log.error('Error in reviveNotebook ', e as any) - throw e - } - const notebook = res.notebook - - if (notebook === undefined) { - throw new Error('deserialization failed to revive notebook') - } - - this.applyIdentity(notebook) - - if (!notebook) { - return this.printCell('⚠️ __Error__: no cells found!') - } - return notebook as any as Serializer.Notebook // ugly cast :( - } - - protected applyIdentity(data: es_proto.Notebook): es_proto.Notebook { - const identity = this.lifecycleIdentity - switch (identity) { - case es_proto.RunmeIdentity.UNSPECIFIED: - case es_proto.RunmeIdentity.DOCUMENT: - break - default: { - data.cells.forEach((cell) => { - if (cell.kind !== es_proto.CellKind.CODE) { - return - } - if (!cell.metadata?.['id'] && cell.metadata?.['runme.dev/id']) { - cell.metadata['id'] = cell.metadata['runme.dev/id'] - } - }) - } - } - - return data - } - - public override async switchLifecycleIdentity( - notebook: NotebookDocument, - identity: es_proto.RunmeIdentity, - ): Promise { - // skip session outputs files - if (!!notebook.metadata['runme.dev/frontmatterParsed']?.runme?.session?.id) { - return false - } - - await notebook.save() - const source = await workspace.fs.readFile(notebook.uri) - const dr = new es_proto.DeserializeRequest({ - source, - options: { identity }, - }) - const deserialized = (await this.client.deserialize(dr)).notebook - - if (!deserialized) { - return false - } - - deserialized.metadata = { ...deserialized.metadata, ...notebook.metadata } - const notebookEdit = NotebookEdit.updateNotebookMetadata(deserialized.metadata) - const edits = [notebookEdit] - notebook.getCells().forEach((cell) => { - const descell = deserialized.cells[cell.index] - // skip if no IDs are present, means no cell identity required - if (!descell.metadata?.['id']) { - return - } - const metadata = { ...descell.metadata, ...cell.metadata } - metadata['id'] = metadata['runme.dev/id'] - edits.push(NotebookEdit.updateCellMetadata(cell.index, metadata)) - }) + static sessionOutputsEnabled() { + const isAutoSaveOn = ContextState.getKey(NOTEBOOK_AUTOSAVE_ON) + const isSessionOutputs = getSessionOutputs() - const edit = new WorkspaceEdit() - edit.set(notebook.uri, edits) - return await workspace.applyEdit(edit) + return isSessionOutputs && isAutoSaveOn } // unable to abstract due to RunmeSession struct potential differences & notebook ts type validation issues - protected async cacheNotebookOutputs( + private async cacheNotebookOutputs( notebook: es_proto.Notebook, cacheId: string | undefined, ): Promise { @@ -988,7 +922,7 @@ export class GrpcSerializer extends GrpcSerializerBase { log.warn('no frontmatter found in metadata') return new es_proto.Frontmatter({ category: '', - // tag: '', //?? es_proto does not have tag, but timostam/prototype-ts does?? + // tag: '', // buf-es does not have tag property, but the old timostam/prototype-ts did cwd: '', runme: { id: '', @@ -1024,7 +958,7 @@ export class GrpcSerializer extends GrpcSerializerBase { session: { id: kernel?.getRunnerEnvironment()?.getSessionId() || '' }, }, category: '', - // tag: '', // es_proto does not have tag + // tag: '', // es_proto does not have a tag property, but timostamm-ts did cwd: '', shell: '', skipPrompts: false, @@ -1084,4 +1018,56 @@ export class GrpcSerializer extends GrpcSerializerBase { }, }) } + + protected async reviveNotebook( + content: Uint8Array, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + token: CancellationToken, + ): Promise { + const identity = this.lifecycleIdentity + const deserializeRequest = new es_proto.DeserializeRequest({ + source: content, + options: { identity }, + }) + let res + try { + res = await this.client.deserialize(deserializeRequest) + } catch (e: any) { + if (e.name === 'ConnectError') { + e.message = `Unable to connect to the serializer service at ${this.serverUrl}` + } + log.error('Error in reviveNotebook ', e as any) + throw e + } + const notebook = res.notebook + + if (notebook === undefined) { + throw new Error('deserialization failed to revive notebook') + } + + this.applyIdentity(notebook) + + if (!notebook) { + return this.printCell('⚠️ __Error__: no cells found!') + } + // we can remove ugly casting once we switch to GRPC + return notebook as unknown as Serializer.Notebook + } + + public dispose(): void { + this.serverReadyListener?.dispose() + super.dispose() + } + + public getMaskedCache(cacheId: string): Promise | undefined { + return this.maskedCache.get(cacheId) + } + + public getPlainCache(cacheId: string): Promise | undefined { + return this.plainCache.get(cacheId) + } + + public getNotebookDataCache(cacheId: string): NotebookData | undefined { + return this.notebookDataCache.get(cacheId) + } } diff --git a/tests/extension/serializer.test.ts b/tests/extension/serializer.test.ts index 614e517bd..7ad268783 100644 --- a/tests/extension/serializer.test.ts +++ b/tests/extension/serializer.test.ts @@ -10,12 +10,7 @@ import { expect, vi, it, describe, beforeEach } from 'vitest' import { isValid } from 'ulidx' import { RunmeIdentity, Notebook } from '@buf/stateful_runme.bufbuild_es/runme/parser/v1/parser_pb' -import { - GrpcSerializer, - SerializerBase, - WasmSerializer, - GrpcSerializerBase, -} from '../../src/extension/serializer' +import { GrpcSerializer, SerializerBase, WasmSerializer } from '../../src/extension/serializer' import type { Kernel } from '../../src/extension/kernel' import { EventEmitter, Uri } from '../../__mocks__/vscode' import { Serializer } from '../../src/types' @@ -523,7 +518,7 @@ describe('GrpcSerializer', () => { const serializer: any = new GrpcSerializer(context, new Server(), new Kernel()) - vi.spyOn(GrpcSerializerBase, 'getOutputsUri').mockReturnValue(fakeSrcDocUri) + vi.spyOn(GrpcSerializer, 'getOutputsUri').mockReturnValue(fakeSrcDocUri) await serializer.handleOpenNotebook({ uri: fakeSrcDocUri, @@ -598,7 +593,7 @@ describe('GrpcSerializer', () => { fixture.metadata['runme.dev/frontmatterParsed'].runme.id, fakeCachedBytes, ) - GrpcSerializerBase.getOutputsUri = vi.fn().mockImplementation(() => undefined) + GrpcSerializer.getOutputsUri = vi.fn().mockImplementation(() => undefined) await serializer.handleSaveNotebookOutputs({ uri: fakeSrcDocUri, metadata: fixture.metadata, @@ -637,7 +632,7 @@ describe('GrpcSerializer', () => { writeableSer.cacheDocUriMapping.set(fixture.metadata['runme.dev/cacheId'], fakeSrcDocUri) ContextState.getKey = vi.fn().mockImplementation(() => true) GrpcSerializer.sessionOutputsEnabled = vi.fn().mockReturnValue(true) - GrpcSerializerBase.getOutputsUri = vi.fn().mockImplementation(() => fakeSrcDocUri) + GrpcSerializer.getOutputsUri = vi.fn().mockImplementation(() => fakeSrcDocUri) const result = await writeableSer.serializeNotebook( { cells: [], metadata: fixture.metadata } as any, From 99964a2eb27330e4657bac69a0a7300b8b645b5c Mon Sep 17 00:00:00 2001 From: hotpocket Date: Sun, 15 Dec 2024 00:32:53 -0800 Subject: [PATCH 14/18] more diff consolidation for PR --- src/extension/serializer.ts | 60 +++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/src/extension/serializer.ts b/src/extension/serializer.ts index 70a173b6f..6ee2558cd 100644 --- a/src/extension/serializer.ts +++ b/src/extension/serializer.ts @@ -27,7 +27,18 @@ import { ParserService } from '@buf/stateful_runme.connectrpc_es/runme/parser/v1 import { createGrpcTransport, GrpcTransportOptions } from '@connectrpc/connect-node' import { createClient as createConnectClient, Client as ConnectClient } from '@connectrpc/connect' // ts bindings generated by protoc-gen-es -import * as es_proto from '@buf/stateful_runme.bufbuild_es/runme/parser/v1/parser_pb' +import { + RunmeIdentity, + RunmeSession, + Notebook, + Frontmatter, + CellOutput, + CellKind, + CellExecutionSummary, + DeserializeRequest, + SerializeRequest, + SerializeRequestOptions, +} from '@buf/stateful_runme.bufbuild_es/runme/parser/v1/parser_pb' import { Serializer } from '../types' import { @@ -46,7 +57,6 @@ import { getTLSEnabled, } from '../utils/configuration' -import { RunmeIdentity } from './grpc/serializerTypes' import { type ReadyPromise } from './grpc/client' import Languages from './languages' import { PLATFORM_OS } from './constants' @@ -683,15 +693,15 @@ export class GrpcSerializer extends SerializerBase { return Uri.parse(GrpcSerializer.getSourceFilePath(outputsUri.fsPath)) } - protected applyIdentity(data: es_proto.Notebook): es_proto.Notebook { + protected applyIdentity(data: Notebook): Notebook { const identity = this.lifecycleIdentity switch (identity) { - case es_proto.RunmeIdentity.UNSPECIFIED: - case es_proto.RunmeIdentity.DOCUMENT: + case RunmeIdentity.UNSPECIFIED: + case RunmeIdentity.DOCUMENT: break default: { data.cells.forEach((cell) => { - if (cell.kind !== es_proto.CellKind.CODE) { + if (cell.kind !== CellKind.CODE) { return } if (!cell.metadata?.['id'] && cell.metadata?.['runme.dev/id']) { @@ -728,7 +738,7 @@ export class GrpcSerializer extends SerializerBase { public override async switchLifecycleIdentity( notebook: NotebookDocument, - identity: es_proto.RunmeIdentity, + identity: RunmeIdentity, ): Promise { // skip session outputs files if (!!notebook.metadata['runme.dev/frontmatterParsed']?.runme?.session?.id) { @@ -737,7 +747,7 @@ export class GrpcSerializer extends SerializerBase { await notebook.save() const source = await workspace.fs.readFile(notebook.uri) - const dreq = new es_proto.DeserializeRequest({ + const dreq = new DeserializeRequest({ source, options: { identity }, }) @@ -784,7 +794,7 @@ export class GrpcSerializer extends SerializerBase { const cacheId = GrpcSerializer.getDocumentCacheId(data.metadata) this.notebookDataCache.set(cacheId as string, data) - const serialRequest = new es_proto.SerializeRequest({ notebook }) + const serialRequest = new SerializeRequest({ notebook }) const cacheOutputs = this.cacheNotebookOutputs(notebook, cacheId) const request = this.client.serialize(serialRequest) @@ -812,22 +822,22 @@ export class GrpcSerializer extends SerializerBase { // unable to abstract due to RunmeSession struct potential differences & notebook ts type validation issues private async cacheNotebookOutputs( - notebook: es_proto.Notebook, + notebook: Notebook, cacheId: string | undefined, ): Promise { - let session: es_proto.RunmeSession | undefined + let session: RunmeSession | undefined const docUri = this.cacheDocUriMapping.get(cacheId ?? '') const sid = this.kernel.getRunnerEnvironment()?.getSessionId() if (sid && docUri) { const relativePath = path.basename(docUri.fsPath) - session = new es_proto.RunmeSession({ + session = new RunmeSession({ id: sid, document: { relativePath }, }) } const outputs = { enabled: true, summary: true } - const options = new es_proto.SerializeRequestOptions({ + const options = new SerializeRequestOptions({ outputs, session, }) @@ -845,10 +855,10 @@ export class GrpcSerializer extends SerializerBase { }) }) - const plainReq = new es_proto.SerializeRequest({ notebook, options }) + const plainReq = new SerializeRequest({ notebook, options }) const plainRes = this.client.serialize(plainReq) - const maskedReq = new es_proto.SerializeRequest({ notebook: maskedNotebook, options }) + const maskedReq = new SerializeRequest({ notebook: maskedNotebook, options }) const masked = this.client.serialize(maskedReq).then((maskedRes) => { if (maskedRes.result === undefined) { console.error('serialization of masked notebook failed') @@ -885,9 +895,9 @@ export class GrpcSerializer extends SerializerBase { marshalFrontmatter?: boolean kernel?: Kernel }, - ): es_proto.Notebook { + ): Notebook { // the bulk copies cleanly except for what's below - const notebook = new es_proto.Notebook(data as any) + const notebook = new Notebook(data as any) // cannot gurantee it wasn't changed if (notebook.metadata[RUNME_FRONTMATTER_PARSED]) { @@ -914,13 +924,13 @@ export class GrpcSerializer extends SerializerBase { static marshalFrontmatter( metadata: { ['runme.dev/frontmatter']?: string }, kernel?: Kernel, - ): es_proto.Frontmatter { + ): Frontmatter { if ( !metadata.hasOwnProperty('runme.dev/frontmatter') || typeof metadata['runme.dev/frontmatter'] !== 'string' ) { log.warn('no frontmatter found in metadata') - return new es_proto.Frontmatter({ + return new Frontmatter({ category: '', // tag: '', // buf-es does not have tag property, but the old timostam/prototype-ts did cwd: '', @@ -951,7 +961,7 @@ export class GrpcSerializer extends SerializerBase { } } - return new es_proto.Frontmatter({ + return new Frontmatter({ runme: { id: data.runme?.id || '', version: data.runme?.version || '', @@ -967,9 +977,9 @@ export class GrpcSerializer extends SerializerBase { } private static marshalCellOutputs( - outputs: es_proto.CellOutput[], + outputs: CellOutput[], dataOutputs: NotebookCellOutput[] | undefined, - ): es_proto.CellOutput[] { + ): CellOutput[] { if (!dataOutputs) { return [] } @@ -1000,7 +1010,7 @@ export class GrpcSerializer extends SerializerBase { private static marshalCellExecutionSummary( executionSummary: NotebookCellExecutionSummary | undefined, - ): es_proto.CellExecutionSummary | undefined { + ): CellExecutionSummary | undefined { if (!executionSummary) { return undefined } @@ -1010,7 +1020,7 @@ export class GrpcSerializer extends SerializerBase { return undefined } - return new es_proto.CellExecutionSummary({ + return new CellExecutionSummary({ success: success, timing: { endTime: BigInt(timing!.endTime), @@ -1025,7 +1035,7 @@ export class GrpcSerializer extends SerializerBase { token: CancellationToken, ): Promise { const identity = this.lifecycleIdentity - const deserializeRequest = new es_proto.DeserializeRequest({ + const deserializeRequest = new DeserializeRequest({ source: content, options: { identity }, }) From f9f3e1bb4be8c146bf8b15e20afd4667943b7e05 Mon Sep 17 00:00:00 2001 From: hotpocket Date: Mon, 16 Dec 2024 17:30:24 -0800 Subject: [PATCH 15/18] pr diff cleanup --- src/extension/extension.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/extension/extension.ts b/src/extension/extension.ts index fedfe59cf..386e01511 100644 --- a/src/extension/extension.ts +++ b/src/extension/extension.ts @@ -70,7 +70,7 @@ import { runForkCommand, selectEnvironment, } from './commands' -import { WasmSerializer, SerializerBase, GrpcSerializer } from './serializer' +import { WasmSerializer, GrpcSerializer, SerializerBase } from './serializer' import { RunmeLauncherProvider, RunmeTreeProvider } from './provider/launcher' import { RunmeLauncherProvider as RunmeLauncherProviderBeta } from './provider/launcherBeta' import { RunmeUriHandler } from './handler/uri' @@ -126,7 +126,6 @@ export class RunmeExtension { ) const reporter = new GrpcReporter(context, server) - const serializer = grpcSerializer ? new GrpcSerializer(context, server, kernel) : new WasmSerializer(context, kernel) From e4c25077c75d58a4645c02e3a9c6b23e95b2b480 Mon Sep 17 00:00:00 2001 From: hotpocket Date: Thu, 19 Dec 2024 18:46:13 -0800 Subject: [PATCH 16/18] fix old schema via foyle npm update --- package-lock.json | 47 ++++++++++--------------------------- src/extension/serializer.ts | 4 ++-- 2 files changed, 14 insertions(+), 37 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0e02ebe51..5977b4571 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2394,23 +2394,6 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, - "node_modules/@buf/bufbuild_protovalidate.bufbuild_es": { - "version": "1.10.0-20240212200630-3014d81c3a48.1", - "resolved": "https://buf.build/gen/npm/v1/@buf/bufbuild_protovalidate.bufbuild_es/-/bufbuild_protovalidate.bufbuild_es-1.10.0-20240212200630-3014d81c3a48.1.tgz", - "peerDependencies": { - "@bufbuild/protobuf": "^1.10.0" - } - }, - "node_modules/@buf/bufbuild_protovalidate.connectrpc_es": { - "version": "1.5.0-20240212200630-3014d81c3a48.1", - "resolved": "https://buf.build/gen/npm/v1/@buf/bufbuild_protovalidate.connectrpc_es/-/bufbuild_protovalidate.connectrpc_es-1.5.0-20240212200630-3014d81c3a48.1.tgz", - "dependencies": { - "@buf/bufbuild_protovalidate.bufbuild_es": "1.10.0-20240212200630-3014d81c3a48.1" - }, - "peerDependencies": { - "@connectrpc/connect": "^1.5.0" - } - }, "node_modules/@buf/googleapis_googleapis.community_timostamm-protobuf-ts": { "version": "2.9.4-20240827201746-e7f8d366f526.4", "resolved": "https://buf.build/gen/npm/v1/@buf/googleapis_googleapis.community_timostamm-protobuf-ts/-/googleapis_googleapis.community_timostamm-protobuf-ts-2.9.4-20240827201746-e7f8d366f526.4.tgz", @@ -2431,34 +2414,29 @@ } }, "node_modules/@buf/jlewi_foyle.bufbuild_es": { - "version": "1.10.0-20241018224331-ac296bc95724.1", - "resolved": "https://buf.build/gen/npm/v1/@buf/jlewi_foyle.bufbuild_es/-/jlewi_foyle.bufbuild_es-1.10.0-20241018224331-ac296bc95724.1.tgz", + "version": "1.10.0-20241219050144-856e893555b3.1", + "resolved": "https://buf.build/gen/npm/v1/@buf/jlewi_foyle.bufbuild_es/-/jlewi_foyle.bufbuild_es-1.10.0-20241219050144-856e893555b3.1.tgz", "dependencies": { - "@buf/bufbuild_protovalidate.bufbuild_es": "1.10.0-20240212200630-3014d81c3a48.1", - "@buf/stateful_runme.bufbuild_es": "1.10.0-20240913234806-45813f39881a.1" + "@buf/stateful_runme.bufbuild_es": "1.10.0-20241021161352-11c142f88aee.1" }, "peerDependencies": { "@bufbuild/protobuf": "^1.10.0" } }, "node_modules/@buf/jlewi_foyle.connectrpc_es": { - "version": "1.5.0-20241018224331-ac296bc95724.1", - "resolved": "https://buf.build/gen/npm/v1/@buf/jlewi_foyle.connectrpc_es/-/jlewi_foyle.connectrpc_es-1.5.0-20241018224331-ac296bc95724.1.tgz", + "version": "1.5.0-20241219050144-856e893555b3.1", + "resolved": "https://buf.build/gen/npm/v1/@buf/jlewi_foyle.connectrpc_es/-/jlewi_foyle.connectrpc_es-1.5.0-20241219050144-856e893555b3.1.tgz", "dependencies": { - "@buf/bufbuild_protovalidate.connectrpc_es": "1.5.0-20240212200630-3014d81c3a48.1", - "@buf/jlewi_foyle.bufbuild_es": "1.10.0-20241018224331-ac296bc95724.1", - "@buf/stateful_runme.connectrpc_es": "1.5.0-20240913234806-45813f39881a.1" + "@buf/jlewi_foyle.bufbuild_es": "1.10.0-20241219050144-856e893555b3.1", + "@buf/stateful_runme.connectrpc_es": "1.5.0-20241021161352-11c142f88aee.1" }, "peerDependencies": { "@connectrpc/connect": "^1.5.0" } }, "node_modules/@buf/stateful_runme.bufbuild_es": { - "version": "1.10.0-20240913234806-45813f39881a.1", - "resolved": "https://buf.build/gen/npm/v1/@buf/stateful_runme.bufbuild_es/-/stateful_runme.bufbuild_es-1.10.0-20240913234806-45813f39881a.1.tgz", - "dependencies": { - "@buf/bufbuild_protovalidate.bufbuild_es": "1.10.0-20240212200630-3014d81c3a48.1" - }, + "version": "1.10.0-20241021161352-11c142f88aee.1", + "resolved": "https://buf.build/gen/npm/v1/@buf/stateful_runme.bufbuild_es/-/stateful_runme.bufbuild_es-1.10.0-20241021161352-11c142f88aee.1.tgz", "peerDependencies": { "@bufbuild/protobuf": "^1.10.0" } @@ -2472,11 +2450,10 @@ } }, "node_modules/@buf/stateful_runme.connectrpc_es": { - "version": "1.5.0-20240913234806-45813f39881a.1", - "resolved": "https://buf.build/gen/npm/v1/@buf/stateful_runme.connectrpc_es/-/stateful_runme.connectrpc_es-1.5.0-20240913234806-45813f39881a.1.tgz", + "version": "1.5.0-20241021161352-11c142f88aee.1", + "resolved": "https://buf.build/gen/npm/v1/@buf/stateful_runme.connectrpc_es/-/stateful_runme.connectrpc_es-1.5.0-20241021161352-11c142f88aee.1.tgz", "dependencies": { - "@buf/bufbuild_protovalidate.connectrpc_es": "1.5.0-20240212200630-3014d81c3a48.1", - "@buf/stateful_runme.bufbuild_es": "1.10.0-20240913234806-45813f39881a.1" + "@buf/stateful_runme.bufbuild_es": "1.10.0-20241021161352-11c142f88aee.1" }, "peerDependencies": { "@connectrpc/connect": "^1.5.0" diff --git a/src/extension/serializer.ts b/src/extension/serializer.ts index 6ee2558cd..b5db9054e 100644 --- a/src/extension/serializer.ts +++ b/src/extension/serializer.ts @@ -932,7 +932,7 @@ export class GrpcSerializer extends SerializerBase { log.warn('no frontmatter found in metadata') return new Frontmatter({ category: '', - // tag: '', // buf-es does not have tag property, but the old timostam/prototype-ts did + tag: '', // buf-es does not have tag property, but the old timostam/prototype-ts did cwd: '', runme: { id: '', @@ -968,7 +968,7 @@ export class GrpcSerializer extends SerializerBase { session: { id: kernel?.getRunnerEnvironment()?.getSessionId() || '' }, }, category: '', - // tag: '', // es_proto does not have a tag property, but timostamm-ts did + tag: '', // es_proto does not have a tag property, but timostamm-ts did cwd: '', shell: '', skipPrompts: false, From 1aa7dfeb246eae73476cb6ed5b4970be8d3e2579 Mon Sep 17 00:00:00 2001 From: hotpocket Date: Thu, 19 Dec 2024 19:03:45 -0800 Subject: [PATCH 17/18] expose TLS methods to avoid code duplication --- src/extension/serializer.ts | 29 ++++++++++++---------------- src/extension/server/kernelServer.ts | 4 ++-- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/extension/serializer.ts b/src/extension/serializer.ts index b5db9054e..199582057 100644 --- a/src/extension/serializer.ts +++ b/src/extension/serializer.ts @@ -1,5 +1,4 @@ import path from 'node:path' -import fs from 'node:fs' import { NotebookSerializer, @@ -50,18 +49,13 @@ import { RUNME_FRONTMATTER_PARSED, VSCODE_LANGUAGEID_MAP, } from '../constants' -import { - ServerLifecycleIdentity, - getSessionOutputs, - getTLSDir, - getTLSEnabled, -} from '../utils/configuration' +import { ServerLifecycleIdentity, getSessionOutputs, getTLSEnabled } from '../utils/configuration' import { type ReadyPromise } from './grpc/client' import Languages from './languages' import { PLATFORM_OS } from './constants' import { initWasm } from './utils' -import { IServer } from './server/kernelServer' +import KernelServer from './server/kernelServer' import { Kernel } from './kernel' import { getCellById } from './cell' import { IProcessInfoState } from './terminal/terminalState' @@ -498,7 +492,7 @@ export class GrpcSerializer extends SerializerBase { constructor( protected context: ExtensionContext, - protected server: IServer, + protected server: KernelServer, kernel: Kernel, ) { super(context, kernel) @@ -512,7 +506,7 @@ export class GrpcSerializer extends SerializerBase { resolve() }) }) - this.serverReadyListener = server.onTransportReady(() => this.initClient()) + this.serverReadyListener = server.onTransportReady(async () => await this.initClient()) this.disposables.push( // todo(sebastian): delete entries on session reset not notebook editor lifecycle // workspace.onDidCloseNotebookDocument(this.handleCloseNotebook.bind(this)), @@ -521,7 +515,7 @@ export class GrpcSerializer extends SerializerBase { ) } - private initClient(): void { + private async initClient(): Promise { // Server options: // (1) pre-existing, started internally by this extension (assuming local execution so it has permissions to start) // (2) pre-existing, started externally, presumably at the this.serverUrl address @@ -531,16 +525,17 @@ export class GrpcSerializer extends SerializerBase { // (c) other, manually configured // assuming 2a, which is the default when starting the server as a result of loading this extension - const addTlsConfigIfEnabled = () => { + const addTlsConfigIfEnabled = async () => { try { if (!getTLSEnabled()) { return {} } - const tlsPath = getTLSDir(Uri.parse(this.context.extensionPath)) + const pems = await KernelServer.getTLS(this.server.getTLSDir()) + return { nodeOptions: { - key: fs.readFileSync(`${tlsPath}/key.pem`), - cert: fs.readFileSync(`${tlsPath}/cert.pem`), + key: pems.privKeyPEM, + cert: pems.certPEM, rejectUnauthorized: false, }, } @@ -551,11 +546,11 @@ export class GrpcSerializer extends SerializerBase { const s = getTLSEnabled() ? 's' : '' this.serverUrl = `http${s}://${this.server.address()}` - + const tlsConfig = await addTlsConfigIfEnabled() let grpcOptions = { baseUrl: this.serverUrl, httpVersion: '2', - ...addTlsConfigIfEnabled(), + ...tlsConfig, } this.client = createConnectClient(ParserService, createGrpcTransport(grpcOptions)) diff --git a/src/extension/server/kernelServer.ts b/src/extension/server/kernelServer.ts index 8c3bf76ea..3576d3f49 100644 --- a/src/extension/server/kernelServer.ts +++ b/src/extension/server/kernelServer.ts @@ -167,7 +167,7 @@ class KernelServer implements IServer { return !!(getCustomServerAddress() || this.#forceExternalServer) } - private static async getTLS(tlsDir: string) { + public static async getTLS(tlsDir: string) { try { const certPEM = await fs.readFile(path.join(tlsDir, 'cert.pem')) const privKeyPEM = await fs.readFile(path.join(tlsDir, 'key.pem')) @@ -178,7 +178,7 @@ class KernelServer implements IServer { } } - protected getTLSDir(): string { + public getTLSDir(): string { return getTLSDir(this.extBasePath) } From f9144e2d23de6d6016a4529460cf411ae520275a Mon Sep 17 00:00:00 2001 From: hotpocket Date: Thu, 19 Dec 2024 22:53:31 -0800 Subject: [PATCH 18/18] remove stale comment --- src/extension/serializer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/extension/serializer.ts b/src/extension/serializer.ts index 199582057..371d1cb4f 100644 --- a/src/extension/serializer.ts +++ b/src/extension/serializer.ts @@ -927,7 +927,7 @@ export class GrpcSerializer extends SerializerBase { log.warn('no frontmatter found in metadata') return new Frontmatter({ category: '', - tag: '', // buf-es does not have tag property, but the old timostam/prototype-ts did + tag: '', cwd: '', runme: { id: '', @@ -963,7 +963,7 @@ export class GrpcSerializer extends SerializerBase { session: { id: kernel?.getRunnerEnvironment()?.getSessionId() || '' }, }, category: '', - tag: '', // es_proto does not have a tag property, but timostamm-ts did + tag: '', cwd: '', shell: '', skipPrompts: false,