diff --git a/scripts/emulator-tests/functionsEmulator.spec.ts b/scripts/emulator-tests/functionsEmulator.spec.ts index 30d336bc0b4..f6591d0e8e3 100644 --- a/scripts/emulator-tests/functionsEmulator.spec.ts +++ b/scripts/emulator-tests/functionsEmulator.spec.ts @@ -38,6 +38,7 @@ const TEST_BACKEND = { secretEnv: [], codebase: "default", bin: process.execPath, + runtime: "nodejs14", // NOTE: Use the following node bin path if you want to run test cases directly from your IDE. // bin: path.join(MODULE_ROOT, "node_modules/.bin/ts-node"), }; diff --git a/src/deploy/functions/runtimes/discovery/index.ts b/src/deploy/functions/runtimes/discovery/index.ts index 809671fa93a..e5e0e8ff71e 100644 --- a/src/deploy/functions/runtimes/discovery/index.ts +++ b/src/deploy/functions/runtimes/discovery/index.ts @@ -72,7 +72,7 @@ export async function detectFromPort( while (true) { try { - res = await Promise.race([fetch(`http://localhost:${port}/__/functions.yaml`), timedOut]); + res = await Promise.race([fetch(`http://127.0.0.1:${port}/__/functions.yaml`), timedOut]); break; } catch (err: any) { // Allow us to wait until the server is listening. diff --git a/src/deploy/functions/runtimes/index.ts b/src/deploy/functions/runtimes/index.ts index ce327dd4f8c..8864b127f44 100644 --- a/src/deploy/functions/runtimes/index.ts +++ b/src/deploy/functions/runtimes/index.ts @@ -1,6 +1,7 @@ import * as backend from "../backend"; import * as build from "../build"; import * as node from "./node"; +import * as python from "./python"; import * as validate from "../validate"; import { FirebaseError } from "../../../error"; @@ -9,7 +10,7 @@ const RUNTIMES: string[] = ["nodejs10", "nodejs12", "nodejs14", "nodejs16", "nod // Experimental runtimes are part of the Runtime type, but are in a // different list to help guard against some day accidentally iterating over // and printing a hidden runtime to the user. -const EXPERIMENTAL_RUNTIMES: string[] = []; +const EXPERIMENTAL_RUNTIMES: string[] = ["python310", "python311"]; export type Runtime = typeof RUNTIMES[number] | typeof EXPERIMENTAL_RUNTIMES[number]; /** Runtimes that can be found in existing backends but not used for new functions. */ @@ -34,6 +35,8 @@ const MESSAGE_FRIENDLY_RUNTIMES: Record = { nodejs14: "Node.js 14", nodejs16: "Node.js 16", nodejs18: "Node.js 18", + python310: "Python 3.10", + python311: "Python 3.11 (Preview)", }; /** @@ -113,7 +116,7 @@ export interface DelegateContext { } type Factory = (context: DelegateContext) => Promise; -const factories: Factory[] = [node.tryCreateDelegate]; +const factories: Factory[] = [node.tryCreateDelegate, python.tryCreateDelegate]; /** * diff --git a/src/deploy/functions/runtimes/python/index.ts b/src/deploy/functions/runtimes/python/index.ts new file mode 100644 index 00000000000..4e4989f9df5 --- /dev/null +++ b/src/deploy/functions/runtimes/python/index.ts @@ -0,0 +1,152 @@ +import * as fs from "fs"; +import * as path from "path"; +import fetch from "node-fetch"; +import { promisify } from "util"; + +import * as portfinder from "portfinder"; + +import * as runtimes from ".."; +import * as backend from "../../backend"; +import * as discovery from "../discovery"; +import { logger } from "../../../../logger"; +import { runWithVirtualEnv } from "../../../../functions/python"; +import { FirebaseError } from "../../../../error"; +import { Build } from "../../build"; + +const LATEST_VERSION: runtimes.Runtime = "python310"; + +/** + * Create a runtime delegate for the Python runtime, if applicable. + * + * @param context runtimes.DelegateContext + * @return Delegate Python runtime delegate + */ +export async function tryCreateDelegate( + context: runtimes.DelegateContext +): Promise { + const requirementsTextPath = path.join(context.sourceDir, "requirements.txt"); + + if (!(await promisify(fs.exists)(requirementsTextPath))) { + logger.debug("Customer code is not Python code."); + return; + } + const runtime = context.runtime ? context.runtime : LATEST_VERSION; + if (!runtimes.isValidRuntime(runtime)) { + throw new FirebaseError(`Runtime ${runtime} is not a valid Python runtime`); + } + return Promise.resolve(new Delegate(context.projectId, context.sourceDir, runtime)); +} + +export class Delegate implements runtimes.RuntimeDelegate { + public readonly name = "python"; + constructor( + private readonly projectId: string, + private readonly sourceDir: string, + public readonly runtime: runtimes.Runtime + ) {} + + private _bin = ""; + private _modulesDir = ""; + + get bin(): string { + if (this._bin === "") { + this._bin = this.getPythonBinary(); + } + return this._bin; + } + + async modulesDir(): Promise { + if (!this._modulesDir) { + const child = runWithVirtualEnv( + [ + this.bin, + "-c", + '"import firebase_functions; import os; print(os.path.dirname(firebase_functions.__file__))"', + ], + this.sourceDir, + {} + ); + let out = ""; + child.stdout?.on("data", (chunk: Buffer) => { + const chunkString = chunk.toString(); + out = out + chunkString; + logger.debug(`stdout: ${chunkString}`); + }); + await new Promise((resolve, reject) => { + child.on("exit", resolve); + child.on("error", reject); + }); + this._modulesDir = out.trim(); + } + return this._modulesDir; + } + + getPythonBinary(): string { + if (process.platform === "win32") { + // There is no easy way to get specific version of python executable in Windows. + return "python.exe"; + } + if (this.runtime === "python310") { + return "python3.10"; + } else if (this.runtime === "python311") { + return "python3.11"; + } + return "python"; + } + + validate(): Promise { + // TODO: make sure firebase-functions is included as a dep + return Promise.resolve(); + } + + watch(): Promise<() => Promise> { + return Promise.resolve(() => Promise.resolve()); + } + + async build(): Promise { + return Promise.resolve(); + } + + async serveAdmin(port: number, envs: backend.EnvironmentVariables): Promise<() => Promise> { + const modulesDir = await this.modulesDir(); + const envWithAdminPort = { + ...envs, + ADMIN_PORT: port.toString(), + }; + const args = [this.bin, path.join(modulesDir, "private", "serving.py")]; + logger.debug( + `Running admin server with args: ${JSON.stringify(args)} and env: ${JSON.stringify( + envWithAdminPort + )} in ${this.sourceDir}` + ); + const childProcess = runWithVirtualEnv(args, this.sourceDir, envWithAdminPort); + return Promise.resolve(async () => { + await fetch(`http://127.0.0.1:${port}/__/quitquitquit`); + const quitTimeout = setTimeout(() => { + if (!childProcess.killed) { + childProcess.kill("SIGKILL"); + } + }, 10_000); + clearTimeout(quitTimeout); + }); + } + + async discoverBuild( + _configValues: backend.RuntimeConfigValues, + envs: backend.EnvironmentVariables + ): Promise { + let discovered = await discovery.detectFromYaml(this.sourceDir, this.projectId, this.runtime); + if (!discovered) { + const adminPort = await portfinder.getPortPromise({ + port: 8081, + }); + const killProcess = await this.serveAdmin(adminPort, envs); + try { + discovered = await discovery.detectFromPort(adminPort, this.projectId, this.runtime); + } finally { + await killProcess(); + } + } + return discovered; + } +} diff --git a/src/emulator/functionsEmulator.ts b/src/emulator/functionsEmulator.ts index d8124797e19..34722120d5e 100644 --- a/src/emulator/functionsEmulator.ts +++ b/src/emulator/functionsEmulator.ts @@ -15,6 +15,7 @@ import { track, trackEmulator } from "../track"; import { Constants } from "./constants"; import { EmulatorInfo, EmulatorInstance, Emulators, FunctionsExecutionMode } from "./types"; import * as chokidar from "chokidar"; +import * as portfinder from "portfinder"; import * as spawn from "cross-spawn"; import { ChildProcess } from "child_process"; @@ -43,7 +44,7 @@ import { RuntimeWorker, RuntimeWorkerPool } from "./functionsRuntimeWorker"; import { PubsubEmulator } from "./pubsubEmulator"; import { FirebaseError } from "../error"; import { WorkQueue, Work } from "./workQueue"; -import { allSettled, connectableHostname, createDestroyer, debounce } from "../utils"; +import { allSettled, connectableHostname, createDestroyer, debounce, randomInt } from "../utils"; import { getCredentialPathAsync } from "../defaultCredentials"; import { AdminSdkConfig, @@ -60,6 +61,7 @@ import { AUTH_BLOCKING_EVENTS, BEFORE_CREATE_EVENT } from "../functions/events/v import { BlockingFunctionsConfig } from "../gcp/identityPlatform"; import { resolveBackend } from "../deploy/functions/build"; import { setEnvVarsForEmulators } from "./env"; +import { runWithVirtualEnv } from "../functions/python"; const EVENT_INVOKE = "functions:invoke"; // event name for UA const EVENT_INVOKE_GA4 = "functions_invoke"; // event name GA4 (alphanumertic) @@ -122,15 +124,41 @@ export interface FunctionsEmulatorArgs { projectAlias?: string; } -// FunctionsRuntimeInstance is the handler for a running function invocation +/** + * IPC connection info of a Function Runtime. + */ +export class IPCConn { + constructor(readonly socketPath: string) {} + + httpReqOpts(): http.RequestOptions { + return { + socketPath: this.socketPath, + }; + } +} + +/** + * TCP/IP connection info of a Function Runtime. + */ +export class TCPConn { + constructor(readonly host: string, readonly port: number) {} + + httpReqOpts(): http.RequestOptions { + return { + host: this.host, + port: this.port, + }; + } +} + export interface FunctionsRuntimeInstance { process: ChildProcess; // An emitter which sends our EmulatorLog events from the runtime. events: EventEmitter; // A cwd of the process cwd: string; - // Path to socket file used for HTTP-over-IPC comms. - socketPath: string; + // Communication info for the runtime + conn: IPCConn | TCPConn; } export interface InvokeRuntimeOpts { @@ -353,8 +381,8 @@ export class FunctionsEmulator implements EmulatorInstance { return new Promise((resolve, reject) => { const req = http.request( { + ...worker.runtime.conn.httpReqOpts(), path: `/`, - socketPath: worker.runtime.socketPath, headers: headers, }, resolve @@ -405,6 +433,7 @@ export class FunctionsEmulator implements EmulatorInstance { /.+?[\\\/]node_modules[\\\/].+?/, // Ignore node_modules /(^|[\/\\])\../, // Ignore files which begin the a period /.+\.log/, // Ignore files which have a .log extension + /.+?[\\\/]venv[\\\/].+?/, // Ignore site-packages in venv ], persistent: true, }); @@ -660,26 +689,30 @@ export class FunctionsEmulator implements EmulatorInstance { // In debug mode, we eagerly start the runtime processes to allow debuggers to attach // before invoking a function. if (this.args.debugPort) { - // Since we're about to start a runtime to be shared by all the functions in this codebase, - // we need to make sure it has all the secrets used by any function in the codebase. - emulatableBackend.secretEnv = Object.values( - triggerDefinitions.reduce( - (acc: Record, curr: EmulatedTriggerDefinition) => { - for (const secret of curr.secretEnvironmentVariables || []) { - acc[secret.key] = secret; - } - return acc; - }, - {} - ) - ); - try { - await this.startRuntime(emulatableBackend); - } catch (e: any) { - this.logger.logLabeled( - "ERROR", - `Failed to start functions in ${emulatableBackend.functionsDir}: ${e}` + if (!emulatableBackend.bin?.startsWith("node")) { + this.logger.log("WARN", "--inspect-functions only supported for Node.js runtimes."); + } else { + // Since we're about to start a runtime to be shared by all the functions in this codebase, + // we need to make sure it has all the secrets used by any function in the codebase. + emulatableBackend.secretEnv = Object.values( + triggerDefinitions.reduce( + (acc: Record, curr: EmulatedTriggerDefinition) => { + for (const secret of curr.secretEnvironmentVariables || []) { + acc[secret.key] = secret; + } + return acc; + }, + {} + ) ); + try { + await this.startRuntime(emulatableBackend); + } catch (e: any) { + this.logger.logLabeled( + "ERROR", + `Failed to start functions in ${emulatableBackend.functionsDir}: ${e}` + ); + } } } } @@ -1266,10 +1299,44 @@ export class FunctionsEmulator implements EmulatorInstance { process: childProcess, events: new EventEmitter(), cwd: backend.functionsDir, - socketPath, + conn: new IPCConn(socketPath), }); } + async startPython( + backend: EmulatableBackend, + envs: Record + ): Promise { + const args = ["functions-framework"]; + + if (this.args.debugPort) { + this.logger.log("WARN", "--inspect-functions not supported for Python functions. Ignored."); + } + + // No support generic socket interface for Unix Domain Socket/Named Pipe in the python. + // Use TCP/IP stack instead. + const port = await portfinder.getPortPromise({ + port: 8081 + randomInt(0, 1000), // Add a small jitter to avoid race condition. + }); + const childProcess = runWithVirtualEnv(args, backend.functionsDir, { + ...process.env, + ...envs, + // Required to flush stdout/stderr immediately to the piped channels. + PYTHONUNBUFFERED: "1", + // Required to prevent flask development server to reload on code changes. + DEBUG: "False", + HOST: "127.0.0.1", + PORT: port.toString(), + }); + + return { + process: childProcess, + events: new EventEmitter(), + cwd: backend.functionsDir, + conn: new TCPConn("127.0.0.1", port), + }; + } + async startRuntime( backend: EmulatableBackend, trigger?: EmulatedTriggerDefinition @@ -1277,7 +1344,12 @@ export class FunctionsEmulator implements EmulatorInstance { const runtimeEnv = this.getRuntimeEnvs(backend, trigger); const secretEnvs = await this.resolveSecretEnvs(backend, trigger); - const runtime = await this.startNode(backend, { ...runtimeEnv, ...secretEnvs }); + let runtime; + if (backend.runtime!.startsWith("python")) { + runtime = await this.startPython(backend, { ...runtimeEnv, ...secretEnvs }); + } else { + runtime = await this.startNode(backend, { ...runtimeEnv, ...secretEnvs }); + } const extensionLogInfo = { instanceId: backend.extensionInstanceId, ref: backend.extensionVersion?.ref, diff --git a/src/emulator/functionsRuntimeWorker.ts b/src/emulator/functionsRuntimeWorker.ts index 3358ac68311..5365b7989d2 100644 --- a/src/emulator/functionsRuntimeWorker.ts +++ b/src/emulator/functionsRuntimeWorker.ts @@ -8,6 +8,7 @@ import { EventEmitter } from "events"; import { EmulatorLogger, ExtensionLogInfo } from "./emulatorLogger"; import { FirebaseError } from "../error"; import { Serializable } from "child_process"; +import { IncomingMessage } from "http"; type LogListener = (el: EmulatorLog) => any; @@ -118,12 +119,12 @@ export class RuntimeWorker { return new Promise((resolve) => { const proxy = http.request( { + ...this.runtime.conn.httpReqOpts(), method: req.method, path: req.path, headers: req.headers, - socketPath: this.runtime.socketPath, }, - (_resp) => { + (_resp: IncomingMessage) => { resp.writeHead(_resp.statusCode || 200, _resp.headers); const piped = _resp.pipe(resp); piped.on("finish", () => { @@ -178,20 +179,19 @@ export class RuntimeWorker { isSocketReady(): Promise { return new Promise((resolve, reject) => { - const req = http - .request( - { - method: "GET", - path: "/__/health", - socketPath: this.runtime.socketPath, - }, - () => { - // Set the worker state to IDLE for new work - this.readyForWork(); - resolve(); - } - ) - .end(); + const req = http.request( + { + ...this.runtime.conn.httpReqOpts(), + method: "GET", + path: "/__/health", + }, + () => { + // Set the worker state to IDLE for new work + this.readyForWork(); + resolve(); + } + ); + req.end(); req.on("error", (error) => { reject(error); }); diff --git a/src/functions/python.ts b/src/functions/python.ts new file mode 100644 index 00000000000..f37b821e1d8 --- /dev/null +++ b/src/functions/python.ts @@ -0,0 +1,32 @@ +import * as path from "path"; +import * as spawn from "cross-spawn"; +import * as cp from "child_process"; +import { logger } from "../logger"; + +const DEFAULT_VENV_DIR = "venv"; + +/** + * Spawn a process inside the Python virtual environment if found. + */ +export function runWithVirtualEnv( + commandAndArgs: string[], + cwd: string, + envs: Record, + venvDir = DEFAULT_VENV_DIR +): cp.ChildProcess { + const activateScriptPath = + process.platform === "win32" ? ["Scripts", "activate.bat"] : ["bin", "activate"]; + const venvActivate = path.join(cwd, venvDir, ...activateScriptPath); + const command = process.platform === "win32" ? venvActivate : "source"; + const args = [process.platform === "win32" ? "" : venvActivate, "&&", ...commandAndArgs]; + logger.debug(`Running command with virtualenv: command=${command}, args=${JSON.stringify(args)}`); + + return spawn(command, args, { + shell: true, + cwd, + stdio: [/* stdin= */ "pipe", /* stdout= */ "pipe", /* stderr= */ "pipe", "pipe"], + // Linting disabled since internal types expect NODE_ENV which does not apply to Python runtimes. + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any + env: envs as any, + }); +} diff --git a/src/test/deploy/functions/runtimes/discovery/index.spec.ts b/src/test/deploy/functions/runtimes/discovery/index.spec.ts index 266cce9997a..0a6f07b3f62 100644 --- a/src/test/deploy/functions/runtimes/discovery/index.spec.ts +++ b/src/test/deploy/functions/runtimes/discovery/index.spec.ts @@ -96,12 +96,12 @@ describe("detectFromPort", () => { }); it("passes as smoke test", async () => { - nock("http://localhost:8080").get("/__/functions.yaml").times(5).replyWithError({ + nock("http://127.0.0.1:8080").get("/__/functions.yaml").times(5).replyWithError({ message: "Still booting", code: "ECONNREFUSED", }); - nock("http://localhost:8080").get("/__/functions.yaml").reply(200, YAML_TEXT); + nock("http://127.0.0.1:8080").get("/__/functions.yaml").reply(200, YAML_TEXT); const parsed = await discovery.detectFromPort(8080, "project", "nodejs16"); expect(parsed).to.deep.equal(BUILD); diff --git a/src/test/deploy/functions/runtimes/python/index.spec.ts b/src/test/deploy/functions/runtimes/python/index.spec.ts new file mode 100644 index 00000000000..c16dae73830 --- /dev/null +++ b/src/test/deploy/functions/runtimes/python/index.spec.ts @@ -0,0 +1,45 @@ +import { expect } from "chai"; +import * as sinon from "sinon"; + +import * as python from "../../../../../deploy/functions/runtimes/python"; + +const PROJECT_ID = "test-project"; +const SOURCE_DIR = "/some/path/fns"; + +describe("PythonDelegate", () => { + describe("getPythonBinary", () => { + let platformMock: sinon.SinonStub; + + beforeEach(() => { + platformMock = sinon.stub(process, "platform"); + }); + + afterEach(() => { + platformMock.restore(); + }); + + it("returns specific version of the python binary corresponding to the runtime", () => { + platformMock.value("darwin"); + const requestedRuntime = "python310"; + const delegate = new python.Delegate(PROJECT_ID, SOURCE_DIR, requestedRuntime); + + expect(delegate.getPythonBinary()).to.equal("python3.10"); + }); + + it("returns generic python binary given non-recognized python runtime", () => { + platformMock.value("darwin"); + const requestedRuntime = "python312"; + const delegate = new python.Delegate(PROJECT_ID, SOURCE_DIR, requestedRuntime); + + expect(delegate.getPythonBinary()).to.equal("python"); + }); + + it("always returns version-neutral, python.exe on windows", () => { + platformMock.value("win32"); + const requestedRuntime = "python310"; + const delegate = new python.Delegate(PROJECT_ID, SOURCE_DIR, requestedRuntime); + + expect(delegate.getPythonBinary()).to.equal("python.exe"); + }); + }); +}); diff --git a/src/test/emulators/functionsRuntimeWorker.spec.ts b/src/test/emulators/functionsRuntimeWorker.spec.ts index d0cb0ca557d..00bc983930c 100644 --- a/src/test/emulators/functionsRuntimeWorker.spec.ts +++ b/src/test/emulators/functionsRuntimeWorker.spec.ts @@ -1,7 +1,7 @@ import * as httpMocks from "node-mocks-http"; import * as nock from "nock"; import { expect } from "chai"; -import { FunctionsRuntimeInstance } from "../../emulator/functionsEmulator"; +import { FunctionsRuntimeInstance, IPCConn } from "../../emulator/functionsEmulator"; import { EventEmitter } from "events"; import { RuntimeWorker, @@ -21,7 +21,7 @@ class MockRuntimeInstance implements FunctionsRuntimeInstance { events: EventEmitter = new EventEmitter(); exit: Promise; cwd = "/home/users/dir"; - socketPath = "/path/to/socket/foo.sock"; + conn = new IPCConn("/path/to/socket/foo.sock"); constructor() { this.exit = new Promise((resolve) => {