feat: add external plugin to build engine (#2994)

Adds `languageplugin.externalPlugin` for working with plugins over gRPC.
This PR does not include any non-test implementations.

Part of #2452

backend/controller/admin/local_client.go

@@ -62,7 +62,7 @@ func (s *diskSchemaRetriever) GetActiveSchema(ctx context.Context, bAllocator op
 			if err != nil {
 				moduleSchemas <- either.RightOf[*schema.Module](fmt.Errorf("could not load plugin for %s: %w", m.Module, err))
 			}
-			defer plugin.Kill(ctx) // nolint:errcheck
+			defer plugin.Kill() // nolint:errcheck
 
 			customDefaults, err := plugin.ModuleConfigDefaults(ctx, m.Dir)
 			if err != nil {

backend/protos/xyz/block/ftl/v1/language/mixins.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/TBD54566975/ftl/internal/builderrors"
+	"github.com/TBD54566975/ftl/internal/log"
 	"github.com/TBD54566975/ftl/internal/slices"
 )
 
@@ -72,3 +73,16 @@ func PosFromProto(pos *Position) builderrors.Position {
 		Filename:    pos.Filename,
 	}
 }
+
+func LogLevelFromProto(level LogMessage_LogLevel) log.Level {
+	switch level {
+	case LogMessage_INFO:
+		return log.Info
+	case LogMessage_WARN:
+		return log.Warn
+	case LogMessage_ERROR:
+		return log.Error
+	default:
+		panic(fmt.Sprintf("unhandled Log_Level %v", level))
+	}
+}

frontend/cli/cmd_new.go

@@ -35,6 +35,8 @@ type newCmd struct {
 // - help text (ftl new go --help)
 // - default values
 // - environment variable overrides
+//
+// Language plugins take time to launch, so we return the one we created so it can be reused in Run().
 func prepareNewCmd(ctx context.Context, k *kong.Kong, args []string) (optionalPlugin optional.Option[languageplugin.LanguagePlugin], err error) {
 	if len(args) < 2 {
 		return optionalPlugin, nil

frontend/cli/cmd_schema_diff.go

@@ -116,7 +116,7 @@ func localSchema(ctx context.Context, projectConfig projectconfig.Config, bindAl
 			if err != nil {
 				moduleSchemas <- either.RightOf[*schema.Module](err)
 			}
-			defer plugin.Kill(ctx) // nolint:errcheck
+			defer plugin.Kill() // nolint:errcheck
 
 			customDefaults, err := plugin.ModuleConfigDefaults(ctx, m.Dir)
 			if err != nil {

frontend/cli/main.go

@@ -85,14 +85,16 @@ func main() {
 	csm := &currentStatusManager{}
 
 	app := createKongApplication(&cli, csm)
-	languagePlugin, err := prepareNewCmd(ctx, app, os.Args[1:])
+
+	// Dynamically update the kong app with language specific flags for the "ftl new" command.
+	languagePlugin, err := prepareNewCmd(log.ContextWithNewDefaultLogger(ctx), app, os.Args[1:])
 	app.FatalIfErrorf(err)
 
 	kctx, err := app.Parse(os.Args[1:])
 	app.FatalIfErrorf(err)
 
 	if plugin, ok := languagePlugin.Get(); ok {
-		// for "ftl new" command, we only need to create the language plugin once
+		// Plugins take time to launch, so we bind the "ftl new" plugin to the kong context.
 		kctx.BindTo(plugin, (*languageplugin.LanguagePlugin)(nil))
 	}
 

internal/buildengine/build.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"time"
 
-	"github.com/alecthomas/types/either"
+	"github.com/alecthomas/types/result"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/TBD54566975/ftl/internal/buildengine/languageplugin"
@@ -17,33 +17,33 @@ import (
 	"github.com/TBD54566975/ftl/internal/schema"
 )
 
+var errInvalidateDependencies = errors.New("dependencies need to be updated")
+
 // Build a module in the given directory given the schema and module config.
 //
 // A lock file is used to ensure that only one build is running at a time.
-func build(ctx context.Context, plugin languageplugin.LanguagePlugin, projectRootDir string, sch *schema.Schema, config moduleconfig.ModuleConfig, buildEnv []string, devMode bool) (moduleSchema *schema.Module, deploy []string, err error) {
-	logger := log.FromContext(ctx).Module(config.Module).Scope("build")
+//
+// Returns invalidateDependenciesError if the build failed due to a change in dependencies.
+func build(ctx context.Context, plugin languageplugin.LanguagePlugin, projectRootDir string, bctx languageplugin.BuildContext, buildEnv []string, devMode bool) (moduleSchema *schema.Module, deploy []string, err error) {
+	logger := log.FromContext(ctx).Module(bctx.Config.Module).Scope("build")
 	ctx = log.ContextWithLogger(ctx, logger)
 
 	logger.Infof("Building module")
-
-	result, err := plugin.Build(ctx, projectRootDir, config, sch, buildEnv, devMode)
-	if err != nil {
-		return handleBuildResult(ctx, config, either.RightOf[languageplugin.BuildResult](err))
-	}
-	return handleBuildResult(ctx, config, either.LeftOf[error](result))
+	return handleBuildResult(ctx, bctx.Config, result.From(plugin.Build(ctx, projectRootDir, bctx, buildEnv, devMode)))
 }
 
 // handleBuildResult processes the result of a build
-func handleBuildResult(ctx context.Context, c moduleconfig.ModuleConfig, eitherResult either.Either[languageplugin.BuildResult, error]) (moduleSchema *schema.Module, deploy []string, err error) {
+func handleBuildResult(ctx context.Context, c moduleconfig.ModuleConfig, eitherResult result.Result[languageplugin.BuildResult]) (moduleSchema *schema.Module, deploy []string, err error) {
 	logger := log.FromContext(ctx)
 	config := c.Abs()
 
-	var result languageplugin.BuildResult
-	switch eitherResult := eitherResult.(type) {
-	case either.Right[languageplugin.BuildResult, error]:
-		return nil, nil, fmt.Errorf("failed to build module: %w", eitherResult.Get())
-	case either.Left[languageplugin.BuildResult, error]:
-		result = eitherResult.Get()
+	result, err := eitherResult.Result()
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to build module: %w", err)
+	}
+
+	if result.InvalidateDependencies {
+		return nil, nil, errInvalidateDependencies
 	}
 
 	var errs []error

internal/buildengine/engine.go

@@ -487,7 +487,7 @@ func (e *Engine) watchForModuleChanges(ctx context.Context, period time.Duration
 				}
 				if meta, ok := e.moduleMetas.Load(event.Config.Module); ok {
 					meta.plugin.Updates().Unsubscribe(meta.events)
-					err := meta.plugin.Kill(ctx)
+					err := meta.plugin.Kill()
 					if err != nil {
 						didError = true
 						e.reportBuildFailed(err)
@@ -633,7 +633,6 @@ func (e *Engine) BuildAndDeploy(ctx context.Context, replicas int32, waitForDepl
 type buildCallback func(ctx context.Context, module Module) error
 
 func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, moduleNames ...string) error {
-
 	if len(moduleNames) == 0 {
 		e.moduleMetas.Range(func(name string, meta moduleMeta) bool {
 			moduleNames = append(moduleNames, name)
@@ -813,9 +812,14 @@ func (e *Engine) build(ctx context.Context, moduleName string, builtModules map[
 		e.listener.OnBuildStarted(meta.module)
 	}
 
-	moduleSchema, deploy, err := build(ctx, meta.plugin, e.projectRoot, sch, meta.module.Config, e.buildEnv, e.devMode)
+	moduleSchema, deploy, err := build(ctx, meta.plugin, e.projectRoot, languageplugin.BuildContext{
+		Config:       meta.module.Config,
+		Schema:       sch,
+		Dependencies: meta.module.Dependencies,
+	}, e.buildEnv, e.devMode)
 	if err != nil {
 		terminal.UpdateModuleState(ctx, moduleName, terminal.BuildStateFailed)
+		// TODO: handle errInvalidateDependencies
 		return err
 	}
 	// update files to deploy