feat: isolate hardcoded language support (#2796)

Part of #2452
In preparation for language plugins, this PR defines a protocol for each
language support to conform to. Later we will define the external plugin
gRPC service and then move each language to being an external plugin one
by one.

### Breakdown of components:

#### Build engine
- Watches for:
    - added and removed modules
    - updates to `ftl.toml`
- Asks a module plugin to build when:
    - explicitly asked (eg: `ftl build`)
    - dependencies are updated
    - `ftl.toml` is updated
- Listens to each module plugin's Updates topic to react to automatic
builds

#### Module plugin
Currently these plugins arent actual plugins, but they will be soon!
- Can be used to scaffold a new module (`ftl new ...`)
- Gathers dependencies for a module
- Builds when asked by the build engine
- Watches for file changes within the module (language dependant.
excludes `ftl.toml`)
- When a change is detected, publishes automatic build events to it's
`Updates` topic

Not included in this PR (coming in a later one):
- allow `ftl new` to have language specific arguments (currently go and
jvm ones are hardcoded into `cmd_new.go`)
- currently no logic for collapsing multiple build requests into one or
detecting if we have already built since a file change event was fired.
This logic needs to change for external plugins anyway.

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

backend/controller/admin/local_client.go

@@ -47,9 +47,10 @@ func (s *diskSchemaRetriever) GetActiveSchema(ctx context.Context) (*schema.Sche
 
 	sch := &schema.Schema{}
 	for _, m := range modules {
-		schemaPath := m.Config.Abs().Schema
+		config := m.Abs()
+		schemaPath := config.Schema()
 		if r, ok := s.deployRoot.Get(); ok {
-			schemaPath = filepath.Join(r, m.Config.Module, m.Config.DeployDir, m.Config.Schema)
+			schemaPath = filepath.Join(r, m.Module, m.DeployDir, m.Schema())
 		}
 
 		module, err := schema.ModuleFromProtoFile(schemaPath)

backend/controller/admin/testdata/go/dischema/go.mod

@@ -5,37 +5,45 @@ go 1.23.0
 require github.com/TBD54566975/ftl v1.1.5
 
 require (
-	connectrpc.com/connect v1.16.1 // indirect
+	connectrpc.com/connect v1.16.2 // indirect
 	connectrpc.com/grpcreflect v1.2.0 // indirect
-	connectrpc.com/otelconnect v0.7.0 // indirect
+	connectrpc.com/otelconnect v0.7.1 // indirect
+	github.com/XSAM/otelsql v0.34.0 // indirect
+	github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
 	github.com/alecthomas/concurrency v0.0.2 // indirect
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect
 	github.com/alecthomas/types v0.16.0 // indirect
 	github.com/alessio/shellescape v1.4.2 // indirect
+	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/danieljoos/wincred v1.2.0 // indirect
+	github.com/deckarep/golang-set/v2 v2.6.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/hashicorp/cronexpr v1.1.2 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
-	github.com/jackc/pgx/v5 v5.6.0 // indirect
-	github.com/jackc/puddle/v2 v2.2.1 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/jackc/pgx/v5 v5.7.1 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/multiformats/go-base36 v0.2.0 // indirect
-	github.com/swaggest/jsonschema-go v0.3.70 // indirect
+	github.com/puzpuzpuz/xsync/v3 v3.4.0 // indirect
+	github.com/swaggest/jsonschema-go v0.3.72 // indirect
 	github.com/swaggest/refl v1.3.0 // indirect
-	github.com/zalando/go-keyring v0.2.4 // indirect
-	go.opentelemetry.io/otel v1.27.0 // indirect
-	go.opentelemetry.io/otel/metric v1.27.0 // indirect
-	go.opentelemetry.io/otel/trace v1.27.0 // indirect
-	golang.org/x/crypto v0.23.0 // indirect
-	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
-	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/net v0.25.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.20.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
-	google.golang.org/protobuf v1.34.1 // indirect
+	github.com/zalando/go-keyring v0.2.5 // indirect
+	go.opentelemetry.io/otel v1.30.0 // indirect
+	go.opentelemetry.io/otel/metric v1.30.0 // indirect
+	go.opentelemetry.io/otel/trace v1.30.0 // indirect
+	golang.org/x/crypto v0.27.0 // indirect
+	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
+	golang.org/x/mod v0.21.0 // indirect
+	golang.org/x/net v0.29.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 )
+
+replace github.com/TBD54566975/ftl => ./../../../../../..

frontend/cli/cmd_box.go

@@ -153,7 +153,7 @@ func (b *boxCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceC
 			return err
 		}
 		files = append(files, filepath.Join(config.Dir, "ftl.toml"))
-		files = append(files, config.Schema)
+		files = append(files, config.Schema())
 		for _, file := range files {
 			relFile, err := filepath.Rel(config.Dir, file)
 			if err != nil {

frontend/cli/cmd_box_run.go

@@ -85,7 +85,7 @@ func (b *boxRunCmd) Run(ctx context.Context, projConfig projectconfig.Config) er
 	// Manually import the schema for each module to get the dependency graph.
 	err = engine.Each(func(m buildengine.Module) error {
 		logger.Debugf("Loading schema for module %q", m.Config.Module)
-		mod, err := schema.ModuleFromProtoFile(m.Config.Abs().Schema)
+		mod, err := schema.ModuleFromProtoFile(m.Config.Abs().Schema())
 		if err != nil {
 			return fmt.Errorf("failed to read schema for module %q: %w", m.Config.Module, err)
 		}

frontend/cli/cmd_init.go

@@ -1,14 +1,18 @@
 package main
 
 import (
+	"archive/zip"
 	"bufio"
 	"context"
 	"fmt"
 	"os"
 	"path"
 	"strings"
 
+	"github.com/TBD54566975/scaffolder"
+
 	"github.com/TBD54566975/ftl"
+	"github.com/TBD54566975/ftl/internal"
 	"github.com/TBD54566975/ftl/internal/configuration"
 	"github.com/TBD54566975/ftl/internal/configuration/providers"
 	"github.com/TBD54566975/ftl/internal/exec"
@@ -110,3 +114,17 @@ func updateGitIgnore(ctx context.Context, gitRoot string) error {
 	// Add .gitignore to git
 	return maybeGitAdd(ctx, gitRoot, ".gitignore")
 }
+
+func scaffold(ctx context.Context, includeBinDir bool, source *zip.Reader, destination string, sctx any, options ...scaffolder.Option) error {
+	logger := log.FromContext(ctx)
+	opts := []scaffolder.Option{scaffolder.Exclude("^go.mod$")}
+	if !includeBinDir {
+		logger.Debugf("Excluding bin directory")
+		opts = append(opts, scaffolder.Exclude("^bin"))
+	}
+	opts = append(opts, options...)
+	if err := internal.ScaffoldZip(source, destination, sctx, opts...); err != nil {
+		return fmt.Errorf("failed to scaffold: %w", err)
+	}
+	return nil
+}

frontend/cli/cmd_new.go

@@ -1,75 +1,58 @@
 package main
 
 import (
-	"archive/zip"
 	"context"
 	"fmt"
 	"go/token"
-	"html/template"
 	"os"
 	"path/filepath"
 	"regexp"
-	"runtime"
-	"strings"
-
-	"github.com/TBD54566975/scaffolder"
 
 	"github.com/TBD54566975/ftl/backend/schema"
-	"github.com/TBD54566975/ftl/backend/schema/strcase"
-	goruntime "github.com/TBD54566975/ftl/go-runtime"
 	"github.com/TBD54566975/ftl/internal"
-	"github.com/TBD54566975/ftl/internal/exec"
+	"github.com/TBD54566975/ftl/internal/buildengine"
 	"github.com/TBD54566975/ftl/internal/log"
+	"github.com/TBD54566975/ftl/internal/moduleconfig"
 	"github.com/TBD54566975/ftl/internal/projectconfig"
-	"github.com/TBD54566975/ftl/jvm-runtime/java"
-	"github.com/TBD54566975/ftl/jvm-runtime/kotlin"
 )
 
 type newCmd struct {
-	Go     newGoCmd     `cmd:"" help:"Initialize a new FTL Go module."`
-	Java   newJavaCmd   `cmd:"" help:"Initialize a new FTL Java module."`
-	Kotlin newKotlinCmd `cmd:"" help:"Initialize a new FTL Kotlin module."`
-}
+	Language string `arg:"" help:"Language of the module to create."`
+	Dir      string `arg:"" help:"Directory to initialize the module in."`
+	Name     string `arg:"" help:"Name of the FTL module to create underneath the base directory."`
 
-type newGoCmd struct {
-	Replace   map[string]string `short:"r" help:"Replace a module import path with a local path in the initialised FTL module." placeholder:"OLD=NEW,..." env:"FTL_INIT_GO_REPLACE"`
-	Dir       string            `arg:"" help:"Directory to initialize the module in."`
-	Name      string            `arg:"" help:"Name of the FTL module to create underneath the base directory."`
-	GoVersion string
-}
+	// Go specific flags
+	Replace map[string]string `short:"r" help:"For Go, replace a module import path with a local path in the initialised FTL module." placeholder:"OLD=NEW,..." env:"FTL_INIT_GO_REPLACE"`
 
-type newJavaCmd struct {
-	Dir   string `arg:"" help:"Directory to initialize the module in."`
-	Name  string `arg:"" help:"Name of the FTL module to create underneath the base directory."`
-	Group string `help:"The Maven groupId of the project." default:"com.example"`
-}
-type newKotlinCmd struct {
-	Dir   string `arg:"" help:"Directory to initialize the module in."`
-	Name  string `arg:"" help:"Name of the FTL module to create underneath the base directory."`
-	Group string `help:"The Maven groupId of the project." default:"com.example"`
+	// Java/Kotlin specific flags
+	Group string `help:"For Java and Kotlin, the Maven groupId of the project." default:"com.example"`
 }
 
-func (i newGoCmd) Run(ctx context.Context, config projectconfig.Config) error {
+func (i newCmd) Run(ctx context.Context, config projectconfig.Config) error {
 	name, path, err := validateModule(i.Dir, i.Name)
 	if err != nil {
 		return err
 	}
 
 	// Validate the module name with custom validation
-	if !isValidGoModuleName(name) {
+	if !isValidModuleName(name) {
 		return fmt.Errorf("module name %q must be a valid Go module name and not a reserved keyword", name)
 	}
 
 	logger := log.FromContext(ctx)
-	logger.Debugf("Creating FTL Go module %q in %s", name, path)
+	logger.Debugf("Creating FTL %s module %q in %s", i.Language, name, path)
 
-	i.GoVersion = runtime.Version()[2:]
-	if err := scaffold(ctx, config.Hermit, goruntime.Files(), i.Dir, i, scaffolder.Exclude("^go.mod$")); err != nil {
+	moduleConfig := moduleconfig.ModuleConfig{
+		Module:   name,
+		Language: i.Language,
+		Dir:      path,
+	}
+	plugin, err := buildengine.PluginFromConfig(ctx, moduleConfig, config.Root())
+	if err != nil {
 		return err
 	}
-
-	logger.Debugf("Running go mod tidy")
-	if err := exec.Command(ctx, log.Debug, path, "go", "mod", "tidy").RunBuffered(ctx); err != nil {
+	err = plugin.CreateModule(ctx, moduleConfig, config.Hermit, i.Replace, i.Group)
+	if err != nil {
 		return err
 	}
 
@@ -88,56 +71,6 @@ func (i newGoCmd) Run(ctx context.Context, config projectconfig.Config) error {
 	return nil
 }
 
-func (i newJavaCmd) Run(ctx context.Context, config projectconfig.Config) error {
-	return RunJvmScaffolding(ctx, config, i.Dir, i.Name, i.Group, java.Files())
-}
-
-func (i newKotlinCmd) Run(ctx context.Context, config projectconfig.Config) error {
-	return RunJvmScaffolding(ctx, config, i.Dir, i.Name, i.Group, kotlin.Files())
-}
-
-func RunJvmScaffolding(ctx context.Context, config projectconfig.Config, dir string, name string, group string, source *zip.Reader) error {
-	name, path, err := validateModule(dir, name)
-	if err != nil {
-		return err
-	}
-
-	logger := log.FromContext(ctx)
-	logger.Debugf("Creating FTL module %q in %s", name, path)
-
-	packageDir := strings.ReplaceAll(group, ".", "/")
-
-	javaContext := struct {
-		Dir        string
-		Name       string
-		Group      string
-		PackageDir string
-	}{
-		Dir:        dir,
-		Name:       name,
-		Group:      group,
-		PackageDir: packageDir,
-	}
-
-	if err := scaffold(ctx, config.Hermit, source, dir, javaContext); err != nil {
-		return err
-	}
-
-	_, ok := internal.GitRoot(dir).Get()
-	if !config.NoGit && ok {
-		logger.Debugf("Adding files to git")
-		if config.Hermit {
-			if err := maybeGitAdd(ctx, dir, "bin/*"); err != nil {
-				return err
-			}
-		}
-		if err := maybeGitAdd(ctx, dir, filepath.Join(name, "*")); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func validateModule(dir string, name string) (string, string, error) {
 	if dir == "" {
 		return "", "", fmt.Errorf("directory is required")
@@ -149,13 +82,17 @@ func validateModule(dir string, name string) (string, string, error) {
 		return "", "", fmt.Errorf("module name %q is invalid", name)
 	}
 	path := filepath.Join(dir, name)
-	if _, err := os.Stat(path); err == nil {
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		return "", "", fmt.Errorf("could not make %q an absolute path: %w", path, err)
+	}
+	if _, err := os.Stat(absPath); err == nil {
 		return "", "", fmt.Errorf("module directory %s already exists", path)
 	}
-	return name, path, nil
+	return name, absPath, nil
 }
 
-func isValidGoModuleName(name string) bool {
+func isValidModuleName(name string) bool {
 	validNamePattern := regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9_]*$`)
 	if !validNamePattern.MatchString(name) {
 		return false
@@ -165,31 +102,3 @@ func isValidGoModuleName(name string) bool {
 	}
 	return true
 }
-
-func scaffold(ctx context.Context, includeBinDir bool, source *zip.Reader, destination string, sctx any, options ...scaffolder.Option) error {
-	logger := log.FromContext(ctx)
-	opts := []scaffolder.Option{scaffolder.Functions(scaffoldFuncs), scaffolder.Exclude("^go.mod$")}
-	if !includeBinDir {
-		logger.Debugf("Excluding bin directory")
-		opts = append(opts, scaffolder.Exclude("^bin"))
-	}
-	opts = append(opts, options...)
-	if err := internal.ScaffoldZip(source, destination, sctx, opts...); err != nil {
-		return fmt.Errorf("failed to scaffold: %w", err)
-	}
-	return nil
-}
-
-var scaffoldFuncs = template.FuncMap{
-	"snake":          strcase.ToLowerSnake,
-	"screamingSnake": strcase.ToUpperSnake,
-	"camel":          strcase.ToUpperCamel,
-	"lowerCamel":     strcase.ToLowerCamel,
-	"strippedCamel":  strcase.ToUpperStrippedCamel,
-	"kebab":          strcase.ToLowerKebab,
-	"screamingKebab": strcase.ToUpperKebab,
-	"upper":          strings.ToUpper,
-	"lower":          strings.ToLower,
-	"title":          strings.Title,
-	"typename":       schema.TypeName,
-}

frontend/cli/cmd_schema_diff.go

@@ -93,9 +93,9 @@ func localSchema(ctx context.Context, projectConfig projectconfig.Config) (*sche
 		return nil, fmt.Errorf("failed to discover modules %w", err)
 	}
 	for _, moduleSettings := range modules {
-		mod, err := schema.ModuleFromProtoFile(moduleSettings.Config.Abs().Schema)
+		mod, err := schema.ModuleFromProtoFile(moduleSettings.Abs().Schema())
 		if err != nil {
-			tried += fmt.Sprintf(" failed to read schema file %s; did you run ftl build?", moduleSettings.Config.Abs().Schema)
+			tried += fmt.Sprintf(" failed to read schema file %s; did you run ftl build?", moduleSettings.Abs().Schema())
 		} else {
 			found = true
 			pb.Modules = append(pb.Modules, mod)

frontend/cli/cmd_schema_generate.go

@@ -144,7 +144,6 @@ func (s *schemaGenerateCmd) regenerateModules(logger *log.Logger, modules []*sch
 
 	for _, module := range modules {
 		if err := scaffolder.Scaffold(s.Template, s.Dest, module,
-			scaffolder.Functions(scaffoldFuncs),
 			scaffolder.Extend(javascript.Extension("template.js", javascript.WithLogger(makeJSLoggerAdapter(logger)))),
 		); err != nil {
 			return err

go-runtime/compile/build.go

@@ -1154,7 +1154,11 @@ func writeSchema(config moduleconfig.ModuleConfig, module *schema.Module) error
 	if err != nil {
 		return fmt.Errorf("failed to marshal schema: %w", err)
 	}
-	return os.WriteFile(config.Abs().Schema, schemaBytes, 0600)
+	err = os.WriteFile(config.Abs().Schema(), schemaBytes, 0600)
+	if err != nil {
+		return fmt.Errorf("could not write schema: %w", err)
+	}
+	return nil
 }
 
 func writeSchemaErrors(config moduleconfig.ModuleConfig, errors []*schema.Error) error {