From 160703210b0ea9fc97af3056d38d5b75b048cf09 Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Sun, 1 Dec 2024 09:13:22 +1100 Subject: [PATCH 01/14] emplemented stdout template extensions --- cli/cli.go | 25 ++- cli/flags.go | 4 + core/plugin_registry.go | 12 +- plugins/template/extension_executor.go | 196 +++++++++++++++++++++ plugins/template/extension_manager.go | 84 +++++++++ plugins/template/extension_registry.go | 229 +++++++++++++++++++++++++ plugins/template/hash.go | 33 ++++ plugins/template/hash_test.go | 119 +++++++++++++ plugins/template/template.go | 44 ++++- plugins/template/utils.go | 41 +++++ 10 files changed, 782 insertions(+), 5 deletions(-) create mode 100644 plugins/template/extension_executor.go create mode 100644 plugins/template/extension_manager.go create mode 100644 plugins/template/extension_registry.go create mode 100644 plugins/template/hash.go create mode 100644 plugins/template/hash_test.go create mode 100644 plugins/template/utils.go diff --git a/cli/cli.go b/cli/cli.go index 4fdcae67a..95313065a 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -2,12 +2,13 @@ package cli import ( "fmt" - "github.com/danielmiessler/fabric/plugins/tools/youtube" "os" "path/filepath" "strconv" "strings" + "github.com/danielmiessler/fabric/plugins/tools/youtube" + "github.com/danielmiessler/fabric/common" "github.com/danielmiessler/fabric/core" "github.com/danielmiessler/fabric/plugins/ai" @@ -42,7 +43,10 @@ func Cli(version string) (err error) { } } - registry := core.NewPluginRegistry(fabricDb) + var registry *core.PluginRegistry + if registry, err = core.NewPluginRegistry(fabricDb); err != nil { + return + } // if the setup flag is set, run the setup function if currentFlags.Setup { @@ -129,6 +133,23 @@ func Cli(version string) (err error) { } } +if currentFlags.ListExtensions { + err = registry.TemplateExtensions.ListExtensions() + return +} + +if currentFlags.AddExtension != "" { + err = registry.TemplateExtensions.RegisterExtension(currentFlags.AddExtension) + return +} + +if currentFlags.RemoveExtension != "" { + err = registry.TemplateExtensions.RemoveExtension(currentFlags.RemoveExtension) + return +} + + + // if the interactive flag is set, run the interactive function // if currentFlags.Interactive { // interactive.Interactive() diff --git a/cli/flags.go b/cli/flags.go index 701ee730c..7bafc7dc4 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -59,6 +59,10 @@ type Flags struct { Serve bool `long:"serve" description:"Serve the Fabric Rest API"` ServeAddress string `long:"address" description:"The address to bind the REST API" default:":8080"` Version bool `long:"version" description:"Print current version"` + ListExtensions bool `long:"listextensions" description:"List all registered extensions"` + AddExtension string `long:"addextension" description:"Register a new extension from config file path"` + RemoveExtension string `long:"rmextension" description:"Remove a registered extension by name"` + } // Init Initialize flags. returns a Flags struct and an error diff --git a/core/plugin_registry.go b/core/plugin_registry.go index dc62a168f..3dc76bbdb 100644 --- a/core/plugin_registry.go +++ b/core/plugin_registry.go @@ -3,6 +3,8 @@ package core import ( "bytes" "fmt" + "os" + "path/filepath" "strconv" "github.com/samber/lo" @@ -21,13 +23,14 @@ import ( "github.com/danielmiessler/fabric/plugins/ai/openrouter" "github.com/danielmiessler/fabric/plugins/ai/siliconcloud" "github.com/danielmiessler/fabric/plugins/db/fsdb" + "github.com/danielmiessler/fabric/plugins/template" "github.com/danielmiessler/fabric/plugins/tools" "github.com/danielmiessler/fabric/plugins/tools/jina" "github.com/danielmiessler/fabric/plugins/tools/lang" "github.com/danielmiessler/fabric/plugins/tools/youtube" ) -func NewPluginRegistry(db *fsdb.Db) (ret *PluginRegistry) { +func NewPluginRegistry(db *fsdb.Db) (ret *PluginRegistry, err error) { ret = &PluginRegistry{ Db: db, VendorManager: ai.NewVendorsManager(), @@ -37,6 +40,12 @@ func NewPluginRegistry(db *fsdb.Db) (ret *PluginRegistry) { Language: lang.NewLanguage(), Jina: jina.NewClient(), } + + var homedir string + if homedir, err = os.UserHomeDir(); err != nil { + return + } + ret.TemplateExtensions = template.NewExtensionManager(filepath.Join(homedir, ".config/fabric")) ret.Defaults = tools.NeeDefaults(ret.GetModels) @@ -60,6 +69,7 @@ type PluginRegistry struct { YouTube *youtube.YouTube Language *lang.Language Jina *jina.Client + TemplateExtensions *template.ExtensionManager } func (o *PluginRegistry) SaveEnvFile() (err error) { diff --git a/plugins/template/extension_executor.go b/plugins/template/extension_executor.go new file mode 100644 index 000000000..9edc07891 --- /dev/null +++ b/plugins/template/extension_executor.go @@ -0,0 +1,196 @@ +package template + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// ExtensionExecutor handles the secure execution of extensions +// It uses the registry to verify extensions before running them +type ExtensionExecutor struct { + registry *ExtensionRegistry +} + +// NewExtensionExecutor creates a new executor instance +// It requires a registry to verify extensions +func NewExtensionExecutor(registry *ExtensionRegistry) *ExtensionExecutor { + return &ExtensionExecutor{ + registry: registry, + } +} + +// Execute runs an extension with the given operation and value string +// name: the registered name of the extension +// operation: the operation to perform +// value: the input value(s) for the operation +// In extension_executor.go +func (e *ExtensionExecutor) Execute(name, operation, value string) (string, error) { + // Get and verify extension from registry + ext, err := e.registry.GetExtension(name) + if err != nil { + return "", fmt.Errorf("failed to get extension: %w", err) + } + + // Format the command using our template system + cmdStr, err := e.formatCommand(ext, operation, value) + if err != nil { + return "", fmt.Errorf("failed to format command: %w", err) + } + + // Split the command string into command and arguments + cmdParts := strings.Fields(cmdStr) + if len(cmdParts) < 1 { + return "", fmt.Errorf("empty command after formatting") + } + + // Create command with the Executable and formatted arguments + cmd := exec.Command("sh", "-c", cmdStr) + //cmd := exec.Command(cmdParts[0], cmdParts[1:]...) + + // Set up environment if specified + if len(ext.Env) > 0 { + cmd.Env = append(os.Environ(), ext.Env...) + } + + // Execute based on output method + outputMethod := ext.GetOutputMethod() + if outputMethod == "file" { + return e.executeWithFile(cmd, ext) + } + return e.executeStdout(cmd, ext) +} + +// formatCommand uses fabric's template system to format the command +// It creates a variables map for the template system using the input values +func (e *ExtensionExecutor) formatCommand(ext *ExtensionDefinition, operation string, value string) (string, error) { + // Get operation config + opConfig, exists := ext.Operations[operation] + if !exists { + return "", fmt.Errorf("operation %s not found for extension %s", operation, ext.Name) + } + + vars := make(map[string]string) + vars["executable"] = ext.Executable + vars["operation"] = operation + vars["value"] = value + + // Split on pipe for numbered variables + values := strings.Split(value, "|") + for i, val := range values { + vars[fmt.Sprintf("%d", i+1)] = val + } + + return ApplyTemplate(opConfig.CmdTemplate, vars, "") +} + +// executeStdout runs the command and captures its stdout +func (e *ExtensionExecutor) executeStdout(cmd *exec.Cmd, ext *ExtensionDefinition) (string, error) { + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + //debug output + fmt.Printf("Executing command: %s\n", cmd.String()) + + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("execution failed: %w\nstderr: %s", err, stderr.String()) + } + + return stdout.String(), nil +} + +// executeWithFile runs the command and handles file-based output +func (e *ExtensionExecutor) executeWithFile(cmd *exec.Cmd, ext *ExtensionDefinition) (string, error) { + // Parse timeout - this is now a first-class field + timeout, err := time.ParseDuration(ext.Timeout) + if err != nil { + return "", fmt.Errorf("invalid timeout format: %w", err) + } + + // Create context with timeout + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + cmd = exec.CommandContext(ctx, cmd.Path, cmd.Args[1:]...) + cmd.Env = cmd.Env + + fileConfig := ext.GetFileConfig() + if fileConfig == nil { + return "", fmt.Errorf("no file configuration found") + } + + // Handle path from stdout case + if pathFromStdout, ok := fileConfig["path_from_stdout"].(bool); ok && pathFromStdout { + return e.handlePathFromStdout(cmd, ext) + } + + // Handle fixed file case + workDir, _ := fileConfig["work_dir"].(string) + outputFile, _ := fileConfig["output_file"].(string) + + if outputFile == "" { + return "", fmt.Errorf("no output file specified in configuration") + } + + // Set working directory if specified + if workDir != "" { + cmd.Dir = workDir + } + + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + if ctx.Err() == context.DeadlineExceeded { + return "", fmt.Errorf("execution timed out after %v", timeout) + } + return "", fmt.Errorf("execution failed: %w\nerr: %s", err, stderr.String()) + } + + // Construct full file path + outputPath := outputFile + if workDir != "" { + outputPath = filepath.Join(workDir, outputFile) + } + + content, err := os.ReadFile(outputPath) + if err != nil { + return "", fmt.Errorf("failed to read output file: %w", err) + } + + // Handle cleanup if enabled + if ext.IsCleanupEnabled() { + defer os.Remove(outputPath) + } + + return string(content), nil +} + +// Helper method to handle path from stdout case +func (e *ExtensionExecutor) handlePathFromStdout(cmd *exec.Cmd, ext *ExtensionDefinition) (string, error) { + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("failed to get output path: %w\nerr: %s", err, stderr.String()) + } + + outputPath := strings.TrimSpace(stdout.String()) + content, err := os.ReadFile(outputPath) + if err != nil { + return "", fmt.Errorf("failed to read output file: %w", err) + } + + if ext.IsCleanupEnabled() { + defer os.Remove(outputPath) + } + + return string(content), nil +} \ No newline at end of file diff --git a/plugins/template/extension_manager.go b/plugins/template/extension_manager.go new file mode 100644 index 000000000..7d0ed1158 --- /dev/null +++ b/plugins/template/extension_manager.go @@ -0,0 +1,84 @@ +package template + +import ( + "fmt" + "path/filepath" +) + +// ExtensionManager handles the high-level operations of the extension system +type ExtensionManager struct { + registry *ExtensionRegistry + executor *ExtensionExecutor + configDir string +} + +// NewExtensionManager creates a new extension manager instance +func NewExtensionManager(configDir string) *ExtensionManager { + registry := NewExtensionRegistry(configDir) + return &ExtensionManager{ + registry: registry, + executor: NewExtensionExecutor(registry), + configDir: configDir, + } +} + +// ListExtensions handles the listextensions flag action +func (em *ExtensionManager) ListExtensions() error { + extensions, err := em.registry.ListExtensions() + if err != nil { + return fmt.Errorf("failed to list extensions: %w", err) + } + + for _, ext := range extensions { + fmt.Printf("Name: %s\n", ext.Name) + fmt.Printf(" Executable: %s\n", ext.Executable) + fmt.Printf(" Type: %s\n", ext.Type) + fmt.Printf(" Timeout: %s\n", ext.Timeout) + fmt.Printf(" Description: %s\n", ext.Description) + fmt.Printf(" Version: %s\n", ext.Version) + + fmt.Printf(" Operations:\n") + for opName, opConfig := range ext.Operations { + fmt.Printf(" %s:\n", opName) + fmt.Printf(" Command Template: %s\n", opConfig.CmdTemplate) + } + + if fileConfig := ext.GetFileConfig(); fileConfig != nil { + fmt.Printf(" File Configuration:\n") + for k, v := range fileConfig { + fmt.Printf(" %s: %v\n", k, v) + } + } + fmt.Printf("\n") + } + + return nil +} + +// RegisterExtension handles the addextension flag action +func (em *ExtensionManager) RegisterExtension(configPath string) error { + absPath, err := filepath.Abs(configPath) + if err != nil { + return fmt.Errorf("invalid config path: %w", err) + } + + if err := em.registry.Register(absPath); err != nil { + return fmt.Errorf("failed to register extension: %w", err) + } + + return nil +} + +// RemoveExtension handles the rmextension flag action +func (em *ExtensionManager) RemoveExtension(name string) error { + if err := em.registry.Remove(name); err != nil { + return fmt.Errorf("failed to remove extension: %w", err) + } + + return nil +} + +// ProcessExtension handles template processing for extension directives +func (em *ExtensionManager) ProcessExtension(name, operation, value string) (string, error) { + return em.executor.Execute(name, operation, value) +} \ No newline at end of file diff --git a/plugins/template/extension_registry.go b/plugins/template/extension_registry.go new file mode 100644 index 000000000..6da115ae5 --- /dev/null +++ b/plugins/template/extension_registry.go @@ -0,0 +1,229 @@ +package template + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + + "gopkg.in/yaml.v3" + // Add this import +) + +// ExtensionDefinition represents a single extension configuration +type ExtensionDefinition struct { + // Global properties + Name string `yaml:"name"` + Executable string `yaml:"executable"` + Type string `yaml:"type"` + Timeout string `yaml:"timeout"` + Description string `yaml:"description"` + Version string `yaml:"version"` + Env []string `yaml:"env"` + + // Operation-specific commands + Operations map[string]OperationConfig `yaml:"operations"` + + // Additional config + Config map[string]interface{} `yaml:"config"` +} + +type OperationConfig struct { + CmdTemplate string `yaml:"cmd_template"` +} + +type ExtensionRegistry struct { + configDir string + registry struct { + Extensions map[string]*ExtensionDefinition + ConfigHashes map[string]string + ExecutableHashes map[string]string + } +} + + +// Helper methods for Config access +func (e *ExtensionDefinition) GetOutputMethod() string { + if output, ok := e.Config["output"].(map[string]interface{}); ok { + if method, ok := output["method"].(string); ok { + return method + } + } + return "stdout" // default to stdout if not specified +} + +func (e *ExtensionDefinition) GetFileConfig() map[string]interface{} { + if output, ok := e.Config["output"].(map[string]interface{}); ok { + if fileConfig, ok := output["file_config"].(map[string]interface{}); ok { + return fileConfig + } + } + return nil +} + +func (e *ExtensionDefinition) IsCleanupEnabled() bool { + if fc := e.GetFileConfig(); fc != nil { + if cleanup, ok := fc["cleanup"].(bool); ok { + return cleanup + } + } + return false // default to no cleanup +} + + +func NewExtensionRegistry(configDir string) *ExtensionRegistry { + r := &ExtensionRegistry{ + configDir: configDir, + } + r.registry.Extensions = make(map[string]*ExtensionDefinition) + r.registry.ConfigHashes = make(map[string]string) + r.registry.ExecutableHashes = make(map[string]string) + + // Ensure extensions directory exists + r.ensureConfigDir() + + // Load existing registry if it exists + if err := r.loadRegistry(); err != nil { + // Since we're in a constructor, we can't return error + // Log it if we have logging, but continue with empty registry + if Debug { + fmt.Printf("Warning: could not load extension registry: %v\n", err) + } + } + + return r +} + +func (r *ExtensionRegistry) ensureConfigDir() error { + extDir := filepath.Join(r.configDir, "extensions") + return os.MkdirAll(extDir, 0755) +} + +func (r *ExtensionRegistry) Register(configPath string) error { + // Read and parse the extension definition + data, err := os.ReadFile(configPath) + if err != nil { + return fmt.Errorf("failed to read config file: %w", err) + } + + var ext ExtensionDefinition + if err := yaml.Unmarshal(data, &ext); err != nil { + return fmt.Errorf("failed to parse config file: %w", err) + } + + // Verify Executable exists + if _, err := os.Stat(ext.Executable); err != nil { + return fmt.Errorf("Executable not found: %w", err) + } + + // Calculate hashes using template package functions + configHash := ComputeStringHash(string(data)) + ExecutableHash, err := ComputeHash(ext.Executable) + if err != nil { + return fmt.Errorf("failed to hash Executable: %w", err) + } + + // Store extension and hashes + r.registry.Extensions[ext.Name] = &ext + r.registry.ConfigHashes[ext.Name] = configHash + r.registry.ExecutableHashes[ext.Name] = ExecutableHash + + return r.saveRegistry() +} + +func (r *ExtensionRegistry) Remove(name string) error { + if _, exists := r.registry.Extensions[name]; !exists { + return fmt.Errorf("extension %s not found", name) + } + + delete(r.registry.Extensions, name) + delete(r.registry.ConfigHashes, name) + delete(r.registry.ExecutableHashes, name) + + return r.saveRegistry() +} + +func (r *ExtensionRegistry) Verify(name string) error { + ext, exists := r.registry.Extensions[name] + if !exists { + return fmt.Errorf("extension %s not found", name) + } + + // Verify Executable hash using template package function + currentExecutableHash, err := ComputeHash(ext.Executable) + if err != nil { + return fmt.Errorf("failed to verify Executable: %w", err) + } + + if currentExecutableHash != r.registry.ExecutableHashes[name] { + return fmt.Errorf("Executable hash mismatch for %s", name) + } + + return nil +} + +func (r *ExtensionRegistry) GetExtension(name string) (*ExtensionDefinition, error) { + ext, exists := r.registry.Extensions[name] + if !exists { + return nil, fmt.Errorf("extension %s not found", name) + } + + if err := r.Verify(name); err != nil { + return nil, err + } + + return ext, nil +} + +func (r *ExtensionRegistry) ListExtensions() ([]*ExtensionDefinition, error) { + exts := make([]*ExtensionDefinition, 0, len(r.registry.Extensions)) + for _, ext := range r.registry.Extensions { + exts = append(exts, ext) + } + return exts, nil +} + +func (r *ExtensionRegistry) calculateFileHash(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + + return hex.EncodeToString(h.Sum(nil)), nil +} + +func (r *ExtensionRegistry) saveRegistry() error { + data, err := yaml.Marshal(r.registry) + if err != nil { + return fmt.Errorf("failed to marshal extension registry: %w", err) + } + + registryPath := filepath.Join(r.configDir, "extensions", "extensions.yaml") + return os.WriteFile(registryPath, data, 0644) +} + +func (r *ExtensionRegistry) loadRegistry() error { + registryPath := filepath.Join(r.configDir, "extensions", "extensions.yaml") + data, err := os.ReadFile(registryPath) + if err != nil { + if os.IsNotExist(err) { + return nil // New registry + } + return fmt.Errorf("failed to read extension registry: %w", err) + } + + // Need to unmarshal the data into our registry + if err := yaml.Unmarshal(data, &r.registry); err != nil { + return fmt.Errorf("failed to parse extension registry: %w", err) + } + + return nil +} \ No newline at end of file diff --git a/plugins/template/hash.go b/plugins/template/hash.go new file mode 100644 index 000000000..38f7dde82 --- /dev/null +++ b/plugins/template/hash.go @@ -0,0 +1,33 @@ +package template + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" +) + +// ComputeHash computes SHA-256 hash of a file at given path. +// Returns the hex-encoded hash string or an error if the operation fails. +func ComputeHash(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("open file: %w", err) + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", fmt.Errorf("read file: %w", err) + } + + return hex.EncodeToString(h.Sum(nil)), nil +} + +// ComputeStringHash returns hex-encoded SHA-256 hash of the given string +func ComputeStringHash(s string) string { + h := sha256.New() + h.Write([]byte(s)) + return hex.EncodeToString(h.Sum(nil)) +} \ No newline at end of file diff --git a/plugins/template/hash_test.go b/plugins/template/hash_test.go new file mode 100644 index 000000000..a7b124217 --- /dev/null +++ b/plugins/template/hash_test.go @@ -0,0 +1,119 @@ +// template/hash_test.go +package template + +import ( + "os" + "path/filepath" + "testing" +) + +func TestComputeHash(t *testing.T) { + // Create a temporary test file + content := []byte("test content for hashing") + tmpfile, err := os.CreateTemp("", "hashtest") + if err != nil { + t.Fatalf("failed to create temp file: %v", err) + } + defer os.Remove(tmpfile.Name()) + + if _, err := tmpfile.Write(content); err != nil { + t.Fatalf("failed to write to temp file: %v", err) + } + if err := tmpfile.Close(); err != nil { + t.Fatalf("failed to close temp file: %v", err) + } + + tests := []struct { + name string + path string + want string // known hash for test content + wantErr bool + }{ + { + name: "valid file", + path: tmpfile.Name(), + want: "e25dd806d495b413931f4eea50b677a7a5c02d00460924661283f211a37f7e7f", // pre-computed hash of "test content for hashing" + wantErr: false, + }, + { + name: "nonexistent file", + path: filepath.Join(os.TempDir(), "nonexistent"), + want: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ComputeHash(tt.path) + if (err != nil) != tt.wantErr { + t.Errorf("ComputeHash() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want && !tt.wantErr { + t.Errorf("ComputeHash() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestComputeStringHash(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + { + name: "empty string", + input: "", + want: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }, + { + name: "simple string", + input: "test", + want: "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + }, + { + name: "longer string with spaces", + input: "this is a test string", + want: "f6774519d1c7a3389ef327e9c04766b999db8cdfb85d1346c471ee86d65885bc", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ComputeStringHash(tt.input); got != tt.want { + t.Errorf("ComputeStringHash() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestHashConsistency ensures both hash functions produce same results for same content +func TestHashConsistency(t *testing.T) { + content := "test content for consistency check" + + // Create a file with the test content + tmpfile, err := os.CreateTemp("", "hashconsistency") + if err != nil { + t.Fatalf("failed to create temp file: %v", err) + } + defer os.Remove(tmpfile.Name()) + + if err := os.WriteFile(tmpfile.Name(), []byte(content), 0644); err != nil { + t.Fatalf("failed to write to temp file: %v", err) + } + + // Get hashes using both methods + fileHash, err := ComputeHash(tmpfile.Name()) + if err != nil { + t.Fatalf("ComputeHash failed: %v", err) + } + + stringHash := ComputeStringHash(content) + + // Compare results + if fileHash != stringHash { + t.Errorf("Hash inconsistency: file hash %v != string hash %v", fileHash, stringHash) + } +} \ No newline at end of file diff --git a/plugins/template/template.go b/plugins/template/template.go index 6ff7cd4e8..382053e0e 100644 --- a/plugins/template/template.go +++ b/plugins/template/template.go @@ -2,6 +2,8 @@ package template import ( "fmt" + "os" + "path/filepath" "regexp" "strings" ) @@ -11,11 +13,24 @@ var ( datetimePlugin = &DateTimePlugin{} filePlugin = &FilePlugin{} fetchPlugin = &FetchPlugin{} - sysPlugin = &SysPlugin{} - Debug = false // Debug flag + sysPlugin = &SysPlugin{} + extensionManager *ExtensionManager + Debug = true // Debug flag ) + +func init() { + homedir, err := os.UserHomeDir() + if err != nil { + // We should probably handle this error appropriately + return + } + configDir := filepath.Join(homedir, ".config/fabric") + extensionManager = NewExtensionManager(configDir) +} + var pluginPattern = regexp.MustCompile(`\{\{plugin:([^:]+):([^:]+)(?::([^}]+))?\}\}`) +var extensionPattern = regexp.MustCompile(`\{\{ext:([^:]+):([^:]+)(?::([^}]+))?\}\}`) func debugf(format string, a ...interface{}) { if Debug { @@ -91,6 +106,31 @@ func ApplyTemplate(content string, variables map[string]string, input string) (s } } + if pluginMatches := extensionPattern.FindStringSubmatch(fullMatch); len(pluginMatches) >= 3 { + name := pluginMatches[1] + operation := pluginMatches[2] + value := "" + if len(pluginMatches) == 4 { + value = pluginMatches[3] + } + + debugf("\nExtension call:\n") + debugf(" Name: %s\n", name) + debugf(" Operation: %s\n", operation) + debugf(" Value: %s\n", value) + + result, err := extensionManager.ProcessExtension(name, operation, value) + if err != nil { + return "", fmt.Errorf("extension %s error: %v", name, err) + } + + content = strings.ReplaceAll(content, fullMatch, result) + replaced = true + continue + } + + + // Handle regular variables and input debugf("Processing variable: %s\n", varName) if varName == "input" { diff --git a/plugins/template/utils.go b/plugins/template/utils.go new file mode 100644 index 000000000..1db7c8913 --- /dev/null +++ b/plugins/template/utils.go @@ -0,0 +1,41 @@ +// utils.go in template package for now +package template + +import ( + "fmt" + "os" + "os/user" + "path/filepath" + "strings" +) + +// ExpandPath expands the ~ to user's home directory and returns absolute path +// It also checks if the path exists +// Returns expanded absolute path or error if: +// - cannot determine user home directory +// - cannot convert to absolute path +// - path doesn't exist +func ExpandPath(path string) (string, error) { + // If path starts with ~ + if strings.HasPrefix(path, "~/") { + usr, err := user.Current() + if err != nil { + return "", fmt.Errorf("failed to get user home directory: %w", err) + } + // Replace ~/ with actual home directory + path = filepath.Join(usr.HomeDir, path[2:]) + } + + // Convert to absolute path + absPath, err := filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("failed to get absolute path: %w", err) + } + + // Check if path exists + if _, err := os.Stat(absPath); err != nil { + return "", fmt.Errorf("path does not exist: %w", err) + } + + return absPath, nil +} From 43597e416883994f8e616cf2ac65254d6b62a4aa Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Sun, 1 Dec 2024 18:04:11 +1100 Subject: [PATCH 02/14] Extension Registry Refinement - Successfully implemented path-based registry storage - Moved to storing paths instead of full configurations - Implemented proper hash verification for both configs and executables - Registry format now clean and minimal. File-Based Output Implementation - Successfully implemented file-based output handling - Demonstrated clean interface requiring only path output - Properly handles cleanup of temporary files - Verified working with both local and remote operations --- plugins/template/extension_registry.go | 154 +++++++++++++++++-------- 1 file changed, 106 insertions(+), 48 deletions(-) diff --git a/plugins/template/extension_registry.go b/plugins/template/extension_registry.go index 6da115ae5..228d8de05 100644 --- a/plugins/template/extension_registry.go +++ b/plugins/template/extension_registry.go @@ -34,16 +34,24 @@ type OperationConfig struct { CmdTemplate string `yaml:"cmd_template"` } + + +// RegistryEntry represents a registered extension +type RegistryEntry struct { + ConfigPath string `yaml:"config_path"` + ConfigHash string `yaml:"config_hash"` + ExecutableHash string `yaml:"executable_hash"` +} + type ExtensionRegistry struct { configDir string registry struct { - Extensions map[string]*ExtensionDefinition - ConfigHashes map[string]string - ExecutableHashes map[string]string + Extensions map[string]*RegistryEntry `yaml:"extensions"` } } + // Helper methods for Config access func (e *ExtensionDefinition) GetOutputMethod() string { if output, ok := e.Config["output"].(map[string]interface{}); ok { @@ -74,26 +82,20 @@ func (e *ExtensionDefinition) IsCleanupEnabled() bool { func NewExtensionRegistry(configDir string) *ExtensionRegistry { - r := &ExtensionRegistry{ - configDir: configDir, - } - r.registry.Extensions = make(map[string]*ExtensionDefinition) - r.registry.ConfigHashes = make(map[string]string) - r.registry.ExecutableHashes = make(map[string]string) - - // Ensure extensions directory exists - r.ensureConfigDir() - - // Load existing registry if it exists - if err := r.loadRegistry(); err != nil { - // Since we're in a constructor, we can't return error - // Log it if we have logging, but continue with empty registry - if Debug { - fmt.Printf("Warning: could not load extension registry: %v\n", err) - } - } - - return r + r := &ExtensionRegistry{ + configDir: configDir, + } + r.registry.Extensions = make(map[string]*RegistryEntry) + + r.ensureConfigDir() + + if err := r.loadRegistry(); err != nil { + if Debug { + fmt.Printf("Warning: could not load extension registry: %v\n", err) + } + } + + return r } func (r *ExtensionRegistry) ensureConfigDir() error { @@ -102,7 +104,7 @@ func (r *ExtensionRegistry) ensureConfigDir() error { } func (r *ExtensionRegistry) Register(configPath string) error { - // Read and parse the extension definition + // Read and parse the extension definition to verify it data, err := os.ReadFile(configPath) if err != nil { return fmt.Errorf("failed to read config file: %w", err) @@ -113,22 +115,30 @@ func (r *ExtensionRegistry) Register(configPath string) error { return fmt.Errorf("failed to parse config file: %w", err) } - // Verify Executable exists + // Verify executable exists if _, err := os.Stat(ext.Executable); err != nil { - return fmt.Errorf("Executable not found: %w", err) + return fmt.Errorf("executable not found: %w", err) + } + + // Get absolute path to config + absPath, err := filepath.Abs(configPath) + if err != nil { + return fmt.Errorf("failed to get absolute path: %w", err) } - // Calculate hashes using template package functions + // Calculate hashes configHash := ComputeStringHash(string(data)) - ExecutableHash, err := ComputeHash(ext.Executable) + executableHash, err := ComputeHash(ext.Executable) if err != nil { - return fmt.Errorf("failed to hash Executable: %w", err) + return fmt.Errorf("failed to hash executable: %w", err) } - // Store extension and hashes - r.registry.Extensions[ext.Name] = &ext - r.registry.ConfigHashes[ext.Name] = configHash - r.registry.ExecutableHashes[ext.Name] = ExecutableHash + // Store entry + r.registry.Extensions[ext.Name] = &RegistryEntry{ + ConfigPath: absPath, + ConfigHash: configHash, + ExecutableHash: executableHash, + } return r.saveRegistry() } @@ -139,49 +149,97 @@ func (r *ExtensionRegistry) Remove(name string) error { } delete(r.registry.Extensions, name) - delete(r.registry.ConfigHashes, name) - delete(r.registry.ExecutableHashes, name) return r.saveRegistry() } func (r *ExtensionRegistry) Verify(name string) error { - ext, exists := r.registry.Extensions[name] + // Get the registry entry + entry, exists := r.registry.Extensions[name] if !exists { return fmt.Errorf("extension %s not found", name) } - // Verify Executable hash using template package function + // Load and parse the config file + data, err := os.ReadFile(entry.ConfigPath) + if err != nil { + return fmt.Errorf("failed to read config file: %w", err) + } + + // Verify config hash + currentConfigHash := ComputeStringHash(string(data)) + if currentConfigHash != entry.ConfigHash { + return fmt.Errorf("config file hash mismatch for %s", name) + } + + // Parse to get executable path + var ext ExtensionDefinition + if err := yaml.Unmarshal(data, &ext); err != nil { + return fmt.Errorf("failed to parse config file: %w", err) + } + + // Verify executable hash currentExecutableHash, err := ComputeHash(ext.Executable) if err != nil { - return fmt.Errorf("failed to verify Executable: %w", err) + return fmt.Errorf("failed to verify executable: %w", err) } - if currentExecutableHash != r.registry.ExecutableHashes[name] { - return fmt.Errorf("Executable hash mismatch for %s", name) + if currentExecutableHash != entry.ExecutableHash { + return fmt.Errorf("executable hash mismatch for %s", name) } return nil } func (r *ExtensionRegistry) GetExtension(name string) (*ExtensionDefinition, error) { - ext, exists := r.registry.Extensions[name] + entry, exists := r.registry.Extensions[name] if !exists { return nil, fmt.Errorf("extension %s not found", name) } - - if err := r.Verify(name); err != nil { - return nil, err + + // Read current config file + data, err := os.ReadFile(entry.ConfigPath) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) } - - return ext, nil + + // Verify config hash + currentHash := ComputeStringHash(string(data)) + if currentHash != entry.ConfigHash { + return nil, fmt.Errorf("config file hash mismatch for %s", name) + } + + // Parse config + var ext ExtensionDefinition + if err := yaml.Unmarshal(data, &ext); err != nil { + return nil, fmt.Errorf("failed to parse config file: %w", err) + } + + // Verify executable hash + currentExecHash, err := ComputeHash(ext.Executable) + if err != nil { + return nil, fmt.Errorf("failed to verify executable: %w", err) + } + + if currentExecHash != entry.ExecutableHash { + return nil, fmt.Errorf("executable hash mismatch for %s", name) + } + + return &ext, nil } + func (r *ExtensionRegistry) ListExtensions() ([]*ExtensionDefinition, error) { - exts := make([]*ExtensionDefinition, 0, len(r.registry.Extensions)) - for _, ext := range r.registry.Extensions { + var exts []*ExtensionDefinition + + for name := range r.registry.Extensions { + ext, err := r.GetExtension(name) + if err != nil { + return nil, fmt.Errorf("failed to load extension %s: %w", name, err) + } exts = append(exts, ext) } + return exts, nil } From b31f094e9bcc4a1889ba485030ee7d6f9cb21864 Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Sun, 1 Dec 2024 22:19:06 +1100 Subject: [PATCH 03/14] Added better messages when adding and listing extensions Fix issuse with listextension where it would fail if any hash had changed, now says hash failed. --- plugins/template/extension_manager.go | 72 +++++++++++++++++++++----- plugins/template/extension_registry.go | 5 +- 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/plugins/template/extension_manager.go b/plugins/template/extension_manager.go index 7d0ed1158..aa3a5b630 100644 --- a/plugins/template/extension_manager.go +++ b/plugins/template/extension_manager.go @@ -2,7 +2,10 @@ package template import ( "fmt" + "os" "path/filepath" + + "gopkg.in/yaml.v3" ) // ExtensionManager handles the high-level operations of the extension system @@ -24,13 +27,23 @@ func NewExtensionManager(configDir string) *ExtensionManager { // ListExtensions handles the listextensions flag action func (em *ExtensionManager) ListExtensions() error { - extensions, err := em.registry.ListExtensions() - if err != nil { - return fmt.Errorf("failed to list extensions: %w", err) + if em.registry == nil || em.registry.registry.Extensions == nil { + return fmt.Errorf("extension registry not initialized") } - for _, ext := range extensions { - fmt.Printf("Name: %s\n", ext.Name) + for name, entry := range em.registry.registry.Extensions { + fmt.Printf("Extension: %s\n", name) + + // Try to load extension details + ext, err := em.registry.GetExtension(name) + if err != nil { + fmt.Printf(" Status: DISABLED - Hash verification failed: %v\n", err) + fmt.Printf(" Config Path: %s\n\n", entry.ConfigPath) + continue + } + + // Print extension details if verification succeeded + fmt.Printf(" Status: ENABLED\n") fmt.Printf(" Executable: %s\n", ext.Executable) fmt.Printf(" Type: %s\n", ext.Type) fmt.Printf(" Timeout: %s\n", ext.Timeout) @@ -57,16 +70,49 @@ func (em *ExtensionManager) ListExtensions() error { // RegisterExtension handles the addextension flag action func (em *ExtensionManager) RegisterExtension(configPath string) error { - absPath, err := filepath.Abs(configPath) - if err != nil { - return fmt.Errorf("invalid config path: %w", err) - } + absPath, err := filepath.Abs(configPath) + if err != nil { + return fmt.Errorf("invalid config path: %w", err) + } - if err := em.registry.Register(absPath); err != nil { - return fmt.Errorf("failed to register extension: %w", err) - } + // Get extension name before registration for status message + data, err := os.ReadFile(absPath) + if err != nil { + return fmt.Errorf("failed to read config file: %w", err) + } - return nil + var ext ExtensionDefinition + if err := yaml.Unmarshal(data, &ext); err != nil { + return fmt.Errorf("failed to parse config file: %w", err) + } + + if err := em.registry.Register(absPath); err != nil { + return fmt.Errorf("failed to register extension: %w", err) + } + + // Print success message with extension details + fmt.Printf("Successfully registered extension:\n") + fmt.Printf("Name: %s\n", ext.Name) + fmt.Printf(" Executable: %s\n", ext.Executable) + fmt.Printf(" Type: %s\n", ext.Type) + fmt.Printf(" Timeout: %s\n", ext.Timeout) + fmt.Printf(" Description: %s\n", ext.Description) + fmt.Printf(" Version: %s\n", ext.Version) + + fmt.Printf(" Operations:\n") + for opName, opConfig := range ext.Operations { + fmt.Printf(" %s:\n", opName) + fmt.Printf(" Command Template: %s\n", opConfig.CmdTemplate) + } + + if fileConfig := ext.GetFileConfig(); fileConfig != nil { + fmt.Printf(" File Configuration:\n") + for k, v := range fileConfig { + fmt.Printf(" %s: %v\n", k, v) + } + } + + return nil } // RemoveExtension handles the rmextension flag action diff --git a/plugins/template/extension_registry.go b/plugins/template/extension_registry.go index 228d8de05..1f1560591 100644 --- a/plugins/template/extension_registry.go +++ b/plugins/template/extension_registry.go @@ -235,7 +235,10 @@ func (r *ExtensionRegistry) ListExtensions() ([]*ExtensionDefinition, error) { for name := range r.registry.Extensions { ext, err := r.GetExtension(name) if err != nil { - return nil, fmt.Errorf("failed to load extension %s: %w", name, err) + // Instead of failing, we'll return nil for this extension + // The manager will handle displaying the error + exts = append(exts, nil) + continue } exts = append(exts, ext) } From 744ec0824be60ab39337d8cc92ea0552fcc2c31c Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Sun, 1 Dec 2024 22:29:59 +1100 Subject: [PATCH 04/14] Fix pattern file usage without stdin When using pattern files with variables but no stdin input, ensure proper template processing by initializing an empty message. This allows patterns like: ./fabric -p pattern.txt -v=name:value to work without requiring stdin input, while maintaining compatibility with existing stdin usage: echo "input" | ./fabric -p pattern.txt -v=name:value Changes: - Add empty message initialization in BuildSession when Message is nil - Remove redundant template processing of message content - Let pattern processing handle all template resolution This simplifies the template processing flow while supporting both stdin and non-stdin use cases. --- core/chatter.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/core/chatter.go b/core/chatter.go index a8f30bbfe..f8e2fa8ce 100644 --- a/core/chatter.go +++ b/core/chatter.go @@ -10,7 +10,6 @@ import ( "github.com/danielmiessler/fabric/common" "github.com/danielmiessler/fabric/plugins/ai" "github.com/danielmiessler/fabric/plugins/db/fsdb" - "github.com/danielmiessler/fabric/plugins/template" ) const NoSessionPatternUserMessages = "no session, pattern or user messages provided" @@ -105,29 +104,26 @@ func (o *Chatter) BuildSession(request *common.ChatRequest, raw bool) (session * // Process any template variables in the message content (user input) // Double curly braces {{variable}} indicate template substitution - // should occur, whether in patterns or direct input - if request.Message != nil { - request.Message.Content, err = template.ApplyTemplate(request.Message.Content, request.PatternVariables, "") - if err != nil { - return nil, err - } + // Ensure we have a message, even if empty + if request.Message == nil { + request.Message = &goopenai.ChatCompletionMessage{ + Role: goopenai.ChatMessageRoleUser, + Content: " ", + } } var patternContent string if request.PatternName != "" { - pattern, err := o.db.Patterns.GetApplyVariables(request.PatternName, request.PatternVariables, request.Message.Content) - // pattrn will now contain user input, and all variables will be resolved, or errored - + pattern, err := o.db.Patterns.GetApplyVariables(request.PatternName, request.PatternVariables, request.Message.Content) if err != nil { return nil, fmt.Errorf("could not get pattern %s: %v", request.PatternName, err) } patternContent = pattern.Pattern } - systemMessage := strings.TrimSpace(contextContent) + strings.TrimSpace(patternContent) if request.Language != "" { - systemMessage = fmt.Sprintf("%s. Please use the language '%s' for the output.", systemMessage, request.Language) + systemMessage = fmt.Sprintf("%s. Please use the language '%s' for the output.", systemMessage, request.Language) } if raw { From db2ba460992d07d72db0d065a2acf98d715a5684 Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Sun, 1 Dec 2024 22:44:17 +1100 Subject: [PATCH 05/14] Revert "Fix pattern file usage without stdin" This reverts commit 744ec0824be60ab39337d8cc92ea0552fcc2c31c. --- core/chatter.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/core/chatter.go b/core/chatter.go index f8e2fa8ce..a8f30bbfe 100644 --- a/core/chatter.go +++ b/core/chatter.go @@ -10,6 +10,7 @@ import ( "github.com/danielmiessler/fabric/common" "github.com/danielmiessler/fabric/plugins/ai" "github.com/danielmiessler/fabric/plugins/db/fsdb" + "github.com/danielmiessler/fabric/plugins/template" ) const NoSessionPatternUserMessages = "no session, pattern or user messages provided" @@ -104,26 +105,29 @@ func (o *Chatter) BuildSession(request *common.ChatRequest, raw bool) (session * // Process any template variables in the message content (user input) // Double curly braces {{variable}} indicate template substitution - // Ensure we have a message, even if empty - if request.Message == nil { - request.Message = &goopenai.ChatCompletionMessage{ - Role: goopenai.ChatMessageRoleUser, - Content: " ", - } + // should occur, whether in patterns or direct input + if request.Message != nil { + request.Message.Content, err = template.ApplyTemplate(request.Message.Content, request.PatternVariables, "") + if err != nil { + return nil, err + } } var patternContent string if request.PatternName != "" { - pattern, err := o.db.Patterns.GetApplyVariables(request.PatternName, request.PatternVariables, request.Message.Content) + pattern, err := o.db.Patterns.GetApplyVariables(request.PatternName, request.PatternVariables, request.Message.Content) + // pattrn will now contain user input, and all variables will be resolved, or errored + if err != nil { return nil, fmt.Errorf("could not get pattern %s: %v", request.PatternName, err) } patternContent = pattern.Pattern } + systemMessage := strings.TrimSpace(contextContent) + strings.TrimSpace(patternContent) if request.Language != "" { - systemMessage = fmt.Sprintf("%s. Please use the language '%s' for the output.", systemMessage, request.Language) + systemMessage = fmt.Sprintf("%s. Please use the language '%s' for the output.", systemMessage, request.Language) } if raw { From da6f9748873b466da1e558abc62430a5ca235485 Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Sun, 1 Dec 2024 22:52:32 +1100 Subject: [PATCH 06/14] fixed : if there is no stdin, then a nil message was passed to pattern.go resulting in segfault. now we make user input ' ', before processing. --- core/chatter.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/core/chatter.go b/core/chatter.go index a8f30bbfe..b7751738e 100644 --- a/core/chatter.go +++ b/core/chatter.go @@ -105,12 +105,18 @@ func (o *Chatter) BuildSession(request *common.ChatRequest, raw bool) (session * // Process any template variables in the message content (user input) // Double curly braces {{variable}} indicate template substitution - // should occur, whether in patterns or direct input - if request.Message != nil { - request.Message.Content, err = template.ApplyTemplate(request.Message.Content, request.PatternVariables, "") - if err != nil { - return nil, err - } + // Ensure we have a message before processing, other wise we'll get an error when we pass to pattern.go + if request.Message == nil { + request.Message = &goopenai.ChatCompletionMessage{ + Role: goopenai.ChatMessageRoleUser, + Content: " ", + } + } + + // Now we know request.Message is not nil, process template variables + request.Message.Content, err = template.ApplyTemplate(request.Message.Content, request.PatternVariables, "") + if err != nil { + return nil, err } var patternContent string From 6fc75282e8802c16948376f06012e0daf4ccaa7f Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Tue, 3 Dec 2024 23:28:47 +1100 Subject: [PATCH 07/14] added tests for extension manager, registration and execution. --- plugins/template/extension_executor_test.go | 360 ++++++++++++++++++++ plugins/template/extension_manager_test.go | 184 ++++++++++ plugins/template/extension_registry.go | 41 +++ plugins/template/extension_registry_test.go | 75 ++++ 4 files changed, 660 insertions(+) create mode 100644 plugins/template/extension_executor_test.go create mode 100644 plugins/template/extension_manager_test.go create mode 100644 plugins/template/extension_registry_test.go diff --git a/plugins/template/extension_executor_test.go b/plugins/template/extension_executor_test.go new file mode 100644 index 000000000..72033979e --- /dev/null +++ b/plugins/template/extension_executor_test.go @@ -0,0 +1,360 @@ +package template + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestExtensionExecutor(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "fabric-ext-executor-*") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + // Create test script that has both stdout and file output modes + testScript := filepath.Join(tmpDir, "test-script.sh") + scriptContent := `#!/bin/bash +case "$1" in + "stdout") + echo "Hello, $2!" + ;; + "file") + echo "Hello, $2!" > "$3" + echo "$3" # Print the filename for path_from_stdout + ;; + *) + echo "Unknown command" >&2 + exit 1 + ;; +esac` + + if err := os.WriteFile(testScript, []byte(scriptContent), 0755); err != nil { + t.Fatalf("Failed to create test script: %v", err) + } + + // Create registry and register our test extensions + registry := NewExtensionRegistry(tmpDir) + executor := NewExtensionExecutor(registry) + + // Test stdout-based extension + t.Run("StdoutExecution", func(t *testing.T) { + configPath := filepath.Join(tmpDir, "stdout-extension.yaml") + configContent := `name: stdout-test +executable: ` + testScript + ` +type: executable +timeout: 30s +operations: + greet: + cmd_template: "{{executable}} stdout {{1}}" +config: + output: + method: stdout` + + if err := os.WriteFile(configPath, []byte(configContent), 0644); err != nil { + t.Fatalf("Failed to create config: %v", err) + } + + if err := registry.Register(configPath); err != nil { + t.Fatalf("Failed to register extension: %v", err) + } + + output, err := executor.Execute("stdout-test", "greet", "World") + if err != nil { + t.Errorf("Failed to execute: %v", err) + } + + expected := "Hello, World!\n" + if output != expected { + t.Errorf("Expected output %q, got %q", expected, output) + } + }) + + // Test file-based extension + t.Run("FileExecution", func(t *testing.T) { + configPath := filepath.Join(tmpDir, "file-extension.yaml") + configContent := `name: file-test +executable: ` + testScript + ` +type: executable +timeout: 30s +operations: + greet: + cmd_template: "{{executable}} file {{1}} {{2}}" +config: + output: + method: file + file_config: + cleanup: true + path_from_stdout: true` + + if err := os.WriteFile(configPath, []byte(configContent), 0644); err != nil { + t.Fatalf("Failed to create config: %v", err) + } + + if err := registry.Register(configPath); err != nil { + t.Fatalf("Failed to register extension: %v", err) + } + + output, err := executor.Execute("file-test", "greet", "World|/tmp/test.txt") + if err != nil { + t.Errorf("Failed to execute: %v", err) + } + + expected := "Hello, World!\n" + if output != expected { + t.Errorf("Expected output %q, got %q", expected, output) + } + }) + + // Test execution errors + t.Run("ExecutionErrors", func(t *testing.T) { + // Test with non-existent extension + _, err := executor.Execute("nonexistent", "test", "value") + if err == nil { + t.Error("Expected error executing non-existent extension, got nil") + } + + // Test with invalid command that should exit non-zero + configPath := filepath.Join(tmpDir, "error-extension.yaml") + configContent := `name: error-test +executable: ` + testScript + ` +type: executable +timeout: 30s +operations: + invalid: + cmd_template: "{{executable}} invalid {{1}}" +config: + output: + method: stdout` + + if err := os.WriteFile(configPath, []byte(configContent), 0644); err != nil { + t.Fatalf("Failed to create config: %v", err) + } + + if err := registry.Register(configPath); err != nil { + t.Fatalf("Failed to register extension: %v", err) + } + + _, err = executor.Execute("error-test", "invalid", "test") + if err == nil { + t.Error("Expected error from invalid command, got nil") + } + if !strings.Contains(err.Error(), "Unknown command") { + t.Errorf("Expected 'Unknown command' in error, got: %v", err) + } + }) +} + +func TestFixedFileExtensionExecutor(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "fabric-ext-executor-fixed-*") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + // Create test script + testScript := filepath.Join(tmpDir, "test-script.sh") + scriptContent := `#!/bin/bash +case "$1" in + "write") + echo "Hello, $2!" > "$3" + ;; + "append") + echo "Hello, $2!" >> "$3" + ;; + "large") + for i in {1..1000}; do + echo "Line $i" >> "$3" + done + ;; + "error") + echo "Error message" >&2 + exit 1 + ;; + *) + echo "Unknown command" >&2 + exit 1 + ;; +esac` + + if err := os.WriteFile(testScript, []byte(scriptContent), 0755); err != nil { + t.Fatalf("Failed to create test script: %v", err) + } + + registry := NewExtensionRegistry(tmpDir) + executor := NewExtensionExecutor(registry) + + // Helper function to create and register extension + createExtension := func(name, opName, cmdTemplate string, config map[string]interface{}) error { + configPath := filepath.Join(tmpDir, name+".yaml") + configContent := `name: ` + name + ` +executable: ` + testScript + ` +type: executable +timeout: 30s +operations: + ` + opName + `: + cmd_template: "` + cmdTemplate + `" +config: + output: + method: file + file_config:` + + // Add config options + for k, v := range config { + configContent += "\n " + k + ": " + strings.TrimSpace(v.(string)) + } + + if err := os.WriteFile(configPath, []byte(configContent), 0644); err != nil { + return err + } + + return registry.Register(configPath) + } + + // Test basic fixed file output + t.Run("BasicFixedFile", func(t *testing.T) { + outputFile := filepath.Join(tmpDir, "output.txt") + config := map[string]interface{}{ + "output_file": `"output.txt"`, + "work_dir": `"` + tmpDir + `"`, + "cleanup": "true", + } + + err := createExtension("basic-test", "write", + "{{executable}} write {{1}} "+outputFile, config) + if err != nil { + t.Fatalf("Failed to create extension: %v", err) + } + + output, err := executor.Execute("basic-test", "write", "World") + if err != nil { + t.Errorf("Failed to execute: %v", err) + } + + expected := "Hello, World!\n" + if output != expected { + t.Errorf("Expected output %q, got %q", expected, output) + } + }) + + // Test no work_dir specified + t.Run("NoWorkDir", func(t *testing.T) { + config := map[string]interface{}{ + "output_file": `"direct-output.txt"`, + "cleanup": "true", + } + + err := createExtension("no-workdir-test", "write", + "{{executable}} write {{1}} direct-output.txt", config) + if err != nil { + t.Fatalf("Failed to create extension: %v", err) + } + + _, err = executor.Execute("no-workdir-test", "write", "World") + if err != nil { + t.Errorf("Failed to execute: %v", err) + } + }) + + // Test cleanup behavior + t.Run("CleanupBehavior", func(t *testing.T) { + outputFile := filepath.Join(tmpDir, "cleanup-test.txt") + + // Test with cleanup enabled + config := map[string]interface{}{ + "output_file": `"cleanup-test.txt"`, + "work_dir": `"` + tmpDir + `"`, + "cleanup": "true", + } + + err := createExtension("cleanup-test", "write", + "{{executable}} write {{1}} "+outputFile, config) + if err != nil { + t.Fatalf("Failed to create extension: %v", err) + } + + _, err = executor.Execute("cleanup-test", "write", "World") + if err != nil { + t.Errorf("Failed to execute: %v", err) + } + + // File should be deleted after execution + if _, err := os.Stat(outputFile); !os.IsNotExist(err) { + t.Error("Expected output file to be cleaned up") + } + + // Test with cleanup disabled + config["cleanup"] = "false" + err = createExtension("no-cleanup-test", "write", + "{{executable}} write {{1}} "+outputFile, config) + if err != nil { + t.Fatalf("Failed to create extension: %v", err) + } + + _, err = executor.Execute("no-cleanup-test", "write", "World") + if err != nil { + t.Errorf("Failed to execute: %v", err) + } + + // File should remain after execution + if _, err := os.Stat(outputFile); os.IsNotExist(err) { + t.Error("Expected output file to remain") + } + }) + + // Test error cases + t.Run("ErrorCases", func(t *testing.T) { + outputFile := filepath.Join(tmpDir, "error-test.txt") + config := map[string]interface{}{ + "output_file": `"error-test.txt"`, + "work_dir": `"` + tmpDir + `"`, + "cleanup": "true", + } + + // Test command error + err := createExtension("error-test", "error", + "{{executable}} error {{1}} "+outputFile, config) + if err != nil { + t.Fatalf("Failed to create extension: %v", err) + } + + _, err = executor.Execute("error-test", "error", "World") + if err == nil { + t.Error("Expected error from failing command, got nil") + } + + // Test invalid work_dir + config["work_dir"] = `"/nonexistent/directory"` + err = createExtension("invalid-dir-test", "write", + "{{executable}} write {{1}} output.txt", config) + if err != nil { + t.Fatalf("Failed to create extension: %v", err) + } + + _, err = executor.Execute("invalid-dir-test", "write", "World") + if err == nil { + t.Error("Expected error from invalid work_dir, got nil") + } + }) + + // Test with missing output_file + t.Run("MissingOutputFile", func(t *testing.T) { + config := map[string]interface{}{ + "work_dir": `"` + tmpDir + `"`, + "cleanup": "true", + } + + err := createExtension("missing-output-test", "write", + "{{executable}} write {{1}} output.txt", config) + if err != nil { + t.Fatalf("Failed to create extension: %v", err) + } + + _, err = executor.Execute("missing-output-test", "write", "World") + if err == nil { + t.Error("Expected error from missing output_file, got nil") + } + }) +} \ No newline at end of file diff --git a/plugins/template/extension_manager_test.go b/plugins/template/extension_manager_test.go new file mode 100644 index 000000000..2deec2f13 --- /dev/null +++ b/plugins/template/extension_manager_test.go @@ -0,0 +1,184 @@ +package template + +import ( + "os" + "path/filepath" + "testing" +) + +// TestExtensionManager is the main test suite for ExtensionManager +func TestExtensionManager(t *testing.T) { + // Create temporary directory for tests + tmpDir, err := os.MkdirTemp("", "fabric-ext-test-*") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + // Create test extension config + testConfig := filepath.Join(tmpDir, "test-extension.yaml") + testScript := filepath.Join(tmpDir, "test-script.sh") + + // Create test script + scriptContent := `#!/bin/bash +if [ "$1" = "echo" ]; then + echo "Hello, $2!" +fi` + + err = os.WriteFile(testScript, []byte(scriptContent), 0755) + if err != nil { + t.Fatalf("Failed to create test script: %v", err) + } + + // Create test config + configContent := `name: test-extension +executable: ` + testScript + ` +type: executable +timeout: 30s +description: "Test extension" +version: "1.0.0" +operations: + echo: + cmd_template: "{{executable}} echo {{1}}" +` + + err = os.WriteFile(testConfig, []byte(configContent), 0644) + if err != nil { + t.Fatalf("Failed to create test config: %v", err) + } + + // Initialize manager + manager := NewExtensionManager(tmpDir) + + // Test cases + t.Run("RegisterExtension", func(t *testing.T) { + err := manager.RegisterExtension(testConfig) + if err != nil { + t.Errorf("Failed to register extension: %v", err) + } + }) + + t.Run("ListExtensions", func(t *testing.T) { + err := manager.ListExtensions() + if err != nil { + t.Errorf("Failed to list extensions: %v", err) + } + // Note: Output validation would require capturing stdout + }) + + t.Run("ProcessExtension", func(t *testing.T) { + output, err := manager.ProcessExtension("test-extension", "echo", "World") + if err != nil { + t.Errorf("Failed to process extension: %v", err) + } + expected := "Hello, World!\n" + if output != expected { + t.Errorf("Expected output %q, got %q", expected, output) + } + }) + + t.Run("RemoveExtension", func(t *testing.T) { + err := manager.RemoveExtension("test-extension") + if err != nil { + t.Errorf("Failed to remove extension: %v", err) + } + + // Verify extension is removed by trying to process it + _, err = manager.ProcessExtension("test-extension", "echo", "World") + if err == nil { + t.Error("Expected error processing removed extension, got nil") + } + }) +} + +// TestExtensionManagerErrors tests error cases +func TestExtensionManagerErrors(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "fabric-ext-test-errors-*") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + manager := NewExtensionManager(tmpDir) + + t.Run("RegisterNonexistentConfig", func(t *testing.T) { + err := manager.RegisterExtension("/nonexistent/config.yaml") + if err == nil { + t.Error("Expected error registering nonexistent config, got nil") + } + }) + + t.Run("ProcessNonexistentExtension", func(t *testing.T) { + _, err := manager.ProcessExtension("nonexistent", "echo", "test") + if err == nil { + t.Error("Expected error processing nonexistent extension, got nil") + } + }) + + t.Run("RemoveNonexistentExtension", func(t *testing.T) { + err := manager.RemoveExtension("nonexistent") + if err == nil { + t.Error("Expected error removing nonexistent extension, got nil") + } + }) +} + +// TestExtensionManagerWithInvalidConfig tests handling of invalid configurations +func TestExtensionManagerWithInvalidConfig(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "fabric-ext-test-invalid-*") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + invalidConfig := filepath.Join(tmpDir, "invalid-extension.yaml") + + // Test cases with different invalid configurations + testCases := []struct { + name string + config string + wantErr bool + }{ + { + name: "MissingExecutable", + config: `name: invalid-extension +type: executable +timeout: 30s`, + wantErr: true, + }, + { + name: "InvalidTimeout", + config: `name: invalid-extension +executable: /bin/echo +type: executable +timeout: invalid`, + wantErr: true, + }, + { + name: "EmptyName", + config: `name: "" +executable: /bin/echo +type: executable +timeout: 30s`, + wantErr: true, + }, + } + + manager := NewExtensionManager(tmpDir) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := os.WriteFile(invalidConfig, []byte(tc.config), 0644) + if err != nil { + t.Fatalf("Failed to create invalid config file: %v", err) + } + + err = manager.RegisterExtension(invalidConfig) + if tc.wantErr && err == nil { + t.Error("Expected error registering invalid config, got nil") + } else if !tc.wantErr && err != nil { + t.Errorf("Unexpected error registering config: %v", err) + } + }) + } +} \ No newline at end of file diff --git a/plugins/template/extension_registry.go b/plugins/template/extension_registry.go index 1f1560591..af75808b4 100644 --- a/plugins/template/extension_registry.go +++ b/plugins/template/extension_registry.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "time" "gopkg.in/yaml.v3" // Add this import @@ -103,6 +104,8 @@ func (r *ExtensionRegistry) ensureConfigDir() error { return os.MkdirAll(extDir, 0755) } +// Update the Register method in extension_registry.go + func (r *ExtensionRegistry) Register(configPath string) error { // Read and parse the extension definition to verify it data, err := os.ReadFile(configPath) @@ -115,6 +118,11 @@ func (r *ExtensionRegistry) Register(configPath string) error { return fmt.Errorf("failed to parse config file: %w", err) } + // Add validation + if err := r.validateExtensionDefinition(&ext); err != nil { + return fmt.Errorf("invalid extension configuration: %w", err) + } + // Verify executable exists if _, err := os.Stat(ext.Executable); err != nil { return fmt.Errorf("executable not found: %w", err) @@ -143,6 +151,39 @@ func (r *ExtensionRegistry) Register(configPath string) error { return r.saveRegistry() } +func (r *ExtensionRegistry) validateExtensionDefinition(ext *ExtensionDefinition) error { + // Validate required fields + if ext.Name == "" { + return fmt.Errorf("extension name is required") + } + if ext.Executable == "" { + return fmt.Errorf("executable path is required") + } + if ext.Type == "" { + return fmt.Errorf("extension type is required") + } + + // Validate timeout format + if ext.Timeout != "" { + if _, err := time.ParseDuration(ext.Timeout); err != nil { + return fmt.Errorf("invalid timeout format: %w", err) + } + } + + // Validate operations + if len(ext.Operations) == 0 { + return fmt.Errorf("at least one operation must be defined") + } + for name, op := range ext.Operations { + if op.CmdTemplate == "" { + return fmt.Errorf("command template is required for operation %s", name) + } + } + + return nil +} + + func (r *ExtensionRegistry) Remove(name string) error { if _, exists := r.registry.Extensions[name]; !exists { return fmt.Errorf("extension %s not found", name) diff --git a/plugins/template/extension_registry_test.go b/plugins/template/extension_registry_test.go new file mode 100644 index 000000000..5dcadc4c9 --- /dev/null +++ b/plugins/template/extension_registry_test.go @@ -0,0 +1,75 @@ +package template + +import ( + "os" + "path/filepath" + "testing" +) + +func TestRegistryPersistence(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "fabric-ext-registry-persist-*") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + // Create test executable + execPath := filepath.Join(tmpDir, "test-exec.sh") + execContent := []byte("#!/bin/bash\necho \"test\"") + err = os.WriteFile(execPath, execContent, 0755) + if err != nil { + t.Fatalf("Failed to create test executable: %v", err) + } + + // Create valid config + configContent := `name: test-extension +executable: ` + execPath + ` +type: executable +timeout: 30s +operations: + test: + cmd_template: "{{executable}} {{operation}}"` + + configPath := filepath.Join(tmpDir, "test-extension.yaml") + err = os.WriteFile(configPath, []byte(configContent), 0644) + if err != nil { + t.Fatalf("Failed to create test config: %v", err) + } + + // Test registry persistence + t.Run("SaveAndReload", func(t *testing.T) { + // Create and populate first registry + registry1 := NewExtensionRegistry(tmpDir) + err := registry1.Register(configPath) + if err != nil { + t.Fatalf("Failed to register extension: %v", err) + } + + // Create new registry instance and verify it loads the saved state + registry2 := NewExtensionRegistry(tmpDir) + ext, err := registry2.GetExtension("test-extension") + if err != nil { + t.Fatalf("Failed to get extension from reloaded registry: %v", err) + } + if ext.Name != "test-extension" { + t.Errorf("Expected extension name 'test-extension', got %q", ext.Name) + } + }) + + // Test hash verification + t.Run("HashVerification", func(t *testing.T) { + registry := NewExtensionRegistry(tmpDir) + + // Modify executable after registration + modifiedExecContent := []byte("#!/bin/bash\necho \"modified\"") + err := os.WriteFile(execPath, modifiedExecContent, 0755) + if err != nil { + t.Fatalf("Failed to modify executable: %v", err) + } + + _, err = registry.GetExtension("test-extension") + if err == nil { + t.Error("Expected error when executable modified, got nil") + } + }) +} \ No newline at end of file From c49f47ecabaa58e0da3066b340b01459cd483d63 Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Thu, 5 Dec 2024 20:39:40 +1100 Subject: [PATCH 08/14] check extension names don't have spoaces --- plugins/template/extension_registry.go | 11 ++++++++--- plugins/template/template.go | 6 +++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/plugins/template/extension_registry.go b/plugins/template/extension_registry.go index af75808b4..43690888f 100644 --- a/plugins/template/extension_registry.go +++ b/plugins/template/extension_registry.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "strings" "time" "gopkg.in/yaml.v3" @@ -118,9 +119,13 @@ func (r *ExtensionRegistry) Register(configPath string) error { return fmt.Errorf("failed to parse config file: %w", err) } - // Add validation - if err := r.validateExtensionDefinition(&ext); err != nil { - return fmt.Errorf("invalid extension configuration: %w", err) + // Validate extension name + if ext.Name == "" { + return fmt.Errorf("extension name cannot be empty") + } + + if strings.Contains(ext.Name, " ") { + return fmt.Errorf("extension name '%s' contains spaces - names must not contain spaces", ext.Name) } // Verify executable exists diff --git a/plugins/template/template.go b/plugins/template/template.go index 382053e0e..d959f230a 100644 --- a/plugins/template/template.go +++ b/plugins/template/template.go @@ -15,18 +15,18 @@ var ( fetchPlugin = &FetchPlugin{} sysPlugin = &SysPlugin{} extensionManager *ExtensionManager - Debug = true // Debug flag + Debug = false // Debug flag ) func init() { homedir, err := os.UserHomeDir() if err != nil { - // We should probably handle this error appropriately - return + debugf("Warning: could not initialize extension manager: %v\n", err) } configDir := filepath.Join(homedir, ".config/fabric") extensionManager = NewExtensionManager(configDir) + // Extensions will work if registry exists, otherwise they'll just fail gracefully } var pluginPattern = regexp.MustCompile(`\{\{plugin:([^:]+):([^:]+)(?::([^}]+))?\}\}`) From d8671ea03afdd4aeae80b51d0d5ab1991b5cdd23 Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Thu, 5 Dec 2024 22:09:47 +1100 Subject: [PATCH 09/14] Added example files and tutorial --- plugins/template/Examples/README.md | 0 plugins/template/Examples/memories.db | Bin 0 -> 8192 bytes .../Examples/remote-security-report.sh | 24 ++++ .../Examples/remote-security-report.yaml | 17 +++ plugins/template/Examples/security-report.sh | 113 ++++++++++++++++++ .../template/Examples/security-report.yaml | 18 +++ plugins/template/Examples/sqlite3_demo.yaml | 23 ++++ plugins/template/Examples/track_packages.sh | 18 +++ plugins/template/Examples/word-generator.py | 36 ++++++ plugins/template/Examples/word-generator.yaml | 16 +++ 10 files changed, 265 insertions(+) create mode 100644 plugins/template/Examples/README.md create mode 100644 plugins/template/Examples/memories.db create mode 100755 plugins/template/Examples/remote-security-report.sh create mode 100644 plugins/template/Examples/remote-security-report.yaml create mode 100755 plugins/template/Examples/security-report.sh create mode 100644 plugins/template/Examples/security-report.yaml create mode 100644 plugins/template/Examples/sqlite3_demo.yaml create mode 100755 plugins/template/Examples/track_packages.sh create mode 100755 plugins/template/Examples/word-generator.py create mode 100644 plugins/template/Examples/word-generator.yaml diff --git a/plugins/template/Examples/README.md b/plugins/template/Examples/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/plugins/template/Examples/memories.db b/plugins/template/Examples/memories.db new file mode 100644 index 0000000000000000000000000000000000000000..045be3cdcf01091087bcdc4991b8d1ca805d38d0 GIT binary patch literal 8192 zcmeI1%~I4r5XX~s6-ARERTc*e9W4>DK&ns=94uYdNNuqoEK3Q+X*LQ4|}o;%E<>{rZzjt^uIfuU(cm$c6ZuT6LchGjx~DfO?kfWy+X+I zyxRDJsTw}~*T?sEvd{k;HSbB{2cYt!(u1EHj({WJ2si?cfFs}tI0BAKpsnxfWLH(rlBXuH$e zY;+FLMtrbX8Fg6@)Q$JMXzMLpd+qjUy^2wuw@fbB-h4V(@uHv6x9C&UiB_U8{2l%X zzlNW~!|+)c1iyms!IxkkL~b|&j({WJ2si?cfFs}tI0BA^a9D792DtnE1~O39EgQRIYcbR=<3j)fd7Oo2md z?9d^YEoquVWWlM1&JaSUSoM1X%k*Ic(&kwDnkp@6&uEgO9!>_PKuR#9so=}uKRIj@ zsRYLi4Y)XFB+Vc;<~W1S6Vgzn(j=b?z@c%Pbi0ML5X_zp!@D&`AZMkbYVJnen~8^* z8S+MepG1@=%qTh}C=q#XAZ2J}=dPd0nv!5RWmZOMOu6lM&yrNLlOTIuMe7h%090fY z#4IEo;-tKH4a_ep^O~V7t@50Tsta3pk61`i-ly zTm#{u6^;Pla~$m#0!$+lVmF`Nl#hLI*s%_qxGInC6SOX@IfLz%+Z-6?nStv60ov#= ATmS$7 literal 0 HcmV?d00001 diff --git a/plugins/template/Examples/remote-security-report.sh b/plugins/template/Examples/remote-security-report.sh new file mode 100755 index 000000000..af063b584 --- /dev/null +++ b/plugins/template/Examples/remote-security-report.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# remote-security-report.sh +# Usage: remote-security-report.sh cert host [report_name] + +cert_path="$1" +host="$2" +report_name="${3:-report}" +temp_file="/tmp/security-report-${report_name}.txt" + +# Copy the security report script to remote host +scp -i "$cert_path" /usr/local/bin/security-report.sh "${host}:~/security-report.sh" >&2 + +# Make it executable and run it on remote host +ssh -i "$cert_path" "$host" "chmod +x ~/security-report.sh && sudo ~/security-report.sh ${temp_file}" >&2 + +# Copy the report back +scp -i "$cert_path" "${host}:${temp_file}" "${temp_file}" >&2 + +# Cleanup remote files +ssh -i "$cert_path" "$host" "rm ~/security-report.sh ${temp_file}" >&2 + +# Output the local file path for fabric to read +echo "${temp_file}" + diff --git a/plugins/template/Examples/remote-security-report.yaml b/plugins/template/Examples/remote-security-report.yaml new file mode 100644 index 000000000..bfe02d096 --- /dev/null +++ b/plugins/template/Examples/remote-security-report.yaml @@ -0,0 +1,17 @@ +name: "remote-security" +executable: "/usr/local/bin/remote-security-report.sh" +type: "executable" +timeout: "60s" +description: "Generate security report from remote system" + +operations: + report: + cmd_template: "{{executable}} {{1}} {{2}} {{3}}" + +config: + output: + method: "file" + file_config: + cleanup: true + path_from_stdout: true + work_dir: "/tmp" diff --git a/plugins/template/Examples/security-report.sh b/plugins/template/Examples/security-report.sh new file mode 100755 index 000000000..2cd7e497e --- /dev/null +++ b/plugins/template/Examples/security-report.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +# security-report.sh - Enhanced system security information collection +# Usage: security-report.sh [output_file] + +output_file=${1:-/tmp/security-report.txt} + +{ + echo "=== System Security Report ===" + echo "Generated: $(date)" + echo "Hostname: $(hostname)" + echo "Kernel: $(uname -r)" + echo + + echo "=== System Updates ===" + echo "Last update: $(stat -c %y /var/cache/apt/pkgcache.bin | cut -d' ' -f1)" + echo "Pending updates:" + apt list --upgradable 2>/dev/null + + echo -e "\n=== Security Updates ===" + echo "Pending security updates:" + apt list --upgradable 2>/dev/null | grep -i security + + echo -e "\n=== User Accounts ===" + echo "Users with login shells:" + grep -v '/nologin\|/false' /etc/passwd + echo -e "\nUsers who can login:" + awk -F: '$2!="*" && $2!="!" {print $1}' /etc/shadow + echo -e "\nUsers with empty passwords:" + awk -F: '$2=="" {print $1}' /etc/shadow + echo -e "\nUsers with UID 0:" + awk -F: '$3==0 {print $1}' /etc/passwd + + echo -e "\n=== Sudo Configuration ===" + echo "Users/groups with sudo privileges:" + grep -h '^[^#]' /etc/sudoers.d/* /etc/sudoers 2>/dev/null + echo -e "\nUsers with passwordless sudo:" + grep -h NOPASSWD /etc/sudoers.d/* /etc/sudoers 2>/dev/null + + echo -e "\n=== SSH Configuration ===" + if [ -f /etc/ssh/sshd_config ]; then + echo "Key SSH settings:" + grep -E '^(PermitRootLogin|PasswordAuthentication|Port|Protocol|X11Forwarding|MaxAuthTries|PermitEmptyPasswords)' /etc/ssh/sshd_config + fi + + echo -e "\n=== SSH Keys ===" + echo "Authorized keys found:" + find /home -name "authorized_keys" -ls 2>/dev/null + + echo -e "\n=== Firewall Status ===" + echo "UFW Status:" + ufw status verbose + echo -e "\nIPTables Rules:" + iptables -L -n + + echo -e "\n=== Network Services ===" + echo "Listening services (port - process):" + netstat -tlpn 2>/dev/null | grep LISTEN + + echo -e "\n=== Recent Authentication Failures ===" + echo "Last 5 failed SSH attempts:" + grep "Failed password" /var/log/auth.log | tail -5 + + echo -e "\n=== File Permissions ===" + echo "World-writable files in /etc:" + find /etc -type f -perm -002 -ls 2>/dev/null + echo -e "\nWorld-writable directories in /etc:" + find /etc -type d -perm -002 -ls 2>/dev/null + + echo -e "\n=== System Resource Usage ===" + echo "Disk Usage:" + df -h + echo -e "\nMemory Usage:" + free -h + echo -e "\nTop 5 CPU-using processes:" + ps aux --sort=-%cpu | head -6 + + echo -e "\n=== System Timers ===" + echo "Active timers (potential scheduled tasks):" + systemctl list-timers --all + + echo -e "\n=== Important Service Status ===" + for service in ssh ufw apparmor fail2ban clamav-freshclam; do + echo "Status of $service:" + systemctl status $service --no-pager 2>/dev/null + done + + echo -e "\n=== Fail2Ban Logs ===" + echo "Recent Fail2Ban activity (fail2ban.log):" + if [ -f /var/log/fail2ban.log ]; then + echo "=== Current log (fail2ban.log) ===" + cat /var/log/fail2ban.log + else + echo "fail2ban.log not found" + fi + + if [ -f /var/log/fail2ban.log.1 ]; then + echo -e "\n=== Previous log (fail2ban.log.1) ===" + cat /var/log/fail2ban.log.1 + else + echo -e "\nfail2ban.log.1 not found" + fi + + echo -e "\n=== Fail2Ban Status ===" + echo "Currently banned IPs:" + sudo fail2ban-client status + + +} > "$output_file" + +# Output the file path for fabric to read +echo "$output_file" + diff --git a/plugins/template/Examples/security-report.yaml b/plugins/template/Examples/security-report.yaml new file mode 100644 index 000000000..bb050e4f3 --- /dev/null +++ b/plugins/template/Examples/security-report.yaml @@ -0,0 +1,18 @@ +name: "security-report" +executable: "/usr/local/bin/security-report.sh" +type: "executable" +timeout: "30s" +description: "Generate system security report" +version: "1.0.0" + +operations: + generate: + cmd_template: "{{executable}} /tmp/security-report-{{1}}.txt" + +config: + output: + method: "file" + file_config: + cleanup: true + path_from_stdout: true + work_dir: "/tmp" diff --git a/plugins/template/Examples/sqlite3_demo.yaml b/plugins/template/Examples/sqlite3_demo.yaml new file mode 100644 index 000000000..0faaaa27a --- /dev/null +++ b/plugins/template/Examples/sqlite3_demo.yaml @@ -0,0 +1,23 @@ +name: memory-query +executable: /usr/bin/sqlite3 +type: executable +timeout: "5s" +description: "Query memories database" +version: "1.0.0" +env: [] + +operations: + goal: + cmd_template: "{{executable}} -json /home/matt/memories.db \"select * from memories where type= 'goal'\"" + value: + cmd_template: "{{executable}} -json /home/matt/memories.db \"select * from memories where type= 'value'\"" + project: + cmd_template: "{{executable}} -json /home/matt/memories.db \"select * from memories where type= 'project'\"" + byid: + cmd_template: "{{executable}} -json /home/matt/memories.db \"select * from memories where uid= {{value}}\"" + all: + cmd_template: "{{executable}} -json ~/memories.db \"select * from memories\"" + +config: + output: + method: stdout diff --git a/plugins/template/Examples/track_packages.sh b/plugins/template/Examples/track_packages.sh new file mode 100755 index 000000000..6970d44c5 --- /dev/null +++ b/plugins/template/Examples/track_packages.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +LOG_DIR="/var/log/package_tracking" +DATE=$(date +%Y%m%d) + +# Ensure directory exists +mkdir -p "$LOG_DIR" + +# Current package list +dpkg -l > "$LOG_DIR/packages_current.list" + +# Create diff if previous exists +if [ -f "$LOG_DIR/packages_previous.list" ]; then + diff "$LOG_DIR/packages_previous.list" "$LOG_DIR/packages_current.list" > "$LOG_DIR/changes_current.diff" +fi + +# Keep copy for next comparison +cp "$LOG_DIR/packages_current.list" "$LOG_DIR/packages_previous.list" diff --git a/plugins/template/Examples/word-generator.py b/plugins/template/Examples/word-generator.py new file mode 100755 index 000000000..eb33c5173 --- /dev/null +++ b/plugins/template/Examples/word-generator.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +import sys +import json +import random + +# A small set of words for demonstration! +WORD_LIST = [ + "apple", "banana", "cherry", "date", "elderberry", + "fig", "grape", "honeydew", "kiwi", "lemon", + "mango", "nectarine", "orange", "papaya", "quince", + "raspberry", "strawberry", "tangerine", "ugli", "watermelon" +] + +def generate_words(count): + try: + count = int(count) + if count < 1: + return json.dumps({"error": "Count must be positive"}) + + # Generate random words + words = random.sample(WORD_LIST, min(count, len(WORD_LIST))) + + # Return JSON formatted result + return json.dumps({ + "words": words, + "count": len(words) + }) + except ValueError: + return json.dumps({"error": "Invalid count parameter"}) + +if __name__ == "__main__": + if len(sys.argv) != 2: + print(json.dumps({"error": "Exactly one argument required"})) + sys.exit(1) + + print(generate_words(sys.argv[1])) diff --git a/plugins/template/Examples/word-generator.yaml b/plugins/template/Examples/word-generator.yaml new file mode 100644 index 000000000..a283b25d4 --- /dev/null +++ b/plugins/template/Examples/word-generator.yaml @@ -0,0 +1,16 @@ +name: word-generator +executable: /usr/local/bin/word-generator.py +type: executable +timeout: "5s" +description: "Generates random words based on count parameter" +version: "1.0.0" +env: [] + +operations: + generate: + cmd_template: "{{executable}} {{value}}" + +config: + output: + method: stdout + From ca55f2375db94330734e20910831baa5c45fd5c6 Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Thu, 5 Dec 2024 22:13:17 +1100 Subject: [PATCH 10/14] actually added tutorial --- plugins/template/Examples/README.md | 234 ++++++++++++++++++++++++++++ 1 file changed, 234 insertions(+) diff --git a/plugins/template/Examples/README.md b/plugins/template/Examples/README.md index e69de29bb..c8c9b08ac 100644 --- a/plugins/template/Examples/README.md +++ b/plugins/template/Examples/README.md @@ -0,0 +1,234 @@ +Let me create a restructured tutorial that builds from simple to complex examples. + + + + + +# Fabric Extensions: Complete Guide + +## Understanding Extension Architecture + +### Registry Structure +The extension registry is stored at `~/.config/fabric/extensions/extensions.yaml` and tracks registered extensions: + +```yaml +extensions: + extension-name: + config_path: /path/to/config.yaml + config_hash: + executable_hash: +``` + +The registry maintains security through hash verification of both configs and executables. + +### Extension Configuration +Each extension requires a YAML configuration file with the following structure: + +```yaml +name: "extension-name" # Unique identifier +executable: "/path/to/binary" # Full path to executable +type: "executable" # Type of extension +timeout: "30s" # Execution timeout +description: "Description" # What the extension does +version: "1.0.0" # Version number +env: [] # Optional environment variables + +operations: # Defined operations + operation-name: + cmd_template: "{{executable}} {{operation}} {{value}}" + +config: # Output configuration + output: + method: "stdout" # or "file" + file_config: # Optional, for file output + cleanup: true + path_from_stdout: true + work_dir: "/tmp" +``` + +### Directory Structure +Recommended organization: +``` +~/.config/fabric/extensions/ +├── bin/ # Extension executables +├── configs/ # Extension YAML configs +└── extensions.yaml # Registry file +``` + +## Example 1: Python Wrapper (Word Generator) +A simple example wrapping a Python script. + +### 1. Position Files +```bash +# Create directories +mkdir -p ~/.config/fabric/extensions/{bin,configs} + +# Install script +cp word-generator.py ~/.config/fabric/extensions/bin/ +chmod +x ~/.config/fabric/extensions/bin/word-generator.py +``` + +### 2. Configure +Create `~/.config/fabric/extensions/configs/word-generator.yaml`: +```yaml +name: word-generator +executable: "~/.config/fabric/extensions/bin/word-generator.py" +type: executable +timeout: "5s" +description: "Generates random words based on count parameter" +version: "1.0.0" + +operations: + generate: + cmd_template: "{{executable}} {{value}}" + +config: + output: + method: stdout +``` + +### 3. Register & Run +```bash +# Register +fabric --addextension ~/.config/fabric/extensions/configs/word-generator.yaml + +# Run (generate 3 random words) +fabric -p "{{ext:word-generator:generate:3}}" +``` + +## Example 2: Direct Executable (SQLite3) +Using a system executable directly. + +### 1. Configure +Create `~/.config/fabric/extensions/configs/memory-query.yaml`: +```yaml +name: memory-query +executable: "/usr/bin/sqlite3" +type: executable +timeout: "5s" +description: "Query memories database" +version: "1.0.0" + +operations: + goal: + cmd_template: "{{executable}} -json ~/memories.db \"select * from memories where type= 'goal'\"" + value: + cmd_template: "{{executable}} -json ~/memories.db \"select * from memories where type= 'value'\"" + byid: + cmd_template: "{{executable}} -json ~/memories.db \"select * from memories where uid= {{value}}\"" + all: + cmd_template: "{{executable}} -json ~/memories.db \"select * from memories\"" + +config: + output: + method: stdout +``` + +### 2. Register & Run +```bash +# Register +fabric --addextension ~/.config/fabric/extensions/configs/memory-query.yaml + +# Run queries +fabric -p "{{ext:memory-query:all}}" +fabric -p "{{ext:memory-query:byid:123}}" +``` + +## Example 3: Local Shell Script (Package Tracker) +Running a local system administration script. + +### 1. Position Files +```bash +# Install script +sudo cp track_packages.sh ~/.config/fabric/extensions/bin/ +sudo chmod +x ~/.config/fabric/extensions/bin/track_packages.sh +``` + +### 2. Configure +Create `~/.config/fabric/extensions/configs/package-tracker.yaml`: +```yaml +name: package-tracker +executable: "~/.config/fabric/extensions/bin/track_packages.sh" +type: executable +timeout: "30s" +description: "Track system package changes" +version: "1.0.0" + +operations: + track: + cmd_template: "{{executable}}" + +config: + output: + method: stdout +``` + +### 3. Register & Run +```bash +# Register +fabric --addextension ~/.config/fabric/extensions/configs/package-tracker.yaml + +# Run +fabric -p "{{ext:package-tracker:track}}" +``` + +## Extension Management Commands + +### List Extensions +```bash +fabric --listextensions +``` +Shows all registered extensions with their status and configuration details. + +### Remove Extension +```bash +fabric --rmextension +``` +Removes an extension from the registry. + +## Security Considerations + +1. **Hash Verification** + - Both configs and executables are verified via SHA-256 hashes + - Changes to either require re-registration + - Prevents tampering with registered extensions + +2. **Execution Safety** + - Extensions run with user permissions + - Timeout constraints prevent runaway processes + - Environment variables can be controlled via config + +3. **Best Practices** + - Review extension code before installation + - Keep executables in protected directories + - Use absolute paths in configurations + - Implement proper error handling in scripts + - Regular security audits of registered extensions + +## Troubleshooting + +### Common Issues +1. **Registration Failures** + - Verify file permissions + - Check executable paths + - Validate YAML syntax + +2. **Execution Errors** + - Check operation exists in config + - Verify timeout settings + - Monitor system resources + - Check extension logs + +3. **Output Issues** + - Verify output method configuration + - Check file permissions for file output + - Monitor disk space for file operations + +### Debug Tips +1. Enable verbose logging when available +2. Check system logs for execution errors +3. Verify extension dependencies +4. Test extensions with minimal configurations first + + +Would you like me to expand on any particular section or add more examples? \ No newline at end of file From 373c1d0858f11639caac7c1009128a993e78eec9 Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Thu, 5 Dec 2024 22:28:38 +1100 Subject: [PATCH 11/14] added test pattern --- plugins/template/Examples/README.md | 68 +++++++++++------------ plugins/template/Examples/test_pattern.md | 8 +++ 2 files changed, 39 insertions(+), 37 deletions(-) create mode 100644 plugins/template/Examples/test_pattern.md diff --git a/plugins/template/Examples/README.md b/plugins/template/Examples/README.md index c8c9b08ac..e32e0005e 100644 --- a/plugins/template/Examples/README.md +++ b/plugins/template/Examples/README.md @@ -93,12 +93,15 @@ config: fabric --addextension ~/.config/fabric/extensions/configs/word-generator.yaml # Run (generate 3 random words) -fabric -p "{{ext:word-generator:generate:3}}" +echo "{{ext:word-generator:generate:3}}" | fabric ``` ## Example 2: Direct Executable (SQLite3) Using a system executable directly. +copy the memories to your home directory + ~/memories.db + ### 1. Configure Create `~/.config/fabric/extensions/configs/memory-query.yaml`: ```yaml @@ -130,49 +133,21 @@ config: fabric --addextension ~/.config/fabric/extensions/configs/memory-query.yaml # Run queries -fabric -p "{{ext:memory-query:all}}" -fabric -p "{{ext:memory-query:byid:123}}" +echo "{{ext:memory-query:all}}" | fabric +echo "{{ext:memory-query:byid:3}}" | fabric ``` -## Example 3: Local Shell Script (Package Tracker) -Running a local system administration script. - -### 1. Position Files -```bash -# Install script -sudo cp track_packages.sh ~/.config/fabric/extensions/bin/ -sudo chmod +x ~/.config/fabric/extensions/bin/track_packages.sh -``` - -### 2. Configure -Create `~/.config/fabric/extensions/configs/package-tracker.yaml`: -```yaml -name: package-tracker -executable: "~/.config/fabric/extensions/bin/track_packages.sh" -type: executable -timeout: "30s" -description: "Track system package changes" -version: "1.0.0" - -operations: - track: - cmd_template: "{{executable}}" -config: - output: - method: stdout -``` +## Extension Management Commands -### 3. Register & Run +### Add Extension ```bash -# Register -fabric --addextension ~/.config/fabric/extensions/configs/package-tracker.yaml - -# Run -fabric -p "{{ext:package-tracker:track}}" +fabric --addextension ~/.config/fabric/extensions/configs/memory-query.yaml ``` -## Extension Management Commands +Note : if the executable or config file changes, you must re-add the extension. +This will recompute the hash for the extension. + ### List Extensions ```bash @@ -186,6 +161,25 @@ fabric --rmextension ``` Removes an extension from the registry. + +## Extensions in patterns + +``` +Create a pattern that use multiple extensions. + +These are my favorite +{{ext:word-generator:generate:3}} + +These are my least favorite +{{ext:word-generator:generate:2}} + +what does this say about me? +``` + +```bash +./fabric -p ./plugins/template/Examples/test_pattern.md +``` + ## Security Considerations 1. **Hash Verification** diff --git a/plugins/template/Examples/test_pattern.md b/plugins/template/Examples/test_pattern.md new file mode 100644 index 000000000..d3de195d1 --- /dev/null +++ b/plugins/template/Examples/test_pattern.md @@ -0,0 +1,8 @@ +These are my favorite +{{ext:word-generator:generate:3}} + +These are my least favorite +{{ext:word-generator:generate:2}} + +what does this say about me? + From ad561248fde806c3f5a265a85e82d5c18af472df Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Wed, 25 Dec 2024 09:41:49 +1100 Subject: [PATCH 12/14] fix:properly instatiated extensionManager var fix:added timeout value validation --- plugins/template/extension_manager.go | 5 +++++ plugins/template/template.go | 1 + 2 files changed, 6 insertions(+) diff --git a/plugins/template/extension_manager.go b/plugins/template/extension_manager.go index aa3a5b630..131c988da 100644 --- a/plugins/template/extension_manager.go +++ b/plugins/template/extension_manager.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "time" "gopkg.in/yaml.v3" ) @@ -90,6 +91,10 @@ func (em *ExtensionManager) RegisterExtension(configPath string) error { return fmt.Errorf("failed to register extension: %w", err) } + if _, err := time.ParseDuration(ext.Timeout); err != nil { + return fmt.Errorf("invalid timeout value '%s': must be a duration like '30s' or '1m': %w", ext.Timeout, err) +} + // Print success message with extension details fmt.Printf("Successfully registered extension:\n") fmt.Printf("Name: %s\n", ext.Name) diff --git a/plugins/template/template.go b/plugins/template/template.go index 12231317a..09e3690e9 100644 --- a/plugins/template/template.go +++ b/plugins/template/template.go @@ -17,6 +17,7 @@ var ( Debug = false // Debug flag ) +var extensionManager *ExtensionManager func init() { homedir, err := os.UserHomeDir() From a5929fcad683d3ff87ab5af27f4e26e5f91b9708 Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Wed, 25 Dec 2024 10:20:26 +1100 Subject: [PATCH 13/14] Fix tests to handle NewPluginRegistry returning error --- core/plugin_registry_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/core/plugin_registry_test.go b/core/plugin_registry_test.go index 76f838260..f0b000005 100644 --- a/core/plugin_registry_test.go +++ b/core/plugin_registry_test.go @@ -1,15 +1,20 @@ package core import ( - "github.com/danielmiessler/fabric/plugins/db/fsdb" "os" "testing" + + "github.com/danielmiessler/fabric/plugins/db/fsdb" ) func TestSaveEnvFile(t *testing.T) { - registry := NewPluginRegistry(fsdb.NewDb(os.TempDir())) + db := fsdb.NewDb(os.TempDir()) + registry, err := NewPluginRegistry(db) + if err != nil { + t.Fatalf("NewPluginRegistry() error = %v", err) + } - err := registry.SaveEnvFile() + err = registry.SaveEnvFile() if err != nil { t.Fatalf("SaveEnvFile() error = %v", err) } From 7a260124575dcf1c2e55b2fb8a86e12ad76d56a0 Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Wed, 25 Dec 2024 10:22:52 +1100 Subject: [PATCH 14/14] Updated extension readme --- plugins/template/Examples/README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/plugins/template/Examples/README.md b/plugins/template/Examples/README.md index e32e0005e..307314016 100644 --- a/plugins/template/Examples/README.md +++ b/plugins/template/Examples/README.md @@ -1,8 +1,3 @@ -Let me create a restructured tutorial that builds from simple to complex examples. - - - - # Fabric Extensions: Complete Guide