From 3264ed1dda24d1d584919e2541d47116dda5e39a Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 4 Apr 2023 16:26:17 +0200 Subject: [PATCH] Refactor profiles to decouple them from stack resources (#1140) Changes included here: * New `elastic-package profile use` command to select profile to use. * Configuration files are not rewritten on upgrade. * Stack and test runner files are managed by their own providers. * Stack and test runner files are only written when the services are started, and only if they are different. * Stack files are templates now. Same files are used for multiple versions. Runtime configuration can be passed now as data to templates instead of requiring environment variables. * Support for multiple stack providers. --- README.md | 8 +- cmd/profiles.go | 68 +++- cmd/stack.go | 166 ++++---- go.mod | 1 + go.sum | 2 + internal/cobraext/flags.go | 3 + internal/configuration/locations/locations.go | 30 +- internal/install/application_configuration.go | 58 ++- .../install/application_configuration_yml.go | 33 -- internal/install/install.go | 184 ++------- internal/install/static.go | 34 -- .../profile/_static/elastic-agent_default.env | 4 - .../_static/elasticsearch_config_8x.yml | 15 - internal/profile/_static/kibana_config_80.yml | 59 --- internal/profile/_static/kibana_config_86.yml | 61 --- .../profile/_static/kibana_config_default.yml | 21 - internal/profile/config_profile_test.go | 82 ---- internal/profile/files.go | 63 --- internal/profile/install_profile.go | 229 ----------- internal/profile/profile.go | 370 +++++++----------- internal/profile/profile_json.go | 50 ++- .../profile/{options.go => profile_test.go} | 18 +- internal/profile/static.go | 220 ----------- .../_static/Dockerfile.package-registry.tmpl} | 7 +- .../_static/GeoLite2-ASN.mmdb | Bin .../_static/GeoLite2-City.mmdb | Bin .../_static/GeoLite2-Country.mmdb | Bin .../_static/docker-compose-stack.yml.tmpl} | 20 +- .../_static/elastic-agent.env.tmpl} | 5 +- .../_static/elasticsearch.yml.tmpl} | 8 +- .../_static/kibana.yml.tmpl} | 21 +- .../_static/kibana_healthcheck.sh.tmpl} | 2 +- .../_static/package-registry.yml} | 0 .../{install => stack}/_static/service_tokens | 0 internal/stack/boot.go | 28 +- internal/{profile => stack}/certs.go | 64 +-- internal/{profile => stack}/certs_test.go | 20 +- internal/stack/compose.go | 10 +- internal/stack/config.go | 92 +++++ internal/stack/dump.go | 2 +- internal/stack/initconfig.go | 65 +-- internal/stack/options.go | 1 + internal/stack/providers.go | 79 ++++ internal/stack/resources.go | 163 ++++++++ internal/stack/shellinit.go | 7 +- internal/stack/update.go | 8 +- .../_static/Dockerfile.terraform_deployer | 0 .../_static/docker-custom-agent-base.yml | 0 .../_static/terraform_deployer.yml | 0 .../_static/terraform_deployer_run.sh | 0 .../runners/system/servicedeployer/compose.go | 20 +- .../system/servicedeployer/custom_agent.go | 48 ++- .../system/servicedeployer/terraform.go | 87 +++- 53 files changed, 1030 insertions(+), 1506 deletions(-) delete mode 100644 internal/install/application_configuration_yml.go delete mode 100644 internal/install/static.go delete mode 100644 internal/profile/_static/elastic-agent_default.env delete mode 100644 internal/profile/_static/elasticsearch_config_8x.yml delete mode 100644 internal/profile/_static/kibana_config_80.yml delete mode 100644 internal/profile/_static/kibana_config_86.yml delete mode 100644 internal/profile/_static/kibana_config_default.yml delete mode 100644 internal/profile/config_profile_test.go delete mode 100644 internal/profile/files.go delete mode 100644 internal/profile/install_profile.go rename internal/profile/{options.go => profile_test.go} (54%) delete mode 100644 internal/profile/static.go rename internal/{profile/_static/Dockerfile.package-registry => stack/_static/Dockerfile.package-registry.tmpl} (50%) rename internal/{install => stack}/_static/GeoLite2-ASN.mmdb (100%) rename internal/{install => stack}/_static/GeoLite2-City.mmdb (100%) rename internal/{install => stack}/_static/GeoLite2-Country.mmdb (100%) rename internal/{profile/_static/docker-compose-stack.yml => stack/_static/docker-compose-stack.yml.tmpl} (84%) rename internal/{profile/_static/elastic-agent_8x.env => stack/_static/elastic-agent.env.tmpl} (64%) rename internal/{profile/_static/elasticsearch_config_default.yml => stack/_static/elasticsearch.yml.tmpl} (86%) rename internal/{profile/_static/kibana_config_8x.yml => stack/_static/kibana.yml.tmpl} (79%) rename internal/{install/_static/kibana_healthcheck.sh => stack/_static/kibana_healthcheck.sh.tmpl} (64%) rename internal/{profile/_static/package_registry.yml => stack/_static/package-registry.yml} (100%) rename internal/{install => stack}/_static/service_tokens (100%) rename internal/{profile => stack}/certs.go (69%) rename internal/{profile => stack}/certs_test.go (81%) create mode 100644 internal/stack/config.go create mode 100644 internal/stack/providers.go create mode 100644 internal/stack/resources.go rename internal/{install => testrunner/runners/system/servicedeployer}/_static/Dockerfile.terraform_deployer (100%) rename internal/{install => testrunner/runners/system/servicedeployer}/_static/docker-custom-agent-base.yml (100%) rename internal/{install => testrunner/runners/system/servicedeployer}/_static/terraform_deployer.yml (100%) rename internal/{install => testrunner/runners/system/servicedeployer}/_static/terraform_deployer_run.sh (100%) diff --git a/README.md b/README.md index d32be77c4..1ef5f5478 100644 --- a/README.md +++ b/README.md @@ -287,6 +287,12 @@ _Context: global_ +### `elastic-package profiles use` + +_Context: global_ + + + ### `elastic-package promote` _Context: global_ @@ -383,7 +389,7 @@ By default the latest released version of the stack is spun up but it is possibl Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. -To ęxpose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. +To expose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. For details on how to connect the service with the Elastic stack, see the [service command](https://github.com/elastic/elastic-package/blob/main/README.md#elastic-package-service). diff --git a/cmd/profiles.go b/cmd/profiles.go index 957301ecb..82b4174b6 100644 --- a/cmd/profiles.go +++ b/cmd/profiles.go @@ -16,7 +16,7 @@ import ( "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/configuration/locations" - "github.com/elastic/elastic-package/internal/environment" + "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/profile" ) @@ -26,9 +26,6 @@ const jsonFormat = "json" // tableFormat is the format for table output const tableFormat = "table" -// profileNameEnvVar is the name of the environment variable to set the default profile -var profileNameEnvVar = environment.WithElasticPackagePrefix("PROFILE") - func setupProfilesCommand() *cobraext.Command { profilesLongDescription := `Use this command to add, remove, and manage multiple config profiles. @@ -65,12 +62,16 @@ User profiles are not overwritten on upgrade of elastic-stack, and can be freely return errors.Wrapf(err, "error creating profile %s from profile %s", newProfileName, fromName) } - fmt.Printf("Created profile %s from %s.\n", newProfileName, fromName) + if fromName == "" { + fmt.Printf("Created profile %s.\n", newProfileName) + } else { + fmt.Printf("Created profile %s from %s.\n", newProfileName, fromName) + } return nil }, } - profileNewCommand.Flags().String(cobraext.ProfileFromFlagName, "default", cobraext.ProfileFromFlagDescription) + profileNewCommand.Flags().String(cobraext.ProfileFromFlagName, "", cobraext.ProfileFromFlagDescription) profileDeleteCommand := &cobra.Command{ Use: "delete", @@ -104,10 +105,14 @@ User profiles are not overwritten on upgrade of elastic-stack, and can be freely if err != nil { return errors.Wrap(err, "error listing all profiles") } + if len(profileList) == 0 { + fmt.Println("There are no profiles yet.") + return nil + } format, err := cmd.Flags().GetString(cobraext.ProfileFormatFlagName) if err != nil { - return cobraext.FlagParsingError(err, cobraext.ProfileFromFlagName) + return cobraext.FlagParsingError(err, cobraext.ProfileFormatFlagName) } switch format { @@ -122,7 +127,45 @@ User profiles are not overwritten on upgrade of elastic-stack, and can be freely } profileListCommand.Flags().String(cobraext.ProfileFormatFlagName, tableFormat, cobraext.ProfileFormatFlagDescription) - profileCommand.AddCommand(profileNewCommand, profileDeleteCommand, profileListCommand) + profileUseCommand := &cobra.Command{ + Use: "use", + Short: "Sets the profile to use when no other is specified", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return errors.New("use requires an argument") + } + profileName := args[0] + + _, err := profile.LoadProfile(profileName) + if err != nil { + return fmt.Errorf("cannot use profile %q: %v", profileName, err) + } + + location, err := locations.NewLocationManager() + if err != nil { + return fmt.Errorf("error fetching profile: %w", err) + } + + config, err := install.Configuration() + if err != nil { + return fmt.Errorf("failed to load current configuration: %w", err) + } + config.SetCurrentProfile(profileName) + + err = install.WriteConfigFile(location, config) + if err != nil { + return fmt.Errorf("failed to store configuration: %w", err) + } + return nil + }, + } + + profileCommand.AddCommand( + profileNewCommand, + profileDeleteCommand, + profileListCommand, + profileUseCommand, + ) return cobraext.NewCommand(profileCommand, cobraext.ContextGlobal) } @@ -175,15 +218,6 @@ func profileToList(profiles []profile.Metadata) [][]string { return profileList } -func lookupEnv() string { - env := os.Getenv(profileNameEnvVar) - if env == "" { - return profile.DefaultProfile - } - return env - -} - func availableProfilesAsAList() ([]string, error) { loc, err := locations.NewLocationManager() if err != nil { diff --git a/cmd/stack.go b/cmd/stack.go index c96193111..4039ad32b 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -39,7 +39,7 @@ By default the latest released version of the stack is spun up but it is possibl Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. -To ęxpose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. +To expose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. For details on how to connect the service with the Elastic stack, see the [service command](https://github.com/elastic/elastic-package/blob/main/README.md#elastic-package-service).` @@ -73,37 +73,24 @@ func setupStackCommand() *cobraext.Command { return cobraext.FlagParsingError(err, cobraext.StackVersionFlagName) } - profileName, err := cmd.Flags().GetString(cobraext.ProfileFlagName) + profile, err := getProfileFlag(cmd) if err != nil { - return cobraext.FlagParsingError(err, cobraext.ProfileFlagName) - } - - userProfile, err := profile.LoadProfile(profileName) - if errors.Is(err, profile.ErrNotAProfile) { - pList, err := availableProfilesAsAList() - if err != nil { - return errors.Wrap(err, "error listing known profiles") - } - return fmt.Errorf("%s is not a valid profile, known profiles are: %s", profileName, pList) - } - if err != nil { - return errors.Wrap(err, "error loading profile") + return err } - // Print information before starting the stack, for cases where - // this is executed in the foreground, without daemon mode. - cmd.Printf("Using profile %s.\n", userProfile.ProfilePath) - cmd.Println(`Remember to load stack environment variables using 'eval "$(elastic-package stack shellinit)"'.`) - err = printInitConfig(cmd, userProfile) + provider, err := getProviderFromProfile(cmd, profile, true) if err != nil { return err } - err = stack.BootUp(stack.Options{ + cmd.Printf("Using profile %s.\n", profile.ProfilePath) + cmd.Println(`Remember to load stack environment variables using 'eval "$(elastic-package stack shellinit)"'.`) + err = provider.BootUp(stack.Options{ DaemonMode: daemonMode, StackVersion: stackVersion, Services: services, - Profile: userProfile, + Profile: profile, + Printer: cmd, }) if err != nil { return errors.Wrap(err, "booting up the stack failed") @@ -117,6 +104,7 @@ func setupStackCommand() *cobraext.Command { upCommand.Flags().StringSliceP(cobraext.StackServicesFlagName, "s", nil, fmt.Sprintf(cobraext.StackServicesFlagDescription, strings.Join(availableServicesAsList(), ","))) upCommand.Flags().StringP(cobraext.StackVersionFlagName, "", install.DefaultStackVersion, cobraext.StackVersionFlagDescription) + upCommand.Flags().String(cobraext.StackProviderFlagName, "", fmt.Sprintf(cobraext.StackProviderFlagDescription, strings.Join(stack.SupportedProviders, ", "))) downCommand := &cobra.Command{ Use: "down", @@ -124,26 +112,19 @@ func setupStackCommand() *cobraext.Command { RunE: func(cmd *cobra.Command, args []string) error { cmd.Println("Take down the Elastic stack") - profileName, err := cmd.Flags().GetString(cobraext.ProfileFlagName) + profile, err := getProfileFlag(cmd) if err != nil { - return cobraext.FlagParsingError(err, cobraext.ProfileFlagName) - } - - userProfile, err := profile.LoadProfile(profileName) - if errors.Is(err, profile.ErrNotAProfile) { - pList, err := availableProfilesAsAList() - if err != nil { - return errors.Wrap(err, "error listing known profiles") - } - return fmt.Errorf("%s is not a valid profile, known profiles are: %s", profileName, pList) + return err } + provider, err := getProviderFromProfile(cmd, profile, false) if err != nil { - return errors.Wrap(err, "error loading profile") + return err } - err = stack.TearDown(stack.Options{ - Profile: userProfile, + err = provider.TearDown(stack.Options{ + Profile: profile, + Printer: cmd, }) if err != nil { return errors.Wrap(err, "tearing down the stack failed") @@ -160,14 +141,14 @@ func setupStackCommand() *cobraext.Command { RunE: func(cmd *cobra.Command, args []string) error { cmd.Println("Update the Elastic stack") - profileName, err := cmd.Flags().GetString(cobraext.ProfileFlagName) + profile, err := getProfileFlag(cmd) if err != nil { - return cobraext.FlagParsingError(err, cobraext.ProfileFlagName) + return err } - profile, err := profile.LoadProfile(profileName) + provider, err := getProviderFromProfile(cmd, profile, false) if err != nil { - return errors.Wrap(err, "error loading profile") + return err } stackVersion, err := cmd.Flags().GetString(cobraext.StackVersionFlagName) @@ -175,9 +156,10 @@ func setupStackCommand() *cobraext.Command { return cobraext.FlagParsingError(err, cobraext.StackVersionFlagName) } - err = stack.Update(stack.Options{ + err = provider.Update(stack.Options{ StackVersion: stackVersion, Profile: profile, + Printer: cmd, }) if err != nil { return errors.Wrap(err, "failed updating the stack images") @@ -193,11 +175,6 @@ func setupStackCommand() *cobraext.Command { Use: "shellinit", Short: "Export environment variables", RunE: func(cmd *cobra.Command, args []string) error { - profileName, err := cmd.Flags().GetString(cobraext.ProfileFlagName) - if err != nil { - return cobraext.FlagParsingError(err, cobraext.ProfileFlagName) - } - shellName, err := cmd.Flags().GetString(cobraext.ShellInitShellFlagName) if err != nil { return cobraext.FlagParsingError(err, cobraext.ShellInitShellFlagName) @@ -207,9 +184,9 @@ func setupStackCommand() *cobraext.Command { fmt.Fprintf(cmd.OutOrStderr(), "Detected shell: %s\n", shellName) } - profile, err := profile.LoadProfile(profileName) + profile, err := getProfileFlag(cmd) if err != nil { - return errors.Wrap(err, "error loading profile") + return err } shellCode, err := stack.ShellInit(profile, shellName) @@ -232,17 +209,17 @@ func setupStackCommand() *cobraext.Command { return cobraext.FlagParsingError(err, cobraext.StackDumpOutputFlagName) } - profileName, err := cmd.Flags().GetString(cobraext.ProfileFlagName) + profile, err := getProfileFlag(cmd) if err != nil { - return cobraext.FlagParsingError(err, cobraext.ProfileFlagName) + return err } - profile, err := profile.LoadProfile(profileName) + provider, err := getProviderFromProfile(cmd, profile, false) if err != nil { - return errors.Wrap(err, "error loading profile") + return err } - target, err := stack.Dump(stack.DumpOptions{ + target, err := provider.Dump(stack.DumpOptions{ Output: output, Profile: profile, }) @@ -262,7 +239,20 @@ func setupStackCommand() *cobraext.Command { Use: "status", Short: "Show status of the stack services", RunE: func(cmd *cobra.Command, args []string) error { - servicesStatus, err := stack.Status() + profile, err := getProfileFlag(cmd) + if err != nil { + return err + } + + provider, err := getProviderFromProfile(cmd, profile, false) + if err != nil { + return err + } + + servicesStatus, err := provider.Status(stack.Options{ + Profile: profile, + Printer: cmd, + }) if err != nil { return errors.Wrap(err, "failed getting stack status") } @@ -278,7 +268,7 @@ func setupStackCommand() *cobraext.Command { Short: "Manage the Elastic stack", Long: stackLongDescription, } - cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", lookupEnv(), fmt.Sprintf(cobraext.ProfileFlagDescription, profileNameEnvVar)) + cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar)) cmd.AddCommand( upCommand, downCommand, @@ -317,18 +307,6 @@ func validateServicesFlag(services []string) error { return nil } -func printInitConfig(cmd *cobra.Command, profile *profile.Profile) error { - initConfig, err := stack.StackInitConfig(profile) - if err != nil { - return nil - } - cmd.Printf("Elasticsearch host: %s\n", initConfig.ElasticsearchHostPort) - cmd.Printf("Kibana host: %s\n", initConfig.KibanaHostPort) - cmd.Printf("Username: %s\n", initConfig.ElasticsearchUsername) - cmd.Printf("Password: %s\n", initConfig.ElasticsearchPassword) - return nil -} - func printStatus(cmd *cobra.Command, servicesStatus []stack.ServiceStatus) { if len(servicesStatus) == 0 { cmd.Printf(" - No service running\n") @@ -343,3 +321,57 @@ func printStatus(cmd *cobra.Command, servicesStatus []stack.ServiceStatus) { t.SetStyle(table.StyleRounded) cmd.Println(t.Render()) } + +func getProfileFlag(cmd *cobra.Command) (*profile.Profile, error) { + profileName, err := cmd.Flags().GetString(cobraext.ProfileFlagName) + if err != nil { + return nil, cobraext.FlagParsingError(err, cobraext.ProfileFlagName) + } + if profileName == "" { + config, err := install.Configuration() + if err != nil { + return nil, fmt.Errorf("cannot read configuration: %w", err) + } + profileName = config.CurrentProfile() + } + + p, err := profile.LoadProfile(profileName) + if errors.Is(err, profile.ErrNotAProfile) { + list, err := availableProfilesAsAList() + if err != nil { + return nil, errors.Wrap(err, "error listing known profiles") + } + if len(list) == 0 { + return nil, fmt.Errorf("%s is not a valid profile", profileName) + } + return nil, fmt.Errorf("%s is not a valid profile, known profiles are: %s", profileName, strings.Join(list, ", ")) + } + if err != nil { + return nil, errors.Wrap(err, "error loading profile") + } + + return p, nil +} + +func getProviderFromProfile(cmd *cobra.Command, profile *profile.Profile, checkFlag bool) (stack.Provider, error) { + var providerName = stack.DefaultProvider + stackConfig, err := stack.LoadConfig(profile) + if err != nil { + return nil, err + } + if stackConfig.Provider != "" { + providerName = stackConfig.Provider + } + + if checkFlag { + providerFlag, err := cmd.Flags().GetString(cobraext.StackProviderFlagName) + if err != nil { + return nil, cobraext.FlagParsingError(err, cobraext.StackProviderFlagName) + } + if providerFlag != "" { + providerName = providerFlag + } + } + + return stack.BuildProvider(providerName, profile) +} diff --git a/go.mod b/go.mod index 60523f8f1..a039a503f 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/elastic/elastic-integration-corpus-generator-tool v0.5.0 github.com/elastic/go-elasticsearch/v7 v7.17.7 github.com/elastic/go-licenser v0.4.1 + github.com/elastic/go-resource v0.1.1 github.com/elastic/go-ucfg v0.8.6 github.com/elastic/package-spec/v2 v2.6.0 github.com/fatih/color v1.15.0 diff --git a/go.sum b/go.sum index 557717508..01545ad26 100644 --- a/go.sum +++ b/go.sum @@ -94,6 +94,8 @@ github.com/elastic/go-elasticsearch/v7 v7.17.7 h1:pcYNfITNPusl+cLwLN6OLmVT+F73El github.com/elastic/go-elasticsearch/v7 v7.17.7/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4= github.com/elastic/go-licenser v0.4.1/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= +github.com/elastic/go-resource v0.1.1 h1:vM173uhPoaZ7C64rKrYbbBn5kxOhstE1+YcGFteAKh8= +github.com/elastic/go-resource v0.1.1/go.mod h1:7F1Wjs6eSFX0i/235yAK/x9bvPNd9/ML92AiULa4XYA= github.com/elastic/go-ucfg v0.8.6 h1:stUeyh2goTgGX+/wb9gzKvTv0YB0231LTpKUgCKj4U0= github.com/elastic/go-ucfg v0.8.6/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul8OsQc= diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 790c5af14..12cad8118 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -145,6 +145,9 @@ const ( TLSSkipVerifyFlagName = "tls-skip-verify" TLSSkipVerifyFlagDescription = "skip TLS verify" + StackProviderFlagName = "provider" + StackProviderFlagDescription = "service provider to start a stack (%s)" + StackServicesFlagName = "services" StackServicesFlagDescription = "component services (comma-separated values: \"%s\")" diff --git a/internal/configuration/locations/locations.go b/internal/configuration/locations/locations.go index 6c912019e..e6252e75a 100644 --- a/internal/configuration/locations/locations.go +++ b/internal/configuration/locations/locations.go @@ -24,20 +24,14 @@ const ( deployerDir = "deployer" fieldsCachedDir = "cache/fields" - - terraformDeployerYmlFile = "terraform-deployer.yml" - - dockerCustomAgentDeployerYmlFile = "docker-custom-agent-base.yml" ) var ( // elasticPackageDataHome is the name of the environment variable used to override data folder for elastic-package elasticPackageDataHome = environment.WithElasticPackagePrefix("DATA_HOME") - serviceLogsDir = filepath.Join(temporaryDir, "service_logs") - kubernetesDeployerDir = filepath.Join(deployerDir, "kubernetes") - terraformDeployerDir = filepath.Join(deployerDir, "terraform") - dockerCustomAgentDeployerDir = filepath.Join(deployerDir, "docker_custom_agent") + serviceLogsDir = filepath.Join(temporaryDir, "service_logs") + kubernetesDeployerDir = filepath.Join(deployerDir, "kubernetes") ) // LocationManager maintains an instance of a config path location @@ -91,26 +85,6 @@ func (loc LocationManager) KubernetesDeployerDir() string { return filepath.Join(loc.stackPath, kubernetesDeployerDir) } -// TerraformDeployerDir returns the Terraform Directory -func (loc LocationManager) TerraformDeployerDir() string { - return filepath.Join(loc.stackPath, terraformDeployerDir) -} - -// TerraformDeployerYml returns the Terraform deployer yml file -func (loc LocationManager) TerraformDeployerYml() string { - return filepath.Join(loc.stackPath, terraformDeployerDir, terraformDeployerYmlFile) -} - -// DockerCustomAgentDeployerDir returns the DockerCustomAgent Directory -func (loc LocationManager) DockerCustomAgentDeployerDir() string { - return filepath.Join(loc.stackPath, dockerCustomAgentDeployerDir) -} - -// DockerCustomAgentDeployerYml returns the DockerCustomAgent deployer yml file -func (loc LocationManager) DockerCustomAgentDeployerYml() string { - return filepath.Join(loc.stackPath, dockerCustomAgentDeployerDir, dockerCustomAgentDeployerYmlFile) -} - // ServiceLogDir returns the log directory func (loc LocationManager) ServiceLogDir() string { return filepath.Join(loc.stackPath, serviceLogsDir) diff --git a/internal/install/application_configuration.go b/internal/install/application_configuration.go index e850c35f0..5213204aa 100644 --- a/internal/install/application_configuration.go +++ b/internal/install/application_configuration.go @@ -14,26 +14,61 @@ import ( "gopkg.in/yaml.v3" "github.com/elastic/elastic-package/internal/configuration/locations" + "github.com/elastic/elastic-package/internal/environment" "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/profile" ) const ( stackVersion715 = "7.15.0-SNAPSHOT" stackVersion820 = "8.2.0-SNAPSHOT" + + elasticAgentImageName = "docker.elastic.co/beats/elastic-agent" + elasticAgentCompleteLegacyImageName = "docker.elastic.co/beats/elastic-agent-complete" + elasticAgentCompleteImageName = "docker.elastic.co/elastic-agent/elastic-agent-complete" + elasticsearchImageName = "docker.elastic.co/elasticsearch/elasticsearch" + kibanaImageName = "docker.elastic.co/kibana/kibana" + + applicationConfigurationYmlFile = "config.yml" ) var ( elasticAgentCompleteFirstSupportedVersion = semver.MustParse(stackVersion715) elasticAgentCompleteOwnNamespaceVersion = semver.MustParse(stackVersion820) + + // ProfileNameEnvVar is the name of the environment variable to set the default profile + ProfileNameEnvVar = environment.WithElasticPackagePrefix("PROFILE") ) +func DefaultConfiguration() *ApplicationConfiguration { + config := ApplicationConfiguration{} + config.c.Profile.Current = profile.DefaultProfile + + // Uncomment and use the commented definition of "stack" in case of emergency + // to define Docker image overrides (stack.image_ref_overrides). + // The following sample defines overrides for the Elastic stack ver. 7.13.0-SNAPSHOT. + // It's advised to use latest stable snapshots for the stack snapshot. + // + // config.c.Stack.ImageRefOverrides = map[string]ImageRefs{ + // "7.13.0-SNAPSHOT": ImageRefs{ + // ElasticAgent: elasticAgentImageName + `@sha256:76c294cf55654bc28dde72ce936032f34ad5f40c345f3df964924778b249e581`, + // Kibana: kibanaImageName + `@sha256:78ae3b1ca09efee242d2c77597dfab18670e984adb96c2407ec03fe07ceca4f6`, + // }, + // } + + return &config +} + // ApplicationConfiguration represents the configuration of the elastic-package. type ApplicationConfiguration struct { c configFile } type configFile struct { - Stack stack `yaml:"stack"` + Stack stack `yaml:"stack"` + Profile struct { + Current string `yaml:"current"` + } `yaml:"profile"` } type stack struct { @@ -79,6 +114,24 @@ func (ac *ApplicationConfiguration) StackImageRefs(version string) ImageRefs { return refs } +// CurrentProfile returns the current profile, or the default one if not set. +func (ac *ApplicationConfiguration) CurrentProfile() string { + fromEnv := os.Getenv(ProfileNameEnvVar) + if fromEnv != "" { + return fromEnv + } + current := ac.c.Profile.Current + if current == "" { + return profile.DefaultProfile + } + return current +} + +// SetCurrentProfile sets the current profile. +func (ac *ApplicationConfiguration) SetCurrentProfile(name string) { + ac.c.Profile.Current = name +} + // selectElasticAgentImageName function returns the appropriate image name for Elastic-Agent depending on the stack version. // This is mandatory as "elastic-agent-complete" is available since 7.15.0-SNAPSHOT. func selectElasticAgentImageName(version string) string { @@ -108,6 +161,9 @@ func Configuration() (*ApplicationConfiguration, error) { } cfg, err := os.ReadFile(filepath.Join(configPath.RootDir(), applicationConfigurationYmlFile)) + if errors.Is(err, os.ErrNotExist) { + return DefaultConfiguration(), nil + } if err != nil { return nil, errors.Wrap(err, "can't read configuration file") } diff --git a/internal/install/application_configuration_yml.go b/internal/install/application_configuration_yml.go deleted file mode 100644 index dbc6811ac..000000000 --- a/internal/install/application_configuration_yml.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package install - -const ( - elasticAgentImageName = "docker.elastic.co/beats/elastic-agent" - elasticAgentCompleteLegacyImageName = "docker.elastic.co/beats/elastic-agent-complete" - elasticAgentCompleteImageName = "docker.elastic.co/elastic-agent/elastic-agent-complete" - elasticsearchImageName = "docker.elastic.co/elasticsearch/elasticsearch" - kibanaImageName = "docker.elastic.co/kibana/kibana" -) - -const applicationConfigurationYmlFile = "config.yml" - -/* - -Uncomment and use the commented definition of "stack" in case of emergency to define Docker image overrides -(stack.image_ref_overrides). The following sample defines overrides for the Elastic stack ver. 7.13.0-SNAPSHOT. -It's advised to use latest stable snapshots for the stack snapshot. - -const applicationConfigurationYml = `stack: - image_ref_overrides: - 7.13.0-SNAPSHOT: - # Use stable image versions for Agent and Kibana - elastic-agent: ` + elasticAgentImageName + `@sha256:76c294cf55654bc28dde72ce936032f34ad5f40c345f3df964924778b249e581 - kibana: ` + kibanaImageName + `@sha256:78ae3b1ca09efee242d2c77597dfab18670e984adb96c2407ec03fe07ceca4f6` -*/ - -const applicationConfigurationYml = `stack: - image_ref_overrides: -` diff --git a/internal/install/install.go b/internal/install/install.go index 23eddf83e..814ba072b 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -8,18 +8,17 @@ import ( "fmt" "os" "path/filepath" - "time" "github.com/pkg/errors" + "gopkg.in/yaml.v3" "github.com/elastic/elastic-package/internal/configuration/locations" - "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" ) const versionFilename = "version" -// EnsureInstalled method installs once static resources for the testing Docker stack. +// EnsureInstalled method installs once the required configuration files. func EnsureInstalled() error { elasticPackagePath, err := locations.NewLocationManager() if err != nil { @@ -31,45 +30,43 @@ func EnsureInstalled() error { return errors.Wrap(err, "failed to check if there is an elastic-package installation") } if installed { - return nil - } - - err = migrateIfNeeded(elasticPackagePath) - if err != nil { - return errors.Wrap(err, "error migrating old install") + latestInstalled, err := checkIfLatestVersionInstalled(elasticPackagePath) + if err != nil { + return errors.Wrap(err, "failed to check if latest version is installed") + } + if latestInstalled { + return nil + } + return migrateConfigDirectory(elasticPackagePath) } - // Create the root .elastic-package path + // Create the root .elastic-package path. err = createElasticPackageDirectory(elasticPackagePath) if err != nil { return errors.Wrap(err, "creating elastic package directory failed") } - // write the root config.yml file - err = writeConfigFile(elasticPackagePath) + // Write the root config.yml file. + err = WriteConfigFile(elasticPackagePath, DefaultConfiguration()) if err != nil { return errors.Wrap(err, "writing configuration file failed") } - // write root version file + // Write root version file. err = writeVersionFile(elasticPackagePath) if err != nil { return errors.Wrap(err, "writing version file failed") } - err = writeStackResources(elasticPackagePath) - if err != nil { - return errors.Wrap(err, "writing stack resources failed") - } - - err = writeTerraformDeployerResources(elasticPackagePath) - if err != nil { - return errors.Wrap(err, "writing Terraform deployer resources failed") + // Create initial profile: + options := profile.Options{ + PackagePath: elasticPackagePath.ProfileDir(), + Name: profile.DefaultProfile, + OverwriteExisting: false, } - - err = writeDockerCustomAgentResources(elasticPackagePath) + err = profile.CreateProfile(options) if err != nil { - return errors.Wrap(err, "writing Terraform deployer resources failed") + return errors.Wrap(err, "creation of initial profile failed") } if err := createServiceLogsDir(elasticPackagePath); err != nil { @@ -81,55 +78,14 @@ func EnsureInstalled() error { } func checkIfAlreadyInstalled(elasticPackagePath *locations.LocationManager) (bool, error) { - _, err := os.Stat(elasticPackagePath.StackDir()) + _, err := os.Stat(elasticPackagePath.RootDir()) if errors.Is(err, os.ErrNotExist) { return false, nil } if err != nil { return false, errors.Wrapf(err, "stat file failed (path: %s)", elasticPackagePath) } - return checkIfLatestVersionInstalled(elasticPackagePath) -} - -// checkIfUnmigrated checks to see if we have a pre-profile config that needs to be migrated -func migrateIfNeeded(elasticPackagePath *locations.LocationManager) error { - // use the snapshot.yml file as a canary to see if we have a pre-profile install - _, err := os.Stat(filepath.Join(elasticPackagePath.StackDir(), string(profile.SnapshotFile))) - if errors.Is(err, os.ErrNotExist) { - return nil - } - if err != nil { - return errors.Wrapf(err, "stat file failed (path: %s)", elasticPackagePath) - } - - profileName := fmt.Sprintf("default_migrated_%d", time.Now().Unix()) - logger.Warnf("Pre-profiles elastic-package detected. Existing config will be migrated to %s", profileName) - // Depending on how old the install is, not all the files will be available to migrate, - // So treat any errors from missing files as "soft" - oldFiles := []string{ - filepath.Join(elasticPackagePath.StackDir(), string(profile.SnapshotFile)), - filepath.Join(elasticPackagePath.StackDir(), string(profile.PackageRegistryDockerfileFile)), - filepath.Join(elasticPackagePath.StackDir(), string(profile.KibanaConfigDefaultFile)), - filepath.Join(elasticPackagePath.StackDir(), string(profile.PackageRegistryConfigFile)), - } - - opts := profile.Options{ - PackagePath: elasticPackagePath.StackDir(), - Name: profileName, - } - err = profile.MigrateProfileFiles(opts, oldFiles) - if err != nil { - return errors.Wrap(err, "error migrating profile config") - } - - // delete the old files - for _, file := range oldFiles { - err = os.Remove(file) - if err != nil { - return errors.Wrapf(err, "error removing config file %s", file) - } - } - return nil + return true, nil } func createElasticPackageDirectory(elasticPackagePath *locations.LocationManager) error { @@ -151,107 +107,37 @@ func createElasticPackageDirectory(elasticPackagePath *locations.LocationManager return nil } -func writeStackResources(elasticPackagePath *locations.LocationManager) error { - err := os.MkdirAll(elasticPackagePath.PackagesDir(), 0755) - if err != nil { - return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath.PackagesDir()) - } - - err = os.MkdirAll(elasticPackagePath.ProfileDir(), 0755) - if err != nil { - return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath.PackagesDir()) - } - - kibanaHealthcheckPath := filepath.Join(elasticPackagePath.StackDir(), "healthcheck.sh") - err = writeStaticResource(err, kibanaHealthcheckPath, kibanaHealthcheckSh) - if err != nil { - return errors.Wrapf(err, "copying healthcheck script failed (%s)", kibanaHealthcheckPath) - } - - // Install GeoIP database - ingestGeoIPDir := filepath.Join(elasticPackagePath.StackDir(), "ingest-geoip") - err = os.MkdirAll(ingestGeoIPDir, 0755) - if err != nil { - return errors.Wrapf(err, "creating directory failed (path: %s)", ingestGeoIPDir) - } - - geoIpAsnMmdbPath := filepath.Join(ingestGeoIPDir, "GeoLite2-ASN.mmdb") - err = writeStaticResource(err, geoIpAsnMmdbPath, geoIpAsnMmdb) - if err != nil { - return errors.Wrapf(err, "copying GeoIP ASN database failed (%s)", geoIpAsnMmdbPath) - } - - geoIpCityMmdbPath := filepath.Join(ingestGeoIPDir, "GeoLite2-City.mmdb") - err = writeStaticResource(err, geoIpCityMmdbPath, geoIpCityMmdb) - if err != nil { - return errors.Wrapf(err, "copying GeoIP city database failed (%s)", geoIpCityMmdbPath) - } - - geoIpCountryMmdbPath := filepath.Join(ingestGeoIPDir, "GeoLite2-Country.mmdb") - err = writeStaticResource(err, geoIpCountryMmdbPath, geoIpCountryMmdb) - if err != nil { - return errors.Wrapf(err, "copying GeoIP country database failed (%s)", geoIpCountryMmdbPath) - } - - serviceTokensPath := filepath.Join(elasticPackagePath.StackDir(), "service_tokens") - err = writeStaticResource(err, serviceTokensPath, serviceTokens) - if err != nil { - return errors.Wrapf(err, "copying service_tokens failed (%s)", serviceTokensPath) - } - - options := profile.Options{ - PackagePath: elasticPackagePath.ProfileDir(), - Name: profile.DefaultProfile, - OverwriteExisting: false, - } - return profile.CreateProfile(options) -} - -func writeTerraformDeployerResources(elasticPackagePath *locations.LocationManager) error { - terraformDeployer := elasticPackagePath.TerraformDeployerDir() - err := os.MkdirAll(terraformDeployer, 0755) +func WriteConfigFile(elasticPackagePath *locations.LocationManager, configuration *ApplicationConfiguration) error { + d, err := yaml.Marshal(configuration.c) if err != nil { - return errors.Wrapf(err, "creating directory failed (path: %s)", terraformDeployer) + return errors.Wrap(err, "failed to encode configuration") } - err = writeStaticResource(err, elasticPackagePath.TerraformDeployerYml(), terraformDeployerYml) - err = writeStaticResource(err, filepath.Join(terraformDeployer, "Dockerfile"), terraformDeployerDockerfile) - err = writeStaticResource(err, filepath.Join(terraformDeployer, "run.sh"), terraformDeployerRun) + err = writeStaticResource(err, filepath.Join(elasticPackagePath.RootDir(), applicationConfigurationYmlFile), string(d)) if err != nil { return errors.Wrap(err, "writing static resource failed") } return nil } -func writeDockerCustomAgentResources(elasticPackagePath *locations.LocationManager) error { - dir := elasticPackagePath.DockerCustomAgentDeployerDir() - if err := os.MkdirAll(dir, 0755); err != nil { - return errors.Wrapf(err, "creating directory failed (path: %s)", dir) - } - if err := writeStaticResource(nil, elasticPackagePath.DockerCustomAgentDeployerYml(), dockerCustomAgentBaseYml); err != nil { - return errors.Wrap(err, "writing static resource failed") +func writeStaticResource(err error, path, content string) error { + if err != nil { + return err } - return nil -} -func writeConfigFile(elasticPackagePath *locations.LocationManager) error { - var err error - err = writeStaticResource(err, filepath.Join(elasticPackagePath.RootDir(), applicationConfigurationYmlFile), applicationConfigurationYml) + err = os.WriteFile(path, []byte(content), 0644) if err != nil { - return errors.Wrap(err, "writing static resource failed") + return errors.Wrapf(err, "writing file failed (path: %s)", path) } return nil } -func writeStaticResource(err error, path, content string) error { +func migrateConfigDirectory(elasticPackagePath *locations.LocationManager) error { + err := writeVersionFile(elasticPackagePath) if err != nil { - return err + return errors.Wrap(err, "writing version file failed") } - err = os.WriteFile(path, []byte(content), 0644) - if err != nil { - return errors.Wrapf(err, "writing file failed (path: %s)", path) - } return nil } diff --git a/internal/install/static.go b/internal/install/static.go deleted file mode 100644 index fb6b6d40c..000000000 --- a/internal/install/static.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package install - -import _ "embed" - -//go:embed _static/kibana_healthcheck.sh -var kibanaHealthcheckSh string - -//go:embed _static/Dockerfile.terraform_deployer -var terraformDeployerDockerfile string - -//go:embed _static/terraform_deployer.yml -var terraformDeployerYml string - -//go:embed _static/terraform_deployer_run.sh -var terraformDeployerRun string - -//go:embed _static/GeoLite2-ASN.mmdb -var geoIpAsnMmdb string - -//go:embed _static/GeoLite2-City.mmdb -var geoIpCityMmdb string - -//go:embed _static/GeoLite2-Country.mmdb -var geoIpCountryMmdb string - -//go:embed _static/service_tokens -var serviceTokens string - -//go:embed _static/docker-custom-agent-base.yml -var dockerCustomAgentBaseYml string diff --git a/internal/profile/_static/elastic-agent_default.env b/internal/profile/_static/elastic-agent_default.env deleted file mode 100644 index f0375c615..000000000 --- a/internal/profile/_static/elastic-agent_default.env +++ /dev/null @@ -1,4 +0,0 @@ -FLEET_ENROLL=1 -FLEET_URL=https://fleet-server:8220 -KIBANA_FLEET_HOST=https://kibana:5601 -KIBANA_HOST=https://kibana:5601 diff --git a/internal/profile/_static/elasticsearch_config_8x.yml b/internal/profile/_static/elasticsearch_config_8x.yml deleted file mode 100644 index f36f2b248..000000000 --- a/internal/profile/_static/elasticsearch_config_8x.yml +++ /dev/null @@ -1,15 +0,0 @@ -network.host: "" -transport.host: "127.0.0.1" -http.host: "0.0.0.0" - -indices.id_field_data.enabled: true - -xpack.license.self_generated.type: "trial" -xpack.security.enabled: true -xpack.security.authc.api_key.enabled: true - -xpack.security.http.ssl.enabled: true -xpack.security.http.ssl.key: "certs/key.pem" -xpack.security.http.ssl.certificate: "certs/cert.pem" - -ingest.geoip.downloader.enabled: false diff --git a/internal/profile/_static/kibana_config_80.yml b/internal/profile/_static/kibana_config_80.yml deleted file mode 100644 index 4e3c72ec0..000000000 --- a/internal/profile/_static/kibana_config_80.yml +++ /dev/null @@ -1,59 +0,0 @@ -server.name: kibana -server.host: "0.0.0.0" -server.ssl.enabled: true -server.ssl.certificate: "/usr/share/kibana/config/certs/cert.pem" -server.ssl.key: "/usr/share/kibana/config/certs/key.pem" -server.ssl.certificateAuthorities: ["/usr/share/kibana/config/certs/ca-cert.pem"] - -elasticsearch.hosts: [ "https://elasticsearch:9200" ] -elasticsearch.ssl.certificateAuthorities: "/usr/share/kibana/config/certs/ca-cert.pem" -elasticsearch.serviceAccountToken: "AAEAAWVsYXN0aWMva2liYW5hL2VsYXN0aWMtcGFja2FnZS1raWJhbmEtdG9rZW46b2x4b051SWNRa0tYMHdXazdLWmFBdw" - -monitoring.ui.container.elasticsearch.enabled: true - -xpack.fleet.registryUrl: "https://package-registry:8080" -xpack.fleet.agents.enabled: true -xpack.fleet.agents.elasticsearch.hosts: ["https://elasticsearch:9200"] -xpack.fleet.agents.fleet_server.hosts: ["https://fleet-server:8220"] - -xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" - -xpack.fleet.packages: - - name: system - version: latest - - name: elastic_agent - version: latest - - name: fleet_server - version: latest -xpack.fleet.agentPolicies: - - name: Elastic-Agent (elastic-package) - id: elastic-agent-managed-ep - is_default: true - is_managed: false - namespace: default - monitoring_enabled: - - logs - - metrics - package_policies: - - name: system-1 - id: default-system - package: - name: system - - name: Fleet Server (elastic-package) - id: fleet-server-policy - is_default_fleet_server: true - is_managed: false - namespace: default - package_policies: - - name: fleet_server-1 - id: default-fleet-server - package: - name: fleet_server -xpack.fleet.outputs: - - id: fleet-default-output - name: default - type: elasticsearch - hosts: [ https://elasticsearch:9200 ] - ca_trusted_fingerprint: "${ELASTIC_PACKAGE_CA_TRUSTED_FINGERPRINT}" - is_default: true - is_default_monitoring: true diff --git a/internal/profile/_static/kibana_config_86.yml b/internal/profile/_static/kibana_config_86.yml deleted file mode 100644 index 6d9564c00..000000000 --- a/internal/profile/_static/kibana_config_86.yml +++ /dev/null @@ -1,61 +0,0 @@ -server.name: kibana -server.host: "0.0.0.0" -server.ssl.enabled: true -server.ssl.certificate: "/usr/share/kibana/config/certs/cert.pem" -server.ssl.key: "/usr/share/kibana/config/certs/key.pem" -server.ssl.certificateAuthorities: ["/usr/share/kibana/config/certs/ca-cert.pem"] - -elasticsearch.hosts: [ "https://elasticsearch:9200" ] -elasticsearch.ssl.certificateAuthorities: "/usr/share/kibana/config/certs/ca-cert.pem" -elasticsearch.serviceAccountToken: "AAEAAWVsYXN0aWMva2liYW5hL2VsYXN0aWMtcGFja2FnZS1raWJhbmEtdG9rZW46b2x4b051SWNRa0tYMHdXazdLWmFBdw" - -monitoring.ui.container.elasticsearch.enabled: true - -xpack.fleet.registryUrl: "https://package-registry:8080" -xpack.fleet.agents.enabled: true -xpack.fleet.agents.elasticsearch.hosts: ["https://elasticsearch:9200"] -xpack.fleet.agents.fleet_server.hosts: ["https://fleet-server:8220"] - -xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" - -xpack.cloudSecurityPosture.enabled: true - -xpack.fleet.packages: - - name: system - version: latest - - name: elastic_agent - version: latest - - name: fleet_server - version: latest -xpack.fleet.agentPolicies: - - name: Elastic-Agent (elastic-package) - id: elastic-agent-managed-ep - is_default: true - is_managed: false - namespace: default - monitoring_enabled: - - logs - - metrics - package_policies: - - name: system-1 - id: default-system - package: - name: system - - name: Fleet Server (elastic-package) - id: fleet-server-policy - is_default_fleet_server: true - is_managed: false - namespace: default - package_policies: - - name: fleet_server-1 - id: default-fleet-server - package: - name: fleet_server -xpack.fleet.outputs: - - id: fleet-default-output - name: default - type: elasticsearch - hosts: [ https://elasticsearch:9200 ] - ca_trusted_fingerprint: "${ELASTIC_PACKAGE_CA_TRUSTED_FINGERPRINT}" - is_default: true - is_default_monitoring: true diff --git a/internal/profile/_static/kibana_config_default.yml b/internal/profile/_static/kibana_config_default.yml deleted file mode 100644 index 124f0ff74..000000000 --- a/internal/profile/_static/kibana_config_default.yml +++ /dev/null @@ -1,21 +0,0 @@ -server.name: kibana -server.host: "0.0.0.0" -server.ssl.enabled: true -server.ssl.certificate: "/usr/share/kibana/config/certs/cert.pem" -server.ssl.key: "/usr/share/kibana/config/certs/key.pem" -server.ssl.certificateAuthorities: ["/usr/share/kibana/config/certs/ca-cert.pem"] - -elasticsearch.hosts: [ "https://elasticsearch:9200" ] -elasticsearch.ssl.certificateAuthorities: "/usr/share/kibana/config/certs/ca-cert.pem" -elasticsearch.username: elastic -elasticsearch.password: changeme - -xpack.monitoring.ui.container.elasticsearch.enabled: true - -xpack.fleet.enabled: true -xpack.fleet.registryUrl: "https://package-registry:8080" -xpack.fleet.agents.enabled: true -xpack.fleet.agents.elasticsearch.host: "https://elasticsearch:9200" -xpack.fleet.agents.fleet_server.hosts: ["https://fleet-server:8220"] - -xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" diff --git a/internal/profile/config_profile_test.go b/internal/profile/config_profile_test.go deleted file mode 100644 index c1f00d967..000000000 --- a/internal/profile/config_profile_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. -package profile - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" -) - -const ( - profileName = "test_profile" -) - -func TestNewProfile(t *testing.T) { - elasticPackageDir, err := os.MkdirTemp("", "package") - defer cleanupProfile(t, elasticPackageDir) - assert.NoError(t, err, "error creating tempdir") - t.Logf("writing to directory %s", elasticPackageDir) - - options := Options{ - PackagePath: elasticPackageDir, - Name: profileName, - OverwriteExisting: false, - } - err = createProfile(options) - assert.NoErrorf(t, err, "error creating profile %s", err) - -} - -func TestNewProfileFrom(t *testing.T) { - elasticPackageDir, err := os.MkdirTemp("", "package") - defer cleanupProfile(t, elasticPackageDir) - assert.NoError(t, err, "error creating tempdir") - t.Logf("writing to directory %s", elasticPackageDir) - - options := Options{ - PackagePath: elasticPackageDir, - Name: profileName, - OverwriteExisting: false, - } - err = createProfile(options) - assert.NoErrorf(t, err, "error creating profile %s", err) - - //update the profile to make sure we're properly copying everything - - testProfile, err := NewConfigProfile(elasticPackageDir, profileName) - assert.NoErrorf(t, err, "error creating profile %s", err) - - pkgRegUpdated := &simpleFile{ - path: filepath.Join(testProfile.ProfilePath, string(PackageRegistryConfigFile)), - body: `package_paths: - - /packages/testing - - /packages/development - - /packages/production - - /packages/staging - - /packages/snapshot - `, - } - t.Logf("updating profile %s", testProfile.ProfilePath) - testProfile.configFiles[PackageRegistryConfigFile] = pkgRegUpdated - err = testProfile.writeProfileResources() - assert.NoErrorf(t, err, "error updating profile %s", err) - - // actually create & check the new profile - option := Options{ - PackagePath: elasticPackageDir, - Name: "test_from", - FromProfile: profileName, - } - err = createProfileFrom(option) - assert.NoErrorf(t, err, "error copying updating profile %s", err) - -} - -func cleanupProfile(t *testing.T, dir string) { - err := os.RemoveAll(dir) - assert.NoErrorf(t, err, "Error cleaning up tempdir %s", dir) -} diff --git a/internal/profile/files.go b/internal/profile/files.go deleted file mode 100644 index c115fd3e9..000000000 --- a/internal/profile/files.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package profile - -import ( - "os" - "path/filepath" - - "github.com/pkg/errors" -) - -// NewConfig is a generic function type to return a new Managed config -type NewConfig = func(profileName string, profilePath string) (*simpleFile, error) - -// simpleFile defines a file that's managed by the profile system -// and doesn't require any rendering -type simpleFile struct { - name string - path string - body string -} - -const profileStackPath = "stack" - -// configFilesDiffer checks to see if a local configItem differs from the one it knows. -func (cfg simpleFile) configFilesDiffer() (bool, error) { - changes, err := os.ReadFile(cfg.path) - if err != nil && errors.Is(err, os.ErrNotExist) { - return false, nil - } - if err != nil { - return false, errors.Wrapf(err, "error reading %s", cfg.path) - } - if string(changes) == cfg.body { - return false, nil - } - return true, nil -} - -// writeConfig writes the config item -func (cfg simpleFile) writeConfig() error { - err := os.MkdirAll(filepath.Dir(cfg.path), 0755) - if err != nil { - return errors.Wrapf(err, "creating parent directories for file failed (path: %s)", cfg.path) - } - err = os.WriteFile(cfg.path, []byte(cfg.body), 0644) - if err != nil { - return errors.Wrapf(err, "writing file failed (path: %s)", cfg.path) - } - return nil -} - -// readConfig reads the config item, overwriting whatever exists in the fileBody. -func (cfg *simpleFile) readConfig() error { - body, err := os.ReadFile(cfg.path) - if err != nil { - return errors.Wrapf(err, "reading file failed (path: %s)", cfg.path) - } - cfg.body = string(body) - return nil -} diff --git a/internal/profile/install_profile.go b/internal/profile/install_profile.go deleted file mode 100644 index eb13dfc16..000000000 --- a/internal/profile/install_profile.go +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package profile - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/pkg/errors" - - "github.com/elastic/elastic-package/internal/configuration/locations" - "github.com/elastic/elastic-package/internal/logger" -) - -// CreateProfile creates an existing profile from the default elastic-package config dir -// if option.Package has a value it'll be used, if not, the default location will be used -// if option.From does not have a supplied value, it'll create a default profile. -func CreateProfile(options Options) error { - if options.PackagePath == "" { - loc, err := locations.NewLocationManager() - if err != nil { - return errors.Wrap(err, "error finding stack dir location") - } - options.PackagePath = loc.ProfileDir() - } - - // If they're creating from Default, assume they want the actual default, and - // not whatever is currently inside default. - if options.FromProfile == "" || options.FromProfile == DefaultProfile { - return createProfile(options) - } - - return createProfileFrom(options) -} - -// MigrateProfileFiles creates a new profile based on existing filepaths -// that are stored elsewhere outside the profile system. -func MigrateProfileFiles(options Options, files []string) error { - profile, err := newProfileFromExistingFiles(options.PackagePath, options.Name, files, true) - if err != nil { - return errors.Wrap(err, "error creating new profile from files") - } - err = os.Mkdir(profile.ProfilePath, 0755) - if err != nil { - return errors.Wrapf(err, "error crating profile directory %s", profile.ProfilePath) - } - err = profile.writeProfileResources() - if err != nil { - return errors.Wrap(err, "error writing out new profile config") - } - return nil -} - -// LoadProfile loads an existing profile from the default elastic-package config dir -func LoadProfile(profileName string) (*Profile, error) { - loc, err := locations.NewLocationManager() - if err != nil { - return nil, errors.Wrap(err, "error finding stack dir location") - } - - return loadProfile(loc.ProfileDir(), profileName) -} - -// DeleteProfile deletes a profile from the default elastic-package config dir -func DeleteProfile(profileName string) error { - loc, err := locations.NewLocationManager() - if err != nil { - return errors.Wrap(err, "error finding stack dir location") - } - return deleteProfile(loc.ProfileDir(), profileName) -} - -// FetchAllProfiles returns a list of profile values -func FetchAllProfiles(elasticPackagePath string) ([]Metadata, error) { - dirList, err := os.ReadDir(elasticPackagePath) - if err != nil { - return []Metadata{}, errors.Wrapf(err, "error reading from directory %s", elasticPackagePath) - } - - var profiles []Metadata - // TODO: this should read a profile.json file or something like that - for _, item := range dirList { - if !item.IsDir() { - continue - } - profile, err := loadProfile(elasticPackagePath, item.Name()) - if errors.Is(err, ErrNotAProfile) { - continue - } - if err != nil { - return profiles, errors.Wrapf(err, "error loading profile %s", item.Name()) - } - metadata, err := profile.metadata() - if err != nil { - return profiles, errors.Wrap(err, "error reading profile metadata") - } - profiles = append(profiles, metadata) - } - return profiles, nil -} - -// createProfile installs a new profile at the given package path location. -// overwriteExisting determines the behavior if a profile with the given name already exists. -// On true, it'll overwrite the profile, on false, it'll backup the existing profile to profilename_VERSION-DATE-CREATED -func createProfile(options Options) error { - profile, err := createAndCheckProfile(options.PackagePath, options.Name, options.OverwriteExisting) - if err != nil { - return errors.Wrap(err, "error creating new profile") - } - // write the resources - err = profile.writeProfileResources() - if err != nil { - return errors.Wrap(err, "error writing profile file") - } - return nil -} - -// createProfileFrom creates a new profile by copying over an existing profile -func createProfileFrom(options Options) error { - fromProfile, err := loadProfile(options.PackagePath, options.FromProfile) - if err != nil { - return errors.Wrapf(err, "error loading %s profile", options.FromProfile) - } - - newProfile, err := createAndCheckProfile(options.PackagePath, options.Name, options.OverwriteExisting) - if err != nil { - return errors.Wrap(err, "error creating new profile") - } - - newProfile.overwrite(fromProfile.configFiles) - err = newProfile.writeProfileResources() - if err != nil { - return errors.Wrap(err, "error writing new profile") - } - return nil -} - -// createAndCheckProfile does most of the heavy lifting for initializing a new profile, -// including dealing with profile overwrites -func createAndCheckProfile(packagePath, packageName string, overwriteExisting bool) (*Profile, error) { - profile, err := NewConfigProfile(packagePath, packageName) - if err != nil { - return nil, errors.Wrap(err, "error creating profile") - } - - // check to see if we have an existing profile at that location. - exists, err := profile.alreadyExists() - if err != nil { - return nil, errors.Wrap(err, "error checking for existing profile") - } - if exists { - localChanges, err := profile.localFilesChanged() - if err != nil { - return nil, errors.Wrapf(err, "error checking for changes in %s", profile.ProfilePath) - } - // If there are changes and we've selected CreateNew, move the old path - if localChanges && !overwriteExisting { - if localChanges && packageName == DefaultProfile { - logger.Warn("Default profile has been changed by user or updated by elastic-package. The current profile will be moved.") - } - // Migrate the existing profile - err = updateExistingDefaultProfile(packagePath) - if err != nil { - return nil, errors.Wrap(err, "error moving old profile") - } - err = os.Mkdir(profile.ProfilePath, 0755) - if err != nil { - return nil, errors.Wrapf(err, "error crating profile directory %s", profile.ProfilePath) - } - err = os.Mkdir(profile.ProfileStackPath, 0755) - if err != nil { - return nil, errors.Wrapf(err, "error crating profile directory %s", profile.ProfilePath) - } - } - } else { - err = os.Mkdir(profile.ProfilePath, 0755) - if err != nil { - return nil, errors.Wrapf(err, "error crating profile directory %s", profile.ProfilePath) - } - err = os.Mkdir(profile.ProfileStackPath, 0755) - if err != nil { - return nil, errors.Wrapf(err, "error crating profile directory %s", profile.ProfilePath) - } - } - - return profile, nil -} - -// updateExistingDefaultProfile migrates the old default profile to profile_VERSION_DATE-CREATED -func updateExistingDefaultProfile(path string) error { - profile, err := NewConfigProfile(path, DefaultProfile) - if err != nil { - return errors.Wrap(err, "error creating profile") - } - meta, err := profile.metadata() - if err != nil { - return errors.Wrap(err, "error updating metadata") - } - newName := fmt.Sprintf("default_%s_%d", meta.Version, meta.DateCreated.Unix()) - newFilePath := filepath.Join(filepath.Dir(profile.ProfilePath), newName) - meta.Name = newName - meta.Path = newFilePath - - err = profile.updateMetadata(meta) - if err != nil { - return errors.Wrap(err, "error updating metadata") - } - - err = os.Rename(profile.ProfilePath, newFilePath) - if err != nil { - return errors.Wrap(err, "error moving default profile") - } - return nil -} - -// deleteProfile deletes a given config profile. -func deleteProfile(elasticPackagePath string, profileName string) error { - if profileName == DefaultProfile { - return errors.New("cannot remove default profile") - } - - pathToDelete := filepath.Join(elasticPackagePath, profileName) - - return os.RemoveAll(pathToDelete) - -} diff --git a/internal/profile/profile.go b/internal/profile/profile.go index 73cebeec4..21403b201 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -5,324 +5,218 @@ package profile import ( - "encoding/json" + "errors" "fmt" "os" "path/filepath" + "strings" - "github.com/pkg/errors" + "github.com/elastic/go-resource" - "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/configuration/locations" + "github.com/elastic/elastic-package/internal/files" ) -// Profile manages a a given user config profile -type Profile struct { - // ProfilePath is the absolute path to the profile - ProfilePath string - ProfileStackPath string - profileName string - configFiles map[configFile]*simpleFile -} - const ( - // DefaultProfile is the name of the default profile + // PackageProfileMetaFile is the filename of the profile metadata file + PackageProfileMetaFile = "profile.json" + + // DefaultProfile is the name of the default profile. DefaultProfile = "default" ) -// ErrNotAProfile is returned in cases where we don't have a valid profile directory -var ErrNotAProfile = errors.New("not a profile") +var ( + profileResources = []resource.Resource{ + &resource.File{ + Path: PackageProfileMetaFile, + Content: profileMetadataContent, + }, + } +) -// configFile is a type for for the config file names in a managed profile config -type configFile string - -// managedProfileFiles is the list of all files managed in a profile -// If you create a new file that's managed by a profile, it needs to go in this list -var managedProfileFiles = map[configFile]NewConfig{ - ElasticAgentDefaultEnvFile: newElasticAgentDefaultEnv, - ElasticAgent8xEnvFile: newElasticAgent8xEnv, - ElasticAgent86EnvFile: newElasticAgent86Env, - ElasticAgent80EnvFile: newElasticAgent80Env, - ElasticsearchConfigDefaultFile: newElasticsearchConfigDefault, - ElasticsearchConfig8xFile: newElasticsearchConfig8x, - ElasticsearchConfig86File: newElasticsearchConfig86, - ElasticsearchConfig80File: newElasticsearchConfig80, - KibanaConfigDefaultFile: newKibanaConfigDefault, - KibanaConfig8xFile: newKibanaConfig8x, - KibanaConfig86File: newKibanaConfig86, - KibanaConfig80File: newKibanaConfig80, - PackageRegistryDockerfileFile: newPackageRegistryDockerfile, - PackageRegistryConfigFile: newPackageRegistryConfig, - SnapshotFile: newSnapshotFile, - PackageProfileMetaFile: createProfileMetadata, +type Options struct { + PackagePath string + Name string + FromProfile string + OverwriteExisting bool } -// NewConfigProfile creates a new config profile manager -func NewConfigProfile(elasticPackagePath string, profileName string) (*Profile, error) { - profilePath := filepath.Join(elasticPackagePath, profileName) - - var configMap = map[configFile]*simpleFile{} - for fileItem, configInit := range managedProfileFiles { - cfg, err := configInit(profileName, profilePath) +func CreateProfile(options Options) error { + if options.PackagePath == "" { + loc, err := locations.NewLocationManager() if err != nil { - return nil, errors.Wrapf(err, "error initializing config %s", cfg) + return fmt.Errorf("error finding profile dir location: %w", err) } - configMap[fileItem] = cfg - } - - err := initTLSCertificates(profilePath, configMap) - if err != nil { - return nil, errors.Wrap(err, "error initializing TLS certificates") + options.PackagePath = loc.ProfileDir() } - newProfile := &Profile{ - profileName: profileName, - ProfilePath: profilePath, - ProfileStackPath: filepath.Join(profilePath, profileStackPath), - configFiles: configMap, + if options.Name == "" { + options.Name = DefaultProfile } - return newProfile, nil -} -// newProfileFromExistingFiles creates a profile from a list of absolute filepaths -// This can be used when migrating a config from a non-profiles-managed config set -// ignoreMissing will treat non-existant files as soft errors -func newProfileFromExistingFiles(elasticPackagePath string, profileName string, files []string, ignoreMissing bool) (*Profile, error) { - profilePath := filepath.Join(elasticPackagePath, profileName) - var configMap = map[configFile]*simpleFile{} - for _, file := range files { - if ignoreMissing { - // if we're treating missing files as soft errors, - // just continue on ErrNotExist - // If it's another kind of error, we'll pick it up in ReadFile - if _, err := os.Stat(file); errors.Is(err, os.ErrNotExist) { - continue - } + if !options.OverwriteExisting { + _, err := loadProfile(options.PackagePath, options.Name) + if err == nil { + return fmt.Errorf("profile %q already exists", options.Name) } - - byteFile, err := os.ReadFile(file) - if err != nil { - return nil, errors.Wrapf(err, "error reading file %s", file) - } - //format this in the way configFile expects - name := filepath.Base(file) - configMap[configFile(name)] = &simpleFile{ - name: name, - path: filepath.Join(profilePath, name), - body: string(byteFile), + if err != nil && err != ErrNotAProfile { + return fmt.Errorf("failed to check if profile %q exists: %w", options.Name, err) } } - //add metadata file - metadata, err := createProfileMetadata(profileName, profilePath) - if err != nil { - return nil, errors.Wrap(err, "error creating profile metadata") + // If they're creating from Default, assume they want the actual default, and + // not whatever is currently inside default. + if from := options.FromProfile; from != "" && from != DefaultProfile { + return createProfileFrom(options) } - configMap[PackageProfileMetaFile] = metadata - newProfile := &Profile{ - profileName: profileName, - ProfilePath: profilePath, - configFiles: configMap, - } - return newProfile, nil + return createProfile(options, profileResources) } -// loadProfile loads an existing profile -func loadProfile(elasticPackagePath string, profileName string) (*Profile, error) { - profilePath := filepath.Join(elasticPackagePath, profileName) +func createProfile(options Options, resources []resource.Resource) error { + profileDir := filepath.Join(options.PackagePath, options.Name) - isValid, err := isProfileDir(profilePath) - if err != nil { - return nil, errors.Wrapf(err, "error checking profile %s", profileName) - } + resourceManager := resource.NewManager() + resourceManager.AddFacter(resource.StaticFacter{ + "profile_name": options.Name, + "profile_path": profileDir, + }) - if !isValid { - return nil, ErrNotAProfile - } + os.MkdirAll(profileDir, 0755) + resourceManager.RegisterProvider("file", &resource.FileProvider{ + Prefix: profileDir, + }) - var configMap = map[configFile]*simpleFile{} - for fileItem, configInit := range managedProfileFiles { - cfg, err := configInit(profileName, profilePath) - if err != nil { - return nil, errors.Wrapf(err, "error initializing config %s", cfg) - } - configMap[fileItem] = cfg - } - - err = initTLSCertificates(profilePath, configMap) + results, err := resourceManager.Apply(resources) if err != nil { - return nil, errors.Wrap(err, "error initializing TLS certificates") + var errors []string + for _, result := range results { + if err := result.Err(); err != nil { + errors = append(errors, err.Error()) + } + } + return fmt.Errorf("%w: %s", err, strings.Join(errors, ", ")) } - profile := &Profile{ - profileName: profileName, - ProfilePath: profilePath, - ProfileStackPath: filepath.Join(profilePath, profileStackPath), - configFiles: configMap, - } + return nil +} - exists, err := profile.alreadyExists() +func createProfileFrom(options Options) error { + from, err := LoadProfile(options.FromProfile) if err != nil { - return nil, errors.Wrapf(err, "error checking if profile %s exists", profileName) + return fmt.Errorf("failed to load profile to copy %q: %w", options.FromProfile, err) } - if !exists { - return nil, fmt.Errorf("profile %s does not exist", profile.ProfilePath) - } - - err = profile.readProfileResources() + profileDir := filepath.Join(options.PackagePath, options.Name) + err = files.CopyAll(from.ProfilePath, profileDir) if err != nil { - return nil, errors.Wrapf(err, "error reading in profile %s", profileName) + return fmt.Errorf("failed to copy files from profile %q to %q", options.FromProfile, options.Name) } - return profile, nil + overwriteOptions := options + overwriteOptions.OverwriteExisting = true + return createProfile(overwriteOptions, profileResources) +} +// Profile manages a a given user config profile +type Profile struct { + // ProfilePath is the absolute path to the profile + ProfilePath string + ProfileName string } -// FetchPath returns an absolute path to the given file -func (profile Profile) FetchPath(file configFile) string { - return profile.configFiles[file].path +// Path returns an absolute path to the given file +func (profile Profile) Path(names ...string) string { + elems := append([]string{profile.ProfilePath}, names...) + return filepath.Join(elems...) } +// ErrNotAProfile is returned in cases where we don't have a valid profile directory +var ErrNotAProfile = errors.New("not a profile") + // ComposeEnvVars returns a list of environment variables that can be passed // to docker-compose for the sake of filling out paths and names in the snapshot.yml file. func (profile Profile) ComposeEnvVars() []string { return []string{ - fmt.Sprintf("PROFILE_NAME=%s", profile.profileName), - fmt.Sprintf("STACK_PATH=%s", profile.ProfileStackPath), + fmt.Sprintf("PROFILE_NAME=%s", profile.ProfileName), } } -// writeProfileResources writes the config files -func (profile Profile) writeProfileResources() error { - return writeConfigFiles(profile.configFiles) -} - -func writeConfigFiles(configFiles map[configFile]*simpleFile) error { - for _, cfgFiles := range configFiles { - err := cfgFiles.writeConfig() - if err != nil { - return errors.Wrap(err, "error writing config file") - } +// DeleteProfile deletes a profile from the default elastic-package config dir +func DeleteProfile(profileName string) error { + if profileName == DefaultProfile { + return errors.New("cannot remove default profile") } - return nil -} - -// overwrite updates the string contents of the config files -func (profile *Profile) overwrite(newBody map[configFile]*simpleFile) { - for key := range profile.configFiles { - // skip metadata - if key == PackageProfileMetaFile { - continue - } - toReplace, ok := newBody[key] - if ok { - updatedProfile := profile.configFiles[key] - updatedProfile.body = toReplace.body - profile.configFiles[key] = updatedProfile - } + loc, err := locations.NewLocationManager() + if err != nil { + return fmt.Errorf("error finding stack dir location: %w", err) } + pathToDelete := filepath.Join(loc.ProfileDir(), profileName) + return os.RemoveAll(pathToDelete) } -// alreadyExists checks to see if a profile with this name already exists -func (profile Profile) alreadyExists() (bool, error) { - packageMetadata := profile.configFiles[PackageProfileMetaFile] - // We do this in stages to make sure we return the right error. - _, err := os.Stat(profile.ProfilePath) - if errors.Is(err, os.ErrNotExist) { - return false, nil - } - if err != nil { - return false, errors.Wrapf(err, "error checking root directory: %s", packageMetadata.path) - } - - // If the folder exists, check to make sure it's a profile folder - _, err = os.Stat(packageMetadata.path) +// FetchAllProfiles returns a list of profile values +func FetchAllProfiles(elasticPackagePath string) ([]Metadata, error) { + dirList, err := os.ReadDir(elasticPackagePath) if errors.Is(err, os.ErrNotExist) { - return false, ErrNotAProfile + return []Metadata{}, nil } if err != nil { - return false, errors.Wrapf(err, "error checking metadata: %s", packageMetadata.path) - } - - //if it is, see if it has the same profile name - profileInfo, err := profile.metadata() - if err != nil { - return false, errors.Wrap(err, "error reading metadata") + return []Metadata{}, fmt.Errorf("error reading from directory %s: %w", elasticPackagePath, err) } - //TODO: this will break default_old, as we don't update the json - if profileInfo.Name != profile.profileName { - return false, nil - } - - return true, nil -} - -func (profile Profile) localFilesChanged() (bool, error) { - for cfgName, cfgFile := range profile.configFiles { - // skip checking the metadata file - // TODO: in the future, we might want to check version to see if the default profile needs to be updated - if cfgName == PackageProfileMetaFile { + var profiles []Metadata + // TODO: this should read a profile.json file or something like that + for _, item := range dirList { + if !item.IsDir() { continue } - changes, err := cfgFile.configFilesDiffer() - if err != nil { - return false, errors.Wrap(err, "error checking config file") - } - if changes { - return true, nil - } - } - return false, nil -} - -// readProfileResources reads the associated files into the config, as opposed to writing them out. -func (profile Profile) readProfileResources() error { - for _, cfgFile := range profile.configFiles { - err := cfgFile.readConfig() - if errors.Is(err, os.ErrNotExist) { - logger.Debugf("File %s not found while reading profile.", cfgFile.path) + profile, err := loadProfile(elasticPackagePath, item.Name()) + if errors.Is(err, ErrNotAProfile) { continue } if err != nil { - return errors.Wrap(err, "error reading in profile") + return profiles, fmt.Errorf("error loading profile %s: %w", item.Name(), err) } + metadata, err := loadProfileMetadata(filepath.Join(profile.ProfilePath, PackageProfileMetaFile)) + if err != nil { + return profiles, fmt.Errorf("error reading profile metadata: %w", err) + } + profiles = append(profiles, metadata) } - return nil + return profiles, nil } -// metadata returns the metadata struct for the profile -func (profile Profile) metadata() (Metadata, error) { - packageMetadata := profile.configFiles[PackageProfileMetaFile] - rawPackageMetadata, err := os.ReadFile(packageMetadata.path) +// LoadProfile loads an existing profile from the default elastic-package config dir. +func LoadProfile(profileName string) (*Profile, error) { + loc, err := locations.NewLocationManager() if err != nil { - return Metadata{}, errors.Wrap(err, "error reading metadata file") + return nil, fmt.Errorf("error finding stack dir location: %w", err) } - profileInfo := Metadata{} - - err = json.Unmarshal(rawPackageMetadata, &profileInfo) - if err != nil { - return Metadata{}, errors.Wrap(err, "error unmarshalling JSON") - } - return profileInfo, nil + return loadProfile(loc.ProfileDir(), profileName) } -// updateMetadata updates the metadata json file -func (profile *Profile) updateMetadata(meta Metadata) error { - packageMetadata := profile.configFiles[PackageProfileMetaFile] - metaString, err := json.Marshal(meta) +// loadProfile loads an existing profile +func loadProfile(elasticPackagePath string, profileName string) (*Profile, error) { + profilePath := filepath.Join(elasticPackagePath, profileName) + + isValid, err := isProfileDir(profilePath) if err != nil { - return errors.Wrap(err, "error marshalling metadata json") + return nil, fmt.Errorf("error checking profile %q: %w", profileName, err) } - err = os.WriteFile(packageMetadata.path, metaString, 0664) - if err != nil { - return errors.Wrap(err, "error writing metadata file") + if !isValid { + return nil, ErrNotAProfile } - return nil + + profile := Profile{ + ProfileName: profileName, + ProfilePath: profilePath, + } + + return &profile, nil } // isProfileDir checks to see if the given path points to a valid profile @@ -333,7 +227,7 @@ func isProfileDir(path string) (bool, error) { return false, nil } if err != nil { - return false, errors.Wrapf(err, "error stat: %s", metaPath) + return false, fmt.Errorf("error stat: %s: %w", metaPath, err) } return true, nil } diff --git a/internal/profile/profile_json.go b/internal/profile/profile_json.go index 449f051ca..edd142099 100644 --- a/internal/profile/profile_json.go +++ b/internal/profile/profile_json.go @@ -6,12 +6,15 @@ package profile import ( "encoding/json" + "io" + "os" "os/user" - "path/filepath" "time" "github.com/pkg/errors" + "github.com/elastic/go-resource" + "github.com/elastic/elastic-package/internal/version" ) @@ -24,14 +27,21 @@ type Metadata struct { Path string `json:"path"` } -// PackageProfileMetaFile is the filename of the profile metadata file -const PackageProfileMetaFile configFile = "profile.json" - -// createProfileMetadata creates the body of the profile.json file -func createProfileMetadata(profileName string, profilePath string) (*simpleFile, error) { +// profileMetadataContent generates the content of the profile.json file. +func profileMetadataContent(applyCtx resource.Context, w io.Writer) error { currentUser, err := user.Current() if err != nil { - return nil, errors.Wrap(err, "error fetching current user") + return errors.Wrap(err, "error fetching current user") + } + + profileName, found := applyCtx.Fact("profile_name") + if !found { + return errors.New("unknown profile name") + } + + profilePath, found := applyCtx.Fact("profile_path") + if !found { + return errors.New("unknown profile path") } profileData := Metadata{ @@ -42,14 +52,26 @@ func createProfileMetadata(profileName string, profilePath string) (*simpleFile, profilePath, } - jsonRaw, err := json.MarshalIndent(profileData, "", " ") + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + err = enc.Encode(profileData) if err != nil { - return nil, errors.Wrap(err, "error marshalling json") + return errors.Wrap(err, "error marshalling json") } - return &simpleFile{ - name: string(PackageProfileMetaFile), - path: filepath.Join(profilePath, string(PackageProfileMetaFile)), - body: string(jsonRaw), - }, nil + return nil +} + +func loadProfileMetadata(path string) (Metadata, error) { + d, err := os.ReadFile(path) + if err != nil { + return Metadata{}, errors.Wrap(err, "error reading metadata file") + } + + metadata := Metadata{} + err = json.Unmarshal(d, &metadata) + if err != nil { + return Metadata{}, errors.Wrapf(err, "error checking profile metadata file %q", path) + } + return metadata, nil } diff --git a/internal/profile/options.go b/internal/profile/profile_test.go similarity index 54% rename from internal/profile/options.go rename to internal/profile/profile_test.go index 7e3fca4a0..5fc03d5f4 100644 --- a/internal/profile/options.go +++ b/internal/profile/profile_test.go @@ -4,10 +4,16 @@ package profile -// Options defines available stack management options -type Options struct { - PackagePath string - Name string - FromProfile string - OverwriteExisting bool +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCreateProfile(t *testing.T) { + options := Options{ + PackagePath: t.TempDir(), + } + err := CreateProfile(options) + require.NoError(t, err) } diff --git a/internal/profile/static.go b/internal/profile/static.go deleted file mode 100644 index ee145b99b..000000000 --- a/internal/profile/static.go +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package profile - -import ( - _ "embed" - "path/filepath" - "strings" -) - -// SnapshotFile is the docker-compose snapshot.yml file name -const SnapshotFile configFile = "snapshot.yml" - -//go:embed _static/docker-compose-stack.yml -var snapshotYml string - -// newSnapshotFile returns a Managed Config -func newSnapshotFile(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(SnapshotFile), - path: filepath.Join(profilePath, profileStackPath, string(SnapshotFile)), - body: snapshotYml, - }, nil -} - -// KibanaConfigDefaultFile is the default kibana config file -const KibanaConfigDefaultFile configFile = "kibana.config.default.yml" - -//go:embed _static/kibana_config_default.yml -var kibanaConfigDefaultYml string - -func newKibanaConfigDefault(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(KibanaConfigDefaultFile), - path: filepath.Join(profilePath, profileStackPath, string(KibanaConfigDefaultFile)), - body: kibanaConfigDefaultYml, - }, nil -} - -// KibanaConfig8xFile is the Kibana config file for 8.x stack family -const KibanaConfig8xFile configFile = "kibana.config.8x.yml" - -//go:embed _static/kibana_config_8x.yml -var kibanaConfig8xYml string - -func newKibanaConfig8x(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(KibanaConfig8xFile), - path: filepath.Join(profilePath, profileStackPath, string(KibanaConfig8xFile)), - body: kibanaConfig8xYml, - }, nil -} - -// KibanaConfig86File is the Kibana config file for the 8.x stack family (8.2 to 8.6) -const KibanaConfig86File configFile = "kibana.config.86.yml" - -//go:embed _static/kibana_config_86.yml -var kibanaConfig86Yml string - -func newKibanaConfig86(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(KibanaConfig86File), - path: filepath.Join(profilePath, profileStackPath, string(KibanaConfig86File)), - body: kibanaConfig86Yml, - }, nil -} - -// KibanaConfig80File is the Kibana config file for 8.0 stack family (8.0 to 8.1) -const KibanaConfig80File configFile = "kibana.config.80.yml" - -//go:embed _static/kibana_config_80.yml -var kibanaConfig80Yml string - -func newKibanaConfig80(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(KibanaConfig80File), - path: filepath.Join(profilePath, profileStackPath, string(KibanaConfig80File)), - body: kibanaConfig80Yml, - }, nil -} - -// ElasticsearchConfigDefaultFile is the default Elasticsearch config file -const ElasticsearchConfigDefaultFile configFile = "elasticsearch.config.default.yml" - -//go:embed _static/elasticsearch_config_default.yml -var elasticsearchConfigDefaultYml string - -func newElasticsearchConfigDefault(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(ElasticsearchConfigDefaultFile), - path: filepath.Join(profilePath, profileStackPath, string(ElasticsearchConfigDefaultFile)), - body: elasticsearchConfigDefaultYml, - }, nil -} - -// ElasticsearchConfig8xFile is the Elasticsearch config file for 8.x stack family -const ElasticsearchConfig8xFile configFile = "elasticsearch.config.8x.yml" - -//go:embed _static/elasticsearch_config_8x.yml -var elasticsearchConfig8xYml string - -func newElasticsearchConfig8x(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(ElasticsearchConfig8xFile), - path: filepath.Join(profilePath, profileStackPath, string(ElasticsearchConfig8xFile)), - body: elasticsearchConfig8xYml, - }, nil -} - -// ElasticsearchConfig8xFile is the Elasticsearch config file for 8.x stack family (8.2 to 8.6) -const ElasticsearchConfig86File configFile = "elasticsearch.config.86.yml" - -func newElasticsearchConfig86(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(ElasticsearchConfig86File), - path: filepath.Join(profilePath, profileStackPath, string(ElasticsearchConfig86File)), - body: elasticsearchConfig8xYml, - }, nil -} - -// ElasticsearchConfig80File is the Elasticsearch virtual config file name for 8.0 stack family (8.0 to 8.1) -// This file does not exist in the source code, since it's identical to the 8x config file. -const ElasticsearchConfig80File configFile = "elasticsearch.config.80.yml" - -func newElasticsearchConfig80(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(ElasticsearchConfig80File), - path: filepath.Join(profilePath, profileStackPath, string(ElasticsearchConfig80File)), - body: elasticsearchConfig8xYml, - }, nil -} - -// PackageRegistryConfigFile is the config file for the Elastic Package registry -const PackageRegistryConfigFile configFile = "package-registry.config.yml" - -//go:embed _static/package_registry.yml -var packageRegistryConfigYml string - -// newPackageRegistryConfig returns a Managed Config -func newPackageRegistryConfig(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(PackageRegistryConfigFile), - path: filepath.Join(profilePath, profileStackPath, string(PackageRegistryConfigFile)), - body: packageRegistryConfigYml, - }, nil -} - -// PackageRegistryBaseImage is the base Docker image of the Elastic Package Registry. -const PackageRegistryBaseImage = "docker.elastic.co/package-registry/package-registry:v1.19.0" - -// PackageRegistryDockerfileFile is the dockerfile for the Elastic package registry -const PackageRegistryDockerfileFile configFile = "Dockerfile.package-registry" - -//go:embed _static/Dockerfile.package-registry -var packageRegistryDockerfileTmpl string -var packageRegistryDockerfile = strings.Replace(packageRegistryDockerfileTmpl, - "__BASE_IMAGE__", PackageRegistryBaseImage, -1) - -// newPackageRegistryDockerfile returns a new config for the package-registry -func newPackageRegistryDockerfile(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(PackageRegistryDockerfileFile), - path: filepath.Join(profilePath, profileStackPath, string(PackageRegistryDockerfileFile)), - body: packageRegistryDockerfile, - }, nil -} - -// ElasticAgent80EnvFile is the .env for the 8.0 stack. -// This file does not exist in the source code, since it's identical to the 8x env file. -const ElasticAgent80EnvFile configFile = "elastic-agent.80.env" - -func newElasticAgent80Env(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(ElasticAgent80EnvFile), - path: filepath.Join(profilePath, profileStackPath, string(ElasticAgent80EnvFile)), - body: elasticAgent8xEnv, - }, nil -} - -// ElasticAgent86EnvFile is the .env for the 8.6 stack. -// This file does not exist in the source code, since it's identical to the 8x env file. -const ElasticAgent86EnvFile configFile = "elastic-agent.86.env" - -func newElasticAgent86Env(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(ElasticAgent86EnvFile), - path: filepath.Join(profilePath, profileStackPath, string(ElasticAgent86EnvFile)), - body: elasticAgent8xEnv, - }, nil -} - -// ElasticAgent8xEnvFile is the .env for the 8x stack. -const ElasticAgent8xEnvFile configFile = "elastic-agent.8x.env" - -//go:embed _static/elastic-agent_8x.env -var elasticAgent8xEnv string - -func newElasticAgent8xEnv(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(ElasticAgent8xEnvFile), - path: filepath.Join(profilePath, profileStackPath, string(ElasticAgent8xEnvFile)), - body: elasticAgent8xEnv, - }, nil -} - -// ElasticAgentDefaultEnvFile is the default .env file. -const ElasticAgentDefaultEnvFile configFile = "elastic-agent.default.env" - -//go:embed _static/elastic-agent_default.env -var elasticAgentDefaultEnv string - -func newElasticAgentDefaultEnv(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(ElasticAgentDefaultEnvFile), - path: filepath.Join(profilePath, profileStackPath, string(ElasticAgentDefaultEnvFile)), - body: elasticAgentDefaultEnv, - }, nil -} diff --git a/internal/profile/_static/Dockerfile.package-registry b/internal/stack/_static/Dockerfile.package-registry.tmpl similarity index 50% rename from internal/profile/_static/Dockerfile.package-registry rename to internal/stack/_static/Dockerfile.package-registry.tmpl index e6ff25594..848cd1aa3 100644 --- a/internal/profile/_static/Dockerfile.package-registry +++ b/internal/stack/_static/Dockerfile.package-registry.tmpl @@ -1,4 +1,5 @@ -FROM __BASE_IMAGE__ +FROM {{ fact "registry_base_image" }} + ARG PROFILE # Disable package validation (already done). @@ -7,5 +8,5 @@ ENV EPR_DISABLE_PACKAGE_VALIDATION=true ENV EPR_FEATURE_PROXY_MODE=true ENV EPR_PROXY_TO=https://epr.elastic.co -COPY profiles/${PROFILE}/stack/package-registry.config.yml /package-registry/config.yml -COPY stack/development/ /packages/development \ No newline at end of file +COPY profiles/${PROFILE}/stack/package-registry.yml /package-registry/config.yml +COPY stack/development/ /packages/development diff --git a/internal/install/_static/GeoLite2-ASN.mmdb b/internal/stack/_static/GeoLite2-ASN.mmdb similarity index 100% rename from internal/install/_static/GeoLite2-ASN.mmdb rename to internal/stack/_static/GeoLite2-ASN.mmdb diff --git a/internal/install/_static/GeoLite2-City.mmdb b/internal/stack/_static/GeoLite2-City.mmdb similarity index 100% rename from internal/install/_static/GeoLite2-City.mmdb rename to internal/stack/_static/GeoLite2-City.mmdb diff --git a/internal/install/_static/GeoLite2-Country.mmdb b/internal/stack/_static/GeoLite2-Country.mmdb similarity index 100% rename from internal/install/_static/GeoLite2-Country.mmdb rename to internal/stack/_static/GeoLite2-Country.mmdb diff --git a/internal/profile/_static/docker-compose-stack.yml b/internal/stack/_static/docker-compose-stack.yml.tmpl similarity index 84% rename from internal/profile/_static/docker-compose-stack.yml rename to internal/stack/_static/docker-compose-stack.yml.tmpl index 0ad5b8b3e..3e7371fb5 100644 --- a/internal/profile/_static/docker-compose-stack.yml +++ b/internal/stack/_static/docker-compose-stack.yml.tmpl @@ -1,19 +1,21 @@ +{{ $username := fact "username" }} +{{ $password := fact "password" }} version: '2.3' services: elasticsearch: image: "${ELASTICSEARCH_IMAGE_REF}" healthcheck: - test: "curl -s --cacert /usr/share/elasticsearch/config/certs/ca-cert.pem -f -u elastic:changeme https://127.0.0.1:9200/_cat/health | cut -f4 -d' ' | grep -E '(green|yellow)'" + test: "curl -s --cacert /usr/share/elasticsearch/config/certs/ca-cert.pem -f -u {{ $username }}:{{ $password }} https://127.0.0.1:9200/_cat/health | cut -f4 -d' ' | grep -E '(green|yellow)'" start_period: 300s interval: 5s environment: - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - - "ELASTIC_PASSWORD=changeme" + - "ELASTIC_PASSWORD={{ $password }}" volumes: - - "./elasticsearch.config.${STACK_VERSION_VARIANT}.yml:/usr/share/elasticsearch/config/elasticsearch.yml" + - "./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml" - "../certs/elasticsearch:/usr/share/elasticsearch/config/certs" - - "../../../stack/ingest-geoip:/usr/share/elasticsearch/config/ingest-geoip" - - "../../../stack/service_tokens:/usr/share/elasticsearch/config/service_tokens" + - "./ingest-geoip:/usr/share/elasticsearch/config/ingest-geoip" + - "./service_tokens:/usr/share/elasticsearch/config/service_tokens" ports: - "127.0.0.1:9200:9200" @@ -40,9 +42,9 @@ services: # Is there a better way to add certificates to Kibana/Fleet? - "NODE_EXTRA_CA_CERTS=/usr/share/kibana/config/certs/ca-cert.pem" volumes: - - "./kibana.config.${STACK_VERSION_VARIANT}.yml:/usr/share/kibana/config/kibana.yml" + - "./kibana.yml:/usr/share/kibana/config/kibana.yml" - "../certs/kibana:/usr/share/kibana/config/certs" - - "../../../stack/healthcheck.sh:/usr/share/kibana/healthcheck.sh" + - "./kibana_healthcheck.sh:/usr/share/kibana/healthcheck.sh" ports: - "127.0.0.1:5601:5601" @@ -55,7 +57,7 @@ services: package-registry: build: context: ../../../ - dockerfile: "${STACK_PATH}/Dockerfile.package-registry" + dockerfile: "./profiles/${PROFILE_NAME}/stack/Dockerfile.package-registry" args: PROFILE: "${PROFILE_NAME}" healthcheck: @@ -129,7 +131,7 @@ services: retries: 180 interval: 5s hostname: docker-fleet-agent - env_file: "./elastic-agent.${STACK_VERSION_VARIANT}.env" + env_file: "./elastic-agent.env" volumes: - "../certs/ca-cert.pem:/etc/ssl/certs/elastic-package.pem" - type: bind diff --git a/internal/profile/_static/elastic-agent_8x.env b/internal/stack/_static/elastic-agent.env.tmpl similarity index 64% rename from internal/profile/_static/elastic-agent_8x.env rename to internal/stack/_static/elastic-agent.env.tmpl index 1fea8128c..70a0b5cb5 100644 --- a/internal/profile/_static/elastic-agent_8x.env +++ b/internal/stack/_static/elastic-agent.env.tmpl @@ -1,5 +1,8 @@ +{{ $version := fact "agent_version" }} FLEET_ENROLL=1 -FLEET_TOKEN_POLICY_NAME=Elastic-Agent (elastic-package) FLEET_URL=https://fleet-server:8220 KIBANA_FLEET_HOST=https://kibana:5601 KIBANA_HOST=https://kibana:5601 +{{ if not (semverLessThan $version "8.0.0") }} +FLEET_TOKEN_POLICY_NAME=Elastic-Agent (elastic-package) +{{ end }} diff --git a/internal/profile/_static/elasticsearch_config_default.yml b/internal/stack/_static/elasticsearch.yml.tmpl similarity index 86% rename from internal/profile/_static/elasticsearch_config_default.yml rename to internal/stack/_static/elasticsearch.yml.tmpl index f72696cd3..9e14ec9a5 100644 --- a/internal/profile/_static/elasticsearch_config_default.yml +++ b/internal/stack/_static/elasticsearch.yml.tmpl @@ -7,15 +7,17 @@ indices.id_field_data.enabled: true xpack.license.self_generated.type: "trial" xpack.security.enabled: true xpack.security.authc.api_key.enabled: true - xpack.security.http.ssl.enabled: true xpack.security.http.ssl.key: "certs/key.pem" xpack.security.http.ssl.certificate: "certs/cert.pem" +ingest.geoip.downloader.enabled: false + +{{ $version := fact "elasticsearch_version" }} +{{ if semverLessThan $version "8.0.0" }} script.max_compilations_rate: "use-context" script.context.template.max_compilations_rate: "unlimited" script.context.ingest.cache_max_size: 2000 script.context.processor_conditional.cache_max_size: 2000 script.context.template.cache_max_size: 2000 - -ingest.geoip.downloader.enabled: false +{{ end }} diff --git a/internal/profile/_static/kibana_config_8x.yml b/internal/stack/_static/kibana.yml.tmpl similarity index 79% rename from internal/profile/_static/kibana_config_8x.yml rename to internal/stack/_static/kibana.yml.tmpl index 72cead61c..a2ace75b1 100644 --- a/internal/profile/_static/kibana_config_8x.yml +++ b/internal/stack/_static/kibana.yml.tmpl @@ -1,3 +1,4 @@ +{{ $version := fact "kibana_version" }} server.name: kibana server.host: "0.0.0.0" server.ssl.enabled: true @@ -7,20 +8,37 @@ server.ssl.certificateAuthorities: ["/usr/share/kibana/config/certs/ca-cert.pem" elasticsearch.hosts: [ "https://elasticsearch:9200" ] elasticsearch.ssl.certificateAuthorities: "/usr/share/kibana/config/certs/ca-cert.pem" + +{{ if semverLessThan $version "8.0.0" }} +elasticsearch.username: {{ fact "username" }} +elasticsearch.password: {{ fact "password" }} + +xpack.monitoring.ui.container.elasticsearch.enabled: true +xpack.fleet.enabled: true +xpack.fleet.agents.elasticsearch.host: "https://elasticsearch:9200" +{{ else }} elasticsearch.serviceAccountToken: "AAEAAWVsYXN0aWMva2liYW5hL2VsYXN0aWMtcGFja2FnZS1raWJhbmEtdG9rZW46b2x4b051SWNRa0tYMHdXazdLWmFBdw" monitoring.ui.container.elasticsearch.enabled: true +xpack.fleet.agents.elasticsearch.hosts: ["https://elasticsearch:9200"] +{{ end }} + xpack.fleet.registryUrl: "https://package-registry:8080" xpack.fleet.agents.enabled: true -xpack.fleet.agents.elasticsearch.hosts: ["https://elasticsearch:9200"] xpack.fleet.agents.fleet_server.hosts: ["https://fleet-server:8220"] + +{{ if not (semverLessThan $version "8.7.0") }} xpack.fleet.enableExperimental: ["experimentalDataStreamSettings"] # Enable experimental toggles in Fleet UI +{{ end }} xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" +{{ if not (semverLessThan $version "8.2.0") }} xpack.cloudSecurityPosture.enabled: true +{{ end }} +{{ if not (semverLessThan $version "8.0.0") }} xpack.fleet.packages: - name: system version: latest @@ -60,3 +78,4 @@ xpack.fleet.outputs: ca_trusted_fingerprint: "${ELASTIC_PACKAGE_CA_TRUSTED_FINGERPRINT}" is_default: true is_default_monitoring: true +{{ end }} diff --git a/internal/install/_static/kibana_healthcheck.sh b/internal/stack/_static/kibana_healthcheck.sh.tmpl similarity index 64% rename from internal/install/_static/kibana_healthcheck.sh rename to internal/stack/_static/kibana_healthcheck.sh.tmpl index 09bc39c35..386b55add 100644 --- a/internal/install/_static/kibana_healthcheck.sh +++ b/internal/stack/_static/kibana_healthcheck.sh.tmpl @@ -3,4 +3,4 @@ set -e curl -s --cacert /usr/share/kibana/config/certs/ca-cert.pem -f https://localhost:5601/login | grep kbn-injected-metadata 2>&1 >/dev/null -curl -s --cacert /usr/share/kibana/config/certs/ca-cert.pem -f -u elastic:changeme "https://elasticsearch:9200/_cat/indices/.security-*?h=health" | grep -v red +curl -s --cacert /usr/share/kibana/config/certs/ca-cert.pem -f -u {{ fact "username" }}:{{ fact "password" }} "https://elasticsearch:9200/_cat/indices/.security-*?h=health" | grep -v red diff --git a/internal/profile/_static/package_registry.yml b/internal/stack/_static/package-registry.yml similarity index 100% rename from internal/profile/_static/package_registry.yml rename to internal/stack/_static/package-registry.yml diff --git a/internal/install/_static/service_tokens b/internal/stack/_static/service_tokens similarity index 100% rename from internal/install/_static/service_tokens rename to internal/stack/_static/service_tokens diff --git a/internal/stack/boot.go b/internal/stack/boot.go index b2262a7ed..f499a8331 100644 --- a/internal/stack/boot.go +++ b/internal/stack/boot.go @@ -23,6 +23,18 @@ const DockerComposeProjectName = "elastic-package-stack" // BootUp function boots up the Elastic stack. func BootUp(options Options) error { + // Print information before starting the stack, for cases where + // this is executed in the foreground, without daemon mode. + config := Config{ + Provider: ProviderCompose, + ElasticsearchHost: "https://127.0.0.1:9200", + ElasticsearchUsername: elasticsearchUsername, + ElasticsearchPassword: elasticsearchPassword, + KibanaHost: "https://127.0.0.1:5601", + CACertFile: options.Profile.Path(CACertificateFile), + } + printUserConfig(options.Printer, config) + buildPackagesPath, found, err := builder.FindBuildPackagesDirectory() if err != nil { return errors.Wrap(err, "finding build packages directory failed") @@ -46,11 +58,16 @@ func BootUp(options Options) error { } } - fmt.Println("Packages from the following directories will be loaded into the package-registry:") - fmt.Println("- built-in packages (package-storage:snapshot Docker image)") + options.Printer.Println("Local package-registry will serve packages from these sources:") + options.Printer.Println("- Proxy to https://epr.elastic.co") if found { - fmt.Printf("- %s\n", buildPackagesPath) + options.Printer.Printf("- Local directory %s\n", buildPackagesPath) + } + + err = applyResources(options.Profile, options.StackVersion) + if err != nil { + return errors.Wrap(err, "creating stack files failed") } err = dockerComposeBuild(options) @@ -73,6 +90,11 @@ func BootUp(options Options) error { return errors.Wrap(err, "running docker-compose failed") } + err = storeConfig(options.Profile, config) + if err != nil { + return errors.Wrap(err, "failed to store config") + } + return nil } diff --git a/internal/profile/certs.go b/internal/stack/certs.go similarity index 69% rename from internal/profile/certs.go rename to internal/stack/certs.go index 49f6b48f2..ac63774e6 100644 --- a/internal/profile/certs.go +++ b/internal/stack/certs.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package profile +package stack import ( "bytes" @@ -11,6 +11,8 @@ import ( "io" "path/filepath" + "github.com/elastic/go-resource" + "github.com/elastic/elastic-package/internal/certs" ) @@ -28,39 +30,41 @@ var ( CertificatesDirectory = "certs" // CACertificateFile is the path to the CA certificate file inside a profile. - CACertificateFile = configFile(filepath.Join(CertificatesDirectory, "ca-cert.pem")) + CACertificateFile = filepath.Join(CertificatesDirectory, "ca-cert.pem") // CAKeyFile is the path to the CA key file inside a profile. - CAKeyFile = configFile(filepath.Join(CertificatesDirectory, "ca-key.pem")) + CAKeyFile = filepath.Join(CertificatesDirectory, "ca-key.pem") // CAEnvFile is the path to the file with environment variables about the CA. - CAEnvFile = configFile(filepath.Join(CertificatesDirectory, "ca.env")) + CAEnvFile = filepath.Join(CertificatesDirectory, "ca.env") ) // initTLSCertificates initializes all the certificates needed to run the services // managed by elastic-package stack. It includes a CA, and a pair of keys and // certificates for each service. -func initTLSCertificates(profilePath string, configMap map[configFile]*simpleFile) error { +func initTLSCertificates(fileProvider string, profilePath string) ([]resource.Resource, error) { certsDir := filepath.Join(profilePath, CertificatesDirectory) caCertFile := filepath.Join(profilePath, string(CACertificateFile)) caKeyFile := filepath.Join(profilePath, string(CAKeyFile)) envFile := filepath.Join(profilePath, string(CAEnvFile)) + var resources []resource.Resource + ca, err := initCA(caCertFile, caKeyFile) if err != nil { - return err + return nil, err } - err = certWriterToSimpleFile(configMap, profilePath, caCertFile, ca.WriteCert) + resources, err = certWriteToResource(resources, fileProvider, profilePath, caCertFile, ca.WriteCert) if err != nil { - return err + return nil, err } - err = certWriterToSimpleFile(configMap, profilePath, caKeyFile, ca.WriteKey) + resources, err = certWriteToResource(resources, fileProvider, profilePath, caKeyFile, ca.WriteKey) if err != nil { - return err + return nil, err } - err = certWriterToSimpleFile(configMap, profilePath, envFile, ca.WriteEnv) + resources, err = certWriteToResource(resources, fileProvider, profilePath, envFile, ca.WriteEnv) if err != nil { - return err + return nil, err } for _, service := range tlsServices { @@ -70,47 +74,47 @@ func initTLSCertificates(profilePath string, configMap map[configFile]*simpleFil keyFile := filepath.Join(certsDir, "key.pem") cert, err := initServiceTLSCertificates(ca, caCertFile, certFile, keyFile, service) if err != nil { - return err + return nil, err } - err = certWriterToSimpleFile(configMap, profilePath, certFile, cert.WriteCert) + resources, err = certWriteToResource(resources, fileProvider, profilePath, certFile, cert.WriteCert) if err != nil { - return err + return nil, err } - err = certWriterToSimpleFile(configMap, profilePath, keyFile, cert.WriteKey) + resources, err = certWriteToResource(resources, fileProvider, profilePath, keyFile, cert.WriteKey) if err != nil { - return err + return nil, err } // Write the CA also in the service directory, so only a directory needs to be mounted // for services that need to configure the CA to validate other services certificates. - err = certWriterToSimpleFile(configMap, profilePath, caFile, ca.WriteCert) + resources, err = certWriteToResource(resources, fileProvider, profilePath, caFile, ca.WriteCert) if err != nil { - return err + return nil, err } } - return nil + return resources, nil } -func certWriterToSimpleFile(configMap map[configFile]*simpleFile, profilePath string, absPath string, writeFile func(w io.Writer) error) error { +func certWriteToResource(resources []resource.Resource, fileProvider string, profilePath string, absPath string, write func(w io.Writer) error) ([]resource.Resource, error) { path, err := filepath.Rel(profilePath, absPath) if err != nil { - return err + return resources, err } var buf bytes.Buffer - err = writeFile(&buf) + err = write(&buf) if err != nil { - return err + return resources, err } - configMap[configFile(path)] = &simpleFile{ - name: path, - path: absPath, - body: buf.String(), - } - return nil + return append(resources, &resource.File{ + Provider: fileProvider, + Path: path, + CreateParent: true, + Content: resource.FileContentLiteral(buf.String()), + }), nil } func initCA(certFile, keyFile string) (*certs.Issuer, error) { diff --git a/internal/profile/certs_test.go b/internal/stack/certs_test.go similarity index 81% rename from internal/profile/certs_test.go rename to internal/stack/certs_test.go index 5d8a9154e..2d7daf2eb 100644 --- a/internal/profile/certs_test.go +++ b/internal/stack/certs_test.go @@ -2,13 +2,14 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package profile +package stack import ( "os" "path/filepath" "testing" + "github.com/elastic/go-resource" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -20,11 +21,15 @@ func TestTLSCertsInitialization(t *testing.T) { assert.Error(t, verifyTLSCertificates(caCertFile, caCertFile, caKeyFile, "")) - configMap := make(map[configFile]*simpleFile) - err := initTLSCertificates(profilePath, configMap) + providerName := "test-file" + resources, err := initTLSCertificates(providerName, profilePath) require.NoError(t, err) - err = writeConfigFiles(configMap) + resourceManager := resource.NewManager() + resourceManager.RegisterProvider(providerName, &resource.FileProvider{ + Prefix: profilePath, + }) + _, err = resourceManager.Apply(resources) require.NoError(t, err) assert.NoError(t, verifyTLSCertificates(caCertFile, caCertFile, caKeyFile, "")) @@ -49,12 +54,11 @@ func TestTLSCertsInitialization(t *testing.T) { assert.Error(t, verifyTLSCertificates(caCertFile, serviceCertFile, serviceKeyFile, service)) // Check it is created again and is validated by the same CA. - configMap := make(map[configFile]*simpleFile) - err := initTLSCertificates(profilePath, configMap) + resources, err := initTLSCertificates(providerName, profilePath) require.NoError(t, err) - - err = writeConfigFiles(configMap) + _, err = resourceManager.Apply(resources) require.NoError(t, err) + assert.NoError(t, verifyTLSCertificates(caCertFile, serviceCertFile, serviceKeyFile, service)) }) } diff --git a/internal/stack/compose.go b/internal/stack/compose.go index a14a3aa22..a1e0bb00e 100644 --- a/internal/stack/compose.go +++ b/internal/stack/compose.go @@ -14,7 +14,6 @@ import ( "github.com/elastic/elastic-package/internal/docker" "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/logger" - "github.com/elastic/elastic-package/internal/profile" ) type ServiceStatus struct { @@ -36,6 +35,7 @@ type envBuilder struct { vars []string } +// TODO: Use template variables instead of environment variables to parameterize docker-compose. func newEnvBuilder() *envBuilder { return new(envBuilder) } @@ -55,7 +55,7 @@ func (eb *envBuilder) build() []string { } func dockerComposeBuild(options Options) error { - c, err := compose.NewProject(DockerComposeProjectName, options.Profile.FetchPath(profile.SnapshotFile)) + c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile)) if err != nil { return errors.Wrap(err, "could not create docker compose project") } @@ -81,7 +81,7 @@ func dockerComposeBuild(options Options) error { } func dockerComposePull(options Options) error { - c, err := compose.NewProject(DockerComposeProjectName, options.Profile.FetchPath(profile.SnapshotFile)) + c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile)) if err != nil { return errors.Wrap(err, "could not create docker compose project") } @@ -107,7 +107,7 @@ func dockerComposePull(options Options) error { } func dockerComposeUp(options Options) error { - c, err := compose.NewProject(DockerComposeProjectName, options.Profile.FetchPath(profile.SnapshotFile)) + c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile)) if err != nil { return errors.Wrap(err, "could not create docker compose project") } @@ -139,7 +139,7 @@ func dockerComposeUp(options Options) error { } func dockerComposeDown(options Options) error { - c, err := compose.NewProject(DockerComposeProjectName, options.Profile.FetchPath(profile.SnapshotFile)) + c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile)) if err != nil { return errors.Wrap(err, "could not create docker compose project") } diff --git a/internal/stack/config.go b/internal/stack/config.go new file mode 100644 index 000000000..01b0cfadf --- /dev/null +++ b/internal/stack/config.go @@ -0,0 +1,92 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stack + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/elastic/elastic-package/internal/profile" +) + +var configFileName = "config.json" + +type Config struct { + Provider string `json:"provider,omitempty"` + Parameters map[string]string `json:"parameters,omitempty"` + + ElasticsearchHost string `json:"elasticsearch_host,omitempty"` + ElasticsearchUsername string `json:"elasticsearch_username,omitempty"` + ElasticsearchPassword string `json:"elasticsearch_password,omitempty"` + KibanaHost string `json:"kibana_host,omitempty"` + CACertFile string `json:"ca_cert_file,omitempty"` +} + +func configPath(profile *profile.Profile) string { + return profile.Path(profileStackPath, configFileName) +} + +func defaultConfig(profile *profile.Profile) Config { + return Config{ + Provider: DefaultProvider, + + // Hard-coded default values for backwards-compatibility. + ElasticsearchHost: "https://127.0.0.1:9200", + ElasticsearchUsername: elasticsearchUsername, + ElasticsearchPassword: elasticsearchPassword, + KibanaHost: "https://127.0.0.1:5601", + CACertFile: profile.Path(CACertificateFile), + } +} + +func LoadConfig(profile *profile.Profile) (Config, error) { + d, err := os.ReadFile(configPath(profile)) + if errors.Is(err, os.ErrNotExist) { + return defaultConfig(profile), nil + } + if err != nil { + return Config{}, fmt.Errorf("failed to read stack config: %w", err) + } + + var config Config + err = json.Unmarshal(d, &config) + if err != nil { + return Config{}, fmt.Errorf("failed to decode stack config: %w", err) + } + + return config, nil +} + +func storeConfig(profile *profile.Profile, config Config) error { + d, err := json.Marshal(config) + if err != nil { + return fmt.Errorf("failed to encode stack config: %w", err) + } + + err = os.MkdirAll(filepath.Dir(configPath(profile)), 0755) + if err != nil { + return fmt.Errorf("failed to create parent directory: %w", err) + } + + err = os.WriteFile(configPath(profile), d, 0644) + if err != nil { + return fmt.Errorf("failed to write stack config: %s", err) + } + + return nil +} + +func printUserConfig(printer Printer, config Config) { + if printer == nil { + return + } + printer.Printf("Elasticsearch host: %s\n", config.ElasticsearchHost) + printer.Printf("Kibana host: %s\n", config.KibanaHost) + printer.Printf("Username: %s\n", config.ElasticsearchUsername) + printer.Printf("Password: %s\n", config.ElasticsearchPassword) +} diff --git a/internal/stack/dump.go b/internal/stack/dump.go index 008272afb..7a80a2707 100644 --- a/internal/stack/dump.go +++ b/internal/stack/dump.go @@ -52,7 +52,7 @@ func dumpStackLogs(options DumpOptions) error { return errors.Wrapf(err, "can't create output location (path: %s)", logsPath) } - snapshotPath := options.Profile.FetchPath(profile.SnapshotFile) + snapshotPath := options.Profile.Path(profileStackPath, SnapshotFile) for _, serviceName := range observedServices { logger.Debugf("Dump stack logs for %s", serviceName) diff --git a/internal/stack/initconfig.go b/internal/stack/initconfig.go index c23859150..916db46be 100644 --- a/internal/stack/initconfig.go +++ b/internal/stack/initconfig.go @@ -5,14 +5,6 @@ package stack import ( - "fmt" - "os" - - "github.com/pkg/errors" - "gopkg.in/yaml.v3" - - "github.com/elastic/elastic-package/internal/compose" - "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/profile" ) @@ -24,58 +16,17 @@ type InitConfig struct { CACertificatePath string } -func StackInitConfig(elasticStackProfile *profile.Profile) (*InitConfig, error) { - // Read Elasticsearch username and password from Kibana configuration file. - // FIXME read credentials from correct Kibana config file, not default - body, err := os.ReadFile(elasticStackProfile.FetchPath(profile.KibanaConfigDefaultFile)) - if err != nil { - return nil, errors.Wrap(err, "error reading Kibana config file") - } - - var kibanaCfg struct { - ElasticsearchUsername string `yaml:"elasticsearch.username"` - ElasticsearchPassword string `yaml:"elasticsearch.password"` - } - err = yaml.Unmarshal(body, &kibanaCfg) - if err != nil { - return nil, errors.Wrap(err, "unmarshalling Kibana configuration failed") - } - - // Read Elasticsearch and Kibana hostnames from Elastic Stack Docker Compose configuration file. - p, err := compose.NewProject(DockerComposeProjectName, elasticStackProfile.FetchPath(profile.SnapshotFile)) +func StackInitConfig(profile *profile.Profile) (*InitConfig, error) { + config, err := LoadConfig(profile) if err != nil { - return nil, errors.Wrap(err, "could not create docker compose project") + return nil, err } - appConfig, err := install.Configuration() - if err != nil { - return nil, errors.Wrap(err, "can't read application configuration") - } - - serviceComposeConfig, err := p.Config(compose.CommandOptions{ - Env: newEnvBuilder(). - withEnvs(appConfig.StackImageRefs(install.DefaultStackVersion).AsEnv()). - withEnvs(elasticStackProfile.ComposeEnvVars()). - withEnv(stackVariantAsEnv(install.DefaultStackVersion)). - build(), - }) - if err != nil { - return nil, errors.Wrap(err, "could not get Docker Compose configuration for service") - } - - kib := serviceComposeConfig.Services["kibana"] - kibHostPort := fmt.Sprintf("https://%s:%d", kib.Ports[0].ExternalIP, kib.Ports[0].ExternalPort) - - es := serviceComposeConfig.Services["elasticsearch"] - esHostPort := fmt.Sprintf("https://%s:%d", es.Ports[0].ExternalIP, es.Ports[0].ExternalPort) - - caCert := elasticStackProfile.FetchPath(profile.CACertificateFile) - return &InitConfig{ - ElasticsearchHostPort: esHostPort, - ElasticsearchUsername: kibanaCfg.ElasticsearchUsername, - ElasticsearchPassword: kibanaCfg.ElasticsearchPassword, - KibanaHostPort: kibHostPort, - CACertificatePath: caCert, + ElasticsearchHostPort: config.ElasticsearchHost, + ElasticsearchUsername: config.ElasticsearchUsername, + ElasticsearchPassword: config.ElasticsearchPassword, + KibanaHostPort: config.KibanaHost, + CACertificatePath: config.CACertFile, }, nil } diff --git a/internal/stack/options.go b/internal/stack/options.go index 9fb263167..a1ed87782 100644 --- a/internal/stack/options.go +++ b/internal/stack/options.go @@ -14,4 +14,5 @@ type Options struct { Services []string Profile *profile.Profile + Printer Printer } diff --git a/internal/stack/providers.go b/internal/stack/providers.go new file mode 100644 index 000000000..7d672cd81 --- /dev/null +++ b/internal/stack/providers.go @@ -0,0 +1,79 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stack + +import ( + "fmt" + "strings" + + "github.com/elastic/elastic-package/internal/profile" +) + +const ( + ProviderCompose = "compose" +) + +var ( + DefaultProvider = ProviderCompose + SupportedProviders = []string{ + ProviderCompose, + } +) + +// Printer is the interface that can be used to print information on operations. +type Printer interface { + Print(i ...interface{}) + Println(i ...interface{}) + Printf(format string, i ...interface{}) +} + +// Provider is the implementation of a stack provider. +type Provider interface { + // BootUp starts a stack. + BootUp(Options) error + + // TearDown stops and/or removes a stack. + TearDown(Options) error + + // Update updates resources associated to a stack. + Update(Options) error + + // Dump dumps data for debug purpouses. + Dump(DumpOptions) (string, error) + + // Status obtains status information of the stack. + Status(Options) ([]ServiceStatus, error) +} + +// BuildProvider returns the provider for the given name. +func BuildProvider(name string, profile *profile.Profile) (Provider, error) { + switch name { + case "compose": + return &composeProvider{}, nil + } + return nil, fmt.Errorf("unknown provider %q, supported providers: %s", name, strings.Join(SupportedProviders, ", ")) +} + +type composeProvider struct{} + +func (*composeProvider) BootUp(options Options) error { + return BootUp(options) +} + +func (*composeProvider) TearDown(options Options) error { + return TearDown(options) +} + +func (*composeProvider) Update(options Options) error { + return Update(options) +} + +func (*composeProvider) Dump(options DumpOptions) (string, error) { + return Dump(options) +} + +func (*composeProvider) Status(options Options) ([]ServiceStatus, error) { + return Status() +} diff --git a/internal/stack/resources.go b/internal/stack/resources.go new file mode 100644 index 000000000..16143334c --- /dev/null +++ b/internal/stack/resources.go @@ -0,0 +1,163 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stack + +import ( + "embed" + "fmt" + "html/template" + "os" + "path/filepath" + "strings" + + "github.com/Masterminds/semver/v3" + "github.com/elastic/go-resource" + + "github.com/elastic/elastic-package/internal/profile" +) + +//go:embed _static +var static embed.FS + +const ( + // SnapshotFile is the docker-compose snapshot.yml file name. + SnapshotFile = "snapshot.yml" + + // ElasticsearchConfigFile is the elasticsearch config file. + ElasticsearchConfigFile = "elasticsearch.yml" + + // KibanaConfigFile is the kibana config file. + KibanaConfigFile = "kibana.yml" + + // KibanaHealthcheckFile is the kibana healthcheck. + KibanaHealthcheckFile = "kibana_healthcheck.sh" + + // PackageRegistryConfigFile is the config file for the Elastic Package registry + PackageRegistryConfigFile = "package-registry.yml" + + // PackageRegistryBaseImage is the base Docker image of the Elastic Package Registry. + PackageRegistryBaseImage = "docker.elastic.co/package-registry/package-registry:v1.19.0" + + // ElasticAgentEnvFile is the elastic agent environment variables file. + ElasticAgentEnvFile = "elastic-agent.env" + + profileStackPath = "stack" + + elasticsearchUsername = "elastic" + elasticsearchPassword = "changeme" +) + +var ( + templateFuncs = template.FuncMap{ + "semverLessThan": semverLessThan, + } + staticSource = resource.NewSourceFS(static).WithTemplateFuncs(templateFuncs) + stackResources = []resource.Resource{ + &resource.File{ + Path: "Dockerfile.package-registry", + Content: staticSource.Template("_static/Dockerfile.package-registry.tmpl"), + }, + &resource.File{ + Path: SnapshotFile, + Content: staticSource.Template("_static/docker-compose-stack.yml.tmpl"), + }, + &resource.File{ + Path: ElasticsearchConfigFile, + Content: staticSource.Template("_static/elasticsearch.yml.tmpl"), + }, + &resource.File{ + Path: "service_tokens", + Content: staticSource.File("_static/service_tokens"), + }, + &resource.File{ + Path: "ingest-geoip/GeoLite2-ASN.mmdb", + CreateParent: true, + Content: staticSource.File("_static/GeoLite2-ASN.mmdb"), + }, + &resource.File{ + Path: "ingest-geoip/GeoLite2-City.mmdb", + CreateParent: true, + Content: staticSource.File("_static/GeoLite2-City.mmdb"), + }, + &resource.File{ + Path: "ingest-geoip/GeoLite2-Country.mmdb", + CreateParent: true, + Content: staticSource.File("_static/GeoLite2-Country.mmdb"), + }, + &resource.File{ + Path: KibanaConfigFile, + Content: staticSource.Template("_static/kibana.yml.tmpl"), + }, + &resource.File{ + Path: KibanaHealthcheckFile, + Content: staticSource.Template("_static/kibana_healthcheck.sh.tmpl"), + }, + &resource.File{ + Path: PackageRegistryConfigFile, + Content: staticSource.File("_static/package-registry.yml"), + }, + &resource.File{ + Path: ElasticAgentEnvFile, + Content: staticSource.Template("_static/elastic-agent.env.tmpl"), + }, + } +) + +func applyResources(profile *profile.Profile, stackVersion string) error { + stackDir := filepath.Join(profile.ProfilePath, profileStackPath) + + resourceManager := resource.NewManager() + resourceManager.AddFacter(resource.StaticFacter{ + "registry_base_image": PackageRegistryBaseImage, + "elasticsearch_version": stackVersion, + "kibana_version": stackVersion, + "agent_version": stackVersion, + + "username": elasticsearchUsername, + "password": elasticsearchPassword, + }) + + os.MkdirAll(stackDir, 0755) + resourceManager.RegisterProvider("file", &resource.FileProvider{ + Prefix: stackDir, + }) + resources := append([]resource.Resource{}, stackResources...) + + // Keeping certificates in the profile directory for backwards compatibility reasons. + resourceManager.RegisterProvider("certs", &resource.FileProvider{ + Prefix: profile.ProfilePath, + }) + certResources, err := initTLSCertificates("certs", profile.ProfilePath) + if err != nil { + return fmt.Errorf("failed to create TLS files: %w", err) + } + resources = append(resources, certResources...) + + results, err := resourceManager.Apply(resources) + if err != nil { + var errors []string + for _, result := range results { + if err := result.Err(); err != nil { + errors = append(errors, err.Error()) + } + } + return fmt.Errorf("%w: %s", err, strings.Join(errors, ", ")) + } + + return nil +} + +func semverLessThan(a, b string) (bool, error) { + sa, err := semver.NewVersion(a) + if err != nil { + return false, err + } + sb, err := semver.NewVersion(b) + if err != nil { + return false, err + } + + return sa.LessThan(sb), nil +} diff --git a/internal/stack/shellinit.go b/internal/stack/shellinit.go index 55f459b60..2e22e1a69 100644 --- a/internal/stack/shellinit.go +++ b/internal/stack/shellinit.go @@ -61,16 +61,15 @@ const ( export %s=%s export %s=%s export %s=%s -export %s=%s -` +export %s=%s` + // fish shell init code. // fish shell is similar but not compliant to POSIX. fishTemplate = `set -x %s %s; set -x %s %s; set -x %s %s; set -x %s %s; -set -x %s %s; -` +set -x %s %s;` // PowerShell init code. // Output to be evaluated with `elastic-package stack shellinit | Invoke-Expression diff --git a/internal/stack/update.go b/internal/stack/update.go index ad5215016..26188d641 100644 --- a/internal/stack/update.go +++ b/internal/stack/update.go @@ -8,12 +8,16 @@ import ( "github.com/pkg/errors" "github.com/elastic/elastic-package/internal/docker" - "github.com/elastic/elastic-package/internal/profile" ) // Update pulls down the most recent versions of the Docker images. func Update(options Options) error { - err := docker.Pull(profile.PackageRegistryBaseImage) + err := applyResources(options.Profile, options.StackVersion) + if err != nil { + return errors.Wrap(err, "creating stack files failed") + } + + err = docker.Pull(PackageRegistryBaseImage) if err != nil { return errors.Wrap(err, "pulling package-registry docker image failed") } diff --git a/internal/install/_static/Dockerfile.terraform_deployer b/internal/testrunner/runners/system/servicedeployer/_static/Dockerfile.terraform_deployer similarity index 100% rename from internal/install/_static/Dockerfile.terraform_deployer rename to internal/testrunner/runners/system/servicedeployer/_static/Dockerfile.terraform_deployer diff --git a/internal/install/_static/docker-custom-agent-base.yml b/internal/testrunner/runners/system/servicedeployer/_static/docker-custom-agent-base.yml similarity index 100% rename from internal/install/_static/docker-custom-agent-base.yml rename to internal/testrunner/runners/system/servicedeployer/_static/docker-custom-agent-base.yml diff --git a/internal/install/_static/terraform_deployer.yml b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer.yml similarity index 100% rename from internal/install/_static/terraform_deployer.yml rename to internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer.yml diff --git a/internal/install/_static/terraform_deployer_run.sh b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh similarity index 100% rename from internal/install/_static/terraform_deployer_run.sh rename to internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh diff --git a/internal/testrunner/runners/system/servicedeployer/compose.go b/internal/testrunner/runners/system/servicedeployer/compose.go index b7be23cd5..568ce7f0a 100644 --- a/internal/testrunner/runners/system/servicedeployer/compose.go +++ b/internal/testrunner/runners/system/servicedeployer/compose.go @@ -24,7 +24,7 @@ import ( // a Docker Compose file. type DockerComposeServiceDeployer struct { ymlPaths []string - sv ServiceVariant + variant ServiceVariant } type dockerComposeDeployedService struct { @@ -32,14 +32,14 @@ type dockerComposeDeployedService struct { ymlPaths []string project string - sv ServiceVariant + variant ServiceVariant } // NewDockerComposeServiceDeployer returns a new instance of a DockerComposeServiceDeployer. func NewDockerComposeServiceDeployer(ymlPaths []string, sv ServiceVariant) (*DockerComposeServiceDeployer, error) { return &DockerComposeServiceDeployer{ ymlPaths: ymlPaths, - sv: sv, + variant: sv, }, nil } @@ -49,7 +49,7 @@ func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer service := dockerComposeDeployedService{ ymlPaths: d.ymlPaths, project: "elastic-package-service", - sv: d.sv, + variant: d.variant, } outCtxt := inCtxt @@ -71,15 +71,15 @@ func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer } // Boot up service - if d.sv.active() { - logger.Infof("Using service variant: %s", d.sv.String()) + if d.variant.active() { + logger.Infof("Using service variant: %s", d.variant.String()) } serviceName := inCtxt.Name opts := compose.CommandOptions{ Env: append( []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, outCtxt.Logs.Folder.Local)}, - d.sv.Env...), + d.variant.Env...), ExtraArgs: []string{"--build", "-d"}, } err = p.Up(opts) @@ -138,7 +138,7 @@ func (s *dockerComposeDeployedService) Signal(signal string) error { opts := compose.CommandOptions{ Env: append( []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, s.ctxt.Logs.Folder.Local)}, - s.sv.Env...), + s.variant.Env...), ExtraArgs: []string{"-s", signal}, } if s.ctxt.Name != "" { @@ -166,14 +166,14 @@ func (s *dockerComposeDeployedService) TearDown() error { opts := compose.CommandOptions{ Env: append( []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, s.ctxt.Logs.Folder.Local)}, - s.sv.Env...), + s.variant.Env...), } processServiceContainerLogs(p, opts, s.ctxt.Name) if err := p.Down(compose.CommandOptions{ Env: append( []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, s.ctxt.Logs.Folder.Local)}, - s.sv.Env...), + s.variant.Env...), ExtraArgs: []string{"--volumes"}, // Remove associated volumes. }); err != nil { return errors.Wrap(err, "could not shut down service using Docker Compose") diff --git a/internal/testrunner/runners/system/servicedeployer/custom_agent.go b/internal/testrunner/runners/system/servicedeployer/custom_agent.go index 206452c2e..02165d0e0 100644 --- a/internal/testrunner/runners/system/servicedeployer/custom_agent.go +++ b/internal/testrunner/runners/system/servicedeployer/custom_agent.go @@ -8,6 +8,7 @@ import ( _ "embed" "fmt" "os" + "path/filepath" "github.com/pkg/errors" @@ -21,18 +22,25 @@ import ( "github.com/elastic/elastic-package/internal/stack" ) -const dockerCustomAgentName = "docker-custom-agent" +const ( + dockerCustomAgentName = "docker-custom-agent" + dockerCustomAgentDir = "docker_custom_agent" + dockerCustomAgentDockerfile = "docker-custom-agent-base.yml" +) + +//go:embed _static/docker-custom-agent-base.yml +var dockerCustomAgentDockerfileContent []byte // CustomAgentDeployer knows how to deploy a custom elastic-agent defined via // a Docker Compose file. type CustomAgentDeployer struct { - cfg string + dockerComposeFile string } // NewCustomAgentDeployer returns a new instance of a deployedCustomAgent. -func NewCustomAgentDeployer(cfgPath string) (*CustomAgentDeployer, error) { +func NewCustomAgentDeployer(dockerComposeFile string) (*CustomAgentDeployer, error) { return &CustomAgentDeployer{ - cfg: cfgPath, + dockerComposeFile: dockerComposeFile, }, nil } @@ -66,15 +74,20 @@ func (d *CustomAgentDeployer) SetUp(inCtxt ServiceContext) (DeployedService, err fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), ) - ymlPaths, err := d.loadComposeDefinitions() + configDir, err := d.installDockerfile() if err != nil { - return nil, err + return nil, errors.Wrap(err, "could not create resources for custom agent") + } + + ymlPaths := []string{ + d.dockerComposeFile, + filepath.Join(configDir, dockerCustomAgentDockerfile), } service := dockerComposeDeployedService{ ymlPaths: ymlPaths, project: "elastic-package-service", - sv: ServiceVariant{ + variant: ServiceVariant{ Name: dockerCustomAgentName, Env: env, }, @@ -149,10 +162,25 @@ func (d *CustomAgentDeployer) SetUp(inCtxt ServiceContext) (DeployedService, err return &service, nil } -func (d *CustomAgentDeployer) loadComposeDefinitions() ([]string, error) { +// installDockerfile creates the files needed to run the custom elastic agent and returns +// the directory with these files. +func (d *CustomAgentDeployer) installDockerfile() (string, error) { locationManager, err := locations.NewLocationManager() if err != nil { - return nil, errors.Wrap(err, "can't locate Docker Compose file for Custom Agent deployer") + return "", errors.Wrap(err, "failed to find the configuration directory") } - return []string{d.cfg, locationManager.DockerCustomAgentDeployerYml()}, nil + + customAgentDir := filepath.Join(locationManager.DeployerDir(), dockerCustomAgentDir) + err = os.MkdirAll(customAgentDir, 0755) + if err != nil { + return "", errors.Wrap(err, "failed to create directory for custom agent files") + } + + customAgentDockerfile := filepath.Join(customAgentDir, dockerCustomAgentDockerfile) + err = os.WriteFile(customAgentDockerfile, dockerCustomAgentDockerfileContent, 0644) + if err != nil { + return "", errors.Wrap(err, "failed to create docker compose file for custom agent") + } + + return customAgentDir, nil } diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 47279581a..329a18b4e 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -5,17 +5,38 @@ package servicedeployer import ( + _ "embed" + "fmt" "os" "path/filepath" + "strings" "github.com/pkg/errors" + "github.com/elastic/go-resource" + "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/logger" ) +const ( + terraformDeployerDir = "terraform" + terraformDeployerYml = "terraform-deployer.yml" + terraformDeployerDockerfile = "Dockerfile" + terraformDeployerRun = "run.sh" +) + +//go:embed _static/terraform_deployer.yml +var terraformDeployerYmlContent string + +//go:embed _static/terraform_deployer_run.sh +var terraformDeployerRunContent string + +//go:embed _static/Dockerfile.terraform_deployer +var terraformDeployerDockerfileContent string + // TerraformServiceDeployer is responsible for deploying infrastructure described with Terraform definitions. type TerraformServiceDeployer struct { definitionsDir string @@ -32,9 +53,16 @@ func NewTerraformServiceDeployer(definitionsDir string) (*TerraformServiceDeploy func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedService, error) { logger.Debug("setting up service using Terraform deployer") - ymlPaths, err := tsd.loadComposeDefinitions() + configDir, err := tsd.installDockerfile() if err != nil { - return nil, errors.Wrap(err, "can't load Docker Compose definitions") + return nil, errors.Wrap(err, "can't install Docker Compose definitions") + } + + ymlPaths := []string{filepath.Join(configDir, terraformDeployerYml)} + envYmlPath := filepath.Join(tsd.definitionsDir, envYmlFile) + _, err = os.Stat(envYmlPath) + if err == nil { + ymlPaths = append(ymlPaths, envYmlPath) } service := dockerComposeDeployedService{ @@ -54,8 +82,12 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic return nil, errors.Wrap(err, "removing service logs failed") } + tfEnvironment := tsd.buildTerraformExecutorEnvironment(inCtxt) + // Set custom aliases, which may be used in agent policies. - serviceComposeConfig, err := p.Config(compose.CommandOptions{}) + serviceComposeConfig, err := p.Config(compose.CommandOptions{ + Env: tfEnvironment, + }) if err != nil { return nil, errors.Wrap(err, "could not get Docker Compose configuration for service") } @@ -65,7 +97,6 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic } // Boot up service - tfEnvironment := tsd.buildTerraformExecutorEnvironment(inCtxt) opts := compose.CommandOptions{ Env: tfEnvironment, ExtraArgs: []string{"--build", "-d"}, @@ -89,25 +120,49 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic return &service, nil } -func (tsd TerraformServiceDeployer) loadComposeDefinitions() ([]string, error) { +func (tsd TerraformServiceDeployer) installDockerfile() (string, error) { locationManager, err := locations.NewLocationManager() if err != nil { - return nil, errors.Wrap(err, "can't locate Docker Compose file for Terraform deployer") + return "", errors.Wrap(err, "failed to find the configuration directory") } - envYmlPath := filepath.Join(tsd.definitionsDir, envYmlFile) - _, err = os.Stat(envYmlPath) - if errors.Is(err, os.ErrNotExist) { - return []string{ - locationManager.TerraformDeployerYml(), - }, nil + tfDir := filepath.Join(locationManager.DeployerDir(), terraformDeployerDir) + + resources := []resource.Resource{ + &resource.File{ + Path: terraformDeployerYml, + Content: resource.FileContentLiteral(terraformDeployerYmlContent), + CreateParent: true, + }, + &resource.File{ + Path: terraformDeployerRun, + Content: resource.FileContentLiteral(terraformDeployerRunContent), + CreateParent: true, + }, + &resource.File{ + Path: terraformDeployerDockerfile, + Content: resource.FileContentLiteral(terraformDeployerDockerfileContent), + CreateParent: true, + }, } + + resourceManager := resource.NewManager() + resourceManager.RegisterProvider("file", &resource.FileProvider{ + Prefix: tfDir, + }) + + results, err := resourceManager.Apply(resources) if err != nil { - return nil, errors.Wrapf(err, "stat failed (path: %s)", envYmlPath) + var errors []string + for _, result := range results { + if err := result.Err(); err != nil { + errors = append(errors, err.Error()) + } + } + return "", fmt.Errorf("%w: %s", err, strings.Join(errors, ", ")) } - return []string{ - locationManager.TerraformDeployerYml(), envYmlPath, - }, nil + + return tfDir, nil } var _ ServiceDeployer = new(TerraformServiceDeployer)