diff --git a/README.md b/README.md index b4d5eabf1..04be56b93 100644 --- a/README.md +++ b/README.md @@ -694,7 +694,11 @@ There are available some environment variables that could be used to change some - `ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS`: If set to `true`, the results from pipeline tests are not compared to avoid errors from GeoIP. - `ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI`: If set to `true`, the Elastic Agent image used for running agents will be using the Ubuntu docker images (e.g. `docker.elastic.co/elastic-agent/elastic-agent-complete`). If set to `false`, the Elastic Agent image used for the running agents will be based on the wolfi +<<<<<<< HEAD + images (e.g. `docker.elastic.co/elastic-agent/elastic-agent-wolfi`). Default: `true`. +======= images (e.g. `docker.elastic.co/elastic-agent/elastic-agent-wolfi`). Default: `false`. +>>>>>>> upstream/main - To configure the Elastic stack to be used by `elastic-package`: - `ELASTIC_PACKAGE_ELASTICSEARCH_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index fdb6aeb05..a523ec152 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -101,7 +101,7 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI logger.Debug("setting up agent using Docker Compose agent deployer") d.agentRunID = agentInfo.Test.RunID - appConfig, err := install.Configuration() + appConfig, err := install.Configuration(install.OptionWithStackVersion(d.stackVersion)) if err != nil { return nil, fmt.Errorf("can't read application configuration: %w", err) } @@ -112,7 +112,7 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI } env := append( - appConfig.StackImageRefs(d.stackVersion).AsEnv(), + appConfig.StackImageRefs().AsEnv(), fmt.Sprintf("%s=%s", serviceLogsDirEnv, agentInfo.Logs.Folder.Local), fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), fmt.Sprintf("%s=%s", fleetPolicyEnv, d.policyName), @@ -264,14 +264,14 @@ func (d *DockerComposeAgentDeployer) installDockerCompose(agentInfo AgentInfo) ( stackVersion = config.Parameters[stack.ParamServerlessLocalStackVersion] } - appConfig, err := install.Configuration() + agentImage, err := selectElasticAgentImage(stackVersion, agentInfo.Agent.BaseImage) if err != nil { - return "", fmt.Errorf("can't read application configuration: %w", err) + return "", nil } resourceManager := resource.NewManager() resourceManager.AddFacter(resource.StaticFacter{ - "agent_image": appConfig.StackImageRefs(stackVersion).ElasticAgent, + "agent_image": agentImage, "user": agentInfo.Agent.User, "capabilities": strings.Join(agentInfo.Agent.LinuxCapabilities, ","), "runtime": agentInfo.Agent.Runtime, @@ -303,6 +303,16 @@ func (d *DockerComposeAgentDeployer) installDockerCompose(agentInfo AgentInfo) ( return customAgentDir, nil } +func selectElasticAgentImage(stackVersion, agentBaseImage string) (string, error) { + appConfig, err := install.Configuration(install.OptionWithAgentBaseImage(agentBaseImage), install.OptionWithStackVersion(stackVersion)) + if err != nil { + return "", fmt.Errorf("can't read application configuration: %w", err) + } + + agentImage := appConfig.StackImageRefs().ElasticAgent + return agentImage, nil +} + func (d *DockerComposeAgentDeployer) installDockerfileResources(agentSettings AgentSettings, folder string) error { agentResources := []resource.Resource{ &resource.File{ diff --git a/internal/agentdeployer/info.go b/internal/agentdeployer/info.go index 15152c34f..919c2ef98 100644 --- a/internal/agentdeployer/info.go +++ b/internal/agentdeployer/info.go @@ -26,6 +26,8 @@ type AgentScript struct { type AgentSettings struct { // User user to run Elastic Agent process User string `config:"user"` + // BaseImage elastic-agent base image to be used for testing + BaseImage string `config:"base_image"` // PidMode selects the host PID mode // (From docker-compose docs) Turns on sharing between container and the host // operating system the PID address space diff --git a/internal/agentdeployer/kubernetes.go b/internal/agentdeployer/kubernetes.go index c6aeb07c8..f29f816f2 100644 --- a/internal/agentdeployer/kubernetes.go +++ b/internal/agentdeployer/kubernetes.go @@ -179,7 +179,7 @@ var elasticAgentManagedYamlTmpl string func getElasticAgentYAML(profile *profile.Profile, stackVersion, policyName, agentName string) ([]byte, error) { logger.Debugf("Prepare YAML definition for Elastic Agent running in stack v%s", stackVersion) - appConfig, err := install.Configuration() + appConfig, err := install.Configuration(install.OptionWithStackVersion(stackVersion)) if err != nil { return nil, fmt.Errorf("can't read application configuration: %w", err) } @@ -196,7 +196,7 @@ func getElasticAgentYAML(profile *profile.Profile, stackVersion, policyName, age "fleetURL": "https://fleet-server:8220", "kibanaURL": "https://kibana:5601", "caCertPem": caCert, - "elasticAgentImage": appConfig.StackImageRefs(stackVersion).ElasticAgent, + "elasticAgentImage": appConfig.StackImageRefs().ElasticAgent, "elasticAgentTokenPolicyName": getTokenPolicyName(stackVersion, policyName), "agentName": agentName, }) diff --git a/internal/install/application_configuration.go b/internal/install/application_configuration.go index ce2f90ffa..8e393cd27 100644 --- a/internal/install/application_configuration.go +++ b/internal/install/application_configuration.go @@ -71,7 +71,9 @@ func DefaultConfiguration() *ApplicationConfiguration { // ApplicationConfiguration represents the configuration of the elastic-package. type ApplicationConfiguration struct { - c configFile + c configFile + agentBaseImage string + stackVersion string } type configFile struct { @@ -119,12 +121,12 @@ func (ir ImageRefs) AsEnv() []string { } // StackImageRefs function selects the appropriate set of Docker image references for the given stack version. -func (ac *ApplicationConfiguration) StackImageRefs(version string) ImageRefs { - refs := ac.c.Stack.ImageRefOverridesForVersion(version) - refs.ElasticAgent = stringOrDefault(refs.ElasticAgent, fmt.Sprintf("%s:%s", selectElasticAgentImageName(version), version)) - refs.Elasticsearch = stringOrDefault(refs.Elasticsearch, fmt.Sprintf("%s:%s", elasticsearchImageName, version)) - refs.Kibana = stringOrDefault(refs.Kibana, fmt.Sprintf("%s:%s", kibanaImageName, version)) - refs.Logstash = stringOrDefault(refs.Logstash, fmt.Sprintf("%s:%s", logstashImageName, version)) +func (ac *ApplicationConfiguration) StackImageRefs() ImageRefs { + refs := ac.c.Stack.ImageRefOverridesForVersion(ac.stackVersion) + refs.ElasticAgent = stringOrDefault(refs.ElasticAgent, fmt.Sprintf("%s:%s", selectElasticAgentImageName(ac.stackVersion, ac.agentBaseImage), ac.stackVersion)) + refs.Elasticsearch = stringOrDefault(refs.Elasticsearch, fmt.Sprintf("%s:%s", elasticsearchImageName, ac.stackVersion)) + refs.Kibana = stringOrDefault(refs.Kibana, fmt.Sprintf("%s:%s", kibanaImageName, ac.stackVersion)) + refs.Logstash = stringOrDefault(refs.Logstash, fmt.Sprintf("%s:%s", logstashImageName, ac.stackVersion)) return refs } @@ -148,7 +150,7 @@ func (ac *ApplicationConfiguration) SetCurrentProfile(name string) { // selectElasticAgentImageName function returns the appropriate image name for Elastic-Agent depending on the stack version. // This is mandatory as "elastic-agent-complete" is available since 7.15.0-SNAPSHOT. -func selectElasticAgentImageName(version string) string { +func selectElasticAgentImageName(version, agentBaseImage string) string { if version == "" { // as version is optional and can be empty return elasticAgentImageName } @@ -164,20 +166,41 @@ func selectElasticAgentImageName(version string) string { if ok && strings.ToLower(valueEnv) != "false" { disableWolfiImages = true } - if !disableWolfiImages && !v.LessThan(elasticAgentWolfiVersion) { + switch { + case !disableWolfiImages && !v.LessThan(elasticAgentWolfiVersion) && agentBaseImage != "complete": return elasticAgentWolfiImageName - } - if !v.LessThan(elasticAgentCompleteOwnNamespaceVersion) { + case !v.LessThan(elasticAgentCompleteOwnNamespaceVersion): return elasticAgentCompleteImageName - } - if !v.LessThan(elasticAgentCompleteFirstSupportedVersion) { + case !v.LessThan(elasticAgentCompleteFirstSupportedVersion): return elasticAgentCompleteLegacyImageName + default: + return elasticAgentImageName + } +} + +type configurationOptions struct { + agentBaseImage string + stackVersion string +} + +type ConfigurationOption func(*configurationOptions) + +// OptionWithAgentBaseImage sets the agent image type to be used. +func OptionWithAgentBaseImage(agentBaseImage string) ConfigurationOption { + return func(opts *configurationOptions) { + opts.agentBaseImage = agentBaseImage + } +} + +// OptionWithStackVersion sets the Elastic Stack version to be used. +func OptionWithStackVersion(stackVersion string) ConfigurationOption { + return func(opts *configurationOptions) { + opts.stackVersion = stackVersion } - return elasticAgentImageName } // Configuration function returns the elastic-package configuration. -func Configuration() (*ApplicationConfiguration, error) { +func Configuration(options ...ConfigurationOption) (*ApplicationConfiguration, error) { configPath, err := locations.NewLocationManager() if err != nil { return nil, fmt.Errorf("can't read configuration directory: %w", err) @@ -197,9 +220,18 @@ func Configuration() (*ApplicationConfiguration, error) { return nil, fmt.Errorf("can't unmarshal configuration file: %w", err) } - return &ApplicationConfiguration{ - c: c, - }, nil + configOptions := configurationOptions{} + for _, option := range options { + option(&configOptions) + } + + configuration := ApplicationConfiguration{ + c: c, + agentBaseImage: configOptions.agentBaseImage, + stackVersion: configOptions.stackVersion, + } + + return &configuration, nil } func stringOrDefault(value string, defaultValue string) string { diff --git a/internal/install/application_configuration_test.go b/internal/install/application_configuration_test.go index 5f20f3419..f41302ae2 100644 --- a/internal/install/application_configuration_test.go +++ b/internal/install/application_configuration_test.go @@ -12,61 +12,72 @@ import ( func TestSelectElasticAgentImageName_NoVersion(t *testing.T) { var version string - selected := selectElasticAgentImageName(version) + selected := selectElasticAgentImageName(version, "") assert.Equal(t, selected, elasticAgentImageName) } func TestSelectElasticAgentImageName_OlderStack(t *testing.T) { version := "7.14.99-SNAPSHOT" - selected := selectElasticAgentImageName(version) + selected := selectElasticAgentImageName(version, "") assert.Equal(t, selected, elasticAgentImageName) } func TestSelectElasticAgentImageName_FirstStackWithCompleteAgent(t *testing.T) { version := stackVersion715 - selected := selectElasticAgentImageName(version) + selected := selectElasticAgentImageName(version, "") assert.Equal(t, selected, elasticAgentCompleteLegacyImageName) } func TestSelectElasticAgentImageName_NextStackWithAgentComplete(t *testing.T) { version := "7.16.0-SNAPSHOT" - selected := selectElasticAgentImageName(version) + selected := selectElasticAgentImageName(version, "") assert.Equal(t, selected, elasticAgentCompleteLegacyImageName) } func TestSelectElasticAgentImageName_OwnNamespace(t *testing.T) { version := "8.2.0-SNAPSHOT" - selected := selectElasticAgentImageName(version) + selected := selectElasticAgentImageName(version, "") assert.Equal(t, selected, elasticAgentCompleteImageName) } func TestSelectElasticAgentImageName_OwnNamespace_Release(t *testing.T) { version := "8.2.0" - selected := selectElasticAgentImageName(version) + selected := selectElasticAgentImageName(version, "") assert.Equal(t, selected, elasticAgentCompleteImageName) } func TestSelectElasticAgentImageName_NextStackInOwnNamespace(t *testing.T) { version := "8.4.0-SNAPSHOT" - selected := selectElasticAgentImageName(version) + selected := selectElasticAgentImageName(version, "") assert.Equal(t, selected, elasticAgentCompleteImageName) } func TestSelectElasticAgentImageName_WolfiImage(t *testing.T) { version := "8.16.0-SNAPSHOT" - selected := selectElasticAgentImageName(version) + selected := selectElasticAgentImageName(version, "") assert.Equal(t, selected, elasticAgentWolfiImageName) } func TestSelectElasticAgentImageName_DisableWolfiImageEnvVar(t *testing.T) { version := "8.16.0-SNAPSHOT" t.Setenv(disableElasticAgentWolfiEnvVar, "true") - selected := selectElasticAgentImageName(version) + selected := selectElasticAgentImageName(version, "") assert.Equal(t, selected, elasticAgentCompleteImageName) } func TestSelectElasticAgentImageName_EnableWolfiImageEnvVar(t *testing.T) { version := "8.16.0-SNAPSHOT" t.Setenv(disableElasticAgentWolfiEnvVar, "false") - selected := selectElasticAgentImageName(version) + selected := selectElasticAgentImageName(version, "") + assert.Equal(t, selected, elasticAgentWolfiImageName) +} + +func TestSelectCompleteElasticAgentImageName_ForceCompleteImage(t *testing.T) { + version := "8.16.0-SNAPSHOT" + selected := selectElasticAgentImageName(version, "complete") + assert.Equal(t, selected, elasticAgentCompleteImageName) +} +func TestSelectCompleteElasticAgentImageName_ForceDefaultImage(t *testing.T) { + version := "8.16.0-SNAPSHOT" + selected := selectElasticAgentImageName(version, "default") assert.Equal(t, selected, elasticAgentWolfiImageName) } diff --git a/internal/servicedeployer/custom_agent.go b/internal/servicedeployer/custom_agent.go index 1931b1bd6..6f9ee5253 100644 --- a/internal/servicedeployer/custom_agent.go +++ b/internal/servicedeployer/custom_agent.go @@ -72,7 +72,7 @@ func NewCustomAgentDeployer(options CustomAgentDeployerOptions) (*CustomAgentDep func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (DeployedService, error) { logger.Warn("DEPRECATED - setting up service using Docker Compose service deployer") - appConfig, err := install.Configuration() + appConfig, err := install.Configuration(install.OptionWithStackVersion(d.stackVersion)) if err != nil { return nil, fmt.Errorf("can't read application configuration: %w", err) } @@ -90,7 +90,7 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D svcInfo.Hostname = dockerCustomAgentName env := append( - appConfig.StackImageRefs(d.stackVersion).AsEnv(), + appConfig.StackImageRefs().AsEnv(), fmt.Sprintf("%s=%s", serviceLogsDirEnv, svcInfo.Logs.Folder.Local), fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), fmt.Sprintf("%s=%s", fleetPolicyEnv, d.policyName), diff --git a/internal/servicedeployer/kubernetes.go b/internal/servicedeployer/kubernetes.go index 3b967285b..2d982d416 100644 --- a/internal/servicedeployer/kubernetes.go +++ b/internal/servicedeployer/kubernetes.go @@ -229,7 +229,7 @@ var elasticAgentManagedYamlTmpl string func getElasticAgentYAML(profile *profile.Profile, stackVersion, policyName string) ([]byte, error) { logger.Debugf("Prepare YAML definition for Elastic Agent running in stack v%s", stackVersion) - appConfig, err := install.Configuration() + appConfig, err := install.Configuration(install.OptionWithStackVersion(stackVersion)) if err != nil { return nil, fmt.Errorf("can't read application configuration: %w", err) } @@ -246,7 +246,7 @@ func getElasticAgentYAML(profile *profile.Profile, stackVersion, policyName stri "fleetURL": "https://fleet-server:8220", "kibanaURL": "https://kibana:5601", "caCertPem": caCert, - "elasticAgentImage": appConfig.StackImageRefs(stackVersion).ElasticAgent, + "elasticAgentImage": appConfig.StackImageRefs().ElasticAgent, "elasticAgentTokenPolicyName": getTokenPolicyName(stackVersion, policyName), }) if err != nil { diff --git a/internal/stack/compose.go b/internal/stack/compose.go index a0afa8b85..483e638e8 100644 --- a/internal/stack/compose.go +++ b/internal/stack/compose.go @@ -56,14 +56,14 @@ func dockerComposeBuild(ctx context.Context, options Options) error { return fmt.Errorf("could not create docker compose project: %w", err) } - appConfig, err := install.Configuration() + appConfig, err := install.Configuration(install.OptionWithStackVersion(options.StackVersion)) if err != nil { return fmt.Errorf("can't read application configuration: %w", err) } opts := compose.CommandOptions{ Env: newEnvBuilder(). - withEnvs(appConfig.StackImageRefs(options.StackVersion).AsEnv()). + withEnvs(appConfig.StackImageRefs().AsEnv()). withEnv(stackVariantAsEnv(options.StackVersion)). withEnvs(options.Profile.ComposeEnvVars()). build(), @@ -82,14 +82,14 @@ func dockerComposePull(ctx context.Context, options Options) error { return fmt.Errorf("could not create docker compose project: %w", err) } - appConfig, err := install.Configuration() + appConfig, err := install.Configuration(install.OptionWithStackVersion(options.StackVersion)) if err != nil { return fmt.Errorf("can't read application configuration: %w", err) } opts := compose.CommandOptions{ Env: newEnvBuilder(). - withEnvs(appConfig.StackImageRefs(options.StackVersion).AsEnv()). + withEnvs(appConfig.StackImageRefs().AsEnv()). withEnv(stackVariantAsEnv(options.StackVersion)). withEnvs(options.Profile.ComposeEnvVars()). build(), @@ -113,14 +113,14 @@ func dockerComposeUp(ctx context.Context, options Options) error { args = append(args, "-d") } - appConfig, err := install.Configuration() + appConfig, err := install.Configuration(install.OptionWithStackVersion(options.StackVersion)) if err != nil { return fmt.Errorf("can't read application configuration: %w", err) } opts := compose.CommandOptions{ Env: newEnvBuilder(). - withEnvs(appConfig.StackImageRefs(options.StackVersion).AsEnv()). + withEnvs(appConfig.StackImageRefs().AsEnv()). withEnv(stackVariantAsEnv(options.StackVersion)). withEnvs(options.Profile.ComposeEnvVars()). build(), @@ -140,14 +140,14 @@ func dockerComposeDown(ctx context.Context, options Options) error { return fmt.Errorf("could not create docker compose project: %w", err) } - appConfig, err := install.Configuration() + appConfig, err := install.Configuration(install.OptionWithStackVersion(options.StackVersion)) if err != nil { return fmt.Errorf("can't read application configuration: %w", err) } downOptions := compose.CommandOptions{ Env: newEnvBuilder(). - withEnvs(appConfig.StackImageRefs(options.StackVersion).AsEnv()). + withEnvs(appConfig.StackImageRefs().AsEnv()). withEnv(stackVariantAsEnv(options.StackVersion)). withEnvs(options.Profile.ComposeEnvVars()). build(), diff --git a/internal/stack/logs.go b/internal/stack/logs.go index 0e821e0e7..7e6356de3 100644 --- a/internal/stack/logs.go +++ b/internal/stack/logs.go @@ -17,7 +17,7 @@ import ( ) func dockerComposeLogsSince(ctx context.Context, serviceName string, profile *profile.Profile, since time.Time) ([]byte, error) { - appConfig, err := install.Configuration() + appConfig, err := install.Configuration(install.OptionWithStackVersion(install.DefaultStackVersion)) if err != nil { return nil, fmt.Errorf("can't read application configuration: %w", err) } @@ -31,7 +31,7 @@ func dockerComposeLogsSince(ctx context.Context, serviceName string, profile *pr opts := compose.CommandOptions{ Env: newEnvBuilder(). - withEnvs(appConfig.StackImageRefs(install.DefaultStackVersion).AsEnv()). + withEnvs(appConfig.StackImageRefs().AsEnv()). withEnv(stackVariantAsEnv(install.DefaultStackVersion)). withEnvs(profile.ComposeEnvVars()). build(), diff --git a/internal/stack/serverlessresources.go b/internal/stack/serverlessresources.go index 3c94d227d..058290665 100644 --- a/internal/stack/serverlessresources.go +++ b/internal/stack/serverlessresources.go @@ -32,18 +32,19 @@ var ( ) func applyServerlessResources(profile *profile.Profile, stackVersion string, config Config) error { - appConfig, err := install.Configuration() + appConfig, err := install.Configuration(install.OptionWithStackVersion(stackVersion)) if err != nil { return fmt.Errorf("can't read application configuration: %w", err) } + imageRefs := appConfig.StackImageRefs() stackDir := filepath.Join(profile.ProfilePath, ProfileStackPath) resourceManager := resource.NewManager() resourceManager.AddFacter(resource.StaticFacter{ "agent_version": stackVersion, - "agent_image": appConfig.StackImageRefs(stackVersion).ElasticAgent, - "logstash_image": appConfig.StackImageRefs(stackVersion).Logstash, + "agent_image": imageRefs.ElasticAgent, + "logstash_image": imageRefs.Logstash, "elasticsearch_host": esHostWithPort(config.ElasticsearchHost), "username": config.ElasticsearchUsername, "password": config.ElasticsearchPassword, diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index 61f2a00ef..db640f693 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -422,6 +422,11 @@ func (r *tester) createAgentInfo(policy *kibana.Policy, config *testConfig, runI info.Agent.User = "root" } + // This could be removed once package-spec adds this new field + if !slices.Contains([]string{"", "default", "complete"}, info.Agent.BaseImage) { + return agentdeployer.AgentInfo{}, fmt.Errorf("invalid value for agent.base_image: %q", info.Agent.BaseImage) + } + return info, nil } @@ -648,7 +653,7 @@ func isSyntheticSourceModeEnabled(ctx context.Context, api *elasticsearch.API, d // It seems that some index modes enable synthetic source mode even when it is not explicitly mentioned // in the mappings. So assume that when these index modes are used, the synthetic mode is also used. - var syntheticsIndexModes = []string{ + syntheticsIndexModes := []string{ "logs", // Replaced in 8.15.0 with "logsdb", see https://github.com/elastic/elasticsearch/pull/111054 "logsdb", "time_series", diff --git a/tools/readme/readme.md.tmpl b/tools/readme/readme.md.tmpl index d916b2ef1..0191f2f65 100644 --- a/tools/readme/readme.md.tmpl +++ b/tools/readme/readme.md.tmpl @@ -236,7 +236,11 @@ There are available some environment variables that could be used to change some - `ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS`: If set to `true`, the results from pipeline tests are not compared to avoid errors from GeoIP. - `ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI`: If set to `true`, the Elastic Agent image used for running agents will be using the Ubuntu docker images (e.g. `docker.elastic.co/elastic-agent/elastic-agent-complete`). If set to `false`, the Elastic Agent image used for the running agents will be based on the wolfi +<<<<<<< HEAD + images (e.g. `docker.elastic.co/elastic-agent/elastic-agent-wolfi`). Default: `true`. +======= images (e.g. `docker.elastic.co/elastic-agent/elastic-agent-wolfi`). Default: `false`. +>>>>>>> upstream/main - To configure the Elastic stack to be used by `elastic-package`: - `ELASTIC_PACKAGE_ELASTICSEARCH_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200)