apmpackage: add more elasticsearch privileges (#6139)

* tools: upgrade elastic-package

* apmpackage: add privileges for sampled_traces

* apmpackage: add config vars for tail sampling

* apmpackage: update changelog

* systemtest: test tail-sampling with Fleet

* systemtest: don't delete integration templates

We now only install the integration once before running
all the tests. We uninstall any existing apm package,
which will take care of removing outdated integration
index templates.

* Update systemtest/fleet_test.go

Co-authored-by: Silvia Mitter <[email protected]>

Co-authored-by: Silvia Mitter <[email protected]>
(cherry picked from commit fa65ccc)

apmpackage/apm/agent/input/template.yml.hbs

@@ -63,3 +63,12 @@ apm-server:
         {{/each}}
     {{/if}}
     write_timeout: {{write_timeout}}
+    {{#if tail_sampling_policies}}
+    sampling:
+      tail:
+        enabled: true
+        {{#if tail_sampling_interval}}
+        interval: {{tail_sampling_interval}}
+        {{/if}}
+        policies: {{tail_sampling_policies}}
+    {{/if}}

apmpackage/apm/changelog.yml

@@ -17,6 +17,12 @@
     - description: remove warm phase from ILM policies
       type: enhancement
       link: https://github.com/elastic/apm-server/pull/6229
+    - description: added privileges to tail-sampled traces data stream
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/6139
+    - description: added tail-sampling config vars
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/6139
 - version: "0.4.0"
   changes:
     - description: add anonymous auth config, replace some RUM config

apmpackage/apm/data_stream/sampled_traces/manifest.yml

@@ -3,6 +3,11 @@ type: traces
 dataset: apm.sampled
 ilm_policy: traces-apm.sampled-default_policy
 elasticsearch:
+  privileges:
+    # We need additional privileges for the sampled traces data stream,
+    # for refreshing indices, querying index stats, and reading documents.
+    # This data stream holds only non-sensitive information.
+    indices: [auto_configure, create_doc, maintenance, monitor, read]
   index_template:
     settings:
       # Create a single shard per index, so we can use

apmpackage/apm/manifest.yml

@@ -137,6 +137,10 @@ policy_templates:
           - name: tls_curve_types
             type: text
             multi: true
+          - name: tail_sampling_policies
+            type: yaml
+          - name: tail_sampling_interval
+            type: text
         template_path: template.yml.hbs
 owner:
   github: elastic/apm-server

systemtest/agentconfig_test.go

@@ -32,6 +32,8 @@ import (
 )
 
 func TestAgentConfig(t *testing.T) {
+	systemtest.CleanupElasticsearch(t)
+
 	serviceName := "systemtest_service"
 	serviceEnvironment := "testing"
 	systemtest.DeleteAgentConfig(t, serviceName, "")
@@ -44,7 +46,7 @@ func TestAgentConfig(t *testing.T) {
 	require.NoError(t, err)
 
 	// Run apm-server under Fleet, exercising the Fleet agent config implementation.
-	apmIntegration := initAPMIntegration(t, map[string]interface{}{})
+	apmIntegration := newAPMIntegration(t, map[string]interface{}{})
 	serverURLs := []string{srv.URL, apmIntegration.URL}
 
 	expectChange := func(serverURL string, etag string) (map[string]string, *http.Response) {

systemtest/containers.go

@@ -39,6 +39,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/go-connections/nat"
 	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/wait"
@@ -301,9 +302,10 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) {
 		return nil, err
 	}
 	stackVersion := agentImageDetails.Config.Labels["org.label-schema.version"]
+	vcsRef := agentImageDetails.Config.Labels["org.label-schema.vcs-ref"]
 
 	// Build a custom elastic-agent image with a locally built apm-server binary injected.
-	agentImage, err = buildElasticAgentImage(context.Background(), docker, stackVersion, agentImageVersion)
+	agentImage, err = buildElasticAgentImage(context.Background(), docker, stackVersion, agentImageVersion, vcsRef)
 	if err != nil {
 		return nil, err
 	}
@@ -415,6 +417,45 @@ func (c *ElasticAgentContainer) Logs(ctx context.Context) (io.ReadCloser, error)
 	return c.container.Logs(ctx)
 }
 
+// Exec executes a command in the container, and returns its stdout and stderr.
+func (c *ElasticAgentContainer) Exec(ctx context.Context, cmd ...string) (stdout, stderr []byte, _ error) {
+	docker, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer docker.Close()
+
+	response, err := docker.ContainerExecCreate(ctx, c.container.GetContainerID(), types.ExecConfig{
+		AttachStderr: true,
+		AttachStdout: true,
+		Cmd:          cmd,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Consume all the exec output.
+	resp, err := docker.ContainerExecAttach(ctx, response.ID, types.ExecStartCheck{})
+	if err != nil {
+		return nil, nil, err
+	}
+	defer resp.Close()
+	var stdoutBuf, stderrBuf bytes.Buffer
+	if _, err := stdcopy.StdCopy(&stdoutBuf, &stderrBuf, resp.Reader); err != nil {
+		return nil, nil, err
+	}
+
+	// Return an error if the command exited non-zero.
+	execResp, err := docker.ContainerExecInspect(ctx, response.ID)
+	if err != nil {
+		return nil, nil, err
+	}
+	if execResp.ExitCode != 0 {
+		return nil, nil, fmt.Errorf("process exited with code %d", execResp.ExitCode)
+	}
+	return stdoutBuf.Bytes(), stderrBuf.Bytes(), nil
+}
+
 func pullDockerImage(ctx context.Context, docker *client.Client, imageRef string) error {
 	rc, err := docker.ImagePull(context.Background(), imageRef, types.ImagePullOptions{})
 	if err != nil {
@@ -438,7 +479,7 @@ func matchFleetServerAPIStatusHealthy(r io.Reader) bool {
 }
 
 // buildElasticAgentImage builds a Docker image from the published image with a locally built apm-server injected.
-func buildElasticAgentImage(ctx context.Context, docker *client.Client, stackVersion, imageVersion string) (string, error) {
+func buildElasticAgentImage(ctx context.Context, docker *client.Client, stackVersion, imageVersion, vcsRef string) (string, error) {
 	imageName := fmt.Sprintf("elastic-agent-systemtest:%s", imageVersion)
 	log.Printf("Building image %s...", imageName)
 
@@ -448,7 +489,8 @@ func buildElasticAgentImage(ctx context.Context, docker *client.Client, stackVer
 	if arch == "amd64" {
 		arch = "x86_64"
 	}
-	apmServerInstallDir := fmt.Sprintf("./state/data/install/apm-server-%s-linux-%s", stackVersion, arch)
+	vcsRefShort := vcsRef[:6]
+	apmServerInstallDir := fmt.Sprintf("./data/elastic-agent-%s/install/apm-server-%s-linux-%s", vcsRefShort, stackVersion, arch)
 	apmServerBinary, err := apmservertest.BuildServerBinary("linux")
 	if err != nil {
 		return "", err

systemtest/elasticsearch.go

@@ -139,15 +139,7 @@ func CleanupElasticsearch(t testing.TB) {
 	)
 
 	// Delete index templates after deleting data streams.
-	doParallel(
-		esapi.IndicesDeleteTemplateRequest{Name: legacyPrefix},
-		esapi.IndicesDeleteIndexTemplateRequest{Name: apmTracesPrefix},
-		esapi.IndicesDeleteIndexTemplateRequest{Name: apmMetricsPrefix},
-		esapi.IndicesDeleteIndexTemplateRequest{Name: apmLogsPrefix},
-	)
-
-	// Delete index templates after deleting data streams.
-	if err := doReq(esapi.IndicesDeleteIndexTemplateRequest{Name: legacyPrefix}); err != nil {
+	if err := doReq(esapi.IndicesDeleteTemplateRequest{Name: legacyPrefix}); err != nil {
 		t.Fatal(err)
 	}