Skip to content

Commit

Permalink
systemtest: fix apm-server binary injection (elastic#5440)
Browse files Browse the repository at this point in the history 
Due to some changes in elastic-agent
(elastic/beats#24817), injection
of the apm-server binary became ineffective and we have
been running system tests with the published artifacts.

Artifacts (such as the apm-server) are now unpacked into
state/data/install/<artifact>. The state/data/install
directory is expected to be owned by the elastic-agent
user, so we can no longer bind mount the apm-server binary.
Instead, we now create a custom Docker image and copy in
the apm-server and apm-server.yml files.
  • Loading branch information
@axw @stuartnelson3
axw authored and stuartnelson3 committed Jun 14, 2021
1 parent 93f7b44 commit 6859ecb
Show file tree
Hide file tree
Showing 2 changed files with 111 additions and 67 deletions.
155 changes: 111 additions & 44 deletions systemtest/containers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
package systemtest

import (
"archive/tar"
"bytes"
"context"
"encoding/json"
"errors"
Expand All @@ -29,8 +31,8 @@ import (
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"strings"
"time"

Expand All @@ -42,6 +44,7 @@ import (
"github.com/testcontainers/testcontainers-go/wait"
"golang.org/x/sync/errgroup"

"github.com/elastic/apm-server/systemtest/apmservertest"
"github.com/elastic/apm-server/systemtest/estest"
"github.com/elastic/go-elasticsearch/v7"
)
Expand Down Expand Up @@ -281,6 +284,11 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) {
Scheme: "https",
Host: net.JoinHostPort(fleetServerIPAddress, fleetServerPort),
}
containerCACertPath := "/etc/pki/tls/certs/fleet-ca.pem"
hostCACertPath, err := filepath.Abs("../testing/docker/fleet-server/ca.pem")
if err != nil {
return nil, err
}

// Use the same stack version as used for fleet-server.
agentImageVersion := fleetServerContainer.Image[strings.LastIndex(fleetServerContainer.Image, ":")+1:]
Expand All @@ -292,43 +300,34 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) {
if err != nil {
return nil, err
}
agentVCSRef := agentImageDetails.Config.Labels["org.label-schema.vcs-ref"]
agentDataHashDir := path.Join("/usr/share/elastic-agent/data", "elastic-agent-"+agentVCSRef[:6])
agentInstallDir := path.Join(agentDataHashDir, "install")
stackVersion := agentImageDetails.Config.Labels["org.label-schema.version"]

// Build a custom elastic-agent image with a locally built apm-server binary injected.
agentImage, err = buildElasticAgentImage(context.Background(), docker, stackVersion, agentImageVersion)
if err != nil {
return nil, err
}

req := testcontainers.ContainerRequest{
Image: agentImage,
AutoRemove: true,
Networks: networks,
BindMounts: map[string]string{hostCACertPath: containerCACertPath},
Env: map[string]string{
// NOTE(axw) because we bind-mount the apm-server artifacts in, they end up owned by the
// current user rather than root. Disable Beats's strict permission checks to avoid resulting
// complaints, as they're irrelevant to these system tests.
"BEAT_STRICT_PERMS": "false",
"FLEET_URL": fleetServerURL.String(),
"FLEET_CA": containerCACertPath,
},
}
return &ElasticAgentContainer{
request: req,
installDir: agentInstallDir,
fleetServerURL: fleetServerURL.String(),
StackVersion: agentImageVersion,
BindMountInstall: make(map[string]string),
request: req,
StackVersion: agentImageVersion,
}, nil
}

// ElasticAgentContainer represents an ephemeral Elastic Agent container.
type ElasticAgentContainer struct {
container testcontainers.Container
request testcontainers.ContainerRequest
fleetServerURL string

// installDir holds the location of the "install" directory inside
// the Elastic Agent container.
//
// This will be set when the ElasticAgentContainer object is created,
// and can be used to anticipate the location into which artifacts
// can be bind-mounted.
installDir string
container testcontainers.Container
request testcontainers.ContainerRequest

// StackVersion holds the stack version of the container image,
// e.g. 8.0.0-SNAPSHOT.
Expand All @@ -344,11 +343,6 @@ type ElasticAgentContainer struct {
// by exposed port. This will be populated by Start.
Addrs map[string]string

// BindMountInstall holds a map of files to bind mount into the
// container, mapping from the host location to target paths relative
// to the install directory in the container.
BindMountInstall map[string]string

// FleetEnrollmentToken holds an optional Fleet enrollment token to
// use for enrolling the agent with Fleet. The agent will only enroll
// if this is specified.
Expand All @@ -366,27 +360,12 @@ func (c *ElasticAgentContainer) Start() error {
defer cancel()

// Update request from user-definable fields.
c.request.Env["FLEET_URL"] = c.fleetServerURL
if c.FleetEnrollmentToken != "" {
c.request.Env["FLEET_ENROLL"] = "1"
c.request.Env["FLEET_ENROLLMENT_TOKEN"] = c.FleetEnrollmentToken
}

c.request.ExposedPorts = c.ExposedPorts
c.request.WaitingFor = c.WaitingFor
c.request.BindMounts = map[string]string{}
for source, target := range c.BindMountInstall {
c.request.BindMounts[source] = path.Join(c.installDir, target)
}

// Inject CA certificate for verifying fleet-server.
containerCACertPath := "/etc/pki/tls/certs/fleet-ca.pem"
hostCACertPath, err := filepath.Abs("../testing/docker/fleet-server/ca.pem")
if err != nil {
return err
}
c.request.BindMounts[hostCACertPath] = containerCACertPath
c.request.Env["FLEET_CA"] = containerCACertPath

container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: c.request,
Expand Down Expand Up @@ -457,3 +436,91 @@ func matchFleetServerAPIStatusHealthy(r io.Reader) bool {
}
return status.Status == "HEALTHY"
}

// buildElasticAgentImage builds a Docker image from the published image with a locally built apm-server injected.
func buildElasticAgentImage(ctx context.Context, docker *client.Client, stackVersion, imageVersion string) (string, error) {
imageName := fmt.Sprintf("elastic-agent-systemtest:%s", imageVersion)
log.Printf("Building image %s...", imageName)

// Build apm-server, and copy it into the elastic-agent container's "install" directory.
// This bypasses downloading the artifact.
arch := runtime.GOARCH
if arch == "amd64" {
arch = "x86_64"
}
apmServerInstallDir := fmt.Sprintf("./state/data/install/apm-server-%s-linux-%s", stackVersion, arch)
apmServerBinary, err := apmservertest.BuildServerBinary("linux")
if err != nil {
return "", err
}

// Binaries to copy from disk into the build context.
binaries := map[string]string{
"apm-server": apmServerBinary,
}

// Generate Dockerfile contents.
var dockerfile bytes.Buffer
fmt.Fprintf(&dockerfile, "FROM docker.elastic.co/beats/elastic-agent:%s\n", imageVersion)
fmt.Fprintf(&dockerfile, "COPY --chown=elastic-agent:elastic-agent apm-server apm-server.yml %s/\n", apmServerInstallDir)

// Files to generate in the build context.
generatedFiles := map[string][]byte{
"Dockerfile": dockerfile.Bytes(),
"apm-server.yml": []byte(""),
}

var buildContext bytes.Buffer
tarw := tar.NewWriter(&buildContext)
for name, path := range binaries {
f, err := os.Open(path)
if err != nil {
return "", err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return "", err
}
if err := tarw.WriteHeader(&tar.Header{
Name: name,
Size: info.Size(),
Mode: 0755,
Uname: "elastic-agent",
Gname: "elastic-agent",
}); err != nil {
return "", err
}
if _, err := io.Copy(tarw, f); err != nil {
return "", err
}
}
for name, content := range generatedFiles {
if err := tarw.WriteHeader(&tar.Header{
Name: name,
Size: int64(len(content)),
Mode: 0644,
Uname: "elastic-agent",
Gname: "elastic-agent",
}); err != nil {
return "", err
}
if _, err := tarw.Write(content); err != nil {
return "", err
}
}
if err := tarw.Close(); err != nil {
return "", err
}

resp, err := docker.ImageBuild(ctx, &buildContext, types.ImageBuildOptions{Tags: []string{imageName}})
if err != nil {
return "", err
}
defer resp.Body.Close()
if _, err := io.Copy(ioutil.Discard, resp.Body); err != nil {
return "", err
}
log.Printf("Built image %s", imageName)
return imageName, nil
}
23 changes: 0 additions & 23 deletions systemtest/fleet_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,8 @@ package systemtest_test

import (
"context"
"fmt"
"io/ioutil"
"net/url"
"path"
"path/filepath"
"runtime"
"testing"
"time"

Expand All @@ -35,7 +31,6 @@ import (
"go.elastic.co/apm/transport"

"github.com/elastic/apm-server/systemtest"
"github.com/elastic/apm-server/systemtest/apmservertest"
"github.com/elastic/apm-server/systemtest/fleettest"
)

Expand Down Expand Up @@ -80,24 +75,6 @@ func TestFleetIntegration(t *testing.T) {
}
}()

// Build apm-server, and bind-mount it into the elastic-agent container's "install"
// directory. This bypasses downloading the artifact.
arch := runtime.GOARCH
if arch == "amd64" {
arch = "x86_64"
}
apmServerArtifactName := fmt.Sprintf("apm-server-%s-linux-%s", agent.StackVersion, arch)

// Bind-mount the apm-server binary and apm-server.yml into the container's
// "install" directory. This causes elastic-agent to skip installing the
// artifact.
apmServerBinary, err := apmservertest.BuildServerBinary("linux")
require.NoError(t, err)
agent.BindMountInstall[apmServerBinary] = path.Join(apmServerArtifactName, "apm-server")
apmServerConfigFile, err := filepath.Abs("../apm-server.yml")
require.NoError(t, err)
agent.BindMountInstall[apmServerConfigFile] = path.Join(apmServerArtifactName, "apm-server.yml")

// Start elastic-agent with port 8200 exposed, and wait for the server to service
// healthcheck requests to port 8200.
agent.ExposedPorts = []string{"8200"}
Expand Down

0 comments on commit 6859ecb

Please sign in to comment.