From 03ed6be94f0249a2ed758a3c4e1551e4cfb92d75 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Mon, 14 Dec 2020 09:18:15 +0800 Subject: [PATCH 01/11] systemtest: test apm-server in Fleet mode We run a custom package registry into which the "apm" integration package is bind-mounted from the tree, enabling us to test with unreleased package changes. Add a system test that runs Elastic Agent in Fleet mode, with a locally built APM Server binary injected into the container. The test adds the "apm" integration package to a new agent policy, waits for elastic-agent to start apm-server, and verifies it is functional. For the time being we must also inject a custom built elastic-agent binary to disable PGP verification of the apm-server artifact. Once we bundle the apm-server artifact with Elastic Agent we can stop doing this. --- docker-compose.yml | 17 ++ go.sum | 9 + systemtest/apmservertest/command.go | 35 ++- systemtest/apmservertest/config.go | 8 +- systemtest/containers.go | 266 +++++++++++++++++++++ systemtest/elasticsearch.go | 5 + systemtest/fleet_test.go | 173 ++++++++++++++ systemtest/fleettest/client.go | 265 ++++++++++++++++++++ systemtest/fleettest/types.go | 69 ++++++ systemtest/kibana.go | 44 ++++ testing/docker/package-registry/config.yml | 9 + 11 files changed, 887 insertions(+), 13 deletions(-) create mode 100644 systemtest/fleet_test.go create mode 100644 systemtest/fleettest/client.go create mode 100644 systemtest/fleettest/types.go create mode 100644 systemtest/kibana.go create mode 100644 testing/docker/package-registry/config.yml diff --git a/docker-compose.yml b/docker-compose.yml index 0deb5d7e0d6..364f586ab07 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,3 +64,20 @@ services: ELASTICSEARCH_URL: elasticsearch:9200 ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-kibana_system_user}" ELASTICSEARCH_PASSWORD: "${KIBANA_ES_PASS:-changeme}" + XPACK_XPACK_MAIN_TELEMETRY_ENABLED: "false" + XPACK_SECURITY_ENCRYPTIONKEY: "fhjskloppd678ehkdfdlliverpoolfcr" + XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: "fhjskloppd678ehkdfdlliverpoolfcr" + XPACK_FLEET_AGENTS_ELASTICSEARCH_HOST: "http://elasticsearch:9200" + XPACK_FLEET_AGENTS_KIBANA_HOST: "http://kibana:5601" + XPACK_FLEET_AGENTS_TLSCHECKDISABLED: "true" + XPACK_FLEET_REGISTRYURL: "http://package-registry:8080" + depends_on: + package-registry: { condition: service_healthy } + + package-registry: + image: docker.elastic.co/package-registry/distribution:snapshot + ports: + - 8080 + volumes: + - "./testing/docker/package-registry/config.yml:/package-registry/config.yml" + - "./apmpackage/apm:/packages/local/apm" diff --git a/go.sum b/go.sum index 9ac3767ff88..b42b51fdde9 100644 --- a/go.sum +++ b/go.sum @@ -138,6 +138,7 @@ github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43/go.mod h1:tJPYQG4mn github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antlr/antlr4 v0.0.0-20200820155224-be881fa6b91d h1:OE3kzLBpy7pOJEzE55j9sdgrSilUPzzj++FWvp1cmIs= github.com/antlr/antlr4 v0.0.0-20200820155224-be881fa6b91d/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y= github.com/antonmedv/expr v1.8.9/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= @@ -239,7 +240,9 @@ github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHo github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.0.0 h1:XJIw/+VlJ+87J+doOxznsAWIdmWuViOVhkQamW5YV28= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -501,7 +504,9 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gocarina/gocsv v0.0.0-20170324095351-ffef3ffc77be/go.mod h1:/oj50ZdPq/cUjA02lMZhijk5kR31SEydKyqah1OgBuo= github.com/gocql/gocql v0.0.0-20200228163523-cd4b606dd2fb/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e h1:BWhy2j3IXJhjCbC68FptL43tDKIq8FladmaTs3Xs7Z8= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godror/godror v0.10.4/go.mod h1:9MVLtu25FBJBMHkPs0m3Ngf/VmwGcLpM2HS8PlNGw9U= github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= @@ -693,6 +698,7 @@ github.com/hashicorp/yamux v0.0.0-20190923154419-df201c70410d/go.mod h1:+NfK9FKe github.com/haya14busa/go-actions-toolkit v0.0.0-20200105081403-ca0307860f01/go.mod h1:1DWDZmeYf0LX30zscWb7K9rUMeirNeBMd5Dum+seUhc= github.com/haya14busa/go-checkstyle v0.0.0-20170303121022-5e9d09f51fa1/go.mod h1:RsN5RGgVYeXpcXNtWyztD5VIe7VNSEqpJvF2iEH7QvI= github.com/haya14busa/secretbox v0.0.0-20180525171038-07c7ecf409f5/go.mod h1:FGO/dXIFZnan7KvvUSFk1hYMnoVNzB6NTMPrmke8SSI= +github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/hetznercloud/hcloud-go v1.22.0/go.mod h1:xng8lbDUg+xM1dgc0yGHX5EeqbwIq7UYlMWMTx3SQVg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -755,6 +761,7 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/justinas/nosurf v1.1.0/go.mod h1:ALpWdSbuNGy2lZWtyXdjkYv4edL23oSEgfBT1gPJ5BQ= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= +github.com/kardianos/service v1.1.0 h1:QV2SiEeWK42P0aEmGcsAgjApw/lRxkwopvT+Gu6t1/0= github.com/kardianos/service v1.1.0/go.mod h1:RrJI2xn5vve/r32U5suTbeaSGoMU6GbNPoj36CVYcHc= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= @@ -892,6 +899,7 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olivere/elastic v6.2.27+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= @@ -944,6 +952,7 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.5/go.mod h1:KpXfKdgRDnnhsxw4pNIH9Md5lyFqKUa4YDFlwRYAMyE= github.com/orijtech/prometheus-go-metrics-exporter v0.0.6/go.mod h1:BiTx/ugZex8LheBk3j53tktWaRdFjV5FCfT2o0P7msE= +github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= diff --git a/systemtest/apmservertest/command.go b/systemtest/apmservertest/command.go index fc554bd8a31..a9c0a82354b 100644 --- a/systemtest/apmservertest/command.go +++ b/systemtest/apmservertest/command.go @@ -33,7 +33,7 @@ import ( // ServerCommand returns a ServerCmd (wrapping os/exec) for running // apm-server with args. func ServerCommand(subcommand string, args ...string) *ServerCmd { - binary, buildErr := buildServer() + binary, buildErr := BuildServerBinary(runtime.GOOS) if buildErr != nil { // Dummy command; Start etc. will return the build error. binary = "/usr/bin/false" @@ -149,22 +149,33 @@ func (c *ServerCmd) cleanup() { } } -// buildServer builds the apm-server binary, returning its absolute path. -func buildServer() (string, error) { +// BuildServerBinary builds the apm-server binary for the given GOOS, +// returning its absolute path. +func BuildServerBinary(goos string) (string, error) { + // Build apm-server binary in the repo root, unless + // we're building for another GOOS, in which case we + // suffix the binary with that GOOS and place it in + // the build directory. + var reldir, suffix string + if goos != runtime.GOOS { + reldir = "build/" + suffix = "-" + goos + if runtime.GOOS == "windows" { + suffix += ".exe" + } + } + apmServerBinaryMu.Lock() defer apmServerBinaryMu.Unlock() - if apmServerBinary != "" { - return apmServerBinary, nil + if binary := apmServerBinary[goos]; binary != "" { + return binary, nil } repoRoot, err := getRepoRoot() if err != nil { return "", err } - abspath := filepath.Join(repoRoot, "apm-server") - if runtime.GOOS == "windows" { - abspath += ".exe" - } + abspath := filepath.Join(repoRoot, reldir, "apm-server"+suffix) log.Println("Building apm-server...") cmd := exec.Command("go", "build", "-o", abspath, "./x-pack/apm-server") @@ -175,8 +186,8 @@ func buildServer() (string, error) { return "", err } log.Println("Built", abspath) - apmServerBinary = abspath - return apmServerBinary, nil + apmServerBinary[goos] = abspath + return abspath, nil } func getRepoRoot() (string, error) { @@ -197,7 +208,7 @@ func getRepoRoot() (string, error) { var ( apmServerBinaryMu sync.Mutex - apmServerBinary string + apmServerBinary = make(map[string]string) repoRootMu sync.Mutex repoRoot string diff --git a/systemtest/apmservertest/config.go b/systemtest/apmservertest/config.go index 9f5e63f8bd7..934e85e58e0 100644 --- a/systemtest/apmservertest/config.go +++ b/systemtest/apmservertest/config.go @@ -405,7 +405,7 @@ func DefaultConfig() Config { Scheme: "http", Host: net.JoinHostPort( getenvDefault("KIBANA_HOST", defaultKibanaHost), - getenvDefault("KIBANA_PORT", defaultKibanaPort), + DefaultKibanaPort(), ), }).String(), Username: getenvDefault("KIBANA_USER", defaultKibanaUser), @@ -436,6 +436,12 @@ func DefaultConfig() Config { } } +// DefaultKibanaPort returns the Kibana port, configured using +// KIBANA_PORT, or otherwise returning the default of 5601. +func DefaultKibanaPort() string { + return getenvDefault("KIBANA_PORT", defaultKibanaPort) +} + func getenvDefault(k, defaultv string) string { v := os.Getenv(k) if v == "" { diff --git a/systemtest/containers.go b/systemtest/containers.go index edb361edf86..c3e19b544ec 100644 --- a/systemtest/containers.go +++ b/systemtest/containers.go @@ -20,12 +20,17 @@ package systemtest import ( "context" "fmt" + "io" + "io/ioutil" "log" "net" "net/url" "os" "os/exec" + "path" + "path/filepath" "strings" + "sync" "time" "github.com/docker/docker/api/types" @@ -34,6 +39,7 @@ import ( "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" + "github.com/elastic/apm-server/systemtest/apmservertest" "github.com/elastic/apm-server/systemtest/estest" "github.com/elastic/go-elasticsearch/v7" ) @@ -234,3 +240,263 @@ func (c *ElasticsearchContainer) Start() error { func (c *ElasticsearchContainer) Close() error { return c.container.Terminate(context.Background()) } + +// NewUnstartedElasticAgentContainer returns a new ElasticAgentContainer. +func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) { + // Create a testcontainer.ContainerRequest to run Elastic Agent. + // We pull some configuration from the Kibana docker-compose service, + // such as the Docker network to use. + + docker, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + return nil, err + } + defer docker.Close() + + kibanaContainer, err := stackContainerInfo(context.Background(), docker, "kibana") + if err != nil { + return nil, err + } + kibanaContainerDetails, err := docker.ContainerInspect(context.Background(), kibanaContainer.ID) + if err != nil { + return nil, err + } + + var kibanaIPAddress string + var networks []string + for network, settings := range kibanaContainerDetails.NetworkSettings.Networks { + networks = append(networks, network) + if kibanaIPAddress == "" && settings.IPAddress != "" { + kibanaIPAddress = settings.IPAddress + } + } + kibanaURL := &url.URL{ + Scheme: "http", + Host: net.JoinHostPort(kibanaIPAddress, apmservertest.DefaultKibanaPort()), + } + + // Use the same stack version as used for Kibana. + agentImageVersion := kibanaContainer.Image[strings.LastIndex(kibanaContainer.Image, ":")+1:] + agentImage := "docker.elastic.co/beats/elastic-agent:" + agentImageVersion + if err := pullDockerImage(context.Background(), docker, agentImage); err != nil { + return nil, err + } + agentImageDetails, _, err := docker.ImageInspectWithRaw(context.Background(), agentImage) + if err != nil { + return nil, err + } + agentVCSRef := agentImageDetails.Config.Labels["org.label-schema.vcs-ref"] + agentDataHashDir := path.Join("/usr/share/elastic-agent/data", "elastic-agent-"+agentVCSRef[:6]) + agentDownloadsDir := path.Join(agentDataHashDir, "downloads") + agentInstallDir := path.Join(agentDataHashDir, "install") + + // Build elastic-agent to replace the binary inside the container. + // + // This enables us to inject a locally built, unsigned, apm-server artifact. + // + // TODO(axw) once apm-server is bundled we can stop building a custom + // elastic-agent. We can then inject a custom apm-server into the *install* + // directory. We do that now, but we still need to inject a custom artifact + // into the *downloads* directory as well, to prevent fetching from the + // internet. + agentBinary, err := buildElasticAgent(agentVCSRef) + if err != nil { + return nil, err + } + + req := testcontainers.ContainerRequest{ + Image: agentImage, + AutoRemove: true, + Networks: networks, + Env: map[string]string{ + "FLEET_ENROLL": "1", + "FLEET_ENROLL_INSECURE": "1", + "FLEET_SETUP": "1", + "KIBANA_HOST": kibanaURL.String(), + "KIBANA_USERNAME": adminKibanaUser, + "KIBANA_PASSWORD": adminKibanaPass, + + // TODO(axw) remove once https://github.com/elastic/elastic-agent-client/issues/20 is fixed + "GODEBUG": "x509ignoreCN=0", + + // NOTE(axw) because we bind-mount the apm-server artifacts in, they end up owned by the + // current user rather than root. Disable Beats's strict permission checks to avoid resulting + // complaints, as they're irrelevant to these system tests. + "BEAT_STRICT_PERMS": "false", + }, + BindMounts: map[string]string{ + agentBinary: path.Join(agentDataHashDir, "elastic-agent"), + }, + } + return &ElasticAgentContainer{ + request: req, + downloadsDir: agentDownloadsDir, + installDir: agentInstallDir, + StackVersion: agentImageVersion, + BindMountDownloads: make(map[string]string), + BindMountInstall: make(map[string]string), + }, nil +} + +// ElasticAgentContainer represents an ephemeral Elastic Agent container. +type ElasticAgentContainer struct { + container testcontainers.Container + request testcontainers.ContainerRequest + + // downloadsDir holds the location of the "downloads" directory inside + // the Elastic Agent container. + // + // This will be set when the ElasticAgentContainer object is created, + // and can be used to anticipate the location into which artifacts + // can be bind-mounted. + downloadsDir string + + // installDir holds the location of the "install" directory inside + // the Elastic Agent container. + // + // This will be set when the ElasticAgentContainer object is created, + // and can be used to anticipate the location into which artifacts + // can be bind-mounted. + installDir string + + // StackVersion holds the stack version of the container image, + // e.g. 8.0.0-SNAPSHOT. + StackVersion string + + // ExposedPorts holds an optional list of ports to expose to the host. + ExposedPorts []string + + // WaitingFor holds an optional wait strategy. + WaitingFor wait.Strategy + + // Addrs holds the "host:port" address for each exposed port. + // This will be populated by Start. + Addrs []string + + // BindMountDownloads holds a map of files to bind mount into the + // container, mapping from the host location to target paths relative + // to the downloads directory in the container. + BindMountDownloads map[string]string + + // BindMountInstall holds a map of files to bind mount into the + // container, mapping from the host location to target paths relative + // to the install directory in the container. + BindMountInstall map[string]string +} + +// Start starts the container. +// +// The Addr and Client fields will be updated on successful return. +// +// The container will be removed when Close() is called, or otherwise by a +// reaper process if the test process is aborted. +func (c *ElasticAgentContainer) Start() error { + ctx, cancel := context.WithTimeout(context.Background(), startContainersTimeout) + defer cancel() + + // Update request from user-definable fields. + c.request.ExposedPorts = c.ExposedPorts + c.request.WaitingFor = c.WaitingFor + for source, target := range c.BindMountDownloads { + c.request.BindMounts[source] = path.Join(c.downloadsDir, target) + } + for source, target := range c.BindMountInstall { + c.request.BindMounts[source] = path.Join(c.installDir, target) + } + + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: c.request, + Started: true, + }) + if err != nil { + return err + } + defer func() { + if c.container == nil { + // Something has gone wrong. + container.Terminate(ctx) + } + }() + + ports, err := container.Ports(ctx) + if err != nil { + return err + } + if len(ports) > 0 { + ip, err := container.Host(ctx) + if err != nil { + return err + } + for _, portbindings := range ports { + for _, pb := range portbindings { + c.Addrs = append(c.Addrs, net.JoinHostPort(ip, pb.HostPort)) + } + } + } + + c.container = container + return nil +} + +// Close terminates and removes the container. +func (c *ElasticAgentContainer) Close() error { + return c.container.Terminate(context.Background()) +} + +func pullDockerImage(ctx context.Context, docker *client.Client, imageRef string) error { + rc, err := docker.ImagePull(context.Background(), imageRef, types.ImagePullOptions{}) + if err != nil { + return err + } + defer rc.Close() + _, err = io.Copy(ioutil.Discard, rc) + return err +} + +// buildElasticAgent builds elastic-agent from the commit defined in go.mod, +// in development mode to enable injecting unsigned artifacts. +// +// The "commit" argumented passed in comes from the Docker image, and is set +// in the binary so that it has a consistent idea of the directory structure. +func buildElasticAgent(commit string) (string, error) { + elasticAgentBinaryMu.Lock() + defer elasticAgentBinaryMu.Unlock() + if elasticAgentBinary != "" { + return elasticAgentBinary, nil + } + + // Build apm-server binary from the repo root, store it in the build dir. + output, err := exec.Command("go", "list", "-m", "-f={{.Dir}}/..").Output() + if err != nil { + return "", err + } + repoRoot := filepath.Clean(strings.TrimSpace(string(output))) + abspath := filepath.Join(repoRoot, "build", "elastic-agent-nopgp") + + log.Println("Building elastic-agent...") + ldflags := "" + + " -X github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.snapshot=true" + + " -X github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.allowEmptyPgp=true" + + " -X github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.allowUpgrade=true" + + " -X github.com/elastic/beats/v7/libbeat/version.commit=" + commit + cmd := exec.Command("go", "build", "-o", abspath, + "-ldflags", ldflags, + "github.com/elastic/beats/v7/x-pack/elastic-agent", + ) + cmd.Dir = repoRoot + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, "GOOS=linux") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return "", err + } + log.Println("Built", abspath) + elasticAgentBinary = abspath + return elasticAgentBinary, nil +} + +var ( + elasticAgentBinaryMu sync.Mutex + elasticAgentBinary string +) diff --git a/systemtest/elasticsearch.go b/systemtest/elasticsearch.go index 39b420ddfc7..88c522e6f57 100644 --- a/systemtest/elasticsearch.go +++ b/systemtest/elasticsearch.go @@ -132,6 +132,11 @@ func CleanupElasticsearch(t testing.TB) { t.Fatal(err) } + // Delete index templates after deleting data streams. + if err := doReq(esapi.IndicesDeleteIndexTemplateRequest{Name: legacyPrefix}); err != nil { + t.Fatal(err) + } + // Delete the ILM policy last or we'll get an error due to it being in use. for { err := doReq(esapi.ILMDeleteLifecycleRequest{Policy: "apm-rollover-30-days"}) diff --git a/systemtest/fleet_test.go b/systemtest/fleet_test.go new file mode 100644 index 00000000000..b438c06148e --- /dev/null +++ b/systemtest/fleet_test.go @@ -0,0 +1,173 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package systemtest_test + +import ( + "crypto/sha512" + "fmt" + "io/ioutil" + "net/url" + "os" + "path" + "path/filepath" + "runtime" + "testing" + + "github.com/elastic/apm-server/systemtest" + "github.com/elastic/apm-server/systemtest/apmservertest" + "github.com/elastic/apm-server/systemtest/fleettest" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go/wait" + "go.elastic.co/apm" + "go.elastic.co/apm/transport" +) + +func TestFleetIntegration(t *testing.T) { + systemtest.CleanupElasticsearch(t) + + fleet := fleettest.NewClient(systemtest.KibanaURL.String()) + require.NoError(t, fleet.Setup()) + cleanupFleet(t, fleet) + defer cleanupFleet(t, fleet) + + agentPolicy, err := fleet.CreateAgentPolicy("apm_systemtest", "default", "Agent policy for APM Server system tests") + require.NoError(t, err) + + // Find the "apm" package to install. + var apmPackage *fleettest.Package + packages, err := fleet.ListPackages() + require.NoError(t, err) + for _, pkg := range packages { + if pkg.Name == "apm" { + apmPackage = &pkg + break + } + } + require.NotNil(t, apmPackage) + + // Add the "apm" integration to the agent policy. + packagePolicy := fleettest.PackagePolicy{ + Name: "apm", + Namespace: "default", + Enabled: true, + AgentPolicyID: agentPolicy.ID, + } + packagePolicy.Package.Name = apmPackage.Name + packagePolicy.Package.Version = apmPackage.Version + packagePolicy.Package.Title = apmPackage.Title + packagePolicy.Inputs = []fleettest.PackagePolicyInput{{ + Type: "apm", + Enabled: true, + Streams: []interface{}{}, + Vars: map[string]interface{}{ + "enable_rum": map[string]interface{}{ + "type": "bool", + "value": false, + }, + "host": map[string]interface{}{ + "type": "string", + "value": ":8200", + }, + }, + }} + err = fleet.CreatePackagePolicy(packagePolicy) + require.NoError(t, err) + + agent, err := systemtest.NewUnstartedElasticAgentContainer() + require.NoError(t, err) + + // Build apm-server, and bind-mount it into the elastic-agent container's "install" + // directory. This bypasses downloading the artifact. + arch := runtime.GOARCH + if arch == "amd64" { + arch = "x86_64" + } + apmServerArtifactName := fmt.Sprintf("apm-server-%s-linux-%s", agent.StackVersion, arch) + + // Bind-mount the apm-server binary and apm-server.yml into the container's + // "install" directory. This causes elastic-agent to skip installing the + // artifact. + apmServerBinary, err := apmservertest.BuildServerBinary("linux") + require.NoError(t, err) + agent.BindMountInstall[apmServerBinary] = path.Join(apmServerArtifactName, "apm-server") + apmServerConfigFile, err := filepath.Abs("../apm-server.yml") + require.NoError(t, err) + agent.BindMountInstall[apmServerConfigFile] = path.Join(apmServerArtifactName, "apm-server.yml") + + // We also place a file (any file) in the download location to skip fetching + // the artifact from the internet. + // + // TODO(axw) we can skip this once apm-server is bundled with Elastic Agent. + tempdir, err := ioutil.TempDir("", "apmservertest") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + apmServerArchive := filepath.Join(tempdir, apmServerArtifactName+".tar.gz") + apmServerArchiveSHA512 := filepath.Join(tempdir, apmServerArtifactName+".tar.gz.sha512") + require.NoError(t, ioutil.WriteFile(apmServerArchive, nil, 0644)) + require.NoError(t, ioutil.WriteFile(apmServerArchiveSHA512, []byte(fmt.Sprintf("%x %s", sha512.New().Sum(nil), filepath.Base(apmServerArchive))), 0644)) + agent.BindMountDownloads[apmServerArchive] = filepath.Base(apmServerArchive) + agent.BindMountDownloads[apmServerArchiveSHA512] = filepath.Base(apmServerArchiveSHA512) + + // Start elastic-agent with port 8200 exposed, and wait for the server to service + // healthcheck requests to port 8200. + agent.ExposedPorts = []string{"8200"} + waitFor := wait.ForHTTP("/") + waitFor.Port = "8200/tcp" + agent.WaitingFor = waitFor + require.NoError(t, agent.Start()) + defer agent.Close() + + // Elastic Agent has started apm-server. Connect to apm-server and send some data, + // and make sure it gets indexed into a data stream. + require.Len(t, agent.Addrs, 1) + transport, err := transport.NewHTTPTransport() + require.NoError(t, err) + transport.SetServerURL(&url.URL{Scheme: "http", Host: agent.Addrs[0]}) + tracer, err := apm.NewTracerOptions(apm.TracerOptions{Transport: transport}) + require.NoError(t, err) + tracer.StartTransaction("name", "type").End() + tracer.Flush(nil) + + systemtest.Elasticsearch.ExpectDocs(t, "traces-*", nil) +} + +func cleanupFleet(t testing.TB, fleet *fleettest.Client) { + apmAgentPolicies, err := fleet.AgentPolicies("ingest-agent-policies.name:apm_systemtest") + require.NoError(t, err) + if len(apmAgentPolicies) == 0 { + return + } + + agents, err := fleet.Agents() + require.NoError(t, err) + agentsByPolicy := make(map[string][]fleettest.Agent) + for _, agent := range agents { + agentsByPolicy[agent.PolicyID] = append(agentsByPolicy[agent.PolicyID], agent) + } + + for _, p := range apmAgentPolicies { + if agents := agentsByPolicy[p.ID]; len(agents) > 0 { + agentIDs := make([]string, len(agents)) + for i, agent := range agents { + agentIDs[i] = agent.ID + } + require.NoError(t, fleet.BulkUnenrollAgents(true, agentIDs...)) + } + require.NoError(t, fleet.DeleteAgentPolicy(p.ID)) + } +} diff --git a/systemtest/fleettest/client.go b/systemtest/fleettest/client.go new file mode 100644 index 00000000000..ae7f7fd7874 --- /dev/null +++ b/systemtest/fleettest/client.go @@ -0,0 +1,265 @@ +package fleettest + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" +) + +// Client provides methods for interacting with the Fleet API. +type Client struct { + fleetURL string +} + +// NewClient returns a new Client for interacting with the Fleet API, +// using the given Kibana URL. +func NewClient(kibanaURL string) *Client { + return &Client{fleetURL: kibanaURL + "/api/fleet"} +} + +// Setup invokes the Fleet Setup API, returning an error if it fails. +func (c *Client) Setup() error { + req, err := http.NewRequest("POST", c.fleetURL+"/setup", nil) + if err != nil { + return err + } + req.Header.Set("kbn-xsrf", "1") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + return nil +} + +// Agents returns the list of enrolled agents. +func (c *Client) Agents() ([]Agent, error) { + resp, err := http.Get(c.fleetURL + "/agents") + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return nil, fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + var result struct { + List []Agent `json:"list"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + return result.List, nil +} + +// BulkUnenrollAgents bulk-unenrolls agents. +func (c *Client) BulkUnenrollAgents(force bool, agentIDs ...string) error { + var body bytes.Buffer + type bulkUnenroll struct { + Agents []string `json:"agents"` + Force bool `json:"force"` + } + if err := json.NewEncoder(&body).Encode(bulkUnenroll{agentIDs, force}); err != nil { + return err + } + req, err := http.NewRequest("POST", c.fleetURL+"/agents/bulk_unenroll", &body) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("kbn-xsrf", "1") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + return nil +} + +// AgentPolicies returns the Agent Policies matching the given KQL query. +func (c *Client) AgentPolicies(kuery string) ([]AgentPolicy, error) { + u, err := url.Parse(c.fleetURL + "/agent_policies") + if err != nil { + return nil, err + } + query := u.Query() + query.Add("kuery", kuery) + u.RawQuery = query.Encode() + resp, err := http.Get(u.String()) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return nil, fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + var result struct { + Items []AgentPolicy `json:"items"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + return result.Items, nil +} + +// DeleteAgentPolicy deletes the Agent Policy with the given ID. +func (c *Client) DeleteAgentPolicy(id string) error { + var body bytes.Buffer + type deleteAgentPolicy struct { + ID string `json:"agentPolicyId"` + } + if err := json.NewEncoder(&body).Encode(deleteAgentPolicy{id}); err != nil { + return err + } + req, err := http.NewRequest("POST", c.fleetURL+"/agent_policies/delete", &body) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("kbn-xsrf", "1") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + return nil +} + +// CreateAgentPolicy returns the default Agent Policy. +func (c *Client) CreateAgentPolicy(name, namespace, description string) (*AgentPolicy, error) { + var body bytes.Buffer + type newAgentPolicy struct { + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` + Description string `json:"description,omitempty"` + } + if err := json.NewEncoder(&body).Encode(newAgentPolicy{name, namespace, description}); err != nil { + return nil, err + } + req, err := http.NewRequest("POST", c.fleetURL+"/agent_policies", &body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("kbn-xsrf", "1") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return nil, fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + var result struct { + Item AgentPolicy `json:"item"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + return &result.Item, nil +} + +// ListPackages lists all packages available for installation. +func (c *Client) ListPackages() ([]Package, error) { + resp, err := http.Get(c.fleetURL + "/epm/packages?experimental=true") + if err != nil { + return nil, err + } + defer resp.Body.Close() + var result struct { + Response []Package `json:"response"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + return result.Response, nil +} + +// PackagePolicy returns information about the package policy with the given ID. +func (c *Client) PackagePolicy(id string) (*PackagePolicy, error) { + resp, err := http.Get(c.fleetURL + "/package_policies/" + id) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return nil, fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + var result struct { + Item PackagePolicy `json:"item"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + return &result.Item, nil +} + +// CreatePackagePolicy adds an integration to a policy. +func (c *Client) CreatePackagePolicy(p PackagePolicy) error { + var body bytes.Buffer + if err := json.NewEncoder(&body).Encode(&p); err != nil { + return err + } + req, err := http.NewRequest("POST", c.fleetURL+"/package_policies", &body) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("kbn-xsrf", "1") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + return nil +} + +// DeletePackagePolicy deletes one or more package policies. +func (c *Client) DeletePackagePolicy(ids ...string) error { + var params struct { + PackagePolicyIDs []string `json:"packagePolicyIds"` + } + params.PackagePolicyIDs = ids + var body bytes.Buffer + if err := json.NewEncoder(&body).Encode(params); err != nil { + return err + } + req, err := http.NewRequest("POST", c.fleetURL+"/package_policies/delete", &body) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("kbn-xsrf", "1") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + return nil +} diff --git a/systemtest/fleettest/types.go b/systemtest/fleettest/types.go new file mode 100644 index 00000000000..0c0872d9d84 --- /dev/null +++ b/systemtest/fleettest/types.go @@ -0,0 +1,69 @@ +package fleettest + +import "time" + +// Agent holds details of a Fleet Agent. +type Agent struct { + ID string `json:"id"` + Active bool `json:"active"` + Status string `json:"status"` + Type string `json:"type"` + PolicyID string `json:"policy_id,omitempty"` + EnrolledAt time.Time `json:"enrolled_at,omitempty"` + UserProvidedMetadata map[string]interface{} `json:"user_provided_metadata,omitempty"` + LocalMetadata map[string]interface{} `json:"local_metadata,omitempty"` +} + +// AgentPolicy holds details of a Fleet Agent Policy. +type AgentPolicy struct { + ID string `json:"id"` + Name string `json:"name"` + Namespace string `json:"namespace"` + Description string `json:"description"` + Revision int `json:"revision"` + + Agents int `json:"agents"` + IsDefault bool `json:"is_default"` + MonitoringEnabled []string `json:"monitoring_enabled"` + PackagePolicies []string `json:"package_policies"` + Status string `json:"status"` + UpdatedAt time.Time `json:"updated_at"` + UpdatedBy string `json:"updated_by"` +} + +// PackagePolicy holds details of a Fleet Package Policy. +type PackagePolicy struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + Namespace string `json:"namespace"` + Enabled bool `json:"enabled"` + Description string `json:"description"` + AgentPolicyID string `json:"policy_id"` + OutputID string `json:"output_id"` + Inputs []PackagePolicyInput `json:"inputs"` + Package struct { + Name string `json:"name"` + Version string `json:"version"` + Title string `json:"title"` + } `json:"package"` +} + +type PackagePolicyInput struct { + Type string `json:"type"` + Enabled bool `json:"enabled"` + Streams []interface{} `json:"streams"` + Config map[string]interface{} `json:"config,omitempty"` + Vars map[string]interface{} `json:"vars,omitempty"` +} + +type Package struct { + Name string `json:"name"` + Version string `json:"version"` + Release string `json:"release"` + Type string `json:"type"` + Title string `json:"title"` + Description string `json:"description"` + Download string `json:"download"` + Path string `json:"path"` + Status string `json:"status"` +} diff --git a/systemtest/kibana.go b/systemtest/kibana.go new file mode 100644 index 00000000000..64d61a30a89 --- /dev/null +++ b/systemtest/kibana.go @@ -0,0 +1,44 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package systemtest + +import ( + "log" + "net/url" + + "github.com/elastic/apm-server/systemtest/apmservertest" +) + +const ( + adminKibanaUser = adminElasticsearchUser + adminKibanaPass = adminElasticsearchPass +) + +// KibanaURL is the base URL for Kibana, including userinfo for +// authenticating as the admin user. +var KibanaURL *url.URL + +func init() { + kibanaConfig := apmservertest.DefaultConfig().Kibana + u, err := url.Parse(kibanaConfig.Host) + if err != nil { + log.Fatal(err) + } + u.User = url.UserPassword(adminKibanaUser, adminKibanaPass) + KibanaURL = u +} diff --git a/testing/docker/package-registry/config.yml b/testing/docker/package-registry/config.yml new file mode 100644 index 00000000000..82fd5881241 --- /dev/null +++ b/testing/docker/package-registry/config.yml @@ -0,0 +1,9 @@ +package_paths: + - /packages/production + - /packages/staging + - /packages/snapshot + - /packages/local + +cache_time.search: 10s +cache_time.categories: 10s +cache_time.catch_all: 10s From 82f9b31ca8172c4b34ad641232aed264fd586d20 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Mon, 25 Jan 2021 18:22:31 +0800 Subject: [PATCH 02/11] make fmt --- beater/http.go | 6 +++--- systemtest/fleet_test.go | 7 ++++--- systemtest/fleettest/client.go | 17 +++++++++++++++++ systemtest/fleettest/types.go | 17 +++++++++++++++++ 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/beater/http.go b/beater/http.go index 9c18d43c696..ae0a7773458 100644 --- a/beater/http.go +++ b/beater/http.go @@ -23,9 +23,6 @@ import ( "net/http" "net/url" - "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/gmux" "go.elastic.co/apm" "go.elastic.co/apm/module/apmhttp" "golang.org/x/net/netutil" @@ -33,6 +30,9 @@ import ( "github.com/elastic/apm-server/beater/api" "github.com/elastic/apm-server/beater/config" "github.com/elastic/apm-server/publish" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/gmux" ) type httpServer struct { diff --git a/systemtest/fleet_test.go b/systemtest/fleet_test.go index b438c06148e..a42bb708e65 100644 --- a/systemtest/fleet_test.go +++ b/systemtest/fleet_test.go @@ -28,13 +28,14 @@ import ( "runtime" "testing" - "github.com/elastic/apm-server/systemtest" - "github.com/elastic/apm-server/systemtest/apmservertest" - "github.com/elastic/apm-server/systemtest/fleettest" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go/wait" "go.elastic.co/apm" "go.elastic.co/apm/transport" + + "github.com/elastic/apm-server/systemtest" + "github.com/elastic/apm-server/systemtest/apmservertest" + "github.com/elastic/apm-server/systemtest/fleettest" ) func TestFleetIntegration(t *testing.T) { diff --git a/systemtest/fleettest/client.go b/systemtest/fleettest/client.go index ae7f7fd7874..f7552b12c6c 100644 --- a/systemtest/fleettest/client.go +++ b/systemtest/fleettest/client.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package fleettest import ( diff --git a/systemtest/fleettest/types.go b/systemtest/fleettest/types.go index 0c0872d9d84..c228db8d9e1 100644 --- a/systemtest/fleettest/types.go +++ b/systemtest/fleettest/types.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package fleettest import "time" From 664dabc6698fa1ed1f3fd4ffacf426d69c04b1e4 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Mon, 1 Feb 2021 20:43:56 +0800 Subject: [PATCH 03/11] Address review comments - Don't create archive in downloads (APM Server is bundled) - DefaultKibanaPort -> KibanaPort --- systemtest/apmservertest/config.go | 6 +++--- systemtest/containers.go | 29 +++++------------------------ systemtest/fleet_test.go | 17 ----------------- 3 files changed, 8 insertions(+), 44 deletions(-) diff --git a/systemtest/apmservertest/config.go b/systemtest/apmservertest/config.go index 934e85e58e0..e5d496fc5c0 100644 --- a/systemtest/apmservertest/config.go +++ b/systemtest/apmservertest/config.go @@ -405,7 +405,7 @@ func DefaultConfig() Config { Scheme: "http", Host: net.JoinHostPort( getenvDefault("KIBANA_HOST", defaultKibanaHost), - DefaultKibanaPort(), + KibanaPort(), ), }).String(), Username: getenvDefault("KIBANA_USER", defaultKibanaUser), @@ -436,9 +436,9 @@ func DefaultConfig() Config { } } -// DefaultKibanaPort returns the Kibana port, configured using +// KibanaPort returns the Kibana port, configured using // KIBANA_PORT, or otherwise returning the default of 5601. -func DefaultKibanaPort() string { +func KibanaPort() string { return getenvDefault("KIBANA_PORT", defaultKibanaPort) } diff --git a/systemtest/containers.go b/systemtest/containers.go index c3e19b544ec..cfa025bd74c 100644 --- a/systemtest/containers.go +++ b/systemtest/containers.go @@ -272,7 +272,7 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) { } kibanaURL := &url.URL{ Scheme: "http", - Host: net.JoinHostPort(kibanaIPAddress, apmservertest.DefaultKibanaPort()), + Host: net.JoinHostPort(kibanaIPAddress, apmservertest.KibanaPort()), } // Use the same stack version as used for Kibana. @@ -287,7 +287,6 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) { } agentVCSRef := agentImageDetails.Config.Labels["org.label-schema.vcs-ref"] agentDataHashDir := path.Join("/usr/share/elastic-agent/data", "elastic-agent-"+agentVCSRef[:6]) - agentDownloadsDir := path.Join(agentDataHashDir, "downloads") agentInstallDir := path.Join(agentDataHashDir, "install") // Build elastic-agent to replace the binary inside the container. @@ -329,12 +328,10 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) { }, } return &ElasticAgentContainer{ - request: req, - downloadsDir: agentDownloadsDir, - installDir: agentInstallDir, - StackVersion: agentImageVersion, - BindMountDownloads: make(map[string]string), - BindMountInstall: make(map[string]string), + request: req, + installDir: agentInstallDir, + StackVersion: agentImageVersion, + BindMountInstall: make(map[string]string), }, nil } @@ -343,14 +340,6 @@ type ElasticAgentContainer struct { container testcontainers.Container request testcontainers.ContainerRequest - // downloadsDir holds the location of the "downloads" directory inside - // the Elastic Agent container. - // - // This will be set when the ElasticAgentContainer object is created, - // and can be used to anticipate the location into which artifacts - // can be bind-mounted. - downloadsDir string - // installDir holds the location of the "install" directory inside // the Elastic Agent container. // @@ -373,11 +362,6 @@ type ElasticAgentContainer struct { // This will be populated by Start. Addrs []string - // BindMountDownloads holds a map of files to bind mount into the - // container, mapping from the host location to target paths relative - // to the downloads directory in the container. - BindMountDownloads map[string]string - // BindMountInstall holds a map of files to bind mount into the // container, mapping from the host location to target paths relative // to the install directory in the container. @@ -397,9 +381,6 @@ func (c *ElasticAgentContainer) Start() error { // Update request from user-definable fields. c.request.ExposedPorts = c.ExposedPorts c.request.WaitingFor = c.WaitingFor - for source, target := range c.BindMountDownloads { - c.request.BindMounts[source] = path.Join(c.downloadsDir, target) - } for source, target := range c.BindMountInstall { c.request.BindMounts[source] = path.Join(c.installDir, target) } diff --git a/systemtest/fleet_test.go b/systemtest/fleet_test.go index a42bb708e65..53d33761dad 100644 --- a/systemtest/fleet_test.go +++ b/systemtest/fleet_test.go @@ -18,11 +18,8 @@ package systemtest_test import ( - "crypto/sha512" "fmt" - "io/ioutil" "net/url" - "os" "path" "path/filepath" "runtime" @@ -110,20 +107,6 @@ func TestFleetIntegration(t *testing.T) { require.NoError(t, err) agent.BindMountInstall[apmServerConfigFile] = path.Join(apmServerArtifactName, "apm-server.yml") - // We also place a file (any file) in the download location to skip fetching - // the artifact from the internet. - // - // TODO(axw) we can skip this once apm-server is bundled with Elastic Agent. - tempdir, err := ioutil.TempDir("", "apmservertest") - require.NoError(t, err) - defer os.RemoveAll(tempdir) - apmServerArchive := filepath.Join(tempdir, apmServerArtifactName+".tar.gz") - apmServerArchiveSHA512 := filepath.Join(tempdir, apmServerArtifactName+".tar.gz.sha512") - require.NoError(t, ioutil.WriteFile(apmServerArchive, nil, 0644)) - require.NoError(t, ioutil.WriteFile(apmServerArchiveSHA512, []byte(fmt.Sprintf("%x %s", sha512.New().Sum(nil), filepath.Base(apmServerArchive))), 0644)) - agent.BindMountDownloads[apmServerArchive] = filepath.Base(apmServerArchive) - agent.BindMountDownloads[apmServerArchiveSHA512] = filepath.Base(apmServerArchiveSHA512) - // Start elastic-agent with port 8200 exposed, and wait for the server to service // healthcheck requests to port 8200. agent.ExposedPorts = []string{"8200"} From 9c615c4a81c53a9258560bb851f3216c2a38f5e6 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Mon, 1 Feb 2021 22:27:10 +0800 Subject: [PATCH 04/11] Fix test Close Tracer when test ends --- systemtest/fleet_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/systemtest/fleet_test.go b/systemtest/fleet_test.go index 53d33761dad..5da8befc041 100644 --- a/systemtest/fleet_test.go +++ b/systemtest/fleet_test.go @@ -124,6 +124,7 @@ func TestFleetIntegration(t *testing.T) { transport.SetServerURL(&url.URL{Scheme: "http", Host: agent.Addrs[0]}) tracer, err := apm.NewTracerOptions(apm.TracerOptions{Transport: transport}) require.NoError(t, err) + defer tracer.Close() tracer.StartTransaction("name", "type").End() tracer.Flush(nil) From 7c862b97d90dc83c782ac79299424b369eab54e2 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Tue, 2 Feb 2021 11:53:34 +0800 Subject: [PATCH 05/11] Specify FLEET_ENROLLMENT_TOKEN Specify the API Key that the agent should use to enroll with Fleet, rather than supplying the Kibana username and password for it to query for a token. Using the latter approach, the agent could end up enrolling with an unrelated policy. --- systemtest/containers.go | 17 ++++-- systemtest/fleet_test.go | 3 +- systemtest/fleettest/client.go | 106 ++++++++++++++++++++++----------- systemtest/fleettest/types.go | 13 ++++ 4 files changed, 98 insertions(+), 41 deletions(-) diff --git a/systemtest/containers.go b/systemtest/containers.go index cfa025bd74c..77c0ad62bf6 100644 --- a/systemtest/containers.go +++ b/systemtest/containers.go @@ -308,12 +308,7 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) { AutoRemove: true, Networks: networks, Env: map[string]string{ - "FLEET_ENROLL": "1", - "FLEET_ENROLL_INSECURE": "1", - "FLEET_SETUP": "1", - "KIBANA_HOST": kibanaURL.String(), - "KIBANA_USERNAME": adminKibanaUser, - "KIBANA_PASSWORD": adminKibanaPass, + "KIBANA_HOST": kibanaURL.String(), // TODO(axw) remove once https://github.com/elastic/elastic-agent-client/issues/20 is fixed "GODEBUG": "x509ignoreCN=0", @@ -366,6 +361,11 @@ type ElasticAgentContainer struct { // container, mapping from the host location to target paths relative // to the install directory in the container. BindMountInstall map[string]string + + // FleetEnrollmentToken holds an optional Fleet enrollment token to + // use for enrolling the agent with Fleet. The agent will only enroll + // if this is specified. + FleetEnrollmentToken string } // Start starts the container. @@ -379,6 +379,11 @@ func (c *ElasticAgentContainer) Start() error { defer cancel() // Update request from user-definable fields. + if c.FleetEnrollmentToken != "" { + c.request.Env["FLEET_ENROLL"] = "1" + c.request.Env["FLEET_ENROLL_INSECURE"] = "1" + c.request.Env["FLEET_ENROLLMENT_TOKEN"] = c.FleetEnrollmentToken + } c.request.ExposedPorts = c.ExposedPorts c.request.WaitingFor = c.WaitingFor for source, target := range c.BindMountInstall { diff --git a/systemtest/fleet_test.go b/systemtest/fleet_test.go index 5da8befc041..e868e79ae45 100644 --- a/systemtest/fleet_test.go +++ b/systemtest/fleet_test.go @@ -43,7 +43,7 @@ func TestFleetIntegration(t *testing.T) { cleanupFleet(t, fleet) defer cleanupFleet(t, fleet) - agentPolicy, err := fleet.CreateAgentPolicy("apm_systemtest", "default", "Agent policy for APM Server system tests") + agentPolicy, enrollmentAPIKey, err := fleet.CreateAgentPolicy("apm_systemtest", "default", "Agent policy for APM Server system tests") require.NoError(t, err) // Find the "apm" package to install. @@ -88,6 +88,7 @@ func TestFleetIntegration(t *testing.T) { agent, err := systemtest.NewUnstartedElasticAgentContainer() require.NoError(t, err) + agent.FleetEnrollmentToken = enrollmentAPIKey.APIKey // Build apm-server, and bind-mount it into the elastic-agent container's "install" // directory. This bypasses downloading the artifact. diff --git a/systemtest/fleettest/client.go b/systemtest/fleettest/client.go index f7552b12c6c..60ff327f656 100644 --- a/systemtest/fleettest/client.go +++ b/systemtest/fleettest/client.go @@ -21,6 +21,7 @@ import ( "bytes" "encoding/json" "fmt" + "io" "io/ioutil" "net/http" "net/url" @@ -39,11 +40,7 @@ func NewClient(kibanaURL string) *Client { // Setup invokes the Fleet Setup API, returning an error if it fails. func (c *Client) Setup() error { - req, err := http.NewRequest("POST", c.fleetURL+"/setup", nil) - if err != nil { - return err - } - req.Header.Set("kbn-xsrf", "1") + req := c.newFleetRequest("POST", "/setup", nil) resp, err := http.DefaultClient.Do(req) if err != nil { return err @@ -86,12 +83,7 @@ func (c *Client) BulkUnenrollAgents(force bool, agentIDs ...string) error { if err := json.NewEncoder(&body).Encode(bulkUnenroll{agentIDs, force}); err != nil { return err } - req, err := http.NewRequest("POST", c.fleetURL+"/agents/bulk_unenroll", &body) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("kbn-xsrf", "1") + req := c.newFleetRequest("POST", "/agents/bulk_unenroll", &body) resp, err := http.DefaultClient.Do(req) if err != nil { return err @@ -140,12 +132,7 @@ func (c *Client) DeleteAgentPolicy(id string) error { if err := json.NewEncoder(&body).Encode(deleteAgentPolicy{id}); err != nil { return err } - req, err := http.NewRequest("POST", c.fleetURL+"/agent_policies/delete", &body) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("kbn-xsrf", "1") + req := c.newFleetRequest("POST", "/agent_policies/delete", &body) resp, err := http.DefaultClient.Do(req) if err != nil { return err @@ -159,7 +146,7 @@ func (c *Client) DeleteAgentPolicy(id string) error { } // CreateAgentPolicy returns the default Agent Policy. -func (c *Client) CreateAgentPolicy(name, namespace, description string) (*AgentPolicy, error) { +func (c *Client) CreateAgentPolicy(name, namespace, description string) (*AgentPolicy, *EnrollmentAPIKey, error) { var body bytes.Buffer type newAgentPolicy struct { Name string `json:"name,omitempty"` @@ -167,32 +154,83 @@ func (c *Client) CreateAgentPolicy(name, namespace, description string) (*AgentP Description string `json:"description,omitempty"` } if err := json.NewEncoder(&body).Encode(newAgentPolicy{name, namespace, description}); err != nil { - return nil, err + return nil, nil, err } req, err := http.NewRequest("POST", c.fleetURL+"/agent_policies", &body) if err != nil { - return nil, err + return nil, nil, err } req.Header.Set("Content-Type", "application/json") req.Header.Set("kbn-xsrf", "1") resp, err := http.DefaultClient.Do(req) if err != nil { - return nil, err + return nil, nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { body, _ := ioutil.ReadAll(resp.Body) - return nil, fmt.Errorf("request failed (%s): %s", resp.Status, body) + return nil, nil, fmt.Errorf("request failed (%s): %s", resp.Status, body) } var result struct { Item AgentPolicy `json:"item"` } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, nil, err + } + enrollmentAPIKey, err := c.getAgentPolicyEnrollmentAPIKey(result.Item.ID) + if err != nil { + return nil, nil, err + } + return &result.Item, enrollmentAPIKey, nil +} + +func (c *Client) getAgentPolicyEnrollmentAPIKey(policyID string) (*EnrollmentAPIKey, error) { + keys, err := c.enrollmentAPIKeys("fleet-enrollment-api-keys.policy_id:" + policyID) + if err != nil { + return nil, err + } + if n := len(keys); n != 1 { + return nil, fmt.Errorf("expected 1 enrollment API key, got %d", n) + } + resp, err := http.Get(c.fleetURL + "/enrollment-api-keys/" + keys[0].ID) + if err != nil { + return nil, err + } + var result struct { + Item EnrollmentAPIKey `json:"item"` + } if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { return nil, err } return &result.Item, nil } +func (c *Client) enrollmentAPIKeys(kuery string) ([]EnrollmentAPIKey, error) { + u, err := url.Parse(c.fleetURL + "/enrollment-api-keys") + if err != nil { + return nil, err + } + query := u.Query() + query.Add("kuery", kuery) + u.RawQuery = query.Encode() + resp, err := http.Get(u.String()) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return nil, fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + var result struct { + Items []EnrollmentAPIKey `json:"list"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + return result.Items, nil +} + // ListPackages lists all packages available for installation. func (c *Client) ListPackages() ([]Package, error) { resp, err := http.Get(c.fleetURL + "/epm/packages?experimental=true") @@ -235,12 +273,7 @@ func (c *Client) CreatePackagePolicy(p PackagePolicy) error { if err := json.NewEncoder(&body).Encode(&p); err != nil { return err } - req, err := http.NewRequest("POST", c.fleetURL+"/package_policies", &body) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("kbn-xsrf", "1") + req := c.newFleetRequest("POST", "/package_policies", &body) resp, err := http.DefaultClient.Do(req) if err != nil { return err @@ -263,12 +296,7 @@ func (c *Client) DeletePackagePolicy(ids ...string) error { if err := json.NewEncoder(&body).Encode(params); err != nil { return err } - req, err := http.NewRequest("POST", c.fleetURL+"/package_policies/delete", &body) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("kbn-xsrf", "1") + req := c.newFleetRequest("POST", "/package_policies/delete", &body) resp, err := http.DefaultClient.Do(req) if err != nil { return err @@ -280,3 +308,13 @@ func (c *Client) DeletePackagePolicy(ids ...string) error { } return nil } + +func (c *Client) newFleetRequest(method string, path string, body io.Reader) *http.Request { + req, err := http.NewRequest(method, c.fleetURL+path, body) + if err != nil { + panic(err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("kbn-xsrf", "1") + return req +} diff --git a/systemtest/fleettest/types.go b/systemtest/fleettest/types.go index c228db8d9e1..0c5fa57fd95 100644 --- a/systemtest/fleettest/types.go +++ b/systemtest/fleettest/types.go @@ -84,3 +84,16 @@ type Package struct { Path string `json:"path"` Status string `json:"status"` } + +type EnrollmentAPIKey struct { + ID string `json:"id"` + Active bool `json:"active"` + APIKeyID string `json:"api_key_id"` + Name string `json:"name"` + PolicyID string `json:"policy_id"` + CreatedAt time.Time `json:"created_at"` + + // APIKey is only returned when querying a specific enrollment API key, + // and not when listing keys. + APIKey string `json:"api_key,omitempty"` +} From a0acac5f93927975458b73687d2a0cfa72719fc2 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Tue, 2 Feb 2021 12:53:27 +0800 Subject: [PATCH 06/11] Invoke /api/fleet/agents/setup too --- systemtest/fleettest/client.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/systemtest/fleettest/client.go b/systemtest/fleettest/client.go index 60ff327f656..a20891ae998 100644 --- a/systemtest/fleettest/client.go +++ b/systemtest/fleettest/client.go @@ -40,15 +40,17 @@ func NewClient(kibanaURL string) *Client { // Setup invokes the Fleet Setup API, returning an error if it fails. func (c *Client) Setup() error { - req := c.newFleetRequest("POST", "/setup", nil) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, _ := ioutil.ReadAll(resp.Body) - return fmt.Errorf("request failed (%s): %s", resp.Status, body) + for _, path := range []string{"/setup", "/agents/setup"} { + req := c.newFleetRequest("POST", path, nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("request failed (%s): %s", resp.Status, body) + } } return nil } From f1fb3898cb346530561f50c1368466b29a69e3b6 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Tue, 2 Feb 2021 12:56:09 +0800 Subject: [PATCH 07/11] Stop building elastic-agent --- systemtest/containers.go | 68 +--------------------------------------- 1 file changed, 1 insertion(+), 67 deletions(-) diff --git a/systemtest/containers.go b/systemtest/containers.go index 77c0ad62bf6..0d6503caf89 100644 --- a/systemtest/containers.go +++ b/systemtest/containers.go @@ -28,9 +28,7 @@ import ( "os" "os/exec" "path" - "path/filepath" "strings" - "sync" "time" "github.com/docker/docker/api/types" @@ -289,20 +287,6 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) { agentDataHashDir := path.Join("/usr/share/elastic-agent/data", "elastic-agent-"+agentVCSRef[:6]) agentInstallDir := path.Join(agentDataHashDir, "install") - // Build elastic-agent to replace the binary inside the container. - // - // This enables us to inject a locally built, unsigned, apm-server artifact. - // - // TODO(axw) once apm-server is bundled we can stop building a custom - // elastic-agent. We can then inject a custom apm-server into the *install* - // directory. We do that now, but we still need to inject a custom artifact - // into the *downloads* directory as well, to prevent fetching from the - // internet. - agentBinary, err := buildElasticAgent(agentVCSRef) - if err != nil { - return nil, err - } - req := testcontainers.ContainerRequest{ Image: agentImage, AutoRemove: true, @@ -318,9 +302,6 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) { // complaints, as they're irrelevant to these system tests. "BEAT_STRICT_PERMS": "false", }, - BindMounts: map[string]string{ - agentBinary: path.Join(agentDataHashDir, "elastic-agent"), - }, } return &ElasticAgentContainer{ request: req, @@ -386,6 +367,7 @@ func (c *ElasticAgentContainer) Start() error { } c.request.ExposedPorts = c.ExposedPorts c.request.WaitingFor = c.WaitingFor + c.request.BindMounts = map[string]string{} for source, target := range c.BindMountInstall { c.request.BindMounts[source] = path.Join(c.installDir, target) } @@ -438,51 +420,3 @@ func pullDockerImage(ctx context.Context, docker *client.Client, imageRef string _, err = io.Copy(ioutil.Discard, rc) return err } - -// buildElasticAgent builds elastic-agent from the commit defined in go.mod, -// in development mode to enable injecting unsigned artifacts. -// -// The "commit" argumented passed in comes from the Docker image, and is set -// in the binary so that it has a consistent idea of the directory structure. -func buildElasticAgent(commit string) (string, error) { - elasticAgentBinaryMu.Lock() - defer elasticAgentBinaryMu.Unlock() - if elasticAgentBinary != "" { - return elasticAgentBinary, nil - } - - // Build apm-server binary from the repo root, store it in the build dir. - output, err := exec.Command("go", "list", "-m", "-f={{.Dir}}/..").Output() - if err != nil { - return "", err - } - repoRoot := filepath.Clean(strings.TrimSpace(string(output))) - abspath := filepath.Join(repoRoot, "build", "elastic-agent-nopgp") - - log.Println("Building elastic-agent...") - ldflags := "" + - " -X github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.snapshot=true" + - " -X github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.allowEmptyPgp=true" + - " -X github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.allowUpgrade=true" + - " -X github.com/elastic/beats/v7/libbeat/version.commit=" + commit - cmd := exec.Command("go", "build", "-o", abspath, - "-ldflags", ldflags, - "github.com/elastic/beats/v7/x-pack/elastic-agent", - ) - cmd.Dir = repoRoot - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, "GOOS=linux") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return "", err - } - log.Println("Built", abspath) - elasticAgentBinary = abspath - return elasticAgentBinary, nil -} - -var ( - elasticAgentBinaryMu sync.Mutex - elasticAgentBinary string -) From 9df5deaa653030bf043426c3ce275f09cf30fe93 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Tue, 2 Feb 2021 17:28:45 +0800 Subject: [PATCH 08/11] systemtest/apmservertest: actually set $GOOS --- systemtest/apmservertest/command.go | 1 + 1 file changed, 1 insertion(+) diff --git a/systemtest/apmservertest/command.go b/systemtest/apmservertest/command.go index a9c0a82354b..aa042486bb2 100644 --- a/systemtest/apmservertest/command.go +++ b/systemtest/apmservertest/command.go @@ -179,6 +179,7 @@ func BuildServerBinary(goos string) (string, error) { log.Println("Building apm-server...") cmd := exec.Command("go", "build", "-o", abspath, "./x-pack/apm-server") + cmd.Env = append(os.Environ(), "GOOS="+goos) cmd.Dir = repoRoot cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr From 1571a11a76513195743f86fa166233e2a0aeaf01 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Wed, 3 Feb 2021 09:58:21 +0800 Subject: [PATCH 09/11] Split assertion to its own line --- systemtest/fleet_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/systemtest/fleet_test.go b/systemtest/fleet_test.go index e868e79ae45..529a177f53e 100644 --- a/systemtest/fleet_test.go +++ b/systemtest/fleet_test.go @@ -114,7 +114,8 @@ func TestFleetIntegration(t *testing.T) { waitFor := wait.ForHTTP("/") waitFor.Port = "8200/tcp" agent.WaitingFor = waitFor - require.NoError(t, agent.Start()) + err = agent.Start() + require.NoError(t, err) defer agent.Close() // Elastic Agent has started apm-server. Connect to apm-server and send some data, From 9b03482dea06473979f8ba84320c68cc1866ed30 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Wed, 3 Feb 2021 12:56:59 +0800 Subject: [PATCH 10/11] systemtest: log elastic-agent output on failure --- systemtest/containers.go | 39 +++++++++++++++++++++++++-------------- systemtest/fleet_test.go | 18 ++++++++++++++++-- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/systemtest/containers.go b/systemtest/containers.go index 0d6503caf89..8a3f9b7035d 100644 --- a/systemtest/containers.go +++ b/systemtest/containers.go @@ -19,6 +19,7 @@ package systemtest import ( "context" + "errors" "fmt" "io" "io/ioutil" @@ -199,18 +200,15 @@ func (c *ElasticsearchContainer) Start() error { container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: c.request, - Started: true, }) if err != nil { return err } - defer func() { - if c.container == nil { - // Something has gone wrong. - container.Terminate(ctx) - } - }() + c.container = container + if err := c.container.Start(ctx); err != nil { + return err + } ip, err := container.Host(ctx) if err != nil { return err @@ -236,6 +234,9 @@ func (c *ElasticsearchContainer) Start() error { // Close terminates and removes the container. func (c *ElasticsearchContainer) Close() error { + if c.container == nil { + return nil + } return c.container.Terminate(context.Background()) } @@ -374,18 +375,15 @@ func (c *ElasticAgentContainer) Start() error { container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: c.request, - Started: true, }) if err != nil { return err } - defer func() { - if c.container == nil { - // Something has gone wrong. - container.Terminate(ctx) - } - }() + c.container = container + if err := container.Start(ctx); err != nil { + return err + } ports, err := container.Ports(ctx) if err != nil { return err @@ -408,9 +406,22 @@ func (c *ElasticAgentContainer) Start() error { // Close terminates and removes the container. func (c *ElasticAgentContainer) Close() error { + if c.container == nil { + return nil + } return c.container.Terminate(context.Background()) } +// Logs returns an io.ReadCloser that can be used for reading the +// container's combined stdout/stderr log. If the container has not +// been created by Start(), Logs will return an error. +func (c *ElasticAgentContainer) Logs(ctx context.Context) (io.ReadCloser, error) { + if c.container == nil { + return nil, errors.New("container not created") + } + return c.container.Logs(ctx) +} + func pullDockerImage(ctx context.Context, docker *client.Client, imageRef string) error { rc, err := docker.ImagePull(context.Background(), imageRef, types.ImagePullOptions{}) if err != nil { diff --git a/systemtest/fleet_test.go b/systemtest/fleet_test.go index 529a177f53e..08885a8db86 100644 --- a/systemtest/fleet_test.go +++ b/systemtest/fleet_test.go @@ -18,7 +18,9 @@ package systemtest_test import ( + "context" "fmt" + "io/ioutil" "net/url" "path" "path/filepath" @@ -89,6 +91,20 @@ func TestFleetIntegration(t *testing.T) { agent, err := systemtest.NewUnstartedElasticAgentContainer() require.NoError(t, err) agent.FleetEnrollmentToken = enrollmentAPIKey.APIKey + defer agent.Close() + + defer func() { + // Log the elastic-agent container output if the test fails. + if !t.Failed() { + return + } + if logs, err := agent.Logs(context.Background()); err == nil { + defer logs.Close() + if out, err := ioutil.ReadAll(logs); err == nil { + t.Logf("elastic-agent logs: %s", out) + } + } + }() // Build apm-server, and bind-mount it into the elastic-agent container's "install" // directory. This bypasses downloading the artifact. @@ -115,8 +131,6 @@ func TestFleetIntegration(t *testing.T) { waitFor.Port = "8200/tcp" agent.WaitingFor = waitFor err = agent.Start() - require.NoError(t, err) - defer agent.Close() // Elastic Agent has started apm-server. Connect to apm-server and send some data, // and make sure it gets indexed into a data stream. From 638dddd0c4ef4d11c3bb82e338a83e0a7b920ac9 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Wed, 3 Feb 2021 12:57:19 +0800 Subject: [PATCH 11/11] systemtest: delete apm templates on cleanup --- systemtest/datastreams_test.go | 2 +- systemtest/elasticsearch.go | 41 +++++++++++++++++++--------------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/systemtest/datastreams_test.go b/systemtest/datastreams_test.go index 9ece776264b..35e261567cf 100644 --- a/systemtest/datastreams_test.go +++ b/systemtest/datastreams_test.go @@ -46,7 +46,7 @@ func TestDataStreamsEnabled(t *testing.T) { // Create a data stream index template. resp, err := systemtest.Elasticsearch.Indices.PutIndexTemplate("apm-data-streams", strings.NewReader(fmt.Sprintf(`{ - "index_patterns": ["traces-*", "logs-*", "metrics-*"], + "index_patterns": ["traces-apm*", "logs-apm*", "metrics-apm*"], "data_stream": {}, "priority": 200, "template": {"settings": {"number_of_shards": 1, "refresh_interval": "250ms"}} diff --git a/systemtest/elasticsearch.go b/systemtest/elasticsearch.go index 88c522e6f57..161765e517d 100644 --- a/systemtest/elasticsearch.go +++ b/systemtest/elasticsearch.go @@ -96,14 +96,6 @@ func CleanupElasticsearch(t testing.TB) { apmMetricsPrefix = "metrics-apm*" apmLogsPrefix = "logs-apm*" ) - requests := []estest.Request{ - esapi.IndicesDeleteRequest{Index: []string{legacyPrefix}}, - esapi.IndicesDeleteDataStreamRequest{Name: apmTracesPrefix}, - esapi.IndicesDeleteDataStreamRequest{Name: apmMetricsPrefix}, - esapi.IndicesDeleteDataStreamRequest{Name: apmLogsPrefix}, - esapi.IngestDeletePipelineRequest{PipelineID: legacyPrefix}, - esapi.IndicesDeleteTemplateRequest{Name: legacyPrefix}, - } doReq := func(req estest.Request) error { _, err := Elasticsearch.Do(context.Background(), req, nil) @@ -113,19 +105,32 @@ func CleanupElasticsearch(t testing.TB) { return err } - var g errgroup.Group - for _, req := range requests { - req := req // copy for closure - g.Go(func() error { return doReq(req) }) - } - if err := g.Wait(); err != nil { - t.Fatal(err) + doParallel := func(requests ...estest.Request) { + var g errgroup.Group + for _, req := range requests { + req := req // copy for closure + g.Go(func() error { return doReq(req) }) + } + if err := g.Wait(); err != nil { + t.Fatal(err) + } } + doParallel( + esapi.IndicesDeleteRequest{Index: []string{legacyPrefix}}, + esapi.IndicesDeleteDataStreamRequest{Name: apmTracesPrefix}, + esapi.IndicesDeleteDataStreamRequest{Name: apmMetricsPrefix}, + esapi.IndicesDeleteDataStreamRequest{Name: apmLogsPrefix}, + esapi.IngestDeletePipelineRequest{PipelineID: legacyPrefix}, + esapi.IndicesDeleteTemplateRequest{Name: legacyPrefix}, + ) // Delete index templates after deleting data streams. - if err := doReq(esapi.IndicesDeleteIndexTemplateRequest{Name: legacyPrefix}); err != nil { - t.Fatal(err) - } + doParallel( + esapi.IndicesDeleteIndexTemplateRequest{Name: legacyPrefix}, // for index template created by tests + esapi.IndicesDeleteIndexTemplateRequest{Name: apmTracesPrefix}, + esapi.IndicesDeleteIndexTemplateRequest{Name: apmMetricsPrefix}, + esapi.IndicesDeleteIndexTemplateRequest{Name: apmLogsPrefix}, + ) // Refresh indices to ensure all recent changes are visible. if err := doReq(esapi.IndicesRefreshRequest{}); err != nil {