From 41b1fcbcfb75a26dde8723453df0eb723147e835 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Wed, 3 Feb 2021 13:47:36 +0800 Subject: [PATCH] systemtest: test apm-server in Fleet mode (#4650) * systemtest: test apm-server in Fleet mode We run a custom package registry into which the "apm" integration package is bind-mounted from the tree, enabling us to test with unreleased package changes. Add a system test that runs Elastic Agent in Fleet mode, with a locally built APM Server binary injected into the container. The test adds the "apm" integration package to a new agent policy, waits for elastic-agent to start apm-server, and verifies it is functional. # Conflicts: # systemtest/elasticsearch.go --- docker-compose.yml | 17 ++ go.sum | 9 + systemtest/apmservertest/command.go | 36 ++- systemtest/apmservertest/config.go | 8 +- systemtest/containers.go | 211 +++++++++++++- systemtest/datastreams_test.go | 2 +- systemtest/elasticsearch.go | 5 - systemtest/fleet_test.go | 174 +++++++++++ systemtest/fleettest/client.go | 322 +++++++++++++++++++++ systemtest/fleettest/types.go | 99 +++++++ systemtest/kibana.go | 44 +++ testing/docker/package-registry/config.yml | 9 + 12 files changed, 910 insertions(+), 26 deletions(-) create mode 100644 systemtest/fleet_test.go create mode 100644 systemtest/fleettest/client.go create mode 100644 systemtest/fleettest/types.go create mode 100644 systemtest/kibana.go create mode 100644 testing/docker/package-registry/config.yml diff --git a/docker-compose.yml b/docker-compose.yml index 0deb5d7e0d6..364f586ab07 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,3 +64,20 @@ services: ELASTICSEARCH_URL: elasticsearch:9200 ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-kibana_system_user}" ELASTICSEARCH_PASSWORD: "${KIBANA_ES_PASS:-changeme}" + XPACK_XPACK_MAIN_TELEMETRY_ENABLED: "false" + XPACK_SECURITY_ENCRYPTIONKEY: "fhjskloppd678ehkdfdlliverpoolfcr" + XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: "fhjskloppd678ehkdfdlliverpoolfcr" + XPACK_FLEET_AGENTS_ELASTICSEARCH_HOST: "http://elasticsearch:9200" + XPACK_FLEET_AGENTS_KIBANA_HOST: "http://kibana:5601" + XPACK_FLEET_AGENTS_TLSCHECKDISABLED: "true" + XPACK_FLEET_REGISTRYURL: "http://package-registry:8080" + depends_on: + package-registry: { condition: service_healthy } + + package-registry: + image: docker.elastic.co/package-registry/distribution:snapshot + ports: + - 8080 + volumes: + - "./testing/docker/package-registry/config.yml:/package-registry/config.yml" + - "./apmpackage/apm:/packages/local/apm" diff --git a/go.sum b/go.sum index 584f58a6499..822bb694d1b 100644 --- a/go.sum +++ b/go.sum @@ -150,6 +150,7 @@ github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43/go.mod h1:tJPYQG4mn github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antlr/antlr4 v0.0.0-20200820155224-be881fa6b91d h1:OE3kzLBpy7pOJEzE55j9sdgrSilUPzzj++FWvp1cmIs= github.com/antlr/antlr4 v0.0.0-20200820155224-be881fa6b91d/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y= github.com/antonmedv/expr v1.8.9 h1:O9stiHmHHww9b4ozhPx7T6BK7fXfOCHJ8ybxf0833zw= github.com/antonmedv/expr v1.8.9/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8= @@ -258,7 +259,9 @@ github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHo github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.0.0 h1:XJIw/+VlJ+87J+doOxznsAWIdmWuViOVhkQamW5YV28= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -523,7 +526,9 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gocarina/gocsv v0.0.0-20170324095351-ffef3ffc77be/go.mod h1:/oj50ZdPq/cUjA02lMZhijk5kR31SEydKyqah1OgBuo= github.com/gocql/gocql v0.0.0-20200228163523-cd4b606dd2fb/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e h1:BWhy2j3IXJhjCbC68FptL43tDKIq8FladmaTs3Xs7Z8= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godror/godror v0.10.4/go.mod h1:9MVLtu25FBJBMHkPs0m3Ngf/VmwGcLpM2HS8PlNGw9U= github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= @@ -737,6 +742,7 @@ github.com/hashicorp/yamux v0.0.0-20190923154419-df201c70410d/go.mod h1:+NfK9FKe github.com/haya14busa/go-actions-toolkit v0.0.0-20200105081403-ca0307860f01/go.mod h1:1DWDZmeYf0LX30zscWb7K9rUMeirNeBMd5Dum+seUhc= github.com/haya14busa/go-checkstyle v0.0.0-20170303121022-5e9d09f51fa1/go.mod h1:RsN5RGgVYeXpcXNtWyztD5VIe7VNSEqpJvF2iEH7QvI= github.com/haya14busa/secretbox v0.0.0-20180525171038-07c7ecf409f5/go.mod h1:FGO/dXIFZnan7KvvUSFk1hYMnoVNzB6NTMPrmke8SSI= +github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/hetznercloud/hcloud-go v1.22.0/go.mod h1:xng8lbDUg+xM1dgc0yGHX5EeqbwIq7UYlMWMTx3SQVg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -801,6 +807,7 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/justinas/nosurf v1.1.0/go.mod h1:ALpWdSbuNGy2lZWtyXdjkYv4edL23oSEgfBT1gPJ5BQ= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= +github.com/kardianos/service v1.1.0 h1:QV2SiEeWK42P0aEmGcsAgjApw/lRxkwopvT+Gu6t1/0= github.com/kardianos/service v1.1.0/go.mod h1:RrJI2xn5vve/r32U5suTbeaSGoMU6GbNPoj36CVYcHc= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= @@ -945,6 +952,7 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olivere/elastic v6.2.27+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= @@ -999,6 +1007,7 @@ github.com/openzipkin/zipkin-go v0.2.5 h1:UwtQQx2pyPIgWYHRg+epgdx1/HnBQTgN3/oIYE github.com/openzipkin/zipkin-go v0.2.5/go.mod h1:KpXfKdgRDnnhsxw4pNIH9Md5lyFqKUa4YDFlwRYAMyE= github.com/orijtech/prometheus-go-metrics-exporter v0.0.6 h1:ExkpQsyDDcyp0U3zhoNUQaCQ/o0Ovq7e1jRCL9lQ/4o= github.com/orijtech/prometheus-go-metrics-exporter v0.0.6/go.mod h1:BiTx/ugZex8LheBk3j53tktWaRdFjV5FCfT2o0P7msE= +github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= diff --git a/systemtest/apmservertest/command.go b/systemtest/apmservertest/command.go index fc554bd8a31..aa042486bb2 100644 --- a/systemtest/apmservertest/command.go +++ b/systemtest/apmservertest/command.go @@ -33,7 +33,7 @@ import ( // ServerCommand returns a ServerCmd (wrapping os/exec) for running // apm-server with args. func ServerCommand(subcommand string, args ...string) *ServerCmd { - binary, buildErr := buildServer() + binary, buildErr := BuildServerBinary(runtime.GOOS) if buildErr != nil { // Dummy command; Start etc. will return the build error. binary = "/usr/bin/false" @@ -149,25 +149,37 @@ func (c *ServerCmd) cleanup() { } } -// buildServer builds the apm-server binary, returning its absolute path. -func buildServer() (string, error) { +// BuildServerBinary builds the apm-server binary for the given GOOS, +// returning its absolute path. +func BuildServerBinary(goos string) (string, error) { + // Build apm-server binary in the repo root, unless + // we're building for another GOOS, in which case we + // suffix the binary with that GOOS and place it in + // the build directory. + var reldir, suffix string + if goos != runtime.GOOS { + reldir = "build/" + suffix = "-" + goos + if runtime.GOOS == "windows" { + suffix += ".exe" + } + } + apmServerBinaryMu.Lock() defer apmServerBinaryMu.Unlock() - if apmServerBinary != "" { - return apmServerBinary, nil + if binary := apmServerBinary[goos]; binary != "" { + return binary, nil } repoRoot, err := getRepoRoot() if err != nil { return "", err } - abspath := filepath.Join(repoRoot, "apm-server") - if runtime.GOOS == "windows" { - abspath += ".exe" - } + abspath := filepath.Join(repoRoot, reldir, "apm-server"+suffix) log.Println("Building apm-server...") cmd := exec.Command("go", "build", "-o", abspath, "./x-pack/apm-server") + cmd.Env = append(os.Environ(), "GOOS="+goos) cmd.Dir = repoRoot cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -175,8 +187,8 @@ func buildServer() (string, error) { return "", err } log.Println("Built", abspath) - apmServerBinary = abspath - return apmServerBinary, nil + apmServerBinary[goos] = abspath + return abspath, nil } func getRepoRoot() (string, error) { @@ -197,7 +209,7 @@ func getRepoRoot() (string, error) { var ( apmServerBinaryMu sync.Mutex - apmServerBinary string + apmServerBinary = make(map[string]string) repoRootMu sync.Mutex repoRoot string diff --git a/systemtest/apmservertest/config.go b/systemtest/apmservertest/config.go index 9f5e63f8bd7..e5d496fc5c0 100644 --- a/systemtest/apmservertest/config.go +++ b/systemtest/apmservertest/config.go @@ -405,7 +405,7 @@ func DefaultConfig() Config { Scheme: "http", Host: net.JoinHostPort( getenvDefault("KIBANA_HOST", defaultKibanaHost), - getenvDefault("KIBANA_PORT", defaultKibanaPort), + KibanaPort(), ), }).String(), Username: getenvDefault("KIBANA_USER", defaultKibanaUser), @@ -436,6 +436,12 @@ func DefaultConfig() Config { } } +// KibanaPort returns the Kibana port, configured using +// KIBANA_PORT, or otherwise returning the default of 5601. +func KibanaPort() string { + return getenvDefault("KIBANA_PORT", defaultKibanaPort) +} + func getenvDefault(k, defaultv string) string { v := os.Getenv(k) if v == "" { diff --git a/systemtest/containers.go b/systemtest/containers.go index edb361edf86..8a3f9b7035d 100644 --- a/systemtest/containers.go +++ b/systemtest/containers.go @@ -19,12 +19,16 @@ package systemtest import ( "context" + "errors" "fmt" + "io" + "io/ioutil" "log" "net" "net/url" "os" "os/exec" + "path" "strings" "time" @@ -34,6 +38,7 @@ import ( "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" + "github.com/elastic/apm-server/systemtest/apmservertest" "github.com/elastic/apm-server/systemtest/estest" "github.com/elastic/go-elasticsearch/v7" ) @@ -195,18 +200,15 @@ func (c *ElasticsearchContainer) Start() error { container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: c.request, - Started: true, }) if err != nil { return err } - defer func() { - if c.container == nil { - // Something has gone wrong. - container.Terminate(ctx) - } - }() + c.container = container + if err := c.container.Start(ctx); err != nil { + return err + } ip, err := container.Host(ctx) if err != nil { return err @@ -232,5 +234,200 @@ func (c *ElasticsearchContainer) Start() error { // Close terminates and removes the container. func (c *ElasticsearchContainer) Close() error { + if c.container == nil { + return nil + } + return c.container.Terminate(context.Background()) +} + +// NewUnstartedElasticAgentContainer returns a new ElasticAgentContainer. +func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) { + // Create a testcontainer.ContainerRequest to run Elastic Agent. + // We pull some configuration from the Kibana docker-compose service, + // such as the Docker network to use. + + docker, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + return nil, err + } + defer docker.Close() + + kibanaContainer, err := stackContainerInfo(context.Background(), docker, "kibana") + if err != nil { + return nil, err + } + kibanaContainerDetails, err := docker.ContainerInspect(context.Background(), kibanaContainer.ID) + if err != nil { + return nil, err + } + + var kibanaIPAddress string + var networks []string + for network, settings := range kibanaContainerDetails.NetworkSettings.Networks { + networks = append(networks, network) + if kibanaIPAddress == "" && settings.IPAddress != "" { + kibanaIPAddress = settings.IPAddress + } + } + kibanaURL := &url.URL{ + Scheme: "http", + Host: net.JoinHostPort(kibanaIPAddress, apmservertest.KibanaPort()), + } + + // Use the same stack version as used for Kibana. + agentImageVersion := kibanaContainer.Image[strings.LastIndex(kibanaContainer.Image, ":")+1:] + agentImage := "docker.elastic.co/beats/elastic-agent:" + agentImageVersion + if err := pullDockerImage(context.Background(), docker, agentImage); err != nil { + return nil, err + } + agentImageDetails, _, err := docker.ImageInspectWithRaw(context.Background(), agentImage) + if err != nil { + return nil, err + } + agentVCSRef := agentImageDetails.Config.Labels["org.label-schema.vcs-ref"] + agentDataHashDir := path.Join("/usr/share/elastic-agent/data", "elastic-agent-"+agentVCSRef[:6]) + agentInstallDir := path.Join(agentDataHashDir, "install") + + req := testcontainers.ContainerRequest{ + Image: agentImage, + AutoRemove: true, + Networks: networks, + Env: map[string]string{ + "KIBANA_HOST": kibanaURL.String(), + + // TODO(axw) remove once https://github.com/elastic/elastic-agent-client/issues/20 is fixed + "GODEBUG": "x509ignoreCN=0", + + // NOTE(axw) because we bind-mount the apm-server artifacts in, they end up owned by the + // current user rather than root. Disable Beats's strict permission checks to avoid resulting + // complaints, as they're irrelevant to these system tests. + "BEAT_STRICT_PERMS": "false", + }, + } + return &ElasticAgentContainer{ + request: req, + installDir: agentInstallDir, + StackVersion: agentImageVersion, + BindMountInstall: make(map[string]string), + }, nil +} + +// ElasticAgentContainer represents an ephemeral Elastic Agent container. +type ElasticAgentContainer struct { + container testcontainers.Container + request testcontainers.ContainerRequest + + // installDir holds the location of the "install" directory inside + // the Elastic Agent container. + // + // This will be set when the ElasticAgentContainer object is created, + // and can be used to anticipate the location into which artifacts + // can be bind-mounted. + installDir string + + // StackVersion holds the stack version of the container image, + // e.g. 8.0.0-SNAPSHOT. + StackVersion string + + // ExposedPorts holds an optional list of ports to expose to the host. + ExposedPorts []string + + // WaitingFor holds an optional wait strategy. + WaitingFor wait.Strategy + + // Addrs holds the "host:port" address for each exposed port. + // This will be populated by Start. + Addrs []string + + // BindMountInstall holds a map of files to bind mount into the + // container, mapping from the host location to target paths relative + // to the install directory in the container. + BindMountInstall map[string]string + + // FleetEnrollmentToken holds an optional Fleet enrollment token to + // use for enrolling the agent with Fleet. The agent will only enroll + // if this is specified. + FleetEnrollmentToken string +} + +// Start starts the container. +// +// The Addr and Client fields will be updated on successful return. +// +// The container will be removed when Close() is called, or otherwise by a +// reaper process if the test process is aborted. +func (c *ElasticAgentContainer) Start() error { + ctx, cancel := context.WithTimeout(context.Background(), startContainersTimeout) + defer cancel() + + // Update request from user-definable fields. + if c.FleetEnrollmentToken != "" { + c.request.Env["FLEET_ENROLL"] = "1" + c.request.Env["FLEET_ENROLL_INSECURE"] = "1" + c.request.Env["FLEET_ENROLLMENT_TOKEN"] = c.FleetEnrollmentToken + } + c.request.ExposedPorts = c.ExposedPorts + c.request.WaitingFor = c.WaitingFor + c.request.BindMounts = map[string]string{} + for source, target := range c.BindMountInstall { + c.request.BindMounts[source] = path.Join(c.installDir, target) + } + + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: c.request, + }) + if err != nil { + return err + } + c.container = container + + if err := container.Start(ctx); err != nil { + return err + } + ports, err := container.Ports(ctx) + if err != nil { + return err + } + if len(ports) > 0 { + ip, err := container.Host(ctx) + if err != nil { + return err + } + for _, portbindings := range ports { + for _, pb := range portbindings { + c.Addrs = append(c.Addrs, net.JoinHostPort(ip, pb.HostPort)) + } + } + } + + c.container = container + return nil +} + +// Close terminates and removes the container. +func (c *ElasticAgentContainer) Close() error { + if c.container == nil { + return nil + } return c.container.Terminate(context.Background()) } + +// Logs returns an io.ReadCloser that can be used for reading the +// container's combined stdout/stderr log. If the container has not +// been created by Start(), Logs will return an error. +func (c *ElasticAgentContainer) Logs(ctx context.Context) (io.ReadCloser, error) { + if c.container == nil { + return nil, errors.New("container not created") + } + return c.container.Logs(ctx) +} + +func pullDockerImage(ctx context.Context, docker *client.Client, imageRef string) error { + rc, err := docker.ImagePull(context.Background(), imageRef, types.ImagePullOptions{}) + if err != nil { + return err + } + defer rc.Close() + _, err = io.Copy(ioutil.Discard, rc) + return err +} diff --git a/systemtest/datastreams_test.go b/systemtest/datastreams_test.go index 9ece776264b..35e261567cf 100644 --- a/systemtest/datastreams_test.go +++ b/systemtest/datastreams_test.go @@ -46,7 +46,7 @@ func TestDataStreamsEnabled(t *testing.T) { // Create a data stream index template. resp, err := systemtest.Elasticsearch.Indices.PutIndexTemplate("apm-data-streams", strings.NewReader(fmt.Sprintf(`{ - "index_patterns": ["traces-*", "logs-*", "metrics-*"], + "index_patterns": ["traces-apm*", "logs-apm*", "metrics-apm*"], "data_stream": {}, "priority": 200, "template": {"settings": {"number_of_shards": 1, "refresh_interval": "250ms"}} diff --git a/systemtest/elasticsearch.go b/systemtest/elasticsearch.go index c3aae25b081..a7122545a24 100644 --- a/systemtest/elasticsearch.go +++ b/systemtest/elasticsearch.go @@ -145,11 +145,6 @@ func CleanupElasticsearch(t testing.TB) { t.Fatal(err) } - // Refresh indices to ensure all recent changes are visible. - if err := doReq(esapi.IndicesRefreshRequest{}); err != nil { - t.Fatal(err) - } - // Delete the ILM policy last or we'll get an error due to it being in use. for { err := doReq(esapi.ILMDeleteLifecycleRequest{Policy: "apm-rollover-30-days"}) diff --git a/systemtest/fleet_test.go b/systemtest/fleet_test.go new file mode 100644 index 00000000000..08885a8db86 --- /dev/null +++ b/systemtest/fleet_test.go @@ -0,0 +1,174 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package systemtest_test + +import ( + "context" + "fmt" + "io/ioutil" + "net/url" + "path" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go/wait" + "go.elastic.co/apm" + "go.elastic.co/apm/transport" + + "github.com/elastic/apm-server/systemtest" + "github.com/elastic/apm-server/systemtest/apmservertest" + "github.com/elastic/apm-server/systemtest/fleettest" +) + +func TestFleetIntegration(t *testing.T) { + systemtest.CleanupElasticsearch(t) + + fleet := fleettest.NewClient(systemtest.KibanaURL.String()) + require.NoError(t, fleet.Setup()) + cleanupFleet(t, fleet) + defer cleanupFleet(t, fleet) + + agentPolicy, enrollmentAPIKey, err := fleet.CreateAgentPolicy("apm_systemtest", "default", "Agent policy for APM Server system tests") + require.NoError(t, err) + + // Find the "apm" package to install. + var apmPackage *fleettest.Package + packages, err := fleet.ListPackages() + require.NoError(t, err) + for _, pkg := range packages { + if pkg.Name == "apm" { + apmPackage = &pkg + break + } + } + require.NotNil(t, apmPackage) + + // Add the "apm" integration to the agent policy. + packagePolicy := fleettest.PackagePolicy{ + Name: "apm", + Namespace: "default", + Enabled: true, + AgentPolicyID: agentPolicy.ID, + } + packagePolicy.Package.Name = apmPackage.Name + packagePolicy.Package.Version = apmPackage.Version + packagePolicy.Package.Title = apmPackage.Title + packagePolicy.Inputs = []fleettest.PackagePolicyInput{{ + Type: "apm", + Enabled: true, + Streams: []interface{}{}, + Vars: map[string]interface{}{ + "enable_rum": map[string]interface{}{ + "type": "bool", + "value": false, + }, + "host": map[string]interface{}{ + "type": "string", + "value": ":8200", + }, + }, + }} + err = fleet.CreatePackagePolicy(packagePolicy) + require.NoError(t, err) + + agent, err := systemtest.NewUnstartedElasticAgentContainer() + require.NoError(t, err) + agent.FleetEnrollmentToken = enrollmentAPIKey.APIKey + defer agent.Close() + + defer func() { + // Log the elastic-agent container output if the test fails. + if !t.Failed() { + return + } + if logs, err := agent.Logs(context.Background()); err == nil { + defer logs.Close() + if out, err := ioutil.ReadAll(logs); err == nil { + t.Logf("elastic-agent logs: %s", out) + } + } + }() + + // Build apm-server, and bind-mount it into the elastic-agent container's "install" + // directory. This bypasses downloading the artifact. + arch := runtime.GOARCH + if arch == "amd64" { + arch = "x86_64" + } + apmServerArtifactName := fmt.Sprintf("apm-server-%s-linux-%s", agent.StackVersion, arch) + + // Bind-mount the apm-server binary and apm-server.yml into the container's + // "install" directory. This causes elastic-agent to skip installing the + // artifact. + apmServerBinary, err := apmservertest.BuildServerBinary("linux") + require.NoError(t, err) + agent.BindMountInstall[apmServerBinary] = path.Join(apmServerArtifactName, "apm-server") + apmServerConfigFile, err := filepath.Abs("../apm-server.yml") + require.NoError(t, err) + agent.BindMountInstall[apmServerConfigFile] = path.Join(apmServerArtifactName, "apm-server.yml") + + // Start elastic-agent with port 8200 exposed, and wait for the server to service + // healthcheck requests to port 8200. + agent.ExposedPorts = []string{"8200"} + waitFor := wait.ForHTTP("/") + waitFor.Port = "8200/tcp" + agent.WaitingFor = waitFor + err = agent.Start() + + // Elastic Agent has started apm-server. Connect to apm-server and send some data, + // and make sure it gets indexed into a data stream. + require.Len(t, agent.Addrs, 1) + transport, err := transport.NewHTTPTransport() + require.NoError(t, err) + transport.SetServerURL(&url.URL{Scheme: "http", Host: agent.Addrs[0]}) + tracer, err := apm.NewTracerOptions(apm.TracerOptions{Transport: transport}) + require.NoError(t, err) + defer tracer.Close() + tracer.StartTransaction("name", "type").End() + tracer.Flush(nil) + + systemtest.Elasticsearch.ExpectDocs(t, "traces-*", nil) +} + +func cleanupFleet(t testing.TB, fleet *fleettest.Client) { + apmAgentPolicies, err := fleet.AgentPolicies("ingest-agent-policies.name:apm_systemtest") + require.NoError(t, err) + if len(apmAgentPolicies) == 0 { + return + } + + agents, err := fleet.Agents() + require.NoError(t, err) + agentsByPolicy := make(map[string][]fleettest.Agent) + for _, agent := range agents { + agentsByPolicy[agent.PolicyID] = append(agentsByPolicy[agent.PolicyID], agent) + } + + for _, p := range apmAgentPolicies { + if agents := agentsByPolicy[p.ID]; len(agents) > 0 { + agentIDs := make([]string, len(agents)) + for i, agent := range agents { + agentIDs[i] = agent.ID + } + require.NoError(t, fleet.BulkUnenrollAgents(true, agentIDs...)) + } + require.NoError(t, fleet.DeleteAgentPolicy(p.ID)) + } +} diff --git a/systemtest/fleettest/client.go b/systemtest/fleettest/client.go new file mode 100644 index 00000000000..a20891ae998 --- /dev/null +++ b/systemtest/fleettest/client.go @@ -0,0 +1,322 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fleettest + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" +) + +// Client provides methods for interacting with the Fleet API. +type Client struct { + fleetURL string +} + +// NewClient returns a new Client for interacting with the Fleet API, +// using the given Kibana URL. +func NewClient(kibanaURL string) *Client { + return &Client{fleetURL: kibanaURL + "/api/fleet"} +} + +// Setup invokes the Fleet Setup API, returning an error if it fails. +func (c *Client) Setup() error { + for _, path := range []string{"/setup", "/agents/setup"} { + req := c.newFleetRequest("POST", path, nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + } + return nil +} + +// Agents returns the list of enrolled agents. +func (c *Client) Agents() ([]Agent, error) { + resp, err := http.Get(c.fleetURL + "/agents") + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return nil, fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + var result struct { + List []Agent `json:"list"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + return result.List, nil +} + +// BulkUnenrollAgents bulk-unenrolls agents. +func (c *Client) BulkUnenrollAgents(force bool, agentIDs ...string) error { + var body bytes.Buffer + type bulkUnenroll struct { + Agents []string `json:"agents"` + Force bool `json:"force"` + } + if err := json.NewEncoder(&body).Encode(bulkUnenroll{agentIDs, force}); err != nil { + return err + } + req := c.newFleetRequest("POST", "/agents/bulk_unenroll", &body) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + return nil +} + +// AgentPolicies returns the Agent Policies matching the given KQL query. +func (c *Client) AgentPolicies(kuery string) ([]AgentPolicy, error) { + u, err := url.Parse(c.fleetURL + "/agent_policies") + if err != nil { + return nil, err + } + query := u.Query() + query.Add("kuery", kuery) + u.RawQuery = query.Encode() + resp, err := http.Get(u.String()) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return nil, fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + var result struct { + Items []AgentPolicy `json:"items"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + return result.Items, nil +} + +// DeleteAgentPolicy deletes the Agent Policy with the given ID. +func (c *Client) DeleteAgentPolicy(id string) error { + var body bytes.Buffer + type deleteAgentPolicy struct { + ID string `json:"agentPolicyId"` + } + if err := json.NewEncoder(&body).Encode(deleteAgentPolicy{id}); err != nil { + return err + } + req := c.newFleetRequest("POST", "/agent_policies/delete", &body) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + return nil +} + +// CreateAgentPolicy returns the default Agent Policy. +func (c *Client) CreateAgentPolicy(name, namespace, description string) (*AgentPolicy, *EnrollmentAPIKey, error) { + var body bytes.Buffer + type newAgentPolicy struct { + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` + Description string `json:"description,omitempty"` + } + if err := json.NewEncoder(&body).Encode(newAgentPolicy{name, namespace, description}); err != nil { + return nil, nil, err + } + req, err := http.NewRequest("POST", c.fleetURL+"/agent_policies", &body) + if err != nil { + return nil, nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("kbn-xsrf", "1") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return nil, nil, fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + var result struct { + Item AgentPolicy `json:"item"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, nil, err + } + enrollmentAPIKey, err := c.getAgentPolicyEnrollmentAPIKey(result.Item.ID) + if err != nil { + return nil, nil, err + } + return &result.Item, enrollmentAPIKey, nil +} + +func (c *Client) getAgentPolicyEnrollmentAPIKey(policyID string) (*EnrollmentAPIKey, error) { + keys, err := c.enrollmentAPIKeys("fleet-enrollment-api-keys.policy_id:" + policyID) + if err != nil { + return nil, err + } + if n := len(keys); n != 1 { + return nil, fmt.Errorf("expected 1 enrollment API key, got %d", n) + } + resp, err := http.Get(c.fleetURL + "/enrollment-api-keys/" + keys[0].ID) + if err != nil { + return nil, err + } + var result struct { + Item EnrollmentAPIKey `json:"item"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + return &result.Item, nil +} + +func (c *Client) enrollmentAPIKeys(kuery string) ([]EnrollmentAPIKey, error) { + u, err := url.Parse(c.fleetURL + "/enrollment-api-keys") + if err != nil { + return nil, err + } + query := u.Query() + query.Add("kuery", kuery) + u.RawQuery = query.Encode() + resp, err := http.Get(u.String()) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return nil, fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + var result struct { + Items []EnrollmentAPIKey `json:"list"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + return result.Items, nil +} + +// ListPackages lists all packages available for installation. +func (c *Client) ListPackages() ([]Package, error) { + resp, err := http.Get(c.fleetURL + "/epm/packages?experimental=true") + if err != nil { + return nil, err + } + defer resp.Body.Close() + var result struct { + Response []Package `json:"response"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + return result.Response, nil +} + +// PackagePolicy returns information about the package policy with the given ID. +func (c *Client) PackagePolicy(id string) (*PackagePolicy, error) { + resp, err := http.Get(c.fleetURL + "/package_policies/" + id) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return nil, fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + var result struct { + Item PackagePolicy `json:"item"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + return &result.Item, nil +} + +// CreatePackagePolicy adds an integration to a policy. +func (c *Client) CreatePackagePolicy(p PackagePolicy) error { + var body bytes.Buffer + if err := json.NewEncoder(&body).Encode(&p); err != nil { + return err + } + req := c.newFleetRequest("POST", "/package_policies", &body) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + return nil +} + +// DeletePackagePolicy deletes one or more package policies. +func (c *Client) DeletePackagePolicy(ids ...string) error { + var params struct { + PackagePolicyIDs []string `json:"packagePolicyIds"` + } + params.PackagePolicyIDs = ids + var body bytes.Buffer + if err := json.NewEncoder(&body).Encode(params); err != nil { + return err + } + req := c.newFleetRequest("POST", "/package_policies/delete", &body) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("request failed (%s): %s", resp.Status, body) + } + return nil +} + +func (c *Client) newFleetRequest(method string, path string, body io.Reader) *http.Request { + req, err := http.NewRequest(method, c.fleetURL+path, body) + if err != nil { + panic(err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("kbn-xsrf", "1") + return req +} diff --git a/systemtest/fleettest/types.go b/systemtest/fleettest/types.go new file mode 100644 index 00000000000..0c5fa57fd95 --- /dev/null +++ b/systemtest/fleettest/types.go @@ -0,0 +1,99 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fleettest + +import "time" + +// Agent holds details of a Fleet Agent. +type Agent struct { + ID string `json:"id"` + Active bool `json:"active"` + Status string `json:"status"` + Type string `json:"type"` + PolicyID string `json:"policy_id,omitempty"` + EnrolledAt time.Time `json:"enrolled_at,omitempty"` + UserProvidedMetadata map[string]interface{} `json:"user_provided_metadata,omitempty"` + LocalMetadata map[string]interface{} `json:"local_metadata,omitempty"` +} + +// AgentPolicy holds details of a Fleet Agent Policy. +type AgentPolicy struct { + ID string `json:"id"` + Name string `json:"name"` + Namespace string `json:"namespace"` + Description string `json:"description"` + Revision int `json:"revision"` + + Agents int `json:"agents"` + IsDefault bool `json:"is_default"` + MonitoringEnabled []string `json:"monitoring_enabled"` + PackagePolicies []string `json:"package_policies"` + Status string `json:"status"` + UpdatedAt time.Time `json:"updated_at"` + UpdatedBy string `json:"updated_by"` +} + +// PackagePolicy holds details of a Fleet Package Policy. +type PackagePolicy struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + Namespace string `json:"namespace"` + Enabled bool `json:"enabled"` + Description string `json:"description"` + AgentPolicyID string `json:"policy_id"` + OutputID string `json:"output_id"` + Inputs []PackagePolicyInput `json:"inputs"` + Package struct { + Name string `json:"name"` + Version string `json:"version"` + Title string `json:"title"` + } `json:"package"` +} + +type PackagePolicyInput struct { + Type string `json:"type"` + Enabled bool `json:"enabled"` + Streams []interface{} `json:"streams"` + Config map[string]interface{} `json:"config,omitempty"` + Vars map[string]interface{} `json:"vars,omitempty"` +} + +type Package struct { + Name string `json:"name"` + Version string `json:"version"` + Release string `json:"release"` + Type string `json:"type"` + Title string `json:"title"` + Description string `json:"description"` + Download string `json:"download"` + Path string `json:"path"` + Status string `json:"status"` +} + +type EnrollmentAPIKey struct { + ID string `json:"id"` + Active bool `json:"active"` + APIKeyID string `json:"api_key_id"` + Name string `json:"name"` + PolicyID string `json:"policy_id"` + CreatedAt time.Time `json:"created_at"` + + // APIKey is only returned when querying a specific enrollment API key, + // and not when listing keys. + APIKey string `json:"api_key,omitempty"` +} diff --git a/systemtest/kibana.go b/systemtest/kibana.go new file mode 100644 index 00000000000..64d61a30a89 --- /dev/null +++ b/systemtest/kibana.go @@ -0,0 +1,44 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package systemtest + +import ( + "log" + "net/url" + + "github.com/elastic/apm-server/systemtest/apmservertest" +) + +const ( + adminKibanaUser = adminElasticsearchUser + adminKibanaPass = adminElasticsearchPass +) + +// KibanaURL is the base URL for Kibana, including userinfo for +// authenticating as the admin user. +var KibanaURL *url.URL + +func init() { + kibanaConfig := apmservertest.DefaultConfig().Kibana + u, err := url.Parse(kibanaConfig.Host) + if err != nil { + log.Fatal(err) + } + u.User = url.UserPassword(adminKibanaUser, adminKibanaPass) + KibanaURL = u +} diff --git a/testing/docker/package-registry/config.yml b/testing/docker/package-registry/config.yml new file mode 100644 index 00000000000..82fd5881241 --- /dev/null +++ b/testing/docker/package-registry/config.yml @@ -0,0 +1,9 @@ +package_paths: + - /packages/production + - /packages/staging + - /packages/snapshot + - /packages/local + +cache_time.search: 10s +cache_time.categories: 10s +cache_time.catch_all: 10s