Merge branch '7.x' into backport/7.x/pr-4650_pr-4683_pr-4693

apmpackage/apm/0.1.0/_dev/docs/README.template.md

@@ -66,7 +66,8 @@ IMPORTANT: If you run APM Server with Elastic Agent manually in standalone mode,
 - `Host`: APM Server host and port to listen on.
 - `Secret token`: Authorization token for sending data to APM Server. See the [documentation](https://www.elastic.co/guide/en/apm/server/current/configuration-rum.html) for details.
 - `Enable RUM`: Enables support for RUM monitoring. See the [documentation](https://www.elastic.co/guide/en/apm/server/current/configuration-rum.html) for details.
-
+- `API Key for Central Configuration`: Gives privileges for APM Agent central configuration. See the [documentation](https://www.elastic.co/guide/en/kibana/master/agent-configuration.html)
+- `API Key for Sourcemaps`: Gives priveleges to read sourcemaps. See the [documentation](https://www.elastic.co/guide/en/apm/agent/rum-js/current/sourcemap.html).
 
 ## Traces
 

apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.json

@@ -3,22 +3,22 @@
   "processors": [
     {
       "pipeline": {
-        "name": "metrics-apm-0.1.0-apm_user_agent"
+        "name": "metrics-apm.app-0.1.0-apm_user_agent"
       }
     },
     {
       "pipeline": {
-        "name": "metrics-apm-0.1.0-apm_user_geo"
+        "name": "metrics-apm.app-0.1.0-apm_user_geo"
       }
     },
     {
       "pipeline": {
-        "name": "metrics-apm-0.1.0-apm_ingest_timestamp"
+        "name": "metrics-apm.app-0.1.0-apm_ingest_timestamp"
       }
     },
     {
       "pipeline": {
-        "name": "metrics-apm-0.1.0-apm_remove_span_metadata"
+        "name": "metrics-apm.app-0.1.0-apm_remove_span_metadata"
       }
     }
   ]

apmpackage/apm/0.1.0/data_stream/app_metrics/manifest.yml

@@ -1,4 +1,5 @@
 title: APM application metrics
 type: metrics
-dataset: apm
+dataset: apm.app
+dataset_is_prefix: true
 ilm_policy: metrics-apm.app_metrics-default_policy

apmpackage/apm/0.1.0/data_stream/error_logs/manifest.yml

@@ -1,4 +1,5 @@
 title: APM logs and errors
 type: logs
 dataset: apm.error
+dataset_is_prefix: true
 ilm_policy: logs-apm.error_logs-default_policy

apmpackage/apm/0.1.0/data_stream/internal_metrics/manifest.yml

@@ -1,4 +1,5 @@
 title: APM internal metrics
 type: metrics
 dataset: apm.internal
+dataset_is_prefix: true
 ilm_policy: metrics-apm.internal_metrics-default_policy

apmpackage/apm/0.1.0/data_stream/profile_metrics/manifest.yml

@@ -1,4 +1,5 @@
 title: APM profiles
 type: metrics
 dataset: apm.profiling
+dataset_is_prefix: true
 ilm_policy: metrics-apm.profile_metrics-default_policy

apmpackage/apm/0.1.0/data_stream/traces/manifest.yml

@@ -1,4 +1,5 @@
 title: APM traces
 type: traces
 dataset: apm
+dataset_is_prefix: true
 ilm_policy: traces-apm.traces-default_policy

apmpackage/apm/0.1.0/docs/README.md

@@ -66,7 +66,8 @@ IMPORTANT: If you run APM Server with Elastic Agent manually in standalone mode,
 - `Host`: APM Server host and port to listen on.
 - `Secret token`: Authorization token for sending data to APM Server. See the [documentation](https://www.elastic.co/guide/en/apm/server/current/configuration-rum.html) for details.
 - `Enable RUM`: Enables support for RUM monitoring. See the [documentation](https://www.elastic.co/guide/en/apm/server/current/configuration-rum.html) for details.
-
+- `API Key for Central Configuration`: Gives privileges for APM Agent central configuration. See the [documentation](https://www.elastic.co/guide/en/kibana/master/agent-configuration.html)
+- `API Key for Sourcemaps`: Gives priveleges to read sourcemaps. See the [documentation](https://www.elastic.co/guide/en/apm/agent/rum-js/current/sourcemap.html).
 
 ## Traces
 

beater/api/config/agent/handler_test.go

@@ -355,7 +355,7 @@ func TestIfNoneMatch(t *testing.T) {
 }
 
 func TestAgentConfigTraceContext(t *testing.T) {
-	kibanaCfg := libkibana.DefaultClientConfig()
+	kibanaCfg := config.KibanaConfig{Enabled: true, ClientConfig: libkibana.DefaultClientConfig()}
 	kibanaCfg.Host = "testKibana:12345"
 	client := kibana.NewConnectingClient(&kibanaCfg)
 	handler := Handler(client, &config.AgentConfig{Cache: &config.Cache{Expiration: 5 * time.Minute}})

beater/api/mux.go

@@ -168,7 +168,7 @@ type middlewareFunc func(*config.Config, *authorization.Handler, map[request.Res
 func agentConfigHandler(cfg *config.Config, authHandler *authorization.Handler, middlewareFunc middlewareFunc) (request.Handler, error) {
 	var client kibana.Client
 	if cfg.Kibana.Enabled {
-		client = kibana.NewConnectingClient(&cfg.Kibana.ClientConfig)
+		client = kibana.NewConnectingClient(&cfg.Kibana)
 	}
 	h := agent.Handler(client, cfg.AgentConfig)
 	msg := "Agent remote configuration is disabled. " +

beater/beater.go

@@ -25,6 +25,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/elastic/beats/v7/libbeat/kibana"
+
 	"github.com/pkg/errors"
 	"go.elastic.co/apm"
 	"golang.org/x/sync/errgroup"
@@ -184,6 +186,7 @@ func (bt *beater) start(ctx context.Context, cancelContext context.CancelFunc, b
 			inputs.Stop()
 		}
 		reload.Register.MustRegisterList("inputs", inputs)
+
 	} else {
 		// Management disabled, use statically defined config.
 		s, err := newServerRunner(ctx, serverRunnerParams{
@@ -236,6 +239,7 @@ func (s *serverCreator) Create(p beat.PipelineConnector, rawConfig *common.Confi
 		sharedServerRunnerParams: s.args,
 		Namespace:                namespace,
 		Pipeline:                 p,
+		KibanaConfig:             &integrationConfig.Fleet.Kibana,
 		RawConfig:                apmServerCommonConfig,
 	})
 }
@@ -268,9 +272,10 @@ type serverRunner struct {
 type serverRunnerParams struct {
 	sharedServerRunnerParams
 
-	Namespace string
-	Pipeline  beat.PipelineConnector
-	RawConfig *common.Config
+	Namespace    string
+	Pipeline     beat.PipelineConnector
+	KibanaConfig *kibana.ClientConfig
+	RawConfig    *common.Config
 }
 
 type sharedServerRunnerParams struct {
@@ -287,6 +292,11 @@ func newServerRunner(ctx context.Context, args serverRunnerParams) (*serverRunne
 	if err != nil {
 		return nil, err
 	}
+
+	if cfg.DataStreams.Enabled && args.KibanaConfig != nil {
+		cfg.Kibana.ClientConfig = *args.KibanaConfig
+	}
+
 	runServerContext, cancel := context.WithCancel(ctx)
 	return &serverRunner{
 		backgroundContext:      ctx,

beater/config/config.go

@@ -40,17 +40,18 @@ const (
 )
 
 type KibanaConfig struct {
-	Enabled             bool `config:"enabled"`
+	Enabled             bool   `config:"enabled"`
+	APIKey              string `config:"api_key"`
 	kibana.ClientConfig `config:",inline"`
 }
 
 func (k *KibanaConfig) Unpack(cfg *common.Config) error {
-	if err := cfg.Unpack(&k.ClientConfig); err != nil {
+	type kibanaConfig KibanaConfig
+	if err := cfg.Unpack((*kibanaConfig)(k)); err != nil {
 		return err
 	}
 	k.Enabled = cfg.Enabled()
 	k.Host = strings.TrimRight(k.Host, "/")
-
 	return nil
 }
 
@@ -119,7 +120,7 @@ func NewConfig(ucfg *common.Config, outputESCfg *common.Config) (*Config, error)
 		return nil, errors.New(msgInvalidConfigAgentCfg)
 	}
 
-	if err := c.RumConfig.setup(logger, outputESCfg); err != nil {
+	if err := c.RumConfig.setup(logger, c.DataStreams.Enabled, outputESCfg); err != nil {
 		return nil, err
 	}
 

beater/config/integration.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/kibana"
 )
 
 func NewIntegrationConfig(rootConfig *common.Config) (*IntegrationConfig, error) {
@@ -48,6 +49,7 @@ type IntegrationConfig struct {
 	Meta       *Meta          `config:"meta"`
 	DataStream *DataStream    `config:"data_stream"`
 	APMServer  *common.Config `config:"apm-server"`
+	Fleet      Fleet          `config:"fleet"`
 }
 
 type DataStream struct {
@@ -62,3 +64,7 @@ type Package struct {
 	Name    string `config:"name"`
 	Version string `config:"version"`
 }
+
+type Fleet struct {
+	Kibana kibana.ClientConfig `config:"kibana"`
+}

beater/config/rum.go

@@ -76,7 +76,7 @@ func (s *SourceMapping) IsEnabled() bool {
 	return s == nil || s.Enabled == nil || *s.Enabled
 }
 
-func (c *RumConfig) setup(log *logp.Logger, outputESCfg *common.Config) error {
+func (c *RumConfig) setup(log *logp.Logger, dataStreamsEnabled bool, outputESCfg *common.Config) error {
 	if !c.IsEnabled() {
 		return nil
 	}
@@ -88,8 +88,14 @@ func (c *RumConfig) setup(log *logp.Logger, outputESCfg *common.Config) error {
 		return errors.Wrapf(err, "Invalid regex for `exclude_from_grouping`: ")
 	}
 
+	var apiKey string
 	if c.SourceMapping == nil || c.SourceMapping.esConfigured {
-		return nil
+		if dataStreamsEnabled {
+			// when running under Fleet, the only setting configured is the api key
+			apiKey = c.SourceMapping.ESConfig.APIKey
+		} else {
+			return nil
+		}
 	}
 
 	// fall back to elasticsearch output configuration for sourcemap storage if possible
@@ -101,6 +107,9 @@ func (c *RumConfig) setup(log *logp.Logger, outputESCfg *common.Config) error {
 	if err := outputESCfg.Unpack(c.SourceMapping.ESConfig); err != nil {
 		return errors.Wrap(err, "unpacking Elasticsearch config into Sourcemap config")
 	}
+	if c.SourceMapping.ESConfig.APIKey == "" {
+		c.SourceMapping.ESConfig.APIKey = apiKey
+	}
 	return nil
 }
 

beater/config/rum_test.go

@@ -20,6 +20,12 @@ package config
 import (
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/apm-server/elasticsearch"
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/logp"
+
 	"github.com/stretchr/testify/assert"
 )
 
@@ -37,6 +43,22 @@ func TestIsRumEnabled(t *testing.T) {
 	}
 }
 
+func TestRumSetup(t *testing.T) {
+	rum := defaultRum()
+	rum.SourceMapping.esConfigured = true
+	rum.Enabled = &rum.SourceMapping.esConfigured
+	rum.SourceMapping.ESConfig = &elasticsearch.Config{APIKey: "id:apikey"}
+	esCfg := common.MustNewConfigFrom(map[string]interface{}{
+		"hosts": []interface{}{"cloud:9200"},
+	})
+
+	err := rum.setup(logp.NewLogger("test"), true, esCfg)
+
+	require.NoError(t, err)
+	assert.Equal(t, elasticsearch.Hosts{"cloud:9200"}, rum.SourceMapping.ESConfig.Hosts)
+	assert.Equal(t, "id:apikey", rum.SourceMapping.ESConfig.APIKey)
+}
+
 func TestDefaultRum(t *testing.T) {
 	c := DefaultConfig()
 	assert.Equal(t, defaultRum(), c.RumConfig)

beater/jaeger/server.go

@@ -102,7 +102,7 @@ func NewServer(logger *logp.Logger, cfg *config.Config, tracer *apm.Tracer, repo
 		var client kibana.Client
 		var fetcher *agentcfg.Fetcher
 		if cfg.Kibana.Enabled {
-			client = kibana.NewConnectingClient(&cfg.Kibana.ClientConfig)
+			client = kibana.NewConnectingClient(&cfg.Kibana)
 			fetcher = agentcfg.NewFetcher(client, cfg.AgentConfig.Cache.Expiration)
 		}
 		RegisterGRPCServices(

beater/server.go

@@ -138,7 +138,7 @@ func newGRPCServer(
 	var kibanaClient kibana.Client
 	var agentcfgFetcher *agentcfg.Fetcher
 	if cfg.Kibana.Enabled {
-		kibanaClient = kibana.NewConnectingClient(&cfg.Kibana.ClientConfig)
+		kibanaClient = kibana.NewConnectingClient(&cfg.Kibana)
 		agentcfgFetcher = agentcfg.NewFetcher(kibanaClient, cfg.AgentConfig.Cache.Expiration)
 	}
 	jaeger.RegisterGRPCServices(srv, authBuilder, jaeger.ElasticAuthTag, logger, reporter, kibanaClient, agentcfgFetcher)

cmd/apikey.go

@@ -49,9 +49,9 @@ func genApikeyCmd(settings instance.Settings) *cobra.Command {
 	apikeyCmd := cobra.Command{
 		Use:   "apikey",
 		Short: short,
-		Long: short + `. 
-Most operations require the "manage_api_key" cluster privilege. Ensure to configure "apm-server.api_key.*" or 
-"output.elasticsearch.*" appropriately. APM Server will create security privileges for the "apm" application; 
+		Long: short + `.
+Most operations require the "manage_api_key" cluster privilege. Ensure to configure "apm-server.api_key.*" or
+"output.elasticsearch.*" appropriately. APM Server will create security privileges for the "apm" application;
 you can freely query them. If you modify or delete apm privileges, APM Server might reject all requests.
 Check the Elastic Security API documentation for details.`,
 	}

docs/getting-started-apm-server.asciidoc

@@ -352,11 +352,12 @@ If you haven't already, you can now install APM Agents in your services!
 * {apm-java-ref-v}/intro.html[Java agent]
 * {apm-dotnet-ref-v}/intro.html[.NET agent]
 * {apm-node-ref-v}/intro.html[Node.js agent]
+* {apm-php-ref-v}/intro.html[PHP agent]
 * {apm-py-ref-v}/getting-started.html[Python agent]
 * {apm-ruby-ref-v}/introduction.html[Ruby agent]
 * {apm-rum-ref-v}/intro.html[JavaScript Real User Monitoring (RUM) agent]
 
-Once you have at least one Agent sending data to APM Server,
+Once you have at least one APM agent sending data to APM Server,
 you can start visualizing your data in the {kibana-ref}/xpack-apm.html[APM app].
 
 If you're migrating from Jaeger, see <<jaeger>>.

docs/guide/agent-server-compatibility.asciidoc

@@ -5,39 +5,43 @@ The chart below outlines the compatibility between different versions of the APM
 
 [options="header"]
 |====
-|Agent |Agent Version |APM Server Version
+|Agent |Agent version |APM Server version
 // Go
-.1+|**Go Agent**
+.1+|**Go agent**
 |`1.x` |≥ `6.5`
 
 // Java
-.1+|**Java Agent**
+.1+|**Java agent**
 |`1.x`|≥ `6.5`
 
 // .NET
-.1+|**.NET Agent**
+.1+|**.NET agent**
 |`1.x` |≥ `6.5`
 
 // Node
-.3+|**Node.js Agent**
+.3+|**Node.js agent**
 |`1.x` |`6.2`-`6.x`
 |`2.x` |≥ `6.5`
 |`3.x` |≥ `6.6`
 
+// PHP
+.1+|**PHP agent**
+|`1.x` |≥ `7.0`
+
 // Python
-.3+|**Python Agent**
+.3+|**Python agent**
 |`2.x`, `3.x` |`6.2`-`6.x`
 |`4.x` |≥ `6.5`
 |`5.x` |≥ `6.6`
 
 // Ruby
-.3+|**Ruby Agent**
+.3+|**Ruby agent**
 |`1.x` |`6.4`-`6.x`
 |`2.x` |≥ `6.5`
 |`3.x` |≥ `6.5`
 
 // RUM
-.4+|**JavaScript RUM Agent**
+.4+|**JavaScript RUM agent**
 |`0.x` |`6.3`-`6.4`
 |`1.x` |`6.4`
 |`2.x`, `3.x`, `4.x` |≥ `6.5`