Merge branch 'main' of github.com:elastic/fleet-server into feat/bulk…

…-api-update
elastic · Sep 19, 2022 · 607ab12 · 607ab12
2 parents f15ff5d + ef9ca2b
commit 607ab12
Show file tree

Hide file tree

Showing 19 changed files with 68 additions and 30 deletions.
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
@@ -10,8 +10,8 @@ pipeline {
     DOCKER_COMPOSE_VERSION = '1.25.5'
     JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba"
     PIPELINE_LOG_LEVEL='INFO'
-    JOB_GCS_BUCKET = 'beats-ci-artifacts'
-    JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin-file-credentials'
+    JOB_GCS_BUCKET = 'fleet-ci-artifacts'
+    JOB_GCS_CREDENTIALS = 'fleet-ci-gcs-plugin-file-credentials'
   }
   options {
     timeout(time: 1, unit: 'HOURS')
@@ -139,7 +139,7 @@ pipeline {
       options { skipDefaultCheckout() }
       when { expression { isBranch() } }
       steps {
-        build(job: "Ingest-manager/fleet-server-package-mbp/${env.JOB_BASE_NAME}",
+        build(job: "fleet-server/fleet-server-package-mbp/${env.JOB_BASE_NAME}",
               propagate: false,
               wait: false,
               parameters: [string(name: 'COMMIT', value: "${env.GIT_BASE_COMMIT}")])

diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml
@@ -3,7 +3,7 @@
 ##### GLOBAL METADATA
 
 - meta:
-    cluster: beats-ci
+    cluster: fleet-ci
 
 ##### JOB DEFAULTS
 
@@ -15,4 +15,4 @@
     publishers:
       - email:
           recipients: [email protected]
-    prune-dead-branches: true
+    prune-dead-branches: true
diff --git a/.ci/jobs/fleet-server-package-mbp.yml b/.ci/jobs/fleet-server-package-mbp.yml
@@ -1,6 +1,6 @@
 ---
 - job:
-    name: Ingest-manager/fleet-server-package-mbp
+    name: fleet-server/fleet-server-package-mbp
     display-name: Fleet Server Package
     description: Jenkins pipeline for the Elastic Fleet Server package process
     project-type: multibranch
@@ -53,4 +53,4 @@
           reference-repo: /var/lib/jenkins/.git-references/fleet-server.git
         timeout: '15'
         use-author: true
-        wipe-workspace: 'True'
+        wipe-workspace: true
diff --git a/.ci/jobs/fleet-server.yml b/.ci/jobs/fleet-server.yml
@@ -1,6 +1,6 @@
 ---
 - job:
-    name: Ingest-manager/fleet-server
+    name: fleet-server/fleet-server-mbp
     display-name: Fleet Server
     description: Jenkins pipeline for the Elastic Fleet Server project
     view: Beats

diff --git a/.ci/jobs/folders.yml b/.ci/jobs/folders.yml
@@ -1,10 +1,10 @@
 ---
 #https://docs.openstack.org/infra/jenkins-job-builder/project_folder.html
 - job:
-    name: Ingest-manager
-    description: Ingest manager related Jobs
+    name: fleet-server
+    description: Fleet Server related Jobs
     project-type: folder
 
 - view:
-    name: Ingest-manager
-    view-type: list
+    name: fleet-server
+    view-type: list
diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -9,7 +9,7 @@ pipeline {
     SLACK_CHANNEL = '#elastic-agent-control-plane'
     NOTIFY_TO = '[email protected]'
     JOB_GCS_BUCKET = credentials('gcs-bucket')
-    JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
+    JOB_GCS_CREDENTIALS = 'fleet-ci-gcs-plugin'
     DOCKER_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_REGISTRY = 'docker.elastic.co'
     DRA_OUTPUT = 'release-manager.out'
@@ -133,7 +133,7 @@ pipeline {
             }
           }
         }
-        stage('DRA Staging') {
+        stage('DRA Release Staging') {
           options { skipDefaultCheckout() }
           when {
             allOf {

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -1,7 +1,15 @@
+==== Breaking Changes
+
+- Upgrade to Go 1.18. Certificates signed with SHA-1 are now rejected. See the Go 1.18 https://tip.golang.org/doc/go1.18#sha1[release notes] for details. {pull}1709[1709]
+
 ==== Bugfixes
 
 - Return a better error on enrolling and the Elasticsearch version is incompatible. {pull}1211[1211]
 - Give a grace period when starting the unenroll monitor. {issue}1500[1500]
+- Fixes a race condition between the unenroller goroutine and the main goroutine for the coordinator monitor. {issues}1738[1738]
+- Remove events from agent checkin body. {issue}1774[1774]
+- Improve authc debug logging. {pull}1870[1870]
+- Add error detail to catch-all HTTP error response. {pull}1854[1854]
 
 ==== New Features
 
@@ -11,4 +19,4 @@
 - Add start_time and minimum_execution_duration to actions to allow fleet-server to schedule agent actions. {pull}1381[1381]
 - Fleet Server now allows setting global labels on APM instrumentation. {pull}1649[1649]
 - Fleet Server now allows setting transaction sample rate on APM instrumentation {pull}1681[1681]
-- Log redacted config when config updates. {issue}1626[1626] {pull}1668[1668]
+- Log redacted config when config updates. {issue}1626[1626] {pull}1668[1668]
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-[![Build Status](https://beats-ci.elastic.co/job/Ingest-manager/job/fleet-server/job/main/badge/icon)](https://beats-ci.elastic.co/job/Ingest-manager/job/fleet-server/job/main/)
+[![Build Status](https://fleet-ci.elastic.co/job/fleet-server/job/fleet-server-mbp/job/main/badge/icon)](https://fleet-ci.elastic.co/job/Ingest-manager/job/fleet-server/job/main/)
 
 # Fleet Server implementation
 

diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env
@@ -1,4 +1,4 @@
-ELASTICSEARCH_VERSION=8.5.0-60a4c029-SNAPSHOT
+ELASTICSEARCH_VERSION=8.5.0-c7913db3-SNAPSHOT
 ELASTICSEARCH_USERNAME=elastic
 ELASTICSEARCH_PASSWORD=changeme
 TEST_ELASTICSEARCH_HOSTS=localhost:9200
diff --git a/internal/pkg/api/auth.go b/internal/pkg/api/auth.go
@@ -33,6 +33,8 @@ var (
 func authAPIKey(r *http.Request, bulker bulk.Bulk, c cache.Cache) (*apikey.APIKey, error) {
 	span, ctx := apm.StartSpan(r.Context(), "authAPIKey", "auth")
 	defer span.End()
+	start := time.Now()
+	reqID := r.Header.Get(logger.HeaderRequestID)
 
 	key, err := apikey.ExtractAPIKey(r)
 	if err != nil {
@@ -41,15 +43,17 @@ func authAPIKey(r *http.Request, bulker bulk.Bulk, c cache.Cache) (*apikey.APIKe
 
 	if c.ValidAPIKey(*key) {
 		span.Context.SetLabel("api_key_cache_hit", true)
+		log.Debug().
+			Str("id", key.ID).
+			Str(ECSHTTPRequestID, reqID).
+			Int64(ECSEventDuration, time.Since(start).Nanoseconds()).
+			Bool("fleet.api_key.cache_hit", true).
+			Msg("ApiKey authenticated")
 		return key, nil
 	} else {
 		span.Context.SetLabel("api_key_cache_hit", false)
 	}
 
-	reqID := r.Header.Get(logger.HeaderRequestID)
-
-	start := time.Now()
-
 	info, err := bulker.APIKeyAuth(ctx, *key)
 
 	if err != nil {
@@ -62,14 +66,15 @@ func authAPIKey(r *http.Request, bulker bulk.Bulk, c cache.Cache) (*apikey.APIKe
 		return nil, err
 	}
 
-	log.Trace().
+	log.Debug().
 		Str("id", key.ID).
 		Str(ECSHTTPRequestID, reqID).
 		Int64(ECSEventDuration, time.Since(start).Nanoseconds()).
 		Str("userName", info.UserName).
 		Strs("roles", info.Roles).
 		Bool("enabled", info.Enabled).
 		RawJSON("meta", info.Metadata).
+		Bool("fleet.api_key.cache_hit", false).
 		Msg("ApiKey authenticated")
 
 	c.SetAPIKey(*key, info.Enabled)

diff --git a/internal/pkg/api/error.go b/internal/pkg/api/error.go
@@ -161,6 +161,7 @@ func NewHTTPErrResp(err error) HTTPErrResp {
 	return HTTPErrResp{
 		StatusCode: http.StatusBadRequest,
 		Error:      "BadRequest",
+		Message:    err.Error(),
 		Level:      zerolog.InfoLevel,
 	}
 }

diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go
@@ -481,6 +481,7 @@ func (ack *AckT) handleUpgrade(ctx context.Context, zlog zerolog.Logger, agent *
 	doc := bulk.UpdateFields{
 		dl.FieldUpgradeStartedAt: nil,
 		dl.FieldUpgradedAt:       now,
+		dl.FieldUpgradeStatus:    "completed",
 	}
 
 	body, err := doc.Marshal()

diff --git a/internal/pkg/api/schema.go b/internal/pkg/api/schema.go
@@ -80,7 +80,6 @@ type EnrollResponse struct {
 type CheckinRequest struct {
 	Status    string          `json:"status"`
 	AckToken  string          `json:"ack_token,omitempty"`
-	Events    []Event         `json:"events"`
 	LocalMeta json.RawMessage `json:"local_metadata"`
 }
 

diff --git a/internal/pkg/bulk/bulk_test.go b/internal/pkg/bulk/bulk_test.go
@@ -251,9 +251,10 @@ func TestCancelCtx(t *testing.T) {
 		},
 	}
 
+	_ = testlog.SetLogger(t)
+
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			_ = testlog.SetLogger(t)
 			ctx, cancelF := context.WithCancel(context.Background())
 
 			var wg sync.WaitGroup

diff --git a/internal/pkg/coordinator/monitor.go b/internal/pkg/coordinator/monitor.go
@@ -74,8 +74,10 @@ type monitorT struct {
 	leadersIndex  string
 	agentsIndex   string
 
-	policies          map[string]policyT
-	policiesCanceller map[string]context.CancelFunc
+	policies map[string]policyT
+
+	muPoliciesCanceller sync.Mutex
+	policiesCanceller   map[string]context.CancelFunc
 }
 
 // NewMonitor creates a new coordinator policy monitor.
@@ -311,7 +313,10 @@ func (m *monitorT) ensureLeadership(ctx context.Context) error {
 		if r.cord == nil {
 			// either failed to take leadership or lost leadership
 			delete(m.policies, r.id)
+
+			m.muPoliciesCanceller.Lock()
 			delete(m.policiesCanceller, r.id)
+			m.muPoliciesCanceller.Unlock()
 		} else {
 			m.policies[r.id] = r
 		}
@@ -396,6 +401,9 @@ func (m *monitorT) getIPs() ([]string, error) {
 }
 
 func (m *monitorT) rescheduleUnenroller(ctx context.Context, pt *policyT, p *model.Policy) {
+	m.muPoliciesCanceller.Lock()
+	defer m.muPoliciesCanceller.Unlock()
+
 	u := uuid.Must(uuid.NewV4())
 	l := m.log.With().Str(dl.FieldPolicyID, pt.id).Str("unenroller_uuid", u.String()).Logger()
 	unenrollTimeout := time.Duration(p.UnenrollTimeout) * time.Second
@@ -418,6 +426,13 @@ func (m *monitorT) rescheduleUnenroller(ctx context.Context, pt *policyT, p *mod
 	}
 }
 
+func (m *monitorT) ActivePoliciesCancellerCount() int {
+	m.muPoliciesCanceller.Lock()
+	defer m.muPoliciesCanceller.Unlock()
+
+	return len(m.policiesCanceller)
+}
+
 func runCoordinator(ctx context.Context, cord Coordinator, l zerolog.Logger, d time.Duration) {
 	cnt := 0
 	for {

diff --git a/internal/pkg/coordinator/monitor_integration_test.go b/internal/pkg/coordinator/monitor_integration_test.go
@@ -226,7 +226,7 @@ func TestMonitorUnenroller(t *testing.T) {
 	assert.NotEmpty(t, agent.UnenrolledAt)
 	assert.Equal(t, unenrolledReasonTimeout, agent.UnenrolledReason)
 	assert.Len(t, pm.(*monitorT).policies, 1)
-	assert.Len(t, pm.(*monitorT).policiesCanceller, 1)
+	assert.Equal(t, pm.(*monitorT).ActivePoliciesCancellerCount(), 1)
 
 	// should error as they are now invalidated
 	_, err = bulker.APIKeyAuth(bulkCtx, *accessKey)
@@ -347,7 +347,7 @@ func TestMonitorUnenrollerSetAndClear(t *testing.T) {
 	assert.True(t, agent.Active)
 	// Make sure canceller is no longer there
 	assert.Len(t, pm.(*monitorT).policies, 1)
-	assert.Len(t, pm.(*monitorT).policiesCanceller, 0)
+	assert.Equal(t, pm.(*monitorT).ActivePoliciesCancellerCount(), 0)
 
 }
 

diff --git a/internal/pkg/dl/constants.go b/internal/pkg/dl/constants.go
@@ -49,6 +49,7 @@ const (
 	FieldUnenrolledAt     = "unenrolled_at"
 	FieldUpgradedAt       = "upgraded_at"
 	FieldUpgradeStartedAt = "upgrade_started_at"
+	FieldUpgradeStatus    = "upgrade_status"
 
 	FieldDecodedSha256 = "decoded_sha256"
 	FieldIdentifier    = "identifier"

diff --git a/internal/pkg/model/schema.go b/internal/pkg/model/schema.go
diff --git a/model/schema.json b/model/schema.json
@@ -401,6 +401,10 @@
           "type": "string",
           "format": "date-time"
         },
+        "upgrade_status": {
+          "description": "Upgrade status",
+          "type": "string"
+        },
         "access_api_key_id": {
           "description": "ID of the API key the Elastic Agent must used to contact Fleet Server",
           "type": "string"
@@ -451,7 +455,7 @@
           "format": "date-time"
         },
         "last_checkin_status": {
-          "description": "Lst checkin status",
+          "description": "Last checkin status",
           "type": "string"
         },
         "default_api_key_id": {