diff --git a/.ci/scripts/install-helm-test-dependencies.sh b/.ci/scripts/install-helm-test-dependencies.sh index 80796977ce..d74446ae13 100755 --- a/.ci/scripts/install-helm-test-dependencies.sh +++ b/.ci/scripts/install-helm-test-dependencies.sh @@ -20,8 +20,8 @@ HOME=${HOME:?$MSG} HELM_VERSION="${HELM_VERSION:-"3.9.0"}" HELM_TAR_GZ_FILE="helm-v${HELM_VERSION}-linux-${GOARCH}.tar.gz" -KIND_VERSION="v${KIND_VERSION:-"0.17.0"}" -KUBERNETES_VERSION="${KUBERNETES_VERSION:-"1.26.0"}" +KIND_VERSION="v${KIND_VERSION:-"0.20.0"}" +KUBERNETES_VERSION="${KUBERNETES_VERSION:-"1.27.3"}" HELM_CMD="${HOME}/bin/helm" KBC_CMD="${HOME}/bin/kubectl" diff --git a/.ci/scripts/install-kubernetes-autodiscover-test-dependencies.sh b/.ci/scripts/install-kubernetes-autodiscover-test-dependencies.sh index fb914ea48a..a17a866b59 100755 --- a/.ci/scripts/install-kubernetes-autodiscover-test-dependencies.sh +++ b/.ci/scripts/install-kubernetes-autodiscover-test-dependencies.sh @@ -16,8 +16,8 @@ set -euxo pipefail MSG="parameter missing." HOME=${HOME:?$MSG} -KIND_VERSION="v${KIND_VERSION:-"0.17.0"}" -KUBERNETES_VERSION="${KUBERNETES_VERSION:-"1.26.0"}" +KIND_VERSION="v${KIND_VERSION:-"0.20.0"}" +KUBERNETES_VERSION="${KUBERNETES_VERSION:-"1.27.3"}" KUBECTL_CMD="${HOME}/bin/kubectl" diff --git a/.go-version b/.go-version index 5fb5a6b4f5..7bf9455f08 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.20 +1.20.5 diff --git a/.stack-version b/.stack-version index 2403ddd309..9a3ba6e2d6 100644 --- a/.stack-version +++ b/.stack-version @@ -1 +1 @@ -8.9.0-473a43eb-SNAPSHOT \ No newline at end of file +8.10.0-c5be8c13-SNAPSHOT \ No newline at end of file diff --git a/e2e/_suites/kubernetes-autodiscover/README.md b/e2e/_suites/kubernetes-autodiscover/README.md index ac7f8d38a0..2cc0a7b140 100644 --- a/e2e/_suites/kubernetes-autodiscover/README.md +++ b/e2e/_suites/kubernetes-autodiscover/README.md @@ -110,8 +110,8 @@ This is an example of the optional configuration: export BEAT_VERSION=7.12.0 # version of beats to use export ELASTIC_AGENT_VERSION=7.12.0 # version of Elastic Agent to use export GITHUB_CHECK_SHA1=0123456789 # to select snapshots built by beats-ci - export KIND_VERSION="0.17.0" # version of kind - export KUBERNETES_VERSION="1.26.0" # version of the cluster to be passed to kind + export KIND_VERSION="0.20.0" # version of kind + export KUBERNETES_VERSION="1.27.3" # version of the cluster to be passed to kind ``` 3. Install dependencies. diff --git a/internal/common/defaults.go b/internal/common/defaults.go index c0e20cc3d3..140b406c77 100644 --- a/internal/common/defaults.go +++ b/internal/common/defaults.go @@ -36,7 +36,7 @@ const FleetProfileName = "fleet" const FleetServerAgentServiceName = "fleet-server" // BeatVersionBase is the base version of the Beat to use -var BeatVersionBase = "8.9.0-473a43eb-SNAPSHOT" +var BeatVersionBase = "8.10.0-c5be8c13-SNAPSHOT" // BeatVersion is the version of the Beat to use // It can be overriden by BEAT_VERSION env var diff --git a/internal/config/compose/profiles/fleet/docker-compose.yml b/internal/config/compose/profiles/fleet/docker-compose.yml index 27f77836e2..c15eefceb1 100644 --- a/internal/config/compose/profiles/fleet/docker-compose.yml +++ b/internal/config/compose/profiles/fleet/docker-compose.yml @@ -18,7 +18,7 @@ services: - xpack.security.authc.token.timeout=60m - ELASTIC_USERNAME=admin - ELASTIC_PASSWORD=changeme - image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion:-8.9.0-473a43eb-SNAPSHOT}" + image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion:-8.10.0-c5be8c13-SNAPSHOT}" platform: ${stackPlatform:-linux/amd64} ports: - "9200:9200" @@ -34,7 +34,7 @@ services: test: ["CMD-SHELL", "curl -u admin:changeme -s http://localhost:5601/api/status | grep -q 'All services are available'"] retries: 60 interval: 10s - image: "docker.elastic.co/${kibanaDockerNamespace:-kibana}/kibana:${kibanaVersion:-8.9.0-473a43eb-SNAPSHOT}" + image: "docker.elastic.co/${kibanaDockerNamespace:-kibana}/kibana:${kibanaVersion:-8.10.0-c5be8c13-SNAPSHOT}" platform: ${stackPlatform:-linux/amd64} ports: - "5601:5601" diff --git a/internal/config/compose/services/elastic-agent/cloud/docker-compose.yml b/internal/config/compose/services/elastic-agent/cloud/docker-compose.yml index b9f74d292c..796ddffab5 100644 --- a/internal/config/compose/services/elastic-agent/cloud/docker-compose.yml +++ b/internal/config/compose/services/elastic-agent/cloud/docker-compose.yml @@ -1,7 +1,7 @@ version: '2.4' services: elastic-agent: - image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.9.0-473a43eb-SNAPSHOT}" + image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.10.0-c5be8c13-SNAPSHOT}" depends_on: elasticsearch: condition: service_healthy diff --git a/internal/config/compose/services/elastic-agent/docker-compose.yml b/internal/config/compose/services/elastic-agent/docker-compose.yml index 91d6bcf695..09660f1520 100644 --- a/internal/config/compose/services/elastic-agent/docker-compose.yml +++ b/internal/config/compose/services/elastic-agent/docker-compose.yml @@ -1,7 +1,7 @@ version: '2.4' services: elastic-agent: - image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.9.0-473a43eb-SNAPSHOT}" + image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.10.0-c5be8c13-SNAPSHOT}" depends_on: elasticsearch: condition: service_healthy diff --git a/internal/config/compose/services/elastic-agent/fleet-server/docker-compose.yml b/internal/config/compose/services/elastic-agent/fleet-server/docker-compose.yml index da97aba0f6..af37277434 100644 --- a/internal/config/compose/services/elastic-agent/fleet-server/docker-compose.yml +++ b/internal/config/compose/services/elastic-agent/fleet-server/docker-compose.yml @@ -1,7 +1,7 @@ version: '2.4' services: fleet-server: - image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.9.0-473a43eb-SNAPSHOT}" + image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.10.0-c5be8c13-SNAPSHOT}" depends_on: elasticsearch: condition: service_healthy diff --git a/internal/config/compose/services/elasticsearch/docker-compose.yml b/internal/config/compose/services/elasticsearch/docker-compose.yml index 11d52c9a44..792fe2c9ae 100644 --- a/internal/config/compose/services/elasticsearch/docker-compose.yml +++ b/internal/config/compose/services/elasticsearch/docker-compose.yml @@ -9,7 +9,7 @@ services: - xpack.monitoring.collection.enabled=true - ELASTIC_USERNAME=elastic - ELASTIC_PASSWORD=changeme - image: "docker.elastic.co/observability-ci/elasticsearch:${elasticsearchTag:-8.9.0-473a43eb-SNAPSHOT}" + image: "docker.elastic.co/observability-ci/elasticsearch:${elasticsearchTag:-8.10.0-c5be8c13-SNAPSHOT}" healthcheck: interval: 10s retries: 100 diff --git a/internal/config/compose/services/kibana/docker-compose.yml b/internal/config/compose/services/kibana/docker-compose.yml index 2ffc184541..16e23e87b4 100644 --- a/internal/config/compose/services/kibana/docker-compose.yml +++ b/internal/config/compose/services/kibana/docker-compose.yml @@ -9,6 +9,6 @@ services: test: ["CMD-SHELL", "curl -u admin:changeme -s http://localhost:5601/api/status | grep -q 'All services are available'"] retries: 60 interval: 10s - image: "docker.elastic.co/kibana/kibana:${kibanaTag:-8.9.0-473a43eb-SNAPSHOT}" + image: "docker.elastic.co/kibana/kibana:${kibanaTag:-8.10.0-c5be8c13-SNAPSHOT}" ports: - "5601:5601" diff --git a/internal/config/compose/services/metricbeat/docker-compose.yml b/internal/config/compose/services/metricbeat/docker-compose.yml index 00a5831f42..773b863ef0 100644 --- a/internal/config/compose/services/metricbeat/docker-compose.yml +++ b/internal/config/compose/services/metricbeat/docker-compose.yml @@ -14,7 +14,7 @@ services: ] environment: - BEAT_STRICT_PERMS=${beatStricPerms:-false} - image: "docker.elastic.co/${metricbeatDockerNamespace:-beats}/metricbeat:${metricbeatTag:-8.9.0-473a43eb-SNAPSHOT}" + image: "docker.elastic.co/${metricbeatDockerNamespace:-beats}/metricbeat:${metricbeatTag:-8.10.0-c5be8c13-SNAPSHOT}" labels: co.elastic.logs/module: "${serviceName}" platform: ${stackPlatform:-linux/amd64} diff --git a/internal/config/kubernetes/README.md b/internal/config/kubernetes/README.md index 96a8fb191f..ad32418f78 100644 --- a/internal/config/kubernetes/README.md +++ b/internal/config/kubernetes/README.md @@ -3,8 +3,8 @@ ## Requirements - docker -- kind (>= 0.17.0) -- kubectl (>= 1.26.0) +- kind (>= 0.20.0) +- kubectl (>= 1.27.3) ## Deployment diff --git a/internal/config/kubernetes/base/elasticsearch/deployment.yaml b/internal/config/kubernetes/base/elasticsearch/deployment.yaml index 8d5e7fe021..0a59ae3692 100644 --- a/internal/config/kubernetes/base/elasticsearch/deployment.yaml +++ b/internal/config/kubernetes/base/elasticsearch/deployment.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - name: elasticsearch - image: docker.elastic.co/elasticsearch/elasticsearch:8.9.0-473a43eb-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.10.0-c5be8c13-SNAPSHOT envFrom: - configMapRef: name: elasticsearch-config diff --git a/internal/config/kubernetes/base/fleet-server/deployment.yaml b/internal/config/kubernetes/base/fleet-server/deployment.yaml index cad5f7046f..093f86472c 100644 --- a/internal/config/kubernetes/base/fleet-server/deployment.yaml +++ b/internal/config/kubernetes/base/fleet-server/deployment.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - name: fleet-server - image: docker.elastic.co/beats/elastic-agent:8.9.0-473a43eb-SNAPSHOT + image: docker.elastic.co/beats/elastic-agent:8.10.0-c5be8c13-SNAPSHOT env: - name: FLEET_SERVER_ENABLE value: "1" diff --git a/internal/config/kubernetes/base/kibana/deployment.yaml b/internal/config/kubernetes/base/kibana/deployment.yaml index c494de2b40..e8f2d52781 100644 --- a/internal/config/kubernetes/base/kibana/deployment.yaml +++ b/internal/config/kubernetes/base/kibana/deployment.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - name: kibana - image: docker.elastic.co/kibana/kibana:8.9.0-473a43eb-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.10.0-c5be8c13-SNAPSHOT env: - name: ELASTICSEARCH_URL value: http://elasticsearch:9200 diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 573869a2b5..b8a005894a 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -5,6 +5,7 @@ package utils import ( + "fmt" "io" "math/rand" "net/http" @@ -53,11 +54,7 @@ func DownloadFile(downloadRequest *DownloadRequest) error { tempParentDir := filepath.Join(os.TempDir(), uuid.NewString()) err := internalio.MkdirAll(tempParentDir) if err != nil { - log.WithFields(log.Fields{ - "error": err, - "path": tempParentDir, - }).Error("Error creating directory") - return err + return fmt.Errorf("creating directory: %w", err) } filePath = filepath.Join(tempParentDir, uuid.NewString()) downloadRequest.DownloadPath = filePath @@ -67,11 +64,7 @@ func DownloadFile(downloadRequest *DownloadRequest) error { tempFile, err := os.Create(filePath) if err != nil { - log.WithFields(log.Fields{ - "error": err, - "url": downloadRequest.URL, - }).Error("Error creating file") - return err + return fmt.Errorf("creating file: %w", err) } defer tempFile.Close() @@ -83,36 +76,19 @@ func DownloadFile(downloadRequest *DownloadRequest) error { download := func() error { resp, err := http.Get(downloadRequest.URL) if err != nil { - log.WithFields(log.Fields{ - "elapsedTime": exp.GetElapsedTime(), - "error": err, - "path": downloadRequest.UnsanitizedFilePath, - "retry": retryCount, - "url": downloadRequest.URL, - }).Warn("Could not download the file") - retryCount++ - - return err + return fmt.Errorf("downloading file %s: %w", downloadRequest.URL, err) } - log.WithFields(log.Fields{ - "elapsedTime": exp.GetElapsedTime(), - "retries": retryCount, - "path": downloadRequest.UnsanitizedFilePath, - "url": downloadRequest.URL, - }).Trace("File downloaded") + if resp != nil && resp.StatusCode == http.StatusNotFound { + return backoff.Permanent(fmt.Errorf("%s not found", downloadRequest.URL)) + } fileReader = resp.Body return nil } - log.WithFields(log.Fields{ - "url": downloadRequest.URL, - "path": downloadRequest.UnsanitizedFilePath, - }).Trace("Downloading file") - err = backoff.Retry(download, exp) if err != nil { return err @@ -121,13 +97,7 @@ func DownloadFile(downloadRequest *DownloadRequest) error { _, err = io.Copy(tempFile, fileReader) if err != nil { - log.WithFields(log.Fields{ - "error": err, - "url": downloadRequest.URL, - "path": downloadRequest.UnsanitizedFilePath, - }).Error("Could not write file") - - return err + return fmt.Errorf("writing file %s: %w", tempFile.Name(), err) } _ = os.Chmod(tempFile.Name(), 0666) diff --git a/pkg/downloads/releases.go b/pkg/downloads/releases.go index 14faebbf22..603d526f67 100644 --- a/pkg/downloads/releases.go +++ b/pkg/downloads/releases.go @@ -7,12 +7,13 @@ package downloads import ( "encoding/json" "fmt" + "io" + "net/http" "strings" "time" "github.com/Jeffail/gabs/v2" "github.com/cenkalti/backoff/v4" - "github.com/elastic/e2e-testing/internal/curl" "github.com/elastic/e2e-testing/internal/utils" log "github.com/sirupsen/logrus" ) @@ -20,6 +21,7 @@ import ( // DownloadURLResolver interface to resolve URLs for downloadable artifacts type DownloadURLResolver interface { Resolve() (url string, shaURL string, err error) + Kind() string } // ArtifactURLResolver type to resolve the URL of artifacts that are currently in development, from the artifacts API @@ -31,24 +33,29 @@ type ArtifactURLResolver struct { // NewArtifactURLResolver creates a new resolver for artifacts that are currently in development, from the artifacts API func NewArtifactURLResolver(fullName string, name string, version string) DownloadURLResolver { - // resolve version alias - resolvedVersion, err := GetElasticArtifactVersion(version) - if err != nil { - return nil - } - - fullName = strings.ReplaceAll(fullName, version, resolvedVersion) - return &ArtifactURLResolver{ FullName: fullName, Name: name, - Version: resolvedVersion, + Version: version, } } +func (r *ArtifactURLResolver) Kind() string { + return fmt.Sprintf("Unified snapshot resolver: %s", r.FullName) +} + // Resolve returns the URL of a released artifact, which its full name is defined in the first argument, // from Elastic's artifact repository, building the JSON path query based on the full name func (r *ArtifactURLResolver) Resolve() (string, string, error) { + resolvedVersion, err := GetElasticArtifactVersion(r.Version) + if err != nil { + return "", "", fmt.Errorf("failed to get version %s: %w", r.Version, err) + } + r.Version = resolvedVersion + + fullName := strings.ReplaceAll(r.FullName, r.Version, resolvedVersion) + r.FullName = fullName + artifactName := r.FullName artifact := r.Name version := r.Version @@ -57,13 +64,14 @@ func (r *ArtifactURLResolver) Resolve() (string, string, error) { retryCount := 1 - body := "" + body := []byte{} tmpVersion := version hasCommit := SnapshotHasCommit(version) if hasCommit { log.WithFields(log.Fields{ - "version": version, + "resolver": r.Kind(), + "version": version, }).Trace("Removing SNAPSHOT from version including commit") // remove the SNAPSHOT from the VERSION as the artifacts API supports commits in the version, but without the snapshot suffix @@ -71,45 +79,52 @@ func (r *ArtifactURLResolver) Resolve() (string, string, error) { } apiStatus := func() error { - r := curl.HTTPRequest{ - URL: fmt.Sprintf("https://artifacts-api.elastic.co/v1/search/%s/%s?x-elastic-no-kpi=true", tmpVersion, artifact), - } - - response, err := curl.Get(r) + url := fmt.Sprintf("https://artifacts-api.elastic.co/v1/search/%s/%s?x-elastic-no-kpi=true", tmpVersion, artifact) + resp, err := http.Get(url) if err != nil { log.WithFields(log.Fields{ + "kind": r.Kind(), "artifact": artifact, "artifactName": artifactName, "version": tmpVersion, "error": err, "retry": retryCount, - "statusEndpoint": r.URL, + "statusEndpoint": url, "elapsedTime": exp.GetElapsedTime(), - }).Warn("The Elastic artifacts API is not available yet") - + }).Warn("Resolver failed") retryCount++ return err } - log.WithFields(log.Fields{ - "retries": retryCount, - "statusEndpoint": r.URL, - "elapsedTime": exp.GetElapsedTime(), - }).Debug("The Elastic artifacts API is available") + defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + if err != nil { + return backoff.Permanent(err) + } + + if resp.StatusCode == http.StatusNotFound { + return backoff.Permanent(fmt.Errorf("not found for url %s", url)) + } - body = response return nil } - err := backoff.Retry(apiStatus, exp) + err = backoff.Retry(apiStatus, exp) if err != nil { + log.WithFields(log.Fields{ + "resolver": r.Kind(), + "artifact": artifact, + "artifactName": artifactName, + "version": tmpVersion, + }).Error("Failed to get artifact") return "", "", err } - jsonParsed, err := gabs.ParseJSON([]byte(body)) + jsonParsed, err := gabs.ParseJSON(body) if err != nil { log.WithFields(log.Fields{ + "resolver": r.Kind(), "artifact": artifact, "artifactName": artifactName, "version": tmpVersion, @@ -118,12 +133,13 @@ func (r *ArtifactURLResolver) Resolve() (string, string, error) { } log.WithFields(log.Fields{ + "resolver": r.Kind(), "retries": retryCount, "artifact": artifact, "artifactName": artifactName, "elapsedTime": exp.GetElapsedTime(), "version": tmpVersion, - }).Trace("Artifact found") + }).Trace("Resolver succeeded") if hasCommit { // remove commit from the artifact as it comes like this: elastic-agent-8.0.0-abcdef-SNAPSHOT-darwin-x86_64.tar.gz @@ -138,7 +154,7 @@ func (r *ArtifactURLResolver) Resolve() (string, string, error) { "artifact": artifact, "name": artifactName, "version": version, - }).Error("object not found in Artifact API") + }).Error("ArtifactURLResolver object not found in Artifact API") return "", "", fmt.Errorf("object not found in Artifact API") } @@ -176,19 +192,24 @@ func NewArtifactsSnapshot() *ArtifactsSnapshotVersion { // 1. Elastic's artifact repository, building the JSON path query based // If the version is a SNAPSHOT including a commit, then it will directly use the version without checking the artifacts API // i.e. GetSnapshotArtifactVersion("$VERSION-abcdef-SNAPSHOT") -func (as *ArtifactsSnapshotVersion) GetSnapshotArtifactVersion(version string) (string, error) { - cacheKey := fmt.Sprintf("%s/beats/latest/%s.json", as.Host, version) +func (as *ArtifactsSnapshotVersion) GetSnapshotArtifactVersion(project string, version string) (string, error) { + cacheKey := fmt.Sprintf("%s/%s/latest/%s.json", as.Host, project, version) - if val, ok := elasticVersionsCache[cacheKey]; ok { + elasticVersionsMutex.RLock() + val, ok := elasticVersionsCache[cacheKey] + elasticVersionsMutex.RUnlock() + if ok { log.WithFields(log.Fields{ "URL": cacheKey, "version": val, - }).Debug("Retrieving version from local cache") + }).Debug("ArtifactsSnapshotVersion Retrieving version from local cache") return val, nil } if SnapshotHasCommit(version) { + elasticVersionsMutex.Lock() elasticVersionsCache[cacheKey] = version + elasticVersionsMutex.Unlock() return version, nil } @@ -196,35 +217,35 @@ func (as *ArtifactsSnapshotVersion) GetSnapshotArtifactVersion(version string) ( retryCount := 1 - body := "" + body := []byte{} apiStatus := func() error { - r := curl.HTTPRequest{ - URL: cacheKey, - } - - response, err := curl.Get(r) + url := cacheKey + resp, err := http.Get(url) if err != nil { log.WithFields(log.Fields{ "version": version, "error": err, "retry": retryCount, - "statusEndpoint": r.URL, + "statusEndpoint": url, "elapsedTime": exp.GetElapsedTime(), - }).Warn("The Elastic artifacts API is not available yet") - + "resp": resp, + }).Warn("ArtifactsSnapshotVersion failed") retryCount++ return err } - log.WithFields(log.Fields{ - "retries": retryCount, - "statusEndpoint": r.URL, - "elapsedTime": exp.GetElapsedTime(), - }).Debug("The Elastic artifacts API is available") + defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + if err != nil { + return backoff.Permanent(err) + } + + if resp.StatusCode == http.StatusNotFound { + return backoff.Permanent(fmt.Errorf("not found for url %s", url)) + } - body = response return nil } @@ -240,13 +261,13 @@ func (as *ArtifactsSnapshotVersion) GetSnapshotArtifactVersion(version string) ( SummaryURL string `json:"summary_url"` // example value: https://artifacts-snapshot.elastic.co/beats/8.8.3-b1d8691a/summary-8.8.3-SNAPSHOT.html } response := ArtifactsSnapshotResponse{} - err = json.Unmarshal([]byte(body), &response) + err = json.Unmarshal(body, &response) if err != nil { log.WithFields(log.Fields{ "error": err, "version": version, "body": body, - }).Error("Could not parse the response body to retrieve the version") + }).Error("ArtifactsSnapshotVersion Could not parse the response body to retrieve the version") return "", fmt.Errorf("could not parse the response body to retrieve the version: %w", err) } @@ -255,7 +276,7 @@ func (as *ArtifactsSnapshotVersion) GetSnapshotArtifactVersion(version string) ( if (len(hashParts) < 2) || (hashParts[1] == "") { log.WithFields(log.Fields{ "buildId": response.BuildID, - }).Error("Could not parse the build_id to retrieve the version hash") + }).Error("ArtifactsSnapshotVersion Could not parse the build_id to retrieve the version hash") return "", fmt.Errorf("could not parse the build_id to retrieve the version hash: %s", response.BuildID) } hash := hashParts[1] @@ -266,41 +287,50 @@ func (as *ArtifactsSnapshotVersion) GetSnapshotArtifactVersion(version string) ( log.WithFields(log.Fields{ "alias": version, "version": latestVersion, - }).Debug("Latest version for current version obtained") + }).Debug("ArtifactsSnapshotVersion got latest version for current version") + elasticVersionsMutex.Lock() elasticVersionsCache[cacheKey] = latestVersion + elasticVersionsMutex.Unlock() return latestVersion, nil } -// NewArtifactURLResolver creates a new resolver for artifacts that are currently in development, from the artifacts API -func NewArtifactSnapshotURLResolver(fullName string, name string, version string) DownloadURLResolver { - return newCustomSnapshotURLResolver(fullName, name, version, "https://artifacts-snapshot.elastic.co") +// NewArtifactSnapshotURLResolver creates a new resolver for artifacts that are currently in development, from the artifacts API +func NewArtifactSnapshotURLResolver(fullName string, name string, project string, version string) DownloadURLResolver { + return newCustomSnapshotURLResolver(fullName, name, project, version, "https://artifacts-snapshot.elastic.co") } // For testing purposes -func newCustomSnapshotURLResolver(fullName string, name string, version string, host string) DownloadURLResolver { +func newCustomSnapshotURLResolver(fullName string, name string, project string, version string, host string) DownloadURLResolver { // resolve version alias - resolvedVersion, err := newArtifactsSnapshotCustom(host).GetSnapshotArtifactVersion(version) + resolvedVersion, err := newArtifactsSnapshotCustom(host).GetSnapshotArtifactVersion(project, version) if err != nil { return nil } return &ArtifactsSnapshotURLResolver{ FullName: fullName, Name: name, + Project: project, Version: resolvedVersion, SnapshotApiHost: host, } } // ArtifactsSnapshotURLResolver type to resolve the URL of artifacts that are currently in development, from the artifacts API +// Takes the artifacts staged for inclusion in the next unified snapshot, before one is available. type ArtifactsSnapshotURLResolver struct { FullName string Name string Version string + Project string SnapshotApiHost string } +func (r *ArtifactsSnapshotURLResolver) Kind() string { + return fmt.Sprintf("Project snapshot resolver: %s", r.FullName) +} + func (asur *ArtifactsSnapshotURLResolver) Resolve() (string, string, error) { artifactName := asur.FullName artifact := asur.Name @@ -311,8 +341,9 @@ func (asur *ArtifactsSnapshotURLResolver) Resolve() (string, string, error) { log.WithFields(log.Fields{ "artifact": artifact, "artifactName": artifactName, + "project": asur.Project, "version": version, - }).Info("The version does not contain a commit hash, it is not a snapshot") + }).Info("ArtifactsSnapshotURLResolver version does not contain a commit hash, it is not a snapshot") return "", "", err } @@ -320,38 +351,39 @@ func (asur *ArtifactsSnapshotURLResolver) Resolve() (string, string, error) { retryCount := 1 - body := "" + body := []byte{} apiStatus := func() error { - r := curl.HTTPRequest{ - // https://artifacts-snapshot.elastic.co/beats/8.9.0-d1b14479/manifest-8.9.0-SNAPSHOT.json - URL: fmt.Sprintf("%s/beats/%s-%s/manifest-%s-SNAPSHOT.json", asur.SnapshotApiHost, semVer, commit, semVer), - } - - response, err := curl.Get(r) + // https://artifacts-snapshot.elastic.co/beats/8.9.0-d1b14479/manifest-8.9.0-SNAPSHOT.json + url := fmt.Sprintf("%s/%s/%s-%s/manifest-%s-SNAPSHOT.json", asur.SnapshotApiHost, asur.Project, semVer, commit, semVer) + resp, err := http.Get(url) if err != nil { log.WithFields(log.Fields{ + "kind": asur.Kind(), "artifact": artifact, "artifactName": artifactName, "version": version, "error": err, "retry": retryCount, - "statusEndpoint": r.URL, + "statusEndpoint": url, "elapsedTime": exp.GetElapsedTime(), - }).Warn("The Elastic artifacts API is not available yet") - + "resp": resp, + }).Warn("resolver failed") retryCount++ return err } - log.WithFields(log.Fields{ - "retries": retryCount, - "statusEndpoint": r.URL, - "elapsedTime": exp.GetElapsedTime(), - }).Debug("The Elastic artifacts API is available") + defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + if err != nil { + return backoff.Permanent(err) + } + + if resp.StatusCode == http.StatusNotFound { + return backoff.Permanent(fmt.Errorf("not found for url %s", url)) + } - body = response return nil } @@ -361,11 +393,13 @@ func (asur *ArtifactsSnapshotURLResolver) Resolve() (string, string, error) { } var jsonParsed map[string]interface{} - err = json.Unmarshal([]byte(body), &jsonParsed) + err = json.Unmarshal(body, &jsonParsed) if err != nil { log.WithFields(log.Fields{ + "kind": asur.Kind(), "artifact": artifact, "artifactName": artifactName, + "project": asur.Project, "version": version, }).Error("Could not parse the response body for the artifact") return "", "", err @@ -377,12 +411,14 @@ func (asur *ArtifactsSnapshotURLResolver) Resolve() (string, string, error) { } log.WithFields(log.Fields{ + "kind": asur.Kind(), "retries": retryCount, "artifact": artifact, "artifactName": artifactName, "elapsedTime": exp.GetElapsedTime(), + "project": asur.Project, "version": version, - }).Trace("Artifact found") + }).Trace("Resolver succeeded") return url, shaURL, nil } @@ -427,6 +463,10 @@ func NewReleaseURLResolver(project string, fullName string, name string) *Releas } } +func (r *ReleaseURLResolver) Kind() string { + return fmt.Sprintf("Official release resolver: %s", r.FullName) +} + // Resolve resolves the URL of a download, which is located in the Elastic. It will use a HEAD request // and if it returns a 200 OK it will return the URL of both file and its SHA512 file func (r *ReleaseURLResolver) Resolve() (string, string, error) { @@ -438,40 +478,34 @@ func (r *ReleaseURLResolver) Resolve() (string, string, error) { found := false apiStatus := func() error { - r := curl.HTTPRequest{ - URL: url, - } - - _, err := curl.Head(r) + resp, err := http.Head(url) if err != nil { - if strings.EqualFold(err.Error(), "HEAD request failed with 404") { - log.WithFields(log.Fields{ - "resolver": r, - "error": err, - "retry": retryCount, - "statusEndpoint": r.URL, - "elapsedTime": exp.GetElapsedTime(), - }).Debug("Download could not be found at the Elastic downloads API") - return nil - } - log.WithFields(log.Fields{ - "resolver": r, + "kind": r.Kind(), "error": err, "retry": retryCount, - "statusEndpoint": r.URL, + "statusEndpoint": url, "elapsedTime": exp.GetElapsedTime(), - }).Debug("The Elastic downloads API is not available yet") + "resp": resp, + }).Debug("Resolver failed") retryCount++ return err } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + + if resp.StatusCode == http.StatusNotFound { + return backoff.Permanent(fmt.Errorf("not found for url %s", url)) + } + found = true log.WithFields(log.Fields{ + "kind": r.Kind(), "retries": retryCount, - "statusEndpoint": r.URL, + "statusEndpoint": url, "elapsedTime": exp.GetElapsedTime(), }).Info("Download was found in the Elastic downloads API") diff --git a/pkg/downloads/releases_test.go b/pkg/downloads/releases_test.go index 17e0abc963..a0efff2c15 100644 --- a/pkg/downloads/releases_test.go +++ b/pkg/downloads/releases_test.go @@ -33,7 +33,7 @@ func TestGetSnapshotArtifactVersion(t *testing.T) { mockURL := mockServer.URL + "/beats/latest/8.8.3-SNAPSHOT.json" artifactsSnapshot := newArtifactsSnapshotCustom(mockURL) - version, err := artifactsSnapshot.GetSnapshotArtifactVersion("8.8.3-SNAPSHOT") + version, err := artifactsSnapshot.GetSnapshotArtifactVersion("beats", "8.8.3-SNAPSHOT") assert.NoError(t, err, "Expected no error") assert.Equal(t, "8.8.3-b1d8691a-SNAPSHOT", version, "Expected version to match") }) @@ -54,7 +54,7 @@ func TestGetSnapshotArtifactVersion(t *testing.T) { mockURL := mockServer.URL + "/beats/latest/8.8.3-SNAPSHOT.json" artifactsSnapshot := newArtifactsSnapshotCustom(mockURL) - version, err := artifactsSnapshot.GetSnapshotArtifactVersion("8.8.3-SNAPSHOT") + version, err := artifactsSnapshot.GetSnapshotArtifactVersion("beats", "8.8.3-SNAPSHOT") assert.ErrorContains(t, err, "could not parse the response body") assert.Empty(t, version) }) @@ -75,7 +75,7 @@ func TestGetSnapshotArtifactVersion(t *testing.T) { mockURL := mockServer.URL + "/beats/latest/8.8.3-SNAPSHOT.json" artifactsSnapshot := newArtifactsSnapshotCustom(mockURL) - version, err := artifactsSnapshot.GetSnapshotArtifactVersion("8.8.3-SNAPSHOT") + version, err := artifactsSnapshot.GetSnapshotArtifactVersion("beats", "8.8.3-SNAPSHOT") assert.ErrorContains(t, err, "could not parse the build_id") assert.ErrorContains(t, err, "bd8691a") assert.Empty(t, version) @@ -121,7 +121,7 @@ func TestArtifactsSnapshotResolver(t *testing.T) { server := httptest.NewServer(mockHandler) defer server.Close() - urlResolver := newCustomSnapshotURLResolver("auditbeat-8.9.0-SNAPSHOT-amd64.deb", "auditbeat", "8.9.0-SNAPSHOT", server.URL) + urlResolver := newCustomSnapshotURLResolver("auditbeat-8.9.0-SNAPSHOT-amd64.deb", "auditbeat", "beats", "8.9.0-SNAPSHOT", server.URL) url, shaUrl, err := urlResolver.Resolve() assert.Equal(t, "https://artifacts-snapshot.elastic.co/auditbeat-8.9.0-SNAPSHOT-amd64.deb", url) assert.Equal(t, "https://artifacts-snapshot.elastic.co/auditbeat-8.9.0-SNAPSHOT-amd64.deb.sha512", shaUrl) diff --git a/pkg/downloads/versions.go b/pkg/downloads/versions.go index 88fe3388d5..61ab2a1ef4 100644 --- a/pkg/downloads/versions.go +++ b/pkg/downloads/versions.go @@ -8,11 +8,14 @@ import ( "context" "errors" "fmt" + "io" + "net/http" "os" "path" "path/filepath" "regexp" "strings" + "sync" "time" "github.com/Jeffail/gabs/v2" @@ -34,11 +37,13 @@ var BeatsLocalPath = "" // the URL of the artifact. If another installer is trying to download the same URL, it will return the location of the // already downloaded artifact. var binariesCache = map[string]string{} +var binariesMutex sync.RWMutex // to avoid fetching the same Elastic artifacts version, we are adding this map to cache the version of the Elastic artifacts, // using as key the URL of the version. If another request is trying to fetch the same URL, it will return the string version // of the already requested one. var elasticVersionsCache = map[string]string{} +var elasticVersionsMutex sync.RWMutex // GithubCommitSha1 represents the value of the "GITHUB_CHECK_SHA1" environment variable var GithubCommitSha1 string @@ -159,7 +164,10 @@ func GetElasticArtifactURL(artifactName string, artifact string, version string) func GetElasticArtifactVersion(version string) (string, error) { cacheKey := fmt.Sprintf("https://artifacts-api.elastic.co/v1/versions/%s/?x-elastic-no-kpi=true", version) - if val, ok := elasticVersionsCache[cacheKey]; ok { + elasticVersionsMutex.RLock() + val, ok := elasticVersionsCache[cacheKey] + elasticVersionsMutex.RUnlock() + if ok { log.WithFields(log.Fields{ "URL": cacheKey, "version": val, @@ -168,43 +176,32 @@ func GetElasticArtifactVersion(version string) (string, error) { } if SnapshotHasCommit(version) { + elasticVersionsMutex.Lock() elasticVersionsCache[cacheKey] = version + elasticVersionsMutex.Unlock() return version, nil } exp := utils.GetExponentialBackOff(time.Minute) - retryCount := 1 - - body := "" + body := []byte{} apiStatus := func() error { - r := curl.HTTPRequest{ - URL: cacheKey, - } - - response, err := curl.Get(r) + url := cacheKey + resp, err := http.Get(url) if err != nil { - log.WithFields(log.Fields{ - "version": version, - "error": err, - "retry": retryCount, - "statusEndpoint": r.URL, - "elapsedTime": exp.GetElapsedTime(), - }).Warn("The Elastic artifacts API is not available yet") - - retryCount++ - - return err + return fmt.Errorf("error getting %s: %w", url, err) } - log.WithFields(log.Fields{ - "retries": retryCount, - "statusEndpoint": r.URL, - "elapsedTime": exp.GetElapsedTime(), - }).Debug("The Elastic artifacts API is available") + if resp.StatusCode == http.StatusNotFound { + return backoff.Permanent(fmt.Errorf("version %s not found at %s", version, url)) + } - body = response + defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + if err != nil { + return backoff.Permanent(err) + } return nil } @@ -213,13 +210,9 @@ func GetElasticArtifactVersion(version string) (string, error) { return "", err } - jsonParsed, err := gabs.ParseJSON([]byte(body)) + jsonParsed, err := gabs.ParseJSON(body) if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": version, - }).Error("Could not parse the response body to retrieve the version") - return "", err + return "", fmt.Errorf("parsing JSON body %s: %w", body, err) } builds := jsonParsed.Path("version.builds") @@ -232,7 +225,9 @@ func GetElasticArtifactVersion(version string) (string, error) { "version": latestVersion, }).Debug("Latest version for current version obtained") + elasticVersionsMutex.Lock() elasticVersionsCache[cacheKey] = latestVersion + elasticVersionsMutex.Unlock() return latestVersion, nil } @@ -383,7 +378,10 @@ func FetchProjectBinaryForSnapshots(ctx context.Context, useCISnapshots bool, pr span.Context.SetLabel("project", project) defer span.End() - if val, ok := binariesCache[URL]; ok { + binariesMutex.RLock() + val, ok := binariesCache[URL] + binariesMutex.RUnlock() + if ok { log.WithFields(log.Fields{ "URL": URL, "path": val, @@ -410,7 +408,9 @@ func FetchProjectBinaryForSnapshots(ctx context.Context, useCISnapshots bool, pr sanitizedFilePath = downloadRequest.UnsanitizedFilePath } + binariesMutex.Lock() binariesCache[URL] = sanitizedFilePath + binariesMutex.Unlock() return sanitizedFilePath, nil } @@ -480,12 +480,13 @@ func FetchProjectBinaryForSnapshots(ctx context.Context, useCISnapshots bool, pr downloadURLResolvers := []DownloadURLResolver{ NewReleaseURLResolver(elasticAgentNamespace, artifactName, artifact), NewArtifactURLResolver(artifactName, artifact, version), - NewArtifactSnapshotURLResolver(artifactName, artifact, version), + NewArtifactSnapshotURLResolver(artifactName, artifact, project, version), } downloadURL, downloadShaURL, err = getDownloadURLFromResolvers(downloadURLResolvers) if err != nil { return "", err } + fmt.Printf("Downloading from %s\n", downloadURL) downloadLocation, err := handleDownload(downloadURL) if err != nil { return "", err @@ -513,15 +514,14 @@ func getDownloadURLFromResolvers(resolvers []DownloadURLResolver) (string, strin continue } + log.WithFields(log.Fields{"kind": resolver.Kind()}).Info("Trying resolver.") url, shaURL, err := resolver.Resolve() if err != nil { if i < len(resolvers)-1 { - log.WithFields(log.Fields{ - "resolver": resolver, - }).Warn("Object not found. Trying with another download resolver") + log.WithFields(log.Fields{"kind": resolver.Kind()}).Warn("Object not found.") continue } else { - log.Error("Object not found. There is no other download resolver") + log.WithFields(log.Fields{"kind": resolver.Kind()}).Error("Object not found. All resolvers failed") return "", "", err } }