Merge branch 'main' into yang.song/otlp-apm-stats

.github/CODEOWNERS

@@ -176,6 +176,7 @@
 /cmd/otel-agent/                        @DataDog/opentelemetry
 /cmd/process-agent/                     @DataDog/processes
 /cmd/serverless/                        @DataDog/serverless
+/cmd/serverless/dependencies*.txt       @DataDog/serverless @DataDog/agent-shared-components
 /cmd/serverless-init/                   @DataDog/serverless
 /cmd/system-probe/                      @DataDog/ebpf-platform
 /cmd/system-probe/config/adjust_npm.go  @DataDog/ebpf-platform @DataDog/Networks
@@ -510,6 +511,7 @@
 /tasks/                                 @DataDog/agent-developer-tools @DataDog/agent-ci-experience
 /tasks/msi.py                           @DataDog/windows-agent
 /tasks/agent.py                         @DataDog/agent-shared-components
+/tasks/go_deps.py                       @DataDog/agent-shared-components
 /tasks/dogstatsd.py                     @DataDog/agent-metrics-logs
 /tasks/update_go.py                     @DataDog/agent-shared-components
 /tasks/unit-tests/update_go_tests.py    @DataDog/agent-shared-components

.gitlab-ci.yml

@@ -175,9 +175,9 @@ variables:
   DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v34815905-ae40295e
 
   DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded
-  DEB_GPG_KEY_ID: ad9589b7
-  DEB_GPG_KEY_NAME: "Datadog, Inc. Master key"
-  RPM_GPG_KEY_ID: fd4bf915
+  DEB_GPG_KEY_ID: c0962c7d
+  DEB_GPG_KEY_NAME: "Datadog, Inc. APT key"
+  RPM_GPG_KEY_ID: b01082d3
   RPM_GPG_KEY_NAME: "Datadog, Inc. RPM key"
   DOCKER_REGISTRY_URL: docker.io
   KITCHEN_INFRASTRUCTURE_FLAKES_RETRY: 2
@@ -205,8 +205,8 @@ variables:
   CHANGELOG_COMMIT_SHA_SSM_NAME: ci.datadog-agent.gitlab_changelog_commit_sha  # agent-ci-experience
   CHOCOLATEY_API_KEY_SSM_NAME: ci.datadog-agent.chocolatey_api_key  # windows-agent
   CODECOV_TOKEN_SSM_NAME: ci.datadog-agent.codecov_token  # agent-ci-experience
-  DEB_GPG_KEY_SSM_NAME: ci.datadog-agent.deb_signing_private_key_v2_${DEB_GPG_KEY_ID}  # agent-build-and-releases
-  DEB_SIGNING_PASSPHRASE_SSM_NAME: ci.datadog-agent.deb_signing_key_passphrase_v2_${DEB_GPG_KEY_ID}  # agent-build-and-releases
+  DEB_GPG_KEY_SSM_NAME: ci.datadog-agent.deb_signing_private_key_${DEB_GPG_KEY_ID}  # agent-build-and-releases
+  DEB_SIGNING_PASSPHRASE_SSM_NAME: ci.datadog-agent.deb_signing_key_passphrase_${DEB_GPG_KEY_ID}  # agent-build-and-releases
   DOCKER_REGISTRY_LOGIN_SSM_KEY: ci.datadog-agent.docker_hub_login  # agent-ci-experience
   DOCKER_REGISTRY_PWD_SSM_KEY: ci.datadog-agent.docker_hub_pwd  # agent-ci-experience
   E2E_TESTS_API_KEY_SSM_NAME: ci.datadog-agent.e2e_tests_api_key  # agent-developer-tools
@@ -227,8 +227,8 @@ variables:
   MACOS_GITHUB_APP_ID_2_SSM_NAME: ci.datadog-agent.macos_github_app_id_2  # agent-ci-experience
   MACOS_GITHUB_INSTALLATION_ID_2_SSM_NAME: ci.datadog-agent.macos_github_installation_id_2  # agent-ci-experience
   MACOS_GITHUB_KEY_2_SSM_NAME: ci.datadog-agent.macos_github_key_b64_2  # agent-ci-experience
-  RPM_GPG_KEY_SSM_NAME: ci.datadog-agent.rpm_signing_private_key_v2_${RPM_GPG_KEY_ID}  # agent-build-and-releases
-  RPM_SIGNING_PASSPHRASE_SSM_NAME: ci.datadog-agent.rpm_signing_key_passphrase_v2_${RPM_GPG_KEY_ID}  # agent-build-and-releases
+  RPM_GPG_KEY_SSM_NAME: ci.datadog-agent.rpm_signing_private_key_${RPM_GPG_KEY_ID}  # agent-build-and-releases
+  RPM_SIGNING_PASSPHRASE_SSM_NAME: ci.datadog-agent.rpm_signing_key_passphrase_${RPM_GPG_KEY_ID}  # agent-build-and-releases
   SMP_ACCOUNT_ID_SSM_NAME: ci.datadog-agent.single-machine-performance-account-id  # single-machine-performance
   SMP_AGENT_TEAM_ID_SSM_NAME: ci.datadog-agent.single-machine-performance-agent-team-id  # single-machine-performance
   SMP_API_SSM_NAME: ci.datadog-agent.single-machine-performance-api  # single-machine-performance

.gitlab/JOBOWNERS

@@ -13,8 +13,10 @@ security_go_generate_check           @DataDog/agent-security
 prepare_sysprobe_ebpf_functional_tests* @DataDog/ebpf-platform
 prepare_secagent_ebpf_functional_tests* @DataDog/agent-security
 
-# Golang dependency list generation (currently disabled)
-golang_deps_generate                 @DataDog/agent-shared-components
+# Send count metrics about Golang dependencies
+golang_deps_send_count_metrics       @DataDog/agent-shared-components
+# Golang test dependecies diff
+golang_deps_test                     @DataDog/agent-shared-components
 # Golang dependency diff generation
 golang_deps_diff                     @DataDog/ebpf-platform
 golang_deps_commenter                @DataDog/ebpf-platform

.gitlab/kernel_matrix_testing/common.yml

@@ -160,6 +160,17 @@
 
 # -- Test runners
 .kmt_run_tests:
+  retry:
+    max: 2
+    when:
+      - job_execution_timeout
+      - runner_system_failure
+      - stuck_or_timeout_failure
+      - unknown_failure
+      - api_failure
+      - scheduler_failure
+      - stale_schedule
+      - data_integrity_failure
   variables:
     AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
     RETRY: 2

.gitlab/kernel_matrix_testing/security_agent.yml

@@ -105,6 +105,7 @@ upload_secagent_tests_arm64:
   allow_failure: true
   stage: kernel_matrix_testing_security_agent
   rules: !reference [.on_security_agent_changes_or_manual]
+  timeout: 1h 30m
   variables:
     TEST_COMPONENT: security-agent
 
@@ -117,7 +118,6 @@ kmt_run_secagent_tests_x64:
     - kmt_setup_env_secagent_x64
     - upload_dependencies_secagent_x64
     - upload_secagent_tests_x64
-  timeout: 3h
   variables:
     ARCH: "x86_64"
   parallel:
@@ -158,7 +158,6 @@ kmt_run_secagent_tests_arm64:
     - kmt_setup_env_secagent_arm64
     - upload_dependencies_secagent_arm64
     - upload_secagent_tests_arm64
-  timeout: 3h
   variables:
     ARCH: "arm64"
   parallel:

.gitlab/kernel_matrix_testing/system_probe.yml

@@ -186,6 +186,7 @@ upload_sysprobe_tests_arm64:
   extends: .kmt_run_tests
   stage: kernel_matrix_testing_system_probe
   rules: !reference [.on_system_probe_or_e2e_changes_or_manual]
+  timeout: 1h 30m
   variables:
     TEST_COMPONENT: system-probe
 
@@ -199,7 +200,6 @@ kmt_run_sysprobe_tests_x64:
     - upload_dependencies_sysprobe_x64
     - upload_sysprobe_tests_x64
     - upload_minimized_btfs_sysprobe_x64
-  timeout: 3h
   variables:
     ARCH: "x86_64"
   parallel:
@@ -236,7 +236,6 @@ kmt_run_sysprobe_tests_arm64:
     - upload_dependencies_sysprobe_arm64
     - upload_sysprobe_tests_arm64
     - upload_minimized_btfs_sysprobe_arm64
-  timeout: 3h
   variables:
     ARCH: "arm64"
   parallel:

.gitlab/package_build/deb.yml

@@ -142,6 +142,7 @@ agent_deb-arm64-a7:
     - !reference [.setup_ruby_mirror_linux]
     - !reference [.setup_python_mirror_linux]
     - !reference [.retrieve_linux_go_deps]
+    - !reference [.cache_omnibus_ruby_deps, setup]
     - echo "About to build for $RELEASE_VERSION_7"
     - echo "Detected host architecture $(uname -m)"
     # $DD_TARGET_ARCH is only set by Arm build images, so assume amd64 if not present
@@ -165,6 +166,8 @@ agent_deb-arm64-a7:
     expire_in: 2 weeks
     paths:
       - $OMNIBUS_PACKAGE_DIR
+  cache:
+    - !reference [.cache_omnibus_ruby_deps, cache]
 
 iot_agent_deb-x64:
   extends: .iot_agent_build_common_deb
@@ -224,6 +227,7 @@ dogstatsd_deb-x64:
   before_script:
     - source /root/.bashrc
     - !reference [.retrieve_linux_go_deps]
+    - !reference [.cache_omnibus_ruby_deps, setup]
   script:
     # remove artifacts from previous pipelines that may come from the cache
     - rm -rf $OMNIBUS_PACKAGE_DIR/*
@@ -241,6 +245,8 @@ dogstatsd_deb-x64:
     expire_in: 2 weeks
     paths:
       - $OMNIBUS_PACKAGE_DIR
+  cache:
+    - !reference [.cache_omnibus_ruby_deps, cache]
 
 dogstatsd_deb-arm64:
   rules:

.gitlab/package_build/installer.yml

@@ -92,6 +92,7 @@ datadog-agent-oci-arm64-a7:
     - source /root/.bashrc
     - !reference [.setup_ruby_mirror_linux]
     - !reference [.retrieve_linux_go_deps]
+    - !reference [.cache_omnibus_ruby_deps, setup]
     - echo "About to build for $RELEASE_VERSION"
     # remove artifacts from previous pipelines that may come from the cache
     - rm -rf $OMNIBUS_PACKAGE_DIR/*
@@ -107,6 +108,8 @@ datadog-agent-oci-arm64-a7:
     expire_in: 2 weeks
     paths:
       - $OMNIBUS_PACKAGE_DIR
+  cache:
+    - !reference [.cache_omnibus_ruby_deps, cache]
 
 # We build a "regular" installer, meant to be packaged as deb/rpm, without a custom install path
 # and an artifact intended for OCI packaging, which has a custom install path

.gitlab/package_build/rpm.yml

@@ -144,6 +144,7 @@ agent_rpm-arm64-a7:
     - !reference [.setup_ruby_mirror_linux]
     - !reference [.setup_python_mirror_linux]
     - !reference [.retrieve_linux_go_deps]
+    - !reference [.cache_omnibus_ruby_deps, setup]
     - echo "Detected host architecture $(uname -m)"
     # $DD_TARGET_ARCH is only set by Arm build images, so assume amd64 if not present
     - echo "Target architecture ${DD_TARGET_ARCH:=amd64}"
@@ -167,6 +168,8 @@ agent_rpm-arm64-a7:
     expire_in: 2 weeks
     paths:
       - $OMNIBUS_PACKAGE_DIR
+  cache:
+    - !reference [.cache_omnibus_ruby_deps, cache]
 
 iot_agent_rpm-x64:
   extends: .iot_agent_build_common_rpm
@@ -223,6 +226,7 @@ dogstatsd_rpm-x64:
   before_script:
     - source /root/.bashrc
     - !reference [.retrieve_linux_go_deps]
+    - !reference [.cache_omnibus_ruby_deps, setup]
   script:
     # remove artifacts from previous pipelines that may come from the cache
     - rm -rf $OMNIBUS_PACKAGE_DIR/*
@@ -242,3 +246,5 @@ dogstatsd_rpm-x64:
     expire_in: 2 weeks
     paths:
       - $OMNIBUS_PACKAGE_DIR
+  cache:
+    - !reference [.cache_omnibus_ruby_deps, cache]

.gitlab/source_test/golang_deps_diff.yml

@@ -47,3 +47,31 @@ golang_deps_commenter:
         exit 0
       fi
       exit $exitcode
+
+golang_deps_send_count_metrics:
+  stage: source_test
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+  rules:
+    - when: on_success
+  needs: ["go_deps"]
+  before_script:
+    - source /root/.bashrc
+    - !reference [.retrieve_linux_go_deps]
+  script:
+    # Get API key to send metrics
+    - export DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $API_KEY_ORG2_SSM_NAME)
+    - inv -e go-deps.send-count-metrics --git-sha "${CI_COMMIT_SHA}" --git-ref "${CI_COMMIT_REF_NAME}"
+
+golang_deps_test:
+  stage: source_test
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+  rules:
+    - when: on_success
+  needs: ["go_deps"]
+  before_script:
+    - source /root/.bashrc
+    - !reference [.retrieve_linux_go_deps]
+  script:
+    - inv -e go-deps.test-list

cmd/agent/subcommands/dogstatsdreplay/command.go

@@ -24,7 +24,7 @@ import (
 	"github.com/DataDog/datadog-agent/comp/core"
 	"github.com/DataDog/datadog-agent/comp/core/config"
 	"github.com/DataDog/datadog-agent/comp/core/log"
-	"github.com/DataDog/datadog-agent/comp/dogstatsd/replay"
+	replay "github.com/DataDog/datadog-agent/comp/dogstatsd/replay/impl"
 	"github.com/DataDog/datadog-agent/pkg/api/security"
 	pkgconfig "github.com/DataDog/datadog-agent/pkg/config"
 	pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/core"

cmd/agent/subcommands/jmx/command.go

@@ -51,7 +51,7 @@ import (
 	"github.com/DataDog/datadog-agent/comp/core/workloadmeta"
 	"github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors"
 	"github.com/DataDog/datadog-agent/comp/dogstatsd/pidmap"
-	"github.com/DataDog/datadog-agent/comp/dogstatsd/replay"
+	replay "github.com/DataDog/datadog-agent/comp/dogstatsd/replay/def"
 	dogstatsdServer "github.com/DataDog/datadog-agent/comp/dogstatsd/server"
 	serverdebug "github.com/DataDog/datadog-agent/comp/dogstatsd/serverDebug"
 	"github.com/DataDog/datadog-agent/comp/forwarder/eventplatformreceiver"

cmd/agent/subcommands/run/command.go

@@ -41,6 +41,7 @@ import (
 	"github.com/DataDog/datadog-agent/comp/agent/jmxlogger/jmxloggerimpl"
 	"github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer"
 	"github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl"
+	demultiplexerendpointfx "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexerendpoint/fx"
 	internalAPI "github.com/DataDog/datadog-agent/comp/api/api"
 	"github.com/DataDog/datadog-agent/comp/api/api/apiimpl"
 	authtokenimpl "github.com/DataDog/datadog-agent/comp/api/authtoken/createandfetchimpl"
@@ -78,7 +79,7 @@ import (
 	"github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors"
 	"github.com/DataDog/datadog-agent/comp/core/workloadmeta/defaults"
 	"github.com/DataDog/datadog-agent/comp/dogstatsd"
-	"github.com/DataDog/datadog-agent/comp/dogstatsd/replay"
+	replay "github.com/DataDog/datadog-agent/comp/dogstatsd/replay/def"
 	dogstatsdServer "github.com/DataDog/datadog-agent/comp/dogstatsd/server"
 	dogstatsddebug "github.com/DataDog/datadog-agent/comp/dogstatsd/serverDebug"
 	dogstatsdStatusimpl "github.com/DataDog/datadog-agent/comp/dogstatsd/status/statusimpl"
@@ -368,6 +369,7 @@ func getSharedFxOption() fx.Option {
 		apiimpl.Module(),
 		compressionimpl.Module(),
 		demultiplexerimpl.Module(),
+		demultiplexerendpointfx.Module(),
 		dogstatsd.Bundle(),
 		fx.Provide(func(logsagent optional.Option[logsAgent.Component]) optional.Option[logsagentpipeline.Component] {
 			if la, ok := logsagent.Get(); ok {

cmd/agent/subcommands/run/command_windows.go

@@ -57,7 +57,7 @@ import (
 	"github.com/DataDog/datadog-agent/comp/core/tagger"
 	"github.com/DataDog/datadog-agent/comp/core/telemetry"
 	"github.com/DataDog/datadog-agent/comp/core/workloadmeta"
-	"github.com/DataDog/datadog-agent/comp/dogstatsd/replay"
+	replay "github.com/DataDog/datadog-agent/comp/dogstatsd/replay/def"
 	dogstatsdServer "github.com/DataDog/datadog-agent/comp/dogstatsd/server"
 	"github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder"
 	logsAgent "github.com/DataDog/datadog-agent/comp/logs/agent"

cmd/cluster-agent/subcommands/start/command.go

@@ -399,13 +399,20 @@ func start(log log.Component,
 	}
 
 	// Autoscaling Product
+	var pa workload.PatcherAdapter
 	if config.GetBool("autoscaling.workload.enabled") {
 		if rcClient == nil {
 			return fmt.Errorf("Remote config is disabled or failed to initialize, remote config is a required dependency for autoscaling")
 		}
 
-		if err := workload.StartWorkloadAutoscaling(mainCtx, apiCl, rcClient); err != nil {
+		if !config.GetBool("admission_controller.enabled") {
+			log.Error("Admission controller is disabled, vertical autoscaling requires the admission controller to be enabled. Vertical scaling will be disabled.")
+		}
+
+		if adapter, err := workload.StartWorkloadAutoscaling(mainCtx, apiCl, rcClient); err != nil {
 			pkglog.Errorf("Error while starting workload autoscaling: %v", err)
+		} else {
+			pa = adapter
 		}
 	}
 
@@ -454,7 +461,7 @@ func start(log log.Component,
 			StopCh:              stopCh,
 		}
 
-		webhooks, err := admissionpkg.StartControllers(admissionCtx, wmeta)
+		webhooks, err := admissionpkg.StartControllers(admissionCtx, wmeta, pa)
 		if err != nil {
 			pkglog.Errorf("Could not start admission controller: %v", err)
 		} else {

cmd/cws-instrumentation/subcommands/setupcmd/setup.go

@@ -68,7 +68,10 @@ func setupCWSInjector(params *setupCliParams) error {
 	targetPath := filepath.Join(params.cwsVolumeMount, filepath.Base(path))
 	target, err := os.Create(targetPath)
 	if err != nil {
-		return fmt.Errorf("couldn't create target cws-instrumentation binary file in the mounted volume")
+		if os.IsPermission(err) && os.Getuid() != 0 {
+			return fmt.Errorf("couldn't copy cws-instrumentation binary file in the mounted volume: %v. Current UID: %d, you may want to use user 0 instead", err, os.Getuid())
+		}
+		return fmt.Errorf("couldn't copy cws-instrumentation binary file in the mounted volume: %v", err)
 	}
 	defer target.Close()