Merge branch 'main' of ssh://github.com/DataDog/datadog-agent into le…

…naic/CONTINT-4412

.gitlab-ci.yml

@@ -305,6 +305,7 @@ variables:
   MACOS_GITHUB_APP_1: macos-github-app-one  # agent-devx-infra
   MACOS_GITHUB_APP_2: macos-github-app-two  # agent-devx-infra
   SLACK_AGENT: slack-agent-ci  # agent-devx-infra
+  SMP_ACCOUNT: smp  # single-machine-performance
   # End vault variables
 
   DD_PKG_VERSION: "latest"

.gitlab/common/macos.yml

@@ -19,14 +19,19 @@
 
 .select_python_env_commands:
   # Select the virtualenv using the current Python version. Create it if it doesn't exist.
+  - PYTHON_VERSION=$(python3 --version | awk '{print $2}')
+  - VENV_NAME="datadog-agent-python-$PYTHON_VERSION"
+  - VENV_PATH="$(pyenv root)/versions/$VENV_NAME"
+  - echo "Using Python $PYTHON_VERSION..."
   - |
-    PYTHON_VERSION=$(python3 --version | awk '{print $2}')
-    VENV_NAME="datadog-agent-python-$PYTHON_VERSION"
-    echo "Using Python $PYTHON_VERSION..."
-    if ! pyenv virtualenvs --bare | grep -q "${VENV_NAME}$"; then
-      pyenv virtualenv $PYTHON_VERSION $VENV_NAME
+    # Check if the virtual environment directory exists
+    if [ ! -d "$VENV_PATH" ]; then
+      echo "Creating virtual environment '$VENV_NAME'..."
+      pyenv virtualenv "$PYTHON_VERSION" "$VENV_NAME"
+    else
+      echo "Virtual environment '$VENV_NAME' already exists. Skipping creation."
     fi
-    pyenv activate $VENV_NAME
+  - pyenv activate $VENV_NAME
 
 .macos_gitlab:
   before_script:

.gitlab/functional_test/regression_detector.yml

@@ -20,7 +20,7 @@ single-machine-performance-regression_detector:
       - outputs/junit.xml # for debugging, also on S3
     when: always
   variables:
-    SMP_VERSION: 0.16.0
+    SMP_VERSION: 0.18.1
   # At present we require two artifacts to exist for the 'baseline' and the
   # 'comparison'. We are guaranteed by the structure of the pipeline that
   # 'comparison' exists, not so much with 'baseline' as it has to come from main
@@ -35,7 +35,6 @@ single-machine-performance-regression_detector:
   script:
     # Ensure output files exist for artifact downloads step
     - mkdir outputs # Also needed for smp job sync step
-    - touch outputs/report.md # Will be emitted by smp job sync
     # Compute merge base of current commit and `main`
     - git fetch origin
     - SMP_BASE_BRANCH=$(inv release.get-release-json-value base_branch)
@@ -44,12 +43,12 @@ single-machine-performance-regression_detector:
     - echo "Merge base is ${SMP_MERGE_BASE}"
     # Setup AWS credentials for single-machine-performance AWS account
     - AWS_NAMED_PROFILE="single-machine-performance"
-    - SMP_ACCOUNT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_ACCOUNT_ID) || exit $?
+    - SMP_ACCOUNT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_ACCOUNT account_id) || exit $?
     - SMP_ECR_URL=${SMP_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com
-    - SMP_AGENT_TEAM_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_AGENT_TEAM_ID) || exit $?
-    - SMP_API=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_API) || exit $?
-    - SMP_BOT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_BOT_ACCESS_KEY_ID) || exit $?
-    - SMP_BOT_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_BOT_ACCESS_KEY) || exit $?
+    - SMP_AGENT_TEAM_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_ACCOUNT agent_team_id) || exit $?
+    - SMP_API=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_ACCOUNT api_url) || exit $?
+    - SMP_BOT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_ACCOUNT bot_login) || exit $?
+    - SMP_BOT_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_ACCOUNT bot_token) || exit $?
     - aws configure set aws_access_key_id "$SMP_BOT_ID" --profile ${AWS_NAMED_PROFILE}
     - aws configure set aws_secret_access_key "$SMP_BOT_KEY" --profile ${AWS_NAMED_PROFILE}
     - aws configure set region us-west-2 --profile ${AWS_NAMED_PROFILE}
@@ -158,6 +157,14 @@ single-machine-performance-regression_detector-pr-comment:
     FF_KUBERNETES_HONOR_ENTRYPOINT: false
   allow_failure: true  # allow_failure here should have same setting as in job above
   script: # ignore error message about no PR, because it happens for dev branches without PRs
+    # Prevent posting empty Regression Detector report if Markdown report is not found or
+    # has zero size.
+    - |
+      if [[ ! -s "outputs/report.md" ]]
+      then
+          echo "ERROR: Regression Detector report not found -- no PR comment posted"
+          exit 1
+      fi
     # We need to transform the Markdown report into a valid JSON string (without
     # quotes) in order to pass a well-formed payload to the PR commenting
     # service. Note that on macOS, the "-z" flag is invalid for `sed` (but

.gitlab/package_build/heroku.yml

@@ -51,7 +51,6 @@
 agent_heroku_deb-x64-a7:
   extends: .heroku_build_base
   variables:
-    DESTINATION_DEB: "datadog-heroku-agent_7_amd64.deb"
     DESTINATION_DBG_DEB: "datadog-heroku-agent-dbg_7_amd64.deb"
     RELEASE_VERSION: $RELEASE_VERSION_7
     AGENT_MAJOR_VERSION: 7

.gitlab/packaging/deb.yml

@@ -46,7 +46,6 @@ agent_deb-x64-a7:
     - when: on_success
   needs: ["datadog-agent-7-x64"]
   variables:
-    DESTINATION_DEB: "datadog-agent_7_amd64.deb"
     DD_PROJECT: "agent"
 
 agent_deb-arm64-a7:
@@ -56,7 +55,6 @@ agent_deb-arm64-a7:
     - when: on_success
   needs: ["datadog-agent-7-arm64"]
   variables:
-    DESTINATION_DEB: "datadog-agent_7_arm64.deb"
     DD_PROJECT: "agent"
 
 .package_ot_deb_common:
@@ -75,7 +73,6 @@ ot_agent_deb-x64-a7:
     - when: on_success
   needs: ["datadog-ot-agent-7-x64"]
   variables:
-    DESTINATION_DEB: "datadog-ot-agent_7_amd64.deb"
     DD_PROJECT: "agent"
 
 ot_agent_deb-arm64-a7:
@@ -85,7 +82,6 @@ ot_agent_deb-arm64-a7:
     - when: on_success
   needs: ["datadog-ot-agent-7-arm64"]
   variables:
-    DESTINATION_DEB: "datadog-ot-agent_7_arm64.deb"
     DD_PROJECT: "agent"
 
 installer_deb-amd64:
@@ -95,7 +91,6 @@ installer_deb-amd64:
     - when: on_success
   needs: ["installer-amd64"]
   variables:
-    DESTINATION_DEB: "datadog-installer_7_amd64.deb"
     DD_PROJECT: "installer"
     # There are currently no files to check for in the installer so we
     # explicitly disable the check
@@ -108,7 +103,6 @@ installer_deb-arm64:
     - when: on_success
   needs: ["installer-arm64"]
   variables:
-    DESTINATION_DEB: "datadog-installer_7_arm64.deb"
     DD_PROJECT: "installer"
     PACKAGE_REQUIRED_FILES_LIST: ""
 
@@ -140,14 +134,10 @@ installer_deb-arm64:
 iot_agent_deb-x64:
   extends: [.package_iot_deb_common, .package_deb_x86]
   needs: ["iot-agent-x64"]
-  variables:
-    DESTINATION_DEB: "datadog-iot-agent_7_amd64.deb"
 
 iot_agent_deb-arm64:
   extends: [.package_iot_deb_common, .package_deb_arm64]
   needs: ["iot-agent-arm64"]
-  variables:
-    DESTINATION_DEB: "datadog-iot-agent_7_arm64.deb"
 
 iot_agent_deb-armhf:
   extends: .package_iot_deb_common
@@ -156,7 +146,6 @@ iot_agent_deb-armhf:
   needs: ["iot-agent-armhf"]
   variables:
     PACKAGE_ARCH: armhf
-    DESTINATION_DEB: "datadog-iot-agent_7_armhf.deb"
     DD_PKG_ARCH: "arm64"
     FORCED_PACKAGE_COMPRESSION_LEVEL: 5
 
@@ -168,7 +157,6 @@ dogstatsd_deb-x64:
   needs: ["dogstatsd-x64"]
   variables:
     DD_PROJECT: dogstatsd
-    DESTINATION_DEB: "datadog-dogstatsd_amd64.deb"
     PACKAGE_REQUIRED_FILES_LIST: "test/required_files/dogstatsd-deb.txt"
 
 dogstatsd_deb-arm64:
@@ -179,6 +167,5 @@ dogstatsd_deb-arm64:
   needs: ["dogstatsd-arm64"]
   variables:
     DD_PROJECT: dogstatsd
-    DESTINATION_DEB: "datadog-dogstatsd_arm64.deb"
     PACKAGE_REQUIRED_FILES_LIST: "test/required_files/dogstatsd-deb.txt"
 

cmd/agent/subcommands/snmp/command.go

@@ -135,7 +135,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
 				fx.Supply(core.BundleParams{
 					ConfigParams: config.NewAgentParams(globalParams.ConfFilePath, config.WithExtraConfFiles(globalParams.ExtraConfFilePath), config.WithFleetPoliciesDirPath(globalParams.FleetPoliciesDirPath)),
 					SecretParams: secrets.NewEnabledParams(),
-					LogParams:    log.ForOneShot(command.LoggerName, "info", true)}),
+					LogParams:    log.ForOneShot(command.LoggerName, "off", true)}),
 				core.Bundle(),
 				snmpscanfx.Module(),
 				demultiplexerimpl.Module(demultiplexerimpl.NewDefaultParams()),

cmd/installer/subcommands/installer/command.go

@@ -9,6 +9,7 @@ package installer
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"os"
 	"runtime"
 	"strings"
@@ -46,6 +47,10 @@ const (
 	envAgentMajorVersion           = "DD_AGENT_MAJOR_VERSION"
 	envAgentMinorVersion           = "DD_AGENT_MINOR_VERSION"
 	envAgentDistChannel            = "DD_AGENT_DIST_CHANNEL"
+	envRemoteUpdates               = "DD_REMOTE_UPDATES"
+	envHTTPProxy                   = "HTTP_PROXY"
+	envHTTPSProxy                  = "HTTPS_PROXY"
+	envNoProxy                     = "NO_PROXY"
 )
 
 // BootstrapCommand returns the bootstrap command.
@@ -161,11 +166,27 @@ func newBootstrapperCmd(operation string) *bootstrapperCmd {
 	cmd.span.SetTag("env.DD_RPM_REPO_GPGCHECK", os.Getenv(envRPMRepoGPGCheck))
 	cmd.span.SetTag("env.DD_AGENT_MAJOR_VERSION", os.Getenv(envAgentMajorVersion))
 	cmd.span.SetTag("env.DD_AGENT_MINOR_VERSION", os.Getenv(envAgentMinorVersion))
+	cmd.span.SetTag("env.DD_AGENT_DIST_CHANNEL", os.Getenv(envAgentDistChannel))
+	cmd.span.SetTag("env.DD_REMOTE_UPDATES", os.Getenv(envRemoteUpdates))
+	cmd.span.SetTag("env.HTTP_PROXY", redactURL(os.Getenv(envHTTPProxy)))
+	cmd.span.SetTag("env.HTTPS_PROXY", redactURL(os.Getenv(envHTTPSProxy)))
+	cmd.span.SetTag("env.NO_PROXY", os.Getenv(envNoProxy))
 	return &bootstrapperCmd{
 		cmd: cmd,
 	}
 }
 
+func redactURL(u string) string {
+	if u == "" {
+		return ""
+	}
+	url, err := url.Parse(u)
+	if err != nil {
+		return "invalid"
+	}
+	return url.Redacted()
+}
+
 type telemetryConfigFields struct {
 	APIKey string `yaml:"api_key"`
 	Site   string `yaml:"site"`

cmd/serverless-init/main.go

@@ -177,6 +177,7 @@ func setupTraceAgent(tags map[string]string, tagger tagger.Component) trace.Serv
 		ColdStartSpanID:       random.Random.Uint64(),
 		AzureContainerAppTags: azureTags.String(),
 	})
+	traceAgent.SetTags(tags)
 	go func() {
 		for range time.Tick(3 * time.Second) {
 			traceAgent.Flush()

cmd/system-probe/modules/gpu.go

@@ -33,7 +33,7 @@ var GPUMonitoring = module.Factory{
 	ConfigNamespaces: gpuMonitoringConfigNamespaces,
 	Fn: func(_ *sysconfigtypes.Config, deps module.FactoryDependencies) (module.Module, error) {
 
-		c := gpuconfig.NewConfig()
+		c := gpuconfig.New()
 		probeDeps := gpu.ProbeDependencies{
 			Telemetry: deps.Telemetry,
 			//if the config parameter doesn't exist or is empty string, the default value is used as defined in go-nvml library

cmd/trace-agent/test/testsuite/cards_test.go

@@ -86,6 +86,17 @@ apm_config:
 			out:     "?",
 			version: "v0.7",
 		},
+		{
+			conf: []byte(`
+apm_config:
+  env: my-env
+  obfuscation:
+    credit_cards:
+      enabled: false
+      keep_values: ["credit_card_number"]`),
+			out:     "4166 6766 6766 6746",
+			version: "v0.5",
+		},
 	} {
 		t.Run(string(tt.version)+"/"+tt.out, func(t *testing.T) {
 			if err := r.RunAgent(tt.conf); err != nil {

docs/cloud-workload-security/linux_expressions.md

@@ -386,6 +386,7 @@ A bind was executed
 | -------- | ------------- |
 | [`bind.addr.family`](#bind-addr-family-doc) | Address family |
 | [`bind.addr.ip`](#common-ipportcontext-ip-doc) | IP address |
+| [`bind.addr.is_public`](#common-ipportcontext-is_public-doc) | Whether the IP address belongs to a public network |
 | [`bind.addr.port`](#common-ipportcontext-port-doc) | Port number |
 | [`bind.retval`](#common-syscallevent-retval-doc) | Return value of the syscall |
 
@@ -520,10 +521,12 @@ A connect was executed
 | -------- | ------------- |
 | [`connect.addr.family`](#connect-addr-family-doc) | Address family |
 | [`connect.addr.ip`](#common-ipportcontext-ip-doc) | IP address |
+| [`connect.addr.is_public`](#common-ipportcontext-is_public-doc) | Whether the IP address belongs to a public network |
 | [`connect.addr.port`](#common-ipportcontext-port-doc) | Port number |
 | [`connect.retval`](#common-syscallevent-retval-doc) | Return value of the syscall |
 | [`connect.server.addr.family`](#connect-server-addr-family-doc) | Server address family |
 | [`connect.server.addr.ip`](#common-ipportcontext-ip-doc) | IP address |
+| [`connect.server.addr.is_public`](#common-ipportcontext-is_public-doc) | Whether the IP address belongs to a public network |
 | [`connect.server.addr.port`](#common-ipportcontext-port-doc) | Port number |
 
 ### Event `dns`
@@ -540,12 +543,14 @@ A DNS request was sent
 | [`dns.question.name.length`](#common-string-length-doc) | Length of the corresponding element |
 | [`dns.question.type`](#dns-question-type-doc) | a two octet code which specifies the DNS question type |
 | [`network.destination.ip`](#common-ipportcontext-ip-doc) | IP address |
+| [`network.destination.is_public`](#common-ipportcontext-is_public-doc) | Whether the IP address belongs to a public network |
 | [`network.destination.port`](#common-ipportcontext-port-doc) | Port number |
 | [`network.device.ifname`](#common-networkdevicecontext-ifname-doc) | Interface ifname |
 | [`network.l3_protocol`](#common-networkcontext-l3_protocol-doc) | L3 protocol of the network packet |
 | [`network.l4_protocol`](#common-networkcontext-l4_protocol-doc) | L4 protocol of the network packet |
 | [`network.size`](#common-networkcontext-size-doc) | Size in bytes of the network packet |
 | [`network.source.ip`](#common-ipportcontext-ip-doc) | IP address |
+| [`network.source.is_public`](#common-ipportcontext-is_public-doc) | Whether the IP address belongs to a public network |
 | [`network.source.port`](#common-ipportcontext-port-doc) | Port number |
 
 ### Event `exec`
@@ -742,12 +747,14 @@ An IMDS event was captured
 | [`imds.url`](#imds-url-doc) | the queried IMDS URL |
 | [`imds.user_agent`](#imds-user_agent-doc) | the user agent of the HTTP client |
 | [`network.destination.ip`](#common-ipportcontext-ip-doc) | IP address |
+| [`network.destination.is_public`](#common-ipportcontext-is_public-doc) | Whether the IP address belongs to a public network |
 | [`network.destination.port`](#common-ipportcontext-port-doc) | Port number |
 | [`network.device.ifname`](#common-networkdevicecontext-ifname-doc) | Interface ifname |
 | [`network.l3_protocol`](#common-networkcontext-l3_protocol-doc) | L3 protocol of the network packet |
 | [`network.l4_protocol`](#common-networkcontext-l4_protocol-doc) | L4 protocol of the network packet |
 | [`network.size`](#common-networkcontext-size-doc) | Size in bytes of the network packet |
 | [`network.source.ip`](#common-ipportcontext-ip-doc) | IP address |
+| [`network.source.is_public`](#common-ipportcontext-is_public-doc) | Whether the IP address belongs to a public network |
 | [`network.source.port`](#common-ipportcontext-port-doc) | Port number |
 
 ### Event `link`
@@ -960,13 +967,15 @@ A raw network packet captured
 | Property | Definition |
 | -------- | ------------- |
 | [`packet.destination.ip`](#common-ipportcontext-ip-doc) | IP address |
+| [`packet.destination.is_public`](#common-ipportcontext-is_public-doc) | Whether the IP address belongs to a public network |
 | [`packet.destination.port`](#common-ipportcontext-port-doc) | Port number |
 | [`packet.device.ifname`](#common-networkdevicecontext-ifname-doc) | Interface ifname |
 | [`packet.filter`](#packet-filter-doc) | pcap filter expression |
 | [`packet.l3_protocol`](#common-networkcontext-l3_protocol-doc) | L3 protocol of the network packet |
 | [`packet.l4_protocol`](#common-networkcontext-l4_protocol-doc) | L4 protocol of the network packet |
 | [`packet.size`](#common-networkcontext-size-doc) | Size in bytes of the network packet |
 | [`packet.source.ip`](#common-ipportcontext-ip-doc) | IP address |
+| [`packet.source.is_public`](#common-ipportcontext-is_public-doc) | Whether the IP address belongs to a public network |
 | [`packet.source.port`](#common-ipportcontext-port-doc) | Port number |
 | [`packet.tls.version`](#packet-tls-version-doc) | TLS version |
 
@@ -2157,6 +2166,15 @@ Definition: Indicates whether the process is a kworker
 `exec` `exit` `process` `process.ancestors` `process.parent` `ptrace.tracee` `ptrace.tracee.ancestors` `ptrace.tracee.parent` `signal.target` `signal.target.ancestors` `signal.target.parent`
 
 
+### `*.is_public` {#common-ipportcontext-is_public-doc}
+Type: bool
+
+Definition: Whether the IP address belongs to a public network
+
+`*.is_public` has 7 possible prefixes:
+`bind.addr` `connect.addr` `connect.server.addr` `network.destination` `network.source` `packet.destination` `packet.source`
+
+
 ### `*.is_thread` {#common-process-is_thread-doc}
 Type: bool