Merge branch 'v1' into mackjmr/grpc-with-span-options

.circleci/config.yml

@@ -15,11 +15,20 @@ jobs:
         description: "go build tags used to compile"
         default: ""
         type: string
+      goflags:
+        description: "extra goflags to pass to go test"
+        default: ""
+        type: string
     docker:
       - image: circleci/golang:1.12
         environment:
           GOPATH: "/home/circleci/go"
     working_directory: /home/circleci/dd-trace-go.v1
+    environment:
+      # Go 1.12 doesn't have the proxy turned on by default but we need it to
+      # fetch some dependencies that aren't handled by fetching directly
+      # (such as github.com/go-chi/chi/[email protected])
+      GOPROXY: "https://proxy.golang.org"
     steps:
     - checkout
 
@@ -106,6 +115,10 @@ jobs:
         description: "go build tags to use to compile the tests"
         default: ""
         type: string
+      goflags:
+        description: "extra goflags to pass to go test"
+        default: ""
+        type: string
     resource_class: xlarge
     environment: # environment variables for the build itself
       TEST_RESULTS: /tmp/test-results # path to where test results will be saved
@@ -119,16 +132,11 @@ jobs:
       - restore_cache: # restores saved cache if no changes are detected since last run
           keys:
             - go-mod-v5-core-{{ checksum "go.sum.orig" }}
-      - run:
-          name: Enforce some dependencies
-          command: |
-            # last version compatible with go1.14, needed for testtraceprof
-            echo 'replace golang.org/x/net => golang.org/x/net d418f374d30933c6c7db22cf349625c295a5afaa' >> go.mod
       - run:
           name: Testing
           command: |
             PACKAGE_NAMES=$(go list ./... | grep -v /contrib/ | circleci tests split --split-by=timings --timings-type=classname)
-            env DD_APPSEC_ENABLED=$(test "<< parameters.build_tags >>" = appsec && echo -n true) gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v -race -coverprofile=coverage.txt -covermode=atomic -tags "<< parameters.build_tags >>"
+            env DD_APPSEC_ENABLED=$(test "<< parameters.build_tags >>" = appsec && echo -n true) gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v << parameters.goflags >> -coverprofile=coverage.txt -covermode=atomic -tags "<< parameters.build_tags >>"
 
       - save_cache:
           key: go-mod-v5-core-{{ checksum "go.sum.orig" }}
@@ -153,6 +161,10 @@ jobs:
         description: "go build tags to use to compile the tests"
         default: ""
         type: string
+      goflags:
+        description: "extra goflags to pass to go test"
+        default: ""
+        type: string
     resource_class: xlarge
     environment: # environment variables for the build itself
       TEST_RESULTS: /tmp/test-results # path to where test results will be saved
@@ -176,6 +188,10 @@ jobs:
           POSTGRES_PASSWORD: postgres
           POSTGRES_USER: postgres
           POSTGRES_DB: postgres
+      - image: mcr.microsoft.com/mssql/server:2019-latest
+        environment:
+         SA_PASSWORD: myPassw0rd
+         ACCEPT_EULA: Y
       - image: consul:1.6.0
       - image: redis:3.2
       - image: elasticsearch:2
@@ -201,16 +217,19 @@ jobs:
           DD_API_KEY: invalid_key_but_this_is_fine
       - image: circleci/mongo:latest-ram
       - image: memcached:1.5.9
-      - image: confluentinc/cp-zookeeper:5.0.0
+      - image: bitnami/zookeeper:latest
+        environment:
+          ALLOW_ANONYMOUS_LOGIN: yes
+      - image: bitnami/kafka:2
         environment:
-          ZOOKEEPER_CLIENT_PORT: "2181"
-      - image: confluentinc/cp-kafka:5.0.0
+          KAFKA_CFG_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
+          KAFKA_CFG_LISTENERS: PLAINTEXT://0.0.0.0:9092
+          ALLOW_PLAINTEXT_LISTENER: yes
+      - image: bitnami/kafka:2
         environment:
-          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
-          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
-          KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
-          KAFKA_CREATE_TOPICS: gotest:1:1
-          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
+          KAFKA_CFG_ZOOKEEPER_CONNECT: localhost:2181
+        command: [kafka-topics.sh, --create, --topic, gosegtest, --bootstrap-server, localhost:9092]
 
     steps:
       - checkout
@@ -239,24 +258,6 @@ jobs:
           paths:
           - /tmp/librdkafka-v1.3.0
 
-      - run:
-          name: Enforce some dependencies
-          command: |
-            go get k8s.io/[email protected]
-            go get k8s.io/[email protected]
-            go get cloud.google.com/go/[email protected]
-            # Temporarily enforce this version. 1.9.0 is incompatible with go < 1.16
-            go get github.com/hashicorp/consul/[email protected]
-            # github.com/hashicorp/vault/sdk > v0.2.0 doesn't compile with go1.14
-            go get github.com/hashicorp/vault/[email protected]
-            # Shopify/sarama > v1.22 doesn't compile with go1.14
-            go get github.com/Shopify/[email protected]
-            # Temporary enforcing gorm to v1.22.4 to avoid the problems of v1.22.5
-            go get -v gorm.io/[email protected]
-            # gofiber >= v2.27.0 has a transitive dependency on a newer version of
-            # golang.org/x/net that requires Go >= 1.15, breaking our build
-            go get github.com/gofiber/fiber/[email protected]
-
       - run:
           name: Wait for MySQL
           command: dockerize -wait tcp://localhost:3306 -timeout 1m
@@ -265,6 +266,10 @@ jobs:
           name: Wait for Postgres
           command: dockerize -wait tcp://localhost:5432 -timeout 1m
 
+      - run:
+          name: Wait for MS SQL Server
+          command: dockerize -wait tcp://localhost:1433 -timeout 1m
+
       - run:
           name: Wait for Redis
           command: dockerize -wait tcp://localhost:6379 -timeout 1m
@@ -301,13 +306,22 @@ jobs:
           name: Wait for Consul
           command: dockerize -wait http://localhost:8500 -timeout 1m
 
+      - run:
+          name: Go module graph (before)
+          command: go mod graph
+
       - run:
           name: Testing integrations
           command: |
             PACKAGE_NAMES=$(go list ./contrib/... | grep -v -e grpc.v12 -e google.golang.org/api | circleci tests split --split-by=timings --timings-type=classname)
             export DD_APPSEC_ENABLED=$(test "<< parameters.build_tags >>" = appsec && echo -n true)
             export INTEGRATION=true
-            gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v -race -coverprofile=coverage.txt -covermode=atomic -tags "<< parameters.build_tags >>"
+            gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v << parameters.goflags >> -coverprofile=coverage.txt -covermode=atomic -tags "<< parameters.build_tags >>"
+
+      - run:
+          name: Go module graph (after)
+          command: go mod graph
+          when: always
 
       - store_artifacts: # upload test summary for display in Artifacts
           path: /tmp/test-results
@@ -368,3 +382,27 @@ workflows:
           matrix:
             parameters:
               build_tags: [ "", "appsec" ]
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - v1
+    jobs:
+      - go1_12-build:
+          matrix:
+            parameters:
+              build_tags: [ "", "appsec" ]
+              goflags: [ "-race" ]
+      - test-core:
+          matrix:
+            parameters:
+              build_tags: [ "", "appsec" ]
+              goflags: [ "-race" ]
+      - test-contrib:
+          matrix:
+            parameters:
+              build_tags: [ "", "appsec" ]
+              goflags: [ "-race" ]

.github/workflows/system-tests.yml

@@ -27,6 +27,8 @@ jobs:
             weblog-variant: echo
           - library: golang
             weblog-variant: chi
+          - library: golang
+            weblog-variant: gin
       fail-fast: false
     env:
       TEST_LIBRARY: golang

CODEOWNERS

@@ -13,5 +13,6 @@
 /internal/traceprof       @DataDog/profiling-go
 
 # appsec
+/appsec                   @DataDog/appsec-go
 /internal/appsec          @DataDog/appsec-go
 /contrib/**/appsec.go     @DataDog/appsec-go

CONTRIBUTING.md

@@ -40,15 +40,9 @@ Please view our contrib [README.md](contrib/README.md) for information on new in
 
 ### Go Modules
 
-This repository currently takes an [idiosyncratic approach](https://github.com/DataDog/dd-trace-go/issues/810) to using Go modules which means that you should not commit modified versions of the `go.mod` or `go.sum` files.
+When adding a new dependency, especially for `contrib/` packages, prefer the minimum secure versions of any modules rather than the latest versions. This is to avoid forcing upgrades on downstream users for modules such as `google.golang.org/grpc` which often introduce breaking changes within minor versions.
 
-The following git command can be used to permanently ignore modifications to these files:
-
-```
-git update-index --assume-unchanged go.*
-```
-
-If you need to undo this for any reason, you can run:
+This repository used to omit many dependencies from the `go.mod` file due to concerns around version compatibility [(ref)](https://github.com/DataDog/dd-trace-go/issues/810). As such, you may have configured git to ignore changes to `go.mod` and `go.sum`. To undo this, run
 
 ```
 git update-index --no-assume-unchanged go.*

FAQ.md

@@ -22,6 +22,8 @@ Integrations that are considered *clients* (http clients, grpc clients, sql clie
 
 While there are good arguments to be made that client integrations should take the same service name as everything else in the service, that's not how the library is intended to function today. As a work-around, most integrations have a `WithServiceName` `Option` that will allow you to override the default. If the integration you are using cannot be configured the way you want, please open an issue to discuss adding as option.
 
+See also: https://github.com/DataDog/dd-trace-go/pull/603
+
 #### Why are client integration spans not measured?
 This is primarily for 2 reasons:
 1. Cost - often a traced client will speak to a traced server. If both are measured, there is duplication of measurement here, and duplication of cost for no benefit. By measuring **only** the server, we get analytics without duplication. 

appsec/appsec.go

@@ -0,0 +1,32 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2022 Datadog, Inc.
+
+// Package appsec provides application security features in the form of SDK
+// functions that can be manually called to monitor specific code paths and data.
+// Application Security is currently transparently integrated into the APM tracer
+// and cannot be used nor started alone at the moment.
+// You can read more on how to enable and start Application Security for Go at
+// https://docs.datadoghq.com/security_platform/application_security/getting_started/go
+package appsec
+
+import (
+	"context"
+
+	"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec"
+	"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo/instrumentation/httpsec"
+)
+
+// MonitorParsedHTTPBody runs the security monitoring rules on the given *parsed*
+// HTTP request body. The given context must be the HTTP request context as returned
+// by the Context() method of an HTTP request. Calls to this function are ignored if
+// AppSec is disabled or the given context is incorrect.
+// Note that passing the raw bytes of the HTTP request body is not expected and would
+// result in inaccurate attack detection.
+func MonitorParsedHTTPBody(ctx context.Context, body interface{}) {
+	if appsec.Enabled() {
+		httpsec.MonitorParsedBody(ctx, body)
+	}
+	// bonus: use sync.Once to log a debug message once if AppSec is disabled
+}

appsec/example_test.go

@@ -0,0 +1,62 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2022 Datadog, Inc.
+
+package appsec_test
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+
+	"gopkg.in/DataDog/dd-trace-go.v1/appsec"
+	echotrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/labstack/echo.v4"
+	httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http"
+
+	"github.com/labstack/echo/v4"
+)
+
+type parsedBodyType struct {
+	Value string `json:"value"`
+}
+
+func customBodyParser(body io.ReadCloser) (*parsedBodyType, error) {
+	var parsedBody parsedBodyType
+	err := json.NewDecoder(body).Decode(&parsedBody)
+	return &parsedBody, err
+}
+
+// Monitor HTTP request parsed body
+func ExampleMonitorParsedHTTPBody() {
+	mux := httptrace.NewServeMux()
+	mux.HandleFunc("/body", func(w http.ResponseWriter, r *http.Request) {
+		// Use the SDK to monitor the request's parsed body
+		body, err := customBodyParser(r.Body)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		appsec.MonitorParsedHTTPBody(r.Context(), body)
+		w.Write([]byte("Body monitored using AppSec SDK\n"))
+	})
+	http.ListenAndServe(":8080", mux)
+}
+
+// Monitor HTTP request parsed body with a framework customized context type
+func ExampleMonitorParsedHTTPBody_CustomContext() {
+	r := echo.New()
+	r.Use(echotrace.Middleware())
+	r.POST("/body", func(c echo.Context) (e error) {
+		req := c.Request()
+		body, err := customBodyParser(req.Body)
+		if err != nil {
+			return c.String(http.StatusInternalServerError, err.Error())
+		}
+		// Use the SDK to monitor the request's parsed body
+		appsec.MonitorParsedHTTPBody(c.Request().Context(), body)
+		return c.String(http.StatusOK, "Body monitored using AppSec SDK")
+	})
+
+	r.Start(":8080")
+}