all: Add integration testing harness and some tests (#37)

* all: Add integration testing harness and some tests This commit adds an integration test harness to the repository. Individual tests consist of a directory under _integration-tests/tests which contains a go program and a file, validation.json. The go program will be instrumented by orchestrion, then run. The traces output by that program go to a fake agent, which records the traces. The validation.json file describes the expected structure of the traces which should be produced by each test. This includes things like name, resource, service, metadata and metrics. Once the traces are recorded and the test program shuts down, the traces recorded by the fake agent are checked against the expected ones described in validation.json to ensure the test produced exactly what we expected, in terms of the number of traces, the structure of those traces, and the fields in the spans of the traces.
DataDog · Sep 19, 2023 · 3fce9ce · 3fce9ce
1 parent 3acdc87
commit 3fce9ce
Show file tree

Hide file tree

Showing 27 changed files with 1,647 additions and 3 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -1,12 +1,15 @@
 name: Tests
 on: [ push ]
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/setup-go@v4
         with:
-          go-version: "1.19"
+          go-version: "stable"
           cache: true
       - uses: actions/checkout@v3
         with:
@@ -24,6 +27,28 @@ jobs:
       - name: Checkout Go
         uses: actions/setup-go@v4
         with:
-          go-version: "1.19"
+          go-version: "stable"
       - name: Run unit tests
         run: make test
+  integration-tests:
+    runs-on: ubuntu-latest
+    services:
+      testagent:
+        image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.11.0
+        ports:
+          - 8126:8126
+        env:
+          # See documentation for configuration:
+          # https://github.com/DataDog/dd-apm-test-agent#environment-variables
+          LOG_LEVEL: DEBUG
+          TRACE_LANGUAGE: golang
+          ENABLED_CHECKS: trace_stall,trace_count_header,trace_peer_service,trace_dd_service
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: 'DataDog/orchestrion'
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "stable"
+      - name: Run Integration Tests
+        run: make integration-tests
diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
 .DEFAULT_GOAL := build
 
-.PHONY: build generate tidy vet test clean fmt licenses verify-licenses
+.PHONY: build generate tidy vet test clean fmt licenses verify-licenses verify-dd-headers integration-tests
 
 build: generate test build-only
 
@@ -41,3 +41,6 @@ bin/go-licenses:
 
 verify-dd-headers:
 	go run tools/headercheck/header_check.go
+
+integration-tests:
+	./integration-tests.sh
diff --git a/_integration-tests/README.md b/_integration-tests/README.md
@@ -0,0 +1,80 @@
+# Integration Tests
+
+The integration tests in this directory run programs instrumented with orchestrion, and compare the traces received by the Trace Agent to a list of expected traces.
+
+Each test is a directory within the `tests` directory. This directory must contain a go program compilable with `go build`. Orchestrion is run on the source before compiling.
+
+## Running Tests Locally
+
+Tests can be run locally by running the `integration-tests.sh` script in the repository root. 
+
+| :exclamation:  Note: The `integration-tests.sh` script modifies the source of the test applications using Orchestrion. Make sure you have any changes saved before running the test suite. |
+|-----------------------------------------|
+
+The script optionally accepts the name of a specific test to run. Otherwise it will run all of the integration tests in the suite.
+
+
+```
+$ ./integration-tests.sh
+... Runs all integration tests
+```
+
+```
+$ ./integration-tests.sh net_http
+... Runs the net_http unit test
+```
+
+
+
+## Creating a New Test
+
+Creating a new test can be done by adding a directory to the `tests` directory. Conventionally, these should be named the same thing as the integrations they test, when adding a test for a new integration.
+
+A test must consist of 2 things:
+- A set of `.go` files which build into an executable program. Orchestrion will be run on these files before compiling to add instrumentation. Such programs must be running an HTTP server, and must trigger the desired trace-generating behavior via a web request to some url. This allows the test harness to trigger the test.
+- A `validation.json` file, containing the URL that will trigger the trace generation, and a list of traces to be expected in the output.
+
+### `validation.json` structure
+
+```
+top level:
+{
+	"url": "http://localhost:8080",
+	"output": [span]
+}
+
+span:
+{
+	"name": "span name",
+	"service": "span service",
+	"resource": "span resource",
+	"type": "span type",
+	"meta": {string -> string},
+	"metrics": {string -> float},
+	"_children": [span]
+}
+```
+
+#### Example:
+```
+{
+	"url": "http://localhost:8080",
+	"output": [
+		{
+			"name": "parent",
+			"service": "myservice",
+			"resource": "parent resource",
+			"type": "parent type",
+			"_children": [
+				{
+					"name": "child",
+					"service": "myservice",
+					"resource": "child resource",
+					"meta": {,
+						"tag": "value"
+					}
+				}
+			]
+	}]
+}
+```
diff --git a/_integration-tests/docker-compose.yaml b/_integration-tests/docker-compose.yaml
@@ -0,0 +1,7 @@
+services:
+  fake-agent:
+    image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:latest
+    ports:
+      - "8126:8126"
+    environment:
+      SNAPSHOT_CI: "0"
diff --git a/_integration-tests/go.mod b/_integration-tests/go.mod
@@ -0,0 +1,73 @@
+module orchestrion/integration
+
+go 1.20
+
+replace github.com/datadog/orchestrion => ../
+
+require (
+	github.com/datadog/orchestrion v0.2.0
+	github.com/gin-gonic/gin v1.9.1
+	github.com/go-chi/chi/v5 v5.0.10
+	github.com/gorilla/mux v1.8.0
+	github.com/labstack/echo/v4 v4.11.1
+	github.com/mattn/go-sqlite3 v1.14.17
+	google.golang.org/grpc v1.58.0
+	google.golang.org/grpc/examples v0.0.0-20230913203803-9deee9ba5f5b
+)
+
+require (
+	github.com/DataDog/appsec-internal-go v1.0.0 // indirect
+	github.com/DataDog/datadog-agent/pkg/obfuscate v0.45.0-rc.1 // indirect
+	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.0-devel.0.20230725154044-2549ba9058df // indirect
+	github.com/DataDog/datadog-go/v5 v5.1.1 // indirect
+	github.com/DataDog/go-libddwaf v1.4.2 // indirect
+	github.com/DataDog/go-tuf v1.0.1-0.5.2 // indirect
+	github.com/DataDog/sketches-go v1.2.1 // indirect
+	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/bytedance/sonic v1.9.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/ebitengine/purego v0.5.0-alpha // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.14.0 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+	github.com/labstack/gommon v0.4.0 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/outcaste-io/ristretto v0.2.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/philhofer/fwd v1.1.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect
+	github.com/tinylib/msgp v1.1.8 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.11 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/valyala/fasttemplate v1.2.2 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect
+	go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect
+	golang.org/x/arch v0.3.0 // indirect
+	golang.org/x/crypto v0.12.0 // indirect
+	golang.org/x/net v0.14.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/text v0.12.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/DataDog/dd-trace-go.v1 v1.54.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	inet.af/netaddr v0.0.0-20220811202034-502d2d690317 // indirect
+)