Skip to content

Commit

Permalink
(feat) internal/civisibility: add support for testify Suites
Browse files Browse the repository at this point in the history
  • Loading branch information
@tonyredondo
tonyredondo committed Jan 16, 2025
1 parent b49d213 commit 5f36bd8
Show file tree
Hide file tree
Showing 5 changed files with 330 additions and 15 deletions.
24 changes: 23 additions & 1 deletion internal/civisibility/integrations/gotesting/instrumentation_orchestrion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,14 @@ func instrumentTestingTFunc(f func(*testing.T)) func(*testing.T) {
}

instrumentedFn := func(t *testing.T) {
// Check if we have testify suite data related to this test
testifyData := getTestifyTest(t)
if testifyData != nil {
// If we have testify data, we need to extract the module and suite name from the testify suite
moduleName = testifyData.moduleName
suiteName = testifyData.suiteName
}

// Initialize module counters if not already present.
if _, ok := modulesCounters[moduleName]; !ok {
var v int32
Expand All @@ -134,7 +142,14 @@ func instrumentTestingTFunc(f func(*testing.T)) func(*testing.T) {
module := session.GetOrCreateModule(moduleName)
suite := module.GetOrCreateSuite(suiteName)
test := suite.CreateTest(t.Name())
test.SetTestFunc(originalFunc)

// If we have testify data we use the method function from testify so the test source is properly set
if testifyData != nil {
test.SetTestFunc(testifyData.methodFunc)
} else {
// If not, let's set the original function
test.SetTestFunc(originalFunc)
}

// Get the metadata regarding the execution (in case is already created from the additional features)
execMeta := getTestMetadata(t)
Expand Down Expand Up @@ -430,3 +445,10 @@ func instrumentTestingBFunc(pb *testing.B, name string, f func(*testing.B)) (str
setCiVisibilityBenchmarkFunc(runtime.FuncForPC(reflect.Indirect(reflect.ValueOf(instrumentedFunc)).Pointer()))
return subBenchmarkAutoName, instrumentedFunc
}

// instrumentTestifySuiteRun helper function to instrument the testify Suite.Run function
//
//go:linkname instrumentTestifySuiteRun
func instrumentTestifySuiteRun(t *testing.T, suite any) {
registerTestifySuite(t, suite)
}
80 changes: 66 additions & 14 deletions internal/civisibility/integrations/gotesting/testcontroller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"net/http/httptest"
"os"
"os/exec"
"strings"
"testing"

"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext"
Expand Down Expand Up @@ -94,7 +95,7 @@ func runFlakyTestRetriesTests(m *testing.M) {

// 1 session span
// 1 module span
// 2 suite span (testing_test.go and reflections_test.go)
// 4 suite span (testing_test.go, testify_test.go, testify_test.go/MySuite and reflections_test.go)
// 5 tests from reflections_test.go
// 1 TestMyTest01
// 1 TestMyTest02 + 2 subtests
Expand All @@ -104,10 +105,13 @@ func runFlakyTestRetriesTests(m *testing.M) {
// 1 TestRetryWithFail + 3 retry tests from testing_test.go
// 1 TestNormalPassingAfterRetryAlwaysFail
// 1 TestEarlyFlakeDetection
// 3 tests from testify_test.go and testify_test.go/MySuite

// check spans by resource name
checkSpansByResourceName(finishedSpans, "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations/gotesting", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest01", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02", 1)
Expand All @@ -122,6 +126,14 @@ func runFlakyTestRetriesTests(m *testing.M) {
checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithFail", 4)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestNormalPassingAfterRetryAlwaysFail", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestEarlyFlakeDetection", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go.TestTestifyLikeTest", 1)
testifySub01 := checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite", 1)[0]
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite/sub01", 1)

// check that testify span has the correct source file
if !strings.HasSuffix(testifySub01.Tag("test.source.file").(string), "/testify_test.go") {
panic(fmt.Sprintf("source file should be testify_test.go, got %s", testifySub01.Tag("test.source.file").(string)))
}

// check spans by tag
checkSpansByTagName(finishedSpans, constants.TestIsRetry, 6)
Expand All @@ -131,8 +143,8 @@ func runFlakyTestRetriesTests(m *testing.M) {
28,
1,
1,
2,
24,
4,
27,
0)

fmt.Println("All tests passed.")
Expand Down Expand Up @@ -171,7 +183,7 @@ func runEarlyFlakyTestDetectionTests(m *testing.M) {

// 1 session span
// 1 module span
// 2 suite span (testing_test.go and reflections_test.go)
// 4 suite span (testing_test.go, testify_test.go, testify_test.go/MySuite and reflections_test.go)
// 5 tests from reflections_test.go
// 11 TestMyTest01
// 11 TestMyTest02 + 22 subtests
Expand All @@ -182,10 +194,13 @@ func runEarlyFlakyTestDetectionTests(m *testing.M) {
// 11 TestNormalPassingAfterRetryAlwaysFail
// 11 TestEarlyFlakeDetection
// 22 normal spans from testing_test.go
// 33 tests from testify_test.go and testify_test.go/MySuite

// check spans by resource name
checkSpansByResourceName(finishedSpans, "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations/gotesting", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest01", 11)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02", 11)
Expand All @@ -200,18 +215,26 @@ func runEarlyFlakyTestDetectionTests(m *testing.M) {
checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithFail", 11)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestNormalPassingAfterRetryAlwaysFail", 11)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestEarlyFlakeDetection", 11)
checkSpansByResourceName(finishedSpans, "testify_test.go.TestTestifyLikeTest", 11)
testifySub01 := checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite", 11)[0]
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite/sub01", 11)

// check that testify span has the correct source file
if !strings.HasSuffix(testifySub01.Tag("test.source.file").(string), "/testify_test.go") {
panic(fmt.Sprintf("source file should be testify_test.go, got %s", testifySub01.Tag("test.source.file").(string)))
}

// check spans by tag
checkSpansByTagName(finishedSpans, constants.TestIsNew, 143)
checkSpansByTagName(finishedSpans, constants.TestIsRetry, 130)
checkSpansByTagName(finishedSpans, constants.TestIsNew, 176)
checkSpansByTagName(finishedSpans, constants.TestIsRetry, 160)

// check spans by type
checkSpansByType(finishedSpans,
152,
1,
1,
2,
148,
4,
181,
0)

fmt.Println("All tests passed.")
Expand Down Expand Up @@ -241,6 +264,13 @@ func runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m *testing.M) {
"TestRetryAlwaysFail",
"TestNormalPassingAfterRetryAlwaysFail",
},
"testify_test.go": []string{
"TestTestifyLikeTest",
},
"testify_test.go/MySuite": []string{
"TestTestifyLikeTest/TestMySuite",
"TestTestifyLikeTest/TestMySuite/sub01",
},
},
},
}, false, nil)
Expand All @@ -264,7 +294,7 @@ func runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m *testing.M) {

// 1 session span
// 1 module span
// 2 suite span (testing_test.go and reflections_test.go)
// 4 suite span (testing_test.go, testify_test.go, testify_test.go/MySuite and reflections_test.go)
// 5 tests from reflections_test.go
// 1 TestMyTest01
// 1 TestMyTest02 + 2 subtests
Expand All @@ -276,10 +306,13 @@ func runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m *testing.M) {
// 1 TestNormalPassingAfterRetryAlwaysFail
// 1 TestEarlyFlakeDetection + 10 EFD retries
// 2 normal spans from testing_test.go
// 3 tests from testify_test.go and testify_test.go/MySuite

// check spans by resource name
checkSpansByResourceName(finishedSpans, "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations/gotesting", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest01", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02", 1)
Expand All @@ -294,6 +327,14 @@ func runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m *testing.M) {
checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithFail", 4)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestNormalPassingAfterRetryAlwaysFail", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestEarlyFlakeDetection", 11)
checkSpansByResourceName(finishedSpans, "testify_test.go.TestTestifyLikeTest", 1)
testifySub01 := checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite", 1)[0]
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite/sub01", 1)

// check that testify span has the correct source file
if !strings.HasSuffix(testifySub01.Tag("test.source.file").(string), "/testify_test.go") {
panic(fmt.Sprintf("source file should be testify_test.go, got %s", testifySub01.Tag("test.source.file").(string)))
}

// check spans by tag
checkSpansByTagName(finishedSpans, constants.TestIsNew, 11)
Expand All @@ -304,8 +345,8 @@ func runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m *testing.M) {
38,
1,
1,
2,
34,
4,
37,
0)

fmt.Println("All tests passed.")
Expand Down Expand Up @@ -361,7 +402,7 @@ func runIntelligentTestRunnerTests(m *testing.M) {

// 1 session span
// 1 module span
// 2 suite span (testing_test.go and reflections_test.go)
// 4 suite span (testing_test.go, testify_test.go, testify_test.go/MySuite and reflections_test.go)
// 5 tests from reflections_test.go
// 1 TestMyTest01
// 1 TestMyTest02
Expand All @@ -372,10 +413,13 @@ func runIntelligentTestRunnerTests(m *testing.M) {
// 1 TestRetryAlwaysFail
// 1 TestNormalPassingAfterRetryAlwaysFail
// 1 TestEarlyFlakeDetection
// 3 tests from testify_test.go and testify_test.go/MySuite

// check spans by resource name
checkSpansByResourceName(finishedSpans, "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations/gotesting", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go.TestGetFieldPointerFrom", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go.TestGetInternalTestArray", 1)
Expand All @@ -395,6 +439,14 @@ func runIntelligentTestRunnerTests(m *testing.M) {
checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithFail", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestNormalPassingAfterRetryAlwaysFail", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestEarlyFlakeDetection", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go.TestTestifyLikeTest", 1)
testifySub01 := checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite", 1)[0]
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite/sub01", 1)

// check that testify span has the correct source file
if !strings.HasSuffix(testifySub01.Tag("test.source.file").(string), "/testify_test.go") {
panic(fmt.Sprintf("source file should be testify_test.go, got %s", testifySub01.Tag("test.source.file").(string)))
}

// check ITR spans
// 5 tests skipped by ITR and 1 normal skipped test
Expand All @@ -411,8 +463,8 @@ func runIntelligentTestRunnerTests(m *testing.M) {
17,
1,
1,
2,
13,
4,
16,
0)

fmt.Println("All tests passed.")
Expand Down
Loading

0 comments on commit 5f36bd8

Please sign in to comment.