Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

(feat) internal/civisibility: add support for testify Suites (v2) #3097

Merged
merged 2 commits into from
Jan 17, 2025
+330 −15
Merged

(feat) internal/civisibility: add support for testify Suites (v2) #3097

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
64a6697
(feat) internal/civisibility: add support for testify Suites
tonyredondo Jan 16, 2025
44b4013
(feat) internal/civisibility: fix test
tonyredondo Jan 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion internal/civisibility/integrations/gotesting/instrumentation_orchestrion.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,14 @@ func instrumentTestingTFunc(f func(*testing.T)) func(*testing.T) {
}

instrumentedFn := func(t *testing.T) {
// Check if we have testify suite data related to this test
testifyData := getTestifyTest(t)
if testifyData != nil {
// If we have testify data, we need to extract the module and suite name from the testify suite
moduleName = testifyData.moduleName
suiteName = testifyData.suiteName
}

// Initialize module counters if not already present.
if _, ok := modulesCounters[moduleName]; !ok {
var v int32
Expand All @@ -123,7 +131,14 @@ func instrumentTestingTFunc(f func(*testing.T)) func(*testing.T) {
module := session.GetOrCreateModule(moduleName)
suite := module.GetOrCreateSuite(suiteName)
test := suite.CreateTest(t.Name())
test.SetTestFunc(originalFunc)

// If we have testify data we use the method function from testify so the test source is properly set
if testifyData != nil {
test.SetTestFunc(testifyData.methodFunc)
} else {
// If not, let's set the original function
test.SetTestFunc(originalFunc)
}

// Get the metadata regarding the execution (in case is already created from the additional features)
execMeta := getTestMetadata(t)
Expand Down Expand Up @@ -407,3 +422,10 @@ func instrumentTestingBFunc(pb *testing.B, name string, f func(*testing.B)) (str
setCiVisibilityBenchmarkFunc(runtime.FuncForPC(reflect.Indirect(reflect.ValueOf(instrumentedFunc)).Pointer()))
return subBenchmarkAutoName, instrumentedFunc
}

// instrumentTestifySuiteRun helper function to instrument the testify Suite.Run function
//
//go:linkname instrumentTestifySuiteRun
func instrumentTestifySuiteRun(t *testing.T, suite any) {
registerTestifySuite(t, suite)
}
80 changes: 66 additions & 14 deletions internal/civisibility/integrations/gotesting/testcontroller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"net/http/httptest"
"os"
"os/exec"
"strings"
"testing"

"github.com/DataDog/dd-trace-go/v2/ddtrace/ext"
Expand Down Expand Up @@ -94,7 +95,7 @@ func runFlakyTestRetriesTests(m *testing.M) {

// 1 session span
// 1 module span
// 2 suite span (testing_test.go and reflections_test.go)
// 4 suite span (testing_test.go, testify_test.go, testify_test.go/MySuite and reflections_test.go)
// 5 tests from reflections_test.go
// 1 TestMyTest01
// 1 TestMyTest02 + 2 subtests
Expand All @@ -104,10 +105,13 @@ func runFlakyTestRetriesTests(m *testing.M) {
// 1 TestRetryWithFail + 3 retry tests from testing_test.go
// 1 TestNormalPassingAfterRetryAlwaysFail
// 1 TestEarlyFlakeDetection
// 3 tests from testify_test.go and testify_test.go/MySuite

// check spans by resource name
checkSpansByResourceName(finishedSpans, "github.com/DataDog/dd-trace-go/v2/internal/civisibility/integrations/gotesting", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest01", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02", 1)
Expand All @@ -122,6 +126,14 @@ func runFlakyTestRetriesTests(m *testing.M) {
checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithFail", 4)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestNormalPassingAfterRetryAlwaysFail", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestEarlyFlakeDetection", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go.TestTestifyLikeTest", 1)
testifySub01 := checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite", 1)[0]
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite/sub01", 1)

// check that testify span has the correct source file
if !strings.HasSuffix(testifySub01.Tag("test.source.file").(string), "/testify_test.go") {
panic(fmt.Sprintf("source file should be testify_test.go, got %s", testifySub01.Tag("test.source.file").(string)))
}

// check spans by tag
checkSpansByTagName(finishedSpans, constants.TestIsRetry, 6)
Expand All @@ -131,8 +143,8 @@ func runFlakyTestRetriesTests(m *testing.M) {
28,
1,
1,
2,
24,
4,
27,
0)

fmt.Println("All tests passed.")
Expand Down Expand Up @@ -171,7 +183,7 @@ func runEarlyFlakyTestDetectionTests(m *testing.M) {

// 1 session span
// 1 module span
// 2 suite span (testing_test.go and reflections_test.go)
// 4 suite span (testing_test.go, testify_test.go, testify_test.go/MySuite and reflections_test.go)
// 5 tests from reflections_test.go
// 11 TestMyTest01
// 11 TestMyTest02 + 22 subtests
Expand All @@ -182,10 +194,13 @@ func runEarlyFlakyTestDetectionTests(m *testing.M) {
// 11 TestNormalPassingAfterRetryAlwaysFail
// 11 TestEarlyFlakeDetection
// 22 normal spans from testing_test.go
// 33 tests from testify_test.go and testify_test.go/MySuite

// check spans by resource name
checkSpansByResourceName(finishedSpans, "github.com/DataDog/dd-trace-go/v2/internal/civisibility/integrations/gotesting", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest01", 11)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02", 11)
Expand All @@ -200,18 +215,26 @@ func runEarlyFlakyTestDetectionTests(m *testing.M) {
checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithFail", 11)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestNormalPassingAfterRetryAlwaysFail", 11)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestEarlyFlakeDetection", 11)
checkSpansByResourceName(finishedSpans, "testify_test.go.TestTestifyLikeTest", 11)
testifySub01 := checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite", 11)[0]
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite/sub01", 11)

// check that testify span has the correct source file
if !strings.HasSuffix(testifySub01.Tag("test.source.file").(string), "/testify_test.go") {
panic(fmt.Sprintf("source file should be testify_test.go, got %s", testifySub01.Tag("test.source.file").(string)))
}

// check spans by tag
checkSpansByTagName(finishedSpans, constants.TestIsNew, 143)
checkSpansByTagName(finishedSpans, constants.TestIsRetry, 130)
checkSpansByTagName(finishedSpans, constants.TestIsNew, 176)
checkSpansByTagName(finishedSpans, constants.TestIsRetry, 160)

// check spans by type
checkSpansByType(finishedSpans,
152,
1,
1,
2,
148,
4,
181,
0)

fmt.Println("All tests passed.")
Expand Down Expand Up @@ -241,6 +264,13 @@ func runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m *testing.M) {
"TestRetryAlwaysFail",
"TestNormalPassingAfterRetryAlwaysFail",
},
"testify_test.go": []string{
"TestTestifyLikeTest",
},
"testify_test.go/MySuite": []string{
"TestTestifyLikeTest/TestMySuite",
"TestTestifyLikeTest/TestMySuite/sub01",
},
},
},
}, false, nil)
Expand All @@ -264,7 +294,7 @@ func runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m *testing.M) {

// 1 session span
// 1 module span
// 2 suite span (testing_test.go and reflections_test.go)
// 4 suite span (testing_test.go, testify_test.go, testify_test.go/MySuite and reflections_test.go)
// 5 tests from reflections_test.go
// 1 TestMyTest01
// 1 TestMyTest02 + 2 subtests
Expand All @@ -276,10 +306,13 @@ func runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m *testing.M) {
// 1 TestNormalPassingAfterRetryAlwaysFail
// 1 TestEarlyFlakeDetection + 10 EFD retries
// 2 normal spans from testing_test.go
// 3 tests from testify_test.go and testify_test.go/MySuite

// check spans by resource name
checkSpansByResourceName(finishedSpans, "github.com/DataDog/dd-trace-go/v2/internal/civisibility/integrations/gotesting", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest01", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02", 1)
Expand All @@ -294,6 +327,14 @@ func runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m *testing.M) {
checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithFail", 4)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestNormalPassingAfterRetryAlwaysFail", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestEarlyFlakeDetection", 11)
checkSpansByResourceName(finishedSpans, "testify_test.go.TestTestifyLikeTest", 1)
testifySub01 := checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite", 1)[0]
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite/sub01", 1)

// check that testify span has the correct source file
if !strings.HasSuffix(testifySub01.Tag("test.source.file").(string), "/testify_test.go") {
panic(fmt.Sprintf("source file should be testify_test.go, got %s", testifySub01.Tag("test.source.file").(string)))
}

// check spans by tag
checkSpansByTagName(finishedSpans, constants.TestIsNew, 11)
Expand All @@ -304,8 +345,8 @@ func runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m *testing.M) {
38,
1,
1,
2,
34,
4,
37,
0)

fmt.Println("All tests passed.")
Expand Down Expand Up @@ -361,7 +402,7 @@ func runIntelligentTestRunnerTests(m *testing.M) {

// 1 session span
// 1 module span
// 2 suite span (testing_test.go and reflections_test.go)
// 4 suite span (testing_test.go, testify_test.go, testify_test.go/MySuite and reflections_test.go)
// 5 tests from reflections_test.go
// 1 TestMyTest01
// 1 TestMyTest02
Expand All @@ -372,10 +413,13 @@ func runIntelligentTestRunnerTests(m *testing.M) {
// 1 TestRetryAlwaysFail
// 1 TestNormalPassingAfterRetryAlwaysFail
// 1 TestEarlyFlakeDetection
// 3 tests from testify_test.go and testify_test.go/MySuite

// check spans by resource name
checkSpansByResourceName(finishedSpans, "github.com/DataDog/dd-trace-go/v2/internal/civisibility/integrations/gotesting", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go.TestGetFieldPointerFrom", 1)
checkSpansByResourceName(finishedSpans, "reflections_test.go.TestGetInternalTestArray", 1)
Expand All @@ -395,6 +439,14 @@ func runIntelligentTestRunnerTests(m *testing.M) {
checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithFail", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestNormalPassingAfterRetryAlwaysFail", 1)
checkSpansByResourceName(finishedSpans, "testing_test.go.TestEarlyFlakeDetection", 1)
checkSpansByResourceName(finishedSpans, "testify_test.go.TestTestifyLikeTest", 1)
testifySub01 := checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite", 1)[0]
checkSpansByResourceName(finishedSpans, "testify_test.go/MySuite.TestTestifyLikeTest/TestMySuite/sub01", 1)

// check that testify span has the correct source file
if !strings.HasSuffix(testifySub01.Tag("test.source.file").(string), "/testify_test.go") {
panic(fmt.Sprintf("source file should be testify_test.go, got %s", testifySub01.Tag("test.source.file").(string)))
}

// check ITR spans
// 5 tests skipped by ITR and 1 normal skipped test
Expand All @@ -411,8 +463,8 @@ func runIntelligentTestRunnerTests(m *testing.M) {
17,
1,
1,
2,
13,
4,
16,
0)

fmt.Println("All tests passed.")
Expand Down
Loading
Loading