diff --git a/tests/jenkins/TestPublishIntegTestResults.groovy b/tests/jenkins/TestPublishIntegTestResults.groovy index 173bc9e7..98ecf0e8 100644 --- a/tests/jenkins/TestPublishIntegTestResults.groovy +++ b/tests/jenkins/TestPublishIntegTestResults.groovy @@ -156,9 +156,6 @@ class TestPublishIntegTestResults extends BuildPipelineTest { "without_security_test_stderr": { "type": "keyword" } - }, - "aliases": { - "opensearch-integration-test-results": {} } } }' @@ -170,6 +167,22 @@ class TestPublishIntegTestResults extends BuildPipelineTest { create_index_response=$(curl -s -XPUT "METRICS_HOST_URL/test-index" --aws-sigv4 "aws:amz:us-east-1:es" --user "null:null" -H "x-amz-security-token:null" -H 'Content-Type: application/json' -d "${INDEX_MAPPING}") if [[ $create_index_response == *'"acknowledged":true'* ]]; then echo "Index created successfully." + echo "Updating alias..." + update_alias_response=\$(curl -s -XPOST "METRICS_HOST_URL/_aliases" --aws-sigv4 "aws:amz:us-east-1:es" --user "null:null" -H "x-amz-security-token:null" -H "Content-Type: application/json" -d '{ + "actions": [ + { + "add": { + "index": "test-index", + "alias": "opensearch-integration-test-results" + } + } + ] + }') + if [[ \$update_alias_response == *'"acknowledged":true'* ]]; then + echo "Alias updated successfully." + else + echo "Failed to update alias. Error message: \$update_alias_response" + fi else echo "Failed to create index. Error message: $create_index_response" exit 1 @@ -185,6 +198,132 @@ class TestPublishIntegTestResults extends BuildPipelineTest { assert normalizeString(calledCommands[0]) == normalizeString(expectedCommandBlock) } + @Test + void testIndexTestFailuresData() { + def indexName = 'opensearch-integration-test-failures-test-index' + def testRecordsFile = 'test-failures.json' + + def script = loadScript('vars/publishIntegTestResults.groovy') + + def calledCommands = new ArrayList() + script.metaClass.sh = { String command -> + calledCommands << command + if (command.contains("curl -I")) { + return "HTTP/1.1 200 OK" + } else if (command.contains("curl -s -XPUT") && command.contains(indexName)) { + return '{"acknowledged":true}' + } else if (command.contains("curl -XPOST") && command.contains(indexName)) { + return '{"took":10, "errors":false}' + } else { + throw new IllegalArgumentException("Unexpected command: $command") + } + } + + script.indexTestFailuresData(indexName, testRecordsFile) + + def expectedCommandBlock = '''set +e + set +x + echo "INDEX NAME IS opensearch-integration-test-failures-test-index" + INDEX_MAPPING='{ + "mappings": { + "properties": { + "component": { + "type": "keyword" + }, + "component_repo": { + "type": "keyword" + }, + "component_repo_url": { + "type": "keyword" + }, + "version": { + "type": "keyword" + }, + "integ_test_build_number": { + "type": "integer" + }, + "integ_test_build_url": { + "type": "keyword" + }, + "distribution_build_number": { + "type": "integer" + }, + "distribution_build_url": { + "type": "keyword" + }, + "build_start_time": { + "type": "date", + "format": "epoch_millis" + }, + "rc": { + "type": "keyword" + }, + "rc_number": { + "type": "integer" + }, + "platform": { + "type": "keyword" + }, + "architecture": { + "type": "keyword" + }, + "distribution": { + "type": "keyword" + }, + "component_category": { + "type": "keyword" + }, + "test_type": { + "type": "keyword" + }, + "test_class": { + "type": "keyword" + }, + "test_name": { + "type": "keyword" + } + } + } + }' + curl -I "METRICS_HOST_URL/opensearch-integration-test-failures-test-index" --aws-sigv4 "aws:amz:us-east-1:es" --user "null:null" -H "x-amz-security-token:null" | grep -E "HTTP\\/[0-9]+(\\.[0-9]+)? 200" + if [ $? -eq 0 ]; then + echo "Index already exists. Indexing Results" + else + echo "Index does not exist. Creating..." + create_index_response=$(curl -s -XPUT "METRICS_HOST_URL/opensearch-integration-test-failures-test-index" --aws-sigv4 "aws:amz:us-east-1:es" --user "null:null" -H "x-amz-security-token:null" -H 'Content-Type: application/json' -d "${INDEX_MAPPING}") + if [[ $create_index_response == *'"acknowledged":true'* ]]; then + echo "Index created successfully." + echo "Updating alias..." + update_alias_response=$(curl -s -XPOST "METRICS_HOST_URL/_aliases" --aws-sigv4 "aws:amz:us-east-1:es" --user "null:null" -H "x-amz-security-token:null" -H "Content-Type: application/json" -d '{ + "actions": [ + { + "add": { + "index": "opensearch-integration-test-failures-test-index", + "alias": "opensearch-integration-test-failures" + } + } + ] + }') + if [[ $update_alias_response == *'"acknowledged":true'* ]]; then + echo "Alias updated successfully." + else + echo "Failed to update alias. Error message: $update_alias_response" + fi + else + echo "Failed to create index. Error message: $create_index_response" + exit 1 + fi + fi + if [ -s test-failures.json ]; then + echo "File Exists, indexing results." + curl -XPOST "METRICS_HOST_URL/opensearch-integration-test-failures-test-index/_bulk" --aws-sigv4 "aws:amz:us-east-1:es" --user "null:null" -H "x-amz-security-token:null" -H "Content-Type: application/x-ndjson" --data-binary "@test-failures.json" + else + echo "File Does not exist. No tests records to process." + fi''' + + assert calledCommands.size() == 1 + assert normalizeString(calledCommands[0]) == normalizeString(expectedCommandBlock) + } @Test void testGenerateJson() { @@ -235,6 +374,185 @@ class TestPublishIntegTestResults extends BuildPipelineTest { assert parsedResult == expectedJson } + @Test + void testGenerateFailedTestJson() { + def script = loadScript('vars/publishIntegTestResults.groovy') + + def result = script.generatefailedTestJson( + 'component1', 'componentRepo', 'https://componentRepoUrl', '1.0', 123, + 'http://example.com/build/123', 456, 'http://example.com/distribution/456', + System.currentTimeMillis(), 'rc1', 1, 'linux', 'x64', 'tar', 'test_category', + 'test_type', 'test_class', 'test_name' + ) + + def parsedResult = new JsonSlurper().parseText(result) + def expectedJson = [ + component: 'component1', + component_repo: 'componentRepo', + component_repo_url: 'https://componentRepoUrl', + version: '1.0', + integ_test_build_number: 123, + integ_test_build_url: 'http://example.com/build/123', + distribution_build_number: 456, + distribution_build_url: 'http://example.com/distribution/456', + rc: 'rc1', + rc_number: 1, + platform: 'linux', + architecture: 'x64', + distribution: 'tar', + component_category: 'test_category', + test_type: 'test_type', + test_class: 'test_class', + test_name: 'test_name' + ] + + // Remove the dynamic field for comparison + parsedResult.remove('build_start_time') + assert parsedResult == expectedJson + } + + @Test + void testProcessFailedTestsWithEmptyList() { + def failedTests = [] + def componentName = "MyComponent" + def componentRepo = "my-repo" + def componentRepoUrl = "https://example.com/my-repo" + def version = "1.0.0" + def integTestBuildNumber = 123 + def integTestBuildUrl = "https://example.com/builds/123" + def distributionBuildNumber = 456 + def distributionBuildUrl = "https://example.com/builds/456" + def buildStartTime = "2023-10-17T12:00:00Z" + def rc = "RC1" + def rcNumber = 1 + def platform = "Linux" + def architecture = "x86_64" + def distribution = "Ubuntu 22.04" + def componentCategory = "Backend" + def securityType = "Web Application" + def testFailuresindexName = "test-failures-index" + + def script = loadScript('vars/publishIntegTestResults.groovy') + def result = script.processFailedTests(failedTests, componentName, componentRepo, componentRepoUrl, version, integTestBuildNumber, integTestBuildUrl, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, platform, architecture, distribution, componentCategory, securityType, testFailuresindexName) + assert result == null + } + + @Test + void testProcessFailedTestsWithEmptyList() { + def script = loadScript('vars/publishIntegTestResults.groovy') + + def result = script.processFailedTests([], 'component1', 'componentRepo', 'https://componentRepoUrl', '1.0', + 123, 'http://example.com/build/123', 456, 'http://example.com/distribution/456', + System.currentTimeMillis(), 'rc1', 1, 'linux', 'x64', 'tar', 'test_category', + 'test_type', 'test_failures_index') + + assertEquals("", result) + } + + @Test + void testProcessFailedTestsWithTestResultNotAvailable() { + def script = loadScript('vars/publishIntegTestResults.groovy') + + def result = script.processFailedTests(['Test Result Not Available'], 'component1', 'componentRepo', + 'https://componentRepoUrl', '1.0', 123, 'http://example.com/build/123', 456, + 'http://example.com/distribution/456', System.currentTimeMillis(), 'rc1', 1, + 'linux', 'x64', 'tar', 'test_category', 'test_type', 'test_failures_index') + + def expectedJson = [ + component: 'component1', + component_repo: 'componentRepo', + component_repo_url: 'https://componentRepoUrl', + version: '1.0', + integ_test_build_number: 123, + integ_test_build_url: 'http://example.com/build/123', + distribution_build_number: 456, + distribution_build_url: 'http://example.com/distribution/456', + rc: 'rc1', + rc_number: 1, + platform: 'linux', + architecture: 'x64', + distribution: 'tar', + component_category: 'test_category', + test_type: 'test_type', + test_class: 'Not Available', + test_name: 'Not Available' + ] + + def parsedResult = new JsonSlurper().parseText(result) + parsedResult.remove('build_start_time') // Remove dynamic fields for comparison + assert parsedResult == expectedJson + } + + @Test + void testProcessFailedTestsWithNoFailedTest() { + def script = loadScript('vars/publishIntegTestResults.groovy') + + def result = script.processFailedTests(['No Failed Test'], 'component1', 'componentRepo', + 'https://componentRepoUrl', '1.0', 123, 'http://example.com/build/123', 456, + 'http://example.com/distribution/456', System.currentTimeMillis(), 'rc1', 1, + 'linux', 'x64', 'tar', 'test_category', 'test_type', 'test_failures_index') + + assertEquals("", result) + } + + @Test + void testProcessFailedTestsWithMultipleFailures() { + def script = loadScript('vars/publishIntegTestResults.groovy') + + def result = script.processFailedTests(['Test1#Failure1', 'Test2#Failure2'], 'component1', + 'componentRepo', 'https://componentRepoUrl', '1.0', 123, 'http://example.com/build/123', 456, + 'http://example.com/distribution/456', System.currentTimeMillis(), 'rc1', 1, + 'linux', 'x64', 'tar', 'test_category', 'test_type', 'test_failures_index') + + def parsedResult = new JsonSlurper().parseText(result) + + def expectedJson1 = [ + component: 'component1', + component_repo: 'componentRepo', + component_repo_url: 'https://componentRepoUrl', + version: '1.0', + integ_test_build_number: 123, + integ_test_build_url: 'http://example.com/build/123', + distribution_build_number: 456, + distribution_build_url: 'http://example.com/distribution/456', + rc: 'rc1', + rc_number: 1, + platform: 'linux', + architecture: 'x64', + distribution: 'tar', + component_category: 'test_category', + test_type: 'test_type', + test_class: 'Test1', + test_name: 'Failure1' + ] + + def expectedJson2 = [ + component: 'component1', + component_repo: 'componentRepo', + component_repo_url: 'https://componentRepoUrl', + version: '1.0', + integ_test_build_number: 123, + integ_test_build_url: 'http://example.com/build/123', + distribution_build_number: 456, + distribution_build_url: 'http://example.com/distribution/456', + rc: 'rc1', + rc_number: 1, + platform: 'linux', + architecture: 'x64', + distribution: 'tar', + component_category: 'test_category', + test_type: 'test_type', + test_class: 'Test2', + test_name: 'Failure2' + ] + + // Remove the dynamic field for comparison + parsedResult.remove('build_start_time') + assert parsedResult.contains(expectedJson1) + assert parsedResult.contains(expectedJson2) + } + + @Test void testComponentResultWithSecurityFail() { def withSecurity = 'fail' diff --git a/vars/publishIntegTestResults.groovy b/vars/publishIntegTestResults.groovy index e3646cf2..989dbdef 100644 --- a/vars/publishIntegTestResults.groovy +++ b/vars/publishIntegTestResults.groovy @@ -42,6 +42,7 @@ void call(Map args = [:]) { def manifestFile = readFile testReportManifestYml def manifest = readYaml text: manifestFile def indexName = "opensearch-integration-test-results-${formattedDate}" + def testFailuresindexName = "opensearch-integration-test-failures-${formattedDate}" def finalJsonDoc = "" def version = manifest.version.toString() def distributionBuildNumber = manifest.id @@ -70,6 +71,19 @@ void call(Map args = [:]) { def withoutSecurityClusterStderr = component.configs.find { it.name == 'without-security' }?.cluster_stderr ?: [] def withoutSecurityTestStdout = component.configs.find { it.name == 'without-security' }?.test_stdout ?: '' def withoutSecurityTestStderr = component.configs.find { it.name == 'without-security' }?.test_stderr ?: '' + + def withSecurityFailedTests = component.configs.find { it.name == 'with-security' }?.failed_test ?: [] + processFailedTests(withSecurityFailedTests, componentName, componentRepo, componentRepoUrl, version, integTestBuildNumber, + integTestBuildUrl, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, + platform, architecture, distribution, componentCategory, "with-security", testFailuresindexName) + + + def withoutSecurityFailedTests = component.configs.find { it.name == 'without-security' }?.failed_test ?: [] + processFailedTests(withoutSecurityFailedTests, componentName, componentRepo, componentRepoUrl, version, integTestBuildNumber, + integTestBuildUrl, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, + platform, architecture, distribution, componentCategory, "without-security", testFailuresindexName) + + def jsonContent = generateJson( componentName, componentRepo, componentRepoUrl, version, integTestBuildNumber, integTestBuildUrl, distributionBuildNumber, distributionBuildUrl, @@ -88,8 +102,161 @@ void call(Map args = [:]) { indexFailedTestData(indexName, "test-records.json") } +def processFailedTests(failedTests, componentName, componentRepo, componentRepoUrl, version, integTestBuildNumber, + integTestBuildUrl, distributionBuildNumber, distributionBuildUrl, buildStartTime, + rc, rcNumber, platform, architecture, distribution, componentCategory, securityType, testFailuresindexName) { + + def finalFailedTestsJsonDoc = "" + switch (true) { + // The tests are not recorded. + case failedTests.isEmpty(): + break + case failedTests.contains("Test Result Not Available"): + def testResultJsonContent = generatefailedTestJson(componentName, componentRepo, componentRepoUrl, version, integTestBuildNumber, + integTestBuildUrl, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, + platform, architecture, distribution, componentCategory, securityType, "Not Available", "Not Available") + finalFailedTestsJsonDoc += "{\"index\": {\"_index\": \"${testFailuresindexName}\"}}\n${testResultJsonContent}\n" + break + // The tests pass and no failed tests. + case failedTests.contains("No Failed Test"): + break + default: + failedTests.collect { failedTest -> + def match = failedTest.split("#") + if (match) { + def testResultJsonContent = generatefailedTestJson(componentName, componentRepo, componentRepoUrl, version, integTestBuildNumber, + integTestBuildUrl, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, + platform, architecture, distribution, componentCategory, securityType, match[0].trim(), match[1].trim()) + finalFailedTestsJsonDoc += "{\"index\": {\"_index\": \"${testFailuresindexName}\"}}\n${testResultJsonContent}\n" + } + } + break + } + if (!finalFailedTestsJsonDoc.isEmpty()) { + writeFile file: "test-failures.json", text: finalFailedTestsJsonDoc + def fileContents = readFile(file: "test-failures.json").trim() + indexTestFailuresData(testFailuresindexName, "test-failures.json") + } + return finalFailedTestsJsonDoc +} + + boolean argCheck(String str) { return (str == null || str.allWhitespace || str.isEmpty()) } +void indexTestFailuresData(testFailuresindexName, testFailuresFile) { + withCredentials([ + string(credentialsId: 'jenkins-health-metrics-account-number', variable: 'METRICS_HOST_ACCOUNT'), + string(credentialsId: 'jenkins-health-metrics-cluster-endpoint', variable: 'METRICS_HOST_URL') + ]) { + withAWS(role: 'OpenSearchJenkinsAccessRole', roleAccount: "${METRICS_HOST_ACCOUNT}", duration: 900, roleSessionName: 'jenkins-session') { + def awsAccessKey = env.AWS_ACCESS_KEY_ID + def awsSecretKey = env.AWS_SECRET_ACCESS_KEY + def awsSessionToken = env.AWS_SESSION_TOKEN + sh """ + set +e + set +x + echo "INDEX NAME IS ${testFailuresindexName}" + INDEX_MAPPING='{ + "mappings": { + "properties": { + "component": { + "type": "keyword" + }, + "component_repo": { + "type": "keyword" + }, + "component_repo_url": { + "type": "keyword" + }, + "version": { + "type": "keyword" + }, + "integ_test_build_number": { + "type": "integer" + }, + "integ_test_build_url": { + "type": "keyword" + }, + "distribution_build_number": { + "type": "integer" + }, + "distribution_build_url": { + "type": "keyword" + }, + "build_start_time": { + "type": "date", + "format": "epoch_millis" + }, + "rc": { + "type": "keyword" + }, + "rc_number": { + "type": "integer" + }, + "platform": { + "type": "keyword" + }, + "architecture": { + "type": "keyword" + }, + "distribution": { + "type": "keyword" + }, + "component_category": { + "type": "keyword" + }, + "test_type": { + "type": "keyword" + }, + "test_class": { + "type": "keyword" + }, + "test_name": { + "type": "keyword" + } + } + } + }' + curl -I "${METRICS_HOST_URL}/${testFailuresindexName}" --aws-sigv4 \"aws:amz:us-east-1:es\" --user \"${awsAccessKey}:${awsSecretKey}\" -H \"x-amz-security-token:${awsSessionToken}\" | grep -E "HTTP\\/[0-9]+(\\.[0-9]+)? 200" + if [ \$? -eq 0 ]; then + echo "Index already exists. Indexing Results" + else + echo "Index does not exist. Creating..." + create_index_response=\$(curl -s -XPUT "${METRICS_HOST_URL}/${testFailuresindexName}" --aws-sigv4 \"aws:amz:us-east-1:es\" --user \"${awsAccessKey}:${awsSecretKey}\" -H \"x-amz-security-token:${awsSessionToken}\" -H 'Content-Type: application/json' -d "\${INDEX_MAPPING}") + if [[ \$create_index_response == *'"acknowledged":true'* ]]; then + echo "Index created successfully." + echo "Updating alias..." + update_alias_response=\$(curl -s -XPOST "${METRICS_HOST_URL}/_aliases" --aws-sigv4 "aws:amz:us-east-1:es" --user "${awsAccessKey}:${awsSecretKey}" -H "x-amz-security-token:${awsSessionToken}" -H "Content-Type: application/json" -d '{ + "actions": [ + { + "add": { + "index": "${testFailuresindexName}", + "alias": "opensearch-integration-test-failures" + } + } + ] + }') + if [[ \$update_alias_response == *'"acknowledged":true'* ]]; then + echo "Alias updated successfully." + else + echo "Failed to update alias. Error message: \$update_alias_response" + fi + else + echo "Failed to create index. Error message: \$create_index_response" + exit 1 + fi + fi + if [ -s ${testFailuresFile} ]; then + echo "File Exists, indexing results." + curl -XPOST "${METRICS_HOST_URL}/$testFailuresindexName/_bulk" --aws-sigv4 \"aws:amz:us-east-1:es\" --user \"${awsAccessKey}:${awsSecretKey}\" -H \"x-amz-security-token:${awsSessionToken}\" -H "Content-Type: application/x-ndjson" --data-binary "@${testFailuresFile}" + else + echo "File Does not exist. No tests records to process." + fi + """ + } + } + } + void indexFailedTestData(indexName, testRecordsFile) { withCredentials([ string(credentialsId: 'jenkins-health-metrics-account-number', variable: 'METRICS_HOST_ACCOUNT'), @@ -194,9 +361,6 @@ void indexFailedTestData(indexName, testRecordsFile) { "without_security_test_stderr": { "type": "keyword" } - }, - "aliases": { - "opensearch-integration-test-results": {} } } }' @@ -208,6 +372,22 @@ void indexFailedTestData(indexName, testRecordsFile) { create_index_response=\$(curl -s -XPUT "${METRICS_HOST_URL}/${indexName}" --aws-sigv4 \"aws:amz:us-east-1:es\" --user \"${awsAccessKey}:${awsSecretKey}\" -H \"x-amz-security-token:${awsSessionToken}\" -H 'Content-Type: application/json' -d "\${INDEX_MAPPING}") if [[ \$create_index_response == *'"acknowledged":true'* ]]; then echo "Index created successfully." + echo "Updating alias..." + update_alias_response=\$(curl -s -XPOST "${METRICS_HOST_URL}/_aliases" --aws-sigv4 "aws:amz:us-east-1:es" --user "${awsAccessKey}:${awsSecretKey}" -H "x-amz-security-token:${awsSessionToken}" -H "Content-Type: application/json" -d '{ + "actions": [ + { + "add": { + "index": "${indexName}", + "alias": "opensearch-integration-test-results" + } + } + ] + }') + if [[ \$update_alias_response == *'"acknowledged":true'* ]]; then + echo "Alias updated successfully." + else + echo "Failed to update alias. Error message: \$update_alias_response" + fi else echo "Failed to create index. Error message: \$create_index_response" exit 1 @@ -224,6 +404,34 @@ void indexFailedTestData(indexName, testRecordsFile) { } } +def generatefailedTestJson(componentName, componentRepo, componentRepoUrl, version, + integTestBuildNumber, integTestBuildUrl, distributionBuildNumber, distributionBuildUrl, + buildStartTime, rc, rcNumber, platform, architecture, distribution, componentCategory, + testType, testClass, testName) { + def json = [ + component: componentName, + component_repo: componentRepo, + component_repo_url: componentRepoUrl, + version: version, + integ_test_build_number: integTestBuildNumber, + integ_test_build_url: integTestBuildUrl, + distribution_build_number: distributionBuildNumber, + distribution_build_url: distributionBuildUrl, + build_start_time: buildStartTime, + rc: rc, + rc_number: rcNumber, + platform: platform, + architecture: architecture, + distribution: distribution, + component_category: componentCategory, + test_type: testType, + test_class: testClass, + test_name: testName + ] + return JsonOutput.toJson(json) +} + + def generateJson(componentName, componentRepo, componentRepoUrl, version, integTestBuildNumber, integTestBuildUrl, distributionBuildNumber, distributionBuildUrl, buildStartTime, rc, rcNumber, platform, architecture, distribution, componentCategory,