Refactor build failure library to use metrics data (#515)

Signed-off-by: Sayali Gaikawad <[email protected]>
opensearch-project · Oct 7, 2024 · 87cf890 · 87cf890
1 parent 3e3106d
commit 87cf890
Show file tree

Hide file tree

Showing 23 changed files with 474 additions and 681 deletions.
diff --git a/build.gradle b/build.gradle
@@ -51,7 +51,7 @@ sourceSets {
 
     test {
         groovy {
-            srcDirs = ['tests/jenkins', 'tests/gradlecheck', 'tests/utils']
+            srcDirs = ['tests/jenkins', 'tests/gradlecheck']
         }
     }
 
@@ -128,7 +128,7 @@ jacocoTestReport {
     }
 }
 
-String version = '6.9.2'
+String version = '7.0.0'
 
 task updateVersion {
     doLast {

diff --git a/src/jenkins/ComponentBuildStatus.groovy b/src/jenkins/ComponentBuildStatus.groovy
@@ -0,0 +1,91 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package jenkins
+
+import groovy.json.JsonOutput
+import utils.OpenSearchMetricsQuery
+
+class ComponentBuildStatus {
+    String metricsUrl
+    String awsAccessKey
+    String awsSecretKey
+    String awsSessionToken
+    String indexName
+    String product 
+    String version
+    String distributionBuildNumber
+    String buildStartTimeFrom
+    String buildStartTimeTo
+    def script
+
+    ComponentBuildStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String product, String version, String distributionBuildNumber, String buildStartTimeFrom, String buildStartTimeTo, def script) {
+        this.metricsUrl = metricsUrl
+        this.awsAccessKey = awsAccessKey
+        this.awsSecretKey = awsSecretKey
+        this.awsSessionToken = awsSessionToken
+        this.indexName = indexName
+        this.product = product
+        this.version = version
+        this.distributionBuildNumber = distributionBuildNumber
+        this.buildStartTimeFrom = buildStartTimeFrom
+        this.buildStartTimeTo = buildStartTimeTo
+        this.script = script
+    }
+
+    def getQuery(String componentBuildResult) {
+        def queryMap = [
+                _source: [
+                        "component",
+                        ], 
+                query: [
+                        bool: [
+                                filter: [
+                                        [
+                                                match_phrase: [
+                                                        component_category: "${this.product}"
+                                                        ]
+                                                ],
+                                        [
+                                                match_phrase: [
+                                                        component_build_result: "${componentBuildResult}"
+                                                ]
+                                        ],
+                                        [
+                                                match_phrase: [
+                                                        version: "${this.version}"
+                                                ]
+                                        ],
+                                        [
+                                                match_phrase : [
+                                                        distribution_build_number : "${this.distributionBuildNumber}"
+                                                ]
+                                        ],
+                                        [
+                                                range: [
+                                                        build_start_time: [
+                                                                from: "${this.buildStartTimeFrom}",
+                                                                to: "${this.buildStartTimeTo}"
+                                                                ]
+                                                        ]
+                                        ]
+                                ]
+                        ]
+                ]
+        ]
+        def query = JsonOutput.toJson(queryMap)
+        return query.replace('"', '\\"')
+    }
+
+    def getComponents(String componentBuildResult) {
+         def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, indexName, script).fetchMetrics(getQuery(componentBuildResult))
+         def components = jsonResponse.hits.hits.collect { it._source.component }
+         return components
+    }
+}
diff --git a/src/utils/OpenSearchMetricsQuery.groovy b/src/utils/OpenSearchMetricsQuery.groovy
@@ -28,15 +28,14 @@ class OpenSearchMetricsQuery {
         this.script = script
     }
 
-    // Ensure the alias `gradle-check` is created targeting all the gradle-check-* indices.
     def fetchMetrics(String query) {
         def response = script.sh(
             script: """
             set -e
             set +x
             curl -s -XGET "${metricsUrl}/${indexName}/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "${awsAccessKey}:${awsSecretKey}" -H "x-amz-security-token:${awsSessionToken}" -H 'Content-Type: application/json' -d "${query}" | jq '.'
         """,
-                returnStdout: true
+        returnStdout: true
         ).trim()
         return new JsonSlurper().parseText(response)
     }

diff --git a/tests/jenkins/TestCloseBuildSuccessGithubIssue.groovy b/tests/jenkins/TestCloseBuildSuccessGithubIssue.groovy
diff --git a/tests/jenkins/TestComponentBuildStatus.groovy b/tests/jenkins/TestComponentBuildStatus.groovy
@@ -0,0 +1,124 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package jenkins
+
+import org.junit.*
+import groovy.json.JsonOutput
+import groovy.mock.interceptor.MockFor
+
+class TestComponentBuildStatus {
+
+    private ComponentBuildStatus componentBuildStatus
+    private final String metricsUrl = 'http://example.com'
+    private final String awsAccessKey = 'testAccessKey'
+    private final String awsSecretKey = 'testSecretKey'
+    private final String awsSessionToken = 'testSessionToken'
+    private final String indexName = 'opensearch-distribution-build-results-*'
+    private final String product = "OpenSearch"
+    private final String version = "2.18.0"
+    private final String distributionBuildNumber = "4891"
+    private final String buildStartTimeFrom = "now-6h"
+    private final String buildStartTimeTo = "now"
+    private def script
+
+    @Before
+    void setUp() {
+        script = new Expando()
+        script.sh = { Map args ->
+            if (args.containsKey("script")) {
+                return """
+                {
+                    "hits": {
+                    "total": {
+                    "value": 2,
+                    "relation": "eq"
+                    },
+                    "max_score": 0,
+                    "hits": [
+                    {
+                        "_index": "opensearch-distribution-build-results-09-2024",
+                        "_id": "QTVbQZIBOi-lzDIlekCk",
+                        "_score": 0,
+                        "_source": {
+                        "component": "performance-analyzer"
+                        }
+                    },
+                    {
+                        "_index": "opensearch-distribution-build-results-09-2024",
+                        "_id": "PzVbQZIBOi-lzDIlekCk",
+                        "_score": 0,
+                        "_source": {
+                        "component": "security-analytics"
+                        }
+                    }
+                    ]
+                }
+                }
+                """
+            }
+            return ""
+        }
+        componentBuildStatus = new ComponentBuildStatus(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, product, version, distributionBuildNumber, buildStartTimeFrom, buildStartTimeTo, script)
+    }
+
+    @Test
+    void testGetQueryReturnsExpectedQuery() {
+        def expectedOutput = JsonOutput.toJson([
+                _source: [
+                        "component",
+                        ], 
+                query: [
+                        bool: [
+                                filter: [
+                                        [
+                                                match_phrase: [
+                                                        component_category: "OpenSearch"
+                                                        ]
+                                                ],
+                                        [
+                                                match_phrase: [
+                                                        component_build_result: "failed"
+                                                ]
+                                        ],
+                                        [
+                                                match_phrase: [
+                                                        version: "2.18.0"
+                                                ]
+                                        ],
+                                        [
+                                                match_phrase : [
+                                                        distribution_build_number : "4891"
+                                                ]
+                                        ],
+                                        [
+                                                range: [
+                                                        build_start_time: [
+                                                                from: "now-6h",
+                                                                to: "now"
+                                                                ]
+                                                        ]
+                                        ]
+                                ]
+                        ]
+                ]
+        ]).replace('"', '\\"')
+
+        def result = componentBuildStatus.getQuery('failed')
+        assert result == expectedOutput
+    }
+
+    @Test
+    void testComponentBuildStatusReturns() {
+        def expectedOutput = ['performance-analyzer', 'security-analytics']
+        def result = componentBuildStatus.getComponents('failed')
+
+        assert result == expectedOutput
+    }
+}
diff --git a/tests/jenkins/TestCreateBuildFailureGithubIssue.groovy b/tests/jenkins/TestCreateBuildFailureGithubIssue.groovy
diff --git a/tests/jenkins/TestPublishDistributionBuildResults.groovy b/tests/jenkins/TestPublishDistributionBuildResults.groovy
@@ -106,6 +106,9 @@ class TestPublishDistributionBuildResults extends BuildPipelineTest {
                         "type": "keyword"
                     }
                 }
+            },
+            "aliases": {
+                "opensearch-distribution-build-results": {}
             }
         }'
         curl -I "METRICS_HOST_URL/test-index" --aws-sigv4 "aws:amz:us-east-1:es" --user "null:null" -H "x-amz-security-token:null" | grep -E "HTTP\\/[0-9]+(\\.[0-9]+)? 200"