Merge branch 'opensearch-project:2.x' into 2.x

YANG-DB · Nov 8, 2024 · 10b03b7 · 10b03b7
2 parents 29a1e55 + 1be7b30
commit 10b03b7
Show file tree

Hide file tree

Showing 196 changed files with 8,844 additions and 4,081 deletions.
diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml
@@ -47,7 +47,7 @@ jobs:
 
     - name: Upload test reports
       if: ${{ always() }}
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       continue-on-error: true
       with:
         name: test-reports-${{ matrix.os }}-${{ matrix.java }}
@@ -79,7 +79,7 @@ jobs:
 
     - name: Upload test reports
       if: ${{ always() }}
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       continue-on-error: true
       with:
         name: test-reports-${{ matrix.os }}-${{ matrix.java }}

diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml
@@ -76,14 +76,14 @@ jobs:
         token: ${{ secrets.CODECOV_TOKEN }}
 
     - name: Upload Artifacts
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: opensearch-sql-ubuntu-latest
         path: opensearch-sql-builds
 
     - name: Upload test reports
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       continue-on-error: true
       with:
         name: test-reports
@@ -130,7 +130,7 @@ jobs:
         cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/
 
     - name: Upload Artifacts
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: opensearch-sql-${{ matrix.entry.os }}
         path: opensearch-sql-builds
diff --git a/async-query-core/README.md b/async-query-core/README.md
@@ -32,3 +32,11 @@ Following is the list of extension points where the consumer of the library need
   - [DataSourceSparkParameterComposer](src/main/java/org/opensearch/sql/spark/parameter/DataSourceSparkParameterComposer.java)
   - [GeneralSparkParameterComposer](src/main/java/org/opensearch/sql/spark/parameter/GeneralSparkParameterComposer.java)
   - [SparkSubmitParameterModifier](src/main/java/org/opensearch/sql/spark/config/SparkSubmitParameterModifier.java) To be deprecated in favor of GeneralSparkParameterComposer
+
+## Update Grammar files
+This package uses ANTLR grammar files from `opensearch-spark` and `Spark` repositories.
+To update the grammar files, update `build.gradle` file (in `downloadG4Files` task) as needed and run:
+```
+./gradlew async-query-core:downloadG4Files
+```
+This will overwrite the files under `src/main/antlr`.
diff --git a/async-query-core/build.gradle b/async-query-core/build.gradle
@@ -21,10 +21,12 @@ tasks.register('downloadG4Files', Exec) {
 
     executable 'curl'
 
-    args '-o', 'src/main/antlr/FlintSparkSqlExtensions.g4', 'https://raw.githubusercontent.com/opensearch-project/opensearch-spark/main/flint-spark-integration/src/main/antlr4/FlintSparkSqlExtensions.g4'
-    args '-o', 'src/main/antlr/SparkSqlBase.g4', 'https://raw.githubusercontent.com/opensearch-project/opensearch-spark/main/flint-spark-integration/src/main/antlr4/SparkSqlBase.g4'
-    args '-o', 'src/main/antlr/SqlBaseParser.g4', 'https://raw.githubusercontent.com/apache/spark/master/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4'
-    args '-o', 'src/main/antlr/SqlBaseLexer.g4', 'https://raw.githubusercontent.com/apache/spark/master/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4'
+    def opensearchSparkBranch = "0.5"
+    def apacheSparkVersionTag = "v3.5.1"
+    args '-o', 'src/main/antlr/FlintSparkSqlExtensions.g4', "https://raw.githubusercontent.com/opensearch-project/opensearch-spark/${opensearchSparkBranch}/flint-spark-integration/src/main/antlr4/FlintSparkSqlExtensions.g4"
+    args '-o', 'src/main/antlr/SparkSqlBase.g4', "https://raw.githubusercontent.com/opensearch-project/opensearch-spark/${opensearchSparkBranch}/flint-spark-integration/src/main/antlr4/SparkSqlBase.g4"
+    args '-o', 'src/main/antlr/SqlBaseParser.g4', "https://raw.githubusercontent.com/apache/spark/${apacheSparkVersionTag}/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4"
+    args '-o', 'src/main/antlr/SqlBaseLexer.g4', "https://raw.githubusercontent.com/apache/spark/${apacheSparkVersionTag}/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4"
 }
 
 generateGrammarSource {
@@ -38,12 +40,6 @@ configurations {
     }
 }
 
-// skip download in case of offline build
-if (!gradle.startParameter.offline) {
-    // Make sure the downloadG4File task runs before the generateGrammarSource task
-    generateGrammarSource.dependsOn downloadG4Files
-}
-
 dependencies {
     antlr "org.antlr:antlr4:4.7.1"
 
@@ -122,7 +118,8 @@ jacocoTestCoverageVerification {
                     'org.opensearch.sql.spark.flint.*',
                     'org.opensearch.sql.spark.flint.operation.*',
                     'org.opensearch.sql.spark.rest.*',
-                    'org.opensearch.sql.spark.utils.SQLQueryUtils.*'
+                    'org.opensearch.sql.spark.utils.SQLQueryUtils.*',
+                    'org.opensearch.sql.spark.validator.SQLQueryValidationVisitor'
             ]
             limit {
                 counter = 'LINE'