From e3a771e33da226975bffed50d9ccc2ce67e77bf4 Mon Sep 17 00:00:00 2001
From: Gera Shegalov <gera@apache.org>
Date: Fri, 27 Jan 2023 20:15:04 -0800
Subject: [PATCH] Centralize source-related properties in parent pom for
 consistent usage in submodules (#7600)

Contributes to #7487. Use a consistent scheme of properties for sql-plugin and tests modules for easier automation.

Signed-off-by: Gera Shegalov <gera@apache.org>
---
 dist/pom.xml       |  43 +----
 pom.xml            | 430 ++++++++++++++++++++++++++++++++++++++++++---
 sql-plugin/pom.xml | 236 +------------------------
 tests/pom.xml      |  34 ----
 4 files changed, 403 insertions(+), 340 deletions(-)

diff --git a/dist/pom.xml b/dist/pom.xml
index 4b508bad0ab..ba86abbe04c 100644
--- a/dist/pom.xml
+++ b/dist/pom.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright (c) 2020-2022, NVIDIA CORPORATION.
+  Copyright (c) 2020-2023, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -40,47 +40,6 @@
 
     <properties>
         <target.classifier/>
-        <noSnapshot.buildvers>
-            311,
-            312,
-            313,
-            320,
-            321,
-            321cdh,
-            322,
-            323,
-            330,
-            331,
-            330cdh
-        </noSnapshot.buildvers>
-        <snapshot.buildvers>
-            332
-        </snapshot.buildvers>
-        <databricks.buildvers>
-            312db,
-            321db,
-            330db
-        </databricks.buildvers>
-        <!--
-          Build and run unit tests on one specific version for each sub-version (e.g. 311, 320, 330)
-          Base shim version (311 currently) should be covered in default mvn verify command of premerge script, 
-          so base shim version is removed from the premergeUT list.
-          Separate the versions to two parts (premergeUT1, premergeUT2) for balancing the duration
-        -->
-        <premergeUT1.buildvers>
-            320
-        </premergeUT1.buildvers>
-        <premergeUT2.buildvers>
-            330
-        </premergeUT2.buildvers>
-        <premergeUTF8.buildvers>
-            320
-        </premergeUTF8.buildvers>
-        <jdk11.buildvers>
-            312,
-            321,
-            331
-        </jdk11.buildvers>
         <dist.jar.name>${project.build.directory}/${project.build.finalName}-${cuda.version}.jar</dist.jar.name>
     </properties>
     <profiles>
diff --git a/pom.xml b/pom.xml
index 804035372f9..4f310d0cc18 100644
--- a/pom.xml
+++ b/pom.xml
@@ -85,7 +85,7 @@
                 <spark.test.version>${spark311.version}</spark.test.version>
                 <parquet.hadoop.version>1.10.1</parquet.hadoop.version>
                 <spark.shim.sources>${spark311.sources}</spark.shim.sources>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark311.test.sources}</spark.shim.test.sources>
             </properties>
             <modules>
                 <module>delta-lake/delta-stub</module>
@@ -125,7 +125,7 @@
                 <rat.consoleOutput>true</rat.consoleOutput>
                 <parquet.hadoop.version>1.10.1</parquet.hadoop.version>
                 <spark.shim.sources>${spark312db.sources}</spark.shim.sources>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark312db.test.sources}</spark.shim.test.sources>
             </properties>
 
             <modules>
@@ -153,7 +153,7 @@
                 <spark.test.version>${spark312.version}</spark.test.version>
                 <parquet.hadoop.version>1.10.1</parquet.hadoop.version>
                 <spark.shim.sources>${spark312.sources}</spark.shim.sources>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark312.test.sources}</spark.shim.test.sources>
             </properties>
             <modules>
                 <module>delta-lake/delta-stub</module>
@@ -181,7 +181,7 @@
                 <spark.test.version>${spark313.version}</spark.test.version>
                 <parquet.hadoop.version>1.10.1</parquet.hadoop.version>
                 <spark.shim.sources>${spark313.sources}</spark.shim.sources>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark313.test.sources}</spark.shim.test.sources>
             </properties>
             <modules>
                 <module>delta-lake/delta-stub</module>
@@ -209,7 +209,7 @@
                 <spark.test.version>${spark314.version}</spark.test.version>
                 <parquet.hadoop.version>1.10.1</parquet.hadoop.version>
                 <spark.shim.sources>${spark314.sources}</spark.shim.sources>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark314.test.sources}</spark.shim.test.sources>
             </properties>
             <modules>
                 <module>delta-lake/delta-stub</module>
@@ -237,7 +237,7 @@
                 <spark.test.version>${spark320.version}</spark.test.version>
                 <parquet.hadoop.version>1.12.1</parquet.hadoop.version>
                 <spark.shim.sources>${spark320.sources}</spark.shim.sources>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark320.test.sources}</spark.shim.test.sources>
             </properties>
             <modules>
                 <module>delta-lake/delta-20x</module>
@@ -264,7 +264,7 @@
                 <spark.test.version>${spark321.version}</spark.test.version>
                 <parquet.hadoop.version>1.12.2</parquet.hadoop.version>
                 <spark.shim.sources>${spark321.sources}</spark.shim.sources>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark321.test.sources}</spark.shim.test.sources>
             </properties>
             <modules>
                 <module>delta-lake/delta-20x</module>
@@ -291,7 +291,7 @@
                 <spark.test.version>${spark321cdh.version}</spark.test.version>
                 <parquet.hadoop.version>1.10.1</parquet.hadoop.version>
                 <spark.shim.sources>${spark321cdh.sources}</spark.shim.sources>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark321cdh.test.sources}</spark.shim.test.sources>
             </properties>
             <repositories>
                 <repository>
@@ -324,7 +324,7 @@
                 <spark.test.version>${spark322.version}</spark.test.version>
                 <parquet.hadoop.version>1.12.2</parquet.hadoop.version>
                 <spark.shim.sources>${spark322.sources}</spark.shim.sources>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark322.test.sources}</spark.shim.test.sources>
             </properties>
             <modules>
                 <module>delta-lake/delta-20x</module>
@@ -351,7 +351,7 @@
                 <spark.test.version>${spark323.version}</spark.test.version>
                 <parquet.hadoop.version>1.12.2</parquet.hadoop.version>
                 <spark.shim.sources>${spark323.sources}</spark.shim.sources>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark323.test.sources}</spark.shim.test.sources>
             </properties>
             <modules>
                 <module>delta-lake/delta-20x</module>
@@ -390,7 +390,7 @@
                 <rat.consoleOutput>true</rat.consoleOutput>
                 <parquet.hadoop.version>1.12.0</parquet.hadoop.version>
                 <spark.shim.sources>${spark321db.sources}</spark.shim.sources>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark321db.test.sources}</spark.shim.test.sources>
             </properties>
             <modules>
                 <module>delta-lake/delta-spark321db</module>
@@ -418,7 +418,7 @@
                 <parquet.hadoop.version>1.12.2</parquet.hadoop.version>
                 <spark.shim.sources>${spark330.sources}</spark.shim.sources>
                 <iceberg.version>${spark330.iceberg.version}</iceberg.version>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark330.test.sources}</spark.shim.test.sources>
             </properties>
             <modules>
                 <module>delta-lake/delta-21x</module>
@@ -447,7 +447,7 @@
                 <parquet.hadoop.version>1.12.2</parquet.hadoop.version>
                 <spark.shim.sources>${spark331.sources}</spark.shim.sources>
                 <iceberg.version>${spark330.iceberg.version}</iceberg.version>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark331.test.sources}</spark.shim.test.sources>
             </properties>
             <modules>
                 <module>delta-lake/delta-21x</module>
@@ -476,7 +476,7 @@
                 <parquet.hadoop.version>1.12.2</parquet.hadoop.version>
                 <spark.shim.sources>${spark332.sources}</spark.shim.sources>
                 <iceberg.version>${spark330.iceberg.version}</iceberg.version>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark332.test.sources}</spark.shim.test.sources>
             </properties>
             <modules>
                 <module>delta-lake/delta-21x</module>
@@ -505,7 +505,7 @@
                 <parquet.hadoop.version>1.12.3</parquet.hadoop.version>
                 <spark.shim.sources>${spark340.sources}</spark.shim.sources>
                 <iceberg.version>${spark330.iceberg.version}</iceberg.version>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark340.test.sources}</spark.shim.test.sources>
             </properties>
             <modules>
                 <module>delta-lake/delta-stub</module>
@@ -533,7 +533,7 @@
                 <parquet.hadoop.version>1.10.99.7.1.8.0-801</parquet.hadoop.version>
                 <spark.shim.sources>${spark330cdh.sources}</spark.shim.sources>
                 <iceberg.version>${spark330.iceberg.version}</iceberg.version>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark330cdh.test.sources}</spark.shim.test.sources>
             </properties>
             <repositories>
                 <repository>
@@ -586,7 +586,7 @@
                 <parquet.hadoop.version>1.12.0</parquet.hadoop.version>
                 <spark.shim.sources>${spark330db.sources}</spark.shim.sources>
                 <iceberg.version>${spark330.iceberg.version}</iceberg.version>
-                <spark.shim.test.sources>${project.basedir}/src/test/${buildver}/scala</spark.shim.test.sources>
+                <spark.shim.test.sources>${spark330db.test.sources}</spark.shim.test.sources>
             </properties>
             <modules>
                 <module>delta-lake/delta-spark330db</module>
@@ -725,6 +725,53 @@
             with the ones deployed to a remote Maven repo
         -->
         <ignore.shim.revisions.check>false</ignore.shim.revisions.check>
+        <noSnapshot.buildvers>
+            311,
+            312,
+            313,
+            320,
+            321,
+            321cdh,
+            322,
+            323,
+            330,
+            331,
+            330cdh
+        </noSnapshot.buildvers>
+        <snapshot.buildvers>
+            332
+        </snapshot.buildvers>
+        <databricks.buildvers>
+            312db,
+            321db,
+            330db
+        </databricks.buildvers>
+        <!--
+          Build and run unit tests on one specific version for each sub-version (e.g. 311, 320, 330)
+          Base shim version (311 currently) should be covered in default mvn verify command of premerge script,
+          so base shim version is removed from the premergeUT list.
+          Separate the versions to two parts (premergeUT1, premergeUT2) for balancing the duration
+        -->
+        <premergeUT1.buildvers>
+            320
+        </premergeUT1.buildvers>
+        <premergeUT2.buildvers>
+            330
+        </premergeUT2.buildvers>
+        <premergeUTF8.buildvers>
+            320
+        </premergeUTF8.buildvers>
+        <jdk11.buildvers>
+            312,
+            321,
+            331
+        </jdk11.buildvers>
+        <all.buildvers>
+            ${noSnapshot.buildvers},
+            ${snapshot.buildvers},
+            ${databricks.buildvers},
+            340
+        </all.buildvers>
     </properties>
 
     <dependencyManagement>
@@ -846,13 +893,315 @@
                 <plugin>
                     <groupId>org.codehaus.mojo</groupId>
                     <artifactId>build-helper-maven-plugin</artifactId>
-                    <version>3.2.0</version>
+                    <version>3.3.0</version>
                 </plugin>
                 <plugin>
                   <groupId>org.apache.maven.plugins</groupId>
                   <artifactId>maven-antrun-plugin</artifactId>
-                  <version>3.0.0</version>
+                  <version>3.1.0</version>
                   <executions>
+                    <execution>
+                        <id>create-source-path-properties</id>
+                        <goals><goal>run</goal></goals>
+                        <phase>initialize</phase>
+                        <configuration>
+                            <exportAntProperties>true</exportAntProperties>
+                            <target xmlns:ac="antlib:net.sf.antcontrib">
+                                <!--
+                                    Rules for adding new shim directories:
+                                    ######################################
+                                    1. Keep includes sorted
+                                    2. Top path components such as 311until320-all should not use
+                                    wildcards until a robust pattern is developed that allows
+                                    refactoring for new shims without breaking build for
+                                    exisiting shims
+                                    3. Using wild cards to pick up scala and java with a single entry
+                                    311until320-all/* is allowed
+                                    4. Using wildcards for trivial excludes is allowed
+                                    - *nondb* dirs in db shims
+                                    - *until330* in 330
+                                    5. Begin with a common reusable pattern, and put the unique includeds
+                                    and excludes at the end
+                                    6. At the same pattern-nesting level list includes before excludes
+                                -->
+
+                                <!-- upstream Spark shims -->
+                                <pathconvert property="spark311.sources" pathsep=",">
+                                    <dirset id="spark311.dirset.id" dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <patternset id="spark311+.pattern">
+                                            <include name="311+-non330db/*"/>
+                                            <include name="311+-nondb/*"/>
+                                            <include name="311until320-all/*"/>
+                                            <include name="311until320-noncdh/*"/>
+                                            <include name="311until320-nondb/*"/>
+                                            <include name="311until330-all/*"/>
+                                            <include name="311until330-nondb/*"/>
+                                            <include name="311until340-all/*"/>
+                                            <include name="311until340-non330db/*"/>
+                                            <include name="311until340-nondb/*"/>
+                                            <include name="pre320-treenode/*"/>
+                                        </patternset>
+                                        <include name="311-nondb/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark312.sources" pathsep=",">
+                                    <dirset id="spark312.dirset.id" dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <patternset refid="spark311+.pattern"/>
+                                        <include name="312-nondb/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark313.sources" pathsep=",">
+                                    <dirset id="spark313.dirset.id" dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <patternset refid="spark311+.pattern"/>
+                                        <include name="313/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark314.sources" pathsep=",">
+                                    <dirset id="spark314.dirset.id" dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <patternset refid="spark311+.pattern"/>
+                                        <include name="314/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark320.sources" pathsep=",">
+                                    <dirset id="spark320.dirset.id" dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <patternset id="spark320+.pattern">
+                                            <!-- inherit from 311+ with exceptions -->
+                                            <patternset refid="spark311+.pattern"/>
+                                            <exclude name="*until320*/*"/>
+                                            <exclude name="pre320-treenode/*"/>
+
+                                            <!-- uniquely 320+ -->
+                                            <include name="320+/*"/>
+                                            <include name="320+-noncdh/*"/>
+                                            <include name="320+-nondb/*"/>
+                                            <include name="320until330-all/*"/>
+                                            <include name="320until330-noncdh/*"/>
+                                            <include name="320until330-nondb/*"/>
+                                            <include name="320until340-all/*"/>
+                                            <include name="320until340-non330db/*"/>
+                                            <include name="delta-lake-common/*"/>
+                                            <include name="post320-treenode/*"/>
+                                        </patternset>
+                                        <include name="320/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark321.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <patternset id="spark321+.pattern">
+                                            <patternset refid="spark320+.pattern"/>
+                                            <include name="321+/*"/>
+                                            <include name="321until330-all/*"/>
+                                        </patternset>
+                                        <include name="321/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark322.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <patternset refid="spark321+.pattern"/>
+                                        <include name="322/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark323.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <patternset refid="spark321+.pattern"/>
+                                        <include name="323/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark330.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <patternset id="spark330+.pattern">
+                                            <patternset refid="spark321+.pattern"/>
+                                            <exclude name="*until330*/*"/>
+
+                                            <!-- uniquely 330+ -->
+                                            <include name="330+/*"/>
+                                            <include name="330+-nondb/*"/>
+                                            <include name="330+-noncdh/*"/>
+                                            <include name="330until340/*"/>
+                                            <include name="330until340-nondb/*"/>
+                                        </patternset>
+                                        <include name="330/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark331.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <patternset id="spark331+.pattern">
+                                            <patternset refid="spark330+.pattern"/>
+                                            <include name="331+/*"/>
+                                        </patternset>
+                                        <include name="331/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark332.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <patternset refid="spark331+.pattern"/>
+                                        <include name="332/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark340.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <patternset id="spark340+.pattern">
+                                            <!-- inherit from 331+ with exceptions -->
+                                            <patternset refid="spark331+.pattern"/>
+                                            <exclude name="*until340*/*"/>
+                                            <include name="340+-and-330db/*"/>
+                                            <!-- uniquely 340+ -->
+                                            <include name="340+/*"/>
+                                        </patternset>
+                                        <include name="340/*"/>
+                                    </dirset>
+                                </pathconvert>
+
+                                <!-- Spark vendor shims -->
+                                <pathconvert property="spark321cdh.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <!-- inherit from 321+ upstream except noncdh -->
+                                        <patternset refid="spark321+.pattern"/>
+                                        <exclude name="*noncdh*/*"/>
+
+                                        <include name="321cdh/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark330cdh.sources" pathsep=",">
+                                    <dirset id="spark330cdh.dirset.id" dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <!-- inherit from 330+ upstream except noncdh -->
+                                        <patternset refid="spark330+.pattern"/>
+                                        <exclude name="*noncdh*/*"/>
+
+                                        <include name="330cdh/*"/>
+                                    </dirset>
+                                </pathconvert>
+
+                                <pathconvert property="spark312db.sources" pathsep=",">
+                                    <dirset id="spark312db.dirset.id" dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <!-- inherit 311+ except nondb -->
+                                        <patternset refid="spark311+.pattern"/>
+                                        <exclude name="pre320-treenode/*"/>
+                                        <exclude name="*nondb*/*"/>
+
+                                        <include name="311+-db/*"/>
+                                        <include name="31xdb/*"/>
+                                        <include name="post320-treenode/*"/>
+
+                                        <include name="312db/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark321db.sources" pathsep=",">
+                                    <dirset id="spark321db.dirset.id" dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <patternset id="spark321db+.pattern">
+                                            <!-- inherit 321+ except nondb -->
+                                            <patternset refid="spark321+.pattern"/>
+                                            <exclude name="*nondb*/*"/>
+
+                                            <include name="311+-db/*"/>
+                                            <include name="321+-db/*"/>
+                                        </patternset>
+                                        <include name="321db/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark330db.sources" pathsep=",">
+                                    <dirset id="spark330db.dirset.id" dir="${project.basedir}/src/main" erroronmissingdir="false">
+                                        <patternset refid="spark321db+.pattern"/>
+                                        <patternset refid="spark330+.pattern"/>
+                                        <include name="340+-and-330db/*"/>
+                                        <include name="330db/*"/>
+                                        <exclude name="*non330db*/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark311.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="311/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark312.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="312/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark313.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="313/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark320.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="320/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark321.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="321/*"/>
+                                        <include name="320+-noncdh-nondb/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark321cdh.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="321cdh/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark322.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="322/*"/>
+                                        <include name="320+-noncdh-nondb/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark323.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="323/*"/>
+                                        <include name="320+-noncdh-nondb/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark330.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="330/*"/>
+                                        <include name="320+-noncdh-nondb/*"/>
+                                        <include name="330+/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark331.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="331/*"/>
+                                        <include name="320+-noncdh-nondb/*"/>
+                                        <include name="330+/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark332.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="332/*"/>
+                                        <include name="320+-noncdh-nondb/*"/>
+                                        <include name="330+/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark330cdh.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="330cdh/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark312db.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="312db/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark321db.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="321db/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark330db.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="330db/*"/>
+                                    </dirset>
+                                </pathconvert>
+                                <pathconvert property="spark340.test.sources" pathsep=",">
+                                    <dirset dir="${project.basedir}/src/test" erroronmissingdir="false">
+                                        <include name="330/*"/>
+                                        <include name="320+-noncdh-nondb/*"/>
+                                        <include name="330+/*"/>
+                                    </dirset>
+                                </pathconvert>
+                            </target>
+                        </configuration>
+                    </execution>
                     <execution>
                         <id>generate-build-info</id>
                         <phase>generate-resources</phase>
@@ -1113,21 +1462,22 @@
             </plugin>
 
             <plugin>
-                <!--
-                This is an alternative implementation of the scalastyle check invocation,
-                a replacement for scalastyle-maven-plugin. It's motivated to address the following:
-                - All scala files are checked at once regardless of the module, so the developer
-                  can focus on addressing violations without being distracted by the build issues
-                  in-between
-                - We don't have to hardcode the source code roots added dynamically by other maven
-                  plugins to the project
-                - The scalastyle launch cost is amortized across all modules
-                -->
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-antrun-plugin</artifactId>
+                <!-- parent-pom only executions -->
                 <inherited>false</inherited>
                 <executions>
                     <execution>
+                        <!--
+                        This is an alternative implementation of the scalastyle check invocation,
+                        a replacement for scalastyle-maven-plugin. It's motivated to address the following:
+                        - All scala files are checked at once regardless of the module, so the developer
+                        can focus on addressing violations without being distracted by the build issues
+                        in-between
+                        - We don't have to hardcode the source code roots added dynamically by other maven
+                        plugins to the project
+                        - The scalastyle launch cost is amortized across all modules
+                        -->
                         <id>scalastyle-all-modules</id>
                         <phase>verify</phase>
                         <goals><goal>run</goal></goals>
@@ -1184,6 +1534,28 @@
                     </dependency>
                 </dependencies>
             </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>build-helper-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>add-shim-sources</id>
+                        <phase>generate-sources</phase>
+                        <goals><goal>add-source</goal></goals>
+                        <configuration>
+                            <sources>${spark.shim.sources}</sources>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>add-shim-test-sources</id>
+                        <phase>generate-test-sources</phase>
+                        <goals><goal>add-test-source</goal></goals>
+                        <configuration>
+                            <sources>${spark.shim.test.sources}</sources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
             <plugin>
                 <groupId>org.jacoco</groupId>
                 <artifactId>jacoco-maven-plugin</artifactId>
diff --git a/sql-plugin/pom.xml b/sql-plugin/pom.xml
index 23f4e44b043..bbc87159e25 100644
--- a/sql-plugin/pom.xml
+++ b/sql-plugin/pom.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright (c) 2020-2022, NVIDIA CORPORATION.
+  Copyright (c) 2020-2023, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -494,216 +494,6 @@
             <plugin>
               <artifactId>maven-antrun-plugin</artifactId>
               <executions>
-                <execution>
-                    <id>create-source-path-properties</id>
-                    <goals><goal>run</goal></goals>
-                    <phase>initialize</phase>
-                    <configuration>
-                        <exportAntProperties>true</exportAntProperties>
-                        <target>
-                            <!--
-                                Rules for adding new shim directories:
-                                ######################################
-                                1. Keep includes sorted
-                                2. Top path components such as 311until320-all should not use
-                                wildcards until a robust pattern is developed that allows
-                                refactoring for new shims without breaking build for
-                                exisiting shims
-                                3. Using wild cards to pick up scala and java with a single entry
-                                311until320-all/* is allowed
-                                4. Using wildcards for trivial excludes is allowed
-                                  - *nondb* dirs in db shims
-                                  - *until330* in 330
-                                5. Begin with a common reusable pattern, and put the unique includeds
-                                and excludes at the end
-                                6. At the same pattern-nesting level list includes before excludes
-                            -->
-
-                            <!-- upstream Spark shims -->
-                            <pathconvert property="spark311.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <patternset id="spark311+.pattern">
-                                        <include name="311+-non330db/*"/>
-                                        <include name="311+-nondb/*"/>
-                                        <include name="311until320-all/*"/>
-                                        <include name="311until320-noncdh/*"/>
-                                        <include name="311until320-nondb/*"/>
-                                        <include name="311until330-all/*"/>
-                                        <include name="311until330-nondb/*"/>
-                                        <include name="311until340-all/*"/>
-                                        <include name="311until340-non330db/*"/>
-                                        <include name="311until340-nondb/*"/>
-                                        <include name="pre320-treenode/*"/>
-                                    </patternset>
-                                    <include name="311-nondb/*"/>
-                                </dirset>
-                            </pathconvert>
-                            <pathconvert property="spark312.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <patternset refid="spark311+.pattern"/>
-                                    <include name="312-nondb/*"/>
-                                </dirset>
-                            </pathconvert>
-                            <pathconvert property="spark313.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <patternset refid="spark311+.pattern"/>
-                                    <include name="313/*"/>
-                                </dirset>
-                            </pathconvert>
-                            <pathconvert property="spark314.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <patternset refid="spark311+.pattern"/>
-                                    <include name="314/*"/>
-                                </dirset>
-                            </pathconvert>
-                            <pathconvert property="spark320.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <patternset id="spark320+.pattern">
-                                        <!-- inherit from 311+ with exceptions -->
-                                        <patternset refid="spark311+.pattern"/>
-                                        <exclude name="*until320*/*"/>
-                                        <exclude name="pre320-treenode/*"/>
-
-                                        <!-- uniquely 320+ -->
-                                        <include name="320+/*"/>
-                                        <include name="320+-noncdh/*"/>
-                                        <include name="320+-nondb/*"/>
-                                        <include name="320until330-all/*"/>
-                                        <include name="320until330-noncdh/*"/>
-                                        <include name="320until330-nondb/*"/>
-                                        <include name="320until340-all/*"/>
-                                        <include name="320until340-non330db/*"/>
-                                        <include name="post320-treenode/*"/>
-                                    </patternset>
-                                    <include name="320/*"/>
-                                </dirset>
-                            </pathconvert>
-                            <pathconvert property="spark321.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <patternset id="spark321+.pattern">
-                                        <patternset refid="spark320+.pattern"/>
-                                        <include name="321+/*"/>
-                                        <include name="321until330-all/*"/>
-                                    </patternset>
-                                    <include name="321/*"/>
-                                </dirset>
-                            </pathconvert>
-                            <pathconvert property="spark322.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <patternset refid="spark321+.pattern"/>
-                                    <include name="322/*"/>
-                                </dirset>
-                            </pathconvert>
-                            <pathconvert property="spark323.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <patternset refid="spark321+.pattern"/>
-                                    <include name="323/*"/>
-                                </dirset>
-                            </pathconvert>
-                            <pathconvert property="spark330.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <patternset id="spark330+.pattern">
-                                        <patternset refid="spark321+.pattern"/>
-                                        <exclude name="*until330*/*"/>
-
-                                        <!-- uniquely 330+ -->
-                                        <include name="330+/*"/>
-                                        <include name="330+-nondb/*"/>
-                                        <include name="330+-noncdh/*"/>
-                                        <include name="330until340/*"/>
-                                        <include name="330until340-nondb/*"/>
-                                    </patternset>
-                                    <include name="330/*"/>
-                                </dirset>
-                            </pathconvert>
-                            <pathconvert property="spark331.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <patternset id="spark331+.pattern">
-                                        <patternset refid="spark330+.pattern"/>
-                                        <include name="331+/*"/>
-                                    </patternset>
-                                    <include name="331/*"/>
-                                </dirset>
-                            </pathconvert>
-                            <pathconvert property="spark332.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <patternset refid="spark331+.pattern"/>
-                                    <include name="332/*"/>
-                                </dirset>
-                            </pathconvert>
-                            <pathconvert property="spark340.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <patternset id="spark340+.pattern">
-                                        <!-- inherit from 331+ with exceptions -->
-                                        <patternset refid="spark331+.pattern"/>
-                                        <exclude name="*until340*/*"/>
-                                        <include name="340+-and-330db/*"/>
-                                        <!-- uniquely 340+ -->
-                                        <include name="340+/*"/>
-                                    </patternset>
-                                    <include name="340/*"/>
-                                </dirset>
-                            </pathconvert>
-
-                            <!-- Spark vendor shims -->
-                            <pathconvert property="spark321cdh.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <!-- inherit from 321+ upstream except noncdh -->
-                                    <patternset refid="spark321+.pattern"/>
-                                    <exclude name="*noncdh*/*"/>
-
-                                    <include name="321cdh/*"/>
-                                </dirset>
-                            </pathconvert>
-                            <pathconvert property="spark330cdh.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <!-- inherit from 330+ upstream except noncdh -->
-                                    <patternset refid="spark330+.pattern"/>
-                                    <exclude name="*noncdh*/*"/>
-
-                                    <include name="330cdh/*"/>
-                                </dirset>
-                            </pathconvert>
-
-                            <pathconvert property="spark312db.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <!-- inherit 311+ except nondb -->
-                                    <patternset refid="spark311+.pattern"/>
-                                    <exclude name="pre320-treenode/*"/>
-                                    <exclude name="*nondb*/*"/>
-
-                                    <include name="311+-db/*"/>
-                                    <include name="31xdb/*"/>
-                                    <include name="post320-treenode/*"/>
-
-                                    <include name="312db/*"/>
-                                </dirset>
-                            </pathconvert>
-                            <pathconvert property="spark321db.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <patternset id="spark321db+.pattern">
-                                        <!-- inherit 321+ except nondb -->
-                                        <patternset refid="spark321+.pattern"/>
-                                        <exclude name="*nondb*/*"/>
-
-                                        <include name="311+-db/*"/>
-                                        <include name="321+-db/*"/>
-                                    </patternset>
-                                    <include name="321db/*"/>
-                                </dirset>
-                            </pathconvert>
-                            <pathconvert property="spark330db.sources" pathsep=",">
-                                <dirset dir="${project.basedir}/src/main">
-                                    <patternset refid="spark321db+.pattern"/>
-                                    <patternset refid="spark330+.pattern"/>
-                                    <include name="340+-and-330db/*"/>
-                                    <include name="330db/*"/>
-                                    <exclude name="*non330db*/*"/>
-                                </dirset>
-                            </pathconvert>
-                        </target>
-                    </configuration>
-                </execution>
                 <execution>
                   <id>copy-notice</id>
                   <goals>
@@ -754,30 +544,6 @@
                 <groupId>org.scalatest</groupId>
                 <artifactId>scalatest-maven-plugin</artifactId>
             </plugin>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>build-helper-maven-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>add-shim-sources</id>
-                        <phase>generate-sources</phase>
-                        <goals><goal>add-source</goal></goals>
-                        <configuration>
-                            <sources>${spark.shim.sources}</sources>
-                        </configuration>
-                    </execution>
-                    <execution>
-                        <id>add-shim-test-sources</id>
-                        <phase>generate-test-sources</phase>
-                        <goals><goal>add-test-source</goal></goals>
-                        <configuration>
-                            <sources>
-                                <source>${spark.shim.test.sources}</source>
-                            </sources>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
         </plugins>
     </build>
 </project>
diff --git a/tests/pom.xml b/tests/pom.xml
index 88b4bcee553..7dcf3aa1982 100644
--- a/tests/pom.xml
+++ b/tests/pom.xml
@@ -350,9 +350,6 @@
                     <value>321</value>
                 </property>
             </activation>
-            <properties>
-                <spark.shim.test.sources>${project.basedir}/src/test/320+-noncdh-nondb/scala</spark.shim.test.sources>
-            </properties>
         </profile>
         <profile>
             <id>release322</id>
@@ -362,9 +359,6 @@
                     <value>322</value>
                 </property>
             </activation>
-            <properties>
-                <spark.shim.test.sources>${project.basedir}/src/test/320+-noncdh-nondb/scala</spark.shim.test.sources>
-            </properties>
         </profile>
         <profile>
             <id>release323</id>
@@ -374,9 +368,6 @@
                     <value>323</value>
                 </property>
             </activation>
-            <properties>
-                <spark.shim.test.sources>${project.basedir}/src/test/320+-noncdh-nondb/scala</spark.shim.test.sources>
-            </properties>
         </profile>
         <profile>
             <id>release330</id>
@@ -386,9 +377,6 @@
                     <value>330</value>
                 </property>
             </activation>
-            <properties>
-                <spark.shim.test.sources>${project.basedir}/src/test/330+/scala,${project.basedir}/src/test/320+-noncdh-nondb/scala</spark.shim.test.sources>
-            </properties>
         </profile>
         <profile>
             <id>release331</id>
@@ -398,9 +386,6 @@
                     <value>331</value>
                 </property>
             </activation>
-            <properties>
-                <spark.shim.test.sources>${project.basedir}/src/test/330+/scala,${project.basedir}/src/test/320+-noncdh-nondb/scala</spark.shim.test.sources>
-            </properties>
         </profile>
         <profile>
             <id>release332</id>
@@ -410,9 +395,6 @@
                     <value>332</value>
                 </property>
             </activation>
-            <properties>
-                <spark.shim.test.sources>${project.basedir}/src/test/330+/scala,${project.basedir}/src/test/320+-noncdh-nondb/scala</spark.shim.test.sources>
-            </properties>
         </profile>
         <profile>
             <id>release340</id>
@@ -422,9 +404,6 @@
                     <value>340</value>
                 </property>
             </activation>
-            <properties>
-                <spark.shim.test.sources>${project.basedir}/src/test/330+/scala,${project.basedir}/src/test/320+-noncdh-nondb/scala</spark.shim.test.sources>
-            </properties>
         </profile>
     </profiles>
 
@@ -449,19 +428,6 @@
                 <groupId>org.apache.rat</groupId>
                 <artifactId>apache-rat-plugin</artifactId>
             </plugin>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>build-helper-maven-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>add-shim-test-sources</id>
-                        <goals><goal>add-test-source</goal></goals>
-                        <configuration>
-                            <sources>${spark.shim.test.sources}</sources>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
         </plugins>
     </build>
 </project>