Add YamlTemplate ITs for JdbcToBigQuery

cicd/cmd/run-it-yaml-tests/main.go

@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package main
+
+import (
+	"flag"
+	"log"
+
+	"github.com/GoogleCloudPlatform/DataflowTemplates/cicd/internal/flags"
+	"github.com/GoogleCloudPlatform/DataflowTemplates/cicd/internal/workflows"
+)
+
+func main() {
+	flags.RegisterCommonFlags()
+	flags.RegisterItFlags()
+	flag.Parse()
+
+	// Run mvn install before running integration tests
+	mvnFlags := workflows.NewMavenFlags()
+	err := workflows.MvnCleanInstall().Run(
+		mvnFlags.IncludeDependencies(),
+		mvnFlags.IncludeDependents(),
+		mvnFlags.SkipDependencyAnalysis(),
+		mvnFlags.SkipCheckstyle(),
+		mvnFlags.SkipJib(),
+		mvnFlags.SkipTests(),
+		mvnFlags.SkipJacoco(),
+		mvnFlags.SkipShade(),
+		mvnFlags.ThreadCount(8),
+		mvnFlags.InternalMaven())
+	if err != nil {
+		log.Fatalf("%v\n", err)
+	}
+
+	// Run integration tests
+	mvnFlags = workflows.NewMavenFlags()
+	err = workflows.MvnVerify().Run(
+		mvnFlags.IncludeDependencies(),
+		mvnFlags.IncludeDependents(),
+		mvnFlags.SkipDependencyAnalysis(),
+		mvnFlags.SkipCheckstyle(),
+		mvnFlags.SkipJib(),
+		mvnFlags.SkipShade(),
+		mvnFlags.RunYamlIntegrationTests(),
+		mvnFlags.ThreadCount(4),
+		mvnFlags.IntegrationTestParallelism(3),
+		mvnFlags.StaticBigtableInstance("teleport"),
+		mvnFlags.StaticSpannerInstance("teleport"),
+		mvnFlags.InternalMaven(),
+		flags.Region(),
+		flags.Project(),
+		flags.ArtifactBucket(),
+		flags.StageBucket(),
+		flags.HostIp(),
+		flags.PrivateConnectivity(),
+		flags.SpannerHost(),
+		flags.FailureMode(),
+		flags.RetryFailures(),
+		flags.StaticOracleHost(),
+		flags.StaticOracleSysPassword(),
+		flags.CloudProxyHost(),
+		flags.CloudProxyMySqlPort(),
+		flags.CloudProxyPostgresPort(),
+		flags.CloudProxyPassword(),
+		flags.UnifiedWorkerHarnessContainerImage(),
+		flags.CloudProxyPassword())
+	if err != nil {
+		log.Fatalf("%v\n", err)
+	}
+	log.Println("Build Successful!")
+}

cicd/internal/flags/common-flags.go

@@ -26,6 +26,7 @@ const (
 	DEFAULT = "DEFAULT" // Modules other than those excluded
 	KAFKA   = "KAFKA"
 	SPANNER = "SPANNER"
+	YAML    = "YAML"
 )
 
 // Avoid making these vars public.
@@ -48,6 +49,9 @@ var (
 			"v2/spanner-to-sourcedb/",
 			"v2/spanner-custom-shard",
 			"plugins/templates-maven-plugin"},
+		YAML: {"yaml/",
+			"plugins/templates-maven-plugin",
+		},
 	}
 )
 

cicd/internal/workflows/maven-workflows.go

@@ -50,6 +50,7 @@ type MavenFlags interface {
 	SkipIntegrationTests() string
 	FailAtTheEnd() string
 	RunIntegrationTests() string
+	RunYamlIntegrationTests() string
 	RunIntegrationSmokeTests() string
 	RunLoadTests() string
 	RunLoadTestObserver() string
@@ -111,6 +112,10 @@ func (*mvnFlags) RunIntegrationTests() string {
 	return "-PtemplatesIntegrationTests"
 }
 
+func (*mvnFlags) RunYamlIntegrationTests() string {
+	return "-PtemplatesYamlIntegrationTests"
+}
+
 func (*mvnFlags) RunIntegrationSmokeTests() string {
 	return "-PtemplatesIntegrationSmokeTests"
 }

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/JDBCBaseIT.java

@@ -20,7 +20,9 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.util.function.Function;
 import org.apache.beam.it.common.utils.IORedirectUtil;
+import org.apache.beam.it.gcp.storage.GcsResourceManager;
 import org.junit.Before;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -36,10 +38,10 @@
   private static final String JAR_SUFFIX = ".jar";
 
   // The JDBC Driver fully-qualified class names
-  protected static final String MYSQL_DRIVER = "com.mysql.jdbc.Driver";
-  protected static final String POSTGRES_DRIVER = "org.postgresql.Driver";
-  protected static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver";
-  protected static final String MSSQL_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver";
+  public static final String MYSQL_DRIVER = "com.mysql.jdbc.Driver";
+  public static final String POSTGRES_DRIVER = "org.postgresql.Driver";
+  public static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver";
+  public static final String MSSQL_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver";
 
   // The relative path to the JDBC drivers under Maven's `.m2/repository` directory
   private static final String MYSQL_LOCAL_PATH = "mysql/mysql-connector-java";
@@ -63,6 +65,10 @@
   @Before
   public void setUpJDBC() throws IOException {
 
+    uploadArtifacts(gcsClient);
+  }
+
+  public static void uploadArtifacts(GcsResourceManager gcsClient) throws IOException {
     String basePath = getMvnBaseRepoPath();
 
     String mySqlDriverGCSRelativePath = GCS_PREFIX + mySqlDriverLocalJar();
@@ -77,19 +83,35 @@
   }
 
   protected String mySqlDriverGCSPath() {
-    return getGcsPath(GCS_PREFIX + mySqlDriverLocalJar());
+    return mySqlDriverGCSPath(this::getGcsPath);
+  }
+
+  public static String mySqlDriverGCSPath(Function<String, String> getGcsPath) {
+    return getGcsPath.apply(GCS_PREFIX + mySqlDriverLocalJar());
   }
 
   protected String postgresDriverGCSPath() {
-    return getGcsPath(GCS_PREFIX + postgresDriverLocalJar());
+    return postgresDriverGCSPath(this::getGcsPath);
+  }
+
+  public static String postgresDriverGCSPath(Function<String, String> getGcsPath) {
+    return getGcsPath.apply(GCS_PREFIX + postgresDriverLocalJar());
   }
 
   protected String oracleDriverGCSPath() {
-    return getGcsPath(GCS_PREFIX + oracleDriverLocalJar());
+    return oracleDriverGCSPath(this::getGcsPath);
+  }
+
+  public static String oracleDriverGCSPath(Function<String, String> getGcsPath) {
+    return getGcsPath.apply(GCS_PREFIX + oracleDriverLocalJar());
   }
 
   protected String msSqlDriverGCSPath() {
-    return getGcsPath(GCS_PREFIX + msSqlDriverLocalJar());
+    return msSqlDriverGCSPath(this::getGcsPath);
+  }
+
+  public static String msSqlDriverGCSPath(Function<String, String> getGcsPath) {
+    return getGcsPath.apply(GCS_PREFIX + msSqlDriverLocalJar());
   }
 
   private static String mySqlDriverLocalJar() {
@@ -125,7 +147,7 @@
     return String.join("/", basePath, MSSQL_LOCAL_PATH, MSSQL_VERSION, msSqlDriverLocalJar());
   }
 
-  private String getMvnBaseRepoPath() {
+  private static String getMvnBaseRepoPath() {
     // Try to get specified maven repo path from args
     if (System.getProperty("mavenRepository") != null) {
       String basePath = System.getProperty("mavenRepository");

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java

@@ -137,16 +137,20 @@
 
   @Before
   public void setUpBase() throws ExecutionException {
+    setUp(null);
+  }
 
+  protected void setUp(TemplateIntegrationTest annotation) throws ExecutionException {
     testId = PipelineUtils.createJobName("test", 10);
 
-    TemplateIntegrationTest annotation = null;
     MultiTemplateIntegrationTest multiAnnotation =
         getClass().getAnnotation(MultiTemplateIntegrationTest.class);
     usingDirectRunner = System.getProperty("directRunnerTest") != null;
     try {
       Method testMethod = getClass().getMethod(testName);
-      annotation = testMethod.getAnnotation(TemplateIntegrationTest.class);
+      if (annotation == null) {
+        annotation = testMethod.getAnnotation(TemplateIntegrationTest.class);
+      }
       Category category = testMethod.getAnnotation(Category.class);
       if (category != null) {
         usingDirectRunner =
@@ -167,7 +171,7 @@
     }
     if (annotation != null && multiAnnotation != null) {
       LOG.warn(
-          "{} specifies both @TemplateIntegrationTest or @MultiTemplateIntegrationTest, please use"
+          "{} specifies both @TemplateIntegrationTest and @MultiTemplateIntegrationTest, please use"
               + " only of either.",
           getClass());
       return;
@@ -371,6 +375,10 @@
       // Flex templates run on parent pom and -pl {path-to-folder}
       moduleBuild = String.join(",", getModulesBuild(pomPath));
       pomPath = pomPath.replaceAll("/v2/.*", "/pom.xml");
+    } else if (pomPath.contains("yaml/pom.xml")) {
+      // YAML templates run on parent pom and -pl {path-to-folder}
+      pomPath = new File(pom.getParentFile().getParentFile(), "pom.xml").getAbsolutePath();
+      moduleBuild = String.join(",", List.of("metadata", "yaml"));
     } else {
       LOG.warn(
           "Specific module POM was not found, so scanning all modules... Stage step may take a"

metadata/src/main/java/com/google/cloud/teleport/metadata/YAMLTemplateIntegrationTest.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.cloud.teleport.metadata;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/** Annotation that marks the test of a Dataflow Template. */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE, ElementType.METHOD})
+public @interface YAMLTemplateIntegrationTest {}

pom.xml

@@ -104,6 +104,7 @@
     <integration.tests>com.google.cloud.teleport.metadata.TemplateIntegrationTest</integration.tests>
     <load.tests>com.google.cloud.teleport.metadata.TemplateLoadTest</load.tests>
     <direct-runner.tests>com.google.cloud.teleport.metadata.DirectRunnerTest</direct-runner.tests>
+    <yaml-integration.tests>com.google.cloud.teleport.metadata.YAMLTemplateIntegrationTest</yaml-integration.tests>
     <excluded.spanner.tests></excluded.spanner.tests>
 
     <licenseHeaderFile>JAVA_LICENSE_HEADER</licenseHeaderFile>
@@ -442,7 +443,10 @@
               <excludes>
                 <exclude>**/KafkaToKafkaIT.java</exclude>
               </excludes>
-              <excludedGroups>${direct-runner.tests}</excludedGroups>
+              <excludedGroups>
+                ${direct-runner.tests}
+                ${yaml-integration.tests}
+              </excludedGroups>
               <groups>
                 ${integration.tests}
               </groups>
@@ -541,6 +545,7 @@
                 <include>**/*IT.java</include>
               </includes>
               <excludedGroups>
+                ${yaml-integration.tests}
                 com.google.cloud.teleport.metadata.SkipDirectRunnerTest
               </excludedGroups>
               <groups>
@@ -587,6 +592,9 @@
               <includes>
                 <include>**/*IT.java</include>
               </includes>
+              <excludedGroups>
+                ${yaml-integration.tests}
+              </excludedGroups>
               <groups>
                 ${direct-runner.tests}
                 org.apache.beam.it.testcontainers.TestContainersIntegrationTest
@@ -601,6 +609,56 @@
         </plugins>
       </build>
     </profile>
+    <profile>
+      <id>templatesYamlIntegrationTests</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <properties>
+        <!-- Skip coverage checks, unit tests are skipped -->
+        <jacoco.skip>true</jacoco.skip>
+        <!-- Some modules may yield no integration tests -->
+        <failIfNoTests>false</failIfNoTests>
+        <!-- Parallelism settings. Default is 2, set to consider methods -->
+        <itParallelismType>classesAndMethods</itParallelismType>
+        <itParallelism>2</itParallelism>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <version>${surefire.version}</version>
+            <configuration combine.self="override">
+              <systemProperties>
+                <property>
+                  <name>beamPythonVersion</name>
+                  <value>${beam-python.version}</value>
+                </property>
+                <property>
+                  <name>beamJavaVersion</name>
+                  <value>${beam.version}</value>
+                </property>
+                <property>
+                  <name>beamMavenRepo</name>
+                  <value>${beam-maven-repo}</value>
+                </property>
+              </systemProperties>
+              <includes>
+                <include>**/*IT.java</include>
+              </includes>
+              <groups>
+                ${yaml-integration.tests}
+              </groups>
+              <reuseForks>true</reuseForks>
+              <parallel>${itParallelismType}</parallel>
+              <threadCount>${itParallelism}</threadCount>
+              <trimStackTrace>false</trimStackTrace>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <profile>
       <id>pluginOutputDir</id>
       <activation>

python/generate_all_dependencies.sh

@@ -22,8 +22,8 @@ sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/default_base_yaml_requiremen
 
 cp $SCRIPTPATH/__build__/default_python_requirements.txt $SCRIPTPATH/../python/src/main/python/word-count-python/requirements.txt
 
-cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../python/src/main/python/yaml-template/requirements.txt
-cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../python/src/main/python/job-builder-server/requirements.txt
+cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../python/src/main/yaml/yaml-template/requirements.txt
+cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../python/src/main/yaml/job-builder-server/requirements.txt
 cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../v2/googlecloud-to-elasticsearch/src/main/resources/requirements.txt
 cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../v2/googlecloud-to-googlecloud/src/main/resources/requirements.txt
 cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../v2/googlecloud-to-splunk/src/main/resources/requirements.txt

python/pom.xml

@@ -36,9 +36,6 @@
   <properties>
     <maven.compiler.source>11</maven.compiler.source>
     <maven.compiler.target>11</maven.compiler.target>
-
-    <!-- TODO - remove line when repo is upgraded to Beam 2.59.0 -->
-    <beam-python.version>2.57.0</beam-python.version>
   </properties>
 
   <dependencies>