From d7c035c1e7f38b46553d8a464d833668bc814678 Mon Sep 17 00:00:00 2001 From: Hector Miuler Malpica Gallegos Date: Wed, 1 Feb 2023 12:10:33 -0500 Subject: [PATCH] test(scio-cosmosdb): Refactor for add the testcontainers-scala-scalatest --- build.sbt | 1 + .../spotify/scio/cosmosdb/CosmosDbIOIT.scala | 52 +++++++++---------- .../ScalaCosmosDBEmulatorContainer.scala | 25 +++++++++ 3 files changed, 51 insertions(+), 27 deletions(-) create mode 100644 scio-cosmosdb/src/it/scala/com/spotify/scio/cosmosdb/ScalaCosmosDBEmulatorContainer.scala diff --git a/build.sbt b/build.sbt index 5fb2b732cc..d1fa7efc41 100644 --- a/build.sbt +++ b/build.sbt @@ -551,6 +551,7 @@ lazy val `scio-cosmosdb`: Project = project "org.mongodb" % "bson" % bsonVersion, "org.slf4j" % "slf4j-api" % slf4jVersion, // TEST + "com.dimafeng" %% "testcontainers-scala-scalatest" % testContainersVersion % "it", "org.testcontainers" % "azure" % cosmosContainerVersion % IntegrationTest, "com.outr" %% "scribe" % scribeVersion % IntegrationTest, "com.outr" %% "scribe-slf4j" % scribeVersion % IntegrationTest diff --git a/scio-cosmosdb/src/it/scala/com/spotify/scio/cosmosdb/CosmosDbIOIT.scala b/scio-cosmosdb/src/it/scala/com/spotify/scio/cosmosdb/CosmosDbIOIT.scala index 04ce682eb8..357facbe8a 100644 --- a/scio-cosmosdb/src/it/scala/com/spotify/scio/cosmosdb/CosmosDbIOIT.scala +++ b/scio-cosmosdb/src/it/scala/com/spotify/scio/cosmosdb/CosmosDbIOIT.scala @@ -1,51 +1,45 @@ package com.spotify.scio.cosmosdb import com.azure.cosmos.CosmosClientBuilder -import com.spotify.scio.ContextAndArgs +import com.dimafeng.testcontainers.ForAllTestContainer import com.spotify.scio.cosmosdb.Utils.initLog +import com.spotify.scio.{ ContextAndArgs, ScioMetrics } import org.bson.Document import org.junit.rules.TemporaryFolder import org.scalatest.BeforeAndAfterAll import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import org.testcontainers.containers.CosmosDBEmulatorContainer -import org.testcontainers.utility.DockerImageName import java.nio.file.Files import scala.util.Using /** sbt scio-cosmosdb/IntegrationTest/test */ -class CosmosDbIOIT extends AnyFlatSpec with Matchers with BeforeAndAfterAll { - private val DOCKER_NAME = "mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:latest" +class CosmosDbIOIT extends AnyFlatSpec with Matchers with BeforeAndAfterAll with ForAllTestContainer { private val DATABASE = "test" private val CONTAINER = "test" - private val cosmosDBEmulatorContainer = new CosmosDBEmulatorContainer( - DockerImageName.parse(DOCKER_NAME) - ) private val tempFolder = new TemporaryFolder tempFolder.create() initLog() - override def beforeAll(): Unit = { - scribe.info("Star CosmosDB emulator") - cosmosDBEmulatorContainer.start() + override val container: ScalaCosmosDBEmulatorContainer = ScalaCosmosDBEmulatorContainer() + override def beforeAll(): Unit = { val keyStoreFile = tempFolder.newFile("azure-cosmos-emulator.keystore").toPath - val keyStore = cosmosDBEmulatorContainer.buildNewKeyStore + val keyStore = container.buildNewKeyStore keyStore.store( Files.newOutputStream(keyStoreFile.toFile.toPath), - cosmosDBEmulatorContainer.getEmulatorKey.toCharArray + container.emulatorKey.toCharArray ) System.setProperty("javax.net.ssl.trustStore", keyStoreFile.toString) - System.setProperty("javax.net.ssl.trustStorePassword", cosmosDBEmulatorContainer.getEmulatorKey) + System.setProperty("javax.net.ssl.trustStorePassword", container.emulatorKey) System.setProperty("javax.net.ssl.trustStoreType", "PKCS12") - scribe.info("Creando la data -------------------------------------------------------->") + scribe.info("Create data -------------------------------------------------------->") val triedCreateData = Using( new CosmosClientBuilder().gatewayMode .endpointDiscoveryEnabled(false) - .endpoint(cosmosDBEmulatorContainer.getEmulatorEndpoint) - .key(cosmosDBEmulatorContainer.getEmulatorKey) + .endpoint(container.emulatorEndpoint) + .key(container.emulatorKey) .buildClient ) { client => client.createDatabase(DATABASE) @@ -61,12 +55,7 @@ class CosmosDbIOIT extends AnyFlatSpec with Matchers with BeforeAndAfterAll { scribe.error("Error creando la data", throwable) throw throwable } - scribe.info("Data creada ------------------------------------------------------------<") - } - - override protected def afterAll(): Unit = { - scribe.info("Stop CosmosDB emulator") - cosmosDBEmulatorContainer.stop() + scribe.info("Data created ------------------------------------------------------------<") } behavior of "CosmosDb with Core (SQL) API" @@ -76,18 +65,27 @@ class CosmosDbIOIT extends AnyFlatSpec with Matchers with BeforeAndAfterAll { scribe.info(s"output path: ${output.getPath}") val (sc, args) = ContextAndArgs(Array()) - val a = sc + + val counter = ScioMetrics.counter("counter") + sc .readCosmosDbCoreApi( - cosmosDBEmulatorContainer.getEmulatorEndpoint, - cosmosDBEmulatorContainer.getEmulatorKey, + container.emulatorEndpoint, + container.emulatorKey, DATABASE, CONTAINER, s"SELECT * FROM c" ) + .tap(_ => counter.inc()) .map(_.toJson) .saveAsTextFile(output.getPath) - sc.run() + val result = sc.run().waitUntilFinish() + + result.counter(counter).committed.get should equal(10) } } + + + + diff --git a/scio-cosmosdb/src/it/scala/com/spotify/scio/cosmosdb/ScalaCosmosDBEmulatorContainer.scala b/scio-cosmosdb/src/it/scala/com/spotify/scio/cosmosdb/ScalaCosmosDBEmulatorContainer.scala new file mode 100644 index 0000000000..e6e86be64d --- /dev/null +++ b/scio-cosmosdb/src/it/scala/com/spotify/scio/cosmosdb/ScalaCosmosDBEmulatorContainer.scala @@ -0,0 +1,25 @@ +package com.spotify.scio.cosmosdb + +import com.dimafeng.testcontainers.SingleContainer +import com.spotify.scio.cosmosdb.ScalaCosmosDBEmulatorContainer.defaultDockerImageName +import org.testcontainers.containers.CosmosDBEmulatorContainer +import org.testcontainers.utility.DockerImageName + +import java.security.KeyStore + +case class ScalaCosmosDBEmulatorContainer( + dockerImageName: DockerImageName = DockerImageName.parse(defaultDockerImageName) +) extends SingleContainer[CosmosDBEmulatorContainer] { + + override val container: CosmosDBEmulatorContainer = new CosmosDBEmulatorContainer(dockerImageName) + + def buildNewKeyStore: KeyStore = container.buildNewKeyStore + def emulatorEndpoint: String = container.getEmulatorEndpoint + def emulatorKey: String = container.getEmulatorKey +} + +object ScalaCosmosDBEmulatorContainer { + val defaultImage = "mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator" + val defaultTag = "latest" + val defaultDockerImageName = s"$defaultImage:$defaultTag" +}