Skip to content

Commit

Permalink
chore: fix errors in build pipeline (#2243)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mhamilton723
mhamilton723 authored Jun 25, 2024
1 parent 440f18e commit 5b2746b
Show file tree
Hide file tree
Showing 6 changed files with 195 additions and 126 deletions.
26 changes: 25 additions & 1 deletion cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/form/FormRecognizerV3.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ class AnalyzeDocument(override val uid: String) extends CognitiveServicesBaseNoH
with HasImageInput with HasSetLocation with SynapseMLLogging with HasSetLinkedService {
logClass(FeatureNames.AiServices.Anomaly)

setDefault(apiVersion -> Left("2022-08-31"))
setDefault(apiVersion -> Left("2023-07-31"))

def this() = this(Identifiable.randomUID("AnalyzeDocument"))

Expand All @@ -60,6 +60,30 @@ class AnalyzeDocument(override val uid: String) extends CognitiveServicesBaseNoH

def getStringIndexTypeCol: String = getVectorParam(stringIndexType)


val features = new ServiceParam[Seq[String]](this, "features",
"List of optional analysis features. (barcodes,formulas,keyValuePairs,languages,ocrHighResolution,styleFont)",
{
case Left(s) => s.forall(entry => Set(
"barcodes",
"formulas",
"keyValuePairs",
"languages",
"ocrHighResolution",
"styleFont"
)(entry))
case Right(_) => true
}, isURLParam = true)

def setFeatures(v: Seq[String]): this.type = setScalarParam(features, v)

def setFeaturesCol(v: String): this.type = setVectorParam(features, v)

def getFeatures: Seq[String] = getScalarParam(features)

def getFeaturesCol: String = getVectorParam(features)


override protected def responseDataType: DataType = AnalyzeDocumentResponse.schema

override protected def prepareEntity: Row => Option[AbstractHttpEntity] = {
Expand Down
9 changes: 8 additions & 1 deletion ...src/main/scala/com/microsoft/azure/synapse/ml/services/form/FormRecognizerV3Schemas.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ case class PageResultV3(pageNumber: Int,
spans: Seq[FormSpan],
words: Option[Seq[FormWord]],
selectionMarks: Option[Seq[FormSelectionMark]],
lines: Option[Seq[FormLine]])
lines: Option[Seq[FormLine]],
barcodes: Option[Seq[FormBarcode]])

case class DocumentParagraph(role: Option[String],
content: String,
Expand All @@ -50,6 +51,12 @@ case class FormSelectionMark(state: String, polygon: Option[Seq[Double]], confid

case class FormLine(content: String, polygon: Option[Seq[Double]], spans: Option[Seq[FormSpan]])

case class FormBarcode(confidence: Option[Double],
kind: Option[String],
polygon: Option[Seq[Double]],
span: Option[FormSpan],
value: Option[String])

case class TableResultV3(rowCount: Int,
columnCount: Int,
boundingRegions: Option[Seq[BoundingRegion]],
Expand Down
1 change: 1 addition & 0 deletions ...tive/src/test/scala/com/microsoft/azure/synapse/ml/services/CognitiveServicesCommon.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,5 @@ import com.microsoft.azure.synapse.ml.Secrets

trait CognitiveKey {
lazy val cognitiveKey = sys.env.getOrElse("COGNITIVE_API_KEY", Secrets.CognitiveApiKey)
lazy val cognitiveLoc = sys.env.getOrElse("COGNITIVE_API_LOC", "eastus")
}
117 changes: 81 additions & 36 deletions ...ive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/FormRecognizerSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,15 @@

package com.microsoft.azure.synapse.ml.services.form

import com.microsoft.azure.synapse.ml.services._
import com.microsoft.azure.synapse.ml.services.bing.BingImageSearch
import com.microsoft.azure.synapse.ml.services.form.FormsFlatteners._
import com.microsoft.azure.synapse.ml.core.env.StreamUtilities.using
import com.microsoft.azure.synapse.ml.core.spark.FluentAPI._
import com.microsoft.azure.synapse.ml.core.test.base.{Flaky, TestBase}
import com.microsoft.azure.synapse.ml.core.test.fuzzing.{TestObject, TransformerFuzzing}
import com.microsoft.azure.synapse.ml.io.http.RESTHelpers
import com.microsoft.azure.synapse.ml.io.http.RESTHelpers.retry
import com.microsoft.azure.synapse.ml.services._
import com.microsoft.azure.synapse.ml.services.bing.BingImageSearch
import com.microsoft.azure.synapse.ml.services.form.FormsFlatteners._
import com.microsoft.azure.synapse.ml.stages.UDFTransformer
import org.apache.commons.io.IOUtils
import org.apache.http.client.methods._
Expand All @@ -23,6 +24,8 @@ import org.scalactic.Equality
import spray.json._

import java.net.URI
import java.time.{ZoneOffset, ZonedDateTime}
import scala.annotation.tailrec

object TrainCustomModelProtocol extends DefaultJsonProtocol {
implicit val SourceFilterEnc: RootJsonFormat[SourceFilter] = jsonFormat2(SourceFilter)
Expand Down Expand Up @@ -173,8 +176,8 @@ class AnalyzeLayoutSuite extends TransformerFuzzing[AnalyzeLayout] with FormReco

test("Basic Usage with URL") {
val results = imageDf1.mlTransform(analyzeLayout,
flattenReadResults("layout", "readlayout"),
flattenPageResults("layout", "pageLayout"))
flattenReadResults("layout", "readlayout"),
flattenPageResults("layout", "pageLayout"))
.select("readlayout", "pageLayout")
.collect()
val headStr = results.head.getString(0)
Expand All @@ -186,8 +189,8 @@ class AnalyzeLayoutSuite extends TransformerFuzzing[AnalyzeLayout] with FormReco

test("Basic Usage with pdf") {
val results = pdfDf1.mlTransform(analyzeLayout,
flattenReadResults("layout", "readlayout"),
flattenPageResults("layout", "pageLayout"))
flattenReadResults("layout", "readlayout"),
flattenPageResults("layout", "pageLayout"))
.select("readlayout", "pageLayout")
.collect()
val headStr = results.head.getString(0)
Expand All @@ -199,8 +202,8 @@ class AnalyzeLayoutSuite extends TransformerFuzzing[AnalyzeLayout] with FormReco

test("Basic Usage with Bytes") {
val results = bytesDF1.mlTransform(bytesAnalyzeLayout,
flattenReadResults("layout", "readlayout"),
flattenPageResults("layout", "pageLayout"))
flattenReadResults("layout", "readlayout"),
flattenPageResults("layout", "pageLayout"))
.select("readlayout", "pageLayout")
.collect()
val headStr = results.head.getString(0)
Expand Down Expand Up @@ -237,8 +240,8 @@ class AnalyzeReceiptsSuite extends TransformerFuzzing[AnalyzeReceipts] with Form

test("Basic Usage with URL") {
val results = imageDf2.mlTransform(analyzeReceipts,
flattenReadResults("receipts", "readReceipts"),
flattenDocumentResults("receipts", "docReceipts"))
flattenReadResults("receipts", "readReceipts"),
flattenDocumentResults("receipts", "docReceipts"))
.select("readReceipts", "docReceipts")
.collect()
val headStr = results.head.getString(0)
Expand All @@ -249,8 +252,8 @@ class AnalyzeReceiptsSuite extends TransformerFuzzing[AnalyzeReceipts] with Form

test("Basic Usage with Bytes") {
val results = bytesDF2.mlTransform(bytesAnalyzeReceipts,
flattenReadResults("receipts", "readReceipts"),
flattenDocumentResults("receipts", "docReceipts"))
flattenReadResults("receipts", "readReceipts"),
flattenDocumentResults("receipts", "docReceipts"))
.select("readReceipts", "docReceipts")
.collect()
val headStr = results.head.getString(0)
Expand Down Expand Up @@ -285,8 +288,8 @@ class AnalyzeBusinessCardsSuite extends TransformerFuzzing[AnalyzeBusinessCards]

test("Basic Usage with URL") {
val results = imageDf3.mlTransform(analyzeBusinessCards,
flattenReadResults("businessCards", "readBusinessCards"),
flattenDocumentResults("businessCards", "docBusinessCards"))
flattenReadResults("businessCards", "readBusinessCards"),
flattenDocumentResults("businessCards", "docBusinessCards"))
.select("readBusinessCards", "docBusinessCards")
.collect()
val headStr = results.head.getString(0)
Expand All @@ -298,8 +301,8 @@ class AnalyzeBusinessCardsSuite extends TransformerFuzzing[AnalyzeBusinessCards]

test("Basic Usage with Bytes") {
val results = bytesDF3.mlTransform(bytesAnalyzeBusinessCards,
flattenReadResults("businessCards", "readBusinessCards"),
flattenDocumentResults("businessCards", "docBusinessCards"))
flattenReadResults("businessCards", "readBusinessCards"),
flattenDocumentResults("businessCards", "docBusinessCards"))
.select("readBusinessCards", "docBusinessCards")
.collect()
val headStr = results.head.getString(0)
Expand Down Expand Up @@ -335,8 +338,8 @@ class AnalyzeInvoicesSuite extends TransformerFuzzing[AnalyzeInvoices] with Form

test("Basic Usage with URL") {
val results = imageDf4.mlTransform(analyzeInvoices,
flattenReadResults("invoices", "readInvoices"),
flattenDocumentResults("invoices", "docInvoices"))
flattenReadResults("invoices", "readInvoices"),
flattenDocumentResults("invoices", "docInvoices"))
.select("readInvoices", "docInvoices")
.collect()
val headStr = results.head.getString(0)
Expand All @@ -347,8 +350,8 @@ class AnalyzeInvoicesSuite extends TransformerFuzzing[AnalyzeInvoices] with Form

test("Basic Usage with pdf") {
val results = pdfDf2.mlTransform(analyzeInvoices,
flattenReadResults("invoices", "readInvoices"),
flattenDocumentResults("invoices", "docInvoices"))
flattenReadResults("invoices", "readInvoices"),
flattenDocumentResults("invoices", "docInvoices"))
.select("readInvoices", "docInvoices")
.collect()
val headStr = results.head.getString(0)
Expand All @@ -359,8 +362,8 @@ class AnalyzeInvoicesSuite extends TransformerFuzzing[AnalyzeInvoices] with Form

test("Basic Usage with Bytes") {
val results = bytesDF4.mlTransform(bytesAnalyzeInvoices,
flattenReadResults("invoices", "readInvoices"),
flattenDocumentResults("invoices", "docInvoices"))
flattenReadResults("invoices", "readInvoices"),
flattenDocumentResults("invoices", "docInvoices"))
.select("readInvoices", "docInvoices")
.collect()
val headStr = results.head.getString(0)
Expand Down Expand Up @@ -395,8 +398,8 @@ class AnalyzeIDDocumentsSuite extends TransformerFuzzing[AnalyzeIDDocuments] wit

test("Basic Usage with URL") {
val results = imageDf5.mlTransform(analyzeIDDocuments,
flattenReadResults("ids", "readIds"),
flattenDocumentResults("ids", "docIds"))
flattenReadResults("ids", "readIds"),
flattenDocumentResults("ids", "docIds"))
.select("readIds", "docIds")
.collect()
val headStr = results.head.getString(0)
Expand All @@ -407,8 +410,8 @@ class AnalyzeIDDocumentsSuite extends TransformerFuzzing[AnalyzeIDDocuments] wit

test("Basic Usage with Bytes") {
val results = bytesDF5.mlTransform(bytesAnalyzeIDDocuments,
flattenReadResults("ids", "readIds"),
flattenDocumentResults("ids", "docIds"))
flattenReadResults("ids", "readIds"),
flattenDocumentResults("ids", "docIds"))
.select("readIds", "docIds")
.collect()
val headStr = results.head.getString(0)
Expand All @@ -424,7 +427,7 @@ class AnalyzeIDDocumentsSuite extends TransformerFuzzing[AnalyzeIDDocuments] wit
override def reader: MLReadable[_] = AnalyzeIDDocuments
}

trait CustomModelUtils extends TestBase {
trait CustomModelUtils extends TestBase with CognitiveKey {

lazy val trainingDataSAS: String = "https://mmlspark.blob.core.windows.net/datasets"

Expand All @@ -433,7 +436,7 @@ trait CustomModelUtils extends TestBase {

var modelToDelete = false

lazy val modelId: Option[String] = retry(List(10000, 20000, 30000), () => {
lazy val modelId: Option[String] = retry(List.fill(60)(10000), () => {
val resp = FormRecognizerUtils.formGet(getRequestUrl)
val modelInfo = resp.parseJson.asJsObject.fields.getOrElse("modelInfo", "")
val status = modelInfo match {
Expand All @@ -452,7 +455,49 @@ trait CustomModelUtils extends TestBase {
}
})

private def fetchModels(url: String, accumulatedModels: Seq[JsObject] = Seq.empty): Seq[JsObject] = {
val request = new HttpGet(url)
request.addHeader("Ocp-Apim-Subscription-Key", cognitiveKey)
val response = RESTHelpers.safeSend(request, close = false)
val content: String = IOUtils.toString(response.getEntity.getContent, "utf-8")
val parsedResponse = JsonParser(content).asJsObject
response.close()

val models = parsedResponse.fields("modelList").convertTo[JsArray].elements.map(_.asJsObject)
println(s"Found ${models.length} more models")
val allModels = accumulatedModels ++ models

parsedResponse.fields.get("nextLink") match {
case Some(JsString(nextLink)) =>
try {
fetchModels(nextLink, allModels)
} catch {
case _: org.apache.http.client.ClientProtocolException =>
allModels.toSet.toList
}
case _ => allModels.toSet.toList
}
}

def deleteOldModels(): Unit = {
val initialUrl = "https://eastus.api.cognitive.microsoft.com/formrecognizer/v2.1/custom/models"
val allModels = fetchModels(initialUrl)
println(s"found ${allModels.length} models")

val modelsToDelete = allModels.filter { model =>
val createdDateTime = ZonedDateTime.parse(model.fields("createdDateTime").convertTo[String])
createdDateTime.isBefore(ZonedDateTime.now(ZoneOffset.UTC).minusHours(24))
}.map(_.fields("modelId").convertTo[String])

modelsToDelete.foreach { modelId =>
FormRecognizerUtils.formDelete(modelId)
println(s"Deleted $modelId")
}

}

override def afterAll(): Unit = {
deleteOldModels()
if (modelToDelete) {
modelId.foreach(FormRecognizerUtils.formDelete(_))
}
Expand Down Expand Up @@ -483,7 +528,7 @@ class ListCustomModelsSuite extends TransformerFuzzing[ListCustomModels]
test("List model list details") {
print(modelId) // Trigger model creation
val results = pathDf.mlTransform(listCustomModels,
flattenModelList("models", "modelIds"))
flattenModelList("models", "modelIds"))
.select("modelIds")
.collect()
assert(results.head.getString(0) != "")
Expand Down Expand Up @@ -570,9 +615,9 @@ class AnalyzeCustomModelSuite extends TransformerFuzzing[AnalyzeCustomModel]

test("Basic Usage with URL") {
val results = imageDf4.mlTransform(analyzeCustomModel,
flattenReadResults("form", "readForm"),
flattenPageResults("form", "pageForm"),
flattenDocumentResults("form", "docForm"))
flattenReadResults("form", "readForm"),
flattenPageResults("form", "pageForm"),
flattenDocumentResults("form", "docForm"))
.select("readForm", "pageForm", "docForm")
.collect()
assert(results.head.getString(0) === "")
Expand All @@ -583,9 +628,9 @@ class AnalyzeCustomModelSuite extends TransformerFuzzing[AnalyzeCustomModel]

test("Basic Usage with Bytes") {
val results = bytesDF4.mlTransform(bytesAnalyzeCustomModel,
flattenReadResults("form", "readForm"),
flattenPageResults("form", "pageForm"),
flattenDocumentResults("form", "docForm"))
flattenReadResults("form", "readForm"),
flattenPageResults("form", "pageForm"),
flattenDocumentResults("form", "docForm"))
.select("readForm", "pageForm", "docForm")
.collect()
assert(results.head.getString(0) === "")
Expand Down
11 changes: 6 additions & 5 deletions ...ve/src/test/scala/com/microsoft/azure/synapse/ml/services/translate/TranslatorSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ trait TranslatorUtils extends TestBase {

lazy val textDf1: DataFrame = Seq(List("Bye")).toDF("text")

lazy val textDf2: DataFrame = Seq(List("Good morning", "Bye")).toDF("text")
lazy val textDf2: DataFrame = Seq(List("Good morning", "Bye")).toDF("text")

lazy val textDf3: DataFrame = Seq(List("This is fucked.")).toDF("text")

Expand All @@ -35,7 +35,7 @@ trait TranslatorUtils extends TestBase {
"or phrase</mstrans:dictionary> is a dictionary entry.")).toDF("text")

lazy val textDf6: DataFrame = Seq(("Hi, this is Synapse!", "zh-Hans"),
(null, "zh-Hans"), ("test", null)) //scalastyle:ignore null
(null, "zh-Hans"), ("test", null)) //scalastyle:ignore null
.toDF("text", "language")

lazy val emptyDf: DataFrame = Seq("").toDF()
Expand All @@ -53,7 +53,7 @@ class TranslateSuite extends TransformerFuzzing[Translate]
.setConcurrency(5)

def getTranslationTextResult(translator: Translate,
df: DataFrame): DataFrame = {
df: DataFrame): DataFrame = {
translator
.transform(df)
.withColumn("translation", flatten(col("translation.translations")))
Expand Down Expand Up @@ -190,8 +190,8 @@ class TransliterateSuite extends TransformerFuzzing[Transliterate]
.withColumn("script", col("result.script"))
.select("text", "script").collect()

assert(TransliterateSuite.stripInvalid(results.head.getSeq(0).mkString("\n")) === "Kon'nichiwa\nsayonara")
assert(TransliterateSuite.stripInvalid(results.head.getSeq(1).mkString("\n")) === "Latn\nLatn")
assert(TransliterateSuite.stripInvalid(results.head.getSeq(0).mkString("\n")).contains("Kon'nichiwa"))
assert(TransliterateSuite.stripInvalid(results.head.getSeq(1).mkString("\n")).contains("Latn"))
}

test("Throw errors if required fields not set") {
Expand All @@ -213,6 +213,7 @@ class TransliterateSuite extends TransformerFuzzing[Transliterate]
o.map(t => (TransliterateSuite.stripInvalid(t._1), t._2))
}
}

override def assertDFEq(df1: DataFrame, df2: DataFrame)(implicit eq: Equality[DataFrame]): Unit = {
val column = "result"
super.assertDFEq(
Expand Down
Loading

0 comments on commit 5b2746b

Please sign in to comment.