Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

If an InvalidFieldNameException occurs while creating struct subfields, then consider field type to be variant instead. #1161

Merged
merged 1 commit into from
Dec 19, 2024
Merged

If an InvalidFieldNameException occurs while creating struct subfields, then consider field type to be variant instead. #1161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/main/scala/ai/starlake/schema/exceptions/InvalidFieldNameException.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package ai.starlake.schema.exceptions

class InvalidFieldNameException(val fieldName: String)
extends RuntimeException(
s"Invalid field name ->${fieldName}<-. Only letters, digits and '_' are allowed. Other characters including spaces are forbidden."
) {}
74 changes: 43 additions & 31 deletions src/main/scala/ai/starlake/schema/handlers/InferSchemaHandler.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
package ai.starlake.schema.handlers

import ai.starlake.config.Settings
import ai.starlake.schema.exceptions.InvalidFieldNameException
import ai.starlake.schema.model._
import ai.starlake.utils.YamlSerde
import org.apache.hadoop.fs.Path
Expand Down Expand Up @@ -101,36 +102,49 @@ object InferSchemaHandler {
currentSchema match {
case st: StructType =>
val schemaWithIndex: Seq[(StructField, Int)] = st.zipWithIndex
val attributes = schemaWithIndex.map { case (field, index) =>
val structLines =
currentLines.flatMap(Option(_)).map {
case r: Row => r.get(index)
case other =>
throw new RuntimeException(
"Encountered " + other.getClass.getName + s" for field path $fieldPath but expected a Row instead for a Struct"
)
}

createAttribute(
structLines,
st(index).dataType,
field,
fieldPath + "." + field.name,
forcePattern
)
}.toList

val rename = container.name.replaceAll("[:.-]", "_")
val renamedField = if (rename != container.name) Some(rename) else None
Attribute(
name = container.name,
`type` = st.typeName,
rename = renamedField,
required = if (!container.nullable) Some(true) else None,
array = Some(false),
attributes = attributes,
sample = None // currentLines.map(Option(_)).headOption.flatten.map(_.toString)
)
try {
val attributes = schemaWithIndex.map { case (field, index) =>
val structLines =
currentLines.flatMap(Option(_)).map {
case r: Row => r.get(index)
case other =>
throw new RuntimeException(
"Encountered " + other.getClass.getName + s" for field path $fieldPath but expected a Row instead for a Struct"
)
}

createAttribute(
structLines,
st(index).dataType,
field,
fieldPath + "." + field.name,
forcePattern
)
}.toList

Attribute(
name = container.name,
`type` = st.typeName,
rename = renamedField,
required = if (!container.nullable) Some(true) else None,
array = Some(false),
attributes = attributes,
sample = None // currentLines.map(Option(_)).headOption.flatten.map(_.toString)
)
} catch {
case _: InvalidFieldNameException =>
Attribute(
name = container.name,
`type` = PrimitiveType.variant.value,
rename = renamedField,
required = if (!container.nullable) Some(true) else None,
array = Some(false),
attributes = Nil,
sample = None // currentLines.map(Option(_)).headOption.flatten.map(_.toString)
)
}
case dt: ArrayType =>
dt.elementType match {
case _: ArrayType =>
Expand Down Expand Up @@ -207,9 +221,7 @@ object InferSchemaHandler {
if (!forcePattern || identifierRegex.pattern.matcher(result.name).matches)
result
else
throw new RuntimeException(
s"Invalid field name ->${result.name}<-. Only letters, digits and '_' are allowed. Other characters including spaces are forbidden."
)
throw new InvalidFieldNameException(result.name)
}
createAttribute(
lines,
Expand Down
Loading