From 17062d8e2639d7feeffe4e58ed2f823d6d730150 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Fri, 31 Mar 2023 12:10:01 -0500 Subject: [PATCH 1/2] refactored literal type casting from flytepropeller Signed-off-by: Daniel Rammer --- clients/go/coreutils/casting.go | 367 ++++++++ clients/go/coreutils/casting_test.go | 868 ++++++++++++++++++ clients/go/coreutils/extract_literal.go | 92 -- clients/go/coreutils/extract_literal_test.go | 202 ---- .../go/coreutils/{literals.go => literal.go} | 91 ++ .../{literals_test.go => literal_test.go} | 188 ++++ clients/go/coreutils/typing.go | 140 +++ clients/go/coreutils/typing_test.go | 52 ++ go.mod | 14 +- go.sum | 28 +- 10 files changed, 1728 insertions(+), 314 deletions(-) create mode 100644 clients/go/coreutils/casting.go create mode 100644 clients/go/coreutils/casting_test.go delete mode 100644 clients/go/coreutils/extract_literal.go delete mode 100644 clients/go/coreutils/extract_literal_test.go rename clients/go/coreutils/{literals.go => literal.go} (81%) rename clients/go/coreutils/{literals_test.go => literal_test.go} (82%) create mode 100644 clients/go/coreutils/typing.go create mode 100644 clients/go/coreutils/typing_test.go diff --git a/clients/go/coreutils/casting.go b/clients/go/coreutils/casting.go new file mode 100644 index 000000000..66dbfa88a --- /dev/null +++ b/clients/go/coreutils/casting.go @@ -0,0 +1,367 @@ +package coreutils + +import ( + "strings" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + structpb "github.com/golang/protobuf/ptypes/struct" +) + +type typeChecker interface { + CastsFrom(*core.LiteralType) bool +} + +type trivialChecker struct { + literalType *core.LiteralType +} + +// CastsFrom is a trivial type checker merely checks if types match exactly. +func (t trivialChecker) CastsFrom(upstreamType *core.LiteralType) bool { + // If upstream is an enum, it can be consumed as a string downstream + if upstreamType.GetEnumType() != nil { + if t.literalType.GetSimple() == core.SimpleType_STRING { + return true + } + } + // If t is an enum, it can be created from a string as Enums as just constrained String aliases + if t.literalType.GetEnumType() != nil { + if upstreamType.GetSimple() == core.SimpleType_STRING { + return true + } + } + + if GetTagForType(upstreamType) != "" && GetTagForType(t.literalType) != GetTagForType(upstreamType) { + return false + } + + // Ignore metadata when comparing types. + upstreamTypeCopy := *upstreamType + downstreamTypeCopy := *t.literalType + upstreamTypeCopy.Structure = &core.TypeStructure{} + downstreamTypeCopy.Structure = &core.TypeStructure{} + upstreamTypeCopy.Metadata = &structpb.Struct{} + downstreamTypeCopy.Metadata = &structpb.Struct{} + upstreamTypeCopy.Annotation = &core.TypeAnnotation{} + downstreamTypeCopy.Annotation = &core.TypeAnnotation{} + return upstreamTypeCopy.String() == downstreamTypeCopy.String() +} + +type noneTypeChecker struct{} + +// CastsFrom matches only void +func (t noneTypeChecker) CastsFrom(upstreamType *core.LiteralType) bool { + return isNoneType(upstreamType) +} + +type mapTypeChecker struct { + literalType *core.LiteralType +} + +// CastsFrom checks that the target map type can be cast to the current map type. We need to ensure both the key types +// and value types match. +func (t mapTypeChecker) CastsFrom(upstreamType *core.LiteralType) bool { + // Empty maps should match any collection. + mapLiteralType := upstreamType.GetMapValueType() + if isNoneType(mapLiteralType) { + return true + } else if mapLiteralType != nil { + return getTypeChecker(t.literalType.GetMapValueType()).CastsFrom(mapLiteralType) + } + + return false +} + +type collectionTypeChecker struct { + literalType *core.LiteralType +} + +// CastsFrom checks whether two collection types match. We need to ensure that the nesting is correct and the final +// subtypes match. +func (t collectionTypeChecker) CastsFrom(upstreamType *core.LiteralType) bool { + // Empty collections should match any collection. + collectionType := upstreamType.GetCollectionType() + if isNoneType(upstreamType.GetCollectionType()) { + return true + } else if collectionType != nil { + return getTypeChecker(t.literalType.GetCollectionType()).CastsFrom(collectionType) + } + + return false +} + +type schemaTypeChecker struct { + literalType *core.LiteralType +} + +// CastsFrom handles type casting to the underlying schema type. +// Schemas are more complex types in the Flyte ecosystem. A schema is considered castable in the following +// cases. +// +// 1. The downstream schema has no column types specified. In such a case, it accepts all schema input since it is +// generic. +// +// 2. The downstream schema has a subset of the upstream columns and they match perfectly. +// +// 3. The upstream type can be Schema type or structured dataset type +func (t schemaTypeChecker) CastsFrom(upstreamType *core.LiteralType) bool { + schemaType := upstreamType.GetSchema() + structuredDatasetType := upstreamType.GetStructuredDatasetType() + if structuredDatasetType == nil && schemaType == nil { + return false + } + + if schemaType != nil { + return schemaCastFromSchema(schemaType, t.literalType.GetSchema()) + } + + // Flyte Schema can only be serialized to parquet + if len(structuredDatasetType.Format) != 0 && !strings.EqualFold(structuredDatasetType.Format, "parquet") { + return false + } + + return schemaCastFromStructuredDataset(structuredDatasetType, t.literalType.GetSchema()) +} + +type structuredDatasetChecker struct { + literalType *core.LiteralType +} + +// CastsFrom for Structured dataset are more complex types in the Flyte ecosystem. A structured dataset is considered +// castable in the following cases: +// +// 1. The downstream structured dataset has no column types specified. In such a case, it accepts all structured dataset input since it is +// generic. +// +// 2. The downstream structured dataset has a subset of the upstream structured dataset columns and they match perfectly. +// +// 3. The upstream type can be Schema type or structured dataset type +func (t structuredDatasetChecker) CastsFrom(upstreamType *core.LiteralType) bool { + // structured datasets are nullable + if isNoneType(upstreamType) { + return true + } + structuredDatasetType := upstreamType.GetStructuredDatasetType() + schemaType := upstreamType.GetSchema() + if structuredDatasetType == nil && schemaType == nil { + return false + } + if schemaType != nil { + // Flyte Schema can only be serialized to parquet + format := t.literalType.GetStructuredDatasetType().Format + if len(format) != 0 && !strings.EqualFold(format, "parquet") { + return false + } + return structuredDatasetCastFromSchema(schemaType, t.literalType.GetStructuredDatasetType()) + } + return structuredDatasetCastFromStructuredDataset(structuredDatasetType, t.literalType.GetStructuredDatasetType()) +} + +// Upstream (schema) -> downstream (schema) +func schemaCastFromSchema(upstream *core.SchemaType, downstream *core.SchemaType) bool { + if len(upstream.Columns) == 0 || len(downstream.Columns) == 0 { + return true + } + + nameToTypeMap := make(map[string]core.SchemaType_SchemaColumn_SchemaColumnType) + for _, column := range upstream.Columns { + nameToTypeMap[column.Name] = column.Type + } + + // Check that the downstream schema is a strict sub-set of the upstream schema. + for _, column := range downstream.Columns { + upstreamType, ok := nameToTypeMap[column.Name] + if !ok { + return false + } + if upstreamType != column.Type { + return false + } + } + return true +} + +type unionTypeChecker struct { + literalType *core.LiteralType +} + +func (t unionTypeChecker) CastsFrom(upstreamType *core.LiteralType) bool { + unionType := t.literalType.GetUnionType() + + upstreamUnionType := upstreamType.GetUnionType() + if upstreamUnionType != nil { + // For each upstream variant we must find a compatible downstream variant + for _, u := range upstreamUnionType.GetVariants() { + found := false + for _, d := range unionType.GetVariants() { + if AreTypesCastable(u, d) { + found = true + break + } + } + if !found { + return false + } + } + + return true + } + + // Matches iff we can unambiguously select a variant + foundOne := false + for _, x := range unionType.GetVariants() { + if AreTypesCastable(upstreamType, x) { + if foundOne { + return false + } + foundOne = true + } + } + + return foundOne +} + +// Upstream (structuredDatasetType) -> downstream (structuredDatasetType) +func structuredDatasetCastFromStructuredDataset(upstream *core.StructuredDatasetType, downstream *core.StructuredDatasetType) bool { + // Skip the format check here when format is empty. https://github.com/flyteorg/flyte/issues/2864 + if len(upstream.Format) != 0 && len(downstream.Format) != 0 && !strings.EqualFold(upstream.Format, downstream.Format) { + return false + } + + if len(upstream.Columns) == 0 || len(downstream.Columns) == 0 { + return true + } + + nameToTypeMap := make(map[string]*core.LiteralType) + for _, column := range upstream.Columns { + nameToTypeMap[column.Name] = column.LiteralType + } + + // Check that the downstream structured dataset is a strict sub-set of the upstream structured dataset. + for _, column := range downstream.Columns { + upstreamType, ok := nameToTypeMap[column.Name] + if !ok { + return false + } + if !getTypeChecker(column.LiteralType).CastsFrom(upstreamType) { + return false + } + } + return true +} + +// Upstream (schemaType) -> downstream (structuredDatasetType) +func structuredDatasetCastFromSchema(upstream *core.SchemaType, downstream *core.StructuredDatasetType) bool { + if len(upstream.Columns) == 0 || len(downstream.Columns) == 0 { + return true + } + nameToTypeMap := make(map[string]core.SchemaType_SchemaColumn_SchemaColumnType) + for _, column := range upstream.Columns { + nameToTypeMap[column.Name] = column.GetType() + } + + // Check that the downstream structuredDataset is a strict sub-set of the upstream schema. + for _, column := range downstream.Columns { + upstreamType, ok := nameToTypeMap[column.Name] + if !ok { + return false + } + if !schemaTypeIsMatchStructuredDatasetType(upstreamType, column.LiteralType.GetSimple()) { + return false + } + } + return true +} + +// Upstream (structuredDatasetType) -> downstream (schemaType) +func schemaCastFromStructuredDataset(upstream *core.StructuredDatasetType, downstream *core.SchemaType) bool { + if len(upstream.Columns) == 0 || len(downstream.Columns) == 0 { + return true + } + nameToTypeMap := make(map[string]core.SimpleType) + for _, column := range upstream.Columns { + nameToTypeMap[column.Name] = column.LiteralType.GetSimple() + } + + // Check that the downstream schema is a strict sub-set of the upstream structuredDataset. + for _, column := range downstream.Columns { + upstreamType, ok := nameToTypeMap[column.Name] + if !ok { + return false + } + if !schemaTypeIsMatchStructuredDatasetType(column.GetType(), upstreamType) { + return false + } + } + return true +} + +func schemaTypeIsMatchStructuredDatasetType(schemaType core.SchemaType_SchemaColumn_SchemaColumnType, structuredDatasetType core.SimpleType) bool { + switch schemaType { + case core.SchemaType_SchemaColumn_INTEGER: + return structuredDatasetType == core.SimpleType_INTEGER + case core.SchemaType_SchemaColumn_FLOAT: + return structuredDatasetType == core.SimpleType_FLOAT + case core.SchemaType_SchemaColumn_STRING: + return structuredDatasetType == core.SimpleType_STRING + case core.SchemaType_SchemaColumn_BOOLEAN: + return structuredDatasetType == core.SimpleType_BOOLEAN + case core.SchemaType_SchemaColumn_DATETIME: + return structuredDatasetType == core.SimpleType_DATETIME + case core.SchemaType_SchemaColumn_DURATION: + return structuredDatasetType == core.SimpleType_DURATION + } + return false +} + +func isNoneType(t *core.LiteralType) bool { + switch t.GetType().(type) { + case *core.LiteralType_Simple: + return t.GetSimple() == core.SimpleType_NONE + default: + return false + } +} + +func getTypeChecker(t *core.LiteralType) typeChecker { + switch t.GetType().(type) { + case *core.LiteralType_CollectionType: + return collectionTypeChecker{ + literalType: t, + } + case *core.LiteralType_MapValueType: + return mapTypeChecker{ + literalType: t, + } + case *core.LiteralType_Schema: + return schemaTypeChecker{ + literalType: t, + } + case *core.LiteralType_UnionType: + return unionTypeChecker{ + literalType: t, + } + case *core.LiteralType_StructuredDatasetType: + return structuredDatasetChecker{ + literalType: t, + } + default: + if isNoneType(t) { + return noneTypeChecker{} + } + + return trivialChecker{ + literalType: t, + } + } +} + +func AreTypesCastable(upstreamType, downstreamType *core.LiteralType) bool { + return getTypeChecker(downstreamType).CastsFrom(upstreamType) +} + +func GetTagForType(x *core.LiteralType) string { + if x.GetStructure() == nil { + return "" + } + return x.GetStructure().GetTag() +} diff --git a/clients/go/coreutils/casting_test.go b/clients/go/coreutils/casting_test.go new file mode 100644 index 000000000..36f06e6d9 --- /dev/null +++ b/clients/go/coreutils/casting_test.go @@ -0,0 +1,868 @@ +package coreutils + +import ( + "testing" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + structpb "github.com/golang/protobuf/ptypes/struct" + "github.com/stretchr/testify/assert" +) + +func TestSimpleLiteralCasting(t *testing.T) { + t.Run("BaseCase_Integer", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + ) + assert.True(t, castable, "Integers should be castable to other integers") + }) + + t.Run("IntegerToFloat", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_FLOAT}, + }, + ) + assert.False(t, castable, "Integers should not be castable to floats") + }) + + t.Run("FloatToInteger", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_FLOAT}, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + ) + assert.False(t, castable, "Floats should not be castable to integers") + }) + + t.Run("VoidToInteger", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_NONE}, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + ) + assert.False(t, castable, "Non-optional types are non-nullable") + }) + + t.Run("IgnoreMetadata", func(t *testing.T) { + s := structpb.Struct{ + Fields: map[string]*structpb.Value{ + "a": {}, + }, + } + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + Metadata: &s, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + ) + assert.True(t, castable, "Metadata should be ignored") + }) + + t.Run("EnumToString", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_EnumType{EnumType: &core.EnumType{ + Values: []string{"x", "y"}, + }}, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}, + }, + ) + assert.True(t, castable, "Enum should be castable to string") + }) + + t.Run("EnumToEnum", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_EnumType{EnumType: &core.EnumType{ + Values: []string{"x", "y"}, + }}, + }, + &core.LiteralType{ + Type: &core.LiteralType_EnumType{EnumType: &core.EnumType{ + Values: []string{"x", "y"}, + }}, + }, + ) + assert.True(t, castable, "Enum should be castable to Enums if they are identical") + }) + + t.Run("EnumToEnum", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_EnumType{EnumType: &core.EnumType{ + Values: []string{"x", "y"}, + }}, + }, + &core.LiteralType{ + Type: &core.LiteralType_EnumType{EnumType: &core.EnumType{ + Values: []string{"m", "n"}, + }}, + }, + ) + assert.False(t, castable, "Enum should not be castable to non matching enums") + }) + + t.Run("StringToEnum", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}, + }, + &core.LiteralType{ + Type: &core.LiteralType_EnumType{EnumType: &core.EnumType{ + Values: []string{"x", "y"}, + }}, + }, + ) + assert.True(t, castable, "Strings should be castable to enums - may result in runtime failure") + }) +} + +func TestUnionCasting(t *testing.T) { + t.Run("StringToUnionUnambiguously", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}, + }, + &core.LiteralType{ + Type: &core.LiteralType_UnionType{ + UnionType: &core.UnionType{ + Variants: []*core.LiteralType{ + { + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + Structure: &core.TypeStructure{ + Tag: "int", + }, + }, + { + Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}, + Structure: &core.TypeStructure{ + Tag: "str", + }, + }, + }, + }, + }, + }, + ) + assert.True(t, castable, "Strings should be castable to (str | int)") + }) + + t.Run("StringToUnionAmbiguously", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}, + }, + &core.LiteralType{ + Type: &core.LiteralType_UnionType{ + UnionType: &core.UnionType{ + Variants: []*core.LiteralType{ + { + Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}, + Structure: &core.TypeStructure{ + Tag: "str1", + }, + }, + { + Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}, + Structure: &core.TypeStructure{ + Tag: "str2", + }, + }, + }, + }, + }, + }, + ) + assert.False(t, castable, "Raw string literals should not be ambiguously castable to (str | str)") + }) + + t.Run("UnionToUnionSuperset", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_UnionType{ + UnionType: &core.UnionType{ + Variants: []*core.LiteralType{ + { + Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}, + Structure: &core.TypeStructure{ + Tag: "str1", + }, + }, + { + Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}, + Structure: &core.TypeStructure{ + Tag: "str2", + }, + }, + }, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_UnionType{ + UnionType: &core.UnionType{ + Variants: []*core.LiteralType{ + { + Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}, + Structure: &core.TypeStructure{ + Tag: "str1", + }, + }, + { + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + Structure: &core.TypeStructure{ + Tag: "int1", + }, + }, + { + Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}, + Structure: &core.TypeStructure{ + Tag: "str2", + }, + }, + }, + }, + }, + }, + ) + assert.True(t, castable, "Union types can be cast to a union that contains a superset of variants") + }) + + t.Run("UnionToUnionTagMismatch", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_UnionType{ + UnionType: &core.UnionType{ + Variants: []*core.LiteralType{ + { + Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}, + Structure: &core.TypeStructure{ + Tag: "str1", + }, + }, + { + Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}, + Structure: &core.TypeStructure{ + Tag: "str2", + }, + }, + }, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_UnionType{ + UnionType: &core.UnionType{ + Variants: []*core.LiteralType{ + { + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + Structure: &core.TypeStructure{ + Tag: "str2", + }, + }, + { + Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}, + Structure: &core.TypeStructure{ + Tag: "str3", + }, + }, + }, + }, + }, + }, + ) + assert.False(t, castable, "Union types can only be cast to a union that contains a superset of variants") + }) + + t.Run("UnionToUnionTypeMismatch", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_UnionType{ + UnionType: &core.UnionType{ + Variants: []*core.LiteralType{ + { + Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}, + Structure: &core.TypeStructure{ + Tag: "test", + }, + }, + }, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_UnionType{ + UnionType: &core.UnionType{ + Variants: []*core.LiteralType{ + { + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + Structure: &core.TypeStructure{ + Tag: "test", + }, + }, + }, + }, + }, + }, + ) + assert.False(t, castable, "Union types can only be cast to a union that contains a superset of variants") + }) +} + +func TestCollectionCasting(t *testing.T) { + t.Run("BaseCase_SingleIntegerCollection", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + }, + }, + ) + assert.True(t, castable, "[Integer] should be castable to [Integer].") + }) + + t.Run("Empty collection", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_NONE}, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + }, + }, + ) + assert.True(t, castable, "[] should be castable to [Integer].") + }) + + t.Run("SingleIntegerCollectionToSingleFloatCollection", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_FLOAT}, + }, + }, + }, + ) + assert.False(t, castable, "[Integer] should not be castable to [Float]") + }) + + t.Run("MismatchedNestLevels_Scalar", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + ) + assert.False(t, castable, "[Integer] should not be castable to Integer") + }) + + t.Run("MismatchedNestLevels_Collections", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + }, + }, + }, + }, + ) + assert.False(t, castable, "[Integer] should not be castable to [[Integer]]") + }) + + t.Run("Nullable_Collections", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_NONE, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + }, + }, + }, + }, + ) + assert.False(t, castable, "Non-optional collections are not nullable") + }) +} + +func TestMapCasting(t *testing.T) { + t.Run("BaseCase_SingleIntegerMap", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_MapValueType{ + MapValueType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_MapValueType{ + MapValueType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + }, + }, + ) + assert.True(t, castable, "{k: Integer} should be castable to {k: Integer}.") + }) + + t.Run("ScalarIntegerMapToScalarFloatMap", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_MapValueType{ + MapValueType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_MapValueType{ + MapValueType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_FLOAT}, + }, + }, + }, + ) + assert.False(t, castable, "{k: Integer} should not be castable to {k: Float}") + }) + + t.Run("ScalarIntegerMapToScalarFloatMap", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_MapValueType{ + MapValueType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_NONE}, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_MapValueType{ + MapValueType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_FLOAT}, + }, + }, + }, + ) + + assert.True(t, castable, "{k: None} should be castable to {k: Float}") + }) + + t.Run("ScalarStructToStruct", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }, + }, + ) + assert.True(t, castable, "castable from Struct to struct") + }) + + t.Run("MismatchedMapNestLevels_Scalar", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_MapValueType{ + MapValueType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + ) + assert.False(t, castable, "{k: Integer} should not be castable to Integer") + }) + + t.Run("MismatchedMapNestLevels_Maps", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_MapValueType{ + MapValueType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + }, + }, + &core.LiteralType{ + Type: &core.LiteralType_MapValueType{ + MapValueType: &core.LiteralType{ + Type: &core.LiteralType_MapValueType{ + MapValueType: &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, + }, + }, + }, + }, + }, + ) + assert.False(t, castable, "{k: Integer} should not be castable to {k: {k: Integer}}") + }) +} + +func TestSchemaCasting(t *testing.T) { + genericSchema := &core.LiteralType{ + Type: &core.LiteralType_Schema{ + Schema: &core.SchemaType{ + Columns: []*core.SchemaType_SchemaColumn{}, + }, + }, + } + genericStructuredDataset := &core.LiteralType{ + Type: &core.LiteralType_StructuredDatasetType{ + StructuredDatasetType: &core.StructuredDatasetType{ + Columns: []*core.StructuredDatasetType_DatasetColumn{}, + Format: "", + }, + }, + } + subsetIntegerSchema := &core.LiteralType{ + Type: &core.LiteralType_Schema{ + Schema: &core.SchemaType{ + Columns: []*core.SchemaType_SchemaColumn{ + { + Name: "a", + Type: core.SchemaType_SchemaColumn_INTEGER, + }, + }, + }, + }, + } + supersetIntegerAndFloatSchema := &core.LiteralType{ + Type: &core.LiteralType_Schema{ + Schema: &core.SchemaType{ + Columns: []*core.SchemaType_SchemaColumn{ + { + Name: "a", + Type: core.SchemaType_SchemaColumn_INTEGER, + }, + { + Name: "b", + Type: core.SchemaType_SchemaColumn_FLOAT, + }, + }, + }, + }, + } + supersetStructuredDataset := &core.LiteralType{ + Type: &core.LiteralType_StructuredDatasetType{ + StructuredDatasetType: &core.StructuredDatasetType{ + Columns: []*core.StructuredDatasetType_DatasetColumn{ + { + Name: "a", + LiteralType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}, + }, + { + Name: "b", + LiteralType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_FLOAT}}, + }, + }, + Format: "parquet", + }, + }, + } + mismatchedSubsetSchema := &core.LiteralType{ + Type: &core.LiteralType_Schema{ + Schema: &core.SchemaType{ + Columns: []*core.SchemaType_SchemaColumn{ + { + Name: "a", + Type: core.SchemaType_SchemaColumn_FLOAT, + }, + }, + }, + }, + } + + t.Run("BaseCase_GenericSchema", func(t *testing.T) { + castable := AreTypesCastable(genericSchema, genericSchema) + assert.True(t, castable, "Schema() should be castable to Schema()") + }) + + t.Run("GenericSchemaToNonGeneric", func(t *testing.T) { + castable := AreTypesCastable(genericSchema, subsetIntegerSchema) + assert.True(t, castable, "Schema() should be castable to Schema(a=Integer)") + }) + + t.Run("NonGenericSchemaToGeneric", func(t *testing.T) { + castable := AreTypesCastable(subsetIntegerSchema, genericSchema) + assert.True(t, castable, "Schema(a=Integer) should be castable to Schema()") + }) + + t.Run("SupersetToSubsetTypedSchema", func(t *testing.T) { + castable := AreTypesCastable(supersetIntegerAndFloatSchema, subsetIntegerSchema) + assert.True(t, castable, "Schema(a=Integer, b=Float) should be castable to Schema(a=Integer)") + }) + + t.Run("GenericToSubsetTypedSchema", func(t *testing.T) { + castable := AreTypesCastable(genericStructuredDataset, subsetIntegerSchema) + assert.True(t, castable, "StructuredDataset() with generic format should be castable to Schema(a=Integer)") + }) + + t.Run("SubsetTypedSchemaToGeneric", func(t *testing.T) { + castable := AreTypesCastable(subsetIntegerSchema, genericStructuredDataset) + assert.True(t, castable, "Schema(a=Integer) should be castable to StructuredDataset() with generic format") + }) + + t.Run("SupersetStructuredToSubsetTypedSchema", func(t *testing.T) { + castable := AreTypesCastable(supersetStructuredDataset, subsetIntegerSchema) + assert.True(t, castable, "StructuredDataset(a=Integer, b=Float) should be castable to Schema(a=Integer)") + }) + + t.Run("SubsetToSupersetSchema", func(t *testing.T) { + castable := AreTypesCastable(subsetIntegerSchema, supersetIntegerAndFloatSchema) + assert.False(t, castable, "Schema(a=Integer) should not be castable to Schema(a=Integer, b=Float)") + }) + + t.Run("MismatchedColumns", func(t *testing.T) { + castable := AreTypesCastable(subsetIntegerSchema, mismatchedSubsetSchema) + assert.False(t, castable, "Schema(a=Integer) should not be castable to Schema(a=Float)") + }) + + t.Run("MismatchedColumnsFlipped", func(t *testing.T) { + castable := AreTypesCastable(mismatchedSubsetSchema, subsetIntegerSchema) + assert.False(t, castable, "Schema(a=Float) should not be castable to Schema(a=Integer)") + }) + + t.Run("SchemasAreNullable", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_NONE, + }, + }, + subsetIntegerSchema) + assert.False(t, castable, "Non-optional schemas are not nullable") + }) +} + +func TestStructuredDatasetCasting(t *testing.T) { + emptyStructuredDataset := &core.LiteralType{ + Type: &core.LiteralType_StructuredDatasetType{ + StructuredDatasetType: &core.StructuredDatasetType{ + Columns: []*core.StructuredDatasetType_DatasetColumn{}, + Format: "", + }, + }, + } + genericStructuredDataset := &core.LiteralType{ + Type: &core.LiteralType_StructuredDatasetType{ + StructuredDatasetType: &core.StructuredDatasetType{ + Columns: []*core.StructuredDatasetType_DatasetColumn{}, + Format: "parquet", + }, + }, + } + subsetStructuredDataset := &core.LiteralType{ + Type: &core.LiteralType_StructuredDatasetType{ + StructuredDatasetType: &core.StructuredDatasetType{ + Columns: []*core.StructuredDatasetType_DatasetColumn{ + { + Name: "a", + LiteralType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}, + }, + { + Name: "b", + LiteralType: &core.LiteralType{Type: &core.LiteralType_CollectionType{CollectionType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}}, + }, + }, + Format: "parquet", + }, + }, + } + supersetStructuredDataset := &core.LiteralType{ + Type: &core.LiteralType_StructuredDatasetType{ + StructuredDatasetType: &core.StructuredDatasetType{ + Columns: []*core.StructuredDatasetType_DatasetColumn{ + { + Name: "a", + LiteralType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}, + }, + { + Name: "b", + LiteralType: &core.LiteralType{Type: &core.LiteralType_CollectionType{CollectionType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}}, + }, + { + Name: "c", + LiteralType: &core.LiteralType{Type: &core.LiteralType_MapValueType{MapValueType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}}, + }, + }, + Format: "parquet", + }, + }, + } + integerSchema := &core.LiteralType{ + Type: &core.LiteralType_Schema{ + Schema: &core.SchemaType{ + Columns: []*core.SchemaType_SchemaColumn{ + { + Name: "a", + Type: core.SchemaType_SchemaColumn_INTEGER, + }, + }, + }, + }, + } + integerStructuredDataset := &core.LiteralType{ + Type: &core.LiteralType_StructuredDatasetType{ + StructuredDatasetType: &core.StructuredDatasetType{ + Columns: []*core.StructuredDatasetType_DatasetColumn{ + { + Name: "a", + LiteralType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}, + }, + }, + Format: "parquet", + }, + }, + } + mismatchedSubsetStructuredDataset := &core.LiteralType{ + Type: &core.LiteralType_StructuredDatasetType{ + StructuredDatasetType: &core.StructuredDatasetType{ + Columns: []*core.StructuredDatasetType_DatasetColumn{ + { + Name: "a", + LiteralType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_FLOAT}}, + }, + }, + }, + }, + } + + t.Run("BaseCase_GenericStructuredDataset", func(t *testing.T) { + castable := AreTypesCastable(genericStructuredDataset, genericStructuredDataset) + assert.True(t, castable, "StructuredDataset() should be castable to StructuredDataset()") + }) + + t.Run("GenericStructuredDatasetToNonGeneric", func(t *testing.T) { + castable := AreTypesCastable(genericStructuredDataset, subsetStructuredDataset) + assert.True(t, castable, "StructuredDataset() should be castable to StructuredDataset(a=Integer, b=Collection)") + }) + + t.Run("NonGenericStructuredDatasetToGeneric", func(t *testing.T) { + castable := AreTypesCastable(subsetStructuredDataset, genericStructuredDataset) + assert.True(t, castable, "StructuredDataset(a=Integer, b=Collection) should be castable to StructuredDataset()") + }) + + t.Run("SupersetToSubsetTypedStructuredDataset", func(t *testing.T) { + castable := AreTypesCastable(supersetStructuredDataset, subsetStructuredDataset) + assert.True(t, castable, "StructuredDataset(a=Integer, b=Collection, c=Map) should be castable to StructuredDataset(a=Integer, b=Collection)") + }) + + t.Run("SubsetToSupersetStructuredDataset", func(t *testing.T) { + castable := AreTypesCastable(subsetStructuredDataset, supersetStructuredDataset) + assert.False(t, castable, "StructuredDataset(a=Integer, b=Collection) should not be castable to StructuredDataset(a=Integer, b=Collection, c=Map)") + }) + + t.Run("SchemaToStructuredDataset", func(t *testing.T) { + castable := AreTypesCastable(integerSchema, integerStructuredDataset) + assert.True(t, castable, "Schema(a=Integer) should be castable to StructuredDataset(a=Integer)") + }) + + t.Run("MismatchedSchemaColumns", func(t *testing.T) { + castable := AreTypesCastable(integerSchema, mismatchedSubsetStructuredDataset) + assert.False(t, castable, "Schema(a=Integer) should not be castable to StructuredDataset(a=Float)") + }) + + t.Run("MismatchedColumns", func(t *testing.T) { + castable := AreTypesCastable(subsetStructuredDataset, mismatchedSubsetStructuredDataset) + assert.False(t, castable, "StructuredDataset(a=Integer, b=Collection) should not be castable to StructuredDataset(a=Float)") + }) + + t.Run("MismatchedColumnsFlipped", func(t *testing.T) { + castable := AreTypesCastable(mismatchedSubsetStructuredDataset, subsetStructuredDataset) + assert.False(t, castable, "StructuredDataset(a=Float) should not be castable to StructuredDataset(a=Integer, b=Collection)") + }) + + t.Run("GenericToEmptyFormat", func(t *testing.T) { + castable := AreTypesCastable(genericStructuredDataset, emptyStructuredDataset) + assert.True(t, castable, "StructuredDataset(format='Parquet') should be castable to StructuredDataset()") + }) + + t.Run("EmptyFormatToGeneric", func(t *testing.T) { + castable := AreTypesCastable(genericStructuredDataset, emptyStructuredDataset) + assert.True(t, castable, "StructuredDataset() should be castable to StructuredDataset(format='Parquet')") + }) + + t.Run("StructuredDatasetsAreNullable", func(t *testing.T) { + castable := AreTypesCastable( + &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_NONE, + }, + }, + subsetStructuredDataset) + assert.True(t, castable, "StructuredDataset are nullable") + }) +} diff --git a/clients/go/coreutils/extract_literal.go b/clients/go/coreutils/extract_literal.go deleted file mode 100644 index afc7fd2a0..000000000 --- a/clients/go/coreutils/extract_literal.go +++ /dev/null @@ -1,92 +0,0 @@ -// extract_literal.go -// Utility methods to extract a native golang value from a given Literal. -// Usage: -// 1] string literal extraction -// lit, _ := MakeLiteral("test_string") -// val, _ := ExtractFromLiteral(lit) -// 2] integer literal extraction. integer would be extracted in type int64. -// lit, _ := MakeLiteral([]interface{}{1, 2, 3}) -// val, _ := ExtractFromLiteral(lit) -// 3] float literal extraction. float would be extracted in type float64. -// lit, _ := MakeLiteral([]interface{}{1.0, 2.0, 3.0}) -// val, _ := ExtractFromLiteral(lit) -// 4] map of boolean literal extraction. -// mapInstance := map[string]interface{}{ -// "key1": []interface{}{1, 2, 3}, -// "key2": []interface{}{5}, -// } -// lit, _ := MakeLiteral(mapInstance) -// val, _ := ExtractFromLiteral(lit) -// For further examples check the test TestFetchLiteral in extract_literal_test.go - -package coreutils - -import ( - "fmt" - - "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" -) - -func ExtractFromLiteral(literal *core.Literal) (interface{}, error) { - switch literalValue := literal.Value.(type) { - case *core.Literal_Scalar: - switch scalarValue := literalValue.Scalar.Value.(type) { - case *core.Scalar_Primitive: - switch scalarPrimitive := scalarValue.Primitive.Value.(type) { - case *core.Primitive_Integer: - scalarPrimitiveInt := scalarPrimitive.Integer - return scalarPrimitiveInt, nil - case *core.Primitive_FloatValue: - scalarPrimitiveFloat := scalarPrimitive.FloatValue - return scalarPrimitiveFloat, nil - case *core.Primitive_StringValue: - scalarPrimitiveString := scalarPrimitive.StringValue - return scalarPrimitiveString, nil - case *core.Primitive_Boolean: - scalarPrimitiveBoolean := scalarPrimitive.Boolean - return scalarPrimitiveBoolean, nil - case *core.Primitive_Datetime: - scalarPrimitiveDateTime := scalarPrimitive.Datetime.AsTime() - return scalarPrimitiveDateTime, nil - case *core.Primitive_Duration: - scalarPrimitiveDuration := scalarPrimitive.Duration.AsDuration() - return scalarPrimitiveDuration, nil - default: - return nil, fmt.Errorf("unsupported literal scalar primitive type %T", scalarValue) - } - case *core.Scalar_Blob: - return scalarValue.Blob.Uri, nil - case *core.Scalar_Schema: - return scalarValue.Schema.Uri, nil - case *core.Scalar_Generic: - return scalarValue.Generic, nil - case *core.Scalar_StructuredDataset: - return scalarValue.StructuredDataset.Uri, nil - default: - return nil, fmt.Errorf("unsupported literal scalar type %T", scalarValue) - } - case *core.Literal_Collection: - collectionValue := literalValue.Collection.Literals - collection := make([]interface{}, len(collectionValue)) - for index, val := range collectionValue { - if collectionElem, err := ExtractFromLiteral(val); err == nil { - collection[index] = collectionElem - } else { - return nil, err - } - } - return collection, nil - case *core.Literal_Map: - mapLiteralValue := literalValue.Map.Literals - mapResult := make(map[string]interface{}, len(mapLiteralValue)) - for key, val := range mapLiteralValue { - if val, err := ExtractFromLiteral(val); err == nil { - mapResult[key] = val - } else { - return nil, err - } - } - return mapResult, nil - } - return nil, fmt.Errorf("unsupported literal type %T", literal) -} diff --git a/clients/go/coreutils/extract_literal_test.go b/clients/go/coreutils/extract_literal_test.go deleted file mode 100644 index 32d392322..000000000 --- a/clients/go/coreutils/extract_literal_test.go +++ /dev/null @@ -1,202 +0,0 @@ -// extract_literal_test.go -// Test class for the utility methods which extract a native golang value from a flyte Literal. - -package coreutils - -import ( - "testing" - "time" - - "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" - - structpb "github.com/golang/protobuf/ptypes/struct" - "github.com/stretchr/testify/assert" -) - -func TestFetchLiteral(t *testing.T) { - t.Run("Primitive", func(t *testing.T) { - lit, err := MakeLiteral("test_string") - assert.NoError(t, err) - val, err := ExtractFromLiteral(lit) - assert.NoError(t, err) - assert.Equal(t, "test_string", val) - }) - - t.Run("Timestamp", func(t *testing.T) { - now := time.Now().UTC() - lit, err := MakeLiteral(now) - assert.NoError(t, err) - val, err := ExtractFromLiteral(lit) - assert.NoError(t, err) - assert.Equal(t, now, val) - }) - - t.Run("Duration", func(t *testing.T) { - duration := time.Second * 10 - lit, err := MakeLiteral(duration) - assert.NoError(t, err) - val, err := ExtractFromLiteral(lit) - assert.NoError(t, err) - assert.Equal(t, duration, val) - }) - - t.Run("Array", func(t *testing.T) { - lit, err := MakeLiteral([]interface{}{1, 2, 3}) - assert.NoError(t, err) - val, err := ExtractFromLiteral(lit) - assert.NoError(t, err) - arr := []interface{}{int64(1), int64(2), int64(3)} - assert.Equal(t, arr, val) - }) - - t.Run("Map", func(t *testing.T) { - mapInstance := map[string]interface{}{ - "key1": []interface{}{1, 2, 3}, - "key2": []interface{}{5}, - } - lit, err := MakeLiteral(mapInstance) - assert.NoError(t, err) - val, err := ExtractFromLiteral(lit) - assert.NoError(t, err) - expectedMapInstance := map[string]interface{}{ - "key1": []interface{}{int64(1), int64(2), int64(3)}, - "key2": []interface{}{int64(5)}, - } - assert.Equal(t, expectedMapInstance, val) - }) - - t.Run("Map_Booleans", func(t *testing.T) { - mapInstance := map[string]interface{}{ - "key1": []interface{}{true, false, true}, - "key2": []interface{}{false}, - } - lit, err := MakeLiteral(mapInstance) - assert.NoError(t, err) - val, err := ExtractFromLiteral(lit) - assert.NoError(t, err) - assert.Equal(t, mapInstance, val) - }) - - t.Run("Map_Floats", func(t *testing.T) { - mapInstance := map[string]interface{}{ - "key1": []interface{}{1.0, 2.0, 3.0}, - "key2": []interface{}{1.0}, - } - lit, err := MakeLiteral(mapInstance) - assert.NoError(t, err) - val, err := ExtractFromLiteral(lit) - assert.NoError(t, err) - expectedMapInstance := map[string]interface{}{ - "key1": []interface{}{float64(1.0), float64(2.0), float64(3.0)}, - "key2": []interface{}{float64(1.0)}, - } - assert.Equal(t, expectedMapInstance, val) - }) - - t.Run("NestedMap", func(t *testing.T) { - mapInstance := map[string]interface{}{ - "key1": map[string]interface{}{"key11": 1.0, "key12": 2.0, "key13": 3.0}, - "key2": map[string]interface{}{"key21": 1.0}, - } - lit, err := MakeLiteral(mapInstance) - assert.NoError(t, err) - val, err := ExtractFromLiteral(lit) - assert.NoError(t, err) - expectedMapInstance := map[string]interface{}{ - "key1": map[string]interface{}{"key11": float64(1.0), "key12": float64(2.0), "key13": float64(3.0)}, - "key2": map[string]interface{}{"key21": float64(1.0)}, - } - assert.Equal(t, expectedMapInstance, val) - }) - - t.Run("Binary", func(t *testing.T) { - s := MakeBinaryLiteral([]byte{'h'}) - assert.Equal(t, []byte{'h'}, s.GetScalar().GetBinary().GetValue()) - _, err := ExtractFromLiteral(s) - assert.NotNil(t, err) - }) - - t.Run("NoneType", func(t *testing.T) { - p, err := MakeLiteral(nil) - assert.NoError(t, err) - assert.NotNil(t, p.GetScalar()) - _, err = ExtractFromLiteral(p) - assert.NotNil(t, err) - }) - - t.Run("Generic", func(t *testing.T) { - literalVal := map[string]interface{}{ - "x": 1, - "y": "ystringvalue", - } - var literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRUCT}} - lit, err := MakeLiteralForType(literalType, literalVal) - assert.NoError(t, err) - extractedLiteralVal, err := ExtractFromLiteral(lit) - assert.NoError(t, err) - fieldsMap := map[string]*structpb.Value{ - "x": { - Kind: &structpb.Value_NumberValue{NumberValue: 1}, - }, - "y": { - Kind: &structpb.Value_StringValue{StringValue: "ystringvalue"}, - }, - } - expectedStructVal := &structpb.Struct{ - Fields: fieldsMap, - } - extractedStructValue := extractedLiteralVal.(*structpb.Struct) - assert.Equal(t, len(expectedStructVal.Fields), len(extractedStructValue.Fields)) - for key, val := range expectedStructVal.Fields { - assert.Equal(t, val.Kind, extractedStructValue.Fields[key].Kind) - } - }) - - t.Run("Generic Passed As String", func(t *testing.T) { - literalVal := "{\"x\": 1,\"y\": \"ystringvalue\"}" - var literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRUCT}} - lit, err := MakeLiteralForType(literalType, literalVal) - assert.NoError(t, err) - extractedLiteralVal, err := ExtractFromLiteral(lit) - assert.NoError(t, err) - fieldsMap := map[string]*structpb.Value{ - "x": { - Kind: &structpb.Value_NumberValue{NumberValue: 1}, - }, - "y": { - Kind: &structpb.Value_StringValue{StringValue: "ystringvalue"}, - }, - } - expectedStructVal := &structpb.Struct{ - Fields: fieldsMap, - } - extractedStructValue := extractedLiteralVal.(*structpb.Struct) - assert.Equal(t, len(expectedStructVal.Fields), len(extractedStructValue.Fields)) - for key, val := range expectedStructVal.Fields { - assert.Equal(t, val.Kind, extractedStructValue.Fields[key].Kind) - } - }) - - t.Run("Structured dataset", func(t *testing.T) { - literalVal := "s3://blah/blah/blah" - var dataSetColumns []*core.StructuredDatasetType_DatasetColumn - dataSetColumns = append(dataSetColumns, &core.StructuredDatasetType_DatasetColumn{ - Name: "Price", - LiteralType: &core.LiteralType{ - Type: &core.LiteralType_Simple{ - Simple: core.SimpleType_FLOAT, - }, - }, - }) - var literalType = &core.LiteralType{Type: &core.LiteralType_StructuredDatasetType{StructuredDatasetType: &core.StructuredDatasetType{ - Columns: dataSetColumns, - Format: "testFormat", - }}} - - lit, err := MakeLiteralForType(literalType, literalVal) - assert.NoError(t, err) - extractedLiteralVal, err := ExtractFromLiteral(lit) - assert.NoError(t, err) - assert.Equal(t, literalVal, extractedLiteralVal) - }) -} diff --git a/clients/go/coreutils/literals.go b/clients/go/coreutils/literal.go similarity index 81% rename from clients/go/coreutils/literals.go rename to clients/go/coreutils/literal.go index 4ad84f181..7b27df563 100644 --- a/clients/go/coreutils/literals.go +++ b/clients/go/coreutils/literal.go @@ -1,4 +1,31 @@ // Contains convenience methods for constructing core types. +// Utility methods to extract a native golang value from a given Literal. +// Usage: +// 1] string literal extraction +// +// lit, _ := MakeLiteral("test_string") +// val, _ := ExtractFromLiteral(lit) +// +// 2] integer literal extraction. integer would be extracted in type int64. +// +// lit, _ := MakeLiteral([]interface{}{1, 2, 3}) +// val, _ := ExtractFromLiteral(lit) +// +// 3] float literal extraction. float would be extracted in type float64. +// +// lit, _ := MakeLiteral([]interface{}{1.0, 2.0, 3.0}) +// val, _ := ExtractFromLiteral(lit) +// +// 4] map of boolean literal extraction. +// +// mapInstance := map[string]interface{}{ +// "key1": []interface{}{1, 2, 3}, +// "key2": []interface{}{5}, +// } +// lit, _ := MakeLiteral(mapInstance) +// val, _ := ExtractFromLiteral(lit) +// +// For further examples check the test TestFetchLiteral in literal_test.go package coreutils import ( @@ -19,6 +46,70 @@ import ( "github.com/pkg/errors" ) +func ExtractFromLiteral(literal *core.Literal) (interface{}, error) { + switch literalValue := literal.Value.(type) { + case *core.Literal_Scalar: + switch scalarValue := literalValue.Scalar.Value.(type) { + case *core.Scalar_Primitive: + switch scalarPrimitive := scalarValue.Primitive.Value.(type) { + case *core.Primitive_Integer: + scalarPrimitiveInt := scalarPrimitive.Integer + return scalarPrimitiveInt, nil + case *core.Primitive_FloatValue: + scalarPrimitiveFloat := scalarPrimitive.FloatValue + return scalarPrimitiveFloat, nil + case *core.Primitive_StringValue: + scalarPrimitiveString := scalarPrimitive.StringValue + return scalarPrimitiveString, nil + case *core.Primitive_Boolean: + scalarPrimitiveBoolean := scalarPrimitive.Boolean + return scalarPrimitiveBoolean, nil + case *core.Primitive_Datetime: + scalarPrimitiveDateTime := scalarPrimitive.Datetime.AsTime() + return scalarPrimitiveDateTime, nil + case *core.Primitive_Duration: + scalarPrimitiveDuration := scalarPrimitive.Duration.AsDuration() + return scalarPrimitiveDuration, nil + default: + return nil, fmt.Errorf("unsupported literal scalar primitive type %T", scalarValue) + } + case *core.Scalar_Blob: + return scalarValue.Blob.Uri, nil + case *core.Scalar_Schema: + return scalarValue.Schema.Uri, nil + case *core.Scalar_Generic: + return scalarValue.Generic, nil + case *core.Scalar_StructuredDataset: + return scalarValue.StructuredDataset.Uri, nil + default: + return nil, fmt.Errorf("unsupported literal scalar type %T", scalarValue) + } + case *core.Literal_Collection: + collectionValue := literalValue.Collection.Literals + collection := make([]interface{}, len(collectionValue)) + for index, val := range collectionValue { + if collectionElem, err := ExtractFromLiteral(val); err == nil { + collection[index] = collectionElem + } else { + return nil, err + } + } + return collection, nil + case *core.Literal_Map: + mapLiteralValue := literalValue.Map.Literals + mapResult := make(map[string]interface{}, len(mapLiteralValue)) + for key, val := range mapLiteralValue { + if val, err := ExtractFromLiteral(val); err == nil { + mapResult[key] = val + } else { + return nil, err + } + } + return mapResult, nil + } + return nil, fmt.Errorf("unsupported literal type %T", literal) +} + func MakePrimitive(v interface{}) (*core.Primitive, error) { switch p := v.(type) { case int: diff --git a/clients/go/coreutils/literals_test.go b/clients/go/coreutils/literal_test.go similarity index 82% rename from clients/go/coreutils/literals_test.go rename to clients/go/coreutils/literal_test.go index 4b16478ad..16d6ac83e 100644 --- a/clients/go/coreutils/literals_test.go +++ b/clients/go/coreutils/literal_test.go @@ -21,6 +21,194 @@ import ( "github.com/stretchr/testify/assert" ) +func TestFetchLiteral(t *testing.T) { + t.Run("Primitive", func(t *testing.T) { + lit, err := MakeLiteral("test_string") + assert.NoError(t, err) + val, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + assert.Equal(t, "test_string", val) + }) + + t.Run("Timestamp", func(t *testing.T) { + now := time.Now().UTC() + lit, err := MakeLiteral(now) + assert.NoError(t, err) + val, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + assert.Equal(t, now, val) + }) + + t.Run("Duration", func(t *testing.T) { + duration := time.Second * 10 + lit, err := MakeLiteral(duration) + assert.NoError(t, err) + val, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + assert.Equal(t, duration, val) + }) + + t.Run("Array", func(t *testing.T) { + lit, err := MakeLiteral([]interface{}{1, 2, 3}) + assert.NoError(t, err) + val, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + arr := []interface{}{int64(1), int64(2), int64(3)} + assert.Equal(t, arr, val) + }) + + t.Run("Map", func(t *testing.T) { + mapInstance := map[string]interface{}{ + "key1": []interface{}{1, 2, 3}, + "key2": []interface{}{5}, + } + lit, err := MakeLiteral(mapInstance) + assert.NoError(t, err) + val, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + expectedMapInstance := map[string]interface{}{ + "key1": []interface{}{int64(1), int64(2), int64(3)}, + "key2": []interface{}{int64(5)}, + } + assert.Equal(t, expectedMapInstance, val) + }) + + t.Run("Map_Booleans", func(t *testing.T) { + mapInstance := map[string]interface{}{ + "key1": []interface{}{true, false, true}, + "key2": []interface{}{false}, + } + lit, err := MakeLiteral(mapInstance) + assert.NoError(t, err) + val, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + assert.Equal(t, mapInstance, val) + }) + + t.Run("Map_Floats", func(t *testing.T) { + mapInstance := map[string]interface{}{ + "key1": []interface{}{1.0, 2.0, 3.0}, + "key2": []interface{}{1.0}, + } + lit, err := MakeLiteral(mapInstance) + assert.NoError(t, err) + val, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + expectedMapInstance := map[string]interface{}{ + "key1": []interface{}{float64(1.0), float64(2.0), float64(3.0)}, + "key2": []interface{}{float64(1.0)}, + } + assert.Equal(t, expectedMapInstance, val) + }) + + t.Run("NestedMap", func(t *testing.T) { + mapInstance := map[string]interface{}{ + "key1": map[string]interface{}{"key11": 1.0, "key12": 2.0, "key13": 3.0}, + "key2": map[string]interface{}{"key21": 1.0}, + } + lit, err := MakeLiteral(mapInstance) + assert.NoError(t, err) + val, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + expectedMapInstance := map[string]interface{}{ + "key1": map[string]interface{}{"key11": float64(1.0), "key12": float64(2.0), "key13": float64(3.0)}, + "key2": map[string]interface{}{"key21": float64(1.0)}, + } + assert.Equal(t, expectedMapInstance, val) + }) + + t.Run("Binary", func(t *testing.T) { + s := MakeBinaryLiteral([]byte{'h'}) + assert.Equal(t, []byte{'h'}, s.GetScalar().GetBinary().GetValue()) + _, err := ExtractFromLiteral(s) + assert.NotNil(t, err) + }) + + t.Run("NoneType", func(t *testing.T) { + p, err := MakeLiteral(nil) + assert.NoError(t, err) + assert.NotNil(t, p.GetScalar()) + _, err = ExtractFromLiteral(p) + assert.NotNil(t, err) + }) + + t.Run("Generic", func(t *testing.T) { + literalVal := map[string]interface{}{ + "x": 1, + "y": "ystringvalue", + } + var literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRUCT}} + lit, err := MakeLiteralForType(literalType, literalVal) + assert.NoError(t, err) + extractedLiteralVal, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + fieldsMap := map[string]*structpb.Value{ + "x": { + Kind: &structpb.Value_NumberValue{NumberValue: 1}, + }, + "y": { + Kind: &structpb.Value_StringValue{StringValue: "ystringvalue"}, + }, + } + expectedStructVal := &structpb.Struct{ + Fields: fieldsMap, + } + extractedStructValue := extractedLiteralVal.(*structpb.Struct) + assert.Equal(t, len(expectedStructVal.Fields), len(extractedStructValue.Fields)) + for key, val := range expectedStructVal.Fields { + assert.Equal(t, val.Kind, extractedStructValue.Fields[key].Kind) + } + }) + + t.Run("Generic Passed As String", func(t *testing.T) { + literalVal := "{\"x\": 1,\"y\": \"ystringvalue\"}" + var literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRUCT}} + lit, err := MakeLiteralForType(literalType, literalVal) + assert.NoError(t, err) + extractedLiteralVal, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + fieldsMap := map[string]*structpb.Value{ + "x": { + Kind: &structpb.Value_NumberValue{NumberValue: 1}, + }, + "y": { + Kind: &structpb.Value_StringValue{StringValue: "ystringvalue"}, + }, + } + expectedStructVal := &structpb.Struct{ + Fields: fieldsMap, + } + extractedStructValue := extractedLiteralVal.(*structpb.Struct) + assert.Equal(t, len(expectedStructVal.Fields), len(extractedStructValue.Fields)) + for key, val := range expectedStructVal.Fields { + assert.Equal(t, val.Kind, extractedStructValue.Fields[key].Kind) + } + }) + + t.Run("Structured dataset", func(t *testing.T) { + literalVal := "s3://blah/blah/blah" + var dataSetColumns []*core.StructuredDatasetType_DatasetColumn + dataSetColumns = append(dataSetColumns, &core.StructuredDatasetType_DatasetColumn{ + Name: "Price", + LiteralType: &core.LiteralType{ + Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_FLOAT, + }, + }, + }) + var literalType = &core.LiteralType{Type: &core.LiteralType_StructuredDatasetType{StructuredDatasetType: &core.StructuredDatasetType{ + Columns: dataSetColumns, + Format: "testFormat", + }}} + + lit, err := MakeLiteralForType(literalType, literalVal) + assert.NoError(t, err) + extractedLiteralVal, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + assert.Equal(t, literalVal, extractedLiteralVal) + }) +} + func TestMakePrimitive(t *testing.T) { { v := 1 diff --git a/clients/go/coreutils/typing.go b/clients/go/coreutils/typing.go new file mode 100644 index 000000000..863000c59 --- /dev/null +++ b/clients/go/coreutils/typing.go @@ -0,0 +1,140 @@ +package coreutils + +import ( + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" + "k8s.io/apimachinery/pkg/util/sets" +) + +func literalTypeForPrimitive(primitive *core.Primitive) *core.LiteralType { + simpleType := core.SimpleType_NONE + switch primitive.GetValue().(type) { + case *core.Primitive_Integer: + simpleType = core.SimpleType_INTEGER + case *core.Primitive_FloatValue: + simpleType = core.SimpleType_FLOAT + case *core.Primitive_StringValue: + simpleType = core.SimpleType_STRING + case *core.Primitive_Boolean: + simpleType = core.SimpleType_BOOLEAN + case *core.Primitive_Datetime: + simpleType = core.SimpleType_DATETIME + case *core.Primitive_Duration: + simpleType = core.SimpleType_DURATION + } + + return &core.LiteralType{Type: &core.LiteralType_Simple{Simple: simpleType}} +} + +// Gets literal type for scalar value. This can be used to compare the underlying type of two scalars for compatibility. +func literalTypeForScalar(scalar *core.Scalar) *core.LiteralType { + // TODO: Should we just pass the type information with the value? That way we don't have to guess? + var literalType *core.LiteralType + switch v := scalar.GetValue().(type) { + case *core.Scalar_Primitive: + literalType = literalTypeForPrimitive(scalar.GetPrimitive()) + case *core.Scalar_Blob: + if scalar.GetBlob().GetMetadata() == nil { + return nil + } + + literalType = &core.LiteralType{Type: &core.LiteralType_Blob{Blob: scalar.GetBlob().GetMetadata().GetType()}} + case *core.Scalar_Binary: + literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_BINARY}} + case *core.Scalar_Schema: + literalType = &core.LiteralType{ + Type: &core.LiteralType_Schema{ + Schema: scalar.GetSchema().Type, + }, + } + case *core.Scalar_StructuredDataset: + if v.StructuredDataset == nil || v.StructuredDataset.Metadata == nil { + return &core.LiteralType{ + Type: &core.LiteralType_StructuredDatasetType{}, + } + } + + literalType = &core.LiteralType{ + Type: &core.LiteralType_StructuredDatasetType{ + StructuredDatasetType: scalar.GetStructuredDataset().GetMetadata().StructuredDatasetType, + }, + } + case *core.Scalar_NoneType: + literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_NONE}} + case *core.Scalar_Error: + literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_ERROR}} + case *core.Scalar_Generic: + literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRUCT}} + case *core.Scalar_Union: + literalType = &core.LiteralType{ + Type: &core.LiteralType_UnionType{ + UnionType: &core.UnionType{ + Variants: []*core.LiteralType{ + scalar.GetUnion().GetType(), + }, + }, + }, + } + default: + return nil + } + + return literalType +} + +// LiteralTypeForLiteral gets LiteralType for literal, nil if the value of literal is unknown, or type collection/map of +// type None if the literal is a non-homogeneous type. +func LiteralTypeForLiteral(l *core.Literal) *core.LiteralType { + switch l.GetValue().(type) { + case *core.Literal_Scalar: + return literalTypeForScalar(l.GetScalar()) + case *core.Literal_Collection: + return &core.LiteralType{ + Type: &core.LiteralType_CollectionType{ + CollectionType: literalTypeForLiterals(l.GetCollection().Literals), + }, + } + case *core.Literal_Map: + return &core.LiteralType{ + Type: &core.LiteralType_MapValueType{ + MapValueType: literalTypeForLiterals(maps.Values(l.GetMap().Literals)), + }, + } + } + + return nil +} + +func literalTypeForLiterals(literals []*core.Literal) *core.LiteralType { + innerType := make([]*core.LiteralType, 0, 1) + innerTypeSet := sets.NewString() + for _, x := range literals { + otherType := LiteralTypeForLiteral(x) + otherTypeKey := otherType.String() + + if !innerTypeSet.Has(otherTypeKey) { + innerType = append(innerType, otherType) + innerTypeSet.Insert(otherTypeKey) + } + } + + if len(innerType) == 0 { + return &core.LiteralType{ + Type: &core.LiteralType_Simple{Simple: core.SimpleType_NONE}, + } + } else if len(innerType) == 1 { + return innerType[0] + } + + // sort inner types to ensure consistent union types are generated + slices.SortFunc(innerType, func(a, b *core.LiteralType) bool { return a.String() < b.String() }) + + return &core.LiteralType{ + Type: &core.LiteralType_UnionType{ + UnionType: &core.UnionType{ + Variants: innerType, + }, + }, + } +} diff --git a/clients/go/coreutils/typing_test.go b/clients/go/coreutils/typing_test.go new file mode 100644 index 000000000..7d7ef36ed --- /dev/null +++ b/clients/go/coreutils/typing_test.go @@ -0,0 +1,52 @@ +package coreutils + +import ( + "testing" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + "github.com/stretchr/testify/assert" +) + +func TestLiteralTypeForLiterals(t *testing.T) { + t.Run("empty", func(t *testing.T) { + lt := literalTypeForLiterals(nil) + assert.Equal(t, core.SimpleType_NONE.String(), lt.GetSimple().String()) + }) + + t.Run("homogenous", func(t *testing.T) { + lt := literalTypeForLiterals([]*core.Literal{ + MustMakeLiteral(5), + MustMakeLiteral(0), + MustMakeLiteral(5), + }) + + assert.Equal(t, core.SimpleType_INTEGER.String(), lt.GetSimple().String()) + }) + + t.Run("non-homogenous", func(t *testing.T) { + lt := literalTypeForLiterals([]*core.Literal{ + MustMakeLiteral("hello"), + MustMakeLiteral(5), + MustMakeLiteral("world"), + MustMakeLiteral(0), + MustMakeLiteral(2), + }) + + assert.Len(t, lt.GetUnionType().Variants, 2) + assert.Equal(t, core.SimpleType_INTEGER.String(), lt.GetUnionType().Variants[0].GetSimple().String()) + assert.Equal(t, core.SimpleType_STRING.String(), lt.GetUnionType().Variants[1].GetSimple().String()) + }) + + t.Run("non-homogenous ensure ordering", func(t *testing.T) { + lt := literalTypeForLiterals([]*core.Literal{ + MustMakeLiteral(5), + MustMakeLiteral("world"), + MustMakeLiteral(0), + MustMakeLiteral(2), + }) + + assert.Len(t, lt.GetUnionType().Variants, 2) + assert.Equal(t, core.SimpleType_INTEGER.String(), lt.GetUnionType().Variants[0].GetSimple().String()) + assert.Equal(t, core.SimpleType_STRING.String(), lt.GetUnionType().Variants[1].GetSimple().String()) + }) +} diff --git a/go.mod b/go.mod index 6858232a9..85833d5f0 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,8 @@ require ( github.com/pkg/errors v0.9.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 - golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f + golang.org/x/exp v0.0.0-20230321023759-10a507213a29 + golang.org/x/net v0.1.0 golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013 google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506 google.golang.org/grpc v1.35.0 @@ -66,14 +67,13 @@ require ( github.com/spf13/cobra v1.1.1 // indirect github.com/stretchr/objx v0.3.0 // indirect go.opencensus.io v0.22.6 // indirect - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect + golang.org/x/crypto v0.1.0 // indirect golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect - golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect - golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/mod v0.6.0 // indirect + golang.org/x/sys v0.1.0 // indirect + golang.org/x/text v0.4.0 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect - golang.org/x/tools v0.1.10 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + golang.org/x/tools v0.2.0 // indirect google.golang.org/api v0.38.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.25.0 // indirect diff --git a/go.sum b/go.sum index 484761370..4c6185d9c 100644 --- a/go.sum +++ b/go.sum @@ -246,8 +246,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= @@ -562,8 +562,8 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -574,6 +574,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -597,8 +599,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -643,8 +645,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -716,8 +718,8 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -726,8 +728,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -795,12 +798,11 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= From 4649cf9c7cc7cc3379189f9a672fede9bc9940ca Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Fri, 31 Mar 2023 13:06:44 -0500 Subject: [PATCH 2/2] exported literal type for functions Signed-off-by: Daniel Rammer --- clients/go/coreutils/typing.go | 14 +++++++------- clients/go/coreutils/typing_test.go | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/clients/go/coreutils/typing.go b/clients/go/coreutils/typing.go index 863000c59..f08e0c1c6 100644 --- a/clients/go/coreutils/typing.go +++ b/clients/go/coreutils/typing.go @@ -7,7 +7,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" ) -func literalTypeForPrimitive(primitive *core.Primitive) *core.LiteralType { +func LiteralTypeForPrimitive(primitive *core.Primitive) *core.LiteralType { simpleType := core.SimpleType_NONE switch primitive.GetValue().(type) { case *core.Primitive_Integer: @@ -28,12 +28,12 @@ func literalTypeForPrimitive(primitive *core.Primitive) *core.LiteralType { } // Gets literal type for scalar value. This can be used to compare the underlying type of two scalars for compatibility. -func literalTypeForScalar(scalar *core.Scalar) *core.LiteralType { +func LiteralTypeForScalar(scalar *core.Scalar) *core.LiteralType { // TODO: Should we just pass the type information with the value? That way we don't have to guess? var literalType *core.LiteralType switch v := scalar.GetValue().(type) { case *core.Scalar_Primitive: - literalType = literalTypeForPrimitive(scalar.GetPrimitive()) + literalType = LiteralTypeForPrimitive(scalar.GetPrimitive()) case *core.Scalar_Blob: if scalar.GetBlob().GetMetadata() == nil { return nil @@ -88,17 +88,17 @@ func literalTypeForScalar(scalar *core.Scalar) *core.LiteralType { func LiteralTypeForLiteral(l *core.Literal) *core.LiteralType { switch l.GetValue().(type) { case *core.Literal_Scalar: - return literalTypeForScalar(l.GetScalar()) + return LiteralTypeForScalar(l.GetScalar()) case *core.Literal_Collection: return &core.LiteralType{ Type: &core.LiteralType_CollectionType{ - CollectionType: literalTypeForLiterals(l.GetCollection().Literals), + CollectionType: LiteralTypeForLiterals(l.GetCollection().Literals), }, } case *core.Literal_Map: return &core.LiteralType{ Type: &core.LiteralType_MapValueType{ - MapValueType: literalTypeForLiterals(maps.Values(l.GetMap().Literals)), + MapValueType: LiteralTypeForLiterals(maps.Values(l.GetMap().Literals)), }, } } @@ -106,7 +106,7 @@ func LiteralTypeForLiteral(l *core.Literal) *core.LiteralType { return nil } -func literalTypeForLiterals(literals []*core.Literal) *core.LiteralType { +func LiteralTypeForLiterals(literals []*core.Literal) *core.LiteralType { innerType := make([]*core.LiteralType, 0, 1) innerTypeSet := sets.NewString() for _, x := range literals { diff --git a/clients/go/coreutils/typing_test.go b/clients/go/coreutils/typing_test.go index 7d7ef36ed..8c676f6ed 100644 --- a/clients/go/coreutils/typing_test.go +++ b/clients/go/coreutils/typing_test.go @@ -9,12 +9,12 @@ import ( func TestLiteralTypeForLiterals(t *testing.T) { t.Run("empty", func(t *testing.T) { - lt := literalTypeForLiterals(nil) + lt := LiteralTypeForLiterals(nil) assert.Equal(t, core.SimpleType_NONE.String(), lt.GetSimple().String()) }) t.Run("homogenous", func(t *testing.T) { - lt := literalTypeForLiterals([]*core.Literal{ + lt := LiteralTypeForLiterals([]*core.Literal{ MustMakeLiteral(5), MustMakeLiteral(0), MustMakeLiteral(5), @@ -24,7 +24,7 @@ func TestLiteralTypeForLiterals(t *testing.T) { }) t.Run("non-homogenous", func(t *testing.T) { - lt := literalTypeForLiterals([]*core.Literal{ + lt := LiteralTypeForLiterals([]*core.Literal{ MustMakeLiteral("hello"), MustMakeLiteral(5), MustMakeLiteral("world"), @@ -38,7 +38,7 @@ func TestLiteralTypeForLiterals(t *testing.T) { }) t.Run("non-homogenous ensure ordering", func(t *testing.T) { - lt := literalTypeForLiterals([]*core.Literal{ + lt := LiteralTypeForLiterals([]*core.Literal{ MustMakeLiteral(5), MustMakeLiteral("world"), MustMakeLiteral(0),