Skip to content

Commit

Permalink
[BitSail][connector]fixed type property shared in same type definitio…
Browse files Browse the repository at this point in the history
…n. (#99)

1. fixed bug for the type info property.
2. remove connector-base from connector directory.

Co-authored-by: haoke <[email protected]>
  • Loading branch information
hk-lrzy and haoke authored Nov 7, 2022
1 parent 88986cd commit 1d66c9a
Show file tree
Hide file tree
Showing 5 changed files with 112 additions and 48 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,10 @@

import com.bytedance.bitsail.common.BitSailException;
import com.bytedance.bitsail.common.exception.CommonErrorCode;
import com.bytedance.bitsail.common.typeinfo.BasicArrayTypeInfo;
import com.bytedance.bitsail.common.typeinfo.ListTypeInfo;
import com.bytedance.bitsail.common.typeinfo.MapTypeInfo;
import com.bytedance.bitsail.common.typeinfo.TypeInfo;
import com.bytedance.bitsail.common.typeinfo.TypeInfos;
import com.bytedance.bitsail.common.typeinfo.TypeInfoBridge;
import com.bytedance.bitsail.common.typeinfo.TypeProperty;
import com.bytedance.bitsail.common.typeinfo.Types;
import com.bytedance.bitsail.common.util.Preconditions;
Expand Down Expand Up @@ -89,52 +88,12 @@ public static TypeInfo<?> fromTypeString(String typeString) {
return new ListTypeInfo<>(fromTypeString(elementTypeString));
}
}
if (equalsTypeString(typeString, Types.VOID)) {
return TypeInfos.VOID_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.SHORT)) {
return TypeInfos.SHORT_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.INT)) {
return TypeInfos.INT_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.LONG) || equalsTypeString(typeString, Types.BIGINT)) {
return TypeInfos.LONG_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.FLOAT)) {
return TypeInfos.FLOAT_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.DOUBLE)) {
return TypeInfos.DOUBLE_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.BIGINTEGER)) {
return TypeInfos.BIG_INTEGER_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.BIGDECIMAL)) {
return TypeInfos.BIG_DECIMAL_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.BOOLEAN)) {
return TypeInfos.BOOLEAN_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.STRING)) {
return TypeInfos.STRING_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.BYTE)) {
return TypeInfos.BYTE_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.BYTES)) {
return BasicArrayTypeInfo.BINARY_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.BINARY)) {
return BasicArrayTypeInfo.BINARY_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.DATE)) {
return TypeInfos.LOCAL_DATE_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.TIME)) {
return TypeInfos.LOCAL_TIME_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.TIMESTAMP)) {
return TypeInfos.LOCAL_DATE_TIME_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.DATE_DATE)) {
return TypeInfos.SQL_DATE_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.DATE_TIME)) {
return TypeInfos.SQL_TIME_TYPE_INFO;
} else if (equalsTypeString(typeString, Types.DATE_DATE_TIME)) {
return TypeInfos.SQL_TIMESTAMP_TYPE_INFO;
TypeInfo<?> typeInfo = TypeInfoBridge.bridgeTypeInfo(typeString);
if (Objects.isNull(typeInfo)) {
throw BitSailException.asBitSailException(CommonErrorCode.INTERNAL_ERROR,
String.format("Not support type string %s.", typeString));
}
throw BitSailException.asBitSailException(CommonErrorCode.INTERNAL_ERROR,
String.format("Not support type string %s.", typeString));
}

private static boolean equalsTypeString(String typeString, Types types) {
return StringUtils.equalsIgnoreCase(typeString, types.name())
|| StringUtils.equalsIgnoreCase(typeString, types.getTypeStringNickName());
return typeInfo;
}

private static String[] parseMapTypeString(String typeString) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package com.bytedance.bitsail.common.typeinfo;

import com.google.common.collect.Maps;
import org.apache.commons.lang3.StringUtils;

import java.util.Map;

public class TypeInfoBridge {

public static final Map<Types, TypeInfo<?>> TYPE_INFO_MAPPING =
Maps.newHashMap();

public static final Map<String, TypeInfo<?>> TYPE_INFO_NAME_MAPPING =
Maps.newHashMap();

static {
TYPE_INFO_MAPPING.put(Types.VOID, TypeInfos.VOID_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.SHORT, TypeInfos.SHORT_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.INT, TypeInfos.INT_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.LONG, TypeInfos.LONG_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.BIGINT, TypeInfos.LONG_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.BIGINTEGER, TypeInfos.BIG_INTEGER_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.FLOAT, TypeInfos.FLOAT_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.DOUBLE, TypeInfos.DOUBLE_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.BIGDECIMAL, TypeInfos.BIG_DECIMAL_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.STRING, TypeInfos.STRING_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.BOOLEAN, TypeInfos.BOOLEAN_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.DATE_DATE, TypeInfos.SQL_DATE_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.DATE_TIME, TypeInfos.SQL_TIME_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.DATE_DATE_TIME, TypeInfos.SQL_TIMESTAMP_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.DATE, TypeInfos.LOCAL_DATE_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.TIME, TypeInfos.LOCAL_TIME_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.TIMESTAMP, TypeInfos.LOCAL_DATE_TIME_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.BYTE, TypeInfos.BYTE_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.BYTES, BasicArrayTypeInfo.BINARY_TYPE_INFO);
TYPE_INFO_MAPPING.put(Types.BINARY, BasicArrayTypeInfo.BINARY_TYPE_INFO);

for (Types type : TYPE_INFO_MAPPING.keySet()) {
TYPE_INFO_NAME_MAPPING.put(StringUtils.upperCase(type.name()), TYPE_INFO_MAPPING.get(type));
if (StringUtils.isNotEmpty(type.getTypeStringNickName())) {
TYPE_INFO_NAME_MAPPING.put(StringUtils.upperCase(type.getTypeStringNickName()),
TYPE_INFO_MAPPING.get(type));
}
}
}

public static TypeInfo<?> bridgeTypeInfo(String typeString) {
return TYPE_INFO_NAME_MAPPING.get(typeString);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.bytedance.bitsail.common.type.BitSailTypeParser;
import com.bytedance.bitsail.common.type.TypeInfoConverter;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;

import java.util.List;
Expand All @@ -45,7 +46,12 @@ public static TypeInfo<?>[] getTypeInfos(TypeInfoConverter converter,
}
List<TypeProperty> typeProperties = BitSailTypeParser
.fromTypePropertyString(columnInfos.get(index).getProperties());
typeInfo.setTypeProperties(typeProperties);

//1. type property should not share in same type definition, so we need create a new one.
//2. type property only support in basic type info.
if (typeInfo instanceof BasicTypeInfo && CollectionUtils.isNotEmpty(typeProperties)) {
typeInfo = new BasicTypeInfo<>(typeInfo.getTypeClass(), typeProperties);
}
fieldTypes[index] = typeInfo;
}
return fieldTypes;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,13 @@

package com.bytedance.bitsail.common.type;

import com.bytedance.bitsail.common.model.ColumnInfo;
import com.bytedance.bitsail.common.typeinfo.TypeInfo;
import com.bytedance.bitsail.common.typeinfo.TypeInfoUtils;
import com.bytedance.bitsail.common.typeinfo.TypeProperty;

import com.google.common.collect.Lists;
import org.apache.commons.collections.CollectionUtils;
import org.junit.Assert;
import org.junit.Test;

Expand All @@ -43,4 +48,27 @@ public void fromTypePropertyString() {
Assert.assertEquals(TypeProperty.NOT_NULL, uniqWithNotNullTypePropertyProperties.get(1));
}

@Test
public void fromTypePropertyForSameTypeString() {
BitSailTypeInfoConverter bitSailTypeInfoConverter = new BitSailTypeInfoConverter();
List<ColumnInfo> columnInfos = Lists.newArrayList();
ColumnInfo columnInfo1 = ColumnInfo
.builder()
.type("bigint")
.name("bigint_1")
.build();
columnInfo1.setProperties("unique");
ColumnInfo columnInfo2 = ColumnInfo
.builder()
.type("bigint")
.name("bigint_2")
.build();
columnInfos.add(columnInfo1);
columnInfos.add(columnInfo2);

TypeInfo<?>[] typeInfos = TypeInfoUtils.getTypeInfos(bitSailTypeInfoConverter, columnInfos);
Assert.assertEquals(CollectionUtils.size(typeInfos[0].getTypeProperties()), 1);
Assert.assertEquals(typeInfos[0].getTypeProperties().get(0), TypeProperty.UNIQUE);
Assert.assertEquals(CollectionUtils.size(typeInfos[1].getTypeProperties()), 0);
}
}
1 change: 1 addition & 0 deletions bitsail-dist/src/main/assemblies/assembly-bin.xml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ under the License.
<excludes>
<exclude>com.bytedance.bitsail:bitsail-connector-messagequeue</exclude>
<exclude>com.bytedance.bitsail:bitsail-connector-streamingfile-common</exclude>
<exclude>com.bytedance.bitsail:connector-base</exclude>
</excludes>
<outputFileNameMapping>${artifact.file.name}</outputFileNameMapping>
</dependencySet>
Expand Down

0 comments on commit 1d66c9a

Please sign in to comment.