[HUDI-4464] Clear warnings in Azure CI (apache#6210)

Co-authored-by: jian.feng <[email protected]>
fengjian428 · Apr 5, 2023 · d83cc64 · d83cc64
1 parent fca9366
commit d83cc64
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 26 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -91,15 +91,15 @@ stages:
         displayName: UT FT common & flink & UT client/spark-client
         timeoutInMinutes: '150'
         steps:
-          - task: Maven@3
+          - task: Maven@3.205.1
             displayName: maven install
             inputs:
               mavenPomFile: 'pom.xml'
               goals: 'clean install'
               options: $(MVN_OPTS_INSTALL)
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-          - task: Maven@3
+          - task: Maven@3.205.1
             displayName: UT common flink client/spark-client
             inputs:
               mavenPomFile: 'pom.xml'
@@ -108,7 +108,7 @@ stages:
               publishJUnitResults: false
               jdkVersionOption: '1.8'
               mavenOptions: '-Xmx4g'
-          - task: Maven@3
+          - task: Maven@3.205.1
             displayName: FT common flink
             inputs:
               mavenPomFile: 'pom.xml'
@@ -121,15 +121,15 @@ stages:
         displayName: FT client/spark-client
         timeoutInMinutes: '150'
         steps:
-          - task: Maven@3
+          - task: Maven@3.205.1
             displayName: maven install
             inputs:
               mavenPomFile: 'pom.xml'
               goals: 'clean install'
               options: $(MVN_OPTS_INSTALL)
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-          - task: Maven@3
+          - task: Maven@3.205.1
             displayName: FT client/spark-client
             inputs:
               mavenPomFile: 'pom.xml'
@@ -142,15 +142,15 @@ stages:
         displayName: UT FT clients & cli & utilities & sync
         timeoutInMinutes: '150'
         steps:
-          - task: Maven@3
+          - task: Maven@3.205.1
             displayName: maven install
             inputs:
               mavenPomFile: 'pom.xml'
               goals: 'clean install'
               options: $(MVN_OPTS_INSTALL)
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-          - task: Maven@3
+          - task: Maven@3.205.1
             displayName: UT clients & cli & utilities & sync
             inputs:
               mavenPomFile: 'pom.xml'
@@ -159,7 +159,7 @@ stages:
               publishJUnitResults: false
               jdkVersionOption: '1.8'
               mavenOptions: '-Xmx4g'
-          - task: Maven@3
+          - task: Maven@3.205.1
             displayName: FT clients & cli & utilities & sync
             inputs:
               mavenPomFile: 'pom.xml'
@@ -172,15 +172,15 @@ stages:
         displayName: UT FT other modules
         timeoutInMinutes: '150'
         steps:
-          - task: Maven@3
+          - task: Maven@3.205.1
             displayName: maven install
             inputs:
               mavenPomFile: 'pom.xml'
               goals: 'clean install'
               options: $(MVN_OPTS_INSTALL)
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-          - task: Maven@3
+          - task: Maven@3.205.1
             displayName: UT other modules
             inputs:
               mavenPomFile: 'pom.xml'
@@ -189,7 +189,7 @@ stages:
               publishJUnitResults: false
               jdkVersionOption: '1.8'
               mavenOptions: '-Xmx4g'
-          - task: Maven@3
+          - task: Maven@3.205.1
             displayName: FT other modules
             inputs:
               mavenPomFile: 'pom.xml'
@@ -202,15 +202,15 @@ stages:
         displayName: IT modules
         timeoutInMinutes: '150'
         steps:
-          - task: Maven@3
+          - task: Maven@3.205.1
             displayName: maven install
             inputs:
               mavenPomFile: 'pom.xml'
               goals: 'clean install'
               options: $(MVN_OPTS_INSTALL) -Pintegration-tests
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-          - task: Maven@3
+          - task: Maven@3.205.1
             displayName: UT integ-test
             inputs:
               mavenPomFile: 'pom.xml'

diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
@@ -64,6 +64,8 @@
 import java.sql.Date;
 import java.sql.Timestamp;
 import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -957,21 +959,28 @@ private static Object rewritePrimaryTypeWithDiffSchemaType(Object oldValue, Sche
     throw new AvroRuntimeException(String.format("cannot support rewrite value for schema type: %s since the old schema type is: %s", newSchema, oldSchema));
   }
 
-  // convert days to Date
-  private static java.sql.Date toJavaDate(int days) {
-    long localMillis = Math.multiplyExact(days, MILLIS_PER_DAY);
-    int timeZoneOffset;
-    TimeZone defaultTimeZone = TimeZone.getDefault();
-    if (defaultTimeZone instanceof sun.util.calendar.ZoneInfo) {
-      timeZoneOffset = ((sun.util.calendar.ZoneInfo) defaultTimeZone).getOffsetsByWall(localMillis, null);
-    } else {
-      timeZoneOffset = defaultTimeZone.getOffset(localMillis - defaultTimeZone.getRawOffset());
-    }
-    return new java.sql.Date(localMillis - timeZoneOffset);
+  /**
+   * convert days to Date
+   *
+   * NOTE: This method could only be used in tests
+   *
+   * @VisibleForTesting
+   */
+  public static java.sql.Date toJavaDate(int days) {
+    LocalDate date = LocalDate.ofEpochDay(days);
+    ZoneId defaultZoneId = ZoneId.systemDefault();
+    ZonedDateTime zonedDateTime = date.atStartOfDay(defaultZoneId);
+    return new java.sql.Date(zonedDateTime.toInstant().toEpochMilli());
   }
 
-  // convert Date to days
-  private static int fromJavaDate(Date date) {
+  /**
+   * convert Date to days
+   *
+   * NOTE: This method could only be used in tests
+   *
+   * @VisibleForTesting
+   */
+  public static int fromJavaDate(Date date) {
     long millisUtc = date.getTime();
     long millisLocal = millisUtc + TimeZone.getDefault().getOffset(millisUtc);
     int julianDays = Math.toIntExact(Math.floorDiv(millisLocal, MILLIS_PER_DAY));

diff --git a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java
@@ -32,6 +32,7 @@
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.nio.ByteBuffer;
+import java.sql.Date;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -372,4 +373,11 @@ public void testReWriteAvroRecordWithNewSchema() {
     GenericRecord studentRecordRename = HoodieAvroUtils.rewriteRecordWithNewSchema(rec3, nestedSchemaRename, colRenames);
     Assertions.assertEquals(GenericData.get().validate(nestedSchemaRename, studentRecordRename), true);
   }
+
+  @Test
+  public void testConvertDaysToDate() {
+    Date now = new Date(System.currentTimeMillis());
+    int days = HoodieAvroUtils.fromJavaDate(now);
+    assertEquals(now.toLocalDate(), HoodieAvroUtils.toJavaDate(days).toLocalDate());
+  }
 }