[GOBBLIN-1667] Create new predicate - ExistingPartitionSkipPredicate

gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/predicates/ExistingPartitionSkipPredicate.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gobblin.data.management.copy.predicates;
+
+import com.google.common.base.Predicate;
+import javax.annotation.Nullable;
+import org.apache.gobblin.data.management.copy.hive.HivePartitionFileSet;
+
+
+/**
+ * This skip predicate will skip any partition that's already registered in the destination
+ * hive table.
+ */
+public class ExistingPartitionSkipPredicate implements Predicate<HivePartitionFileSet> {
+  @Override
+  public boolean apply(@Nullable HivePartitionFileSet input) {
+    if (input == null) {
+      return true;
+    }
+
+    return input.getExistingTargetPartition().isPresent();
+  }
+}

gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/predicates/ExistingPartitionSkipPredicateTest.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gobblin.data.management.copy.predicates;
+
+import com.google.common.base.Optional;
+import org.apache.gobblin.data.management.copy.hive.HivePartitionFileSet;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import static org.mockito.Mockito.*;
+
+
+public class ExistingPartitionSkipPredicateTest {
+  ExistingPartitionSkipPredicate predicate = new ExistingPartitionSkipPredicate();
+
+  @Test
+  public void shouldSkipHiveDatasetWithExistingPartition() {
+    HivePartitionFileSet fileSetWithExistingPartition = mock(HivePartitionFileSet.class);
+    HivePartitionFileSet fileSetWithoutExistingPartition = mock(HivePartitionFileSet.class);
+    Partition partition = mock(Partition.class);
+    when(fileSetWithExistingPartition.getExistingTargetPartition()).thenReturn(Optional.of(partition));
+    when(fileSetWithoutExistingPartition.getExistingTargetPartition()).thenReturn(Optional.absent());
+    Assert.assertTrue(predicate.apply(fileSetWithExistingPartition));
+    Assert.assertFalse(predicate.apply(fileSetWithoutExistingPartition));
+  }
+}

gobblin-docs/case-studies/Hive-Distcp.md

@@ -101,7 +101,9 @@ A partition filter can be applied when copying partitioned tables. Filters can o
 
 ## Fast partition skip predicate
 
-A predicate that operates on partitions can be provided to distcp-ng to allow it to quickly skip partitions without having to list all of the source and target files and do a diff on those sets (a costly operation). To set this predicate, provide the class name of the predicate with the key `hive.dataset.copy.fast.partition.skip.predicate`. Currently only one such predicate exists:
+A predicate that operates on partitions can be provided to distcp-ng to allow it to quickly skip partitions without having to list all of the source and target files and do a diff on those sets (a costly operation). To set this predicate, provide the class name of the predicate with the key `hive.dataset.copy.fast.partition.skip.predicate`. Below are the following predicates that exist
 
 * `RegistrationTimeSkipPredicate`: This predicate compares the Hive partition attribute `registrationGenerationTimeMillis` in the target with the modification time of the partition directory in the source. The partition is skipped unless the directory was modified more recently than the registrationGenerationTime. The attribute `registrationGenerationTimeMillis` is an attribute set by distcp-ng representing (for all practical purposes) the time at which the distcp-ng job that registered that table started.
-
+* `NonPartitionTableRegistrationTimeSkipPredicate`: This predicate can be used on non partition tables and compares the `registrationGenerationTimeMillis` in the target.
+* `ExistingPartitionSkipPredicate`: This predicate can be used to skip any partition that already exists in the target table.
+* `RootDirectoryModtimeSkipPredicate`: This predicate can be used to skip any partition whose root directory modified time is later than the copy source.