From ad2014b090427c858425a7319ad4d971ee8aebf0 Mon Sep 17 00:00:00 2001 From: Rajat Khandelwal Date: Mon, 20 Feb 2017 14:03:30 +0530 Subject: [PATCH 1/2] Fixing Expression resolver tests --- .../apache/lens/cube/metadata/DateUtil.java | 10 +++---- .../apache/lens/cube/metadata/TimeRange.java | 6 ++++- .../cube/parse/CandidateTablePruneCause.java | 6 +++-- .../lens/cube/parse/StorageCandidate.java | 11 ++++++-- .../lens/cube/parse/StorageTableResolver.java | 27 +++++++++++-------- .../cube/parse/TestExpressionResolver.java | 6 ++--- pom.xml | 4 +-- 7 files changed, 43 insertions(+), 27 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java index 771708132..27400501b 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java @@ -329,12 +329,10 @@ private static CoveringInfo getMilliSecondCoveringInfo(Date from, Date to, long } static boolean isCoverableBy(Date from, Date to, Set intervals) { - for (UpdatePeriod period : intervals) { - if (getCoveringInfo(from, to, period).isCoverable()) { - return true; - } - } - return false; + return intervals.stream().anyMatch(period->isCoverableBy(from, to, period)); + } + static boolean isCoverableBy(Date from, Date to, UpdatePeriod period) { + return getCoveringInfo(from, to, period).isCoverable(); } public static int getTimeDiff(Date fromDate, Date toDate, UpdatePeriod updatePeriod) { diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java index bf6cc5c3e..d7691fa68 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java @@ -22,6 +22,7 @@ import java.util.Calendar; import java.util.Date; +import java.util.Set; import java.util.TreeSet; import org.apache.lens.cube.error.LensCubeErrorCode; @@ -48,10 +49,13 @@ public class TimeRange { private ASTNode parent; private int childIndex; - public boolean isCoverableBy(TreeSet updatePeriods) { + public boolean isCoverableBy(Set updatePeriods) { return DateUtil.isCoverableBy(fromDate, toDate, updatePeriods); } + public boolean isCoverableBy(UpdatePeriod updatePeriod) { + return DateUtil.isCoverableBy(fromDate, toDate, updatePeriod); + } public static class TimeRangeBuilder { private final TimeRange range; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java index c7f2047b5..00161f13f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java @@ -227,8 +227,10 @@ String getBriefError(Set causes) { public enum SkipUpdatePeriodCode { // invalid update period INVALID, - // Query max interval is more than update period - QUERY_INTERVAL_BIGGER + //TODO: add property name this update period is greater than the Query max interval as provided by user + QUERY_INTERVAL_BIGGER_THAN_MAX, + // Query range is too small to be answered by this update period. It can only be answered by smaller update periods. + QUERY_INTERVAL_SMALL } // Used for Test cases only. diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index fca86ec88..273b9b5f5 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -506,8 +506,8 @@ public boolean evaluateCompleteness(TimeRange timeRange, TimeRange parentTimeRan boolean partColNotSupported = rangeParts.isEmpty(); String storageTableName = getName(); - if (storagePruningMsgs.containsKey(storageTableName)) { - List causes = storagePruningMsgs.get(storageTableName); + if (storagePruningMsgs.containsKey(this)) { + List causes = storagePruningMsgs.get(this); // Find the PART_COL_DOES_NOT_EXISTS for (CandidateTablePruneCause cause : causes) { if (cause.getCause().equals(CandidateTablePruneCode.PART_COL_DOES_NOT_EXIST)) { @@ -744,4 +744,11 @@ public String getAliasForTable(String alias) { } return ret; } + + public boolean canCoverTimeRanges(CubeQueryContext cubeql) { + return cubeql.getTimeRanges().stream().allMatch(this::isTimeRangeCoverable); + } + public boolean isTimeRangeCoverable(TimeRange timeRange) { + return timeRange.isCoverableBy(getValidUpdatePeriods()); + } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java index 3029589ae..ede10932c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java @@ -30,6 +30,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import lombok.extern.slf4j.Slf4j; /** * Resolve storages and partitions of all candidate tables and prunes candidate tables with missing storages or @@ -46,7 +48,7 @@ class StorageTableResolver implements ContextRewriter { private final UpdatePeriod maxInterval; // TODO union : Remove this. All partitions are stored in the StorageCandidate. private final Map> nonExistingPartitions = new HashMap<>(); - CubeMetastoreClient client; + private CubeMetastoreClient client; private PHASE phase; private float completenessThreshold; private String completenessPartCol; @@ -268,7 +270,6 @@ private void resolveStorageTable(CubeQueryContext cubeql) throws LensException { boolean partitionColumnExists = client.partColExists(storageTable, range.getPartitionColumn()); valid = partitionColumnExists; if (!partitionColumnExists) { - String timeDim = cubeql.getBaseCube().getTimeDimOfPartitionColumn(range.getPartitionColumn()); TimeRange fallBackRange = StorageUtil.getFallbackRange(range, sc.getFact().getName(), cubeql); if (fallBackRange == null) { log.info("No partitions for range:{}. fallback range: {}", range, fallBackRange); @@ -299,17 +300,21 @@ private void resolveStorageTable(CubeQueryContext cubeql) throws LensException { // Check for update period. for (UpdatePeriod updatePeriod : sc.getFact().getUpdatePeriods().get(sc.getStorageName())) { if (maxInterval != null && updatePeriod.compareTo(maxInterval) > 0) { - log.info("Skipping update period {} for fact {}", updatePeriod, sc.getFact()); - skipUpdatePeriodCauses.put(updatePeriod.toString(), SkipUpdatePeriodCode.QUERY_INTERVAL_BIGGER); - continue; - } - if (validUpdatePeriods != null && !validUpdatePeriods.contains(updatePeriod.name().toLowerCase())) { - log.info("Skipping update period {} for fact {} for storage {}", updatePeriod, sc.getFact(), storageTable); + log.info("Skipping update period {} for fact {} since it's more than max interval supplied", + updatePeriod, sc.getFact()); + skipUpdatePeriodCauses.put(updatePeriod.toString(), SkipUpdatePeriodCode.QUERY_INTERVAL_BIGGER_THAN_MAX); + } else if (validUpdatePeriods != null && !validUpdatePeriods.contains(updatePeriod.name().toLowerCase())) { + log.info("Skipping update period {} for fact {} for storage {} since it's invalid", + updatePeriod, sc.getFact(), storageTable); skipUpdatePeriodCauses.put(updatePeriod.toString(), SkipUpdatePeriodCode.INVALID); - continue; + } else if (cubeql.getTimeRanges().stream().noneMatch(timeRange -> timeRange.isCoverableBy(updatePeriod))) { + log.info("Skipping update period {} for fact {} for storage {} since it's bigger than query range", + updatePeriod, sc.getFact(), storageTable); + skipUpdatePeriodCauses.put(updatePeriod.toString(), SkipUpdatePeriodCode.QUERY_INTERVAL_SMALL); + } else { + isStorageAdded = true; + sc.addValidUpdatePeriod(updatePeriod); } - isStorageAdded = true; - sc.addValidUpdatePeriod(updatePeriod); } if (!isStorageAdded) { cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.updatePeriodsRejected(skipUpdatePeriodCauses)); diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java index dd18ffd49..6ad3091f5 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java @@ -446,7 +446,7 @@ public void testDimensionQueryWithTableAliasColumnAlias() throws Exception { @Test public void testSingleColExpression() throws Exception { Configuration tconf = new Configuration(conf); - tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); + tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); // time_range_in not translating CubeQueryContext rewrittenQuery = rewriteCtx("select singlecolmsr2expr from testCube where " + TWO_DAYS_RANGE, tconf); String expected = @@ -547,7 +547,7 @@ public void testSingleChainRefColExpressionWithAlias() throws Exception { @Test public void testSingleColExpressionWithAlias() throws Exception { Configuration tconf = new Configuration(conf); - tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); + tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); // time_range_in not getting translated CubeQueryContext rewrittenQuery = rewriteCtx("select singlecolmsr2expr as msr2 from testCube where " + TWO_DAYS_RANGE, tconf); String expected = @@ -559,7 +559,7 @@ public void testSingleColExpressionWithAlias() throws Exception { @Test public void testSingleColQualifiedExpression() throws Exception { Configuration tconf = new Configuration(conf); - tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); + tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); // time_range_in not getting converted CubeQueryContext rewrittenQuery = rewriteCtx("select singlecolmsr2qualifiedexpr from testCube where " + TWO_DAYS_RANGE, tconf); String expected = diff --git a/pom.xml b/pom.xml index 8ea64b77f..12cd9cc93 100644 --- a/pom.xml +++ b/pom.xml @@ -34,8 +34,8 @@ UTF-8 - 1.7 - 1.7 + 1.8 + 1.8 3.0.1 2.5 2.0.1 From d65e6d5caf148c8dc1538d5a80b4e922be2d1e9f Mon Sep 17 00:00:00 2001 From: Rajat Khandelwal Date: Mon, 20 Feb 2017 14:43:01 +0530 Subject: [PATCH 2/2] merge with apache/lens-1381 --- .../java/org/apache/lens/cube/parse/TestBaseCubeQueries.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java index 44847484d..de01150cb 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java @@ -838,7 +838,7 @@ public void testFallbackPartCol() throws Exception { getLensExceptionInRewrite("select msr12 from basecube where " + TWO_DAYS_RANGE, conf); NoCandidateFactAvailableException ne = (NoCandidateFactAvailableException) exc; PruneCauses.BriefAndDetailedError pruneCause = ne.getJsonMessage(); - assertTrue(pruneCause.getBrief().contains("Missing partitions")); + assertTrue(pruneCause.getBrief().contains("Missing partitions")); // error in this. assertEquals(pruneCause.getDetails().get("c4_testfact2_base").iterator().next().getCause(), MISSING_PARTITIONS); assertEquals(pruneCause.getDetails().get("c4_testfact2_base").iterator().next().getMissingPartitions().size(), 1); assertEquals(