diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcBatchPageSourceFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcBatchPageSourceFactory.java index f9f9967492cb..f26622a8b41c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcBatchPageSourceFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcBatchPageSourceFactory.java @@ -193,7 +193,7 @@ public static OrcBatchPageSource createOrcPageSource( AggregatedMemoryContext systemMemoryUsage = newSimpleAggregatedMemoryContext(); try { - OrcReader reader = new OrcReader(orcDataSource, orcEncoding, maxMergeDistance, maxBufferSize, tinyStripeThreshold, maxReadBlockSize); + OrcReader reader = new OrcReader(orcDataSource, orcEncoding, maxMergeDistance, tinyStripeThreshold, maxReadBlockSize); List physicalColumns = getPhysicalHiveColumnHandles(columns, useOrcColumnNames, reader, path); ImmutableMap.Builder includedColumns = ImmutableMap.builder(); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSourceFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSourceFactory.java index cdbf669c37d3..1bf75c8a0d83 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSourceFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSourceFactory.java @@ -232,7 +232,7 @@ public static OrcSelectivePageSource createOrcPageSource( AggregatedMemoryContext systemMemoryUsage = newSimpleAggregatedMemoryContext(); try { - OrcReader reader = new OrcReader(orcDataSource, orcEncoding, maxMergeDistance, maxBufferSize, tinyStripeThreshold, maxReadBlockSize); + OrcReader reader = new OrcReader(orcDataSource, orcEncoding, maxMergeDistance, tinyStripeThreshold, maxReadBlockSize); checkArgument(!domainPredicate.isNone(), "Unexpected NONE domain"); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/TempFileReader.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/TempFileReader.java index fb3a3ffc775c..878ad68fd366 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/TempFileReader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/TempFileReader.java @@ -55,7 +55,6 @@ public TempFileReader(List types, OrcDataSource dataSource) ORC, new DataSize(1, MEGABYTE), new DataSize(8, MEGABYTE), - new DataSize(8, MEGABYTE), new DataSize(16, MEGABYTE)); Map includedColumns = new HashMap<>(); diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/OrcReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/OrcReader.java index d0626bebe2e0..75bbc5e83a23 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/OrcReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/OrcReader.java @@ -62,7 +62,6 @@ public class OrcReader private final OrcDataSource orcDataSource; private final ExceptionWrappingMetadataReader metadataReader; private final DataSize maxMergeDistance; - private final DataSize maxReadSize; private final DataSize tinyStripeThreshold; private final DataSize maxBlockSize; private final HiveWriterVersion hiveWriterVersion; @@ -75,17 +74,16 @@ public class OrcReader private final Optional writeValidation; // This is based on the Apache Hive ORC code - public OrcReader(OrcDataSource orcDataSource, OrcEncoding orcEncoding, DataSize maxMergeDistance, DataSize maxReadSize, DataSize tinyStripeThreshold, DataSize maxBlockSize) + public OrcReader(OrcDataSource orcDataSource, OrcEncoding orcEncoding, DataSize maxMergeDistance, DataSize tinyStripeThreshold, DataSize maxBlockSize) throws IOException { - this(orcDataSource, orcEncoding, maxMergeDistance, maxReadSize, tinyStripeThreshold, maxBlockSize, Optional.empty()); + this(orcDataSource, orcEncoding, maxMergeDistance, tinyStripeThreshold, maxBlockSize, Optional.empty()); } OrcReader( OrcDataSource orcDataSource, OrcEncoding orcEncoding, DataSize maxMergeDistance, - DataSize maxReadSize, DataSize tinyStripeThreshold, DataSize maxBlockSize, Optional writeValidation) @@ -96,7 +94,6 @@ public OrcReader(OrcDataSource orcDataSource, OrcEncoding orcEncoding, DataSize requireNonNull(orcEncoding, "orcEncoding is null"); this.metadataReader = new ExceptionWrappingMetadataReader(orcDataSource.getId(), orcEncoding.createMetadataReader()); this.maxMergeDistance = requireNonNull(maxMergeDistance, "maxMergeDistance is null"); - this.maxReadSize = requireNonNull(maxReadSize, "maxReadSize is null"); this.tinyStripeThreshold = requireNonNull(tinyStripeThreshold, "tinyStripeThreshold is null"); this.maxBlockSize = requireNonNull(maxBlockSize, "maxBlockSize is null"); @@ -378,7 +375,7 @@ static void validateFile( readTypes.put(columnIndex, types.get(columnIndex)); } try { - OrcReader orcReader = new OrcReader(input, orcEncoding, new DataSize(1, MEGABYTE), new DataSize(8, MEGABYTE), new DataSize(8, MEGABYTE), new DataSize(16, MEGABYTE), Optional.of(writeValidation)); + OrcReader orcReader = new OrcReader(input, orcEncoding, new DataSize(1, MEGABYTE), new DataSize(8, MEGABYTE), new DataSize(16, MEGABYTE), Optional.of(writeValidation)); try (OrcBatchRecordReader orcRecordReader = orcReader.createBatchRecordReader(readTypes.build(), OrcPredicate.TRUE, hiveStorageTimeZone, newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE)) { while (orcRecordReader.nextBatch() >= 0) { // ignored diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkBatchStreamReaders.java b/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkBatchStreamReaders.java index 03a9143a6998..567598facdc5 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkBatchStreamReaders.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkBatchStreamReaders.java @@ -264,7 +264,7 @@ OrcBatchRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(orcFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); - OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); + OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createBatchRecordReader( ImmutableMap.of(0, type), OrcPredicate.TRUE, diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkSelectiveStreamReaders.java b/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkSelectiveStreamReaders.java index f2dd98cc1d3a..3b8665f48c43 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkSelectiveStreamReaders.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkSelectiveStreamReaders.java @@ -162,7 +162,7 @@ public OrcSelectiveRecordReader createRecordReader(Optional f throws IOException { OrcDataSource dataSource = new FileOrcDataSource(orcFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); - OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); + OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createSelectiveRecordReader( ImmutableMap.of(0, type), diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/OrcTester.java b/presto-orc/src/test/java/com/facebook/presto/orc/OrcTester.java index 0268faa6fd06..c081fd867daa 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/OrcTester.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/OrcTester.java @@ -842,7 +842,7 @@ private static OrcBatchRecordReader createCustomOrcRecordReader(TempFile tempFil throws IOException { OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); - OrcReader orcReader = new OrcReader(orcDataSource, orcEncoding, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), MAX_BLOCK_SIZE); + OrcReader orcReader = new OrcReader(orcDataSource, orcEncoding, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), MAX_BLOCK_SIZE); assertEquals(orcReader.getColumnNames(), ImmutableList.of("test")); assertEquals(orcReader.getFooter().getRowsInRowGroup(), 10_000); @@ -901,7 +901,7 @@ static OrcSelectiveRecordReader createCustomOrcSelectiveRecordReader( throws IOException { OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); - OrcReader orcReader = new OrcReader(orcDataSource, orcEncoding, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), MAX_BLOCK_SIZE); + OrcReader orcReader = new OrcReader(orcDataSource, orcEncoding, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), MAX_BLOCK_SIZE); assertEquals(orcReader.getColumnNames(), makeColumnNames(types.size())); assertEquals(orcReader.getFooter().getRowsInRowGroup(), 10_000); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestCachingOrcDataSource.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestCachingOrcDataSource.java index 5dde0ff5cea8..1fd025d5fa40 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestCachingOrcDataSource.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestCachingOrcDataSource.java @@ -192,7 +192,7 @@ public void testIntegration() public void doIntegration(TestingOrcDataSource orcDataSource, DataSize maxMergeDistance, DataSize maxReadSize, DataSize tinyStripeThreshold) throws IOException { - OrcReader orcReader = new OrcReader(orcDataSource, ORC, maxMergeDistance, maxReadSize, tinyStripeThreshold, new DataSize(1, Unit.MEGABYTE)); + OrcReader orcReader = new OrcReader(orcDataSource, ORC, maxMergeDistance, tinyStripeThreshold, new DataSize(1, Unit.MEGABYTE)); // 1 for reading file footer assertEquals(orcDataSource.getReadCount(), 1); List stripes = orcReader.getFooter().getStripes(); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestFlatMap.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestFlatMap.java index 65a9d04a23e4..46f1f3a768ae 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestFlatMap.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestFlatMap.java @@ -397,7 +397,6 @@ private void runTest(String testOrcFileName, Type keyType, Type valueType OrcEncoding.DWRF, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), - new DataSize(1, MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE)); Type mapType = TYPE_MANAGER.getParameterizedType( StandardTypes.MAP, diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcLz4.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcLz4.java index 6d4e8997da92..b4abe37e26da 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcLz4.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcLz4.java @@ -46,7 +46,7 @@ public void testReadLz4() // TODO: use Apache ORC library in OrcTester byte[] data = toByteArray(getResource("apache-lz4.orc")); - OrcReader orcReader = new OrcReader(new InMemoryOrcDataSource(data), ORC, SIZE, SIZE, SIZE, SIZE); + OrcReader orcReader = new OrcReader(new InMemoryOrcDataSource(data), ORC, SIZE, SIZE, SIZE); assertEquals(orcReader.getCompressionKind(), LZ4); assertEquals(orcReader.getFooter().getNumberOfRows(), 10_000); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderPositions.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderPositions.java index 7d9bbe7c5280..86571f4ac9a7 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderPositions.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderPositions.java @@ -297,7 +297,7 @@ public void testReadUserMetadata() createFileWithOnlyUserMetadata(tempFile.getFile(), metadata); OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); - OrcReader orcReader = new OrcReader(orcDataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); + OrcReader orcReader = new OrcReader(orcDataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); Footer footer = orcReader.getFooter(); Map readMetadata = Maps.transformValues(footer.getUserMetadata(), Slice::toStringAscii); assertEquals(readMetadata, metadata); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcWriter.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcWriter.java index 70c8d19c3273..11933d3641cf 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcWriter.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcWriter.java @@ -95,7 +95,7 @@ public void testWriteOutputStreamsInOrder() // read the footer and verify the streams are ordered by size DataSize dataSize = new DataSize(1, MEGABYTE); OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), dataSize, dataSize, dataSize, true); - Footer footer = new OrcReader(orcDataSource, ORC, dataSize, dataSize, dataSize, dataSize).getFooter(); + Footer footer = new OrcReader(orcDataSource, ORC, dataSize, dataSize, dataSize).getFooter(); for (StripeInformation stripe : footer.getStripes()) { // read the footer diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestStructBatchStreamReader.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestStructBatchStreamReader.java index 635b1dac1d2d..90a207c89eab 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestStructBatchStreamReader.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestStructBatchStreamReader.java @@ -263,7 +263,7 @@ private RowBlock read(TempFile tempFile, Type readerType) { DataSize dataSize = new DataSize(1, MEGABYTE); OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), dataSize, dataSize, dataSize, true); - OrcReader orcReader = new OrcReader(orcDataSource, ORC, dataSize, dataSize, dataSize, dataSize); + OrcReader orcReader = new OrcReader(orcDataSource, ORC, dataSize, dataSize, dataSize); Map includedColumns = new HashMap<>(); includedColumns.put(0, readerType); diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java index b0fd3201b6b0..fa637e8179d4 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java @@ -95,7 +95,6 @@ public OrcFileInfo rewrite(Map allColumnTypes, Path input, Path ou orcDataEnvironment.createOrcDataSource(input, readerAttributes), ORC, readerAttributes.getMaxMergeDistance(), - readerAttributes.getMaxReadSize(), readerAttributes.getTinyStripeThreshold(), HUGE_MAX_READ_BLOCK_SIZE); diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java index 0a567d927335..756823398119 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java @@ -261,7 +261,7 @@ public ConnectorPageSource getPageSource( AggregatedMemoryContext systemMemoryUsage = newSimpleAggregatedMemoryContext(); try { - OrcReader reader = new OrcReader(dataSource, ORC, readerAttributes.getMaxMergeDistance(), readerAttributes.getMaxReadSize(), readerAttributes.getTinyStripeThreshold(), HUGE_MAX_READ_BLOCK_SIZE); + OrcReader reader = new OrcReader(dataSource, ORC, readerAttributes.getMaxMergeDistance(), readerAttributes.getTinyStripeThreshold(), HUGE_MAX_READ_BLOCK_SIZE); Map indexMap = columnIdIndex(reader.getColumnNames()); ImmutableMap.Builder includedColumns = ImmutableMap.builder(); @@ -411,7 +411,7 @@ private ShardInfo createShardInfo(UUID shardUuid, OptionalInt bucketNumber, Path private List computeShardStats(Path file) { try (OrcDataSource dataSource = orcDataEnvironment.createOrcDataSource(file, defaultReaderAttributes)) { - OrcReader reader = new OrcReader(dataSource, ORC, defaultReaderAttributes.getMaxMergeDistance(), defaultReaderAttributes.getMaxReadSize(), defaultReaderAttributes.getTinyStripeThreshold(), HUGE_MAX_READ_BLOCK_SIZE); + OrcReader reader = new OrcReader(dataSource, ORC, defaultReaderAttributes.getMaxMergeDistance(), defaultReaderAttributes.getTinyStripeThreshold(), HUGE_MAX_READ_BLOCK_SIZE); ImmutableList.Builder list = ImmutableList.builder(); for (ColumnInfo info : getColumnInfo(reader)) { diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java index fa76f5d4041b..02432917718c 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java @@ -58,7 +58,7 @@ public static OrcDataSource fileOrcDataSource(File file) public static OrcBatchRecordReader createReader(OrcDataSource dataSource, List columnIds, List types) throws IOException { - OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); + OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); List columnNames = orcReader.getColumnNames(); assertEquals(columnNames.size(), columnIds.size()); @@ -77,7 +77,7 @@ public static OrcBatchRecordReader createReader(OrcDataSource dataSource, List