From dab2fae16cbdf3e7a843ad0e4c51d4b23e0b2695 Mon Sep 17 00:00:00 2001 From: Andrii Rosa Date: Tue, 4 Jan 2022 15:08:50 -0500 Subject: [PATCH] Disable streaming upload for MinIO failure recovery tests Streaming upload to S3 allocates a 16MB buffer (by default) for each output stream. Failure recovery tests create a table partitioned into ~60 partitions. Since for each partition at least one file must be created the engine has to allocate ~1GB of buffer space. These buffer allocations push the memory reservation beyond the maximum heap size. --- .../plugin/hive/TestHiveMinioQueryFailureRecoveryTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveMinioQueryFailureRecoveryTest.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveMinioQueryFailureRecoveryTest.java index b354b508c1df..f93f27bfd8d2 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveMinioQueryFailureRecoveryTest.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveMinioQueryFailureRecoveryTest.java @@ -50,6 +50,12 @@ protected QueryRunner createQueryRunner(List> requiredTpchTables, M .setInitialTables(requiredTpchTables) .setExtraProperties(configProperties) .setCoordinatorProperties(coordinatorProperties) + .setHiveProperties(ImmutableMap.builder() + // Streaming upload allocates non trivial amount of memory for buffering (16MB per output file by default). + // When streaming upload is enabled insert into a table with high number of buckets / partitions may cause + // the tests to run out of memory as the buffer space is eagerly allocated for each output file. + .put("hive.s3.streaming.enabled", "false") + .build()) .build(); }