Skip to content

Commit

Permalink
Deduce correct partition location in Hive partition projection
Browse files Browse the repository at this point in the history
  • Loading branch information
mayankvadariya authored and wendigo committed Sep 11, 2024
1 parent e21984d commit 85af13a
Show file tree
Hide file tree
Showing 2 changed files with 54 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -102,12 +102,20 @@ private Partition buildPartitionObject(Table table, String partitionName)
table.getPartitionColumns().stream()
.map(Column::getName).collect(Collectors.toList()),
partitionValues))
.orElseGet(() -> format("%s/%s/", table.getStorage().getLocation(), partitionName)))
.orElseGet(() -> getPartitionLocation(table.getStorage().getLocation(), partitionName)))
.setBucketProperty(table.getStorage().getBucketProperty())
.setSerdeParameters(table.getStorage().getSerdeParameters()))
.build();
}

private static String getPartitionLocation(String tableLocation, String partitionName)
{
if (tableLocation.endsWith("/")) {
return format("%s%s/", tableLocation, partitionName);
}
return format("%s/%s/", tableLocation, partitionName);
}

private static String expandStorageLocationTemplate(String template, List<String> partitionColumns, List<String> partitionValues)
{
Matcher matcher = PROJECTION_LOCATION_TEMPLATE_PLACEHOLDER_PATTERN.matcher(template);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import static io.trino.testing.MaterializedResult.resultBuilder;
import static io.trino.testing.TestingNames.randomNameSuffix;
import static io.trino.testing.TestingSession.testSessionBuilder;
import static java.lang.String.format;
import static java.util.Objects.requireNonNull;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
Expand All @@ -61,6 +62,7 @@ protected QueryRunner createQueryRunner()
.addHiveProperty("hive.metastore.glue.default-warehouse-dir", schemaPath())
.addHiveProperty("hive.security", "allow-all")
.addHiveProperty("hive.non-managed-table-writes-enabled", "true")
.addHiveProperty("hive.partition-projection-enabled", "true")
.addHiveProperty("fs.hadoop.enabled", "false")
.addHiveProperty("fs.native-s3.enabled", "true")
.build();
Expand Down Expand Up @@ -299,6 +301,49 @@ private void testAnalyzeWithProvidedTableLocation(boolean partitioned, LocationP
}
}

@Test
public void testPartitionProjectionWithProvidedTableLocation()
{
for (LocationPattern locationPattern : LocationPattern.values()) {
if (locationPattern == DOUBLE_SLASH || locationPattern == TRIPLE_SLASH || locationPattern == TWO_TRAILING_SLASHES) {
assertThatThrownBy(() -> testPartitionProjectionWithProvidedTableLocation(locationPattern))
.hasMessageStartingWith("Unsupported location that cannot be internally represented: ")
.hasStackTraceContaining("SQL: CREATE TABLE");
continue;
}
testPartitionProjectionWithProvidedTableLocation(locationPattern);
}
}

private void testPartitionProjectionWithProvidedTableLocation(LocationPattern locationPattern)
{
String tableName = "test_partition_projection_" + randomNameSuffix();
String tableLocation = locationPattern.locationForTable(bucketName, schemaName, tableName);

computeActual(format("""
CREATE TABLE %s (
name varchar(25),
short_name varchar WITH (
partition_projection_type='date',
partition_projection_format='yyyy-MM-dd HH',
partition_projection_range=ARRAY['2001-01-22 00', '2001-01-22 06'],
partition_projection_interval=1,
partition_projection_interval_unit='HOURS'
)
)
WITH (
partitioned_by=ARRAY['short_name'],
partition_projection_enabled=true,
external_location = '%s'
)""",
tableName,
tableLocation));

assertUpdate("INSERT INTO " + tableName + " VALUES ('name1', '2001-01-22 00')", 1);

assertQuery(format("SELECT name FROM %s", tableName), "VALUES ('name1')");
}

@Test
public void testInvalidSchemaNameLocation()
{
Expand Down

0 comments on commit 85af13a

Please sign in to comment.