Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Deduplication buffer spooling #10507

Merged
merged 7 commits into from
Jan 27, 2022
Merged

Deduplication buffer spooling #10507

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ae3fa18
Consolidate failure handling in DeduplicatingDirectExchangeBuffer
arhimondr Jan 7, 2022
ac3a156
Support order preservation in ExchangeSink
arhimondr Jan 14, 2022
6fc8f7f
Identify exchanges by ExchangeId
arhimondr Jan 21, 2022
2b468b8
Add DeduplicatingDirectExchangeBuffer capacity config
arhimondr Jan 14, 2022
0ce87e1
Check for failure first in DeduplicatingDirectExchangeBuffer
arhimondr Jan 26, 2022
30529c7
Implement spilling in DeduplicatingDirectExchangeBuffer
arhimondr Jan 14, 2022
b4d213b
Improve error message when exchange manager is not configured
arhimondr Jan 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion core/trino-main/src/main/java/io/trino/exchange/ExchangeManagerRegistry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ public ExchangeManager getExchangeManager()
{
ExchangeManager exchangeManager = this.exchangeManager;
if (exchangeManager == null) {
throw new TrinoException(EXCHANGE_MANAGER_NOT_CONFIGURED, "Exchange manager is not configured");
throw new TrinoException(EXCHANGE_MANAGER_NOT_CONFIGURED, "Exchange manager must be configured for the failure recovery capabilities to be fully functional");
}
return exchangeManager;
}
Expand Down
2 changes: 1 addition & 1 deletion core/trino-main/src/main/java/io/trino/execution/buffer/LazyOutputBuffer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ public void setOutputBuffers(OutputBuffers newOutputBuffers)
ExchangeSinkInstanceHandle exchangeSinkInstanceHandle = newOutputBuffers.getExchangeSinkInstanceHandle()
.orElseThrow(() -> new IllegalArgumentException("exchange sink handle is expected to be present for buffer type EXTERNAL"));
ExchangeManager exchangeManager = exchangeManagerRegistry.getExchangeManager();
ExchangeSink exchangeSink = exchangeManager.createSink(exchangeSinkInstanceHandle);
ExchangeSink exchangeSink = exchangeManager.createSink(exchangeSinkInstanceHandle, false);
outputBuffer = new SpoolingExchangeOutputBuffer(state, newOutputBuffers, exchangeSink, memoryContextSupplier);
break;
default:
Expand Down
4 changes: 3 additions & 1 deletion core/trino-main/src/main/java/io/trino/execution/scheduler/SqlQueryScheduler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
import io.trino.spi.connector.ConnectorPartitionHandle;
import io.trino.spi.exchange.Exchange;
import io.trino.spi.exchange.ExchangeContext;
import io.trino.spi.exchange.ExchangeId;
import io.trino.spi.exchange.ExchangeManager;
import io.trino.split.SplitSource;
import io.trino.sql.planner.NodePartitionMap;
Expand Down Expand Up @@ -1741,7 +1742,8 @@ public static FaultTolerantDistributedStagesScheduler create(
}
else {
// create external exchange
exchange = Optional.of(exchangeManager.createExchange(new ExchangeContext(session.getQueryId(), stage.getStageId().getId()), hashPartitionCount));
ExchangeContext context = new ExchangeContext(session.getQueryId(), new ExchangeId("external-exchange-" + stage.getStageId().getId()));
exchange = Optional.of(exchangeManager.createExchange(context, hashPartitionCount));
exchanges.put(fragment.getId(), exchange.get());
taskLifecycleListener = TaskLifecycleListener.NO_OP;
}
Expand Down
Loading