Skip to content

Commit

Permalink
Remove distinction between system and user memory
Browse files Browse the repository at this point in the history
  • Loading branch information
@findepi
findepi committed Jan 24, 2022
1 parent c36a280 commit 9ab09e9
Show file tree
Hide file tree
Showing 200 changed files with 602 additions and 1,250 deletions.
10 changes: 2 additions & 8 deletions core/trino-main/src/main/java/io/trino/ExceededMemoryLimitException.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,19 +37,13 @@ public static ExceededMemoryLimitException exceededGlobalTotalLimit(DataSize max
public static ExceededMemoryLimitException exceededLocalUserMemoryLimit(DataSize maxMemory, String additionalFailureInfo)
{
return new ExceededMemoryLimitException(EXCEEDED_LOCAL_MEMORY_LIMIT,
format("Query exceeded per-node user memory limit of %s [%s]", maxMemory, additionalFailureInfo));
}

public static ExceededMemoryLimitException exceededLocalTotalMemoryLimit(DataSize maxMemory, String additionalFailureInfo)
{
return new ExceededMemoryLimitException(EXCEEDED_LOCAL_MEMORY_LIMIT,
format("Query exceeded per-node total memory limit of %s [%s]", maxMemory, additionalFailureInfo));
format("Query exceeded per-node memory limit of %s [%s]", maxMemory, additionalFailureInfo));
}

public static ExceededMemoryLimitException exceededTaskMemoryLimit(DataSize maxMemory, String additionalFailureInfo)
{
return new ExceededMemoryLimitException(EXCEEDED_LOCAL_MEMORY_LIMIT,
format("Query exceeded per-task total memory limit of %s [%s]", maxMemory, additionalFailureInfo));
format("Query exceeded per-task memory limit of %s [%s]", maxMemory, additionalFailureInfo));
}

private ExceededMemoryLimitException(StandardErrorCode errorCode, String message)
Expand Down
19 changes: 4 additions & 15 deletions core/trino-main/src/main/java/io/trino/SystemSessionProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,7 @@ public final class SystemSessionProperties
public static final String ENABLE_COORDINATOR_DYNAMIC_FILTERS_DISTRIBUTION = "enable_coordinator_dynamic_filters_distribution";
public static final String ENABLE_LARGE_DYNAMIC_FILTERS = "enable_large_dynamic_filters";
public static final String QUERY_MAX_MEMORY_PER_NODE = "query_max_memory_per_node";
public static final String QUERY_MAX_TOTAL_MEMORY_PER_NODE = "query_max_total_memory_per_node";
public static final String QUERY_MAX_TOTAL_MEMORY_PER_TASK = "query_max_total_memory_per_task";
public static final String QUERY_MAX_MEMORY_PER_TASK = "query_max_memory_per_task";
public static final String IGNORE_DOWNSTREAM_PREFERENCES = "ignore_downstream_preferences";
public static final String FILTERING_SEMI_JOIN_TO_INNER = "rewrite_filtering_semi_join_to_inner_join";
public static final String OPTIMIZE_DUPLICATE_INSENSITIVE_JOINS = "optimize_duplicate_insensitive_joins";
Expand Down Expand Up @@ -581,14 +580,9 @@ public SystemSessionProperties(
nodeMemoryConfig.getMaxQueryMemoryPerNode(),
true),
dataSizeProperty(
QUERY_MAX_TOTAL_MEMORY_PER_NODE,
"Maximum amount of total memory a query can use per node",
nodeMemoryConfig.getMaxQueryTotalMemoryPerNode(),
true),
dataSizeProperty(
QUERY_MAX_TOTAL_MEMORY_PER_TASK,
QUERY_MAX_MEMORY_PER_TASK,
"Maximum amount of memory a single task can use",
nodeMemoryConfig.getMaxQueryTotalMemoryPerTask().orElse(null),
nodeMemoryConfig.getMaxQueryMemoryPerTask().orElse(null),
true),
booleanProperty(
IGNORE_DOWNSTREAM_PREFERENCES,
Expand Down Expand Up @@ -1151,14 +1145,9 @@ public static DataSize getQueryMaxMemoryPerNode(Session session)
return session.getSystemProperty(QUERY_MAX_MEMORY_PER_NODE, DataSize.class);
}

public static DataSize getQueryMaxTotalMemoryPerNode(Session session)
{
return session.getSystemProperty(QUERY_MAX_TOTAL_MEMORY_PER_NODE, DataSize.class);
}

public static Optional<DataSize> getQueryMaxTotalMemoryPerTask(Session session)
{
return Optional.ofNullable(session.getSystemProperty(QUERY_MAX_TOTAL_MEMORY_PER_TASK, DataSize.class));
return Optional.ofNullable(session.getSystemProperty(QUERY_MAX_MEMORY_PER_TASK, DataSize.class));
}

public static boolean ignoreDownStreamPreferences(Session session)
Expand Down
2 changes: 1 addition & 1 deletion ...-main/src/main/java/io/trino/connector/informationschema/InformationSchemaPageSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ public Page getNextPage()
}

@Override
public long getSystemMemoryUsage()
public long getMemoryUsage()
{
return memoryUsageBytes + pageBuilder.getRetainedSizeInBytes();
}
Expand Down
2 changes: 0 additions & 2 deletions core/trino-main/src/main/java/io/trino/dispatcher/FailedDispatchQuery.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,8 +272,6 @@ private static QueryStats immediateFailureQueryStats()
0,
0,
0,
0,
DataSize.ofBytes(0),
DataSize.ofBytes(0),
DataSize.ofBytes(0),
DataSize.ofBytes(0),
Expand Down
4 changes: 0 additions & 4 deletions core/trino-main/src/main/java/io/trino/event/QueryMonitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,6 @@ public void queryImmediateFailureEvent(BasicQueryInfo queryInfo, ExecutionFailur
0,
0,
0,
0,
0,
ImmutableList.of(),
0,
true,
Expand Down Expand Up @@ -263,7 +261,6 @@ private QueryStatistics createQueryStatistics(QueryInfo queryInfo)
Optional.of(ofMillis(queryStats.getPlanningTime().toMillis())),
Optional.of(ofMillis(queryStats.getExecutionTime().toMillis())),
queryStats.getPeakUserMemoryReservation().toBytes(),
queryStats.getPeakNonRevocableMemoryReservation().toBytes(),
queryStats.getPeakTaskUserMemory().toBytes(),
queryStats.getPeakTaskTotalMemory().toBytes(),
queryStats.getPhysicalInputDataSize().toBytes(),
Expand All @@ -277,7 +274,6 @@ private QueryStatistics createQueryStatistics(QueryInfo queryInfo)
queryStats.getLogicalWrittenDataSize().toBytes(),
queryStats.getWrittenPositions(),
queryStats.getCumulativeUserMemory(),
queryStats.getCumulativeSystemMemory(),
queryStats.getStageGcStatistics(),
queryStats.getCompletedDrivers(),
queryInfo.isCompleteInfo(),
Expand Down
12 changes: 0 additions & 12 deletions core/trino-main/src/main/java/io/trino/execution/BasicStageStats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ public class BasicStageStats
DataSize.ofBytes(0),
0,

0,
0,
DataSize.ofBytes(0),
DataSize.ofBytes(0),
Expand All @@ -73,7 +72,6 @@ public class BasicStageStats
private final DataSize rawInputDataSize;
private final long rawInputPositions;
private final long cumulativeUserMemory;
private final long cumulativeSystemMemory;
private final DataSize userMemoryReservation;
private final DataSize totalMemoryReservation;
private final Duration totalCpuTime;
Expand Down Expand Up @@ -101,7 +99,6 @@ public BasicStageStats(
long rawInputPositions,

long cumulativeUserMemory,
long cumulativeSystemMemory,
DataSize userMemoryReservation,
DataSize totalMemoryReservation,

Expand All @@ -126,7 +123,6 @@ public BasicStageStats(
this.rawInputDataSize = requireNonNull(rawInputDataSize, "rawInputDataSize is null");
this.rawInputPositions = rawInputPositions;
this.cumulativeUserMemory = cumulativeUserMemory;
this.cumulativeSystemMemory = cumulativeSystemMemory;
this.userMemoryReservation = requireNonNull(userMemoryReservation, "userMemoryReservation is null");
this.totalMemoryReservation = requireNonNull(totalMemoryReservation, "totalMemoryReservation is null");
this.totalCpuTime = requireNonNull(totalCpuTime, "totalCpuTime is null");
Expand Down Expand Up @@ -201,11 +197,6 @@ public long getCumulativeUserMemory()
return cumulativeUserMemory;
}

public long getCumulativeSystemMemory()
{
return cumulativeSystemMemory;
}

public DataSize getUserMemoryReservation()
{
return userMemoryReservation;
Expand Down Expand Up @@ -249,7 +240,6 @@ public static BasicStageStats aggregateBasicStageStats(Iterable<BasicStageStats>
int completedDrivers = 0;

long cumulativeUserMemory = 0;
long cumulativeSystemMemory = 0;
long userMemoryReservation = 0;
long totalMemoryReservation = 0;

Expand Down Expand Up @@ -278,7 +268,6 @@ public static BasicStageStats aggregateBasicStageStats(Iterable<BasicStageStats>
completedDrivers += stageStats.getCompletedDrivers();

cumulativeUserMemory += stageStats.getCumulativeUserMemory();
cumulativeSystemMemory += stageStats.getCumulativeSystemMemory();
userMemoryReservation += stageStats.getUserMemoryReservation().toBytes();
totalMemoryReservation += stageStats.getTotalMemoryReservation().toBytes();

Expand Down Expand Up @@ -325,7 +314,6 @@ public static BasicStageStats aggregateBasicStageStats(Iterable<BasicStageStats>
rawInputPositions,

cumulativeUserMemory,
cumulativeSystemMemory,
succinctBytes(userMemoryReservation),
succinctBytes(totalMemoryReservation),

Expand Down
7 changes: 2 additions & 5 deletions core/trino-main/src/main/java/io/trino/execution/MemoryTrackingRemoteTaskFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@ private static final class UpdatePeakMemory
{
private final QueryStateMachine stateMachine;
private long previousUserMemory;
private long previousSystemMemory;
private long previousRevocableMemory;

public UpdatePeakMemory(QueryStateMachine stateMachine)
Expand All @@ -83,14 +82,12 @@ public UpdatePeakMemory(QueryStateMachine stateMachine)
public synchronized void stateChanged(TaskStatus newStatus)
{
long currentUserMemory = newStatus.getMemoryReservation().toBytes();
long currentSystemMemory = newStatus.getSystemMemoryReservation().toBytes();
long currentRevocableMemory = newStatus.getRevocableMemoryReservation().toBytes();
long currentTotalMemory = currentUserMemory + currentSystemMemory + currentRevocableMemory;
long currentTotalMemory = currentUserMemory + currentRevocableMemory;
long deltaUserMemoryInBytes = currentUserMemory - previousUserMemory;
long deltaRevocableMemoryInBytes = currentRevocableMemory - previousRevocableMemory;
long deltaTotalMemoryInBytes = currentTotalMemory - (previousUserMemory + previousSystemMemory + previousRevocableMemory);
long deltaTotalMemoryInBytes = currentTotalMemory - (previousUserMemory + previousRevocableMemory);
previousUserMemory = currentUserMemory;
previousSystemMemory = currentSystemMemory;
previousRevocableMemory = currentRevocableMemory;
stateMachine.updateMemoryUsage(deltaUserMemoryInBytes, deltaRevocableMemoryInBytes, deltaTotalMemoryInBytes, currentUserMemory, currentRevocableMemory, currentTotalMemory);
}
Expand Down
14 changes: 0 additions & 14 deletions core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,6 @@ public class QueryStateMachine

private final AtomicLong currentRevocableMemory = new AtomicLong();
private final AtomicLong peakRevocableMemory = new AtomicLong();
private final AtomicLong peakNonRevocableMemory = new AtomicLong();

// peak of the user + system + revocable memory reservation
private final AtomicLong currentTotalMemory = new AtomicLong();
Expand Down Expand Up @@ -289,11 +288,6 @@ public long getPeakRevocableMemoryInBytes()
return peakRevocableMemory.get();
}

public long getPeakNonRevocableMemoryInBytes()
{
return peakNonRevocableMemory.get();
}

public long getPeakTotalMemoryInBytes()
{
return peakTotalMemory.get();
Expand Down Expand Up @@ -332,7 +326,6 @@ public void updateMemoryUsage(
currentTotalMemory.addAndGet(deltaTotalMemoryInBytes);
peakUserMemory.updateAndGet(currentPeakValue -> Math.max(currentUserMemory.get(), currentPeakValue));
peakRevocableMemory.updateAndGet(currentPeakValue -> Math.max(currentRevocableMemory.get(), currentPeakValue));
peakNonRevocableMemory.updateAndGet(currentPeakValue -> Math.max(currentTotalMemory.get() - currentRevocableMemory.get(), currentPeakValue));
peakTotalMemory.updateAndGet(currentPeakValue -> Math.max(currentTotalMemory.get(), currentPeakValue));
peakTaskUserMemory.accumulateAndGet(taskUserMemoryInBytes, Math::max);
peakTaskRevocableMemory.accumulateAndGet(taskRevocableMemoryInBytes, Math::max);
Expand Down Expand Up @@ -373,7 +366,6 @@ public BasicQueryInfo getBasicQueryInfo(Optional<BasicStageStats> rootStage)
stageStats.getPhysicalInputDataSize(),

stageStats.getCumulativeUserMemory(),
stageStats.getCumulativeSystemMemory(),
stageStats.getUserMemoryReservation(),
stageStats.getTotalMemoryReservation(),
succinctBytes(getPeakUserMemoryInBytes()),
Expand Down Expand Up @@ -472,7 +464,6 @@ private QueryStats getQueryStats(Optional<StageInfo> rootStage)
int completedDrivers = 0;

long cumulativeUserMemory = 0;
long cumulativeSystemMemory = 0;
long userMemoryReservation = 0;
long revocableMemoryReservation = 0;
long totalMemoryReservation = 0;
Expand Down Expand Up @@ -519,7 +510,6 @@ private QueryStats getQueryStats(Optional<StageInfo> rootStage)
completedDrivers += stageStats.getCompletedDrivers();

cumulativeUserMemory += stageStats.getCumulativeUserMemory();
cumulativeSystemMemory += stageStats.getCumulativeSystemMemory();
userMemoryReservation += stageStats.getUserMemoryReservation().toBytes();
revocableMemoryReservation += stageStats.getRevocableMemoryReservation().toBytes();
totalMemoryReservation += stageStats.getTotalMemoryReservation().toBytes();
Expand Down Expand Up @@ -589,13 +579,11 @@ private QueryStats getQueryStats(Optional<StageInfo> rootStage)
completedDrivers,

cumulativeUserMemory,
cumulativeSystemMemory,
succinctBytes(userMemoryReservation),
succinctBytes(revocableMemoryReservation),
succinctBytes(totalMemoryReservation),
succinctBytes(getPeakUserMemoryInBytes()),
succinctBytes(getPeakRevocableMemoryInBytes()),
succinctBytes(getPeakNonRevocableMemoryInBytes()),
succinctBytes(getPeakTotalMemoryInBytes()),
succinctBytes(getPeakTaskUserMemory()),
succinctBytes(getPeakTaskRevocableMemory()),
Expand Down Expand Up @@ -1153,13 +1141,11 @@ private static QueryStats pruneQueryStats(QueryStats queryStats)
queryStats.getBlockedDrivers(),
queryStats.getCompletedDrivers(),
queryStats.getCumulativeUserMemory(),
queryStats.getCumulativeSystemMemory(),
queryStats.getUserMemoryReservation(),
queryStats.getRevocableMemoryReservation(),
queryStats.getTotalMemoryReservation(),
queryStats.getPeakUserMemoryReservation(),
queryStats.getPeakRevocableMemoryReservation(),
queryStats.getPeakNonRevocableMemoryReservation(),
queryStats.getPeakTotalMemoryReservation(),
queryStats.getPeakTaskUserMemory(),
queryStats.getPeakTaskRevocableMemory(),
Expand Down
19 changes: 0 additions & 19 deletions core/trino-main/src/main/java/io/trino/execution/QueryStats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,13 +65,11 @@ public class QueryStats
private final int completedDrivers;

private final double cumulativeUserMemory;
private final double cumulativeSystemMemory;
private final DataSize userMemoryReservation;
private final DataSize revocableMemoryReservation;
private final DataSize totalMemoryReservation;
private final DataSize peakUserMemoryReservation;
private final DataSize peakRevocableMemoryReservation;
private final DataSize peakNonRevocableMemoryReservation;
private final DataSize peakTotalMemoryReservation;
private final DataSize peakTaskUserMemory;
private final DataSize peakTaskRevocableMemory;
Expand Down Expand Up @@ -135,13 +133,11 @@ public QueryStats(
@JsonProperty("completedDrivers") int completedDrivers,

@JsonProperty("cumulativeUserMemory") double cumulativeUserMemory,
@JsonProperty("cumulativeSystemMemory") double cumulativeSystemMemory,
@JsonProperty("userMemoryReservation") DataSize userMemoryReservation,
@JsonProperty("revocableMemoryReservation") DataSize revocableMemoryReservation,
@JsonProperty("totalMemoryReservation") DataSize totalMemoryReservation,
@JsonProperty("peakUserMemoryReservation") DataSize peakUserMemoryReservation,
@JsonProperty("peakRevocableMemoryReservation") DataSize peakRevocableMemoryReservation,
@JsonProperty("peakNonRevocableMemoryReservation") DataSize peakNonRevocableMemoryReservation,
@JsonProperty("peakTotalMemoryReservation") DataSize peakTotalMemoryReservation,
@JsonProperty("peakTaskUserMemory") DataSize peakTaskUserMemory,
@JsonProperty("peakTaskRevocableMemory") DataSize peakTaskRevocableMemory,
Expand Down Expand Up @@ -211,14 +207,11 @@ public QueryStats(
this.completedDrivers = completedDrivers;
checkArgument(cumulativeUserMemory >= 0, "cumulativeUserMemory is negative");
this.cumulativeUserMemory = cumulativeUserMemory;
checkArgument(cumulativeSystemMemory >= 0, "cumulativeSystemMemory is negative");
this.cumulativeSystemMemory = cumulativeSystemMemory;
this.userMemoryReservation = requireNonNull(userMemoryReservation, "userMemoryReservation is null");
this.revocableMemoryReservation = requireNonNull(revocableMemoryReservation, "revocableMemoryReservation is null");
this.totalMemoryReservation = requireNonNull(totalMemoryReservation, "totalMemoryReservation is null");
this.peakUserMemoryReservation = requireNonNull(peakUserMemoryReservation, "peakUserMemoryReservation is null");
this.peakRevocableMemoryReservation = requireNonNull(peakRevocableMemoryReservation, "peakRevocableMemoryReservation is null");
this.peakNonRevocableMemoryReservation = requireNonNull(peakNonRevocableMemoryReservation, "peakNonRevocableMemoryReservation is null");
this.peakTotalMemoryReservation = requireNonNull(peakTotalMemoryReservation, "peakTotalMemoryReservation is null");
this.peakTaskUserMemory = requireNonNull(peakTaskUserMemory, "peakTaskUserMemory is null");
this.peakTaskRevocableMemory = requireNonNull(peakTaskRevocableMemory, "peakTaskRevocableMemory is null");
Expand Down Expand Up @@ -387,12 +380,6 @@ public double getCumulativeUserMemory()
return cumulativeUserMemory;
}

@JsonProperty
public double getCumulativeSystemMemory()
{
return cumulativeSystemMemory;
}

@JsonProperty
public DataSize getUserMemoryReservation()
{
Expand Down Expand Up @@ -423,12 +410,6 @@ public DataSize getPeakRevocableMemoryReservation()
return peakRevocableMemoryReservation;
}

@JsonProperty
public DataSize getPeakNonRevocableMemoryReservation()
{
return peakNonRevocableMemoryReservation;
}

@JsonProperty
public DataSize getPeakTotalMemoryReservation()
{
Expand Down
Loading

0 comments on commit 9ab09e9

Please sign in to comment.