Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

compaction: incorporate L0 size into compaction priority #1624

Closed
wants to merge 1 commit into from
Closed

compaction: incorporate L0 size into compaction priority #1624

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 15 additions & 17 deletions compaction_picker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -717,6 +717,8 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp
p.estimatedMaxWAmp = float64(numLevels-p.baseLevel) * (smoothedLevelMultiplier + 1)

levelSize := float64(baseBytesMax)
// L0 also uses LBaseMaxBytes as its max size.
p.levelMaxBytes[0] = baseBytesMax
for level := p.baseLevel; level < numLevels; level++ {
if level > p.baseLevel && levelSize > 0 {
levelSize *= smoothedLevelMultiplier
Expand Down Expand Up @@ -770,15 +772,25 @@ func (p *compactionPickerByScore) calculateScores(
scores[i].level = i
scores[i].outputLevel = i + 1
}
scores[0] = p.calculateL0Score(inProgressCompactions)

sizeAdjust := calculateSizeAdjust(inProgressCompactions)
for level := 1; level < numLevels; level++ {
for level := 0; level < numLevels; level++ {
levelSize := int64(levelCompensatedSize(p.vers.Levels[level])) + sizeAdjust[level]
scores[level].score = float64(levelSize) / float64(p.levelMaxBytes[level])
scores[level].origScore = scores[level].score
}

// Calculate a L0 score based on the sublevel count. The base vs intra-L0
// compaction determination happens in pickAuto, not here. If the
// sublevel-based score is higher than the size-based score, use the
// sublevel-based score.
sublevelScore := float64(2*p.vers.L0Sublevels.MaxDepthAfterOngoingCompactions()) /
float64(p.opts.L0CompactionThreshold)
if sublevelScore > scores[0].score {
scores[0].score = sublevelScore
scores[0].origScore = sublevelScore
}
scores[0].outputLevel = p.baseLevel

// Adjust each level's score by the score of the next level. If the next
// level has a high score, and is thus a priority for compaction, this
// reduces the priority for compacting the current level. If the next level
Expand Down Expand Up @@ -819,20 +831,6 @@ func (p *compactionPickerByScore) calculateScores(
return scores
}

func (p *compactionPickerByScore) calculateL0Score(
inProgressCompactions []compactionInfo,
) candidateLevelInfo {
var info candidateLevelInfo
info.outputLevel = p.baseLevel

// If L0Sublevels are present, we use the sublevel count as opposed to
// the L0 file count to score this level. The base vs intra-L0
// compaction determination happens in pickAuto, not here.
info.score = float64(2*p.vers.L0Sublevels.MaxDepthAfterOngoingCompactions()) /
float64(p.opts.L0CompactionThreshold)
return info
}

func (p *compactionPickerByScore) pickFile(
level, outputLevel int, earliestSnapshotSeqNum uint64,
) (manifest.LevelFile, bool) {
Expand Down
10 changes: 5 additions & 5 deletions testdata/compaction_picker_target_level
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ L4->L5: 7.7

pick ongoing=(5,6)
----
no compaction
L0->L4: 1.0

pick ongoing=(0,4)
----
Expand Down Expand Up @@ -224,11 +224,11 @@ base: 4

queue
----
L0->L4: 1000.0
L0->L4: 2000.0

pick
----
L0->L4: 1000.0
L0->L4: 2000.0

pick ongoing=(0,4)
----
Expand All @@ -249,15 +249,15 @@ base: 4

queue
----
L0->L4: 1000.0
L0->L4: 2000.0

pick ongoing=(0,4,4,5)
----
no compaction

pick ongoing=(4,5)
----
L0->L4: 1000.0
L0->L4: 2000.0

# Verify we can start concurrent Ln->Ln+1 compactions given sufficient
# priority.
Expand Down