diff --git a/client/gc.go b/client/gc.go index 2276b56c75f..190e7bbff80 100644 --- a/client/gc.go +++ b/client/gc.go @@ -23,6 +23,9 @@ const ( // maintain, whenever we are over it we will attempt to GC terminal // allocations inodeUsageThreshold = 70 + + // MB is a constant which converts values in bytes to MB + MB = 1024 * 1024 ) type GCAlloc struct { @@ -180,6 +183,11 @@ func (a *AllocGarbageCollector) keepUsageBelowThreshold() error { // See if we are below thresholds for used disk space and inode usage diskStats := a.statsCollector.Stats().AllocDirStats + + if diskStats == nil { + break + } + if diskStats.UsedPercent <= diskUsageThreshold && diskStats.InodesUsedPercent <= inodeUsageThreshold { break @@ -249,18 +257,31 @@ func (a *AllocGarbageCollector) MakeRoomFor(allocations []*structs.Allocation) e // If the host has enough free space to accomodate the new allocations then // we don't need to garbage collect terminated allocations - hostStats := a.statsCollector.Stats() - if hostStats != nil && uint64(totalResource.DiskMB*1024*1024) < hostStats.AllocDirStats.Available { - return nil + if hostStats := a.statsCollector.Stats(); hostStats != nil { + var availableForAllocations uint64 + if hostStats.AllocDirStats.Available < uint64(a.reservedDiskMB*MB) { + availableForAllocations = 0 + } else { + availableForAllocations = hostStats.AllocDirStats.Available - uint64(a.reservedDiskMB*MB) + } + if uint64(totalResource.DiskMB*MB) < availableForAllocations { + return nil + } } var diskCleared int for { // Collect host stats and see if we still need to remove older // allocations + var allocDirStats *stats.DiskStats if err := a.statsCollector.Collect(); err == nil { - hostStats := a.statsCollector.Stats() - if hostStats.AllocDirStats.Available >= uint64(totalResource.DiskMB*1024*1024) { + if hostStats := a.statsCollector.Stats(); hostStats != nil { + allocDirStats = hostStats.AllocDirStats + } + } + + if allocDirStats != nil { + if allocDirStats.Available >= uint64(totalResource.DiskMB*1024*1024) { break } } else { diff --git a/client/gc_test.go b/client/gc_test.go index c85b0ccd22f..5e3572023d5 100644 --- a/client/gc_test.go +++ b/client/gc_test.go @@ -1,9 +1,13 @@ package client import ( + "log" + "os" "testing" + "github.com/hashicorp/nomad/client/stats" "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" ) func TestIndexedGCAllocPQ(t *testing.T) { @@ -44,3 +48,278 @@ func TestIndexedGCAllocPQ(t *testing.T) { t.Fatalf("expected nil, got %v", gcAlloc) } } + +type MockStatsCollector struct { + availableValues []uint64 + usedPercents []float64 + inodePercents []float64 + index int +} + +func (m *MockStatsCollector) Collect() error { + return nil +} + +func (m *MockStatsCollector) Stats() *stats.HostStats { + if len(m.availableValues) == 0 { + return nil + } + + available := m.availableValues[m.index] + usedPercent := m.usedPercents[m.index] + inodePercent := m.inodePercents[m.index] + + if m.index < len(m.availableValues)-1 { + m.index = m.index + 1 + } + return &stats.HostStats{ + AllocDirStats: &stats.DiskStats{ + Available: available, + UsedPercent: usedPercent, + InodesUsedPercent: inodePercent, + }, + } +} + +func TestAllocGarbageCollector_MarkForCollection(t *testing.T) { + logger := log.New(os.Stdout, "", 0) + gc := NewAllocGarbageCollector(logger, &MockStatsCollector{}, 0) + + _, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false) + if err := gc.MarkForCollection(ar1); err != nil { + t.Fatalf("err: %v", err) + } + + gcAlloc := gc.allocRunners.Pop() + if gcAlloc == nil || gcAlloc.allocRunner != ar1 { + t.Fatalf("bad gcAlloc: %v", gcAlloc) + } +} + +func TestAllocGarbageCollector_Collect(t *testing.T) { + logger := log.New(os.Stdout, "", 0) + gc := NewAllocGarbageCollector(logger, &MockStatsCollector{}, 0) + + _, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false) + _, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false) + if err := gc.MarkForCollection(ar1); err != nil { + t.Fatalf("err: %v", err) + } + if err := gc.MarkForCollection(ar2); err != nil { + t.Fatalf("err: %v", err) + } + + if err := gc.Collect(ar1.Alloc().ID); err != nil { + t.Fatalf("err: %v", err) + } + gcAlloc := gc.allocRunners.Pop() + if gcAlloc == nil || gcAlloc.allocRunner != ar2 { + t.Fatalf("bad gcAlloc: %v", gcAlloc) + } +} + +func TestAllocGarbageCollector_CollectAll(t *testing.T) { + logger := log.New(os.Stdout, "", 0) + gc := NewAllocGarbageCollector(logger, &MockStatsCollector{}, 0) + + _, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false) + _, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false) + if err := gc.MarkForCollection(ar1); err != nil { + t.Fatalf("err: %v", err) + } + if err := gc.MarkForCollection(ar2); err != nil { + t.Fatalf("err: %v", err) + } + + if err := gc.CollectAll(); err != nil { + t.Fatalf("err: %v", err) + } + gcAlloc := gc.allocRunners.Pop() + if gcAlloc != nil { + t.Fatalf("bad gcAlloc: %v", gcAlloc) + } +} + +func TestAllocGarbageCollector_MakeRoomForAllocations_EnoughSpace(t *testing.T) { + logger := log.New(os.Stdout, "", 0) + statsCollector := &MockStatsCollector{} + gc := NewAllocGarbageCollector(logger, statsCollector, 20) + + _, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false) + close(ar1.waitCh) + _, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false) + close(ar2.waitCh) + if err := gc.MarkForCollection(ar1); err != nil { + t.Fatalf("err: %v", err) + } + if err := gc.MarkForCollection(ar2); err != nil { + t.Fatalf("err: %v", err) + } + + // Make stats collector report 200MB free out of which 20MB is reserved + statsCollector.availableValues = []uint64{200 * MB} + statsCollector.usedPercents = []float64{0} + statsCollector.inodePercents = []float64{0} + + alloc := mock.Alloc() + if err := gc.MakeRoomFor([]*structs.Allocation{alloc}); err != nil { + t.Fatalf("err: %v", err) + } + + // When we have enough disk available and don't need to do any GC so we + // should have two ARs in the GC queue + for i := 0; i < 2; i++ { + if gcAlloc := gc.allocRunners.Pop(); gcAlloc == nil { + t.Fatalf("err: %v", gcAlloc) + } + } +} + +func TestAllocGarbageCollector_MakeRoomForAllocations_GC_Partial(t *testing.T) { + logger := log.New(os.Stdout, "", 0) + statsCollector := &MockStatsCollector{} + gc := NewAllocGarbageCollector(logger, statsCollector, 20) + + _, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false) + close(ar1.waitCh) + _, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false) + close(ar2.waitCh) + if err := gc.MarkForCollection(ar1); err != nil { + t.Fatalf("err: %v", err) + } + if err := gc.MarkForCollection(ar2); err != nil { + t.Fatalf("err: %v", err) + } + + // Make stats collector report 80MB and 175MB free in subsequent calls + statsCollector.availableValues = []uint64{80 * MB, 80 * MB, 175 * MB} + statsCollector.usedPercents = []float64{0, 0, 0} + statsCollector.inodePercents = []float64{0, 0, 0} + + alloc := mock.Alloc() + if err := gc.MakeRoomFor([]*structs.Allocation{alloc}); err != nil { + t.Fatalf("err: %v", err) + } + + // We should be GC-ing one alloc + if gcAlloc := gc.allocRunners.Pop(); gcAlloc == nil { + t.Fatalf("err: %v", gcAlloc) + } + + if gcAlloc := gc.allocRunners.Pop(); gcAlloc != nil { + t.Fatalf("gcAlloc: %v", gcAlloc) + } +} + +func TestAllocGarbageCollector_MakeRoomForAllocations_GC_All(t *testing.T) { + logger := log.New(os.Stdout, "", 0) + statsCollector := &MockStatsCollector{} + gc := NewAllocGarbageCollector(logger, statsCollector, 20) + + _, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false) + close(ar1.waitCh) + _, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false) + close(ar2.waitCh) + if err := gc.MarkForCollection(ar1); err != nil { + t.Fatalf("err: %v", err) + } + if err := gc.MarkForCollection(ar2); err != nil { + t.Fatalf("err: %v", err) + } + + // Make stats collector report 80MB and 95MB free in subsequent calls + statsCollector.availableValues = []uint64{80 * MB, 80 * MB, 95 * MB} + statsCollector.usedPercents = []float64{0, 0, 0} + statsCollector.inodePercents = []float64{0, 0, 0} + + alloc := mock.Alloc() + if err := gc.MakeRoomFor([]*structs.Allocation{alloc}); err != nil { + t.Fatalf("err: %v", err) + } + + // We should be GC-ing all the alloc runners + if gcAlloc := gc.allocRunners.Pop(); gcAlloc != nil { + t.Fatalf("gcAlloc: %v", gcAlloc) + } +} + +func TestAllocGarbageCollector_MakeRoomForAllocations_GC_Fallback(t *testing.T) { + logger := log.New(os.Stdout, "", 0) + statsCollector := &MockStatsCollector{} + gc := NewAllocGarbageCollector(logger, statsCollector, 20) + + _, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false) + close(ar1.waitCh) + _, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false) + close(ar2.waitCh) + if err := gc.MarkForCollection(ar1); err != nil { + t.Fatalf("err: %v", err) + } + if err := gc.MarkForCollection(ar2); err != nil { + t.Fatalf("err: %v", err) + } + + alloc := mock.Alloc() + if err := gc.MakeRoomFor([]*structs.Allocation{alloc}); err != nil { + t.Fatalf("err: %v", err) + } + + // We should be GC-ing one alloc + if gcAlloc := gc.allocRunners.Pop(); gcAlloc == nil { + t.Fatalf("err: %v", gcAlloc) + } + + if gcAlloc := gc.allocRunners.Pop(); gcAlloc != nil { + t.Fatalf("gcAlloc: %v", gcAlloc) + } +} + +func TestAllocGarbageCollector_UsageBelowThreshold(t *testing.T) { + logger := log.New(os.Stdout, "", 0) + statsCollector := &MockStatsCollector{} + gc := NewAllocGarbageCollector(logger, statsCollector, 20) + + _, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false) + close(ar1.waitCh) + _, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false) + close(ar2.waitCh) + if err := gc.MarkForCollection(ar1); err != nil { + t.Fatalf("err: %v", err) + } + if err := gc.MarkForCollection(ar2); err != nil { + t.Fatalf("err: %v", err) + } + + statsCollector.availableValues = []uint64{1000} + statsCollector.usedPercents = []float64{20} + statsCollector.inodePercents = []float64{10} + + if err := gc.keepUsageBelowThreshold(); err != nil { + t.Fatalf("err: %v", err) + } +} + +func TestAllocGarbageCollector_UsedPercentThreshold(t *testing.T) { + logger := log.New(os.Stdout, "", 0) + statsCollector := &MockStatsCollector{} + gc := NewAllocGarbageCollector(logger, statsCollector, 20) + + _, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false) + close(ar1.waitCh) + _, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false) + close(ar2.waitCh) + if err := gc.MarkForCollection(ar1); err != nil { + t.Fatalf("err: %v", err) + } + if err := gc.MarkForCollection(ar2); err != nil { + t.Fatalf("err: %v", err) + } + + statsCollector.availableValues = []uint64{1000, 800} + statsCollector.usedPercents = []float64{85, 60} + statsCollector.inodePercents = []float64{50, 30} + + if err := gc.keepUsageBelowThreshold(); err != nil { + t.Fatalf("err: %v", err) + } +}