Skip to content

Commit

Permalink
Added tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@diptanu
diptanu committed Dec 19, 2016
1 parent 61e534d commit 6143d8d
Show file tree
Hide file tree
Showing 2 changed files with 305 additions and 5 deletions.
31 changes: 26 additions & 5 deletions client/gc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ const (
// maintain, whenever we are over it we will attempt to GC terminal
// allocations
inodeUsageThreshold = 70

// MB is a constant which converts values in bytes to MB
MB = 1024 * 1024
)

type GCAlloc struct {
Expand Down Expand Up @@ -180,6 +183,11 @@ func (a *AllocGarbageCollector) keepUsageBelowThreshold() error {

// See if we are below thresholds for used disk space and inode usage
diskStats := a.statsCollector.Stats().AllocDirStats

if diskStats == nil {
break
}

if diskStats.UsedPercent <= diskUsageThreshold &&
diskStats.InodesUsedPercent <= inodeUsageThreshold {
break
Expand Down Expand Up @@ -249,18 +257,31 @@ func (a *AllocGarbageCollector) MakeRoomFor(allocations []*structs.Allocation) e

// If the host has enough free space to accomodate the new allocations then
// we don't need to garbage collect terminated allocations
hostStats := a.statsCollector.Stats()
if hostStats != nil && uint64(totalResource.DiskMB*1024*1024) < hostStats.AllocDirStats.Available {
return nil
if hostStats := a.statsCollector.Stats(); hostStats != nil {
var availableForAllocations uint64
if hostStats.AllocDirStats.Available < uint64(a.reservedDiskMB*MB) {
availableForAllocations = 0
} else {
availableForAllocations = hostStats.AllocDirStats.Available - uint64(a.reservedDiskMB*MB)
}
if uint64(totalResource.DiskMB*MB) < availableForAllocations {
return nil
}
}

var diskCleared int
for {
// Collect host stats and see if we still need to remove older
// allocations
var allocDirStats *stats.DiskStats
if err := a.statsCollector.Collect(); err == nil {
hostStats := a.statsCollector.Stats()
if hostStats.AllocDirStats.Available >= uint64(totalResource.DiskMB*1024*1024) {
if hostStats := a.statsCollector.Stats(); hostStats != nil {
allocDirStats = hostStats.AllocDirStats
}
}

if allocDirStats != nil {
if allocDirStats.Available >= uint64(totalResource.DiskMB*1024*1024) {
break
}
} else {
Expand Down
279 changes: 279 additions & 0 deletions client/gc_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
package client

import (
"log"
"os"
"testing"

"github.com/hashicorp/nomad/client/stats"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/nomad/structs"
)

func TestIndexedGCAllocPQ(t *testing.T) {
Expand Down Expand Up @@ -44,3 +48,278 @@ func TestIndexedGCAllocPQ(t *testing.T) {
t.Fatalf("expected nil, got %v", gcAlloc)
}
}

type MockStatsCollector struct {
availableValues []uint64
usedPercents []float64
inodePercents []float64
index int
}

func (m *MockStatsCollector) Collect() error {
return nil
}

func (m *MockStatsCollector) Stats() *stats.HostStats {
if len(m.availableValues) == 0 {
return nil
}

available := m.availableValues[m.index]
usedPercent := m.usedPercents[m.index]
inodePercent := m.inodePercents[m.index]

if m.index < len(m.availableValues)-1 {
m.index = m.index + 1
}
return &stats.HostStats{
AllocDirStats: &stats.DiskStats{
Available: available,
UsedPercent: usedPercent,
InodesUsedPercent: inodePercent,
},
}
}

func TestAllocGarbageCollector_MarkForCollection(t *testing.T) {
logger := log.New(os.Stdout, "", 0)
gc := NewAllocGarbageCollector(logger, &MockStatsCollector{}, 0)

_, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false)
if err := gc.MarkForCollection(ar1); err != nil {
t.Fatalf("err: %v", err)
}

gcAlloc := gc.allocRunners.Pop()
if gcAlloc == nil || gcAlloc.allocRunner != ar1 {
t.Fatalf("bad gcAlloc: %v", gcAlloc)
}
}

func TestAllocGarbageCollector_Collect(t *testing.T) {
logger := log.New(os.Stdout, "", 0)
gc := NewAllocGarbageCollector(logger, &MockStatsCollector{}, 0)

_, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false)
_, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false)
if err := gc.MarkForCollection(ar1); err != nil {
t.Fatalf("err: %v", err)
}
if err := gc.MarkForCollection(ar2); err != nil {
t.Fatalf("err: %v", err)
}

if err := gc.Collect(ar1.Alloc().ID); err != nil {
t.Fatalf("err: %v", err)
}
gcAlloc := gc.allocRunners.Pop()
if gcAlloc == nil || gcAlloc.allocRunner != ar2 {
t.Fatalf("bad gcAlloc: %v", gcAlloc)
}
}

func TestAllocGarbageCollector_CollectAll(t *testing.T) {
logger := log.New(os.Stdout, "", 0)
gc := NewAllocGarbageCollector(logger, &MockStatsCollector{}, 0)

_, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false)
_, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false)
if err := gc.MarkForCollection(ar1); err != nil {
t.Fatalf("err: %v", err)
}
if err := gc.MarkForCollection(ar2); err != nil {
t.Fatalf("err: %v", err)
}

if err := gc.CollectAll(); err != nil {
t.Fatalf("err: %v", err)
}
gcAlloc := gc.allocRunners.Pop()
if gcAlloc != nil {
t.Fatalf("bad gcAlloc: %v", gcAlloc)
}
}

func TestAllocGarbageCollector_MakeRoomForAllocations_EnoughSpace(t *testing.T) {
logger := log.New(os.Stdout, "", 0)
statsCollector := &MockStatsCollector{}
gc := NewAllocGarbageCollector(logger, statsCollector, 20)

_, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false)
close(ar1.waitCh)
_, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false)
close(ar2.waitCh)
if err := gc.MarkForCollection(ar1); err != nil {
t.Fatalf("err: %v", err)
}
if err := gc.MarkForCollection(ar2); err != nil {
t.Fatalf("err: %v", err)
}

// Make stats collector report 200MB free out of which 20MB is reserved
statsCollector.availableValues = []uint64{200 * MB}
statsCollector.usedPercents = []float64{0}
statsCollector.inodePercents = []float64{0}

alloc := mock.Alloc()
if err := gc.MakeRoomFor([]*structs.Allocation{alloc}); err != nil {
t.Fatalf("err: %v", err)
}

// When we have enough disk available and don't need to do any GC so we
// should have two ARs in the GC queue
for i := 0; i < 2; i++ {
if gcAlloc := gc.allocRunners.Pop(); gcAlloc == nil {
t.Fatalf("err: %v", gcAlloc)
}
}
}

func TestAllocGarbageCollector_MakeRoomForAllocations_GC_Partial(t *testing.T) {
logger := log.New(os.Stdout, "", 0)
statsCollector := &MockStatsCollector{}
gc := NewAllocGarbageCollector(logger, statsCollector, 20)

_, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false)
close(ar1.waitCh)
_, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false)
close(ar2.waitCh)
if err := gc.MarkForCollection(ar1); err != nil {
t.Fatalf("err: %v", err)
}
if err := gc.MarkForCollection(ar2); err != nil {
t.Fatalf("err: %v", err)
}

// Make stats collector report 80MB and 175MB free in subsequent calls
statsCollector.availableValues = []uint64{80 * MB, 80 * MB, 175 * MB}
statsCollector.usedPercents = []float64{0, 0, 0}
statsCollector.inodePercents = []float64{0, 0, 0}

alloc := mock.Alloc()
if err := gc.MakeRoomFor([]*structs.Allocation{alloc}); err != nil {
t.Fatalf("err: %v", err)
}

// We should be GC-ing one alloc
if gcAlloc := gc.allocRunners.Pop(); gcAlloc == nil {
t.Fatalf("err: %v", gcAlloc)
}

if gcAlloc := gc.allocRunners.Pop(); gcAlloc != nil {
t.Fatalf("gcAlloc: %v", gcAlloc)
}
}

func TestAllocGarbageCollector_MakeRoomForAllocations_GC_All(t *testing.T) {
logger := log.New(os.Stdout, "", 0)
statsCollector := &MockStatsCollector{}
gc := NewAllocGarbageCollector(logger, statsCollector, 20)

_, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false)
close(ar1.waitCh)
_, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false)
close(ar2.waitCh)
if err := gc.MarkForCollection(ar1); err != nil {
t.Fatalf("err: %v", err)
}
if err := gc.MarkForCollection(ar2); err != nil {
t.Fatalf("err: %v", err)
}

// Make stats collector report 80MB and 95MB free in subsequent calls
statsCollector.availableValues = []uint64{80 * MB, 80 * MB, 95 * MB}
statsCollector.usedPercents = []float64{0, 0, 0}
statsCollector.inodePercents = []float64{0, 0, 0}

alloc := mock.Alloc()
if err := gc.MakeRoomFor([]*structs.Allocation{alloc}); err != nil {
t.Fatalf("err: %v", err)
}

// We should be GC-ing all the alloc runners
if gcAlloc := gc.allocRunners.Pop(); gcAlloc != nil {
t.Fatalf("gcAlloc: %v", gcAlloc)
}
}

func TestAllocGarbageCollector_MakeRoomForAllocations_GC_Fallback(t *testing.T) {
logger := log.New(os.Stdout, "", 0)
statsCollector := &MockStatsCollector{}
gc := NewAllocGarbageCollector(logger, statsCollector, 20)

_, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false)
close(ar1.waitCh)
_, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false)
close(ar2.waitCh)
if err := gc.MarkForCollection(ar1); err != nil {
t.Fatalf("err: %v", err)
}
if err := gc.MarkForCollection(ar2); err != nil {
t.Fatalf("err: %v", err)
}

alloc := mock.Alloc()
if err := gc.MakeRoomFor([]*structs.Allocation{alloc}); err != nil {
t.Fatalf("err: %v", err)
}

// We should be GC-ing one alloc
if gcAlloc := gc.allocRunners.Pop(); gcAlloc == nil {
t.Fatalf("err: %v", gcAlloc)
}

if gcAlloc := gc.allocRunners.Pop(); gcAlloc != nil {
t.Fatalf("gcAlloc: %v", gcAlloc)
}
}

func TestAllocGarbageCollector_UsageBelowThreshold(t *testing.T) {
logger := log.New(os.Stdout, "", 0)
statsCollector := &MockStatsCollector{}
gc := NewAllocGarbageCollector(logger, statsCollector, 20)

_, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false)
close(ar1.waitCh)
_, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false)
close(ar2.waitCh)
if err := gc.MarkForCollection(ar1); err != nil {
t.Fatalf("err: %v", err)
}
if err := gc.MarkForCollection(ar2); err != nil {
t.Fatalf("err: %v", err)
}

statsCollector.availableValues = []uint64{1000}
statsCollector.usedPercents = []float64{20}
statsCollector.inodePercents = []float64{10}

if err := gc.keepUsageBelowThreshold(); err != nil {
t.Fatalf("err: %v", err)
}
}

func TestAllocGarbageCollector_UsedPercentThreshold(t *testing.T) {
logger := log.New(os.Stdout, "", 0)
statsCollector := &MockStatsCollector{}
gc := NewAllocGarbageCollector(logger, statsCollector, 20)

_, ar1 := testAllocRunnerFromAlloc(mock.Alloc(), false)
close(ar1.waitCh)
_, ar2 := testAllocRunnerFromAlloc(mock.Alloc(), false)
close(ar2.waitCh)
if err := gc.MarkForCollection(ar1); err != nil {
t.Fatalf("err: %v", err)
}
if err := gc.MarkForCollection(ar2); err != nil {
t.Fatalf("err: %v", err)
}

statsCollector.availableValues = []uint64{1000, 800}
statsCollector.usedPercents = []float64{85, 60}
statsCollector.inodePercents = []float64{50, 30}

if err := gc.keepUsageBelowThreshold(); err != nil {
t.Fatalf("err: %v", err)
}
}

0 comments on commit 6143d8d

Please sign in to comment.