Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add more details in gc history #18779

Merged
merged 2 commits into from
Jun 2, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 44 additions & 1 deletion src/controller/gc/callback.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,11 @@ func init() {
}

if err := task.RegisterTaskStatusChangePostFunc(job.GarbageCollectionVendorType, gcTaskStatusChange); err != nil {
log.Fatalf("failed to register the task status change post for the gc job, error %v", err)
log.Fatalf("failed to register the task status change post for the garbage collection job, error %v", err)
}

if err := task.RegisterCheckInProcessor(job.GarbageCollectionVendorType, gcCheckIn); err != nil {
log.Fatalf("failed to register the checkin processor for the garbage collection job, error %v", err)
}
}

Expand All @@ -60,3 +64,42 @@ func gcTaskStatusChange(ctx context.Context, taskID int64, status string) error

return nil
}

func gcCheckIn(ctx context.Context, t *task.Task, sc *job.StatusChange) error {
taskID := t.ID
status := t.Status

log.Infof("received garbage collection task status update event: task-%d, status-%s", taskID, status)
if sc.CheckIn != "" {
var gcObj struct {
SweepSize int64 `json:"freed_space"`
Blobs int64 `json:"purged_blobs"`
Manifests int64 `json:"purged_manifests"`
}
if err := json.Unmarshal([]byte(sc.CheckIn), &gcObj); err != nil {
log.Errorf("failed to resolve checkin of garbage collection task %d: %v", taskID, err)

return err
}
t, err := task.Mgr.Get(ctx, taskID)
if err != nil {
return err
}

e, err := task.ExecMgr.Get(ctx, t.ExecutionID)
if err != nil {
return err
}

e.ExtraAttrs["freed_space"] = gcObj.SweepSize
e.ExtraAttrs["purged_blobs"] = gcObj.Blobs
e.ExtraAttrs["purged_manifests"] = gcObj.Manifests

err = task.ExecMgr.UpdateExtraAttrs(ctx, e.ID, e.ExtraAttrs)
if err != nil {
log.G(ctx).WithField("error", err).Errorf("failed to update of garbage collection task %d", taskID)
return err
}
}
return nil
}
44 changes: 44 additions & 0 deletions src/controller/gc/callback_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package gc

import (
"context"
"testing"

"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/task"
"github.com/goharbor/harbor/src/testing/mock"
tasktesting "github.com/goharbor/harbor/src/testing/pkg/task"
"github.com/stretchr/testify/suite"
)

type callbackTestSuite struct {
suite.Suite
execMgr *tasktesting.ExecutionManager
taskMgr *tasktesting.Manager
}

func (c *callbackTestSuite) SetupTest() {
c.execMgr = &tasktesting.ExecutionManager{}
c.taskMgr = &tasktesting.Manager{}
}

func (c *callbackTestSuite) TestCheckIn() {
t := &task.Task{
ID: 1,
Status: "Success",
}

sc := &job.StatusChange{
CheckIn: "",
}

c.taskMgr.On("Get", mock.Anything, int64(1)).Return(&task.Task{ID: 1, ExecutionID: 1}, nil)
c.execMgr.On("Get", mock.Anything, mock.Anything).Return(&task.Execution{ID: 1}, nil)
c.execMgr.On("UpdateExtraAttrs", mock.Anything, mock.Anything, mock.Anything).Return(nil)

gcCheckIn(context.Background(), t, sc)
}

func TestCallBackTestSuite(t *testing.T) {
suite.Run(t, &callbackTestSuite{})
}
28 changes: 26 additions & 2 deletions src/jobservice/job/impl/gc/garbage_collection.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package gc

import (
"encoding/json"
"os"
"time"

Expand Down Expand Up @@ -255,8 +256,8 @@ func (gc *GarbageCollector) mark(ctx job.Context) error {
func (gc *GarbageCollector) sweep(ctx job.Context) error {
gc.logger = ctx.GetLogger()
sweepSize := int64(0)
blobCnt := 0
mfCnt := 0
blobCnt := int64(0)
mfCnt := int64(0)
total := len(gc.deleteSet)
for i, blob := range gc.deleteSet {
if gc.shouldStop(ctx) {
Expand Down Expand Up @@ -413,6 +414,11 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error {
}
gc.logger.Infof("%d blobs and %d manifests are actually deleted", blobCnt, mfCnt)
gc.logger.Infof("The GC job actual frees up %d MB space.", sweepSize/1024/1024)

if err := saveGCRes(ctx, sweepSize, blobCnt, mfCnt); err != nil {
gc.logger.Errorf("failed to save the garbage collection results, errMsg=%v", err)
}

return nil
}

Expand Down Expand Up @@ -647,3 +653,21 @@ func (gc *GarbageCollector) shouldStop(ctx job.Context) bool {
}
return false
}

func saveGCRes(ctx job.Context, sweepSize, blobs, manifests int64) error {
gcObj := struct {
SweepSize int64 `json:"freed_space"`
Blobs int64 `json:"purged_blobs"`
Manifests int64 `json:"purged_manifests"`
}{
SweepSize: sweepSize,
Blobs: blobs,
Manifests: manifests,
}
c, err := json.Marshal(gcObj)
if err != nil {
return err
}
_ = ctx.Checkin(string(c))
return nil
}
10 changes: 10 additions & 0 deletions src/jobservice/job/impl/gc/garbage_collection_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ func (suite *gcTestSuite) TestRun() {
ctx.On("GetLogger").Return(logger)
ctx.On("OPCommand").Return(job.NilCommand, true)
mock.OnAnything(ctx, "Get").Return("core url", true)
mock.OnAnything(ctx, "Checkin").Return(nil)

suite.artifactCtl.On("List").Return([]*artifact.Artifact{
{
Expand Down Expand Up @@ -357,6 +358,7 @@ func (suite *gcTestSuite) TestSweep() {
logger := &mockjobservice.MockJobLogger{}
ctx.On("GetLogger").Return(logger)
ctx.On("OPCommand").Return(job.NilCommand, false)
mock.OnAnything(ctx, "Checkin").Return(nil)

mock.OnAnything(suite.blobMgr, "UpdateBlobStatus").Return(int64(1), nil)
mock.OnAnything(suite.blobMgr, "Delete").Return(nil)
Expand All @@ -378,6 +380,14 @@ func (suite *gcTestSuite) TestSweep() {
suite.Nil(gc.sweep(ctx))
}

func (suite *gcTestSuite) TestSaveRes() {
ctx := &mockjobservice.MockJobContext{}
logger := &mockjobservice.MockJobLogger{}
ctx.On("GetLogger").Return(logger)
mock.OnAnything(ctx, "Checkin").Return(nil)
suite.Nil(saveGCRes(ctx, 123456, 100, 100))
wy65701436 marked this conversation as resolved.
Show resolved Hide resolved
}

func TestGCTestSuite(t *testing.T) {
t.Setenv("UTTEST", "true")
suite.Run(t, &gcTestSuite{})
Expand Down