Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(bigtable): add GC policy to FamilyInfo. #5993

Closed
Closed
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
dc6e491
Add GC policy proto to FamilyInfo.
hoangpham95 May 4, 2022
736e1bf
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 May 4, 2022
265d7f6
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 May 5, 2022
1861f7f
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 May 9, 2022
992dbff
Add PolicyType and expose GCPolicyObject.
hoangpham95 May 9, 2022
dc64bdd
Add PolicyType alias to int.
hoangpham95 May 9, 2022
5741a5a
Add new TypedGCPolicy to prevent breaking changes
hoangpham95 May 12, 2022
f7a68fc
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 May 12, 2022
a215371
Remove unused methods
hoangpham95 May 12, 2022
ee17e07
Address PR comments.
hoangpham95 May 16, 2022
c2eff7c
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 May 16, 2022
f93358d
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion May 16, 2022
3b2c864
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 2, 2022
6597b60
Change variable name to Unspecified instead of Unknown.
hoangpham95 Jun 2, 2022
bace825
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 2, 2022
61dcb35
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 4, 2022
696172a
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 7, 2022
8eb8f75
Fix go fmt.
hoangpham95 Jun 7, 2022
f3df3da
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 7, 2022
330fecb
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 8, 2022
8c1b978
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 8, 2022
71fb066
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 9, 2022
0e2a815
Address build failure.
hoangpham95 Jun 10, 2022
0d12c0a
Add FullGCPolicy to admin.go.
hoangpham95 Jun 13, 2022
d10c1cf
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 13, 2022
674f607
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 13, 2022
83108bc
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 13, 2022
778031f
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 14, 2022
d30b02f
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 14, 2022
b19cdbc
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 16, 2022
cf1fae8
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 16, 2022
b2f7785
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 16, 2022
de859e9
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 16, 2022
849e7fb
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 17, 2022
444f66b
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 20, 2022
e7f871d
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 21, 2022
149d191
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 21, 2022
4b9651b
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 21, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions bigtable/admin.go
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ type TableInfo struct {
type FamilyInfo struct {
Name string
GCPolicy string
GCPolicyObject GCPolicy
TypedGCPolicy TypedGCPolicy
}

func (ac *AdminClient) getTable(ctx context.Context, table string, view btapb.Table_View) (*btapb.Table, error) {
Expand Down Expand Up @@ -351,7 +351,7 @@ func (ac *AdminClient) TableInfo(ctx context.Context, table string) (*TableInfo,
ti.FamilyInfos = append(ti.FamilyInfos, FamilyInfo{
Name: name,
GCPolicy: GCRuleToString(fam.GcRule),
GCPolicyObject: GCRuleToPolicy(fam.GcRule),
TypedGCPolicy: fromPolicyToTypedPolicy(gcRuleToPolicy(fam.GcRule)),
})
}
return ti, nil
Expand Down
71 changes: 35 additions & 36 deletions bigtable/gc.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,21 +25,25 @@ import (
bttdpb "google.golang.org/genproto/googleapis/bigtable/admin/v2"
)

type PolicyType = int
hoangpham95 marked this conversation as resolved.
Show resolved Hide resolved

const (
PolicyType_None = iota
PolicyType_MaxAge
PolicyType_MaxVersion
PolicyType_Union
PolicyType_Intersection
PolicyTypeNone PolicyType = iota
hoangpham95 marked this conversation as resolved.
Show resolved Hide resolved
PolicyTypeMaxAge
PolicyTypeMaxVersion
PolicyTypeUnion
PolicyTypeIntersection
)

type PolicyType = int

// A GCPolicy represents a rule that determines which cells are eligible for garbage collection.
type GCPolicy interface {
String() string
proto() *bttdpb.GcRule
PolicyType() PolicyType
}

type TypedGCPolicy struct {
GCPolicy
PolicyType PolicyType
}

// IntersectionPolicy returns a GC policy that only applies when all its sub-policies apply.
Expand Down Expand Up @@ -67,10 +71,6 @@ func (ip intersectionPolicy) proto() *bttdpb.GcRule {
}
}

func (intersectionPolicy) PolicyType() PolicyType {
return PolicyType_Intersection
}

// UnionPolicy returns a GC policy that applies when any of its sub-policies apply.
func UnionPolicy(sub ...GCPolicy) GCPolicy { return unionPolicy{sub} }

Expand All @@ -96,10 +96,6 @@ func (up unionPolicy) proto() *bttdpb.GcRule {
}
}

func (unionPolicy) PolicyType() PolicyType {
return PolicyType_Union
}

// MaxVersionsPolicy returns a GC policy that applies to all versions of a cell
// except for the most recent n.
func MaxVersionsPolicy(n int) GCPolicy { return maxVersionsPolicy(n) }
Expand All @@ -112,10 +108,6 @@ func (mvp maxVersionsPolicy) proto() *bttdpb.GcRule {
return &bttdpb.GcRule{Rule: &bttdpb.GcRule_MaxNumVersions{MaxNumVersions: int32(mvp)}}
}

func (maxVersionsPolicy) PolicyType() PolicyType {
return PolicyType_MaxVersion
}

// MaxAgePolicy returns a GC policy that applies to all cells
// older than the given age.
func MaxAgePolicy(d time.Duration) GCPolicy { return maxAgePolicy(d) }
Expand Down Expand Up @@ -153,20 +145,12 @@ func (ma maxAgePolicy) proto() *bttdpb.GcRule {
}
}

func (maxAgePolicy) PolicyType() PolicyType {
return PolicyType_MaxAge
}

type noGCPolicy struct{}

func (n noGCPolicy) String() string { return "" }

func (n noGCPolicy) proto() *bttdpb.GcRule { return &bttdpb.GcRule{Rule: nil} }

func (n noGCPolicy) PolicyType() PolicyType {
return PolicyType_None
}

// NoGcPolicy applies to all cells setting maxage and maxversions to nil implies no gc policies
func NoGcPolicy() GCPolicy { return noGCPolicy{} }

Expand All @@ -189,15 +173,15 @@ func GCRuleToString(rule *bttdpb.GcRule) string {
}
}

func GCRuleToPolicy(rule *bttdpb.GcRule) GCPolicy {
func gcRuleToPolicy(rule *bttdpb.GcRule) GCPolicy {
if rule == nil {
return NoGcPolicy()
}
switch r := rule.Rule.(type) {
case *bttdpb.GcRule_Intersection_:
return fromRuleToPolicy(r.Intersection.Rules, PolicyType_Intersection)
return compoundRuleToPolicy(r.Intersection.Rules, PolicyTypeIntersection)
case *bttdpb.GcRule_Union_:
return fromRuleToPolicy(r.Union.Rules, PolicyType_Union)
return compoundRuleToPolicy(r.Union.Rules, PolicyTypeUnion)
case *bttdpb.GcRule_MaxAge:
return MaxAgePolicy(time.Duration(r.MaxAge.Seconds) * time.Second)
case *bttdpb.GcRule_MaxNumVersions:
Expand All @@ -215,21 +199,36 @@ func joinRules(rules []*bttdpb.GcRule, sep string) string {
return "(" + strings.Join(chunks, sep) + ")"
}

func fromRuleToPolicy(rules []*bttdpb.GcRule, mode PolicyType) GCPolicy {
func compoundRuleToPolicy(rules []*bttdpb.GcRule, mode PolicyType) GCPolicy {
sub := []GCPolicy{}
for _, r := range rules {
p := GCRuleToPolicy(r)
p := gcRuleToPolicy(r)
if p.String() != "" {
sub = append(sub, GCRuleToPolicy(r))
sub = append(sub, gcRuleToPolicy(r))
}
}

switch mode {
case PolicyType_Union:
case PolicyTypeUnion:
return unionPolicy{sub: sub}
case PolicyType_Intersection:
case PolicyTypeIntersection:
return intersectionPolicy{sub: sub}
default:
return NoGcPolicy()
}
}

func fromPolicyToTypedPolicy(gcPolicy GCPolicy) TypedGCPolicy {
switch gcPolicy.proto().Rule.(type) {
case *bttdpb.GcRule_Intersection_:
return TypedGCPolicy{GCPolicy: gcPolicy, PolicyType: PolicyTypeIntersection}
case *bttdpb.GcRule_Union_:
return TypedGCPolicy{GCPolicy: gcPolicy, PolicyType: PolicyTypeUnion}
case *bttdpb.GcRule_MaxAge:
return TypedGCPolicy{GCPolicy: gcPolicy, PolicyType: PolicyTypeMaxAge}
case *bttdpb.GcRule_MaxNumVersions:
return TypedGCPolicy{GCPolicy: gcPolicy, PolicyType: PolicyTypeMaxVersion}
default:
return TypedGCPolicy{GCPolicy: gcPolicy, PolicyType: PolicyTypeNone}
}
}
2 changes: 1 addition & 1 deletion bigtable/gc_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func TestGCRuleToPolicy(t *testing.T) {
}

for _, test := range tc {
got := GCRuleToPolicy(test.proto)
got := gcRuleToPolicy(test.proto)
if got.String() != test.want {
t.Errorf("got gc rule %v, want: %v", got, test.want)
}
Expand Down