Skip to content

Commit

Permalink
Merge branch 'master' into segment-type
Browse files Browse the repository at this point in the history 
* master:
  Add evaluation benchmarks (#185)
  • Loading branch information
@markphelps
markphelps committed Nov 23, 2019
2 parents 60ed6ab + 0b111ec commit 1e2344d
Show file tree
Hide file tree
Showing 3 changed files with 330 additions and 1 deletion.
5 changes: 5 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ setup: ## Install dev tools
@echo ">> installing dev tools"
go install -v $(TOOLS)

.PHONY: bench
bench: ## Run all the benchmarks
@echo ">> running benchmarks"
go test -v -bench=. $(SOURCE_FILES) -run=XXX

.PHONY: test
test: ## Run all the tests
@echo ">> running tests"
Expand Down
2 changes: 1 addition & 1 deletion storage/db_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,10 @@ func TestMain(m *testing.M) {

func run(m *testing.M) int {
logger = logrus.New()
logger.Level = logrus.DebugLevel

debug := os.Getenv("DEBUG")
if debug == "" {
logger.Level = logrus.DebugLevel
logger.Out = ioutil.Discard
}

Expand Down
324 changes: 324 additions & 0 deletions storage/evaluator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1974,3 +1974,327 @@ func Test_evaluate(t *testing.T) {
assert.Empty(t, d)
})
}

func BenchmarkEvaluate_SingleVariantDistribution(b *testing.B) {
flag, _ := flagStore.CreateFlag(context.TODO(), &flipt.CreateFlagRequest{
Key: b.Name(),
Name: b.Name(),
Description: "foo",
Enabled: true,
})

var variants []*flipt.Variant

for _, req := range []*flipt.CreateVariantRequest{
{
FlagKey: flag.Key,
Key: fmt.Sprintf("foo_%s", b.Name()),
},
{
FlagKey: flag.Key,
Key: fmt.Sprintf("bar_%s", b.Name()),
},
} {
variant, _ := flagStore.CreateVariant(context.TODO(), req)
variants = append(variants, variant)
}

segment, _ := segmentStore.CreateSegment(context.TODO(), &flipt.CreateSegmentRequest{
Key: b.Name(),
Name: b.Name(),
Description: "foo",
})

_, _ = segmentStore.CreateConstraint(context.TODO(), &flipt.CreateConstraintRequest{
SegmentKey: segment.Key,
Type: flipt.ComparisonType_STRING_COMPARISON_TYPE,
Property: "bar",
Operator: opEQ,
Value: "baz",
})

_, _ = segmentStore.CreateConstraint(context.TODO(), &flipt.CreateConstraintRequest{
SegmentKey: segment.Key,
Type: flipt.ComparisonType_BOOLEAN_COMPARISON_TYPE,
Property: "admin",
Operator: opTrue,
})

rule, _ := ruleStore.CreateRule(context.TODO(), &flipt.CreateRuleRequest{
FlagKey: flag.Key,
SegmentKey: segment.Key,
})

_, _ = ruleStore.CreateDistribution(context.TODO(), &flipt.CreateDistributionRequest{
FlagKey: flag.Key,
RuleId: rule.Id,
VariantId: variants[0].Id,
Rollout: 100,
})

runs := []struct {
name string
req *flipt.EvaluationRequest
}{
{
name: "match string value",
req: &flipt.EvaluationRequest{
FlagKey: flag.Key,
EntityId: "1",
Context: map[string]string{
"bar": "baz",
"admin": "true",
},
},
},
{
name: "no match string value",
req: &flipt.EvaluationRequest{
FlagKey: flag.Key,
EntityId: "1",
Context: map[string]string{
"bar": "boz",
"admin": "true",
},
},
},
{
name: "no match just bool value",
req: &flipt.EvaluationRequest{
FlagKey: flag.Key,
EntityId: "1",
Context: map[string]string{
"admin": "true",
},
},
},
{
name: "no match just string value",
req: &flipt.EvaluationRequest{
FlagKey: flag.Key,
EntityId: "1",
Context: map[string]string{
"bar": "baz",
},
},
},
}

for _, bb := range runs {
req := bb.req

b.Run(bb.name, func(b *testing.B) {
b.ReportAllocs()

for i := 0; i < b.N; i++ {
_, _ = evaluator.Evaluate(context.TODO(), req)
}
})
}
}

func BenchmarkEvaluate_RolloutDistribution(b *testing.B) {
flag, _ := flagStore.CreateFlag(context.TODO(), &flipt.CreateFlagRequest{
Key: b.Name(),
Name: b.Name(),
Description: "foo",
Enabled: true,
})

var variants []*flipt.Variant

for _, req := range []*flipt.CreateVariantRequest{
{
FlagKey: flag.Key,
Key: fmt.Sprintf("foo_%s", b.Name()),
},
{
FlagKey: flag.Key,
Key: fmt.Sprintf("bar_%s", b.Name()),
},
} {
variant, _ := flagStore.CreateVariant(context.TODO(), req)
variants = append(variants, variant)
}

segment, _ := segmentStore.CreateSegment(context.TODO(), &flipt.CreateSegmentRequest{
Key: b.Name(),
Name: b.Name(),
Description: "foo",
})

_, _ = segmentStore.CreateConstraint(context.TODO(), &flipt.CreateConstraintRequest{
SegmentKey: segment.Key,
Type: flipt.ComparisonType_STRING_COMPARISON_TYPE,
Property: "bar",
Operator: opEQ,
Value: "baz",
})

rule, _ := ruleStore.CreateRule(context.TODO(), &flipt.CreateRuleRequest{
FlagKey: flag.Key,
SegmentKey: segment.Key,
})

for _, req := range []*flipt.CreateDistributionRequest{
{
FlagKey: flag.Key,
RuleId: rule.Id,
VariantId: variants[0].Id,
Rollout: 50,
},
{
FlagKey: flag.Key,
RuleId: rule.Id,
VariantId: variants[1].Id,
Rollout: 50,
},
} {
_, _ = ruleStore.CreateDistribution(context.TODO(), req)
}

runs := []struct {
name string
req *flipt.EvaluationRequest
}{
{
name: "match string value - variant 1",
req: &flipt.EvaluationRequest{
FlagKey: flag.Key,
EntityId: "1",
Context: map[string]string{
"bar": "baz",
},
},
},
{
name: "match string value - variant 2",
req: &flipt.EvaluationRequest{
FlagKey: flag.Key,
EntityId: "10",
Context: map[string]string{
"bar": "baz",
},
},
},
{
name: "no match string value",
req: &flipt.EvaluationRequest{
FlagKey: flag.Key,
EntityId: "1",
Context: map[string]string{
"bar": "boz",
},
},
},
}

for _, bb := range runs {
req := bb.req

b.Run(bb.name, func(b *testing.B) {
b.ReportAllocs()

for i := 0; i < b.N; i++ {
_, _ = evaluator.Evaluate(context.TODO(), req)
}
})
}
}

func BenchmarkEvaluate_NoConstraints(b *testing.B) {
flag, _ := flagStore.CreateFlag(context.TODO(), &flipt.CreateFlagRequest{
Key: b.Name(),
Name: b.Name(),
Description: "foo",
Enabled: true,
})

var variants []*flipt.Variant

for _, req := range []*flipt.CreateVariantRequest{
{
FlagKey: flag.Key,
Key: fmt.Sprintf("foo_%s", b.Name()),
},
{
FlagKey: flag.Key,
Key: fmt.Sprintf("bar_%s", b.Name()),
},
} {
variant, _ := flagStore.CreateVariant(context.TODO(), req)
variants = append(variants, variant)
}

segment, _ := segmentStore.CreateSegment(context.TODO(), &flipt.CreateSegmentRequest{
Key: b.Name(),
Name: b.Name(),
Description: "foo",
})

rule, _ := ruleStore.CreateRule(context.TODO(), &flipt.CreateRuleRequest{
FlagKey: flag.Key,
SegmentKey: segment.Key,
})

for _, req := range []*flipt.CreateDistributionRequest{
{
FlagKey: flag.Key,
RuleId: rule.Id,
VariantId: variants[0].Id,
Rollout: 50,
},
{
FlagKey: flag.Key,
RuleId: rule.Id,
VariantId: variants[1].Id,
Rollout: 50,
},
} {
_, _ = ruleStore.CreateDistribution(context.TODO(), req)
}

runs := []struct {
name string
req *flipt.EvaluationRequest
}{
{
name: "match no value - variant 1",
req: &flipt.EvaluationRequest{
FlagKey: flag.Key,
EntityId: "01",
Context: map[string]string{},
},
},
{
name: "match no value - variant 2",
req: &flipt.EvaluationRequest{
FlagKey: flag.Key,
EntityId: "10",
Context: map[string]string{},
},
},
{
name: "match string value - variant 2",
req: &flipt.EvaluationRequest{
FlagKey: flag.Key,
EntityId: "10",
Context: map[string]string{
"bar": "boz",
},
},
},
}

for _, bb := range runs {
req := bb.req

b.Run(bb.name, func(b *testing.B) {
b.ReportAllocs()

for i := 0; i < b.N; i++ {
_, _ = evaluator.Evaluate(context.TODO(), req)
}
})
}
}

0 comments on commit 1e2344d

Please sign in to comment.