From 839fd4ca6dbe9da57fa029d067014be851981ed0 Mon Sep 17 00:00:00 2001 From: Mark Phelps Date: Sat, 23 Nov 2019 16:03:59 -0500 Subject: [PATCH] Add evaluation benchmarks --- Makefile | 5 + storage/db_test.go | 2 +- storage/evaluator_test.go | 324 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 330 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 266c2bdf0e..ba90c58bbe 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,11 @@ setup: ## Install dev tools @echo ">> installing dev tools" go install -v $(TOOLS) +.PHONY: bench +bench: ## Run all the benchmarks + @echo ">> running benchmarks" + go test -v -bench=. $(SOURCE_FILES) -run=XXX + .PHONY: test test: ## Run all the tests @echo ">> running tests" diff --git a/storage/db_test.go b/storage/db_test.go index 98272ecf2c..19b1f0ad00 100644 --- a/storage/db_test.go +++ b/storage/db_test.go @@ -93,10 +93,10 @@ func TestMain(m *testing.M) { func run(m *testing.M) int { logger = logrus.New() - logger.Level = logrus.DebugLevel debug := os.Getenv("DEBUG") if debug == "" { + logger.Level = logrus.DebugLevel logger.Out = ioutil.Discard } diff --git a/storage/evaluator_test.go b/storage/evaluator_test.go index bc46dac5e5..8ffcf485cc 100644 --- a/storage/evaluator_test.go +++ b/storage/evaluator_test.go @@ -1323,3 +1323,327 @@ func Test_evaluate(t *testing.T) { assert.Empty(t, d) }) } + +func BenchmarkEvaluate_SingleVariantDistribution(b *testing.B) { + flag, _ := flagStore.CreateFlag(context.TODO(), &flipt.CreateFlagRequest{ + Key: b.Name(), + Name: b.Name(), + Description: "foo", + Enabled: true, + }) + + var variants []*flipt.Variant + + for _, req := range []*flipt.CreateVariantRequest{ + { + FlagKey: flag.Key, + Key: fmt.Sprintf("foo_%s", b.Name()), + }, + { + FlagKey: flag.Key, + Key: fmt.Sprintf("bar_%s", b.Name()), + }, + } { + variant, _ := flagStore.CreateVariant(context.TODO(), req) + variants = append(variants, variant) + } + + segment, _ := segmentStore.CreateSegment(context.TODO(), &flipt.CreateSegmentRequest{ + Key: b.Name(), + Name: b.Name(), + Description: "foo", + }) + + _, _ = segmentStore.CreateConstraint(context.TODO(), &flipt.CreateConstraintRequest{ + SegmentKey: segment.Key, + Type: flipt.ComparisonType_STRING_COMPARISON_TYPE, + Property: "bar", + Operator: opEQ, + Value: "baz", + }) + + _, _ = segmentStore.CreateConstraint(context.TODO(), &flipt.CreateConstraintRequest{ + SegmentKey: segment.Key, + Type: flipt.ComparisonType_BOOLEAN_COMPARISON_TYPE, + Property: "admin", + Operator: opTrue, + }) + + rule, _ := ruleStore.CreateRule(context.TODO(), &flipt.CreateRuleRequest{ + FlagKey: flag.Key, + SegmentKey: segment.Key, + }) + + _, _ = ruleStore.CreateDistribution(context.TODO(), &flipt.CreateDistributionRequest{ + FlagKey: flag.Key, + RuleId: rule.Id, + VariantId: variants[0].Id, + Rollout: 100, + }) + + runs := []struct { + name string + req *flipt.EvaluationRequest + }{ + { + name: "match string value", + req: &flipt.EvaluationRequest{ + FlagKey: flag.Key, + EntityId: "1", + Context: map[string]string{ + "bar": "baz", + "admin": "true", + }, + }, + }, + { + name: "no match string value", + req: &flipt.EvaluationRequest{ + FlagKey: flag.Key, + EntityId: "1", + Context: map[string]string{ + "bar": "boz", + "admin": "true", + }, + }, + }, + { + name: "no match just bool value", + req: &flipt.EvaluationRequest{ + FlagKey: flag.Key, + EntityId: "1", + Context: map[string]string{ + "admin": "true", + }, + }, + }, + { + name: "no match just string value", + req: &flipt.EvaluationRequest{ + FlagKey: flag.Key, + EntityId: "1", + Context: map[string]string{ + "bar": "baz", + }, + }, + }, + } + + for _, bb := range runs { + req := bb.req + + b.Run(bb.name, func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + _, _ = evaluator.Evaluate(context.TODO(), req) + } + }) + } +} + +func BenchmarkEvaluate_RolloutDistribution(b *testing.B) { + flag, _ := flagStore.CreateFlag(context.TODO(), &flipt.CreateFlagRequest{ + Key: b.Name(), + Name: b.Name(), + Description: "foo", + Enabled: true, + }) + + var variants []*flipt.Variant + + for _, req := range []*flipt.CreateVariantRequest{ + { + FlagKey: flag.Key, + Key: fmt.Sprintf("foo_%s", b.Name()), + }, + { + FlagKey: flag.Key, + Key: fmt.Sprintf("bar_%s", b.Name()), + }, + } { + variant, _ := flagStore.CreateVariant(context.TODO(), req) + variants = append(variants, variant) + } + + segment, _ := segmentStore.CreateSegment(context.TODO(), &flipt.CreateSegmentRequest{ + Key: b.Name(), + Name: b.Name(), + Description: "foo", + }) + + _, _ = segmentStore.CreateConstraint(context.TODO(), &flipt.CreateConstraintRequest{ + SegmentKey: segment.Key, + Type: flipt.ComparisonType_STRING_COMPARISON_TYPE, + Property: "bar", + Operator: opEQ, + Value: "baz", + }) + + rule, _ := ruleStore.CreateRule(context.TODO(), &flipt.CreateRuleRequest{ + FlagKey: flag.Key, + SegmentKey: segment.Key, + }) + + for _, req := range []*flipt.CreateDistributionRequest{ + { + FlagKey: flag.Key, + RuleId: rule.Id, + VariantId: variants[0].Id, + Rollout: 50, + }, + { + FlagKey: flag.Key, + RuleId: rule.Id, + VariantId: variants[1].Id, + Rollout: 50, + }, + } { + _, _ = ruleStore.CreateDistribution(context.TODO(), req) + } + + runs := []struct { + name string + req *flipt.EvaluationRequest + }{ + { + name: "match string value - variant 1", + req: &flipt.EvaluationRequest{ + FlagKey: flag.Key, + EntityId: "1", + Context: map[string]string{ + "bar": "baz", + }, + }, + }, + { + name: "match string value - variant 2", + req: &flipt.EvaluationRequest{ + FlagKey: flag.Key, + EntityId: "10", + Context: map[string]string{ + "bar": "baz", + }, + }, + }, + { + name: "no match string value", + req: &flipt.EvaluationRequest{ + FlagKey: flag.Key, + EntityId: "1", + Context: map[string]string{ + "bar": "boz", + }, + }, + }, + } + + for _, bb := range runs { + req := bb.req + + b.Run(bb.name, func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + _, _ = evaluator.Evaluate(context.TODO(), req) + } + }) + } +} + +func BenchmarkEvaluate_NoConstraints(b *testing.B) { + flag, _ := flagStore.CreateFlag(context.TODO(), &flipt.CreateFlagRequest{ + Key: b.Name(), + Name: b.Name(), + Description: "foo", + Enabled: true, + }) + + var variants []*flipt.Variant + + for _, req := range []*flipt.CreateVariantRequest{ + { + FlagKey: flag.Key, + Key: fmt.Sprintf("foo_%s", b.Name()), + }, + { + FlagKey: flag.Key, + Key: fmt.Sprintf("bar_%s", b.Name()), + }, + } { + variant, _ := flagStore.CreateVariant(context.TODO(), req) + variants = append(variants, variant) + } + + segment, _ := segmentStore.CreateSegment(context.TODO(), &flipt.CreateSegmentRequest{ + Key: b.Name(), + Name: b.Name(), + Description: "foo", + }) + + rule, _ := ruleStore.CreateRule(context.TODO(), &flipt.CreateRuleRequest{ + FlagKey: flag.Key, + SegmentKey: segment.Key, + }) + + for _, req := range []*flipt.CreateDistributionRequest{ + { + FlagKey: flag.Key, + RuleId: rule.Id, + VariantId: variants[0].Id, + Rollout: 50, + }, + { + FlagKey: flag.Key, + RuleId: rule.Id, + VariantId: variants[1].Id, + Rollout: 50, + }, + } { + _, _ = ruleStore.CreateDistribution(context.TODO(), req) + } + + runs := []struct { + name string + req *flipt.EvaluationRequest + }{ + { + name: "match no value - variant 1", + req: &flipt.EvaluationRequest{ + FlagKey: flag.Key, + EntityId: "01", + Context: map[string]string{}, + }, + }, + { + name: "match no value - variant 2", + req: &flipt.EvaluationRequest{ + FlagKey: flag.Key, + EntityId: "10", + Context: map[string]string{}, + }, + }, + { + name: "match string value - variant 2", + req: &flipt.EvaluationRequest{ + FlagKey: flag.Key, + EntityId: "10", + Context: map[string]string{ + "bar": "boz", + }, + }, + }, + } + + for _, bb := range runs { + req := bb.req + + b.Run(bb.name, func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + _, _ = evaluator.Evaluate(context.TODO(), req) + } + }) + } +}