-
Notifications
You must be signed in to change notification settings - Fork 177
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fully remove multierr.Errors system test for testing connection errors bugfix issue 350 View.Get fails on reconnecting view bugfix processor freeze on error (334)
Showing
17 changed files
with
731 additions
and
209 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
|
||
|
||
test: | ||
go test -race ./... | ||
|
||
test-systemtest: | ||
GOKA_SYSTEMTEST=y go test -v github.com/lovoo/goka/systemtest | ||
|
||
test-all: test test-systemtest |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,92 @@ | ||
package systemtest | ||
|
||
import ( | ||
"errors" | ||
"fmt" | ||
"log" | ||
"sync" | ||
"sync/atomic" | ||
"syscall" | ||
"testing" | ||
"time" | ||
|
||
"github.com/lovoo/goka" | ||
"github.com/lovoo/goka/codec" | ||
"github.com/lovoo/goka/internal/test" | ||
) | ||
|
||
func TestEmitter_KafkaDisconnect(t *testing.T) { | ||
brokers := initSystemTest(t) | ||
var ( | ||
topic = goka.Stream(fmt.Sprintf("goka_systemtest_emitter_disconnect-%d", time.Now().Unix())) | ||
) | ||
|
||
tmgr, err := goka.DefaultTopicManagerBuilder(brokers) | ||
test.AssertNil(t, err) | ||
test.AssertNil(t, tmgr.EnsureStreamExists(string(topic), 10)) | ||
|
||
cfg := goka.DefaultConfig() | ||
|
||
fi := NewFIProxy() | ||
cfg.Net.Proxy.Enable = true | ||
cfg.Net.Proxy.Dialer = fi | ||
|
||
// get it faster over with | ||
cfg.Producer.Retry.Max = 1 | ||
cfg.Producer.Retry.Backoff = 0 | ||
|
||
em, err := goka.NewEmitter(brokers, topic, new(codec.Int64), | ||
goka.WithEmitterProducerBuilder(goka.ProducerBuilderWithConfig(cfg)), | ||
) | ||
test.AssertNil(t, err) | ||
var ( | ||
i int64 | ||
success int64 | ||
) | ||
|
||
done := make(chan struct{}) | ||
go func() { | ||
defer close(done) | ||
var closeOnce sync.Once | ||
stop := make(chan struct{}) | ||
for { | ||
select { | ||
case <-stop: | ||
return | ||
default: | ||
} | ||
|
||
prom, err := em.Emit(fmt.Sprintf("key-%d", i%20), i) | ||
if err != nil { | ||
if errors.Is(err, goka.ErrEmitterAlreadyClosed) { | ||
return | ||
} | ||
log.Printf("error emitting: %v", err) | ||
} | ||
prom.Then(func(err error) { | ||
if err != nil { | ||
log.Printf("error emitting (async): %v", err) | ||
closeOnce.Do(func() { | ||
close(stop) | ||
}) | ||
return | ||
} | ||
if err == nil { | ||
atomic.AddInt64(&success, 1) | ||
} | ||
|
||
}) | ||
time.Sleep(10 * time.Millisecond) | ||
i++ | ||
} | ||
|
||
}() | ||
|
||
pollTimed(t, "emitter emitted something successfully", 10, func() bool { | ||
return atomic.LoadInt64(&success) > 0 | ||
}) | ||
|
||
fi.SetWriteError(syscall.EPIPE) | ||
<-done | ||
test.AssertNil(t, em.Finish()) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
package systemtest | ||
|
||
import ( | ||
"os" | ||
"strings" | ||
"testing" | ||
) | ||
|
||
// Checks if the env-variable to activate system test is set and returns a broker | ||
// If system tests are not activated, will skip the test | ||
func initSystemTest(t *testing.T) []string { | ||
if _, isIntegration := os.LookupEnv("GOKA_SYSTEMTEST"); !isIntegration { | ||
t.Skip("*** skip integration test ***") | ||
} | ||
if brokers, ok := os.LookupEnv("GOKA_SYSTEMTEST_BROKERS"); ok { | ||
return strings.Split(brokers, ",") | ||
} | ||
return []string{"localhost:9092"} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,130 @@ | ||
package systemtest | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"log" | ||
"math/rand" | ||
"testing" | ||
"time" | ||
|
||
"github.com/lovoo/goka" | ||
"github.com/lovoo/goka/codec" | ||
"github.com/lovoo/goka/internal/test" | ||
"github.com/lovoo/goka/multierr" | ||
) | ||
|
||
// Tests a processor with multiple input topics. Random values are emitted to random topics, the values are accumulated | ||
// for a single key and checked for correctness after emitting a couple of messages. | ||
// This is a regression/showcase test for https://github.com/lovoo/goka/issues/332 | ||
func TestMultiTopics(t *testing.T) { | ||
|
||
brokers := initSystemTest(t) | ||
var ( | ||
group goka.Group = goka.Group(fmt.Sprintf("%s-%d", "goka-systemtest-multitopic", time.Now().Unix())) | ||
table = goka.GroupTable(group) | ||
inputStreams []goka.Stream | ||
) | ||
|
||
for i := 0; i < 5; i++ { | ||
inputStreams = append(inputStreams, goka.Stream(fmt.Sprintf("%s-input-%d", string(group), i))) | ||
} | ||
|
||
tmc := goka.NewTopicManagerConfig() | ||
tmc.Table.Replication = 1 | ||
tmc.Stream.Replication = 1 | ||
cfg := goka.DefaultConfig() | ||
tm, err := goka.TopicManagerBuilderWithConfig(cfg, tmc)(brokers) | ||
test.AssertNil(t, err) | ||
|
||
for _, inStream := range inputStreams { | ||
err = tm.EnsureStreamExists(string(inStream), 1) | ||
test.AssertNil(t, err) | ||
} | ||
// let the cluster create it | ||
time.Sleep(5 * time.Second) | ||
|
||
proc, err := goka.NewProcessor(brokers, | ||
goka.DefineGroup( | ||
group, | ||
goka.Inputs(inputStreams, new(codec.Int64), func(ctx goka.Context, msg interface{}) { | ||
var oldVal int64 | ||
|
||
if val := ctx.Value(); val != nil { | ||
oldVal = val.(int64) | ||
} | ||
|
||
// accumulate with old value | ||
ctx.SetValue(msg.(int64) + oldVal) | ||
}), | ||
goka.Persist(new(codec.Int64)), | ||
), | ||
goka.WithTopicManagerBuilder(goka.TopicManagerBuilderWithTopicManagerConfig(tmc)), | ||
) | ||
test.AssertNil(t, err) | ||
|
||
view, err := goka.NewView(brokers, table, new(codec.Int64)) | ||
test.AssertNil(t, err) | ||
|
||
var emitters []*goka.Emitter | ||
|
||
for _, input := range inputStreams { | ||
emitter, err := goka.NewEmitter(brokers, input, new(codec.Int64)) | ||
test.AssertNil(t, err) | ||
emitters = append(emitters, emitter) | ||
} | ||
|
||
ctx, cancel := context.WithCancel(context.Background()) | ||
defer cancel() | ||
errg, ctx := multierr.NewErrGroup(ctx) | ||
|
||
errg.Go(func() error { | ||
return proc.Run(ctx) | ||
}) | ||
errg.Go(func() error { | ||
return view.Run(ctx) | ||
}) | ||
|
||
log.Printf("waiting for processor/view to be running") | ||
pollTimed(t, "proc and view are recovered", 10.0, proc.Recovered, view.Recovered) | ||
log.Printf("...done") | ||
|
||
var sum int64 | ||
for i := int64(0); i < 100; i++ { | ||
value := rand.Int63n(100) | ||
// emit to random emitters in sync | ||
err := emitters[rand.Intn(len(emitters))].EmitSync("key", value) | ||
test.AssertNil(t, err) | ||
// ... and batched | ||
prom, err := emitters[rand.Intn(len(emitters))].Emit("key", value) | ||
test.AssertNil(t, err) | ||
prom.Then(func(err error) { | ||
test.AssertNil(t, err) | ||
}) | ||
|
||
// accumulate what we have sent so far | ||
sum += (value * 2) | ||
} | ||
|
||
for _, emitter := range emitters { | ||
test.AssertNil(t, emitter.Finish()) | ||
} | ||
|
||
// poll the view and the processor until we're sure that we have | ||
pollTimed(t, "all messages have been transferred", 10.0, | ||
func() bool { | ||
value, err := view.Get("key") | ||
test.AssertNil(t, err) | ||
return value != nil && value.(int64) == sum | ||
}, | ||
func() bool { | ||
value, err := proc.Get("key") | ||
test.AssertNil(t, err) | ||
return value != nil && value.(int64) == sum | ||
}, | ||
) | ||
|
||
// stop everything and wait until it's shut down | ||
cancel() | ||
test.AssertNil(t, errg.Wait().ErrorOrNil()) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,96 @@ | ||
package systemtest | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"io" | ||
"log" | ||
"testing" | ||
"time" | ||
|
||
"github.com/lovoo/goka" | ||
"github.com/lovoo/goka/codec" | ||
"github.com/lovoo/goka/internal/test" | ||
"github.com/lovoo/goka/multierr" | ||
) | ||
|
||
func TestProcessorShutdown_KafkaDisconnect(t *testing.T) { | ||
brokers := initSystemTest(t) | ||
var ( | ||
topic = goka.Stream(fmt.Sprintf("goka_systemtest_proc_shutdown_disconnect-%d", time.Now().Unix())) | ||
group = goka.Group(topic) | ||
) | ||
|
||
ctx, cancel := context.WithCancel(context.Background()) | ||
defer cancel() | ||
|
||
errg, ctx := multierr.NewErrGroup(ctx) | ||
|
||
tmgr, err := goka.DefaultTopicManagerBuilder(brokers) | ||
test.AssertNil(t, err) | ||
test.AssertNil(t, tmgr.EnsureStreamExists(string(topic), 10)) | ||
|
||
// emit values | ||
errg.Go(func() error { | ||
em, err := goka.NewEmitter(brokers, topic, new(codec.Int64)) | ||
test.AssertNil(t, err) | ||
defer em.Finish() | ||
var i int64 | ||
for { | ||
select { | ||
case <-ctx.Done(): | ||
return nil | ||
default: | ||
} | ||
|
||
prom, err := em.Emit(fmt.Sprintf("key-%d", i%20), i) | ||
test.AssertNil(t, err) | ||
prom.Then(func(err error) { | ||
test.AssertNil(t, err) | ||
}) | ||
time.Sleep(100 * time.Millisecond) | ||
i++ | ||
} | ||
}) | ||
|
||
cfg := goka.DefaultConfig() | ||
|
||
fi := NewFIProxy() | ||
cfg.Net.Proxy.Enable = true | ||
cfg.Net.Proxy.Dialer = fi | ||
|
||
proc, err := goka.NewProcessor(brokers, | ||
goka.DefineGroup( | ||
group, | ||
goka.Input(topic, new(codec.Int64), func(ctx goka.Context, msg interface{}) { | ||
if val := ctx.Value(); val != nil { | ||
ctx.SetValue(val.(int64) + msg.(int64)) | ||
} else { | ||
ctx.SetValue(msg) | ||
} | ||
}), | ||
goka.Persist(new(codec.Int64)), | ||
), | ||
goka.WithConsumerGroupBuilder(goka.ConsumerGroupBuilderWithConfig(cfg)), | ||
goka.WithProducerBuilder(goka.ProducerBuilderWithConfig(cfg)), | ||
goka.WithConsumerSaramaBuilder(goka.SaramaConsumerBuilderWithConfig(cfg)), | ||
) | ||
test.AssertNil(t, err) | ||
|
||
errg.Go(func() error { | ||
return proc.Run(ctx) | ||
}) | ||
pollTimed(t, "proc running", 10, proc.Recovered, func() bool { | ||
if val, _ := proc.Get("key-15"); val != nil && val.(int64) > 0 { | ||
return true | ||
} | ||
return false | ||
}) | ||
|
||
log.Printf("disconnecting consumer-group") | ||
fi.SetReadError(io.EOF) | ||
fi.SetWriteError(io.ErrClosedPipe) | ||
err = errg.Wait().ErrorOrNil() | ||
|
||
test.AssertNotNil(t, err) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,110 @@ | ||
package systemtest | ||
|
||
import ( | ||
"net" | ||
"sync" | ||
) | ||
|
||
// FIProxy is a fault injecting proxy hooked into the sarama-config | ||
// to proxy connections to kafka and inject connection loss etc. | ||
type FIProxy struct { | ||
m sync.RWMutex | ||
|
||
readErr error | ||
writeErr error | ||
|
||
conns map[string]*Conn | ||
} | ||
|
||
type Conn struct { | ||
net.Conn | ||
fip *FIProxy | ||
} | ||
|
||
func (c *Conn) Close() error { | ||
defer c.fip.removeConn(c.Conn) | ||
return c.Conn.Close() | ||
} | ||
|
||
func (c *Conn) Read(b []byte) (int, error) { | ||
c.fip.m.RLock() | ||
defer c.fip.m.RUnlock() | ||
|
||
if c.fip.readErr != nil { | ||
return 0, c.fip.readErr | ||
} | ||
return c.Conn.Read(b) | ||
} | ||
|
||
func (c *Conn) Write(b []byte) (int, error) { | ||
c.fip.m.RLock() | ||
defer c.fip.m.RUnlock() | ||
if c.fip.writeErr != nil { | ||
return 0, c.fip.writeErr | ||
} | ||
return c.Conn.Write(b) | ||
} | ||
|
||
func NewFIProxy() *FIProxy { | ||
return &FIProxy{ | ||
conns: make(map[string]*Conn), | ||
} | ||
} | ||
|
||
func (fip *FIProxy) Dial(network, addr string) (c net.Conn, err error) { | ||
fip.m.Lock() | ||
defer fip.m.Unlock() | ||
|
||
conn, err := net.Dial(network, addr) | ||
|
||
wrappedConn := &Conn{ | ||
Conn: conn, | ||
fip: fip, | ||
} | ||
key := conn.LocalAddr().String() | ||
|
||
fip.conns[key] = wrappedConn | ||
return wrappedConn, err | ||
} | ||
|
||
func (fip *FIProxy) removeConn(c net.Conn) { | ||
fip.m.Lock() | ||
defer fip.m.Unlock() | ||
|
||
delete(fip.conns, c.LocalAddr().String()) | ||
} | ||
|
||
func (fip *FIProxy) getConns() []string { | ||
fip.m.Lock() | ||
defer fip.m.Unlock() | ||
var conns []string | ||
|
||
for c := range fip.conns { | ||
conns = append(conns, c) | ||
} | ||
|
||
return conns | ||
} | ||
|
||
func (fip *FIProxy) SetReadError(err error) { | ||
fip.m.Lock() | ||
defer fip.m.Unlock() | ||
fip.readErr = err | ||
} | ||
|
||
func (fip *FIProxy) SetWriteError(err error) { | ||
fip.m.Lock() | ||
defer fip.m.Unlock() | ||
fip.writeErr = err | ||
} | ||
|
||
func (fip *FIProxy) ResetErrors() { | ||
fip.m.Lock() | ||
defer fip.m.Unlock() | ||
fip.readErr = nil | ||
fip.writeErr = nil | ||
} | ||
|
||
func (fip *FIProxy) String() string { | ||
return "Fault Injecting Proxy (FIP)" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,115 @@ | ||
package systemtest | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"io" | ||
"testing" | ||
"time" | ||
|
||
"github.com/lovoo/goka" | ||
"github.com/lovoo/goka/codec" | ||
"github.com/lovoo/goka/internal/test" | ||
"github.com/lovoo/goka/multierr" | ||
) | ||
|
||
// Tests the following scenario: | ||
// A view started with `WithViewAutoReconnect` should still return values even after losing connection to kafka. | ||
// Therefore we start a view on a topic fed by an emitter, the view proxies through the FIProxy and loses connection | ||
// after recovering. The values are still be served/returned | ||
func TestView_Reconnect(t *testing.T) { | ||
var topic = fmt.Sprintf("goka_systemtest_view_reconnect_test-%d", time.Now().Unix()) | ||
brokers := initSystemTest(t) | ||
|
||
ctx, cancel := context.WithCancel(context.Background()) | ||
defer cancel() | ||
|
||
errg, ctx := multierr.NewErrGroup(ctx) | ||
|
||
tmgr, err := goka.DefaultTopicManagerBuilder(brokers) | ||
test.AssertNil(t, err) | ||
test.AssertNil(t, tmgr.EnsureStreamExists(topic, 10)) | ||
|
||
errg.Go(func() error { | ||
em, err := goka.NewEmitter(brokers, goka.Stream(topic), new(codec.Int64)) | ||
if err != nil { | ||
return err | ||
} | ||
defer em.Finish() | ||
var i int64 | ||
for { | ||
select { | ||
case <-ctx.Done(): | ||
return nil | ||
default: | ||
} | ||
|
||
test.AssertNil(t, em.EmitSync("key", i)) | ||
time.Sleep(10 * time.Millisecond) | ||
i++ | ||
} | ||
}) | ||
|
||
cfg := goka.DefaultConfig() | ||
|
||
fi := NewFIProxy() | ||
cfg.Net.Proxy.Enable = true | ||
cfg.Net.Proxy.Dialer = fi | ||
|
||
// we'll use a view on the stream. | ||
view, err := goka.NewView(brokers, goka.Table(topic), new(codec.Int64), | ||
goka.WithViewAutoReconnect(), | ||
goka.WithViewConsumerSaramaBuilder(goka.SaramaConsumerBuilderWithConfig(cfg)), | ||
goka.WithViewTopicManagerBuilder(goka.TopicManagerBuilderWithConfig(cfg, goka.NewTopicManagerConfig())), | ||
) | ||
test.AssertNil(t, err) | ||
|
||
// Start view and wait for it to be recovered | ||
errg.Go(func() error { | ||
return view.Run(ctx) | ||
}) | ||
pollTimed(t, "view-recovered", 10, view.Recovered) | ||
|
||
val := func() int64 { | ||
val, err := view.Get("key") | ||
test.AssertNil(t, err) | ||
if val == nil { | ||
return 0 | ||
} | ||
return val.(int64) | ||
} | ||
|
||
pollTimed(t, "wait-first-value", 3, func() bool { | ||
return val() > 0 | ||
}) | ||
firstVal := val() | ||
|
||
time.Sleep(500 * time.Millisecond) | ||
|
||
// kill kafka connection | ||
fi.SetReadError(io.EOF) | ||
pollTimed(t, "view-reconnecting", 10, func() bool { | ||
return view.CurrentState() == goka.ViewStateConnecting | ||
}) | ||
|
||
// the view still should have gotten the update before the EOF | ||
secondVal := val() | ||
test.AssertTrue(t, secondVal > firstVal) | ||
|
||
// let some time pass -> the value should not have updated | ||
time.Sleep(500 * time.Millisecond) | ||
test.AssertTrue(t, val() == secondVal) | ||
|
||
// connect kafka again, wait until it's running -> the value should have changed | ||
fi.ResetErrors() | ||
pollTimed(t, "view-running", 10, func() bool { | ||
return view.CurrentState() == goka.ViewStateRunning | ||
}) | ||
pollTimed(t, "view-running", 5, func() bool { | ||
return val() > secondVal | ||
}) | ||
|
||
// shut everything down | ||
cancel() | ||
test.AssertNil(t, errg.Wait().ErrorOrNil()) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters