Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ccl/sqlccl: TestBackupRestoreLocal failed under stress #15195

Closed
cockroach-teamcity opened this issue Apr 20, 2017 · 3 comments
Closed

ccl/sqlccl: TestBackupRestoreLocal failed under stress #15195

cockroach-teamcity opened this issue Apr 20, 2017 · 3 comments
Assignees
@a-robinson
Labels
C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot.
Milestone
1.0

Comments

@cockroach-teamcity
Copy link
Member

SHA: https://github.com/cockroachdb/cockroach/commits/f6847aa114df055c7cbb8b8ba6e942c8c5b65fb8

Parameters:

COCKROACH_PROPOSER_EVALUATED_KV=false
TAGS=deadlock
GOFLAGS=

Stress build found a failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=229314&tab=buildLog

	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/ambient_context.go:139 +0xba
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc421ad1800, 0x7f3d823b1f28, 0xc421223500, 0x14b70a7212ba1030, 0x0, 0x100000001, 0x1, 0x8, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1387 +0xd4
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420cb8380, 0x7f3d823b1f28, 0xc421223500, 0x14b70a7212ba1030, 0x0, 0x100000001, 0x1, 0x8, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2558 +0x7a9
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc421519b20, 0x7f3d823b1f28, 0xc4212234a0, 0x0, 0x0, 0x100000001, 0x1, 0x8, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x1cf
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:843 +0x18c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc4228b0f30, 0xc425005830, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:263 +0x105
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc42128c2c0, 0x7f3d823b1f28, 0xc421223410, 0xc4200b1110, 0xc421223410, 0xc4254a1ad8, 0x5ac3e3)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:854 +0x11d
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc42128c2c0, 0x7f3d823b1f28, 0xc421223410, 0xc4200b1110, 0xc42128c2c0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:871 +0x99
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x1a09f20, 0xc42128c2c0, 0x7f3d823b1f28, 0xc421223170, 0xc4200b0af0, 0x0, 0x0, 0x0, 0xc4218c3440, 0x5adfe2)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1869 +0x28d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc42001c9a0, 0x264ce40, 0xc42001cfc0, 0xc422a01c20, 0xc422cc0e70, 0x2610dc0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:738 +0xaa0
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc42001c9a0, 0x264ce40, 0xc42001cfc0, 0xc422a01c20, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:932 +0x1339
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc422bf92b0, 0xc42001c9a0, 0x264ce40, 0xc42001cfc0, 0xc422a01c20)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:497 +0xa9
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:498 +0xa1

goroutine 433649 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422a019e0, 0x1aabe50, 0xc42611d650, 0x264cde0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:252 +0x19b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x7f3d823b1f28, 0xc421222b40, 0x0, 0x0, 0x2645420, 0x2aad418, 0x0, 0x0, 0x263a660, 0x2aad418, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:63 +0xaa
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7f3d823b1f28, 0xc421222b40, 0x1a5f502, 0x21, 0x1a19980, 0xc4209471a8, 0x19843c0, 0xc422806420, 0xc4233d7980, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:265 +0xa4b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x7f3d823b1f28, 0xc421222b40, 0x1a5f502, 0x21, 0x1a19980, 0xc4209471a8, 0x19843c0, 0xc422806420, 0xc4233d7980, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:146 +0x19c
github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc42098e620, 0x7f3d823b1f28, 0xc421222b40, 0xc4209471a8, 0x0, 0x0, 0x0, 0x1, 0x8, 0xc4201532f8)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1846 +0xd2
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc423e279f0, 0xc420947180, 0x7f3d823b1f28, 0xc421222b40, 0x0, 0x0, 0xc4238d24e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:196 +0x17f
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423e279f0, 0xc420947180, 0x7f3d823b1f28, 0xc421222b40, 0x2651500, 0x2aad418, 0xc423cb9bc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:205 +0x7e
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:209 +0x140

goroutine 396879 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4250f67c0, 0xc4252a3590, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:143 +0x556
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc425748fc0, 0xc4252a3590, 0x5, 0x5, 0xc42546db60, 0x415090abf3, 0x59f8aa)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:333 +0x5c
io.ReadAtLeast(0x26325a0, 0xc425748fc0, 0xc4252a3590, 0x5, 0x5, 0x5, 0xc422d1d1b8, 0x59f43f, 0xc42546dc90)
	/usr/local/go/src/io/io.go:307 +0xa9
io.ReadFull(0x26325a0, 0xc425748fc0, 0xc4252a3590, 0x5, 0x5, 0x20, 0xc42546dc60, 0x5a801c)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4252a3580, 0x7fffffff, 0xc42546dc88, 0x188c8a0, 0xc420f40c60, 0xc42546dcc0, 0x9, 0x3)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:243 +0x69
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4252a3580, 0x2645420, 0x2aad418, 0xc425748fc0, 0x263a660, 0x2aad418, 0x1961860, 0xc42027bbc0, 0x7fffffff, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:339 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc4224318c0, 0x1961860, 0xc42027bbc0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:386 +0x120
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc4244d5ba0, 0x7f3d823b1f28, 0xc42027bb60, 0xc420e0c390)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:283 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x264e9a0, 0xc4244d5ba0, 0xc425714880, 0xc420429570, 0x7f3d823b1f28, 0xc4237e67b0, 0xc422ff9f98, 0x5c7fce)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:467 +0x49
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:484 +0xdd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42237f4d0, 0xc423c240f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:211 +0x66

goroutine 433124 [runnable]:
sync.runtime_SemacquireMutex(0xc4200fcf24)
	/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4200fcf20)
	/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock.(*lockOrder).PreLock(0xc4200fcf20, 0x4, 0x188c8a0, 0xc421673b10)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock/deadlock.go:243 +0x91
github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock.PreLock(0x4, 0x188c8a0, 0xc421673b10)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock/deadlock.go:129 +0x4b
github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock.lock(0xc422bf91b0, 0x188c8a0, 0xc421673b10)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock/deadlock.go:145 +0x7b
github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock.(*Mutex).Lock(0xc421673b10)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock/deadlock.go:55 +0x7a
github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).Update(0xc421673b00, 0x14b70a7212434609, 0x0, 0x0, 0xc400000000)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:196 +0x54
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send.func1(0xc424599528, 0xc4245995a0, 0x14b70a721243ad8d, 0x0, 0xc424599598, 0xc4222e6e00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2461 +0x20c
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4222e6e00, 0x7f3d823b1f28, 0xc4228ca5a0, 0x14b70a7212434609, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2560 +0x1015
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc420429ab0, 0x7f3d823b1f28, 0xc4228ca570, 0x0, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x1cf
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:843 +0x18c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc42237f4d0, 0xc424599830, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:263 +0x105
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420bed080, 0x7f3d823b1f28, 0xc4228ca540, 0xc423afcc08, 0xc4228ca540, 0x1f, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:854 +0x11d
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420bed080, 0x7f3d823b1f28, 0xc4228ca540, 0xc423afcc08, 0x1f, 0x5df7bb, 0x10)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:871 +0x99
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc4250796d0, 0xc423afcbe0, 0x7f3d823b1f28, 0xc4228ca420, 0x0, 0x0, 0xc4220e2d00)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:192 +0x49f
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4250796d0, 0xc423afcbe0, 0x7f3d823b1f28, 0xc4228ca420, 0x2651500, 0x2aad418, 0xc420f94b40)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:205 +0x7e
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:209 +0x140

goroutine 432091 [syscall, locked to thread]:
github.com/cockroachdb/cockroach/pkg/storage/engine._Cfunc_DBOpen(0xc4220d40b0, 0xc423e27770, 0x4c, 0x7f3d73414820, 0x8000, 0x0, 0x800000000, 0xffffffff, 0x0, 0x0)
	github.com/cockroachdb/cockroach/pkg/storage/engine/_obj/_cgo_gotypes.go:565 +0x5d
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open.func1(0xc4220d40b0, 0xc423e27770, 0x4c, 0x7f3d73414820, 0x8000, 0x0, 0x800000000, 0xffffffff, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:405 +0x1a1
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open(0xc4220d40b0, 0xc423e27680, 0x50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:405 +0x37d
github.com/cockroachdb/cockroach/pkg/storage/engine.NewRocksDB(0x0, 0x0, 0x0, 0xc423e274a0, 0x4c, 0x7f3d734147f0, 0x20000000, 0xffffffffffffffff, 0x18b9a80, 0xc421970101, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:346 +0x203
github.com/cockroachdb/cockroach/pkg/storage/engine.MakeRocksDBSstFileReader(0xc423e274a0, 0x4c, 0xc423e274a0, 0x4c, 0x7f3d823b1f28)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:1702 +0x86
github.com/cockroachdb/cockroach/pkg/ccl/storageccl.evalImport(0x7f3d823b1f28, 0xc421167b30, 0xc42387a600, 0x0, 0x14b70a72117aa9cc, 0x0, 0x200000002, 0x3, 0x16, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/ccl/storageccl/import.go:149 +0xac4
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeAdminBatch(0xc42387a600, 0x7f3d823b1f28, 0xc421167aa0, 0x14b70a72117aa9cc, 0x0, 0x200000002, 0x3, 0x16, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1957 +0x9d5
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc42387a600, 0x7f3d823b1f28, 0xc421167aa0, 0x14b70a72117aa9cc, 0x0, 0x200000002, 0x3, 0x16, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1404 +0x66c
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420fdea80, 0x7f3d823b1f28, 0xc4216136b0, 0x14b70a72117aa9cc, 0x0, 0x200000002, 0x3, 0x16, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2558 +0x7a9
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc420f8e620, 0x7f3d823b1f28, 0xc421613680, 0x0, 0x0, 0x200000002, 0x3, 0x16, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x1cf
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:843 +0x18c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc42296b5f0, 0xc421971830, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:263 +0x105
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420830000, 0x7f3d823b1f28, 0xc421613650, 0xc42388b9d0, 0xc421613650, 0xc4254a1ad8, 0x5ac3e3)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:854 +0x11d
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420830000, 0x7f3d823b1f28, 0xc421613650, 0xc42388b9d0, 0xc420830000, 0xf2df7d616fa89874, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:871 +0x99
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x1a09f20, 0xc420830000, 0x7f3d823b1f28, 0xc421612f90, 0xc42388b960, 0x0, 0x0, 0x0, 0x83350f, 0x1)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1869 +0x28d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc42163ab60, 0x264ce40, 0xc42163afc0, 0xc420edf320, 0xc422892630, 0x2610dc0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:738 +0xaa0
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc42163ab60, 0x264ce40, 0xc42163afc0, 0xc420edf320, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:932 +0x1339
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4212b1520, 0xc42163ab60, 0x264ce40, 0xc42163afc0, 0xc420edf320)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:497 +0xa9
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:498 +0xa1

goroutine 389862 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc425cd7ba0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:859 +0x666
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc425cd7ba0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:604 +0x1d9
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:605 +0x715

goroutine 391625 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc420a1da40)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:742 +0x743
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:161 +0x896

goroutine 433402 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422a7db00, 0x1aabe50, 0xc422f2b650, 0x264cde0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:252 +0x19b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x7f3d823b1f28, 0xc4229d5620, 0x0, 0x0, 0x2645420, 0x2aad418, 0x0, 0x0, 0x263a660, 0x2aad418, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:63 +0xaa
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7f3d823b1f28, 0xc4229d5620, 0x1a5f502, 0x21, 0x1a19980, 0xc423da9608, 0x19843c0, 0xc42305e780, 0xc42147f980, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:265 +0xa4b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x7f3d823b1f28, 0xc4229d5620, 0x1a5f502, 0x21, 0x1a19980, 0xc423da9608, 0x19843c0, 0xc42305e780, 0xc42147f980, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:146 +0x19c
github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc420b9c8b8, 0x7f3d823b1f28, 0xc4229d5620, 0xc423da9608, 0x0, 0x0, 0x0, 0x1, 0x5ae838, 0x70)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1846 +0xd2
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc4245cf7c0, 0xc423da95e0, 0x7f3d823b1f28, 0xc4229d5620, 0x0, 0x0, 0xc420ec6d00)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:196 +0x17f
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4245cf7c0, 0xc423da95e0, 0x7f3d823b1f28, 0xc4229d5620, 0x2651500, 0x2aad418, 0xc4237b5080)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:205 +0x7e
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:209 +0x140

goroutine 394889 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func3(0x264cde0, 0xc425425980, 0xc424a6e120, 0xc42037b8c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:236 +0x40f
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:256 +0xc94

goroutine 433041 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc423f5b500, 0x7f3d823b1f28, 0xc421103bc0, 0xc422f7ecb0, 0xc420f94060, 0x5, 0x8, 0xc42504ccf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1707 +0xccd
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc423f5b500, 0x7f3d823b1f28, 0xc421103bc0, 0x14b70a721233277a, 0x0, 0x100000001, 0x1, 0x11, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2162 +0xf64
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc423f5b500, 0x7f3d823b1f28, 0xc421103bc0, 0x14b70a721233277a, 0x0, 0x100000001, 0x1, 0x11, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2070 +0x98
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc423f5b500, 0x7f3d823b1f28, 0xc421103bc0, 0x14b70a721233277a, 0x0, 0x100000001, 0x1, 0x11, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1398 +0x229
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420cb8380, 0x7f3d823b1f28, 0xc421103320, 0x14b70a721233277a, 0x0, 0x100000001, 0x1, 0x11, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2558 +0x7a9
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc421519b20, 0x7f3d823b1f28, 0xc421103290, 0x0, 0x0, 0x100000001, 0x1, 0x11, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x1cf
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:843 +0x18c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc4228b0f30, 0xc426145830, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:263 +0x105
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc42128c2c0, 0x7f3d823b1f28, 0xc421103260, 0xc422f7ec40, 0xc421103260, 0xc425a9fad8, 0x5ac3e3)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:854 +0x11d
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc42128c2c0, 0x7f3d823b1f28, 0xc421103260, 0xc422f7ec40, 0xc42128c2c0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:871 +0x99
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x1a09f20, 0xc42128c2c0, 0x7f3d823b1f28, 0xc421102f30, 0xc422f7ebd0, 0x0, 0x0, 0x0, 0x5d98e9, 0xc420a3ef30)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1869 +0x28d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc42001c9a0, 0x264ce40, 0xc420a1d5e0, 0xc4233d2360, 0xc422cc0e70, 0x2610dc0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:738 +0xaa0
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc42001c9a0, 0x264ce40, 0xc420a1d5e0, 0xc4233d2360, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:932 +0x1339
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420ce8700, 0xc42001c9a0, 0x264ce40, 0xc420a1d5e0, 0xc4233d2360)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:497 +0xa9
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:498 +0xa1

goroutine 433909 [runnable]:
github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock.lock.func1(0xc422bf9a90, 0xc42274eea0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock/deadlock.go:150
created by github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock.lock
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock/deadlock.go:153 +0xdb

goroutine 430061 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc421436a00, 0x7f3d823b1f68, 0xc422f83700, 0x1dcd6500, 0x0, 0xc421436a48, 0x1, 0xc4213e4690, 0x3, 0x3, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1175 +0x19e1
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc421436a00, 0x7f3d823b1f68, 0xc422f83700, 0x1, 0xc4213e4690, 0x3, 0x3, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:408 +0x311
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc421436a00, 0x7f3d823b1f28, 0xc4204fb230, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:477 +0x16e
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc421436a00, 0x7f3d823b1f28, 0xc4204fb230, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:944 +0x3db
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatchAsync.func1(0x7f3d823b1f28, 0xc4204fb230)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:879 +0x103
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc42296b5f0, 0x234ee09, 0x11, 0x371, 0xc420a66b40, 0x2651500, 0x2aad418, 0xc4238b76c0, 0x7f3d823b1f28, 0xc4204fb230)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:348 +0x10c
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x285

goroutine 393208 [IO wait]:
net.runtime_pollWait(0x7f3d822aa580, 0x72, 0x3f)
	/usr/local/go/src/runtime/netpoll.go:164 +0x59
net.(*pollDesc).wait(0xc422b534f8, 0x72, 0x26363a0, 0x262b898)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x38
net.(*pollDesc).waitRead(0xc422b534f8, 0xc422bd1400, 0x400)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x34
net.(*netFD).Read(0xc422b53490, 0xc422bd1400, 0x400, 0x400, 0x0, 0x26363a0, 0x262b898)
	/usr/local/go/src/net/fd_unix.go:250 +0x1b7
net.(*conn).Read(0xc420d78338, 0xc422bd1400, 0x400, 0x400, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:181 +0x70
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc420ac3860, 0xc422bd1400, 0x400, 0x400, 0x1971da0, 0x0, 0x7f3d816b8028)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x113
crypto/tls.(*block).readFromUntil(0xc4256844b0, 0x7f3d816b8028, 0xc420ac3860, 0x5, 0xc420ac3860, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:488 +0x98
crypto/tls.(*Conn).readRecord(0xc421ac4380, 0x1aaca17, 0xc421ac44a0, 0x5c8696)
	/usr/local/go/src/crypto/tls/conn.go:590 +0xc4
crypto/tls.(*Conn).Read(0xc421ac4380, 0xc426052000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1134 +0x11d
bufio.(*Reader).Read(0xc4244c5d40, 0xc421919298, 0x9, 0x9, 0xc422cbc870, 0x2316dae0, 0xc42316db48)
	/usr/local/go/src/bufio/bufio.go:213 +0x312
io.ReadAtLeast(0x262d4e0, 0xc4244c5d40, 0xc421919298, 0x9, 0x9, 0x9, 0xc4232dae78, 0x26326e0, 0xc421102a50)
	/usr/local/go/src/io/io.go:307 +0xa9
io.ReadFull(0x262d4e0, 0xc4244c5d40, 0xc421919298, 0x9, 0x9, 0xa6669d, 0xc425e5afc0, 0xc42237bdc0)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc421919298, 0x9, 0x9, 0x262d4e0, 0xc4244c5d40, 0x0, 0x100000000, 0x100000039, 0x3900000001)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc421919260, 0x39, 0x39, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc4235322a0, 0xc421102a20, 0xc421102a20, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x2f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc421919420, 0xc423532480, 0x1aabd88)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:307 +0x2bf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc42163ab60, 0x264ce40, 0xc421919420)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:505 +0x162
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc42163ab60, 0x264d920, 0xc421ac4380, 0x2635820, 0xc420472fd0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:486 +0x42c
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc42163ab60, 0x264eb20, 0xc420ac3860)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:457 +0x4aa
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:426 +0x587

goroutine 396482 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc424fd3140, 0xc42573fd90, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:143 +0x556
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc425e5afc0, 0xc42573fd90, 0x5, 0x5, 0xc4238c2960, 0x3a0261fa70, 0x5cb5f4)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:333 +0x5c
io.ReadAtLeast(0x26325a0, 0xc425e5afc0, 0xc42573fd90, 0x5, 0x5, 0x5, 0x4, 0xc4229390e0, 0x0)
	/usr/local/go/src/io/io.go:307 +0xa9
io.ReadFull(0x26325a0, 0xc425e5afc0, 0xc42573fd90, 0x5, 0x5, 0xc422ded180, 0x1, 0xc422b91ce8)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc42573fd80, 0x7fffffff, 0xc4238c22d8, 0x20, 0x20, 0xc422b91db8, 0xf, 0x6)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:243 +0x69
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc42573fd80, 0x2645420, 0x2aad418, 0xc425e5afc0, 0x263a660, 0x2aad418, 0x1961780, 0xc422539760, 0x7fffffff, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:339 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc42111d0e0, 0x1961780, 0xc422539760, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:613 +0x11d
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc424c794c0, 0x7f3d823b1f28, 0xc4232daf30, 0xc422c0d080)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:351 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x264ea00, 0xc424c794c0, 0xc4226516c0, 0xc4227824e0, 0xc4e88f)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:300 +0x45
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:323 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42296b5f0, 0xc4232daf60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:210 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:211 +0x66

goroutine 433853 [runnable]:
github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock.lock.func1(0xc423c2c310, 0xc4222125a0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock/deadlock.go:150
created by github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock.lock
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock/deadlock.go:153 +0xdb

goroutine 433040 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4233d2120, 0x1aabe50, 0xc421ae7650, 0x264cde0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:252 +0x19b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x7f3d823b1f28, 0xc421102c90, 0x0, 0x0, 0x2645420, 0x2aad418, 0x0, 0x0, 0x263a660, 0x2aad418, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:63 +0xaa
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7f3d823b1f28, 0xc421102c90, 0x1a5f502, 0x21, 0x1a19980, 0xc423afca28, 0x19843c0, 0xc42236dbc0, 0xc42271a180, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:265 +0xa4b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x7f3d823b1f28, 0xc421102c90, 0x1a5f502, 0x21, 0x1a19980, 0xc423afca28, 0x19843c0, 0xc42236dbc0, 0xc42271a180, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:146 +0x19c
github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4209b0780, 0x7f3d823b1f28, 0xc421102c90, 0xc423afca28, 0x0, 0x0, 0x0, 0x1, 0xc425accf60, 0xc420026600)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1846 +0xd2
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc423327590, 0xc423afca00, 0x7f3d823b1f28, 0xc421102c90, 0x0, 0x0, 0xc4260c5ba0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:196 +0x17f
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423327590, 0xc423afca00, 0x7f3d823b1f28, 0xc421102c90, 0x2651500, 0x2aad418, 0xc422e86480)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:205 +0x7e
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:209 +0x140

goroutine 430852 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc421436a00, 0x7f3d823b1f68, 0xc422abe440, 0x1dcd6500, 0x0, 0xc421436a48, 0x11, 0xc423704be0, 0x3, 0x3, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1175 +0x19e1
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc421436a00, 0x7f3d823b1f68, 0xc422abe440, 0x11, 0xc423704be0, 0x3, 0x3, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:408 +0x311
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc421436a00, 0x7f3d823b1f28, 0xc422cf3b90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:477 +0x16e
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc421436a00, 0x7f3d823b1f28, 0xc422cf3b90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:944 +0x3db
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc421436a00, 0x7f3d823b1f28, 0xc422cf3b90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:817 +0xa64
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc421436a00, 0x7f3d823b1f28, 0xc422cf3b90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:632 +0x31f
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422c31970, 0x7f3d823b1f28, 0xc422cf3b90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:438 +0x382
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4216e7ac0, 0x7f3d82385140, 0xc4223dc060, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:528 +0x21a
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x7f3d82385140, 0xc4223dc060, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:464 +0x76
github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x7f3d82385140, 0xc4223dc060, 0xc42612ee90, 0xc4230e6700, 0x1, 0xc422e9a770)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:436 +0xf7
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4216e7ac0, 0x7f3d82385140, 0xc4223dc060, 0xc4230e6700, 0x1, 0xc422e9a760)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:464 +0x9d
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421d54740, 0x7f3d82385140, 0xc4223dc060, 0xc426183ef0, 0x1, 0x1, 0x14b70a7210428f60, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:202 +0xc6c
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntents(0xc421d54740, 0x7f3d823795c0, 0xc420016e50, 0xc422424f00, 0x264e400, 0xc4214dfd40, 0xc426183ef0, 0x1, 0x1, 0x14b70a7210428f60, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:286 +0x199
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x7f3d823795c0, 0xc420016e50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:258 +0x7f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc42296b5f0, 0x235304c, 0x1a, 0x103, 0xc422208de0, 0x0, 0x0, 0xc422cf3a70, 0x7f3d823795c0, 0xc420016e50)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:348 +0x10c
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x285

goroutine 432967 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc421436a00, 0x7f3d823b1f68, 0xc421053ac0, 0x1dcd6500, 0x0, 0xc421436a48, 0x1, 0xc4245cf2c0, 0x3, 0x3, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1175 +0x19e1
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc421436a00, 0x7f3d823b1f68, 0xc421053ac0, 0x1, 0xc4245cf2c0, 0x3, 0x3, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:408 +0x311
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc421436a00, 0x7f3d823b1f28, 0xc422792090, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:477 +0x16e
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc421436a00, 0x7f3d823b1f28, 0xc422792090, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:944 +0x3db
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatchAsync.func1(0x7f3d823b1f28, 0xc422792090)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:879 +0x103
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc42296b5f0, 0x234ee09, 0x11, 0x371, 0xc420a66b40, 0x2651500, 0x2aad418, 0xc4220d9520, 0x7f3d823b1f28, 0xc422792090)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:348 +0x10c
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x285

goroutine 433752 [runnable]:
sync.runtime_Semacquire(0xc420cb8698)
	/usr/local/go/src/runtime/sema.go:47 +0x34
sync.(*RWMutex).Lock(0xc420cb8690)
	/usr/local/go/src/sync/rwmutex.go:91 +0x6e
sync.(*RWMutex).Lock-fm()
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock/deadlock.go:87 +0x2a
github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock.lock.func1(0xc422bf91a0, 0xc42142c420)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock/deadlock.go:151 +0x27
created by github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock.lock
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/sasha-s/go-deadlock/deadlock.go:153 +0xdb

goroutine 430154 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc422e34c00, 0x7f3d823b1f28, 0xc422cf31d0, 0x14b70a72100b7909, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2235 +0xc06
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc422e34c00, 0x7f3d823b1f28, 0xc422cf31d0, 0x14b70a72100b7909, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2070 +0x98
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc422e34c00, 0x7f3d823b1f28, 0xc422cf31d0, 0x14b70a72100b7909, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1398 +0x229
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420cb8380, 0x7f3d823b1f28, 0xc42027a5a0, 0x14b70a72100b7909, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2558 +0x7a9
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc421519b20, 0x7f3d823b1f28, 0xc42027a390, 0x0, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x1cf
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:843 +0x18c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc4228b0f30, 0xc424d9d830, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:263 +0x105
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc42128c2c0, 0x7f3d823b1f28, 0xc420200360, 0xc42422fab0, 0xc420200360, 0xc423171ad8, 0x5ac3e3)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:854 +0x11d
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc42128c2c0, 0x7f3d823b1f28, 0xc420200360, 0xc42422fab0, 0xc42128c2c0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:871 +0x99
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x1a09f20, 0xc42128c2c0, 0x7f3d823b1f28, 0xc420593ef0, 0xc42422fa40, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1869 +0x28d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc42001c9a0, 0x264ce40, 0xc42001cfc0, 0xc42126e6c0, 0xc422cc0e70, 0x2610dc0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:738 +0xaa0
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc42001c9a0, 0x264ce40, 0xc42001cfc0, 0xc42126e6c0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:932 +0x1339
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc422bf92b0, 0xc42001c9a0, 0x264ce40, 0xc42001cfc0, 0xc42126e6c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:497 +0xa9
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:498 +0xa1

goroutine 428912 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc421436a00, 0x7f3d823b1f68, 0xc422f83100, 0x1dcd6500, 0x0, 0xc421436a48, 0x1, 0xc4213e4550, 0x3, 0x3, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1175 +0x19e1
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc421436a00, 0x7f3d823b1f68, 0xc422f83100, 0x1, 0xc4213e4550, 0x3, 0x3, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:408 +0x311
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc421436a00, 0x7f3d823b1f28, 0xc424675f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:477 +0x16e
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc421436a00, 0x7f3d823b1f28, 0xc424675f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:944 +0x3db
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc421436a00, 0x7f3d823b1f28, 0xc424675f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:817 +0xa64
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc421436a00, 0x7f3d823b1f28, 0xc424675f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:632 +0x31f
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422c31970, 0x7f3d823b1f28, 0xc424675f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:438 +0x382
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4216e7ac0, 0x7f3d823b1f28, 0xc424675ef0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:528 +0x21a
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x7f3d823b1f28, 0xc424675ef0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:464 +0x76
github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x7f3d823b1f28, 0xc424675ef0, 0xc42350da10, 0xc422eabc00, 0x7f3d8240b0d0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:436 +0xf7
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4216e7ac0, 0x7f3d823b1f28, 0xc424675ef0, 0xc422eabc00, 0x264e501, 0xc424675f20)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:464 +0x9d
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).resolveIntents.func1(0x19410e0, 0xc424675f20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:431 +0x4c
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).resolveIntents(0xc421d54740, 0x7f3d823b1f28, 0xc424675ef0, 0xc425b93a70, 0x1, 0x1, 0x1, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:442 +0x5e3
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntents(0xc421d54740, 0x7f3d823795c0, 0xc420016e50, 0xc42387a600, 0x264dec0, 0xc42512ee70, 0xc425b93a70, 0x1, 0x1, 0x14b70a720eee8dfc, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:326 +0x42c
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x7f3d823795c0, 0xc420016e50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:258 +0x7f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc42296b5f0, 0x235304c, 0x1a, 0x103, 0xc422208de0, 0x0, 0x0, 0xc422faf0e0, 0x7f3d823795c0, 0xc420016e50)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:348 +0x10c
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x285

goroutine 419916 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc425676ea0, 0x1aabe50, 0xc425497650, 0x264cde0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:252 +0x19b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x7f3d823b1f28, 0xc423bb8120, 0x0, 0x0, 0x2645420, 0x2aad418, 0x0, 0x0, 0x263a660, 0x2aad418, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:63 +0xaa
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7f3d823b1f28, 0xc423bb8120, 0x1a5f502, 0x21, 0x1a19980, 0xc4235b82a8, 0x19843c0, 0xc4260716e0, 0xc42271a180, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:265 +0xa4b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x7f3d823b1f28, 0xc423bb8120, 0x1a5f502, 0x21, 0x1a19980, 0xc4235b82a8, 0x19843c0, 0xc4260716e0, 0xc42271a180, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:146 +0x19c
github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc420fb4c30, 0x7f3d823b1f28, 0xc423bb8120, 0xc4235b82a8, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1846 +0xd2
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc425755590, 0xc4235b8280, 0x7f3d823b1f28, 0xc423bb8120, 0x0, 0x0, 0xc4219aad00)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:196 +0x17f
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc425755590, 0xc4235b8280, 0x7f3d823b1f28, 0xc423bb8120, 0x2651500, 0x2aad418, 0xc4253a5740)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:205 +0x7e
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:209 +0x140

goroutine 430295 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc422d1c700, 0x7f3d823b1f68, 0xc420dd40c0, 0x1dcd6500, 0x0, 0xc422d1c748, 0x11, 0xc4245ceaa0, 0x3, 0x3, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1175 +0x19e1
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc422d1c700, 0x7f3d823b1f68, 0xc420dd40c0, 0x11, 0xc4245ceaa0, 0x3, 0x3, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:408 +0x311
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc422d1c700, 0x7f3d823b1f28, 0xc4228f3860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:477 +0x16e
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc422d1c700, 0x7f3d823b1f28, 0xc4228f3860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:944 +0x3db
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc422d1c700, 0x7f3d823b1f28, 0xc4228f3860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:817 +0xa64
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc422d1c700, 0x7f3d823b1f28, 0xc4228f3860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:632 +0x31f
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423c16420, 0x7f3d823b1f28, 0xc4228f3860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:438 +0x382
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc420a45d40, 0x7f3d82385140, 0xc422ea8e40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:528 +0x21a
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x7f3d82385140, 0xc422ea8e40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:464 +0x76
github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x7f3d82385140, 0xc422ea8e40, 0xc422bcce90, 0xc422eac300, 0x1, 0xc420527ae0)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:436 +0xf7
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc420a45d40, 0x7f3d82385140, 0xc422ea8e40, 0xc422eac300, 0x1, 0xc420527ad0)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:464 +0x9d
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc420f38c80, 0x7f3d82385140, 0xc422ea8e40, 0xc425f52bd0, 0x1, 0x1, 0x14b70a720fe6d8dd, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:202 +0xc6c
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntents(0xc420f38c80, 0x7f3d823795c0, 0xc420016e50, 0xc424798f00, 0x264e400, 0xc4224c4080, 0xc425f52bd0, 0x1, 0x1, 0x14b70a720fe6d8dd, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:286 +0x199
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x7f3d823795c0, 0xc420016e50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:258 +0x7f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc42237f4d0, 0x235304c, 0x1a, 0x103, 0xc420e1ff20, 0x0, 0x0, 0xc4228f37a0, 0x7f3d823795c0, 0xc420016e50)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:348 +0x10c
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x285

goroutine 431883 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc423f5b500, 0x7f3d823b1f28, 0xc4209ca150, 0xc4213669a0, 0xc421e60660, 0x5, 0x8, 0xc42504ccf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1707 +0xccd
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc423f5b500, 0x7f3d823b1f28, 0xc4209ca150, 0x14b70a7211643368, 0x0, 0x100000001, 0x1, 0x11, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2162 +0xf64
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc423f5b500, 0x7f3d823b1f28, 0xc4209ca150, 0x14b70a7211643368, 0x0, 0x100000001, 0x1, 0x11, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2070 +0x98
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc423f5b500, 0x7f3d823b1f28, 0xc4209ca150, 0x14b70a7211643368, 0x0, 0x100000001, 0x1, 0x11, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1398 +0x229
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420cb8380, 0x7f3d823b1f28, 0xc421166570, 0x14b70a7211643368, 0x0, 0x100000001, 0x1, 0x11, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2558 +0x7a9
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc421519b20, 0x7f3d823b1f28, 0xc420f06cf0, 0x0, 0x0, 0x100000001, 0x1, 0x11, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x1cf
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:843 +0x18c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc4228b0f30, 0xc424777830, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:263 +0x105
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc42128c2c0, 0x7f3d823b1f28, 0xc420f068d0, 0xc422a7bf10, 0xc420f068d0, 0xc422f2fad8, 0x5ac3e3)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:854 +0x11d
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc42128c2c0, 0x7f3d823b1f28, 0xc420f068d0, 0xc422a7bf10, 0xc42128c2c0, 0x4, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:871 +0x99
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x1a09f20, 0xc42128c2c0, 0x7f3d823b1f28, 0xc420f06450, 0xc422a7bea0, 0x0, 0x0, 0x0, 0xc420804c40, 0x5adfe2)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1869 +0x28d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc42001c9a0, 0x264ce40, 0xc42001cfc0, 0xc423ce9e60, 0xc422cc0e70, 0x2610dc0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:738 +0xaa0
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc42001c9a0, 0x264ce40, 0xc42001cfc0, 0xc423ce9e60, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:932 +0x1339
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc422bf92b0, 0xc42001c9a0, 0x264ce40, 0xc42001cfc0, 0xc423ce9e60)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:497 +0xa9
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:498 +0xa1

goroutine 428893 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc421436a00, 0x7f3d823b1f68, 0xc423024140, 0x1dcd6500, 0x0, 0xc421436a48, 0x8, 0xc4213e4780, 0x3, 0x3, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1175 +0x19e1
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc421436a00, 0x7f3d823b1f68, 0xc423024140, 0x8, 0xc4213e4780, 0x3, 0x3, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:408 +0x311
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc421436a00, 0x7f3d823b1f28, 0xc422aa9650, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:477 +0x16e
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc421436a00, 0x7f3d823b1f28, 0xc422aa9650, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:944 +0x3db
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc421436a00, 0x7f3d823b1f28, 0xc422aa9650, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:817 +0xa64
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc421436a00, 0x7f3d823b1f28, 0xc422aa9650, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:632 +0x31f
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422c31970, 0x7f3d823b1f28, 0xc422aa9650, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:438 +0x382
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4216e7ac0, 0x7f3d823b1f28, 0xc422aa95f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:528 +0x21a
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x7f3d823b1f28, 0xc422aa95f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:464 +0x76
github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x7f3d823b1f28, 0xc422aa95f0, 0xc424270a10, 0xc42154e700, 0x7f3d8240a2c0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:436 +0xf7
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4216e7ac0, 0x7f3d823b1f28, 0xc422aa95f0, 0xc42154e700, 0x264e501, 0xc422aa9620)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:464 +0x9d
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).resolveIntents.func1(0x19410e0, 0xc422aa9620)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:431 +0x4c
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).resolveIntents(0xc421d54740, 0x7f3d823b1f28, 0xc422aa95f0, 0xc422776900, 0x6, 0x8, 0x1, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:442 +0x5e3
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntents(0xc421d54740, 0x7f3d823795c0, 0xc420016e50, 0xc424693800, 0x264dec0, 0xc425ee7650, 0xc422776900, 0x6, 0x8, 0x14b70a720ef36ae8, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:326 +0x42c
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x7f3d823795c0, 0xc420016e50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:258 +0x7f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc42296b5f0, 0x235304c, 0x1a, 0x103, 0xc422208de0, 0x0, 0x0, 0xc422aa9500, 0x7f3d823795c0, 0xc420016e50)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:348 +0x10c
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x285

goroutine 431526 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc420ede900, 0x1aabe50, 0xc426117650, 0x264cde0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:252 +0x19b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x7f3d823b1f28, 0xc4210e9c20, 0x0, 0x0, 0x2645420, 0x2aad418, 0x0, 0x0, 0x263a660, 0x2aad418, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:63 +0xaa
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7f3d823b1f28, 0xc4210e9c20, 0x1a5f502, 0x21, 0x1a19980, 0xc42057dce8, 0x19843c0, 0xc422902a80, 0xc4233d7980, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:265 +0xa4b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x7f3d823b1f28, 0xc4210e9c20, 0x1a5f502, 0x21, 0x1a19980, 0xc42057dce8, 0x19843c0, 0xc422902a80, 0xc4233d7980, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:146 +0x19c
github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc420fb4990, 0x7f3d823b1f28, 0xc4210e9c20, 0xc42057dce8, 0x0, 0x0, 0x0, 0x1, 0x983a9d, 0x6)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1846 +0xd2
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc423e26a00, 0xc42057dcc0, 0x7f3d823b1f28, 0xc4210e9c20, 0x0, 0x0, 0xc42391c4e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:196 +0x17f
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423e26a00, 0xc42057dcc0, 0x7f3d823b1f28, 0xc4210e9c20, 0x2651500, 0x2aad418, 0xc4229024e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:205 +0x7e
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:209 +0x140

goroutine 433017 [chan receive]:
github.com/cockroachdb/cockroach/pkg/kv.(*RangeDescriptorCache).lookupRangeDescriptorInternal(0xc420faa300, 0x7f3d823b1f28, 0xc4228ca0c0, 0xc420eaf3d3, 0x5, 0xd, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/range_cache.go:345 +0x696
github.com/cockroachdb/cockroach/pkg/kv.(*RangeDescriptorCache).LookupRangeDescriptor(0xc420faa300, 0x7f3d823b1f28, 0xc4228ca0c0, 0xc420eaf3d3, 0x5, 0xd, 0x0, 0x3fc3333333333300, 0xc420c18f00, 0xc422f4f9e0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/range_cache.go:243 +0x92
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).getDescriptor(0xc422d1c700, 0x7f3d823b1f28, 0xc4228ca0c0, 0xc420eaf3d3, 0x5, 0xd, 0x0, 0xc420c18f00, 0x989680, 0x3b9aca00, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:446 +0x8d
github.com/cockroachdb/cockroach/pkg/kv.(*RangeIterator).Seek(0xc4255fbfb8, 0x7f3d823b1f28, 0xc4228ca0c0, 0xc420eaf3d3, 0x5, 0xd, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/range_iter.go:163 +0x1c9
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc422d1c700, 0x7f3d823b1f28, 0xc4228ca0c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:732 +0x16a
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc422d1c700, 0x7f3d823b1f28, 0xc4228ca0c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:632 +0x31f
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423c16420, 0x7f3d823b1f28, 0xc4228ca0c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:438 +0x382
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc420a45d40, 0x7f3d82385140, 0xc422f4fa40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:528 +0x21a
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x7f3d82385140, 0xc422f4fa40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:464 +0x76
github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x7f3d82385140, 0xc422f4fa40, 0xc4254c2e90, 0xc42414ae00, 0x2, 0xc422e71580)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:436 +0xf7
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc420a45d40, 0x7f3d82385140, 0xc422f4fa40, 0xc42414ae00, 0x2, 0xc422e71560)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:464 +0x9d
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc420f38c80, 0x7f3d82385140, 0xc422f4fa40, 0xc421fc9440, 0x3, 0x4, 0x14b70a7212314a86, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:202 +0xc6c
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntents(0xc420f38c80, 0x7f3d823795c0, 0xc420016e50, 0xc424798f00, 0x264e400, 0xc4208ce800, 0xc421fc9440, 0x3, 0x4, 0x14b70a7212314a86, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:286 +0x199
github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x7f3d823795c0, 0xc420016e50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:258 +0x7f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc42237f4d0, 0x235304c, 0x1a, 0x103, 0xc420e1ff20, 0x0, 0x0, 0xc422792630, 0x7f3d823795c0, 0xc420016e50)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:348 +0x10c
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x285


ERROR: exit status 255

make: *** [stress] Error 1
25 runs completed, 1 failures, over 3m47s
Makefile:192: recipe for target 'stress' failed
@cockroach-teamcity cockroach-teamcity added O-robot Originated from a bot. C-test-failure Broken test (automatically or manually discovered). labels Apr 20, 2017
@cockroach-teamcity
Copy link
Member Author

SHA: https://github.com/cockroachdb/cockroach/commits/9ead8a7d2b1802e0749d12ee0767cd8b95349680

Parameters:

TAGS=
GOFLAGS=-race

Stress build found a failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=233297&tab=buildLog

@a-robinson
Copy link
Contributor

Same as #15003/#15023

@a-robinson a-robinson self-assigned this Apr 25, 2017
@a-robinson a-robinson added this to the 1.0 milestone Apr 25, 2017
@a-robinson
Copy link
Contributor

Fixed by #15355

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees

@a-robinson a-robinson

Labels
C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot.
Projects
None yet
Milestone
1.0
Development

No branches or pull requests
2 participants
@a-robinson @cockroach-teamcity