I160415 16:35:27.938240 kv/range_cache.go:614 Recovered: llrb: inverted range: from="\"\\x00\"" to="/Min" I160415 16:35:27.938265 kv/range_cache.go:615 from: /Min (start: "\x00") to: /Min (/Min) I160415 16:35:27.938401 kv/range_cache.go:617 key=/Meta2/System/"tsd\x12cr.node.sql.latency-1h-p99.99\x00\x01\x89\xf8\x060\xc21" desc=range_id:622 start_key:"\004tsd\022cr.node.sql.latency-10m-p75\000\001\211\370\0060\2736" end_key:"\004tsd\022cr.node.sql.latency-1h-p99.99\000\001\211\370\0060\3021" replicas: replicas: replicas: next_replica_id:4 I160415 16:35:27.938450 kv/range_cache.go:617 key=/Meta2/System/"tsd\x12cr.node.sql.txn.commit.count\x00\x01\x89\xf8\x060\xba1" desc=range_id:535 start_key:"\004tsd\022cr.node.sql.latency-1h-p99.99\000\001\211\370\0060\3021" end_key:"\004tsd\022cr.node.sql.txn.commit.count\000\001\211\370\0060\2721" replicas: replicas: replicas: next_replica_id:6 I160415 16:35:27.938479 kv/range_cache.go:617 key=/Meta2/System/"tsd\x12cr.node.txn.abandons-1m\x00\x01\x89\xf8\x060\xb72" desc=range_id:524 start_key:"\004tsd\022cr.node.sql.txn.commit.count\000\001\211\370\0060\2721" end_key:"\004tsd\022cr.node.txn.abandons-1m\000\001\211\370\0060\2672" replicas: replicas: replicas: next_replica_id:6 I160415 16:35:27.938511 kv/range_cache.go:617 key=/Meta2/System/"tsd\x12cr.node.txn.durations-1h-max\x00\x01\x89\xf8\x060\xb84" desc=range_id:523 start_key:"\004tsd\022cr.node.txn.abandons-1m\000\001\211\370\0060\2672" end_key:"\004tsd\022cr.node.txn.durations-1h-max\000\001\211\370\0060\2704" replicas: replicas: replicas: next_replica_id:8 I160415 16:35:27.938540 kv/range_cache.go:617 key=/Meta2/System/"tsd\x12cr.store.intentcount\x00\x01\x89\xf8\x060\xb95" desc=range_id:448 start_key:"\004tsd\022cr.node.txn.durations-1h-max\000\001\211\370\0060\2704" end_key:"\004tsd\022cr.store.intentcount\000\001\211\370\0060\2715" replicas: replicas: replicas: next_replica_id:7 I160415 16:35:27.938576 kv/range_cache.go:617 key=/Meta2/System/"tsd\x12cr.store.intentcount\x00\x01\x89\xf8\x060\xb96" desc=range_id:529 start_key:"\004tsd\022cr.store.intentcount\000\001\211\370\0060\2715" end_key:"\004tsd\022cr.store.intentcount\000\001\211\370\0060\2716" replicas: replicas: replicas: next_replica_id:5 I160415 16:35:27.938614 kv/range_cache.go:617 key=/Meta2/Table/11 desc=range_id:119 start_key:"\004update-cluster" end_key:"\223" replicas: replicas: replicas: next_replica_id:5 I160415 16:35:27.938639 kv/range_cache.go:617 key=/Meta2/Table/12 desc=range_id:2 start_key:"\223" end_key:"\224" replicas: replicas: replicas: next_replica_id:12 I160415 16:35:27.938664 kv/range_cache.go:617 key=/Meta2/Table/13 desc=range_id:3 start_key:"\224" end_key:"\225" replicas: replicas: replicas: next_replica_id:11 I160415 16:35:27.938686 kv/range_cache.go:617 key=/Meta2/Table/14 desc=range_id:4 start_key:"\225" end_key:"\226" replicas: replicas: replicas: next_replica_id:12 I160415 16:35:27.938707 kv/range_cache.go:617 key=/Meta2/Table/50 desc=range_id:5 start_key:"\226" end_key:"\272" replicas: replicas: replicas: next_replica_id:15 I160415 16:35:27.938756 kv/range_cache.go:617 key=/Meta2/Table/51 desc=range_id:21 start_key:"\272" end_key:"\273" replicas: replicas: replicas: next_replica_id:15 I160415 16:35:27.938788 kv/range_cache.go:617 key=/Meta2/Table/52 desc=range_id:22 start_key:"\273" end_key:"\274" replicas: replicas: replicas: next_replica_id:11 I160415 16:35:27.938813 kv/range_cache.go:617 key=/Meta2/Table/53 desc=range_id:23 start_key:"\274" end_key:"\275" replicas: replicas: replicas: next_replica_id:12 F160415 16:35:27.939101 util/log/log.go:42 unexpected panic: Dying anyway!!! goroutine 376 [running]: github.com/cockroachdb/cockroach/util/log.getStacks(0xecea30f01, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/log/clog.go:859 +0xb8 github.com/cockroachdb/cockroach/util/log.(*loggingT).outputLogEntry(0x247e160, 0x3, 0x1ebfb7a, 0xf, 0x2a, 0xc822880150, 0x21) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/log/clog.go:725 +0x9c5 github.com/cockroachdb/cockroach/util/log.addStructured(0x0, 0x0, 0x3, 0x2, 0x17f3690, 0x14, 0xc830cacda8, 0x1, 0x1) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/log/structured.go:57 +0xfc github.com/cockroachdb/cockroach/util/log.logDepth(0x0, 0x0, 0x1, 0x3, 0x17f3690, 0x14, 0xc830cacda8, 0x1, 0x1) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/log/log.go:73 +0x7b github.com/cockroachdb/cockroach/util/log.Fatalf(0x17f3690, 0x14, 0xc830cacda8, 0x1, 0x1) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/log/log.go:179 +0x66 github.com/cockroachdb/cockroach/util/log.FatalOnPanic() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/log/log.go:42 +0xb3 panic(0x135e2a0, 0xc82287ba40) /home/marc/go/src/runtime/panic.go:426 +0x4e9 github.com/cockroachdb/cockroach/kv.(*rangeDescriptorCache).clearOverlappingCachedRangeDescriptors.func1(0x24a46e0, 0x0, 0x0, 0x24a46e0, 0x0, 0x0, 0xc82043d2c0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/range_cache.go:619 +0x526 panic(0x135e2a0, 0xc8227f37d0) /home/marc/go/src/runtime/panic.go:426 +0x4e9 github.com/biogo/store/llrb.(*Tree).DoRange(0xc82044c448, 0xc830cad250, 0x7fe17a727d30, 0xc822867aa0, 0x7fe17a727d30, 0xc822867ad0, 0x18) /home/marc/cockroach/src/github.com/biogo/store/llrb/llrb.go:507 +0x1e9 github.com/cockroachdb/cockroach/util/cache.(*OrderedCache).DoRange(0xc82044c3f0, 0xc830cad308, 0x159f3a0, 0xc822864ba0, 0x159f3a0, 0xc822864bc0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/cache/cache.go:370 +0x159 github.com/cockroachdb/cockroach/kv.(*rangeDescriptorCache).clearOverlappingCachedRangeDescriptors(0xc82043d2c0, 0xc8227f9c80, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/range_cache.go:627 +0x823 github.com/cockroachdb/cockroach/kv.(*rangeDescriptorCache).insertRangeDescriptorsLocked(0xc82043d2c0, 0xc8227f9c80, 0x1, 0x1, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/range_cache.go:561 +0xa7 github.com/cockroachdb/cockroach/kv.(*rangeDescriptorCache).(github.com/cockroachdb/cockroach/kv.insertRangeDescriptorsLocked)-fm(0xc8227f9c80, 0x1, 0x1, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/range_cache.go:217 +0x4c github.com/cockroachdb/cockroach/kv.(*evictionToken).EvictAndReplace(0xc830cadd18, 0xc8227f9c80, 0x1, 0x1, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/range_cache.go:244 +0x18d github.com/cockroachdb/cockroach/kv.(*DistSender).sendChunk(0xc820427680, 0x7fe17a727330, 0xc8228671d0, 0x14459596899160c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:772 +0x3199 github.com/cockroachdb/cockroach/kv.(*DistSender).Send(0xc820427680, 0x7fe17a727330, 0xc8228671d0, 0x14459596899160c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:554 +0x6d5 github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc8203f8360, 0x7fe17a727330, 0xc8228671d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:398 +0xc4a github.com/cockroachdb/cockroach/client.(*DB).send(0xc8203fe050, 0x0, 0x2, 0xc8227f3580, 0x1, 0x1, 0x2d, 0x30) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:474 +0x421 github.com/cockroachdb/cockroach/client.(*DB).(github.com/cockroachdb/cockroach/client.send)-fm(0x0, 0x2, 0xc8227f3580, 0x1, 0x1, 0xc8228671a0, 0x2d) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:421 +0x50 github.com/cockroachdb/cockroach/client.sendAndFill(0xc830caefc8, 0xc822868a80, 0x2d, 0x2d) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:386 +0x54 github.com/cockroachdb/cockroach/client.(*DB).RunWithResponse(0xc8203fe050, 0xc822868a80, 0x1662600, 0xc8228648c0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:421 +0x6b github.com/cockroachdb/cockroach/client.(*DB).Run(0xc8203fe050, 0xc822868a80, 0xc822868a00) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:412 +0x2b github.com/cockroachdb/cockroach/client.runOneResult(0x7fe17a727c00, 0xc8203fe050, 0xc822868a80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:490 +0x5f github.com/cockroachdb/cockroach/client.(*DB).scan(0xc8203fe050, 0x1662600, 0xc8228648c0, 0x1662600, 0xc8228648e0, 0x0, 0x2005a5400, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:293 +0x145 github.com/cockroachdb/cockroach/client.(*DB).ScanInconsistent(0xc8203fe050, 0x1662600, 0xc8228648c0, 0x1662600, 0xc8228648e0, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:309 +0x84 github.com/cockroachdb/cockroach/ts.(*DB).Query(0xc820120058, 0xc825c18ba0, 0x21, 0xc82845b158, 0xc82845b164, 0xc82845b170, 0x24a6c10, 0x0, 0x0, 0x1, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/ts/query.go:435 +0x2fa github.com/cockroachdb/cockroach/ts.(*Server).handleQuery(0xc8203b8510, 0x7fe17a632e68, 0xc8207d2e00, 0xc82026a380, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/ts/server.go:84 +0x4f4 github.com/cockroachdb/cockroach/ts.(*Server).(github.com/cockroachdb/cockroach/ts.handleQuery)-fm(0x7fe17a632e68, 0xc8207d2e00, 0xc82026a380, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/ts/server.go:49 +0x5c github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc8203fc400, 0x7fe17a632e68, 0xc8207d2e00, 0xc82026a380) /home/marc/cockroach/src/github.com/julienschmidt/httprouter/router.go:344 +0x195 github.com/cockroachdb/cockroach/ts.(*Server).ServeHTTP(0xc8203b8510, 0x7fe17a632e68, 0xc8207d2e00, 0xc82026a380) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/ts/server.go:55 +0x43 net/http.(*ServeMux).ServeHTTP(0xc820448000, 0x7fe17a632e68, 0xc8207d2e00, 0xc82026a380) /home/marc/go/src/net/http/server.go:1910 +0x17d github.com/cockroachdb/cockroach/server.(*Server).ServeHTTP(0xc82034de00, 0x7fe17a632e68, 0xc8207d2e00, 0xc82026a380) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/server.go:440 +0x1d4 net/http.serverHandler.ServeHTTP(0xc8203c4200, 0x7fe17a632dd8, 0xc828036000, 0xc82026a380) /home/marc/go/src/net/http/server.go:2081 +0x19e net/http.initNPNRequest.ServeHTTP(0xc8207b6f00, 0xc8203c4200, 0x7fe17a632dd8, 0xc828036000, 0xc82026a380) /home/marc/go/src/net/http/server.go:2489 +0x221 net/http.(*initNPNRequest).ServeHTTP(0xc820c6f3b0, 0x7fe17a632dd8, 0xc828036000, 0xc82026a380) :253 +0xb6 net/http.(Handler).ServeHTTP-fm(0x7fe17a632dd8, 0xc828036000, 0xc82026a380) /home/marc/go/src/net/http/h2_bundle.go:3847 +0x50 golang.org/x/net/http2.(*serverConn).runHandler(0xc821500c00, 0xc828036000, 0xc82026a380, 0xc8207d2de0) /home/marc/cockroach/src/golang.org/x/net/http2/server.go:1679 +0x9f created by golang.org/x/net/http2.(*serverConn).processHeaders /home/marc/cockroach/src/golang.org/x/net/http2/server.go:1444 +0x8e7 goroutine 1 [select]: github.com/cockroachdb/cockroach/cli.runStart(0x21a2600, 0xc8203bbdc0, 0x0, 0x7, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/cli/start.go:215 +0x217d github.com/spf13/cobra.(*Command).execute(0x21a2600, 0xc8203bbc00, 0x7, 0x7, 0x0, 0x0) /home/marc/cockroach/src/github.com/spf13/cobra/command.go:561 +0x62c github.com/spf13/cobra.(*Command).ExecuteC(0x219f000, 0x21a2600, 0x0, 0x0) /home/marc/cockroach/src/github.com/spf13/cobra/command.go:651 +0x55c github.com/spf13/cobra.(*Command).Execute(0x219f000, 0x0, 0x0) /home/marc/cockroach/src/github.com/spf13/cobra/command.go:610 +0x2d github.com/cockroachdb/cockroach/cli.Run(0xc82000a0a0, 0x8, 0x8, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/cli/cli.go:87 +0x7c main.main() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/main.go:37 +0xe6 goroutine 17 [syscall, locked to thread]: runtime.goexit() /home/marc/go/src/runtime/asm_amd64.s:1998 +0x1 goroutine 7 [chan receive]: github.com/cockroachdb/cockroach/util/log.(*loggingT).flushDaemon(0x247e160) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/log/clog.go:994 +0x64 created by github.com/cockroachdb/cockroach/util/log.init.1 /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/log/clog.go:595 +0x8a goroutine 20 [select]: github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).startStats(0xc8203f8360) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:239 +0xd21 github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).(github.com/cockroachdb/cockroach/kv.startStats)-fm() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:224 +0x20 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8203b8000) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 34 [syscall]: os/signal.signal_recv(0x0) /home/marc/go/src/runtime/sigqueue.go:116 +0x132 os/signal.loop() /home/marc/go/src/os/signal/signal_unix.go:22 +0x18 created by os/signal.init.1 /home/marc/go/src/os/signal/signal_unix.go:28 +0x37 goroutine 35 [chan receive]: github.com/cockroachdb/cockroach/rpc.NewContext.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:89 +0x57 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc82043eec0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 36 [select]: github.com/cockroachdb/cockroach/storage.(*StorePool).start.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store_pool.go:219 +0x23f github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc82043f080) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 21 [select]: github.com/cockroachdb/cockroach/sql.(*LeaseManager).RefreshLeases.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/sql/lease.go:613 +0xde4 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8204362d0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 22 [select]: github.com/cockroachdb/cockroach/sql.NewExecutor.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/sql/executor.go:207 +0x15c github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8203fe0f0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 23 [chan receive]: github.com/cockroachdb/cockroach/server.(*Server).Start.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/server.go:259 +0x62 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820410340) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 24 [chan receive]: github.com/cockroachdb/cockroach/server.(*Server).Start.func2() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/server.go:280 +0x59 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820410440) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 25 [IO wait]: net.runtime_pollWait(0x7fe17a6a43d0, 0x72, 0x0) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc82042e3e0, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc82042e3e0, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).accept(0xc82042e380, 0x0, 0x7fe17a6ef6c0, 0xc821230000) /home/marc/go/src/net/fd_unix.go:426 +0x27c net.(*TCPListener).AcceptTCP(0xc820120258, 0xc820034ed8, 0x0, 0x0) /home/marc/go/src/net/tcpsock_posix.go:254 +0x4d net.(*TCPListener).Accept(0xc820120258, 0x0, 0x0, 0x0, 0x0) /home/marc/go/src/net/tcpsock_posix.go:264 +0x3d github.com/cockroachdb/cmux.(*cMux).Serve(0xc8203fc640, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cmux/cmux.go:110 +0x9d github.com/cockroachdb/cockroach/server.(*Server).Start.func3() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/server.go:292 +0x32 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8204105c0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 26 [chan receive]: github.com/cockroachdb/cmux.(*muxListener).Accept(0xc820410580, 0x0, 0x0, 0x0, 0x0) :9 +0x125 net/http.(*Server).Serve(0xc8203c4180, 0x7fe17a6a4588, 0xc820410580, 0x0, 0x0) /home/marc/go/src/net/http/server.go:2117 +0x129 github.com/cockroachdb/cockroach/util.ServeHandler.func2() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/net.go:90 +0x66 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8203fc680) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 27 [chan receive]: github.com/cockroachdb/cmux.(*muxListener).Accept(0xc8204105a0, 0x0, 0x0, 0x0, 0x0) :9 +0x125 crypto/tls.(*listener).Accept(0xc820410660, 0x0, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/tls.go:52 +0x60 net/http.(*Server).Serve(0xc8203c4200, 0x7fe17a6a45c0, 0xc820410660, 0x0, 0x0) /home/marc/go/src/net/http/server.go:2117 +0x129 github.com/cockroachdb/cockroach/util.ServeHandler.func2() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/net.go:90 +0x66 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8203fc700) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 28 [chan receive]: github.com/cockroachdb/cmux.(*muxListener).Accept(0xc820410380, 0x0, 0x0, 0x0, 0x0) :9 +0x125 google.golang.org/grpc.(*Server).Serve(0xc8203c4000, 0x7fe17a6a4588, 0xc820410380, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:252 +0x1b5 github.com/cockroachdb/cockroach/server.(*Server).Start.func5() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/server.go:306 +0x43 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8204106e0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 29 [chan receive]: github.com/cockroachdb/cmux.(*muxListener).Accept(0xc820410360, 0x0, 0x0, 0x0, 0x0) :9 +0x125 github.com/cockroachdb/cockroach/util.ServeHandler.func3(0x7fe17a6a4588, 0xc820410360, 0xc8203a0000, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/net.go:105 +0x66 github.com/cockroachdb/cockroach/server.(*Server).Start.func6() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/server.go:314 +0x96 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820436c60) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 30 [chan receive]: github.com/cockroachdb/cockroach/gossip.(*server).start.func3() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/gossip/server.go:298 +0x5c github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820410760) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 31 [select]: github.com/cockroachdb/cockroach/gossip.(*Gossip).bootstrap.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/gossip/gossip.go:758 +0x219 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820410780) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 32 [select]: github.com/cockroachdb/cockroach/gossip.(*Gossip).manage.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/gossip/gossip.go:787 +0x33b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8204107a0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 51 [select]: github.com/cockroachdb/cockroach/gossip.(*client).gossip(0xc8204000c0, 0xc820364dc0, 0x7fe17a727308, 0xc8201340f0, 0xc8203bbea0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/gossip/client.go:236 +0x619 github.com/cockroachdb/cockroach/gossip.(*client).start.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/gossip/client.go:77 +0x33a github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820400120) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 10 [select]: google.golang.org/grpc/transport.(*http2Client).controller(0xc820576e10) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:835 +0x5da created by google.golang.org/grpc/transport.newHTTP2Client /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:194 +0x153b goroutine 11 [IO wait]: net.runtime_pollWait(0x7fe17a6a4310, 0x72, 0xc8202ba000) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc8203f2140, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc8203f2140, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc8203f20e0, 0xc8202ba000, 0x2000, 0x2000, 0x0, 0x7fe17a71e050, 0xc820012008) /home/marc/go/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc820026000, 0xc8202ba000, 0x2000, 0x2000, 0x0, 0x0, 0x0) /home/marc/go/src/net/net.go:172 +0xe4 crypto/tls.(*block).readFromUntil(0xc8203d0480, 0x7fe17a726dc0, 0xc820026000, 0x5, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:460 +0xcc crypto/tls.(*Conn).readRecord(0xc8201ea000, 0x1a68017, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:562 +0x2d1 crypto/tls.(*Conn).Read(0xc8201ea000, 0xc820546000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:939 +0x167 bufio.(*Reader).fill(0xc820391c80) /home/marc/go/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc820391c80, 0xc8203ccc38, 0x9, 0x9, 0xc81fed6a95, 0x0, 0x0) /home/marc/go/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fe17a727270, 0xc820391c80, 0xc8203ccc38, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727270, 0xc820391c80, 0xc8203ccc38, 0x9, 0x9, 0xc82252aab8, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 golang.org/x/net/http2.readFrameHeader(0xc8203ccc38, 0x9, 0x9, 0x7fe17a727270, 0xc820391c80, 0x0, 0xc800000000, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:236 +0xa5 golang.org/x/net/http2.(*Framer).ReadFrame(0xc8203ccc00, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:463 +0x106 google.golang.org/grpc/transport.(*framer).readFrame(0xc820437680, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http_util.go:406 +0x3d google.golang.org/grpc/transport.(*http2Client).reader(0xc820576e10) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:757 +0x109 created by google.golang.org/grpc/transport.newHTTP2Client /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:200 +0x159a goroutine 12 [select]: google.golang.org/grpc.(*Conn).transportMonitor(0xc8200ca000) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:510 +0x1d3 created by google.golang.org/grpc.NewConn /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:312 +0x499 goroutine 13 [select]: github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc82042fc70, 0xc8203b2f00, 0xc8203caa80, 0x15, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0x5e5 github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:157 +0x66 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820437740) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 14 [select]: google.golang.org/grpc.NewClientStream.func1(0x7fe17a7272c0, 0xc820576e10, 0xc82026a0e0, 0xc820380b40) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:149 +0x258 created by google.golang.org/grpc.NewClientStream /home/marc/cockroach/src/google.golang.org/grpc/stream.go:157 +0xab2 goroutine 15 [select]: google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc820437800, 0xc820402570, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:141 +0x7e6 google.golang.org/grpc/transport.(*Stream).Read(0xc82026a0e0, 0xc820402570, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fe17a727398, 0xc82026a0e0, 0xc820402570, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727398, 0xc82026a0e0, 0xc820402570, 0x5, 0x5, 0xc820675b30, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 google.golang.org/grpc.(*parser).recvMsg(0xc820402560, 0xc82bf076e0, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:218 +0xb9 google.golang.org/grpc.recv(0xc820402560, 0x7fe17a6a3208, 0x24a6c10, 0xc82026a0e0, 0x0, 0x0, 0x166c2e0, 0xc820675b80, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:302 +0x45 google.golang.org/grpc.(*clientStream).RecvMsg(0xc820380b40, 0x166c2e0, 0xc820675b80, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:232 +0xac github.com/cockroachdb/cockroach/gossip.(*gossipGossipClient).Recv(0xc8202c83b0, 0xc820364dc0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:176 +0x7e github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2.1(0x7fe17a727410, 0xc8202c83b0, 0xc8204000c0, 0xc820364dc0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/gossip/client.go:224 +0x37 github.com/cockroachdb/cockroach/gossip.(*client).gossip.func2() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/gossip/client.go:232 +0x51 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8204378f0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 353 [select]: golang.org/x/net/http2.(*serverConn).serve(0xc821500c00) /home/marc/cockroach/src/golang.org/x/net/http2/server.go:706 +0xedc golang.org/x/net/http2.(*Server).ServeConn(0xc8204106a0, 0x7fe17a7271c0, 0xc8207b6f00, 0xc8211fbbb0) /home/marc/cockroach/src/golang.org/x/net/http2/server.go:334 +0xa51 golang.org/x/net/http2.ConfigureServer.func1(0xc8203c4200, 0xc8207b6f00, 0x7fe17a632ca8, 0xc820c6f3b0) /home/marc/cockroach/src/golang.org/x/net/http2/server.go:200 +0xae net/http.(*conn).serve(0xc82baaf500) /home/marc/go/src/net/http/server.go:1414 +0x777 created by net/http.(*Server).Serve /home/marc/go/src/net/http/server.go:2137 +0x44e goroutine 199 [select]: github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc82042fc70, 0xc820c960a0, 0xc820410a40, 0x17, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0x5e5 github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:157 +0x66 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820c8d140) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 306 [select]: google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc821130180, 0xc82107a030, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:141 +0x7e6 google.golang.org/grpc/transport.(*Stream).Read(0xc820fce000, 0xc82107a030, 0x5, 0x5, 0xc8215ce010, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fe17a727398, 0xc820fce000, 0xc82107a030, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727398, 0xc820fce000, 0xc82107a030, 0x5, 0x5, 0x163fe00, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 google.golang.org/grpc.(*parser).recvMsg(0xc82107a020, 0x22, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:218 +0xb9 google.golang.org/grpc.recv(0xc82107a020, 0x7fe17a6a3208, 0x24a6c10, 0xc820fce000, 0x0, 0x0, 0x167e2a0, 0xc82153a000, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:302 +0x45 google.golang.org/grpc.(*serverStream).RecvMsg(0xc821506000, 0x167e2a0, 0xc82153a000, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:411 +0xe4 github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc8215ce010, 0x27, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:174 +0x7e github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x7fe17a6f1e38, 0xc8215ce010, 0xc820436150, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:111 +0x4b github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:128 +0x48 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc82100c060) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 198 [select]: google.golang.org/grpc.(*Conn).transportMonitor(0xc820873880) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:510 +0x1d3 google.golang.org/grpc.NewConn.func1(0xc820873880) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:321 +0x1b5 created by google.golang.org/grpc.NewConn /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:322 +0x4dd goroutine 361 [select]: google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc820bc24e0, 0xc820396410, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:141 +0x7e6 google.golang.org/grpc/transport.(*Stream).Read(0xc82087e000, 0xc820396410, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fe17a727398, 0xc82087e000, 0xc820396410, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727398, 0xc82087e000, 0xc820396410, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 google.golang.org/grpc.(*parser).recvMsg(0xc820396400, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:218 +0xb9 google.golang.org/grpc.recv(0xc820396400, 0x7fe17a6a3208, 0x24a6c10, 0xc82087e000, 0x0, 0x0, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:302 +0x45 google.golang.org/grpc.(*clientStream).RecvMsg(0xc82588a780, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:232 +0xac github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).RecvMsg(0xc8202c8210, 0x16734e0, 0x24a6c10, 0x0, 0x0) :236 +0x6f github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func2.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:226 +0x79 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820396480) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 224 [select]: google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc820b72180, 0xc8210dc1f0, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:141 +0x7e6 google.golang.org/grpc/transport.(*Stream).Read(0xc8211a8fc0, 0xc8210dc1f0, 0x5, 0x5, 0x5eb9e0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fe17a727398, 0xc8211a8fc0, 0xc8210dc1f0, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727398, 0xc8211a8fc0, 0xc8210dc1f0, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 google.golang.org/grpc.(*parser).recvMsg(0xc8210dc1e0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:218 +0xb9 google.golang.org/grpc.recv(0xc8210dc1e0, 0x7fe17a6a3208, 0x24a6c10, 0xc8211a8fc0, 0x0, 0x0, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:302 +0x45 google.golang.org/grpc.(*clientStream).RecvMsg(0xc820b9c1e0, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:232 +0xac github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).RecvMsg(0xc8210d98b0, 0x16734e0, 0x24a6c10, 0x0, 0x0) :236 +0x6f github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func2.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:226 +0x79 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8210dc280) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 80 [select]: google.golang.org/grpc/transport.(*http2Server).controller(0xc8209b75f0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:620 +0x5da created by google.golang.org/grpc/transport.newHTTP2Server /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:134 +0x853 goroutine 81 [select]: github.com/cockroachdb/cockroach/storage.(*Replica).requestLeaderLease.func1.1(0x144595942d40a23a, 0x0, 0xc82014e460, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:492 +0x7c4 github.com/cockroachdb/cockroach/storage.(*Replica).requestLeaderLease.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:455 +0x45 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask.func1(0xc8203bbea0, 0x1ec2389, 0x12, 0x200, 0xc820a54320) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:181 +0x62 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:182 +0xc9 goroutine 79 [chan receive]: github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open.func1(0xc8203c1b00) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:164 +0x3a created by github.com/cockroachdb/cockroach/storage/engine.(*RocksDB).Open /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:165 +0x8f4 goroutine 502 [select]: github.com/cockroachdb/cockroach/storage.(*Replica).redirectOnOrAcquireLeaderLease(0xc82014e460, 0x7fe17a727330, 0xc823709020, 0xc821638a00) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:558 +0x6f3 github.com/cockroachdb/cockroach/storage.(*Replica).addWriteCmd(0xc82014e460, 0x7fe17a727330, 0xc823709020, 0x1445959687f4446f, 0x0, 0x500000005, 0x1, 0x217, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:1130 +0x1af github.com/cockroachdb/cockroach/storage.(*Replica).Send(0xc82014e460, 0x7fe17a727330, 0xc823709020, 0x1445959687f4446f, 0x0, 0x500000005, 0x1, 0x217, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:768 +0x5ba github.com/cockroachdb/cockroach/storage.(*Store).Send(0xc820156200, 0x7fe17a727330, 0xc823709020, 0x1445959687f4446f, 0x0, 0x500000005, 0x1, 0x217, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:1640 +0xd71 github.com/cockroachdb/cockroach/storage.(*Stores).Send(0xc8203fc380, 0x7fe17a727330, 0xc823708fc0, 0x0, 0x0, 0x500000005, 0x1, 0x217, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/stores.go:178 +0x501 github.com/cockroachdb/cockroach/server.(*Node).Batch.func2() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:729 +0x55b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc8203bbea0, 0xc821639880, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:166 +0xd9 github.com/cockroachdb/cockroach/server.(*Node).Batch(0xc82034cd80, 0x7fe17a727330, 0xc823708870, 0xc825e728c0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:741 +0x2c0 github.com/cockroachdb/cockroach/roachpb._Internal_Batch_Handler(0x16c7380, 0xc82034cd80, 0x7fe17a727330, 0xc823708870, 0xc8239fecc0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/roachpb/api.pb.go:1306 +0x132 google.golang.org/grpc.(*Server).processUnaryRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc820a18bd0, 0xc8214468c0, 0xc820410180, 0x2195870, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:497 +0xe13 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc820a18bd0, 0xc8214468c0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:646 +0x109d google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc820886b10, 0xc8203c4000, 0x7fe17a6aeb08, 0xc820a18bd0, 0xc8214468c0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 16 [runnable, locked to thread]: github.com/cockroachdb/cockroach/storage/engine._Cfunc_DBClose(0x7fe170001490) ??:0 +0x36 github.com/cockroachdb/cockroach/storage/engine.(*rocksDBBatch).Commit(0xc8228825d0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/engine/rocksdb.go:581 +0x1f2 github.com/cockroachdb/cockroach/storage.(*Replica).applyRaftCommand(0xc82014f420, 0xc8237446c8, 0x8, 0x7fe17a727330, 0xc8228825a0, 0x8abf, 0x100000001, 0x8, 0x1445895deb8ba16c, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:1544 +0x81a github.com/cockroachdb/cockroach/storage.(*Replica).processRaftCommand(0xc82014f420, 0xc8237446c8, 0x8, 0x8abf, 0x4, 0x100000001, 0x8, 0x1445895deb8ba16c, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:1494 +0x2be github.com/cockroachdb/cockroach/storage.(*Replica).handleRaftReady(0xc82014f420, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:1351 +0xa49 github.com/cockroachdb/cockroach/storage.(*Store).processRaft.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:1872 +0x35a github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8203d27c0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 82 [select]: github.com/cockroachdb/cockroach/storage.(*Store).Start.func3() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:794 +0x13d github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820331ba0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 83 [select]: github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:865 +0x3cf github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820331bc0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 84 [select]: github.com/cockroachdb/cockroach/storage.(*Replica).redirectOnOrAcquireLeaderLease(0xc82014f0a0, 0x7fe17a727330, 0xc82080e510, 0x5c4295) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:558 +0x6f3 github.com/cockroachdb/cockroach/storage.(*Replica).getLeaseForGossip.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:1896 +0x5d github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc8203bbea0, 0xc820795d28, 0xc8207b4100) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:166 +0xd9 github.com/cockroachdb/cockroach/storage.(*Replica).getLeaseForGossip(0xc82014f0a0, 0x7fe17a727330, 0xc82080e510, 0x7fe17a727330, 0xc82080e510) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:1915 +0xaf github.com/cockroachdb/cockroach/storage.(*Store).maybeGossipSystemConfig(0xc820156200, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:937 +0x12c github.com/cockroachdb/cockroach/storage.(*Store).startGossip.func2() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:877 +0x67 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820331be0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 59 [select]: github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/queue.go:356 +0x303 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820187420) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 52 [select]: github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/queue.go:356 +0x303 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820187300) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 53 [select]: github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/queue.go:356 +0x303 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820187320) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 54 [select]: github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/queue.go:356 +0x303 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820187340) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 55 [select]: github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/queue.go:356 +0x303 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820187360) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 56 [select]: github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/queue.go:356 +0x303 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820187380) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 57 [select]: github.com/cockroachdb/cockroach/storage.(*baseQueue).processLoop.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/queue.go:356 +0x303 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8201873a0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 58 [select]: github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc82042e3f0, 0xecea30f45, 0x2e463445, 0x247d840, 0xc82043d1c0, 0xc8203bbea0, 0xc82014ea80, 0x3def1f32) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/scanner.go:162 +0x568 github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1.1(0xc82014ea80, 0xc820436d50) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/scanner.go:204 +0x78 github.com/cockroachdb/cockroach/storage.(*storeRangeSet).Visit(0xc820436e40, 0xc8203fc5c0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:237 +0x246 github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/scanner.go:206 +0x21e github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8201873c0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 60 [select]: github.com/cockroachdb/cockroach/storage.(*replicaScanner).waitAndProcess(0xc82042e690, 0xecea30f45, 0x2e4a1d55, 0x247d840, 0xc82043d1c0, 0xc8203bbea0, 0xc82014e460, 0x3def1f32) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/scanner.go:162 +0x568 github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1.1(0xc82014e460, 0xc820436d50) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/scanner.go:204 +0x78 github.com/cockroachdb/cockroach/storage.(*storeRangeSet).Visit(0xc820437020, 0xc8203fcdc0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:237 +0x246 github.com/cockroachdb/cockroach/storage.(*replicaScanner).scanLoop.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/scanner.go:206 +0x21e github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820187460) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 219 [select]: google.golang.org/grpc.(*Conn).transportMonitor(0xc8211a8e00) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:510 +0x1d3 google.golang.org/grpc.NewConn.func1(0xc8211a8e00) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:321 +0x1b5 created by google.golang.org/grpc.NewConn /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:322 +0x4dd goroutine 88 [select]: github.com/cockroachdb/cockroach/server.(*Node).startComputePeriodicMetrics.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:561 +0x23b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8201225a0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 89 [select]: github.com/cockroachdb/cockroach/server.(*Node).startGossip.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:528 +0x3e3 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820122620) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 90 [select]: github.com/cockroachdb/cockroach/storage.(*Replica).redirectOnOrAcquireLeaderLease(0xc82014f0a0, 0x7fe17a727330, 0xc82252a960, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:558 +0x6f3 github.com/cockroachdb/cockroach/storage.(*Replica).addReadOnlyCmd(0xc82014f0a0, 0x7fe17a727330, 0xc82252a960, 0x144595942c119afd, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc8212338c0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:1046 +0xcc github.com/cockroachdb/cockroach/storage.(*Replica).Send(0xc82014f0a0, 0x7fe17a727330, 0xc82252a960, 0x144595942c119afd, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc8212338c0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:765 +0x500 github.com/cockroachdb/cockroach/storage.(*Store).Send(0xc820156200, 0x7fe17a727330, 0xc82252a960, 0x144595942c119afd, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc8212338c0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:1640 +0xd71 github.com/cockroachdb/cockroach/storage.(*Stores).Send(0xc8203fc380, 0x7fe17a727330, 0xc82252a8d0, 0x0, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc8212338c0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/stores.go:178 +0x501 github.com/cockroachdb/cockroach/server.(*Node).Batch.func2() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:729 +0x55b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc8203bbea0, 0xc820713258, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:166 +0xd9 github.com/cockroachdb/cockroach/server.(*Node).Batch(0xc82034cd80, 0x7fe17a6a4cc0, 0xc82252a750, 0xc820b3e028, 0xb2d05e00, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:741 +0x2c0 github.com/cockroachdb/cockroach/kv.sendOne(0x7fe17a727330, 0xc820340690, 0x1, 0x2540be400, 0xb2d05e00, 0xc82042fc70, 0xc820410a40, 0x17, 0xc820c960a0, 0x7fe17a727dc0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/send.go:263 +0x3a8 github.com/cockroachdb/cockroach/kv.send(0x7fe17a727330, 0xc820340690, 0x1, 0x2540be400, 0xb2d05e00, 0xc821102230, 0x3, 0x3, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/send.go:212 +0x1294 github.com/cockroachdb/cockroach/kv.(*DistSender).sendRPC(0xc820427680, 0x7fe17a727330, 0xc820340690, 0x77, 0xc821102230, 0x3, 0x3, 0x1, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:345 +0x1e9 github.com/cockroachdb/cockroach/kv.(*DistSender).sendSingleRange(0xc820427680, 0x7fe17a727330, 0xc820340690, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc820660240, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:450 +0x300 github.com/cockroachdb/cockroach/kv.(*DistSender).sendChunk.func2(0xc8207144c0, 0xc8207140d0, 0xc820714cd0, 0xc820427680, 0xc820714cc0, 0xf, 0x10) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:710 +0x448 github.com/cockroachdb/cockroach/kv.(*DistSender).sendChunk(0xc820427680, 0x7fe17a727330, 0xc820340690, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc820660240, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:711 +0x2244 github.com/cockroachdb/cockroach/kv.(*DistSender).Send(0xc820427680, 0x7fe17a727330, 0xc820340690, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc820660240, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:554 +0x6d5 github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc8203f8360, 0x7fe17a727330, 0xc820340690, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc820660180, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:398 +0xc4a github.com/cockroachdb/cockroach/client.(*txnSender).Send(0xc820150420, 0x7fe17a6a3398, 0xc8200131f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc820150478, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/txn.go:59 +0xb8 github.com/cockroachdb/cockroach/client.(*DB).send(0xc820150420, 0x0, 0x0, 0xc8206bad70, 0x1, 0x1, 0x0, 0xc8206bab30) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:474 +0x421 github.com/cockroachdb/cockroach/client.(*Txn).send(0xc820150420, 0x0, 0x0, 0xc8206bad70, 0x1, 0x1, 0x20, 0x5c319d) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/txn.go:649 +0xa46 github.com/cockroachdb/cockroach/client.(*Txn).(github.com/cockroachdb/cockroach/client.send)-fm(0x0, 0xc800000000, 0xc8206bad70, 0x1, 0x1, 0xc820715ca0, 0x976d66) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/txn.go:361 +0x50 github.com/cockroachdb/cockroach/client.sendAndFill(0xc820715cb8, 0xc820190000, 0xd, 0x16ecab0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:386 +0x54 github.com/cockroachdb/cockroach/client.(*Txn).RunWithResponse(0xc820150420, 0xc820190000, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/txn.go:361 +0xa9 github.com/cockroachdb/cockroach/client.(*Txn).Run(0xc820150420, 0xc820190000, 0x7fe17a65f090) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/txn.go:349 +0x2b github.com/cockroachdb/cockroach/client.runOneRow(0x7fe17a65f090, 0xc820150420, 0xc820190000, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:498 +0x4b github.com/cockroachdb/cockroach/client.(*Txn).Get(0xc820150420, 0x1662600, 0xc8200103a0, 0x0, 0x0, 0x0, 0x0, 0xc800000000) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/txn.go:225 +0x120 github.com/cockroachdb/cockroach/sql.(*planner).getDescriptor(0xc820662000, 0x7fe17a65f060, 0xc8206bad40, 0x7fe17a65eff0, 0xc820010380, 0x7fe17a65f060) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/sql/descriptor.go:130 +0xb5 github.com/cockroachdb/cockroach/sql.(*planner).getDatabaseDesc(0xc820662000, 0x1a7586d, 0x6, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/sql/database.go:81 +0x102 github.com/cockroachdb/cockroach/sql.(*planner).getTableDesc(0xc820662000, 0xc8203fd900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/sql/table.go:226 +0x10f github.com/cockroachdb/cockroach/sql.(*planner).getTableLease(0xc820662000, 0xc8203fd900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/sql/table.go:267 +0x164 github.com/cockroachdb/cockroach/sql.(*planner).Insert(0xc820662000, 0xc8203fd8c0, 0xe984f100, 0x0, 0x0, 0x599618) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/sql/insert.go:40 +0xe5 github.com/cockroachdb/cockroach/sql.(*planner).makePlan(0xc820662000, 0x7fe17a65eb10, 0xc8203fd8c0, 0xc820010200, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/sql/plan.go:225 +0x1313 github.com/cockroachdb/cockroach/sql.(*planner).query(0xc820662000, 0x1a75860, 0x74, 0xc82069e280, 0x5, 0x5, 0x0, 0x0, 0x800) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/sql/plan.go:320 +0x1b9 github.com/cockroachdb/cockroach/sql.(*planner).exec(0xc820662000, 0x1a75860, 0x74, 0xc82069e280, 0x5, 0x5, 0x5de998, 0xc820660000) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/sql/plan.go:345 +0x60 github.com/cockroachdb/cockroach/sql.InternalExecutor.ExecuteStatementInTransaction(0xc82042e000, 0xc820150420, 0x1a75860, 0x74, 0xc82069e280, 0x5, 0x5, 0xc820028800, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/sql/internal.go:44 +0x1b7 github.com/cockroachdb/cockroach/sql.EventLogger.InsertEventRecord(0xc82042e000, 0xc820150420, 0x173a170, 0xc, 0x500000005, 0x15af5c0, 0xc820622600, 0x3) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/sql/event_log.go:106 +0x3fc github.com/cockroachdb/cockroach/server.(*Node).recordJoinEvent.func1.1(0xc820150420, 0xc820187fa0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:663 +0x122 github.com/cockroachdb/cockroach/client.(*DB).Txn.func1(0xc820150420, 0xc820187fa0, 0x4) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:438 +0x25 github.com/cockroachdb/cockroach/client.(*Txn).Exec(0xc820150420, 0x1bd0101, 0x0, 0x0, 0xc820717c20, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/txn.go:531 +0x40b github.com/cockroachdb/cockroach/client.(*DB).Txn(0xc8203fe050, 0xc820717dc8, 0x7fe17a724678) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:436 +0x261 github.com/cockroachdb/cockroach/server.(*Node).recordJoinEvent.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:653 +0x31e github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820122660) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 91 [select]: github.com/cockroachdb/cockroach/server.(*Server).startSampleEnvironment.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/server.go:387 +0x15d github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820122980) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 92 [select]: github.com/cockroachdb/cockroach/kv.send(0x7fe17a727330, 0xc820804000, 0x1, 0x2540be400, 0xb2d05e00, 0xc821102460, 0x3, 0x3, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/send.go:171 +0x1336 github.com/cockroachdb/cockroach/kv.(*DistSender).sendRPC(0xc820427680, 0x7fe17a727330, 0xc820804000, 0x1, 0xc821102460, 0x3, 0x3, 0x1, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:345 +0x1e9 github.com/cockroachdb/cockroach/kv.(*DistSender).sendSingleRange(0xc820427680, 0x7fe17a727330, 0xc820804000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:450 +0x300 github.com/cockroachdb/cockroach/kv.(*DistSender).sendChunk.func2(0xc8206345a8, 0xc8206341b8, 0xc820634db8, 0xc820427680, 0xc820634da8, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:710 +0x448 github.com/cockroachdb/cockroach/kv.(*DistSender).sendChunk(0xc820427680, 0x7fe17a727330, 0xc820804000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:711 +0x2244 github.com/cockroachdb/cockroach/kv.(*DistSender).Send(0xc820427680, 0x7fe17a727330, 0xc820804000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:554 +0x6d5 github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc8203f8360, 0x7fe17a727330, 0xc820804000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:398 +0xc4a github.com/cockroachdb/cockroach/client.(*DB).send(0xc8203fe050, 0x0, 0x0, 0xc82081b000, 0xa6, 0x100, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:474 +0x421 github.com/cockroachdb/cockroach/client.(*DB).(github.com/cockroachdb/cockroach/client.send)-fm(0x0, 0x0, 0xc82081b000, 0xa6, 0x100, 0xc82081b000, 0x80) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:421 +0x50 github.com/cockroachdb/cockroach/client.sendAndFill(0xc820635978, 0xc820191180, 0xc82081b000, 0xc820653b10) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:386 +0x54 github.com/cockroachdb/cockroach/client.(*DB).RunWithResponse(0xc8203fe050, 0xc820191180, 0x1, 0x1) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:421 +0x6b github.com/cockroachdb/cockroach/client.(*DB).Run(0xc8203fe050, 0xc820191180, 0x1) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:412 +0x2b github.com/cockroachdb/cockroach/ts.(*DB).StoreData(0xc820120058, 0x1, 0xc8207f2000, 0xa6, 0x124, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/ts/db.go:138 +0x88d github.com/cockroachdb/cockroach/ts.(*poller).poll.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/ts/db.go:97 +0xac github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc8203bbea0, 0xc820635e70, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:166 +0xd9 github.com/cockroachdb/cockroach/ts.(*poller).poll(0xc82000e870) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/ts/db.go:100 +0x3d github.com/cockroachdb/cockroach/ts.(*poller).start.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/ts/db.go:74 +0x31 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8203d2c50) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 93 [select]: github.com/cockroachdb/cockroach/kv.send(0x7fe17a727330, 0xc82080e7e0, 0x1, 0x2540be400, 0xb2d05e00, 0xc820fbc050, 0x3, 0x3, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/send.go:171 +0x1336 github.com/cockroachdb/cockroach/kv.(*DistSender).sendRPC(0xc820427680, 0x7fe17a727330, 0xc82080e7e0, 0x1, 0xc820fbc050, 0x3, 0x3, 0x1, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:345 +0x1e9 github.com/cockroachdb/cockroach/kv.(*DistSender).sendSingleRange(0xc820427680, 0x7fe17a727330, 0xc82080e7e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:450 +0x300 github.com/cockroachdb/cockroach/kv.(*DistSender).sendChunk.func2(0xc8207a6730, 0xc8207a6340, 0xc8207a6f40, 0xc820427680, 0xc8207a6f30, 0xf, 0x16) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:710 +0x448 github.com/cockroachdb/cockroach/kv.(*DistSender).sendChunk(0xc820427680, 0x7fe17a727330, 0xc82080e7e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:711 +0x2244 github.com/cockroachdb/cockroach/kv.(*DistSender).Send(0xc820427680, 0x7fe17a727330, 0xc82080e7e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:554 +0x6d5 github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc8203f8360, 0x7fe17a727330, 0xc82080e7e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:398 +0xc4a github.com/cockroachdb/cockroach/client.(*DB).send(0xc8203fe050, 0x0, 0x0, 0xc82083cab0, 0x1, 0x1, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:474 +0x421 github.com/cockroachdb/cockroach/client.(*DB).(github.com/cockroachdb/cockroach/client.send)-fm(0x0, 0x0, 0xc82083cab0, 0x1, 0x1, 0xc82083cab0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:421 +0x50 github.com/cockroachdb/cockroach/client.sendAndFill(0xc8207a7b00, 0xc820190e00, 0x16, 0xc820702000) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:386 +0x54 github.com/cockroachdb/cockroach/client.(*DB).RunWithResponse(0xc8203fe050, 0xc820190e00, 0xc820190e00, 0x1662600) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:421 +0x6b github.com/cockroachdb/cockroach/client.(*DB).Run(0xc8203fe050, 0xc820190e00, 0xc820190e01) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:412 +0x2b github.com/cockroachdb/cockroach/client.runOneResult(0x7fe17a727c00, 0xc8203fe050, 0xc820190e00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:490 +0x5f github.com/cockroachdb/cockroach/client.(*DB).PutInline(0xc8203fe050, 0x1662600, 0xc8207a8260, 0x16722e0, 0xc820225ba0, 0xc82062ad78) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:251 +0x102 github.com/cockroachdb/cockroach/server.(*Node).writeSummaries.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:622 +0x12d github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc8203bbea0, 0xc8207a7df0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:166 +0xd9 github.com/cockroachdb/cockroach/server.(*Node).writeSummaries(0xc82034cd80, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:634 +0x5d github.com/cockroachdb/cockroach/server.(*Node).startWriteSummaries.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:589 +0x55 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8201229a0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 94 [select]: github.com/cockroachdb/cockroach/sql.(*SchemaChangeManager).Start.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/sql/schema_changer.go:450 +0x16f8 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8201229c0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 95 [select]: github.com/cockroachdb/cockroach/kv.send(0x7fe17a727330, 0xc82000f2c0, 0x1, 0x2540be400, 0xb2d05e00, 0xc8284de050, 0x3, 0x3, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/send.go:171 +0x1336 github.com/cockroachdb/cockroach/kv.(*DistSender).sendRPC(0xc820427680, 0x7fe17a727330, 0xc82000f2c0, 0x77, 0xc8284de050, 0x3, 0x3, 0x1, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:345 +0x1e9 github.com/cockroachdb/cockroach/kv.(*DistSender).sendSingleRange(0xc820427680, 0x7fe17a727330, 0xc82000f2c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:450 +0x300 github.com/cockroachdb/cockroach/kv.(*DistSender).sendChunk.func2(0xc820fea6d8, 0xc820fea2e8, 0xc820feaee8, 0xc820427680, 0xc820feaed8, 0x10, 0x10) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:710 +0x448 github.com/cockroachdb/cockroach/kv.(*DistSender).sendChunk(0xc820427680, 0x7fe17a727330, 0xc82000f2c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:711 +0x2244 github.com/cockroachdb/cockroach/kv.(*DistSender).Send(0xc820427680, 0x7fe17a727330, 0xc82000f2c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/dist_sender.go:554 +0x6d5 github.com/cockroachdb/cockroach/kv.(*TxnCoordSender).Send(0xc8203f8360, 0x7fe17a727330, 0xc82000f2c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/txn_coord_sender.go:398 +0xc4a github.com/cockroachdb/cockroach/client.(*DB).send(0xc8203fe050, 0x0, 0x0, 0xc8203d2c90, 0x1, 0x1, 0x17, 0x1) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:474 +0x421 github.com/cockroachdb/cockroach/client.(*DB).(github.com/cockroachdb/cockroach/client.send)-fm(0x0, 0x0, 0xc8203d2c90, 0x1, 0x1, 0x139c580, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:421 +0x50 github.com/cockroachdb/cockroach/client.sendAndFill(0xc820febaa8, 0xc820440a80, 0xc82000f260, 0x16dc3d0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:386 +0x54 github.com/cockroachdb/cockroach/client.(*DB).RunWithResponse(0xc8203fe050, 0xc820440a80, 0x5a535d, 0x24a4be0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:421 +0x6b github.com/cockroachdb/cockroach/client.(*DB).Run(0xc8203fe050, 0xc820440a80, 0x7fe17a727c00) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:412 +0x2b github.com/cockroachdb/cockroach/client.runOneRow(0x7fe17a727c00, 0xc8203fe050, 0xc820440a80, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:498 +0x4b github.com/cockroachdb/cockroach/client.(*DB).Get(0xc8203fe050, 0x1662600, 0xc820122a20, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/client/db.go:198 +0xf6 github.com/cockroachdb/cockroach/server.(*Server).maybeRunPeriodicCheck(0xc82034de00, 0x1742430, 0xd, 0xc82011ef00, 0xf, 0xf, 0xc820febe18, 0xecea30f45) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/updates.go:121 +0xb0 github.com/cockroachdb/cockroach/server.(*Server).maybeCheckForUpdates(0xc82034de00, 0xc82e61a239) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/updates.go:111 +0x75 github.com/cockroachdb/cockroach/server.(*Server).periodicallyCheckForUpdates.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/updates.go:93 +0x59 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8203d2c60) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 44 [select]: google.golang.org/grpc/transport.(*http2Client).controller(0xc820a8e000) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:835 +0x5da created by google.golang.org/grpc/transport.newHTTP2Client /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:194 +0x153b goroutine 364 [chan send]: github.com/cockroachdb/cockroach/storage.(*Store).(github.com/cockroachdb/cockroach/storage.enqueueRaftMessage)-fm(0xc820a38a20, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x61 github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x7fe17a6f1e38, 0xc8202c8300, 0xc820436150, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:124 +0x259 github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:128 +0x48 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820bc2930) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 190 [select]: google.golang.org/grpc/transport.(*http2Client).controller(0xc8216dcff0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:835 +0x5da created by google.golang.org/grpc/transport.newHTTP2Client /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:194 +0x153b goroutine 223 [select]: google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc820b720f0, 0xc8210dc150, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:141 +0x7e6 google.golang.org/grpc/transport.(*Stream).Read(0xc8211a8ee0, 0xc8210dc150, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fe17a727398, 0xc8211a8ee0, 0xc8210dc150, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727398, 0xc8211a8ee0, 0xc8210dc150, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 google.golang.org/grpc.(*parser).recvMsg(0xc8210dc140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:218 +0xb9 google.golang.org/grpc.recv(0xc8210dc140, 0x7fe17a6a3208, 0x24a6c10, 0xc8211a8ee0, 0x0, 0x0, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:302 +0x45 google.golang.org/grpc.(*clientStream).RecvMsg(0xc820b9c0a0, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:232 +0xac github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).RecvMsg(0xc8210d9870, 0x16734e0, 0x24a6c10, 0x0, 0x0) :236 +0x6f github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func2.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:226 +0x79 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8210dc260) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 101 [select]: google.golang.org/grpc.(*Conn).transportMonitor(0xc8200cac40) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:510 +0x1d3 google.golang.org/grpc.NewConn.func1(0xc8200cac40) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:321 +0x1b5 created by google.golang.org/grpc.NewConn /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:322 +0x4dd goroutine 102 [select]: github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc82042fc70, 0xc82041f7c0, 0xc820410b40, 0x17, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0x5e5 github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:157 +0x66 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc82000f920) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 115 [IO wait]: net.runtime_pollWait(0x7fe17a6a4490, 0x72, 0x0) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc82042e370, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc82042e370, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).accept(0xc82042e310, 0x0, 0x7fe17a6ef6c0, 0xc8282d7140) /home/marc/go/src/net/fd_unix.go:426 +0x27c net.(*TCPListener).AcceptTCP(0xc820120238, 0xc82071bed8, 0x0, 0x0) /home/marc/go/src/net/tcpsock_posix.go:254 +0x4d net.(*TCPListener).Accept(0xc820120238, 0x0, 0x0, 0x0, 0x0) /home/marc/go/src/net/tcpsock_posix.go:264 +0x3d github.com/cockroachdb/cmux.(*cMux).Serve(0xc8203fc580, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cmux/cmux.go:110 +0x9d github.com/cockroachdb/cockroach/server.(*Server).Start.func9() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/server.go:370 +0x32 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820123d80) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 436 [select]: google.golang.org/grpc.(*Conn).Wait(0xc8211a8000, 0x7fe17a6a3408, 0xc820a34140, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:551 +0x360 google.golang.org/grpc.(*unicastPicker).Pick(0xc827fea020, 0x7fe17a6a3408, 0xc820a34140, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/picker.go:84 +0x51 google.golang.org/grpc.NewClientStream(0x7fe17a6a3408, 0xc820a34140, 0x2197920, 0xc82bf12000, 0x18670e0, 0x28, 0x0, 0x0, 0x0, 0x0, ...) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:105 +0xb8 github.com/cockroachdb/cockroach/storage.(*multiRaftClient).RaftMessage(0xc8215e0000, 0x7fe17a6a3408, 0xc820a34140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:111 +0xe9 github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc820436150, 0xc800000003) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:204 +0x997 github.com/cockroachdb/cockroach/storage.(*RaftTransport).Send.func1.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:289 +0x2b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8210dd200) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 207 [select]: google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc820b9a4b0, 0xc8208fec70, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:141 +0x7e6 google.golang.org/grpc/transport.(*Stream).Read(0xc820cfc0e0, 0xc8208fec70, 0x5, 0x5, 0x10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fe17a727398, 0xc820cfc0e0, 0xc8208fec70, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727398, 0xc820cfc0e0, 0xc8208fec70, 0x5, 0x5, 0x1, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 google.golang.org/grpc.(*parser).recvMsg(0xc8208fec60, 0xc820c900b8, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:218 +0xb9 google.golang.org/grpc.recv(0xc8208fec60, 0x7fe17a6a3208, 0x24a6c10, 0xc820cfc0e0, 0x0, 0x0, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:302 +0x45 google.golang.org/grpc.(*clientStream).RecvMsg(0xc820b681e0, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:232 +0xac github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).RecvMsg(0xc820c889d0, 0x16734e0, 0x24a6c10, 0x0, 0x0) :236 +0x6f github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func2.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:226 +0x79 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8208fecc0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 47 [IO wait]: net.runtime_pollWait(0x7fe17a6a3c50, 0x72, 0xc82078e400) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc820b42060, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc820b42060, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc820b42000, 0xc82078e400, 0x400, 0x400, 0x0, 0x7fe17a71e050, 0xc820012008) /home/marc/go/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc8201340f8, 0xc82078e400, 0x400, 0x400, 0x0, 0x0, 0x0) /home/marc/go/src/net/net.go:172 +0xe4 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc820b36000, 0xc82078e400, 0x400, 0x400, 0xd, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cmux/cmux.go:204 +0x117 crypto/tls.(*block).readFromUntil(0xc8206a8180, 0x7fe17a728178, 0xc820b36000, 0x5, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:460 +0xcc crypto/tls.(*Conn).readRecord(0xc8207b6000, 0x1a68017, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:562 +0x2d1 crypto/tls.(*Conn).Read(0xc8207b6000, 0xc8214d6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:939 +0x167 bufio.(*Reader).fill(0xc821483ce0) /home/marc/go/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc821483ce0, 0xc82145d6b8, 0x9, 0x9, 0x6, 0x0, 0x0) /home/marc/go/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fe17a727270, 0xc821483ce0, 0xc82145d6b8, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727270, 0xc821483ce0, 0xc82145d6b8, 0x9, 0x9, 0xc8214a9a48, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 golang.org/x/net/http2.readFrameHeader(0xc82145d6b8, 0x9, 0x9, 0x7fe17a727270, 0xc821483ce0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:236 +0xa5 golang.org/x/net/http2.(*Framer).ReadFrame(0xc82145d680, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:463 +0x106 google.golang.org/grpc/transport.(*framer).readFrame(0xc821487c20, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http_util.go:406 +0x3d google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc821476ab0, 0xc821487cb0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:247 +0x646 google.golang.org/grpc.(*Server).serveStreams(0xc8203c4000, 0x7fe17a6aeb08, 0xc821476ab0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:325 +0x159 google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc8203c4000, 0x7fe17a7271c0, 0xc8207b6000, 0x7fe17a6aeae0, 0xc821467080) /home/marc/cockroach/src/google.golang.org/grpc/server.go:312 +0x49d google.golang.org/grpc.(*Server).handleRawConn(0xc8203c4000, 0x7fe17a728118, 0xc820b36000) /home/marc/cockroach/src/google.golang.org/grpc/server.go:289 +0x4ee created by google.golang.org/grpc.(*Server).Serve /home/marc/cockroach/src/google.golang.org/grpc/server.go:261 +0x372 goroutine 130 [IO wait]: net.runtime_pollWait(0x7fe17a6a4190, 0x72, 0xc820480800) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc8202740d0, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc8202740d0, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc820274070, 0xc820480800, 0x400, 0x400, 0x0, 0x7fe17a71e050, 0xc820012008) /home/marc/go/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc820120260, 0xc820480800, 0x400, 0x400, 0x0, 0x0, 0x0) /home/marc/go/src/net/net.go:172 +0xe4 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc8206580b0, 0xc820480800, 0x400, 0x400, 0xd, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cmux/cmux.go:204 +0x117 crypto/tls.(*block).readFromUntil(0xc820804780, 0x7fe17a728178, 0xc8206580b0, 0x5, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:460 +0xcc crypto/tls.(*Conn).readRecord(0xc8201ea600, 0x1a68017, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:562 +0x2d1 crypto/tls.(*Conn).Read(0xc8201ea600, 0xc820a98000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:939 +0x167 bufio.(*Reader).fill(0xc820a2d860) /home/marc/go/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc820a2d860, 0xc8209fef38, 0x9, 0x9, 0x9, 0x0, 0x0) /home/marc/go/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fe17a727270, 0xc820a2d860, 0xc8209fef38, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727270, 0xc820a2d860, 0xc8209fef38, 0x9, 0x9, 0x601b4e68ec47bc01, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 golang.org/x/net/http2.readFrameHeader(0xc8209fef38, 0x9, 0x9, 0x7fe17a727270, 0xc820a2d860, 0x7a000000, 0xc800000000, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:236 +0xa5 golang.org/x/net/http2.(*Framer).ReadFrame(0xc8209fef00, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:463 +0x106 google.golang.org/grpc/transport.(*framer).readFrame(0xc82072a9c0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http_util.go:406 +0x3d google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc820a197a0, 0xc82072aa50) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:247 +0x646 google.golang.org/grpc.(*Server).serveStreams(0xc8203c4000, 0x7fe17a6aeb08, 0xc820a197a0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:325 +0x159 google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc8203c4000, 0x7fe17a7271c0, 0xc8201ea600, 0x7fe17a6aeae0, 0xc820a40790) /home/marc/cockroach/src/google.golang.org/grpc/server.go:312 +0x49d google.golang.org/grpc.(*Server).handleRawConn(0xc8203c4000, 0x7fe17a728118, 0xc8206580b0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:289 +0x4ee created by google.golang.org/grpc.(*Server).Serve /home/marc/cockroach/src/google.golang.org/grpc/server.go:261 +0x372 goroutine 48 [select]: google.golang.org/grpc/transport.(*http2Server).controller(0xc821476ab0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:620 +0x5da created by google.golang.org/grpc/transport.newHTTP2Server /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:134 +0x853 goroutine 169 [select]: github.com/cockroachdb/cockroach/storage.(*Replica).requestLeaderLease.func1.1(0x144595949ed45b7f, 0xc800000000, 0xc82014ee00, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:492 +0x7c4 github.com/cockroachdb/cockroach/storage.(*Replica).requestLeaderLease.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:455 +0x45 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask.func1(0xc8203bbea0, 0x1ec2389, 0x12, 0x200, 0xc821626160) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:181 +0x62 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:182 +0xc9 goroutine 189 [select]: google.golang.org/grpc/transport.(*http2Server).controller(0xc8214fe990) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:620 +0x5da created by google.golang.org/grpc/transport.newHTTP2Server /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:134 +0x853 goroutine 188 [IO wait]: net.runtime_pollWait(0x7fe17a6a3e90, 0x72, 0xc820481000) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc820c9c220, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc820c9c220, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc820c9c1c0, 0xc820481000, 0x400, 0x400, 0x0, 0x7fe17a71e050, 0xc820012008) /home/marc/go/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc8215e0050, 0xc820481000, 0x400, 0x400, 0x0, 0x0, 0x0) /home/marc/go/src/net/net.go:172 +0xe4 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc820ede2c0, 0xc820481000, 0x400, 0x400, 0xd, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cmux/cmux.go:204 +0x117 crypto/tls.(*block).readFromUntil(0xc820c70d20, 0x7fe17a728178, 0xc820ede2c0, 0x5, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:460 +0xcc crypto/tls.(*Conn).readRecord(0xc8201eac00, 0x1a68017, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:562 +0x2d1 crypto/tls.(*Conn).Read(0xc8201eac00, 0xc821542000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:939 +0x167 bufio.(*Reader).fill(0xc82152e7e0) /home/marc/go/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc82152e7e0, 0xc8216f87b8, 0x9, 0x9, 0x19, 0x0, 0x0) /home/marc/go/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fe17a727270, 0xc82152e7e0, 0xc8216f87b8, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727270, 0xc82152e7e0, 0xc8216f87b8, 0x9, 0x9, 0xc821635a48, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 golang.org/x/net/http2.readFrameHeader(0xc8216f87b8, 0x9, 0x9, 0x7fe17a727270, 0xc82152e7e0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:236 +0xa5 golang.org/x/net/http2.(*Framer).ReadFrame(0xc8216f8780, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:463 +0x106 google.golang.org/grpc/transport.(*framer).readFrame(0xc82153e180, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http_util.go:406 +0x3d google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc8214fe990, 0xc82153e210) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:247 +0x646 google.golang.org/grpc.(*Server).serveStreams(0xc8203c4000, 0x7fe17a6aeb08, 0xc8214fe990) /home/marc/cockroach/src/google.golang.org/grpc/server.go:325 +0x159 google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc8203c4000, 0x7fe17a7271c0, 0xc8201eac00, 0x7fe17a6aeae0, 0xc8214eba20) /home/marc/cockroach/src/google.golang.org/grpc/server.go:312 +0x49d google.golang.org/grpc.(*Server).handleRawConn(0xc8203c4000, 0x7fe17a728118, 0xc820ede2c0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:289 +0x4ee created by google.golang.org/grpc.(*Server).Serve /home/marc/cockroach/src/google.golang.org/grpc/server.go:261 +0x372 goroutine 131 [select]: google.golang.org/grpc/transport.(*http2Server).controller(0xc820a197a0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:620 +0x5da created by google.golang.org/grpc/transport.newHTTP2Server /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:134 +0x853 goroutine 132 [select]: google.golang.org/grpc/transport.(*http2Client).controller(0xc8209ce960) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:835 +0x5da created by google.golang.org/grpc/transport.newHTTP2Client /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:194 +0x153b goroutine 40 [select]: github.com/cockroachdb/cockroach/storage.(*Replica).requestLeaderLease.func1.1(0x144595942c68c095, 0x0, 0xc82014f0a0, 0x5ebaf0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:492 +0x7c4 github.com/cockroachdb/cockroach/storage.(*Replica).requestLeaderLease.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:455 +0x45 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask.func1(0xc8203bbea0, 0x1ec2389, 0x12, 0x200, 0xc8207a8160) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:181 +0x62 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:182 +0xc9 goroutine 45 [IO wait]: net.runtime_pollWait(0x7fe17a6a40d0, 0x72, 0xc820328800) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc82044c370, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc82044c370, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc82044c310, 0xc820328800, 0x400, 0x400, 0x0, 0x7fe17a71e050, 0xc820012008) /home/marc/go/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc8201201d8, 0xc820328800, 0x400, 0x400, 0x0, 0x0, 0x0) /home/marc/go/src/net/net.go:172 +0xe4 crypto/tls.(*block).readFromUntil(0xc82080ecc0, 0x7fe17a726dc0, 0xc8201201d8, 0x5, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:460 +0xcc crypto/tls.(*Conn).readRecord(0xc8201ea300, 0x1a68017, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:562 +0x2d1 crypto/tls.(*Conn).Read(0xc8201ea300, 0xc820a78000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:939 +0x167 bufio.(*Reader).fill(0xc820434ba0) /home/marc/go/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc820434ba0, 0xc8206780f8, 0x9, 0x9, 0xc81febff77, 0x0, 0x0) /home/marc/go/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fe17a727270, 0xc820434ba0, 0xc8206780f8, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727270, 0xc820434ba0, 0xc8206780f8, 0x9, 0x9, 0xc822800e78, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 golang.org/x/net/http2.readFrameHeader(0xc8206780f8, 0x9, 0x9, 0x7fe17a727270, 0xc820434ba0, 0x0, 0xc800000000, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:236 +0xa5 golang.org/x/net/http2.(*Framer).ReadFrame(0xc8206780c0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:463 +0x106 google.golang.org/grpc/transport.(*framer).readFrame(0xc82080f4a0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http_util.go:406 +0x3d google.golang.org/grpc/transport.(*http2Client).reader(0xc820a8e000) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:757 +0x109 created by google.golang.org/grpc/transport.newHTTP2Client /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:200 +0x159a goroutine 120 [IO wait]: net.runtime_pollWait(0x7fe17a6a3d10, 0x72, 0xc822e08000) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc8202cb560, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc8202cb560, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc8202cb500, 0xc822e08000, 0x8000, 0x8000, 0x0, 0x7fe17a71e050, 0xc820012008) /home/marc/go/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc820120388, 0xc822e08000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/net/net.go:172 +0xe4 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc8202ae9a0, 0xc822e08000, 0x8000, 0x8000, 0xd, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cmux/cmux.go:204 +0x117 crypto/tls.(*block).readFromUntil(0xc82079e420, 0x7fe17a728178, 0xc8202ae9a0, 0x5, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:460 +0xcc crypto/tls.(*Conn).readRecord(0xc8201eaf00, 0x1a68017, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:562 +0x2d1 crypto/tls.(*Conn).Read(0xc8201eaf00, 0xc820a5a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:939 +0x167 bufio.(*Reader).fill(0xc8209e4960) /home/marc/go/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc8209e4960, 0xc820977178, 0x9, 0x9, 0x19, 0x0, 0x0) /home/marc/go/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fe17a727270, 0xc8209e4960, 0xc820977178, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727270, 0xc8209e4960, 0xc820977178, 0x9, 0x9, 0xc82145ba58, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 golang.org/x/net/http2.readFrameHeader(0xc820977178, 0x9, 0x9, 0x7fe17a727270, 0xc8209e4960, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:236 +0xa5 golang.org/x/net/http2.(*Framer).ReadFrame(0xc820977140, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:463 +0x106 google.golang.org/grpc/transport.(*framer).readFrame(0xc8201ae5d0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http_util.go:406 +0x3d google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc8209b75f0, 0xc8201ae690) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:247 +0x646 google.golang.org/grpc.(*Server).serveStreams(0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:325 +0x159 google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc8203c4000, 0x7fe17a7271c0, 0xc8201eaf00, 0x7fe17a6aeae0, 0xc820996dc0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:312 +0x49d google.golang.org/grpc.(*Server).handleRawConn(0xc8203c4000, 0x7fe17a728118, 0xc8202ae9a0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:289 +0x4ee created by google.golang.org/grpc.(*Server).Serve /home/marc/cockroach/src/google.golang.org/grpc/server.go:261 +0x372 goroutine 206 [select]: google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc820b9a420, 0xc8208febd0, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:141 +0x7e6 google.golang.org/grpc/transport.(*Stream).Read(0xc820cfc000, 0xc8208febd0, 0x5, 0x5, 0x2, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fe17a727398, 0xc820cfc000, 0xc8208febd0, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727398, 0xc820cfc000, 0xc8208febd0, 0x5, 0x5, 0x5a65e4, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 google.golang.org/grpc.(*parser).recvMsg(0xc8208febc0, 0xc820622838, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:218 +0xb9 google.golang.org/grpc.recv(0xc8208febc0, 0x7fe17a6a3208, 0x24a6c10, 0xc820cfc000, 0x0, 0x0, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:302 +0x45 google.golang.org/grpc.(*clientStream).RecvMsg(0xc820b680a0, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:232 +0xac github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).RecvMsg(0xc820c88990, 0x16734e0, 0x24a6c10, 0x0, 0x0) :236 +0x6f github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func2.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:226 +0x79 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8208feca0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 133 [IO wait]: net.runtime_pollWait(0x7fe17a6a3dd0, 0x72, 0xc820938000) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc82044c680, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc82044c680, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc82044c620, 0xc820938000, 0x400, 0x400, 0x0, 0x7fe17a71e050, 0xc820012008) /home/marc/go/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc820640048, 0xc820938000, 0x400, 0x400, 0x0, 0x0, 0x0) /home/marc/go/src/net/net.go:172 +0xe4 crypto/tls.(*block).readFromUntil(0xc820437200, 0x7fe17a726dc0, 0xc820640048, 0x5, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:460 +0xcc crypto/tls.(*Conn).readRecord(0xc8207b6300, 0x1a68017, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:562 +0x2d1 crypto/tls.(*Conn).Read(0xc8207b6300, 0xc820aaa000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:939 +0x167 bufio.(*Reader).fill(0xc820a2daa0) /home/marc/go/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc820a2daa0, 0xc8209feff8, 0x9, 0x9, 0x9, 0x0, 0x0) /home/marc/go/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fe17a727270, 0xc820a2daa0, 0xc8209feff8, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727270, 0xc820a2daa0, 0xc8209feff8, 0x9, 0x9, 0x601b4e68ec47bc01, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 golang.org/x/net/http2.readFrameHeader(0xc8209feff8, 0x9, 0x9, 0x7fe17a727270, 0xc820a2daa0, 0x7a000000, 0xc800000000, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:236 +0xa5 golang.org/x/net/http2.(*Framer).ReadFrame(0xc8209fefc0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:463 +0x106 google.golang.org/grpc/transport.(*framer).readFrame(0xc82072ade0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http_util.go:406 +0x3d google.golang.org/grpc/transport.(*http2Client).reader(0xc8209ce960) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:757 +0x109 created by google.golang.org/grpc/transport.newHTTP2Client /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:200 +0x159a goroutine 134 [select]: google.golang.org/grpc.(*Conn).transportMonitor(0xc8200cad20) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:510 +0x1d3 created by google.golang.org/grpc.NewConn /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:312 +0x499 goroutine 135 [chan receive]: github.com/cockroachdb/cockroach/server.RegisterAdminHandlerFromEndpoint.func1.1(0x7fe17a6a3408, 0xc8203fc3c0, 0xc82041f860, 0xc820410300, 0x17) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/admin.pb.gw.go:186 +0x51 created by github.com/cockroachdb/cockroach/server.RegisterAdminHandlerFromEndpoint.func1 /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/admin.pb.gw.go:190 +0x205 goroutine 129 [select, locked to thread]: runtime.gopark(0x1a67fa8, 0xc8205b2728, 0x171b6c0, 0x6, 0x18, 0x2) /home/marc/go/src/runtime/proc.go:262 +0x163 runtime.selectgoImpl(0xc8205b2728, 0x0, 0x18) /home/marc/go/src/runtime/select.go:392 +0xa67 runtime.selectgo(0xc8205b2728) /home/marc/go/src/runtime/select.go:215 +0x12 runtime.ensureSigM.func1() /home/marc/go/src/runtime/signal1_unix.go:279 +0x358 runtime.goexit() /home/marc/go/src/runtime/asm_amd64.s:1998 +0x1 goroutine 146 [IO wait]: net.runtime_pollWait(0x7fe17a6a4250, 0x72, 0xc823780000) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc820684060, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc820684060, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc820684000, 0xc823780000, 0x8000, 0x8000, 0x0, 0x7fe17a71e050, 0xc820012008) /home/marc/go/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc8201202d8, 0xc823780000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/net/net.go:172 +0xe4 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc820ede000, 0xc823780000, 0x8000, 0x8000, 0xd, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cmux/cmux.go:204 +0x117 crypto/tls.(*block).readFromUntil(0xc820800360, 0x7fe17a728178, 0xc820ede000, 0x5, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:460 +0xcc crypto/tls.(*Conn).readRecord(0xc8201ea900, 0x1a68017, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:562 +0x2d1 crypto/tls.(*Conn).Read(0xc8201ea900, 0xc82140e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:939 +0x167 bufio.(*Reader).fill(0xc8206d5ce0) /home/marc/go/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc8206d5ce0, 0xc8209776b8, 0x9, 0x9, 0x6, 0x0, 0x0) /home/marc/go/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fe17a727270, 0xc8206d5ce0, 0xc8209776b8, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727270, 0xc8206d5ce0, 0xc8209776b8, 0x9, 0x9, 0xc820729a48, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 golang.org/x/net/http2.readFrameHeader(0xc8209776b8, 0x9, 0x9, 0x7fe17a727270, 0xc8206d5ce0, 0x1000000, 0xc800000000, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:236 +0xa5 golang.org/x/net/http2.(*Framer).ReadFrame(0xc820977680, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:463 +0x106 google.golang.org/grpc/transport.(*framer).readFrame(0xc820807a70, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http_util.go:406 +0x3d google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc820a18bd0, 0xc820807b00) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:247 +0x646 google.golang.org/grpc.(*Server).serveStreams(0xc8203c4000, 0x7fe17a6aeb08, 0xc820a18bd0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:325 +0x159 google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc8203c4000, 0x7fe17a7271c0, 0xc8201ea900, 0x7fe17a6aeae0, 0xc820f22160) /home/marc/cockroach/src/google.golang.org/grpc/server.go:312 +0x49d google.golang.org/grpc.(*Server).handleRawConn(0xc8203c4000, 0x7fe17a728118, 0xc820ede000) /home/marc/cockroach/src/google.golang.org/grpc/server.go:289 +0x4ee created by google.golang.org/grpc.(*Server).Serve /home/marc/cockroach/src/google.golang.org/grpc/server.go:261 +0x372 goroutine 147 [select]: google.golang.org/grpc/transport.(*http2Server).controller(0xc820a18bd0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:620 +0x5da created by google.golang.org/grpc/transport.newHTTP2Server /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:134 +0x853 goroutine 322 [select]: google.golang.org/grpc.(*Conn).transportMonitor(0xc820fce2a0) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:510 +0x1d3 google.golang.org/grpc.NewConn.func1(0xc820fce2a0) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:321 +0x1b5 created by google.golang.org/grpc.NewConn /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:322 +0x4dd goroutine 222 [select]: google.golang.org/grpc.NewClientStream.func1(0x7fe17a7272c0, 0xc820a8e000, 0xc8211a8fc0, 0xc820b9c1e0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:149 +0x258 created by google.golang.org/grpc.NewClientStream /home/marc/cockroach/src/google.golang.org/grpc/stream.go:157 +0xab2 goroutine 253 [select]: google.golang.org/grpc.(*Conn).transportMonitor(0xc820812620) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:510 +0x1d3 google.golang.org/grpc.NewConn.func1(0xc820812620) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:321 +0x1b5 created by google.golang.org/grpc.NewConn /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:322 +0x4dd goroutine 236 [select]: github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc820436150, 0xc800000001) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:238 +0x1720 github.com/cockroachdb/cockroach/storage.(*RaftTransport).Send.func1.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:289 +0x2b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820c37ce0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 463 [select]: google.golang.org/grpc/transport.(*Stream).Header(0xc82087e460, 0xc82087e460, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:221 +0x242 google.golang.org/grpc.recvResponse(0x7fe17a6a3208, 0x24a6c10, 0x0, 0x0, 0x0, 0x0, 0x7fe17a65e078, 0xc820d82020, 0x0, 0x0, ...) /home/marc/cockroach/src/google.golang.org/grpc/call.go:54 +0x4e google.golang.org/grpc.Invoke(0x7fe17a6a4cc0, 0xc8225d3a10, 0x1822fe0, 0x21, 0x16e0cc0, 0xc82092a148, 0x1692520, 0xc820ba6c00, 0xc820a3e0a0, 0x0, ...) /home/marc/cockroach/src/google.golang.org/grpc/call.go:178 +0xae6 github.com/cockroachdb/cockroach/roachpb.(*internalClient).Batch(0xc82108a050, 0x7fe17a6a4cc0, 0xc8225d3a10, 0xc82092a148, 0x0, 0x0, 0x0, 0xc820fae970, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/roachpb/api.pb.go:1284 +0xec github.com/cockroachdb/cockroach/kv.sendOne.func1(0x7fe17a727330, 0xc820804000, 0x1, 0x2540be400, 0xb2d05e00, 0xc82092a120, 0xc820bb4960, 0xc8204109c0, 0x17) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/send.go:286 +0x658 created by github.com/cockroachdb/cockroach/kv.sendOne /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/send.go:288 +0x4c9 goroutine 360 [select]: google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc82252b710, 0xc8258d8670, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:141 +0x7e6 google.golang.org/grpc/transport.(*Stream).Read(0xc821447260, 0xc8258d8670, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fe17a727398, 0xc821447260, 0xc8258d8670, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727398, 0xc821447260, 0xc8258d8670, 0x5, 0x5, 0x5a81df, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 google.golang.org/grpc.(*parser).recvMsg(0xc8258d8660, 0xd63770, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:218 +0xb9 google.golang.org/grpc.recv(0xc8258d8660, 0x7fe17a6a3208, 0x24a6c10, 0xc821447260, 0x0, 0x0, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:302 +0x45 google.golang.org/grpc.(*clientStream).RecvMsg(0xc82578e000, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:232 +0xac github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).RecvMsg(0xc8285010f0, 0x16734e0, 0x24a6c10, 0x0, 0x0) :236 +0x6f github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func2.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:226 +0x79 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820396440) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 221 [select]: google.golang.org/grpc.NewClientStream.func1(0x7fe17a7272c0, 0xc820a8e000, 0xc8211a8ee0, 0xc820b9c0a0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:149 +0x258 created by google.golang.org/grpc.NewClientStream /home/marc/cockroach/src/google.golang.org/grpc/stream.go:157 +0xab2 goroutine 477 [select]: google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc8237c6180, 0xc82148aa30, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:141 +0x7e6 google.golang.org/grpc/transport.(*Stream).Read(0xc8214462a0, 0xc82148aa30, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fe17a727398, 0xc8214462a0, 0xc82148aa30, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727398, 0xc8214462a0, 0xc82148aa30, 0x5, 0x5, 0xc82001f450, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 google.golang.org/grpc.(*parser).recvMsg(0xc82148aa20, 0x1a668c8, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:218 +0xb9 google.golang.org/grpc.recv(0xc82148aa20, 0x7fe17a6a3208, 0x24a6c10, 0xc8214462a0, 0x0, 0x0, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:302 +0x45 google.golang.org/grpc.(*clientStream).RecvMsg(0xc820d74640, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:232 +0xac github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).RecvMsg(0xc8284ba7b0, 0x16734e0, 0x24a6c10, 0x0, 0x0) :236 +0x6f github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func2.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:226 +0x79 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc82148ab00) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 191 [IO wait]: net.runtime_pollWait(0x7fe17a6a3b90, 0x72, 0xc820939400) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc820c56530, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc820c56530, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc820c564d0, 0xc820939400, 0x400, 0x400, 0x0, 0x7fe17a71e050, 0xc820012008) /home/marc/go/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc821736050, 0xc820939400, 0x400, 0x400, 0x0, 0x0, 0x0) /home/marc/go/src/net/net.go:172 +0xe4 crypto/tls.(*block).readFromUntil(0xc820c630b0, 0x7fe17a726dc0, 0xc821736050, 0x5, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:460 +0xcc crypto/tls.(*Conn).readRecord(0xc8207b6900, 0x1a68017, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:562 +0x2d1 crypto/tls.(*Conn).Read(0xc8207b6900, 0xc821552000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:939 +0x167 bufio.(*Reader).fill(0xc82152ea20) /home/marc/go/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc82152ea20, 0xc8216f8878, 0x9, 0x9, 0xc81ff5c5e0, 0x0, 0x0) /home/marc/go/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fe17a727270, 0xc82152ea20, 0xc8216f8878, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727270, 0xc82152ea20, 0xc8216f8878, 0x9, 0x9, 0xc8210742a8, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 golang.org/x/net/http2.readFrameHeader(0xc8216f8878, 0x9, 0x9, 0x7fe17a727270, 0xc82152ea20, 0x0, 0xc800000000, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:236 +0xa5 golang.org/x/net/http2.(*Framer).ReadFrame(0xc8216f8840, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:463 +0x106 google.golang.org/grpc/transport.(*framer).readFrame(0xc82153e2d0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http_util.go:406 +0x3d google.golang.org/grpc/transport.(*http2Client).reader(0xc8216dcff0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:757 +0x109 created by google.golang.org/grpc/transport.newHTTP2Client /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:200 +0x159a goroutine 479 [select]: github.com/cockroachdb/cockroach/storage.(*Replica).redirectOnOrAcquireLeaderLease(0xc82014f0a0, 0x7fe17a727330, 0xc8235a0240, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:558 +0x6f3 github.com/cockroachdb/cockroach/storage.(*Replica).addReadOnlyCmd(0xc82014f0a0, 0x7fe17a727330, 0xc8235a0240, 0x14459595bccb4dd0, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc821233c80, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:1046 +0xcc github.com/cockroachdb/cockroach/storage.(*Replica).Send(0xc82014f0a0, 0x7fe17a727330, 0xc8235a0240, 0x14459595bccb4dd0, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc821233c80, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:765 +0x500 github.com/cockroachdb/cockroach/storage.(*Store).Send(0xc820156200, 0x7fe17a727330, 0xc8235a0240, 0x14459595bccb4dd0, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc821233c80, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:1640 +0xd71 github.com/cockroachdb/cockroach/storage.(*Stores).Send(0xc8203fc380, 0x7fe17a727330, 0xc8235a01e0, 0x0, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc821233c80, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/stores.go:178 +0x501 github.com/cockroachdb/cockroach/server.(*Node).Batch.func2() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:729 +0x55b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc8203bbea0, 0xc820631880, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:166 +0xd9 github.com/cockroachdb/cockroach/server.(*Node).Batch(0xc82034cd80, 0x7fe17a727330, 0xc8237c7f80, 0xc823857c00, 0xc828500070, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:741 +0x2c0 github.com/cockroachdb/cockroach/roachpb._Internal_Batch_Handler(0x16c7380, 0xc82034cd80, 0x7fe17a727330, 0xc8237c7f80, 0xc82091dd80, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/roachpb/api.pb.go:1306 +0x132 google.golang.org/grpc.(*Server).processUnaryRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0, 0xc821446460, 0xc820410180, 0x2195870, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:497 +0xe13 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0, 0xc821446460, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:646 +0x109d google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc82079d690, 0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0, 0xc821446460) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 395 [IO wait]: net.runtime_pollWait(0x7fe17a6a4010, 0x72, 0xc82300c000) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc82be4e060, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc82be4e060, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc82be4e000, 0xc82300c000, 0x1000, 0x1000, 0x0, 0x7fe17a71e050, 0xc820012008) /home/marc/go/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc820026040, 0xc82300c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /home/marc/go/src/net/net.go:172 +0xe4 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc82ba9dd90, 0xc82300c000, 0x1000, 0x1000, 0xd, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cmux/cmux.go:204 +0x117 crypto/tls.(*block).readFromUntil(0xc82bb10150, 0x7fe17a728178, 0xc82ba9dd90, 0x5, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:460 +0xcc crypto/tls.(*Conn).readRecord(0xc8207b6f00, 0x1a68017, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:562 +0x2d1 crypto/tls.(*Conn).Read(0xc8207b6f00, 0xc821233e38, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:939 +0x167 io.ReadAtLeast(0x7fe17a727220, 0xc8207b6f00, 0xc821233e38, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727220, 0xc8207b6f00, 0xc821233e38, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 golang.org/x/net/http2.readFrameHeader(0xc821233e38, 0x9, 0x9, 0x7fe17a727220, 0xc8207b6f00, 0x0, 0xc800000000, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:236 +0xa5 golang.org/x/net/http2.(*Framer).ReadFrame(0xc821233e00, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:463 +0x106 golang.org/x/net/http2.(*serverConn).readFrames(0xc821500c00) /home/marc/cockroach/src/golang.org/x/net/http2/server.go:602 +0x99 created by golang.org/x/net/http2.(*serverConn).serve /home/marc/cockroach/src/golang.org/x/net/http2/server.go:700 +0x720 goroutine 238 [select]: github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc820436150, 0xc800000002) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:238 +0x1720 github.com/cockroachdb/cockroach/storage.(*RaftTransport).Send.func1.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:289 +0x2b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820c37d60) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 220 [select]: github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc82042fc70, 0xc820b9c000, 0xc820410a60, 0x17, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0x5e5 github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:157 +0x66 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820b72000) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 205 [select]: google.golang.org/grpc.NewClientStream.func1(0x7fe17a7272c0, 0xc8209d74a0, 0xc820cfc0e0, 0xc820b681e0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:149 +0x258 created by google.golang.org/grpc.NewClientStream /home/marc/cockroach/src/google.golang.org/grpc/stream.go:157 +0xab2 goroutine 476 [select]: google.golang.org/grpc.NewClientStream.func1(0x7fe17a7272c0, 0xc820fc3a40, 0xc821446380, 0xc820d74780) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:149 +0x258 created by google.golang.org/grpc.NewClientStream /home/marc/cockroach/src/google.golang.org/grpc/stream.go:157 +0xab2 goroutine 254 [select]: github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc82042fc70, 0xc82112af00, 0xc8204108e0, 0x17, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0x5e5 github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:157 +0x66 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc821150690) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 265 [select]: google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc82157c330, 0xc820d83250, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:141 +0x7e6 google.golang.org/grpc/transport.(*Stream).Read(0xc821584000, 0xc820d83250, 0x5, 0x5, 0xc820d84720, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fe17a727398, 0xc821584000, 0xc820d83250, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727398, 0xc821584000, 0xc820d83250, 0x5, 0x5, 0x163fe00, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 google.golang.org/grpc.(*parser).recvMsg(0xc820d83240, 0x22, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:218 +0xb9 google.golang.org/grpc.recv(0xc820d83240, 0x7fe17a6a3208, 0x24a6c10, 0xc821584000, 0x0, 0x0, 0x167e2a0, 0xc820d8f320, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:302 +0x45 google.golang.org/grpc.(*serverStream).RecvMsg(0xc820d64a80, 0x167e2a0, 0xc820d8f320, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:411 +0xe4 github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc820d84720, 0x27, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:174 +0x7e github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x7fe17a6f1e38, 0xc820d84720, 0xc820436150, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:111 +0x4b github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:128 +0x48 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc821582180) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 292 [select]: google.golang.org/grpc/transport.(*http2Client).controller(0xc820fc3a40) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:835 +0x5da created by google.golang.org/grpc/transport.newHTTP2Client /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:194 +0x153b goroutine 241 [select]: google.golang.org/grpc/transport.(*http2Client).controller(0xc8209d74a0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:835 +0x5da created by google.golang.org/grpc/transport.newHTTP2Client /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:194 +0x153b goroutine 204 [select]: google.golang.org/grpc.NewClientStream.func1(0x7fe17a7272c0, 0xc8209d74a0, 0xc820cfc000, 0xc820b680a0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:149 +0x258 created by google.golang.org/grpc.NewClientStream /home/marc/cockroach/src/google.golang.org/grpc/stream.go:157 +0xab2 goroutine 274 [IO wait]: net.runtime_pollWait(0x7fe17a6a3f50, 0x72, 0xc820b5e000) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc82042ebc0, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc82042ebc0, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc82042eb60, 0xc820b5e000, 0x400, 0x400, 0x0, 0x7fe17a71e050, 0xc820012008) /home/marc/go/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc820026258, 0xc820b5e000, 0x400, 0x400, 0x0, 0x0, 0x0) /home/marc/go/src/net/net.go:172 +0xe4 crypto/tls.(*block).readFromUntil(0xc8214b0d50, 0x7fe17a726dc0, 0xc820026258, 0x5, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:460 +0xcc crypto/tls.(*Conn).readRecord(0xc8207b6c00, 0x1a68017, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:562 +0x2d1 crypto/tls.(*Conn).Read(0xc8207b6c00, 0xc820cf0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:939 +0x167 bufio.(*Reader).fill(0xc820cfa0c0) /home/marc/go/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc820cfa0c0, 0xc8214fcff8, 0x9, 0x9, 0xc81fe2e6bf, 0x0, 0x0) /home/marc/go/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fe17a727270, 0xc820cfa0c0, 0xc8214fcff8, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727270, 0xc820cfa0c0, 0xc8214fcff8, 0x9, 0x9, 0xc8235a0698, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 golang.org/x/net/http2.readFrameHeader(0xc8214fcff8, 0x9, 0x9, 0x7fe17a727270, 0xc820cfa0c0, 0x0, 0xc800000000, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:236 +0xa5 golang.org/x/net/http2.(*Framer).ReadFrame(0xc8214fcfc0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:463 +0x106 google.golang.org/grpc/transport.(*framer).readFrame(0xc821193dd0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http_util.go:406 +0x3d google.golang.org/grpc/transport.(*http2Client).reader(0xc8209d74a0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:757 +0x109 created by google.golang.org/grpc/transport.newHTTP2Client /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:200 +0x159a goroutine 262 [IO wait]: net.runtime_pollWait(0x7fe17a6a3a10, 0x72, 0xc820d27400) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc82044dc60, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc82044dc60, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc82044dc00, 0xc820d27400, 0x400, 0x400, 0x0, 0x7fe17a71e050, 0xc820012008) /home/marc/go/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc820121ab8, 0xc820d27400, 0x400, 0x400, 0x0, 0x0, 0x0) /home/marc/go/src/net/net.go:172 +0xe4 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc820b36420, 0xc820d27400, 0x400, 0x400, 0xd, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cmux/cmux.go:204 +0x117 crypto/tls.(*block).readFromUntil(0xc821150cc0, 0x7fe17a728178, 0xc820b36420, 0x5, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:460 +0xcc crypto/tls.(*Conn).readRecord(0xc8201eb800, 0x1a68017, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:562 +0x2d1 crypto/tls.(*Conn).Read(0xc8201eb800, 0xc820d90000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:939 +0x167 bufio.(*Reader).fill(0xc820d8c4e0) /home/marc/go/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc820d8c4e0, 0xc82166c338, 0x9, 0x9, 0x19, 0x0, 0x0) /home/marc/go/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fe17a727270, 0xc820d8c4e0, 0xc82166c338, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727270, 0xc820d8c4e0, 0xc82166c338, 0x9, 0x9, 0xc820d81a48, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 golang.org/x/net/http2.readFrameHeader(0xc82166c338, 0x9, 0x9, 0x7fe17a727270, 0xc820d8c4e0, 0x1000000, 0xc800000000, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:236 +0xa5 golang.org/x/net/http2.(*Framer).ReadFrame(0xc82166c300, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:463 +0x106 google.golang.org/grpc/transport.(*framer).readFrame(0xc820d6b7d0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http_util.go:406 +0x3d google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc820d58ea0, 0xc820d6b860) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:247 +0x646 google.golang.org/grpc.(*Server).serveStreams(0xc8203c4000, 0x7fe17a6aeb08, 0xc820d58ea0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:325 +0x159 google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc8203c4000, 0x7fe17a7271c0, 0xc8201eb800, 0x7fe17a6aeae0, 0xc821678e70) /home/marc/cockroach/src/google.golang.org/grpc/server.go:312 +0x49d google.golang.org/grpc.(*Server).handleRawConn(0xc8203c4000, 0x7fe17a728118, 0xc820b36420) /home/marc/cockroach/src/google.golang.org/grpc/server.go:289 +0x4ee created by google.golang.org/grpc.(*Server).Serve /home/marc/cockroach/src/google.golang.org/grpc/server.go:261 +0x372 goroutine 263 [select]: google.golang.org/grpc/transport.(*http2Server).controller(0xc820d58ea0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:620 +0x5da created by google.golang.org/grpc/transport.newHTTP2Server /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:134 +0x853 goroutine 458 [select]: google.golang.org/grpc/transport.(*Stream).Header(0xc8211a82a0, 0xc8211a82a0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:221 +0x242 google.golang.org/grpc.recvResponse(0x7fe17a6a3208, 0x24a6c10, 0x0, 0x0, 0x0, 0x0, 0x7fe17a65e078, 0xc820122f00, 0x0, 0x0, ...) /home/marc/cockroach/src/google.golang.org/grpc/call.go:54 +0x4e google.golang.org/grpc.Invoke(0x7fe17a6a4cc0, 0xc820faccf0, 0x1822fe0, 0x21, 0x16e0cc0, 0xc82114eb68, 0x1692520, 0xc827fee720, 0xc82041f7c0, 0x0, ...) /home/marc/cockroach/src/google.golang.org/grpc/call.go:178 +0xae6 github.com/cockroachdb/cockroach/roachpb.(*internalClient).Batch(0xc8215e0040, 0x7fe17a6a4cc0, 0xc820faccf0, 0xc82114eb68, 0x0, 0x0, 0x0, 0xc8285231e0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/roachpb/api.pb.go:1284 +0xec github.com/cockroachdb/cockroach/kv.sendOne.func1(0x7fe17a727330, 0xc82000f2c0, 0x1, 0x2540be400, 0xb2d05e00, 0xc82114eb40, 0xc827fee600, 0xc820410b40, 0x17) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/send.go:286 +0x658 created by github.com/cockroachdb/cockroach/kv.sendOne /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/send.go:288 +0x4c9 goroutine 293 [IO wait]: net.runtime_pollWait(0x7fe17a6a3950, 0x72, 0xc82174a800) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc8202cbdb0, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc8202cbdb0, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc8202cbd50, 0xc82174a800, 0x400, 0x400, 0x0, 0x7fe17a71e050, 0xc820012008) /home/marc/go/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc82108a1f0, 0xc82174a800, 0x400, 0x400, 0x0, 0x0, 0x0) /home/marc/go/src/net/net.go:172 +0xe4 crypto/tls.(*block).readFromUntil(0xc820f2eb70, 0x7fe17a726dc0, 0xc82108a1f0, 0x5, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:460 +0xcc crypto/tls.(*Conn).readRecord(0xc821748000, 0x1a68017, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:562 +0x2d1 crypto/tls.(*Conn).Read(0xc821748000, 0xc821568000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:939 +0x167 bufio.(*Reader).fill(0xc82100ab40) /home/marc/go/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc82100ab40, 0xc820fe3d78, 0x9, 0x9, 0xc81febcc4e, 0x0, 0x0) /home/marc/go/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fe17a727270, 0xc82100ab40, 0xc820fe3d78, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727270, 0xc82100ab40, 0xc820fe3d78, 0x9, 0x9, 0xc8228673b8, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 golang.org/x/net/http2.readFrameHeader(0xc820fe3d78, 0x9, 0x9, 0x7fe17a727270, 0xc82100ab40, 0x0, 0xc800000000, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:236 +0xa5 golang.org/x/net/http2.(*Framer).ReadFrame(0xc820fe3d40, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:463 +0x106 google.golang.org/grpc/transport.(*framer).readFrame(0xc82100db00, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http_util.go:406 +0x3d google.golang.org/grpc/transport.(*http2Client).reader(0xc820fc3a40) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:757 +0x109 created by google.golang.org/grpc/transport.newHTTP2Client /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:200 +0x159a goroutine 511 [runnable]: github.com/cockroachdb/cockroach/util/interval.Range.OverlapExclusive(0xc8237af91a, 0x0, 0x1, 0xc8237af91a, 0x1, 0x1, 0xc824dc32a7, 0x0, 0x1, 0xc824dc32aa, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/interval/interval.go:63 github.com/cockroachdb/cockroach/util/interval.(*Node).doMatch(0xc823735b00, 0xc82284a060, 0xc8237af91a, 0x0, 0x1, 0xc8237af91a, 0x1, 0x1, 0x1a668a0, 0xc824dc3200) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/interval/interval.go:673 +0x32d github.com/cockroachdb/cockroach/util/interval.(*Node).doMatch(0xc823735800, 0xc82284a060, 0xc8237af91a, 0x0, 0x1, 0xc8237af91a, 0x1, 0x1, 0x1a668a0, 0xc824dc3200) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/interval/interval.go:674 +0x3b8 github.com/cockroachdb/cockroach/util/interval.(*Node).doMatch(0xc823735e00, 0xc82284a060, 0xc8237af91a, 0x0, 0x1, 0xc8237af91a, 0x1, 0x1, 0x1a668a0, 0xc824dc3200) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/interval/interval.go:662 +0x142 github.com/cockroachdb/cockroach/util/interval.(*Node).doMatch(0xc82375ea20, 0xc82284a060, 0xc8237af91a, 0x0, 0x1, 0xc8237af91a, 0x1, 0x1, 0x1a668a0, 0xc823708d00) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/interval/interval.go:662 +0x142 github.com/cockroachdb/cockroach/util/interval.(*Node).doMatch(0xc823735200, 0xc82284a060, 0xc8237af91a, 0x0, 0x1, 0xc8237af91a, 0x1, 0x1, 0x1a668a0, 0xc823708d00) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/interval/interval.go:674 +0x3b8 github.com/cockroachdb/cockroach/util/interval.(*Tree).DoMatching(0xc820089758, 0xc82284a060, 0xc8237af91a, 0x0, 0x1, 0xc8237af91a, 0x1, 0x1, 0x135e7a0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/interval/interval.go:655 +0x11a github.com/cockroachdb/cockroach/util/cache.(*IntervalCache).GetOverlaps(0xc820089700, 0xc8237af91a, 0x0, 0x1, 0xc8237af91a, 0x1, 0x1, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/cache/cache.go:493 +0x138 github.com/cockroachdb/cockroach/storage.(*CommandQueue).GetWait(0xc820293270, 0xc8237aef00, 0xc8237aeff0, 0xc82101d000, 0xa6, 0x100) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/command_queue.go:102 +0x378 github.com/cockroachdb/cockroach/storage.(*Replica).beginCmds(0xc82014e460, 0xc822842000, 0x1730f90) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:852 +0x340 github.com/cockroachdb/cockroach/storage.(*Replica).addWriteCmd(0xc82014e460, 0x7fe17a727330, 0xc82281de90, 0x144595968971da2f, 0x0, 0x500000005, 0x1, 0x217, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:1121 +0x13f github.com/cockroachdb/cockroach/storage.(*Replica).Send(0xc82014e460, 0x7fe17a727330, 0xc82281de90, 0x144595968971da2f, 0x0, 0x500000005, 0x1, 0x217, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:768 +0x5ba github.com/cockroachdb/cockroach/storage.(*Store).Send(0xc820156200, 0x7fe17a727330, 0xc82281de90, 0x144595968971da2f, 0x0, 0x500000005, 0x1, 0x217, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:1640 +0xd71 github.com/cockroachdb/cockroach/storage.(*Stores).Send(0xc8203fc380, 0x7fe17a727330, 0xc82281de30, 0x0, 0x0, 0x500000005, 0x1, 0x217, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/stores.go:178 +0x501 github.com/cockroachdb/cockroach/server.(*Node).Batch.func2() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:729 +0x55b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc8203bbea0, 0xc82284b880, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:166 +0xd9 github.com/cockroachdb/cockroach/server.(*Node).Batch(0xc82034cd80, 0x7fe17a727330, 0xc82281d6e0, 0xc825e73f80, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:741 +0x2c0 github.com/cockroachdb/cockroach/roachpb._Internal_Batch_Handler(0x16c7380, 0xc82034cd80, 0x7fe17a727330, 0xc82281d6e0, 0xc8237b3980, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/roachpb/api.pb.go:1306 +0x132 google.golang.org/grpc.(*Server).processUnaryRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0, 0xc8214470a0, 0xc820410180, 0x2195870, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:497 +0xe13 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0, 0xc8214470a0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:646 +0x109d google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc82079d690, 0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0, 0xc8214470a0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 415 [select]: google.golang.org/grpc.(*Conn).Wait(0xc8211a8000, 0x7fe17a6a4cc0, 0xc824cba030, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:551 +0x360 google.golang.org/grpc.(*unicastPicker).Pick(0xc827fea020, 0x7fe17a6a4cc0, 0xc824cba030, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/picker.go:84 +0x51 google.golang.org/grpc.Invoke(0x7fe17a6a4cc0, 0xc824cba030, 0x17f9b40, 0x1d, 0x167cf80, 0xc820a34200, 0x166ece0, 0xc827fea040, 0xc82bf12000, 0x0, ...) /home/marc/cockroach/src/google.golang.org/grpc/call.go:155 +0x6d0 github.com/cockroachdb/cockroach/rpc.(*heartbeatClient).Ping(0xc8215e0018, 0x7fe17a6a4cc0, 0xc824cba030, 0xc820a34200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/heartbeat.pb.go:115 +0xec github.com/cockroachdb/cockroach/rpc.(*Context).heartbeat(0xc82042fc70, 0x7fe17a6a4c68, 0xc8215e0018, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc820410300, 0x17, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:221 +0x150 github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc82042fc70, 0xc82bf12000, 0xc820410960, 0x17, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:176 +0x247 github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:157 +0x66 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc824cba000) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 281 [select]: github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc820436150, 0x7fe17a6f1e38, 0xc820f314d0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:132 +0x1e1 github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x1621840, 0xc820436150, 0x7fe17a6f1df0, 0xc821004a00, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:155 +0xd8 google.golang.org/grpc.(*Server).processStreamingRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc820d58ea0, 0xc820d0a8c0, 0xc820410080, 0x2197920, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:575 +0x448 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc820d58ea0, 0xc820d0a8c0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:650 +0x114e google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc820d84310, 0xc8203c4000, 0x7fe17a6aeb08, 0xc820d58ea0, 0xc820d0a8c0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 282 [select]: github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc820436150, 0x7fe17a6f1e38, 0xc820b595d0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:132 +0x1e1 github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x1621840, 0xc820436150, 0x7fe17a6f1df0, 0xc82118f780, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:155 +0xd8 google.golang.org/grpc.(*Server).processStreamingRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc820d58ea0, 0xc820d0a9a0, 0xc820410080, 0x2197920, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:575 +0x448 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc820d58ea0, 0xc820d0a9a0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:650 +0x114e google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc820d84310, 0xc8203c4000, 0x7fe17a6aeb08, 0xc820d58ea0, 0xc820d0a9a0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 283 [chan send]: github.com/cockroachdb/cockroach/storage.(*Store).(github.com/cockroachdb/cockroach/storage.enqueueRaftMessage)-fm(0xc8202765a0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x61 github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x7fe17a6f1e38, 0xc820b595d0, 0xc820436150, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:124 +0x259 github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:128 +0x48 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc820f33050) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 298 [select]: google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc820f32db0, 0xc821011530, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:141 +0x7e6 google.golang.org/grpc/transport.(*Stream).Read(0xc820d0a8c0, 0xc821011530, 0x5, 0x5, 0xc820f314d0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fe17a727398, 0xc820d0a8c0, 0xc821011530, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727398, 0xc820d0a8c0, 0xc821011530, 0x5, 0x5, 0x163fe00, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 google.golang.org/grpc.(*parser).recvMsg(0xc821011520, 0x22, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:218 +0xb9 google.golang.org/grpc.recv(0xc821011520, 0x7fe17a6a3208, 0x24a6c10, 0xc820d0a8c0, 0x0, 0x0, 0x167e2a0, 0xc821018d80, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:302 +0x45 google.golang.org/grpc.(*serverStream).RecvMsg(0xc821004a00, 0x167e2a0, 0xc821018d80, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:411 +0xe4 github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc820f314d0, 0x27, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:174 +0x7e github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x7fe17a6f1e38, 0xc820f314d0, 0xc820436150, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:111 +0x4b github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:128 +0x48 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc82157c000) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 299 [select]: github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc820436150, 0x7fe17a6f1e38, 0xc820d84720, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:132 +0x1e1 github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x1621840, 0xc820436150, 0x7fe17a6f1df0, 0xc820d64a80, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:155 +0xd8 google.golang.org/grpc.(*Server).processStreamingRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc820a18bd0, 0xc821584000, 0xc820410080, 0x2197920, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:575 +0x448 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc820a18bd0, 0xc821584000, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:650 +0x114e google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc820886b10, 0xc8203c4000, 0x7fe17a6aeb08, 0xc820a18bd0, 0xc821584000) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 300 [select]: github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc820436150, 0x7fe17a6f1e38, 0xc820f315d0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:132 +0x1e1 github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x1621840, 0xc820436150, 0x7fe17a6f1df0, 0xc821004a80, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:155 +0xd8 google.golang.org/grpc.(*Server).processStreamingRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc820a18bd0, 0xc8215840e0, 0xc820410080, 0x2197920, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:575 +0x448 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc820a18bd0, 0xc8215840e0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:650 +0x114e google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc820886b10, 0xc8203c4000, 0x7fe17a6aeb08, 0xc820a18bd0, 0xc8215840e0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 301 [chan send]: github.com/cockroachdb/cockroach/storage.(*Store).(github.com/cockroachdb/cockroach/storage.enqueueRaftMessage)-fm(0xc821018240, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x61 github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x7fe17a6f1e38, 0xc820f315d0, 0xc820436150, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:124 +0x259 github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:128 +0x48 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc82157c5a0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 303 [select]: github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc820436150, 0x7fe17a6f1e38, 0xc8215ce010, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:132 +0x1e1 github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x1621840, 0xc820436150, 0x7fe17a6f1df0, 0xc821506000, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:155 +0xd8 google.golang.org/grpc.(*Server).processStreamingRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0, 0xc820fce000, 0xc820410080, 0x2197920, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:575 +0x448 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0, 0xc820fce000, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:650 +0x114e google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc82079d690, 0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0, 0xc820fce000) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 304 [select]: github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc820436150, 0x7fe17a6f1e38, 0xc8218de0b0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:132 +0x1e1 github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x1621840, 0xc820436150, 0x7fe17a6f1df0, 0xc820c64000, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:155 +0xd8 google.golang.org/grpc.(*Server).processStreamingRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0, 0xc820fce0e0, 0xc820410080, 0x2197920, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:575 +0x448 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0, 0xc820fce0e0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:650 +0x114e google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc82079d690, 0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0, 0xc820fce0e0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 305 [chan send]: github.com/cockroachdb/cockroach/storage.(*Store).(github.com/cockroachdb/cockroach/storage.enqueueRaftMessage)-fm(0xc820a386c0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:782 +0x61 github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x7fe17a6f1e38, 0xc8218de0b0, 0xc820436150, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:124 +0x259 github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:128 +0x48 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8211303f0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 267 [select]: github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc820436150, 0xc800000004) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:238 +0x1720 github.com/cockroachdb/cockroach/storage.(*RaftTransport).Send.func1.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:289 +0x2b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8210310a0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 323 [select]: github.com/cockroachdb/cockroach/rpc.(*Context).runHeartbeat(0xc82042fc70, 0xc820a3e0a0, 0xc8204109c0, 0x17, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:209 +0x5e5 github.com/cockroachdb/cockroach/rpc.(*Context).GRPCDial.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/rpc/context.go:157 +0x66 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc821192030) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 363 [select]: github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc820436150, 0x7fe17a6f1e38, 0xc8202c8300, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:132 +0x1e1 github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x1621840, 0xc820436150, 0x7fe17a6f1df0, 0xc82baae080, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:155 +0xd8 google.golang.org/grpc.(*Server).processStreamingRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120, 0xc82087e1c0, 0xc820410080, 0x2197920, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:575 +0x448 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120, 0xc82087e1c0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:650 +0x114e google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc82845a4d0, 0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120, 0xc82087e1c0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 462 [select]: google.golang.org/grpc/transport.(*Stream).Header(0xc82087e380, 0xc82087e380, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:221 +0x242 google.golang.org/grpc.recvResponse(0x7fe17a6a3208, 0x24a6c10, 0x0, 0x0, 0x0, 0x0, 0x7fe17a65e078, 0xc820d82020, 0x0, 0x0, ...) /home/marc/cockroach/src/google.golang.org/grpc/call.go:54 +0x4e google.golang.org/grpc.Invoke(0x7fe17a6a4cc0, 0xc8225d38f0, 0x1822fe0, 0x21, 0x16e0cc0, 0xc82092a0b8, 0x1692520, 0xc820ba6a20, 0xc820a3e0a0, 0x0, ...) /home/marc/cockroach/src/google.golang.org/grpc/call.go:178 +0xae6 github.com/cockroachdb/cockroach/roachpb.(*internalClient).Batch(0xc8200261b8, 0x7fe17a6a4cc0, 0xc8225d38f0, 0xc82092a0b8, 0x0, 0x0, 0x0, 0xc820fae8d0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/roachpb/api.pb.go:1284 +0xec github.com/cockroachdb/cockroach/kv.sendOne.func1(0x7fe17a727330, 0xc82080e7e0, 0x1, 0x2540be400, 0xb2d05e00, 0xc82092a090, 0xc820ba77a0, 0xc8204109c0, 0x17) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/send.go:286 +0x658 created by github.com/cockroachdb/cockroach/kv.sendOne /home/marc/cockroach/src/github.com/cockroachdb/cockroach/kv/send.go:288 +0x4c9 goroutine 414 [sleep]: time.Sleep(0x638342fa) /home/marc/go/src/runtime/time.go:59 +0xf9 google.golang.org/grpc.(*Conn).resetTransport(0xc8211a8000, 0x5eb900, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:460 +0x79f google.golang.org/grpc.NewConn.func1(0xc8211a8000) /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:316 +0x2a created by google.golang.org/grpc.NewConn /home/marc/cockroach/src/google.golang.org/grpc/clientconn.go:322 +0x4dd goroutine 420 [select]: google.golang.org/grpc/transport.(*http2Server).controller(0xc82845e120) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:620 +0x5da created by google.golang.org/grpc/transport.newHTTP2Server /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:134 +0x853 goroutine 475 [select]: google.golang.org/grpc.NewClientStream.func1(0x7fe17a7272c0, 0xc820fc3a40, 0xc8214462a0, 0xc820d74640) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:149 +0x258 created by google.golang.org/grpc.NewClientStream /home/marc/cockroach/src/google.golang.org/grpc/stream.go:157 +0xab2 goroutine 409 [select]: google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc820bc26c0, 0xc82043e270, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:141 +0x7e6 google.golang.org/grpc/transport.(*Stream).Read(0xc82087e0e0, 0xc82043e270, 0x5, 0x5, 0xc8206ba2e0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fe17a727398, 0xc82087e0e0, 0xc82043e270, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727398, 0xc82087e0e0, 0xc82043e270, 0x5, 0x5, 0x163fe00, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 google.golang.org/grpc.(*parser).recvMsg(0xc82043e260, 0x22, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:218 +0xb9 google.golang.org/grpc.recv(0xc82043e260, 0x7fe17a6a3208, 0x24a6c10, 0xc82087e0e0, 0x0, 0x0, 0x167e2a0, 0xc82126e000, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:302 +0x45 google.golang.org/grpc.(*serverStream).RecvMsg(0xc820c64080, 0x167e2a0, 0xc82126e000, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:411 +0xe4 github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageServer).Recv(0xc8206ba2e0, 0x27, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:174 +0x7e github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1.1(0x7fe17a6f1e38, 0xc8206ba2e0, 0xc820436150, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:111 +0x4b github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage.func1.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:128 +0x48 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc82bb10ba0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 362 [select]: github.com/cockroachdb/cockroach/storage.(*RaftTransport).RaftMessage(0xc820436150, 0x7fe17a6f1e38, 0xc8206ba2e0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:132 +0x1e1 github.com/cockroachdb/cockroach/storage._MultiRaft_RaftMessage_Handler(0x1621840, 0xc820436150, 0x7fe17a6f1df0, 0xc820c64080, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft.pb.go:155 +0xd8 google.golang.org/grpc.(*Server).processStreamingRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120, 0xc82087e0e0, 0xc820410080, 0x2197920, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:575 +0x448 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120, 0xc82087e0e0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:650 +0x114e google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc82845a4d0, 0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120, 0xc82087e0e0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 485 [select]: github.com/cockroachdb/cockroach/storage.(*Replica).redirectOnOrAcquireLeaderLease(0xc82014f0a0, 0x7fe17a727330, 0xc8237423c0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:558 +0x6f3 github.com/cockroachdb/cockroach/storage.(*Replica).addReadOnlyCmd(0xc82014f0a0, 0x7fe17a727330, 0xc8237423c0, 0x1445959688021926, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:1046 +0xcc github.com/cockroachdb/cockroach/storage.(*Replica).Send(0xc82014f0a0, 0x7fe17a727330, 0xc8237423c0, 0x1445959688021926, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:765 +0x500 github.com/cockroachdb/cockroach/storage.(*Store).Send(0xc820156200, 0x7fe17a727330, 0xc8237423c0, 0x1445959688021926, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:1640 +0xd71 github.com/cockroachdb/cockroach/storage.(*Stores).Send(0xc8203fc380, 0x7fe17a727330, 0xc823742360, 0x0, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/stores.go:178 +0x501 github.com/cockroachdb/cockroach/server.(*Node).Batch.func2() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:729 +0x55b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc8203bbea0, 0xc82375d880, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:166 +0xd9 github.com/cockroachdb/cockroach/server.(*Node).Batch(0xc82034cd80, 0x7fe17a727330, 0xc823742180, 0xc8237460e0, 0xc8284ba550, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:741 +0x2c0 github.com/cockroachdb/cockroach/roachpb._Internal_Batch_Handler(0x16c7380, 0xc82034cd80, 0x7fe17a727330, 0xc823742180, 0xc82374c080, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/roachpb/api.pb.go:1306 +0x132 google.golang.org/grpc.(*Server).processUnaryRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120, 0xc82087e540, 0xc820410180, 0x2195870, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:497 +0xe13 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120, 0xc82087e540, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:646 +0x109d google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc82845a4d0, 0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120, 0xc82087e540) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 419 [IO wait]: net.runtime_pollWait(0x7fe17a6a37d0, 0x72, 0xc8284b6000) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc8282c7560, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc8282c7560, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc8282c7500, 0xc8284b6000, 0x2000, 0x2000, 0x0, 0x7fe17a71e050, 0xc820012008) /home/marc/go/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc8201213a0, 0xc8284b6000, 0x2000, 0x2000, 0x0, 0x0, 0x0) /home/marc/go/src/net/net.go:172 +0xe4 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc8282ab6b0, 0xc8284b6000, 0x2000, 0x2000, 0xd, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cmux/cmux.go:204 +0x117 crypto/tls.(*block).readFromUntil(0xc82842ea20, 0x7fe17a728178, 0xc8282ab6b0, 0x5, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:460 +0xcc crypto/tls.(*Conn).readRecord(0xc821748900, 0x1a68017, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:562 +0x2d1 crypto/tls.(*Conn).Read(0xc821748900, 0xc828486000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:939 +0x167 bufio.(*Reader).fill(0xc828425200) /home/marc/go/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc828425200, 0xc8283f1a78, 0x9, 0x9, 0x6, 0x0, 0x0) /home/marc/go/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fe17a727270, 0xc828425200, 0xc8283f1a78, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727270, 0xc828425200, 0xc8283f1a78, 0x9, 0x9, 0xc8211f7a58, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 golang.org/x/net/http2.readFrameHeader(0xc8283f1a78, 0x9, 0x9, 0x7fe17a727270, 0xc828425200, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:236 +0xa5 golang.org/x/net/http2.(*Framer).ReadFrame(0xc8283f1a40, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:463 +0x106 google.golang.org/grpc/transport.(*framer).readFrame(0xc82842ec90, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http_util.go:406 +0x3d google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc82845e120, 0xc82842ed20) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_server.go:247 +0x646 google.golang.org/grpc.(*Server).serveStreams(0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120) /home/marc/cockroach/src/google.golang.org/grpc/server.go:325 +0x159 google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc8203c4000, 0x7fe17a7271c0, 0xc821748900, 0x7fe17a6aeae0, 0xc8283fee70) /home/marc/cockroach/src/google.golang.org/grpc/server.go:312 +0x49d google.golang.org/grpc.(*Server).handleRawConn(0xc8203c4000, 0x7fe17a728118, 0xc8282ab6b0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:289 +0x4ee created by google.golang.org/grpc.(*Server).Serve /home/marc/cockroach/src/google.golang.org/grpc/server.go:261 +0x372 goroutine 430 [select]: github.com/cockroachdb/cockroach/storage.(*Replica).redirectOnOrAcquireLeaderLease(0xc82014f420, 0x7fe17a727330, 0xc824cbaed0, 0xc8207a2a00) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:558 +0x6f3 github.com/cockroachdb/cockroach/storage.(*Replica).addWriteCmd(0xc82014f420, 0x7fe17a727330, 0xc824cbaed0, 0x14459595f7d2146b, 0x0, 0x500000005, 0xb, 0x4, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:1130 +0x1af github.com/cockroachdb/cockroach/storage.(*Replica).Send(0xc82014f420, 0x7fe17a727330, 0xc824cbaed0, 0x14459595f7d2146b, 0x0, 0x500000005, 0xb, 0x4, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:768 +0x5ba github.com/cockroachdb/cockroach/storage.(*Store).Send(0xc820156200, 0x7fe17a727330, 0xc824cbaed0, 0x14459595f7d2146b, 0x0, 0x500000005, 0xb, 0x4, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:1640 +0xd71 github.com/cockroachdb/cockroach/storage.(*Stores).Send(0xc8203fc380, 0x7fe17a727330, 0xc824cbae70, 0x0, 0x0, 0x500000005, 0xb, 0x4, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/stores.go:178 +0x501 github.com/cockroachdb/cockroach/server.(*Node).Batch.func2() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:729 +0x55b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc8203bbea0, 0xc8207a3880, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:166 +0xd9 github.com/cockroachdb/cockroach/server.(*Node).Batch(0xc82034cd80, 0x7fe17a727330, 0xc824cbacc0, 0xc82590a0e0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:741 +0x2c0 github.com/cockroachdb/cockroach/roachpb._Internal_Batch_Handler(0x16c7380, 0xc82034cd80, 0x7fe17a727330, 0xc824cbacc0, 0xc82842afc0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/roachpb/api.pb.go:1306 +0x132 google.golang.org/grpc.(*Server).processUnaryRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0, 0xc8283dc8c0, 0xc820410180, 0x2195870, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:497 +0xe13 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0, 0xc8283dc8c0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:646 +0x109d google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc82079d690, 0xc8203c4000, 0x7fe17a6aeb08, 0xc8209b75f0, 0xc8283dc8c0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 379 [select]: github.com/cockroachdb/cockroach/gossip.(*server).Gossip(0xc8204349c0, 0x7fe17a632ec8, 0xc820f30e60, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/gossip/server.go:134 +0xa0f github.com/cockroachdb/cockroach/gossip._Gossip_Gossip_Handler(0x1636f00, 0xc8204349c0, 0x7fe17a6f1df0, 0xc82b9ceb00, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:193 +0xd8 google.golang.org/grpc.(*Server).processStreamingRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120, 0xc82026a540, 0xc820410700, 0x21978c0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:575 +0x448 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120, 0xc82026a540, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:650 +0x114e google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc82845a4d0, 0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120, 0xc82026a540) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 380 [select]: google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc8227adce0, 0xc8207d30d0, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:141 +0x7e6 google.golang.org/grpc/transport.(*Stream).Read(0xc82026a540, 0xc8207d30d0, 0x5, 0x5, 0x61, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fe17a727398, 0xc82026a540, 0xc8207d30d0, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727398, 0xc82026a540, 0xc8207d30d0, 0x5, 0x5, 0xc8284b5a18, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 google.golang.org/grpc.(*parser).recvMsg(0xc8207d30c0, 0x7, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:218 +0xb9 google.golang.org/grpc.recv(0xc8207d30c0, 0x7fe17a6a3208, 0x24a6c10, 0xc82026a540, 0x0, 0x0, 0x167c4e0, 0xc82842ab40, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:302 +0x45 google.golang.org/grpc.(*serverStream).RecvMsg(0xc82b9ceb00, 0x167c4e0, 0xc82842ab40, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:411 +0xe4 github.com/cockroachdb/cockroach/gossip.(*gossipGossipServer).Recv(0xc820f30e60, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/gossip/gossip.pb.go:212 +0x7e github.com/cockroachdb/cockroach/gossip.(Gossip_GossipServer).Recv-fm(0xc8204349c8, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/gossip/server.go:101 +0x40 github.com/cockroachdb/cockroach/gossip.(*server).gossipReceiver(0xc8204349c0, 0xc828036018, 0xc8284aa030, 0xc8284b5f50, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/gossip/server.go:227 +0x747 github.com/cockroachdb/cockroach/gossip.(*server).Gossip.func3.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/gossip/server.go:101 +0x76 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc8215af0c0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 382 [select]: github.com/cockroachdb/cockroach/storage.(*Replica).redirectOnOrAcquireLeaderLease(0xc82014f0a0, 0x7fe17a727330, 0xc82bf0f650, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:558 +0x6f3 github.com/cockroachdb/cockroach/storage.(*Replica).addReadOnlyCmd(0xc82014f0a0, 0x7fe17a727330, 0xc82bf0f650, 0x14459595f1cb7a99, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:1046 +0xcc github.com/cockroachdb/cockroach/storage.(*Replica).Send(0xc82014f0a0, 0x7fe17a727330, 0xc82bf0f650, 0x14459595f1cb7a99, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:765 +0x500 github.com/cockroachdb/cockroach/storage.(*Store).Send(0xc820156200, 0x7fe17a727330, 0xc82bf0f650, 0x14459595f1cb7a99, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:1640 +0xd71 github.com/cockroachdb/cockroach/storage.(*Stores).Send(0xc8203fc380, 0x7fe17a727330, 0xc82bf0f5f0, 0x0, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0x0, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/stores.go:178 +0x501 github.com/cockroachdb/cockroach/server.(*Node).Batch.func2() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:729 +0x55b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc8203bbea0, 0xc8214a5880, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:166 +0xd9 github.com/cockroachdb/cockroach/server.(*Node).Batch(0xc82034cd80, 0x7fe17a727330, 0xc8284aa750, 0xc82befff10, 0x180001, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:741 +0x2c0 github.com/cockroachdb/cockroach/roachpb._Internal_Batch_Handler(0x16c7380, 0xc82034cd80, 0x7fe17a727330, 0xc8284aa750, 0xc8284d5240, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/roachpb/api.pb.go:1306 +0x132 google.golang.org/grpc.(*Server).processUnaryRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc820d58ea0, 0xc82026a700, 0xc820410180, 0x2195870, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:497 +0xe13 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc820d58ea0, 0xc82026a700, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:646 +0x109d google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc820d84310, 0xc8203c4000, 0x7fe17a6aeb08, 0xc820d58ea0, 0xc82026a700) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 454 [select]: google.golang.org/grpc/transport.(*http2Client).controller(0xc821af6ff0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:835 +0x5da created by google.golang.org/grpc/transport.newHTTP2Client /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:194 +0x153b goroutine 383 [select]: github.com/cockroachdb/cockroach/storage.(*Replica).redirectOnOrAcquireLeaderLease(0xc82014f0a0, 0x7fe17a727330, 0xc8284aac30, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:558 +0x6f3 github.com/cockroachdb/cockroach/storage.(*Replica).addReadOnlyCmd(0xc82014f0a0, 0x7fe17a727330, 0xc8284aac30, 0x144595953dfe6fc1, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc82b9d4c00, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:1046 +0xcc github.com/cockroachdb/cockroach/storage.(*Replica).Send(0xc82014f0a0, 0x7fe17a727330, 0xc8284aac30, 0x144595953dfe6fc1, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc82b9d4c00, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:765 +0x500 github.com/cockroachdb/cockroach/storage.(*Store).Send(0xc820156200, 0x7fe17a727330, 0xc8284aac30, 0x144595953dfe6fc1, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc82b9d4c00, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:1640 +0xd71 github.com/cockroachdb/cockroach/storage.(*Stores).Send(0xc8203fc380, 0x7fe17a727330, 0xc8284aabd0, 0x0, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc82b9d4c00, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/stores.go:178 +0x501 github.com/cockroachdb/cockroach/server.(*Node).Batch.func2() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:729 +0x55b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc8203bbea0, 0xc820725880, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:166 +0xd9 github.com/cockroachdb/cockroach/server.(*Node).Batch(0xc82034cd80, 0x7fe17a727330, 0xc8284aa900, 0xc820956700, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:741 +0x2c0 github.com/cockroachdb/cockroach/roachpb._Internal_Batch_Handler(0x16c7380, 0xc82034cd80, 0x7fe17a727330, 0xc8284aa900, 0xc8215aff80, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/roachpb/api.pb.go:1306 +0x132 google.golang.org/grpc.(*Server).processUnaryRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc820d58ea0, 0xc82026a7e0, 0xc820410180, 0x2195870, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:497 +0xe13 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc820d58ea0, 0xc82026a7e0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:646 +0x109d google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc820d84310, 0xc8203c4000, 0x7fe17a6aeb08, 0xc820d58ea0, 0xc82026a7e0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a goroutine 431 [select]: github.com/cockroachdb/cockroach/storage.(*Replica).requestLeaderLease.func1.1(0x14459595f7d342a0, 0x0, 0xc82014f420, 0xd6730b) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:492 +0x7c4 github.com/cockroachdb/cockroach/storage.(*Replica).requestLeaderLease.func1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:455 +0x45 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask.func1(0xc8203bbea0, 0x1ec2389, 0x12, 0x200, 0xc8284313a0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:181 +0x62 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunAsyncTask /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:182 +0xc9 goroutine 359 [select]: google.golang.org/grpc.NewClientStream.func1(0x7fe17a7272c0, 0xc821af6ff0, 0xc82087e000, 0xc82588a780) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:149 +0x258 created by google.golang.org/grpc.NewClientStream /home/marc/cockroach/src/google.golang.org/grpc/stream.go:157 +0xab2 goroutine 455 [IO wait]: net.runtime_pollWait(0x7fe17a6a3ad0, 0x72, 0xc8284ce400) /home/marc/go/src/runtime/netpoll.go:160 +0x60 net.(*pollDesc).Wait(0xc8284e0140, 0x72, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:73 +0x3a net.(*pollDesc).WaitRead(0xc8284e0140, 0x0, 0x0) /home/marc/go/src/net/fd_poll_runtime.go:78 +0x36 net.(*netFD).Read(0xc8284e00e0, 0xc8284ce400, 0x400, 0x400, 0x0, 0x7fe17a71e050, 0xc820012008) /home/marc/go/src/net/fd_unix.go:250 +0x23a net.(*conn).Read(0xc8215e0030, 0xc8284ce400, 0x400, 0x400, 0x0, 0x0, 0x0) /home/marc/go/src/net/net.go:172 +0xe4 crypto/tls.(*block).readFromUntil(0xc8225d3560, 0x7fe17a726dc0, 0xc8215e0030, 0x5, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:460 +0xcc crypto/tls.(*Conn).readRecord(0xc8201eb200, 0x1a68017, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:562 +0x2d1 crypto/tls.(*Conn).Read(0xc8201eb200, 0xc8312a2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /home/marc/go/src/crypto/tls/conn.go:939 +0x167 bufio.(*Reader).fill(0xc820bb50e0) /home/marc/go/src/bufio/bufio.go:97 +0x1e9 bufio.(*Reader).Read(0xc820bb50e0, 0xc8209ff3b8, 0x9, 0x9, 0xc81fe44a26, 0x0, 0x0) /home/marc/go/src/bufio/bufio.go:207 +0x260 io.ReadAtLeast(0x7fe17a727270, 0xc820bb50e0, 0xc8209ff3b8, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727270, 0xc820bb50e0, 0xc8209ff3b8, 0x9, 0x9, 0xc82376b688, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 golang.org/x/net/http2.readFrameHeader(0xc8209ff3b8, 0x9, 0x9, 0x7fe17a727270, 0xc820bb50e0, 0x0, 0xc800000000, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:236 +0xa5 golang.org/x/net/http2.(*Framer).ReadFrame(0xc8209ff380, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/golang.org/x/net/http2/frame.go:463 +0x106 google.golang.org/grpc/transport.(*framer).readFrame(0xc82252b290, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http_util.go:406 +0x3d google.golang.org/grpc/transport.(*http2Client).reader(0xc821af6ff0) /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:757 +0x109 created by google.golang.org/grpc/transport.newHTTP2Client /home/marc/cockroach/src/google.golang.org/grpc/transport/http2_client.go:200 +0x159a goroutine 456 [select]: google.golang.org/grpc.NewClientStream.func1(0x7fe17a7272c0, 0xc821af6ff0, 0xc821447260, 0xc82578e000) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:149 +0x258 created by google.golang.org/grpc.NewClientStream /home/marc/cockroach/src/google.golang.org/grpc/stream.go:157 +0xab2 goroutine 441 [select]: github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue(0xc820436150, 0xc800000006) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:238 +0x1720 github.com/cockroachdb/cockroach/storage.(*RaftTransport).Send.func1.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:289 +0x2b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc82ba06580) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 478 [select]: google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc8237c6210, 0xc82148aad0, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:141 +0x7e6 google.golang.org/grpc/transport.(*Stream).Read(0xc821446380, 0xc82148aad0, 0x5, 0x5, 0x5dea87, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/transport/transport.go:294 +0x71 io.ReadAtLeast(0x7fe17a727398, 0xc821446380, 0xc82148aad0, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0) /home/marc/go/src/io/io.go:297 +0xe6 io.ReadFull(0x7fe17a727398, 0xc821446380, 0xc82148aad0, 0x5, 0x5, 0x1, 0x0, 0x0) /home/marc/go/src/io/io.go:315 +0x62 google.golang.org/grpc.(*parser).recvMsg(0xc82148aac0, 0xc8208fe070, 0x0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:218 +0xb9 google.golang.org/grpc.recv(0xc82148aac0, 0x7fe17a6a3208, 0x24a6c10, 0xc821446380, 0x0, 0x0, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/rpc_util.go:302 +0x45 google.golang.org/grpc.(*clientStream).RecvMsg(0xc820d74780, 0x16734e0, 0x24a6c10, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/stream.go:232 +0xac github.com/cockroachdb/cockroach/storage.(*multiRaftRaftMessageClient).RecvMsg(0xc8284ba7f0, 0x16734e0, 0x24a6c10, 0x0, 0x0) :236 +0x6f github.com/cockroachdb/cockroach/storage.(*RaftTransport).processQueue.func2.1() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/raft_transport.go:226 +0x79 github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker.func1(0xc8203bbea0, 0xc82148ab20) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:139 +0x52 created by github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunWorker /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:140 +0x62 goroutine 506 [select]: github.com/cockroachdb/cockroach/storage.(*Replica).redirectOnOrAcquireLeaderLease(0xc82014f0a0, 0x7fe17a727330, 0xc8237ab590, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:558 +0x6f3 github.com/cockroachdb/cockroach/storage.(*Replica).addReadOnlyCmd(0xc82014f0a0, 0x7fe17a727330, 0xc8237ab590, 0x14459595eed05cb5, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc820678600, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:1046 +0xcc github.com/cockroachdb/cockroach/storage.(*Replica).Send(0xc82014f0a0, 0x7fe17a727330, 0xc8237ab590, 0x14459595eed05cb5, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc820678600, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/replica.go:765 +0x500 github.com/cockroachdb/cockroach/storage.(*Store).Send(0xc820156200, 0x7fe17a727330, 0xc8237ab590, 0x14459595eed05cb5, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc820678600, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/store.go:1640 +0xd71 github.com/cockroachdb/cockroach/storage.(*Stores).Send(0xc8203fc380, 0x7fe17a727330, 0xc8237ab530, 0x0, 0x0, 0x500000005, 0x1, 0x77, 0x0, 0xc820678600, ...) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/storage/stores.go:178 +0x501 github.com/cockroachdb/cockroach/server.(*Node).Batch.func2() /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:729 +0x55b github.com/cockroachdb/cockroach/util/stop.(*Stopper).RunTask(0xc8203bbea0, 0xc823759880, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/util/stop/stopper.go:166 +0xd9 github.com/cockroachdb/cockroach/server.(*Node).Batch(0xc82034cd80, 0x7fe17a727330, 0xc8237ab2f0, 0xc825e73730, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/server/node.go:741 +0x2c0 github.com/cockroachdb/cockroach/roachpb._Internal_Batch_Handler(0x16c7380, 0xc82034cd80, 0x7fe17a727330, 0xc8237ab2f0, 0xc8237b22c0, 0x0, 0x0, 0x0, 0x0) /home/marc/cockroach/src/github.com/cockroachdb/cockroach/roachpb/api.pb.go:1306 +0x132 google.golang.org/grpc.(*Server).processUnaryRPC(0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120, 0xc821446c40, 0xc820410180, 0x2195870, 0x0, 0x0, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:497 +0xe13 google.golang.org/grpc.(*Server).handleStream(0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120, 0xc821446c40, 0x0) /home/marc/cockroach/src/google.golang.org/grpc/server.go:646 +0x109d google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc82845a4d0, 0xc8203c4000, 0x7fe17a6aeb08, 0xc82845e120, 0xc821446c40) /home/marc/cockroach/src/google.golang.org/grpc/server.go:323 +0xa0 created by google.golang.org/grpc.(*Server).serveStreams.func1 /home/marc/cockroach/src/google.golang.org/grpc/server.go:324 +0x9a