Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

panic: unsupported type when running tpc-c workload #37414

Closed
awoods187 opened this issue May 8, 2019 · 5 comments
Closed

panic: unsupported type when running tpc-c workload #37414

awoods187 opened this issue May 8, 2019 · 5 comments
Labels
C-bug Code not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior. S-2-temp-unavailability Temp crashes or other availability problems. Can be worked around or resolved by restarting.

Comments

@awoods187
Copy link
Contributor

Describe the problem
Saw a panic when running tpc-c on a geo-distributed cluster using master.

panic: unsupported type []uint8: [48 48 52 49 56 57 51 55 45 52 98 99 54 45 97 55 101 102 45 48 48 48 48 45 48 48 48 48 48 48 48 48 48 48 48 48]

goroutine 1 [running]:
panic(0x1315340, 0xc046e2c300)
	/usr/local/go/src/runtime/panic.go:556 +0x2cb fp=0xc0016b91d8 sp=0xc0016b9148 pc=0x436e6b
github.com/cockroachdb/cockroach/pkg/workload.sliceSliceInterface.Less(0xc02cdb8000, 0x3e7, 0x3e7, 0x0, 0x7c, 0xc0016b93b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/workload/workload.go:703 +0x48d fp=0xc0016b9278 sp=0xc0016b91d8 pc=0xf089ad
github.com/cockroachdb/cockroach/pkg/workload.(*sliceSliceInterface).Less(0xc02f4bf9c0, 0x0, 0x7c, 0xc02f4bf9c0)
	<autogenerated>:1 +0x62 fp=0xc0016b92b8 sp=0xc0016b9278 pc=0xf1f972
sort.medianOfThree(0x19923e0, 0xc02f4bf9c0, 0x0, 0x7c, 0xf8)
	/usr/local/go/src/sort/sort.go:76 +0x49 fp=0xc0016b92e8 sp=0xc0016b92b8 pc=0x4ed179
sort.doPivot(0x19923e0, 0xc02f4bf9c0, 0x0, 0x3e7, 0xf1f8de, 0xc02cdb8000)
	/usr/local/go/src/sort/sort.go:101 +0x5cd fp=0xc0016b9378 sp=0xc0016b92e8 pc=0x4ed8dd
sort.quickSort(0x19923e0, 0xc02f4bf9c0, 0x0, 0x3e7, 0x14)
	/usr/local/go/src/sort/sort.go:190 +0x9a fp=0xc0016b93d0 sp=0xc0016b9378 pc=0x4eda2a
sort.Sort(0x19923e0, 0xc02f4bf9c0)
	/usr/local/go/src/sort/sort.go:218 +0x79 fp=0xc0016b9410 sp=0xc0016b93d0 pc=0x4edc29
github.com/cockroachdb/cockroach/pkg/workload.Split(0x1994de0, 0xc0000d2010, 0xc000570000, 0x1530624, 0x7, 0x1593c57, 0x125, 0xc845880, 0x0, 0xc04038f030, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/workload/workload.go:554 +0x2cf fp=0xc0016b9510 sp=0xc0016b9410 pc=0xf078ef
github.com/cockroachdb/cockroach/pkg/workload/cli.runRun(0x1982ea0, 0xc0001b7080, 0xc0001fc870, 0x9, 0xf, 0x152c77d, 0x4, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/workload/cli/run.go:342 +0x1341 fp=0xc0016b9b90 sp=0xc0016b9510 pc=0x11a7df1
github.com/cockroachdb/cockroach/pkg/workload/cli.CmdHelper.func1(0xc000692780, 0xc0001fc870, 0x9, 0xf, 0x4, 0x119ba55)
	/go/src/github.com/cockroachdb/cockroach/pkg/workload/cli/run.go:177 +0x203 fp=0xc0016b9ca0 sp=0xc0016b9b90 pc=0x11a9603
github.com/cockroachdb/cockroach/pkg/workload/cli.HandleErrs.func1(0xc000692780, 0xc0001fc870, 0x9, 0xf)
	/go/src/github.com/cockroachdb/cockroach/pkg/workload/cli/cli.go:79 +0x57 fp=0xc0016b9d10 sp=0xc0016b9ca0 pc=0x11a8857
github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra.(*Command).execute(0xc000692780, 0xc0001fc780, 0xf, 0xf, 0xc000692780, 0xc0001fc780)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra/command.go:766 +0x2cc fp=0xc0016b9e00 sp=0xc0016b9d10 pc=0x119a9cc
github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0xc000283b80, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra/command.go:852 +0x2fd fp=0xc0016b9f40 sp=0xc0016b9e00 pc=0x119b56d
github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra.(*Command).Execute(0xc000283b80, 0xc000283b80, 0xc0000bc058)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/spf13/cobra/command.go:800 +0x2b fp=0xc0016b9f70 sp=0xc0016b9f40 pc=0x119b24b
main.main()
	/go/src/github.com/cockroachdb/cockroach/pkg/cmd/workload/main.go:27 +0x34 fp=0xc0016b9f98 sp=0xc0016b9f70 pc=0x11af814
runtime.main()
	/usr/local/go/src/runtime/proc.go:201 +0x207 fp=0xc0016b9fe0 sp=0xc0016b9f98 pc=0x438b97
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0016b9fe8 sp=0xc0016b9fe0 pc=0x466f51

goroutine 2 [force gc (idle)]:
runtime.gopark(0x15ed530, 0x28b9230, 0x1410, 0x1)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000078f80 sp=0xc000078f60 pc=0x438f8b
runtime.goparkunlock(0x28b9230, 0x1410, 0x1)
	/usr/local/go/src/runtime/proc.go:308 +0x53 fp=0xc000078fb0 sp=0xc000078f80 pc=0x439033
runtime.forcegchelper()
	/usr/local/go/src/runtime/proc.go:251 +0xb3 fp=0xc000078fe0 sp=0xc000078fb0 pc=0x438e03
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc000078fe8 sp=0xc000078fe0 pc=0x466f51
created by runtime.init.4
	/usr/local/go/src/runtime/proc.go:240 +0x35

goroutine 3 [GC sweep wait]:
runtime.gopark(0x15ed530, 0x28bc120, 0x46140c, 0x1)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000079780 sp=0xc000079760 pc=0x438f8b
runtime.goparkunlock(0x28bc120, 0x197140c, 0x1)
	/usr/local/go/src/runtime/proc.go:308 +0x53 fp=0xc0000797b0 sp=0xc000079780 pc=0x439033
runtime.bgsweep(0xc0000a2000)
	/usr/local/go/src/runtime/mgcsweep.go:71 +0x102 fp=0xc0000797d8 sp=0xc0000797b0 pc=0x42ac02
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0000797e0 sp=0xc0000797d8 pc=0x466f51
created by runtime.gcenable
	/usr/local/go/src/runtime/mgc.go:216 +0x58

goroutine 18 [finalizer wait]:
runtime.gopark(0x15ed530, 0x28e2dd8, 0x140f, 0x1)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000074728 sp=0xc000074708 pc=0x438f8b
runtime.goparkunlock(0x28e2dd8, 0x140f, 0x1)
	/usr/local/go/src/runtime/proc.go:308 +0x53 fp=0xc000074758 sp=0xc000074728 pc=0x439033
runtime.runfinq()
	/usr/local/go/src/runtime/mfinal.go:175 +0x99 fp=0xc0000747e0 sp=0xc000074758 pc=0x422519
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0000747e8 sp=0xc0000747e0 pc=0x466f51
created by runtime.createfing
	/usr/local/go/src/runtime/mfinal.go:156 +0x61

goroutine 19 [syscall]:
runtime.notetsleepg(0x28c6e60, 0xae01fb83, 0x0)
	/usr/local/go/src/runtime/lock_futex.go:227 +0x37 fp=0xc000078758 sp=0xc000078728 pc=0x415b97
runtime.timerproc(0x28c6e40)
	/usr/local/go/src/runtime/time.go:288 +0x30e fp=0xc0000787d8 sp=0xc000078758 pc=0x4561de
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0000787e0 sp=0xc0000787d8 pc=0x466f51
created by runtime.(*timersBucket).addtimerLocked
	/usr/local/go/src/runtime/time.go:170 +0x114

goroutine 20 [syscall]:
runtime.notetsleepg(0x28e3800, 0xffffffffffffffff, 0x0)
	/usr/local/go/src/runtime/lock_futex.go:227 +0x37 fp=0xc000074f88 sp=0xc000074f58 pc=0x415b97
os/signal.signal_recv(0x0)
	/usr/local/go/src/runtime/sigqueue.go:139 +0x9c fp=0xc000074fb0 sp=0xc000074f88 pc=0x44da3c
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:23 +0x22 fp=0xc000074fe0 sp=0xc000074fb0 pc=0xad4a82
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc000074fe8 sp=0xc000074fe0 pc=0x466f51
created by os/signal.init.0
	/usr/local/go/src/os/signal/signal_unix.go:29 +0x41

goroutine 21 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc000040070, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000075760 sp=0xc000075740 pc=0x438f8b
runtime.gcBgMarkWorker(0xc00004a000)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc0000757d8 sp=0xc000075760 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0000757e0 sp=0xc0000757d8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 22 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc000470000, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000075f60 sp=0xc000075f40 pc=0x438f8b
runtime.gcBgMarkWorker(0xc00004c500)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc000075fd8 sp=0xc000075f60 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc000075fe0 sp=0xc000075fd8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 34 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc000040080, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000478760 sp=0xc000478740 pc=0x438f8b
runtime.gcBgMarkWorker(0xc00004ea00)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc0004787d8 sp=0xc000478760 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0004787e0 sp=0xc0004787d8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 4 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc000040090, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000079f60 sp=0xc000079f40 pc=0x438f8b
runtime.gcBgMarkWorker(0xc000050f00)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc000079fd8 sp=0xc000079f60 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc000079fe0 sp=0xc000079fd8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 23 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc0000400a0, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000076760 sp=0xc000076740 pc=0x438f8b
runtime.gcBgMarkWorker(0xc000053400)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc0000767d8 sp=0xc000076760 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0000767e0 sp=0xc0000767d8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 35 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc0000400b0, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000478f60 sp=0xc000478f40 pc=0x438f8b
runtime.gcBgMarkWorker(0xc000055900)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc000478fd8 sp=0xc000478f60 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc000478fe0 sp=0xc000478fd8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 50 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc000470010, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000474760 sp=0xc000474740 pc=0x438f8b
runtime.gcBgMarkWorker(0xc000058000)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc0004747d8 sp=0xc000474760 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0004747e0 sp=0xc0004747d8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 24 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc000470020, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000076f60 sp=0xc000076f40 pc=0x438f8b
runtime.gcBgMarkWorker(0xc00005a500)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc000076fd8 sp=0xc000076f60 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc000076fe0 sp=0xc000076fd8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 51 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc000470030, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000474f60 sp=0xc000474f40 pc=0x438f8b
runtime.gcBgMarkWorker(0xc00005ca00)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc000474fd8 sp=0xc000474f60 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc000474fe0 sp=0xc000474fd8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 36 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc000449ed0, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000479760 sp=0xc000479740 pc=0x438f8b
runtime.gcBgMarkWorker(0xc00005ef00)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc0004797d8 sp=0xc000479760 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0004797e0 sp=0xc0004797d8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 52 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc000449ee0, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000475760 sp=0xc000475740 pc=0x438f8b
runtime.gcBgMarkWorker(0xc000061400)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc0004757d8 sp=0xc000475760 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0004757e0 sp=0xc0004757d8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 37 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc000470040, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000479f60 sp=0xc000479f40 pc=0x438f8b
runtime.gcBgMarkWorker(0xc000063900)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc000479fd8 sp=0xc000479f60 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc000479fe0 sp=0xc000479fd8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 38 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc000470050, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc00047a760 sp=0xc00047a740 pc=0x438f8b
runtime.gcBgMarkWorker(0xc000066000)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc00047a7d8 sp=0xc00047a760 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00047a7e0 sp=0xc00047a7d8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 39 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc000470060, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc00047af60 sp=0xc00047af40 pc=0x438f8b
runtime.gcBgMarkWorker(0xc000068500)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc00047afd8 sp=0xc00047af60 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00047afe0 sp=0xc00047afd8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 40 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc000470070, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc00047b760 sp=0xc00047b740 pc=0x438f8b
runtime.gcBgMarkWorker(0xc00006aa00)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc00047b7d8 sp=0xc00047b760 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00047b7e0 sp=0xc00047b7d8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 41 [GC worker (idle)]:
runtime.gopark(0x15ed3e0, 0xc000470080, 0x1417, 0x0)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc00047bf60 sp=0xc00047bf40 pc=0x438f8b
runtime.gcBgMarkWorker(0xc00006cf00)
	/usr/local/go/src/runtime/mgc.go:1772 +0xfc fp=0xc00047bfd8 sp=0xc00047bf60 pc=0x425e0c
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00047bfe0 sp=0xc00047bfd8 pc=0x466f51
created by runtime.gcBgMarkStartWorkers
	/usr/local/go/src/runtime/mgc.go:1720 +0x77

goroutine 5 [chan receive]:
runtime.gopark(0x15ed530, 0xc0000a40b8, 0x170d, 0x3)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc00007be70 sp=0xc00007be50 pc=0x438f8b
runtime.goparkunlock(0xc0000a40b8, 0x170d, 0x3)
	/usr/local/go/src/runtime/proc.go:308 +0x53 fp=0xc00007bea0 sp=0xc00007be70 pc=0x439033
runtime.chanrecv(0xc0000a4060, 0xc00007bfb8, 0x3b9aca01, 0xc000360101)
	/usr/local/go/src/runtime/chan.go:520 +0x2be fp=0xc00007bf30 sp=0xc00007bea0 pc=0x4109de
runtime.chanrecv2(0xc0000a4060, 0xc00007bfb8, 0x1)
	/usr/local/go/src/runtime/chan.go:407 +0x2b fp=0xc00007bf60 sp=0xc00007bf30 pc=0x41070b
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1226 +0xf2 fp=0xc00007bfe0 sp=0xc00007bf60 pc=0xae3cd2
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00007bfe8 sp=0xc00007bfe0 pc=0x466f51
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:629 +0x11e

goroutine 6 [chan receive]:
runtime.gopark(0x15ed530, 0xc000200058, 0xad170d, 0x3)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000077680 sp=0xc000077660 pc=0x438f8b
runtime.goparkunlock(0xc000200058, 0x170d, 0x3)
	/usr/local/go/src/runtime/proc.go:308 +0x53 fp=0xc0000776b0 sp=0xc000077680 pc=0x439033
runtime.chanrecv(0xc000200000, 0xc0000777b0, 0x1985d01, 0x19725f8)
	/usr/local/go/src/runtime/chan.go:520 +0x2be fp=0xc000077740 sp=0xc0000776b0 pc=0x4109de
runtime.chanrecv2(0xc000200000, 0xc0000777b0, 0x0)
	/usr/local/go/src/runtime/chan.go:407 +0x2b fp=0xc000077770 sp=0xc000077740 pc=0x41070b
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:636 +0xab fp=0xc0000777e0 sp=0xc000077770 pc=0xae13ab
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0000777e8 sp=0xc0000777e0 pc=0x466f51
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:630 +0x136

goroutine 53 [syscall]:
runtime.notetsleepg(0x28c6f60, 0x3b98cb05, 0x0)
	/usr/local/go/src/runtime/lock_futex.go:227 +0x37 fp=0xc000491f58 sp=0xc000491f28 pc=0x415b97
runtime.timerproc(0x28c6f40)
	/usr/local/go/src/runtime/time.go:288 +0x30e fp=0xc000491fd8 sp=0xc000491f58 pc=0x4561de
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc000491fe0 sp=0xc000491fd8 pc=0x466f51
created by runtime.(*timersBucket).addtimerLocked
	/usr/local/go/src/runtime/time.go:170 +0x114

goroutine 25 [select, locked to thread]:
runtime.gopark(0x15ed570, 0x0, 0x1809, 0x1)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc00048dde0 sp=0xc00048ddc0 pc=0x438f8b
runtime.selectgo(0xc00048df80, 0xc00048df78, 0x2, 0x0, 0x1)
	/usr/local/go/src/runtime/select.go:313 +0xcc6 fp=0xc00048df40 sp=0xc00048dde0 pc=0x448e56
runtime.ensureSigM.func1()
	/usr/local/go/src/runtime/signal_unix.go:547 +0x1bf fp=0xc00048dfe0 sp=0xc00048df40 pc=0x4646af
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00048dfe8 sp=0xc00048dfe0 pc=0x466f51
created by runtime.ensureSigM
	/usr/local/go/src/runtime/signal_unix.go:530 +0xd5

goroutine 26 [select]:
runtime.gopark(0x15ed570, 0x0, 0x1809, 0x1)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000475da8 sp=0xc000475d88 pc=0x438f8b
runtime.selectgo(0xc000475f50, 0xc000475f34, 0x3, 0x28bcf20, 0x1)
	/usr/local/go/src/runtime/select.go:313 +0xcc6 fp=0xc000475f08 sp=0xc000475da8 pc=0x448e56
github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view.(*worker).start(0xc000599a00)
	/go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:152 +0xdd fp=0xc000475fd8 sp=0xc000475f08 pc=0x116ae9d
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc000475fe0 sp=0xc000475fd8 pc=0x466f51
created by github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view.init.0
	/go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:29 +0x57

goroutine 8 [select]:
runtime.gopark(0x15ed570, 0x0, 0x1809, 0x1)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc0004765d8 sp=0xc0004765b8 pc=0x438f8b
runtime.selectgo(0xc000476768, 0xc000476760, 0x2, 0x159916c, 0xc0004767a8)
	/usr/local/go/src/runtime/select.go:313 +0xcc6 fp=0xc000476738 sp=0xc0004765d8 pc=0x448e56
database/sql.(*DB).connectionOpener(0xc000570000, 0x1994da0, 0xc0000a10c0)
	/usr/local/go/src/database/sql/sql.go:1001 +0xe8 fp=0xc0004767c8 sp=0xc000476738 pc=0x4f8898
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0004767d0 sp=0xc0004767c8 pc=0x466f51
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:671 +0x15d

goroutine 28 [IO wait]:
runtime.gopark(0x15ed500, 0x7f5880e0ff28, 0x1b02, 0x5)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc000086b58 sp=0xc000086b38 pc=0x438f8b
runtime.netpollblock(0x7f5880e0ff00, 0x72, 0xc0004aca80)
	/usr/local/go/src/runtime/netpoll.go:366 +0x99 fp=0xc000086b90 sp=0xc000086b58 pc=0x4342b9
internal/poll.runtime_pollWait(0x7f5880e0ff00, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:173 +0x66 fp=0xc000086bc0 sp=0xc000086b90 pc=0x433966
internal/poll.(*pollDesc).wait(0xc000049698, 0x72, 0xc0006f6200, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9a fp=0xc000086bf0 sp=0xc000086bc0 pc=0x49044a
internal/poll.(*pollDesc).waitRead(0xc000049698, 0xffffffffffffff00, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d fp=0xc000086c28 sp=0xc000086bf0 pc=0x49055d
internal/poll.(*FD).Accept(0xc000049680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1a0 fp=0xc000086c90 sp=0xc000086c28 pc=0x492860
net.(*netFD).accept(0xc000049680, 0xc00007ce00, 0x30, 0x30)
	/usr/local/go/src/net/fd_unix.go:238 +0x42 fp=0xc000086d60 sp=0xc000086c90 pc=0x59cc52
net.(*TCPListener).accept(0xc0006e87b0, 0x50, 0x7f5880e6b6c0, 0x0)
	/usr/local/go/src/net/tcpsock_posix.go:139 +0x2e fp=0xc000086d90 sp=0xc000086d60 pc=0x5ba42e
net.(*TCPListener).AcceptTCP(0xc0006e87b0, 0x4173f8, 0x30, 0x146ed00)
	/usr/local/go/src/net/tcpsock.go:247 +0x47 fp=0xc000086dd0 sp=0xc000086d90 pc=0x5b8e67
net/http.tcpKeepAliveListener.Accept(0xc0006e87b0, 0x146ed00, 0xc000286c30, 0x1379980, 0x2654570)
	/usr/local/go/src/net/http/server.go:3232 +0x2f fp=0xc000086e08 sp=0xc000086dd0 pc=0x70408f
net/http.(*onceCloseListener).Accept(0xc000286b70, 0xc0000d2010, 0x1379980, 0x2654570, 0x1501fa0)
	<autogenerated>:1 +0x3c fp=0xc000086e40 sp=0xc000086e08 pc=0x724afc
net/http.(*Server).Serve(0xc0001dc820, 0x1992a60, 0xc0006e87b0, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2826 +0x22f fp=0xc000086f00 sp=0xc000086e40 pc=0x702ebf
net/http.(*Server).ListenAndServe(0xc0001dc820, 0xc0001dc820, 0x6)
	/usr/local/go/src/net/http/server.go:2764 +0xb6 fp=0xc000086f50 sp=0xc000086f00 pc=0x702b76
net/http.ListenAndServe(0xc000378be6, 0x6, 0x0, 0x0, 0x5, 0xc000378be6)
	/usr/local/go/src/net/http/server.go:3004 +0x74 fp=0xc000086f78 sp=0xc000086f50 pc=0x703b64
github.com/cockroachdb/cockroach/pkg/workload/cli.startPProfEndPoint.func1(0x1994de0, 0xc0000d2010)
	/go/src/github.com/cockroachdb/cockroach/pkg/workload/cli/run.go:288 +0x8a fp=0xc000086fd0 sp=0xc000086f78 pc=0x11a98ca
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc000086fd8 sp=0xc000086fd0 pc=0x466f51
created by github.com/cockroachdb/cockroach/pkg/workload/cli.startPProfEndPoint
	/go/src/github.com/cockroachdb/cockroach/pkg/workload/cli/run.go:287 +0xbd

goroutine 9 [select]:
runtime.gopark(0x15ed570, 0x0, 0x1809, 0x1)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc00048edc0 sp=0xc00048eda0 pc=0x438f8b
runtime.selectgo(0xc00048ef68, 0xc00048ef48, 0x2, 0x0, 0x0)
	/usr/local/go/src/runtime/select.go:313 +0xcc6 fp=0xc00048ef20 sp=0xc00048edc0 pc=0x448e56
database/sql.(*DB).connectionResetter(0xc000570000, 0x1994da0, 0xc0000a10c0)
	/usr/local/go/src/database/sql/sql.go:1014 +0xfb fp=0xc00048efc8 sp=0xc00048ef20 pc=0x4f89cb
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00048efd0 sp=0xc00048efc8 pc=0x466f51
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:672 +0x193

goroutine 13008070 [timer goroutine (idle)]:
runtime.gopark(0x15ed530, 0x28c70c0, 0x2a09c1414, 0x1)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc0243b3728 sp=0xc0243b3708 pc=0x438f8b
runtime.goparkunlock(0x28c70c0, 0xc03d731414, 0x1)
	/usr/local/go/src/runtime/proc.go:308 +0x53 fp=0xc0243b3758 sp=0xc0243b3728 pc=0x439033
runtime.timerproc(0x28c70c0)
	/usr/local/go/src/runtime/time.go:280 +0x288 fp=0xc0243b37d8 sp=0xc0243b3758 pc=0x456158
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0243b37e0 sp=0xc0243b37d8 pc=0x466f51
created by runtime.(*timersBucket).addtimerLocked
	/usr/local/go/src/runtime/time.go:170 +0x114

goroutine 13008157 [timer goroutine (idle)]:
runtime.gopark(0x15ed530, 0x28c72c0, 0x2a2f21414, 0x1)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc02a371f28 sp=0xc02a371f08 pc=0x438f8b
runtime.goparkunlock(0x28c72c0, 0xc03d721414, 0x1)
	/usr/local/go/src/runtime/proc.go:308 +0x53 fp=0xc02a371f58 sp=0xc02a371f28 pc=0x439033
runtime.timerproc(0x28c72c0)
	/usr/local/go/src/runtime/time.go:280 +0x288 fp=0xc02a371fd8 sp=0xc02a371f58 pc=0x456158
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc02a371fe0 sp=0xc02a371fd8 pc=0x466f51
created by runtime.(*timersBucket).addtimerLocked
	/usr/local/go/src/runtime/time.go:170 +0x114

goroutine 13007872 [timer goroutine (idle)]:
runtime.gopark(0x15ed530, 0x28c6ec0, 0x3f6301414, 0x1)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc0298c0f28 sp=0xc0298c0f08 pc=0x438f8b
runtime.goparkunlock(0x28c6ec0, 0xc02cd01414, 0x1)
	/usr/local/go/src/runtime/proc.go:308 +0x53 fp=0xc0298c0f58 sp=0xc0298c0f28 pc=0x439033
runtime.timerproc(0x28c6ec0)
	/usr/local/go/src/runtime/time.go:280 +0x288 fp=0xc0298c0fd8 sp=0xc0298c0f58 pc=0x456158
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0298c0fe0 sp=0xc0298c0fd8 pc=0x466f51
created by runtime.(*timersBucket).addtimerLocked
	/usr/local/go/src/runtime/time.go:170 +0x114

goroutine 13007592 [timer goroutine (idle)]:
runtime.gopark(0x15ed530, 0x28c6dc0, 0x3fc3e1414, 0x1)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc00048cf28 sp=0xc00048cf08 pc=0x438f8b
runtime.goparkunlock(0x28c6dc0, 0xc02cd01414, 0x1)
	/usr/local/go/src/runtime/proc.go:308 +0x53 fp=0xc00048cf58 sp=0xc00048cf28 pc=0x439033
runtime.timerproc(0x28c6dc0)
	/usr/local/go/src/runtime/time.go:280 +0x288 fp=0xc00048cfd8 sp=0xc00048cf58 pc=0x456158
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00048cfe0 sp=0xc00048cfd8 pc=0x466f51
created by runtime.(*timersBucket).addtimerLocked
	/usr/local/go/src/runtime/time.go:170 +0x114

goroutine 13007760 [timer goroutine (idle)]:
runtime.gopark(0x15ed530, 0x28c74c0, 0x28a431414, 0x1)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc0066c8f28 sp=0xc0066c8f08 pc=0x438f8b
runtime.goparkunlock(0x28c74c0, 0xc03d731414, 0x1)
	/usr/local/go/src/runtime/proc.go:308 +0x53 fp=0xc0066c8f58 sp=0xc0066c8f28 pc=0x439033
runtime.timerproc(0x28c74c0)
	/usr/local/go/src/runtime/time.go:280 +0x288 fp=0xc0066c8fd8 sp=0xc0066c8f58 pc=0x456158
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0066c8fe0 sp=0xc0066c8fd8 pc=0x466f51
created by runtime.(*timersBucket).addtimerLocked
	/usr/local/go/src/runtime/time.go:170 +0x114

goroutine 13008546 [timer goroutine (idle)]:
runtime.gopark(0x15ed530, 0x28c7240, 0x3fbc71414, 0x1)
	/usr/local/go/src/runtime/proc.go:302 +0xeb fp=0xc0298fdf28 sp=0xc0298fdf08 pc=0x438f8b
runtime.goparkunlock(0x28c7240, 0xc02cd11414, 0x1)
	/usr/local/go/src/runtime/proc.go:308 +0x53 fp=0xc0298fdf58 sp=0xc0298fdf28 pc=0x439033
runtime.timerproc(0x28c7240)
	/usr/local/go/src/runtime/time.go:280 +0x288 fp=0xc0298fdfd8 sp=0xc0298fdf58 pc=0x456158
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0298fdfe0 sp=0xc0298fdfd8 pc=0x466f51
created by runtime.(*timersBucket).addtimerLocked
	/usr/local/go/src/runtime/time.go:170 +0x114
bash: line 1:  4396 Aborted                 (core dumped) bash -c "./workload run tpcc --ramp=5m --warehouses=7000 --active-warehouses=4000 --duration=15m --split --scatter 'postgres://[email protected]:26257?sslmode=disable' 'postgres://[email protected]:26257?sslmode=disable' 'postgres://[email protected]:26257?sslmode=disable' 'postgres://[email protected]:26257?sslmode=disable' 'postgres://[email protected]:26257?sslmode=disable' 'postgres://[email protected]:26257?sslmode=disable' 'postgres://[email protected]:26257?sslmode=disable' 'postgres://[email protected]:26257?sslmode=disable' 'postgres://[email protected]:26257?sslmode=disable'"
Error:  exit status 134

To Reproduce

set -euxo pipefail
export CLUSTER=andy-okr4
roachprod create $CLUSTER -n 10 --gce-machine-type=n1-standard-16 --geo --gce-zones=us-east1-b --gce-zones=us-east1-c --gce-zones=us-east1-d
roachprod run $CLUSTER --  "sudo umount /mnt/data1; sudo mount -o discard,defaults,nobarrier /mnt/data1/; mount | grep /mnt/data1"
roachprod stage $CLUSTER:1,3-10 cockroach
roachprod stage $CLUSTER:2 workload
roachprod start $CLUSTER:1,3-10 
roachprod adminurl --open $CLUSTER:1
roachprod run $CLUSTER:1 -- "./cockroach workload fixtures import tpcc --warehouses=7000 --db=tpcc"
roachprod run $CLUSTER:2 "./workload run tpcc --ramp=5m --warehouses=7000 --active-warehouses=4000 --duration=15m --split --scatter {pgurl:1,3-10}"`

Environment:
v19.2.0-alpha.00000000-322-gc67e77c
@awoods187 awoods187 added C-bug Code not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior. S-2-temp-unavailability Temp crashes or other availability problems. Can be worked around or resolved by restarting. labels May 8, 2019
@awoods187
Copy link
Contributor Author

cockroach.log

@yuzefovich
Copy link
Member

The core dumped message at the bottom of the stack trace suggests that it might be the issue with the workload that has been fixed by #37401.

@awoods187
Copy link
Contributor Author

@danhhz do you concur? Should I try this out again?

@danhhz
Copy link
Contributor

danhhz commented May 9, 2019 •
edited
Loading

Yep, this got fixed yesterday. Pulling and rebuilding your workload binary should fix it. No need to update the cockroach binary

@awoods187
Copy link
Contributor Author

Awesome thanks!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
C-bug Code not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior. S-2-temp-unavailability Temp crashes or other availability problems. Can be worked around or resolved by restarting.
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
3 participants
@danhhz @yuzefovich @awoods187