kv/kvserver: TestFlowControlClassPrioritization failed

[cockroach-teamcity]

kv/kvserver.TestFlowControlClassPrioritization failed with artifacts on master @ fba93856e7fe76c1573c8c9ebeac6b6395e3e730:

github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).next(0x4008c6b2c0)
	github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:124 +0x38
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).nextAdapter(...)
	github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:141
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x400ea64a10?)
	github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:152 +0x58
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Next(0x4008c6b2c0)
	github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:146 +0x38
github.com/cockroachdb/cockroach/pkg/sql/execinfra.Run({0x7d52ab0, 0x400d1e1aa0}, {0x7d83fa0, 0x4008c6b2c0}, {0x7d1b328, 0x40035a6808})
	github.com/cockroachdb/cockroach/pkg/sql/execinfra/base.go:198 +0x48
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0x4008c6b2c0, {0x7d52a78?, 0x4009d76be0?}, {0x7d1b328, 0x40035a6808})
	github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:727 +0x6c
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0x400d196788, {0x7d52a78, 0x4009d76be0}, 0x0?)
	github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:579 +0x180
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0x7dd2fe0?, {0x7d52a78?, 0x4009d76be0?}, 0xe0?)
	github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:305 +0x1d0
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0x4001e0d400, {0x7d52ab0, 0x400d1e1350}, 0x4008a0a750, 0x400c6170e0, 0x400d1c8f00, 0x40035a6808, 0x400e9ef040, 0x400ea657b0)
	github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:932 +0xa0c
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x4001e0d400, {0x7d52ab0, 0x400d1e1350}, 0x400e9ef040, 0x4008a0a750, 0x400c6170e0, {{0x7d53f18, 0x400d1c8980}, 0x0}, 0x40035a6808, ...)
	github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1994 +0x174
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll.func3(0x0?, 0x0?, {0x7d52ab0?, 0x400d1e1350?}, 0x8?, 0x400d20f008?, 0x400ed658e8?)
	github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1708 +0xa0
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0x4001e0d400, {0x7d52ab0, 0x400d1e1350}, 0x400e9ef040, 0x4008a0a750, 0x400e9eeb38, 0x40035a6808, 0x0)
	github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1711 +0x178
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0x400e9ee488, {0x7d52ab0, 0x400d1e1350}, 0x400e9eeb38, 0x0?, {0x7dd6030?, 0x4009d76320?}, 0x0, 0x4009bf3718, {0x0, ...})
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2419 +0x360
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0x400e9ee488, {0x7d52a78, 0x4009d76370}, 0x400e9eeb38, {0x7dd6030, 0x4009d76320})
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1966 +0xd64
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0x400e9ee488, {0x7d52ab0, 0x400ce3f920}, {{0x7d7c1e0, 0x4009d76050}, {0x0, 0x0, 0x0}, {0x669315f, 0x296}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1173 +0x3e34
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x7d52ab0?, 0x400ce3f920?})
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:145 +0x8c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x7d52ab0?, {0x7d52ab0?, 0x400ce3f920?}, {0x7d7c1e0?, 0x4009d76050?}, 0x0?, 0x400daacfa8?)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:3425 +0x25c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0x400e9ee488, {0x7d52ab0, 0x400ce3f920}, {{0x7d7c1e0, 0x4009d76050}, {0x0, 0x0, 0x0}, {0x669315f, 0x296}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:144 +0x544
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0x400e9ee488, {0x7d52ab0, 0x400ce3f920}, {{0x0, 0x0}, 0x400c661560, {0x4004f22510, 0x1, 0x1}, {0x0, ...}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:250 +0x35c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0x0, 0x0}, 0x0, {0xc1994b7e4dee1e16, 0x192fc559c, 0x0}, 0x1}, 0x400e9ee488, 0x400ea67b80, 0x400ea67b70, ...)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2421 +0x7fc
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0x400e9ee488)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2423 +0x5c4
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0x400e9ee488, {0x7d52ab0, 0x400ce3f9b0}, 0xa?, 0x400acdaef8?, 0xccf1c8?)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2238 +0x168
github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).runWithEx.func2({0x7d52ab0, 0x400ce3f9b0})
	github.com/cockroachdb/cockroach/pkg/sql/internal.go:247 +0x90
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:485 +0x128
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx in goroutine 26624
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:476 +0x31c

Help

See also: How To Investigate a Go Test Failure (internal)

/cc @cockroachdb/kv

This test on roachdash | Improve this report!

Jira issue: CRDB-39993

[arulajmani]

The panic here seems to originate from SQL, so I'll throw this over to @cockroachdb/sql-queries to have a look.

[yuzefovich]

This is not us

      * ERROR: a panic has occurred!
      * panic: grunning.Time() should be non-decreasing
      * (1) attached stack trace
      *   -- stack trace:
      *   | runtime.gopanic
      *   |   GOROOT/src/runtime/panic.go:770
      *   | github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission.(*controllerImpl).AdmittedKVWorkDone
      *   |   github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission/kvadmission.go:456
      *   | github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func2
      *   |   github.com/cockroachdb/cockroach/pkg/server/node.go:1404
      *   | github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal
      *   |   github.com/cockroachdb/cockroach/pkg/server/node.go:1485
      *   | github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch
      *   |   github.com/cockroachdb/cockroach/pkg/server/node.go:1626
      *   | github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.func1
      *   |   github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:704
      *   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.ServerInterceptor.func12
      *   |   github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:97
      *   | github.com/cockroachdb/cockroach/pkg/rpc.makeInternalClientAdapter.chainUna

[aadityasondhi]

Being tracked by #126681.