Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

sentry: conn_executor.go:434: panic while executing 1 statements: SELECT _._ AS _ FROM _ AS _ INNER JOIN _ AS _ ON (_._ = _._) AND (_._ = _._) WHERE (_._ = _) AND ((_._ IN (SELECT DISTINCT _ FROM _ WHERE (_ = _) AND (_ IN (_, _)))) AND (_._ LIKE ANY (ARRAY[_]))) GROUP BY _._ ORDER BY _._ DESC LIMIT _ OFFSET _: caused by <redacted> #40841

Closed
cockroach-teamcity opened this issue Sep 17, 2019 · 2 comments · Fixed by #41042
Closed

sentry: conn_executor.go:434: panic while executing 1 statements: SELECT _._ AS _ FROM _ AS _ INNER JOIN _ AS _ ON (_._ = _._) AND (_._ = _._) WHERE (_._ = _) AND ((_._ IN (SELECT DISTINCT _ FROM _ WHERE (_ = _) AND (_ IN (_, _)))) AND (_._ LIKE ANY (ARRAY[_]))) GROUP BY _._ ORDER BY _._ DESC LIMIT _ OFFSET _: caused by <redacted> #40841

cockroach-teamcity opened this issue Sep 17, 2019 · 2 comments · Fixed by #41042
Assignees
@rafiss
Labels
C-bug Code not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior. O-sentry Originated from an in-the-wild panic report. S-2-temp-unavailability Temp crashes or other availability problems. Can be worked around or resolved by restarting.

Comments

@cockroach-teamcity
Copy link
Member

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/1229810161/?referrer=webhooks_plugin

Panic message:

conn_executor.go:434: panic while executing 1 statements: SELECT . AS _ FROM _ AS _ INNER JOIN _ AS _ ON (. = .) AND (. = .) WHERE (. = ) AND ((._ IN (SELECT DISTINCT _ FROM _ WHERE (_ = ) AND ( IN (, )))) AND (. LIKE ANY (ARRAY[_]))) GROUP BY . ORDER BY . DESC LIMIT _ OFFSET _: caused by

Stacktrace (expand for inline code snippets):

cockroach/pkg/sql/conn_executor.go

Lines 433 to 435 in 1a14d34

r := recover()
h.ex.closeWrapper(ctx, r)
}()
in pkg/sql.(*Server).ServeConn.func1
/usr/local/go/src/runtime/asm_amd64.s#L521-L523 in runtime.call32
/usr/local/go/src/runtime/panic.go#L512-L514 in runtime.gopanic

cockroach/pkg/sql/sem/tree/datum.go

Lines 1051 to 1053 in 1a14d34

if !ok {
panic(pgerror.NewAssertionErrorf("expected *DString, found %T", e))
}
in pkg/sql/sem/tree.MustBeDString

cockroach/pkg/sql/sem/tree/eval.go

Lines 2332 to 2334 in 1a14d34

func matchLike(ctx *EvalContext, left, right Datum, caseInsensitive bool) (Datum, error) {
s, pattern := string(MustBeDString(left)), string(MustBeDString(right))
if len(s) == 0 {
in pkg/sql/sem/tree.matchLike

cockroach/pkg/sql/sem/tree/eval.go

Lines 1917 to 1919 in 1a14d34

Fn: func(ctx *EvalContext, left Datum, right Datum) (Datum, error) {
return matchLike(ctx, left, right, false)
},
in pkg/sql/sem/tree.glob..func113

cockroach/pkg/sql/sem/tree/eval.go

Lines 2251 to 2253 in 1a14d34

_, newLeft, newRight, _, not := foldComparisonExpr(subOp, left, elem)
d, err := fn.Fn(ctx, newLeft.(Datum), newRight.(Datum))
if err != nil {
in pkg/sql/sem/tree.evalDatumsCmp

cockroach/pkg/sql/sem/tree/eval.go

Lines 3603 to 3605 in 1a14d34

}
return evalDatumsCmp(ctx, op, expr.SubOperator, expr.fn, left, datums)
}
in pkg/sql/sem/tree.(*ComparisonExpr).Eval

cockroach/pkg/sql/sqlbase/expr_filter.go

Lines 25 to 27 in 1a14d34

d, err := filter.Eval(evalCtx)
if err != nil {
in pkg/sql/sqlbase.RunFilter

cockroach/pkg/sql/distsqlrun/expr.go

Lines 178 to 180 in 1a14d34

eh.evalCtx.PushIVarContainer(eh)
pass, err := sqlbase.RunFilter(eh.expr, eh.evalCtx)
eh.evalCtx.PopIVarContainer()
in pkg/sql/distsqlrun.(*exprHelper).evalFilter

cockroach/pkg/sql/distsqlrun/processors.go

Lines 354 to 356 in 1a14d34

// Filtering.
passes, err := h.filter.evalFilter(row)
if err != nil {
in pkg/sql/distsqlrun.(*ProcOutputHelper).ProcessRow

cockroach/pkg/sql/distsqlrun/processors.go

Lines 777 to 779 in 1a14d34

func (pb *ProcessorBase) ProcessRowHelper(row sqlbase.EncDatumRow) sqlbase.EncDatumRow {
outRow, ok, err := pb.out.ProcessRow(pb.Ctx, row)
if err != nil {
in pkg/sql/distsqlrun.(*ProcessorBase).ProcessRowHelper

cockroach/pkg/sql/distsqlrun/tablereader.go

Lines 306 to 308 in 1a14d34

if outRow := tr.ProcessRowHelper(row); outRow != nil {
return outRow, nil
in pkg/sql/distsqlrun.(*tableReader).Next

cockroach/pkg/sql/distsqlrun/hashjoiner.go

Lines 638 to 640 in 1a14d34

}
row, meta := source.Next()
if meta != nil {
in pkg/sql/distsqlrun.(*hashJoiner).receiveNext

cockroach/pkg/sql/distsqlrun/hashjoiner.go

Lines 302 to 304 in 1a14d34

row, meta, emitDirectly, err := h.receiveNext(side)
if err != nil {
in pkg/sql/distsqlrun.(*hashJoiner).build

cockroach/pkg/sql/distsqlrun/hashjoiner.go

Lines 236 to 238 in 1a14d34

case hjBuilding:
h.runningState, row, meta = h.build()
case hjConsumingStoredSide:
in pkg/sql/distsqlrun.(*hashJoiner).Next

cockroach/pkg/sql/distsqlrun/sorter.go

Lines 304 to 306 in 1a14d34

for {
row, meta := s.input.Next()
if meta != nil {
in pkg/sql/distsqlrun.(*sortAllProcessor).fill

cockroach/pkg/sql/distsqlrun/sorter.go

Lines 285 to 287 in 1a14d34

valid, err := s.fill()
if !valid || err != nil {
in pkg/sql/distsqlrun.(*sortAllProcessor).Start

cockroach/pkg/sql/distsqlrun/distinct.go

Lines 140 to 142 in 1a14d34

func (d *Distinct) Start(ctx context.Context) context.Context {
d.input.Start(ctx)
return d.StartInternal(ctx, distinctProcName)
in pkg/sql/distsqlrun.(*Distinct).Start

cockroach/pkg/sql/distsqlrun/processors.go

Lines 799 to 801 in 1a14d34

}
ctx = pb.self.Start(ctx)
Run(ctx, pb.self, pb.out.output)
in pkg/sql/distsqlrun.(*ProcessorBase).Run

cockroach/pkg/sql/distsqlrun/flow.go

Lines 625 to 627 in 1a14d34

}
headProc.Run(ctx)
return nil
in pkg/sql/distsqlrun.(*Flow).Run

cockroach/pkg/sql/distsql_running.go

Lines 251 to 253 in 1a14d34

// TODO(radu): this should go through the flow scheduler.
if err := flow.Run(ctx, func() {}); err != nil {
log.Fatalf(ctx, "unexpected error from syncFlow.Start(): %s "+
in pkg/sql.(*DistSQLPlanner).Run

cockroach/pkg/sql/distsql_running.go

Lines 838 to 840 in 1a14d34

dsp.FinalizePlan(planCtx, &physPlan)
dsp.Run(planCtx, txn, &physPlan, recv, evalCtx, nil /* finishedSetupFn */)
}
in pkg/sql.(*DistSQLPlanner).PlanAndRun

cockroach/pkg/sql/conn_executor_exec.go

Lines 1124 to 1126 in 1a14d34

// the planner whether or not to plan remote table readers.
ex.server.cfg.DistSQLPlanner.PlanAndRun(
ctx, evalCtx, planCtx, planner.txn, planner.curPlan.plan, recv)
in pkg/sql.(*connExecutor).execWithDistSQLEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 960 to 962 in 1a14d34

ex.sessionTracing.TraceExecStart(ctx, "distributed")
err = ex.execWithDistSQLEngine(ctx, planner, stmt.AST.StatementType(), res, distributePlan)
ex.sessionTracing.TraceExecEnd(ctx, res.Err(), res.RowsAffected())
in pkg/sql.(*connExecutor).dispatchToExecutionEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 455 to 457 in 1a14d34

p.autoCommit = os.ImplicitTxn.Get() && !ex.server.cfg.TestingKnobs.DisableAutoCommit
if err := ex.dispatchToExecutionEngine(ctx, p, res); err != nil {
return nil, nil, err
in pkg/sql.(*connExecutor).execStmtInOpenState

cockroach/pkg/sql/conn_executor_exec.go

Lines 101 to 103 in 1a14d34

} else {
ev, payload, err = ex.execStmtInOpenState(ctx, stmt, pinfo, res)
}
in pkg/sql.(*connExecutor).execStmt

cockroach/pkg/sql/conn_executor.go

Lines 1185 to 1187 in 1a14d34

stmtCtx := withStatement(ex.Ctx(), ex.curStmt)
ev, payload, err = ex.execStmt(stmtCtx, curStmt, stmtRes, nil /* pinfo */)
if err != nil {
in pkg/sql.(*connExecutor).run

cockroach/pkg/sql/conn_executor.go

Lines 435 to 437 in 1a14d34

}()
return h.ex.run(ctx, s.pool, reserved, cancel)
}
in pkg/sql.(*Server).ServeConn

cockroach/pkg/sql/pgwire/conn.go

Lines 579 to 581 in 1a14d34

reservedOwned = false // We're about to pass ownership away.
retErr = sqlServer.ServeConn(ctx, connHandler, reserved, cancelConn)
}()
in pkg/sql/pgwire.(*conn).processCommandsAsync.func1

pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn.func1 at line 434
/usr/local/go/src/runtime/asm_amd64.s in runtime.call32 at line 522
/usr/local/go/src/runtime/panic.go in runtime.gopanic at line 513
pkg/sql/sem/tree/datum.go in pkg/sql/sem/tree.MustBeDString at line 1052
pkg/sql/sem/tree/eval.go in pkg/sql/sem/tree.matchLike at line 2333
pkg/sql/sem/tree/eval.go in pkg/sql/sem/tree.glob..func113 at line 1918
pkg/sql/sem/tree/eval.go in pkg/sql/sem/tree.evalDatumsCmp at line 2252
pkg/sql/sem/tree/eval.go in pkg/sql/sem/tree.(*ComparisonExpr).Eval at line 3604
pkg/sql/sqlbase/expr_filter.go in pkg/sql/sqlbase.RunFilter at line 26
pkg/sql/distsqlrun/expr.go in pkg/sql/distsqlrun.(*exprHelper).evalFilter at line 179
pkg/sql/distsqlrun/processors.go in pkg/sql/distsqlrun.(*ProcOutputHelper).ProcessRow at line 355
pkg/sql/distsqlrun/processors.go in pkg/sql/distsqlrun.(*ProcessorBase).ProcessRowHelper at line 778
pkg/sql/distsqlrun/tablereader.go in pkg/sql/distsqlrun.(*tableReader).Next at line 307
pkg/sql/distsqlrun/hashjoiner.go in pkg/sql/distsqlrun.(*hashJoiner).receiveNext at line 639
pkg/sql/distsqlrun/hashjoiner.go in pkg/sql/distsqlrun.(*hashJoiner).build at line 303
pkg/sql/distsqlrun/hashjoiner.go in pkg/sql/distsqlrun.(*hashJoiner).Next at line 237
pkg/sql/distsqlrun/sorter.go in pkg/sql/distsqlrun.(*sortAllProcessor).fill at line 305
pkg/sql/distsqlrun/sorter.go in pkg/sql/distsqlrun.(*sortAllProcessor).Start at line 286
pkg/sql/distsqlrun/distinct.go in pkg/sql/distsqlrun.(*Distinct).Start at line 141
pkg/sql/distsqlrun/processors.go in pkg/sql/distsqlrun.(*ProcessorBase).Run at line 800
pkg/sql/distsqlrun/flow.go in pkg/sql/distsqlrun.(*Flow).Run at line 626
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).Run at line 252
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).PlanAndRun at line 839
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execWithDistSQLEngine at line 1125
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).dispatchToExecutionEngine at line 961
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmtInOpenState at line 456
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt at line 102
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).run at line 1186
pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn at line 436
pkg/sql/pgwire/conn.go in pkg/sql/pgwire.(*conn).processCommandsAsync.func1 at line 580
Tag Value
Cockroach Release v19.1.3
Cockroach SHA: 1a14d34
Platform linux amd64
Distribution CCL
Environment v19.1.3
Command server
Go Version go1.11.6
# of CPUs 4
# of Goroutines 214
@cockroach-teamcity cockroach-teamcity added C-bug Code not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior. O-sentry Originated from an in-the-wild panic report. labels Sep 17, 2019
@jordanlewis jordanlewis added the S-2-temp-unavailability Temp crashes or other availability problems. Can be worked around or resolved by restarting. label Sep 19, 2019
@jordanlewis jordanlewis mentioned this issue Sep 19, 2019
Closed
@jordanlewis
Copy link
Member

Figured out a repro:

[email protected]:53919/movr> create table a (a string);
CREATE TABLE

Time: 2.487ms

[email protected]:53919/movr> insert into a values(null);
INSERT 1

Time: 2.692ms

[email protected]:53919/movr> select a like any(array['foo']) from a;
*
* ERROR: [n1,client=127.0.0.1:53925,user=root] a SQL panic has occurred while executing "SELECT a LIKE ANY (ARRAY['foo']) FROM a": expected *DString, found tree.dNull
*
*
* ERROR: [n1,client=127.0.0.1:53925,user=root] a panic has occurred!
*
panic while executing 1 statements: SELECT _ LIKE ANY (ARRAY[_]) FROM _

goroutine 4173 [running]:
runtime/debug.Stack(0x8448b60, 0xc0022a7040, 0x3)
        /usr/local/Cellar/[email protected]/1.12.9/libexec/src/runtime/debug/stack.go:24 +0x9d
github.com/cockroachdb/cockroach/pkg/util/log.ReportPanic(0x8448b60, 0xc0022a7040, 0xc000c78000, 0x791bfa0, 0xc0023650b0, 0x1)
        /Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/util/log/crash_reporting.go:217 +0xb6
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).closeWrapper(0xc002111000, 0x8448b60, 0xc0022a7040, 0x7794180, 0xc001a64400)
        /Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:723 +0x2a0
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn.func1(0xc002111000, 0x8448b60, 0xc0022a7040)
        /Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:442 +0x61
panic(0x7794180, 0xc001a64400)
        /usr/local/Cellar/[email protected]/1.12.9/libexec/src/runtime/panic.go:522 +0x1b5
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.MustBeDString(0x846f9e0, 0xae7c3d0, 0xae7c3d0, 0x846f9e0)
        /Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/sql/sem/tree/datum.go:1098 +0xe3
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.matchLike(0xc00280f1e0, 0x84d0da0, 0xae7c3d0, 0x84d04e0, 0xc002074a18, 0x400ca00, 0x78b09a0, 0x79c5780, 0x0, 0x84d04e0)
        /Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/sql/sem/tree/eval.go:2456 +0x71
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.glob..func113(0xc00280f1e0, 0x84d0da0, 0xae7c3d0, 0x84d04e0, 0xc002074a18, 0x8, 0x846f9e0, 0xae7c3d0, 0x846caa0)
        /Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/sql/sem/tree/eval.go:1977 +0x5f
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.evalDatumsCmp(0xc00280f1e0, 0x1a, 0x8, 0xc000506800, 0x84d0da0, 0xae7c3d0, 0xc001a64350, 0x1, 0x1, 0x0, ...)
        /Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/sql/sem/tree/eval.go:2363 +0x301
github.com/cockroachdb/cockroach/pkg/sql/sem/tree.(*ComparisonExpr).Eval(0xc002074b00, 0xc00280f1e0, 0x0, 0x0, 0x1, 0xc001a643c0)
        /Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/sql/sem/tree/eval.go:3775 +0x201
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ExprHelper).Eval(0xc0027a2d80, 0xc000c69020, 0x2, 0x2, 0x0, 0x0, 0xc002364d20, 0xc002074bc0)
        /Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/expr.go:192 +0xef
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcOutputHelper).ProcessRow(0xc000bcd518, 0x8448b60, 0xc002074bc0, 0xc000c69020, 0x2, 0x2, 0xc00225f000, 0xc00225f070, 0x0, 0x0, ...)
        /Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:293 +0x2d8
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBase).ProcessRowHelper(0xc000bcd500, 0xc000c69020, 0x2, 0x2, 0x0, 0x0, 0xc001ab404d)
        /Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:702 +0x78
github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*tableReader).Next(0xc000bcd500, 0x18, 0x0, 0x0, 0x0)

I think this might be related to #37775. @rafiss could you PTAL?

@rafiss
Copy link
Collaborator

rafiss commented Sep 24, 2019 •
edited
Loading

I found the issue -- it relates to LIKE specifically. fix incoming

@rafiss rafiss mentioned this issue Sep 24, 2019
Merged
craig bot pushed a commit that referenced this issue Sep 24, 2019
@rytaft @rafiss
Merge #41037 #41042
ec3947e 
41037: stats: add logging to TestDefaultColumns to help debug flake r=rytaft a=rytaft

This commit adds some logging to the `TestDefaultColumns` test in order
to help debug a test failure.

Informs #38572

Release note: None
Release justification: category 1, non-production code change

41042: tree: fix LIKE suboperator with NULL LHS r=rafiss a=rafiss

If a NULL value was present in the LHS of a suboperator with a LIKE
comparison, then previously the server would crash with a panic. Now it
is handled and there are tests.

closes #40841

Release justification: Low impact bug fix that prevents a server crash.

Release note: None

Co-authored-by: Rebecca Taft <[email protected]>
Co-authored-by: Rafi Shamim <[email protected]>
@craig craig bot closed this as completed in 477d887 Sep 24, 2019
@rafiss rafiss mentioned this issue Sep 25, 2019
Merged
rafiss added a commit to rafiss/cockroach that referenced this issue Sep 25, 2019
@rafiss
tree: fix LIKE suboperator with NULL LHS
422d027 
If a NULL value was present in the LHS of a suboperator with a LIKE
comparison, then previously the server would crash with a panic. Now it
is handled and there are tests.

closes cockroachdb#40841

Release note (bug fix): Fix a crash that occurs when a suboperator with
a LIKE comparison has a NULL left-hand side.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees

@rafiss rafiss

Labels
C-bug Code not up to spec/doc, specs & docs deemed correct. Solution expected to change code/behavior. O-sentry Originated from an in-the-wild panic report. S-2-temp-unavailability Temp crashes or other availability problems. Can be worked around or resolved by restarting.
Projects
None yet
Milestone
No milestone
Development

Successfully merging a pull request may close this issue.

tree: fix LIKE suboperator with NULL LHS rafiss/cockroach
3 participants
@jordanlewis @rafiss @cockroach-teamcity