logic_test.go

// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Peter Mattis (peter@cockroachlabs.com)

package sql_test

import (
	"bufio"
	"bytes"
	"crypto/md5"
	gosql "database/sql"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"runtime/trace"
	"sort"
	"strconv"
	"strings"
	"testing"
	"text/tabwriter"
	"time"
	"unicode/utf8"

	"github.com/cockroachdb/cockroach/config"
	"github.com/cockroachdb/cockroach/security"
	"github.com/cockroachdb/cockroach/server"
	"github.com/cockroachdb/cockroach/sql"
	"github.com/cockroachdb/cockroach/testutils"
	"github.com/cockroachdb/cockroach/testutils/sqlutils"
	"github.com/cockroachdb/cockroach/util/leaktest"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/timeutil"
	"github.com/cockroachdb/pq"
)

// This file is home to TestLogic, a general-purpose engine for
// running SQL logic tests.
//
// TestLogic is used both for direct unit testing of CockroachDB
// during development cycles, and regular (nightly) stress testing of
// the product.
//
// In a nutshell, TestLogic reads one or more test input files
// containing sequences of SQL statements and queries. Each input file
// is meant to test a feature group. The reason why tests can/should
// be split across multiple files is that each test input file gets
// its own fresh, empty database.
//
// Input files for unit testing are stored alongside the source code
// in the `testdata` subdirectory. The input files for the larger
// `bigtest` are stored in a separate repository.
//
// The test input is expressed using a domain-specific language, called
// Test-Script, defined by SQLite's "Sqllogictest".  The official home
// for Sqllogictest and Test-Script is
//      https://www.sqlite.org/sqllogictest/
//
// (CockroachDB's `bigtest` is actually a fork of the Sqllogictest
// test files; its input files are hosted at
// https://github.com/cockroachdb/sqllogictest )
//
// The Test-Script language is extended here for use with CockroachDB,
// for example it introduces the "traceon" and "traceoff"
// directives. See processTestFile() for all supported test
// directives.
//
// The Test-Script is line-oriented. It supports both statements which
// generate no result rows, and queries that produce result rows. The
// result of queries can be checked either using an explicit reference
// output in the test file, or using the expected row count and a hash
// of the expected output. A test can also check for expected column
// names for query results, or expected errors.
//
// The overall architecture of TestLogic is as follows:

// - TestLogic() selects the input files and instantiates
//   a `logicTest` object for each input file.
//
// - logicTest.run() sets up a new database.
// - logicTest.processTestFile() runs all tests from that input file.
//
// - each test is instantiated either as a logicStatement or
//   logicQuery object. These are then processed by either
//   logicTest.execStatement() or logicTest.execQuery().
//
// TestLogic has three main parameter groups:
//
// - Selection of the input files.
// - How and how early to stop when tests fail.
// - What gets reported.
//
// The parameters are typically set using the TESTFLAGS `make`
// parameter, as in:
//
//   make test PKG=./sql TESTS=TestLogic TESTFLAGS='....'
//
// Selection of input files:
//
// -d <glob>  selects all files matching <glob>. This can mix and
//            match wildcards (*/?) or groups like {a,b,c}.
//
// -bigtest   cancels any -d setting and selects all relevant input
//            files from CockroachDB's fork of Sqllogictest.
//
// How and how early to stop when tests fail:
//
// -try-harder    continue testing after the first error encountered.
//                This can help find more errors to troubleshoot during
//                one development iteration.
//
// -max-errors N  when -try-harder is specified, this stops testing
//                after N errors have been encountered. Default 10.
//
// -soft          tolerate / ignore errors that occur during query
//                preparation. With -soft you can indicate that it is
//                OK as long as the database reports early to the
//                client that it does not support a given query.
//                Errors are still reported if queries fail during
//                execution only or if a statement fails.
//
// -flex-types    tolerate when a result column is produced
//                with another numeric type than the one expected by
//                the test. This enables reusing tests designed for
//                database with sligtly different typing semantics.
//
// What gets reported:
//
// -v             be more verbose. This causes database log messages
//                to be output alongside the test results. It also
//                enables progress messages during long tests.
//
// -show-sql      show SQL statements/queries immediately before they
//                are tested. This can be useful for example when
//                troubleshooting errors that cause the database/test
//                to stop before the test completes.  When -show-sql
//                is set, individual test results are annoted with
//                either "OK" (test passed as expected), "XFAIL"
//                (expected failure, test failed as expected), or
//                "FAIL" to indicate an unexpected/undesired test
//                failure.
//                Works only in combination with -v.
//
// -error-summary produces a report organized by error message
//                of all queries that have caused that error.
//                Useful with -soft and/or -flex-types.
//                Works only in combination with -v.
//
// -full-messages by default -error-summary shortens error messages
//                and queries so they fit in a moderately large
//                terminal screen. With this parameter, the
//                full text of errors and queries is printed.
//                Works only in combination with -v.
//
// Suggested use:
//
// - For validation testing: just -d or -bigtest.
// - For compatibility testing: add -soft -flex-types.
// - For troubleshooting / analysis: add -v -show-sql -error-summary.

var (
	resultsRE = regexp.MustCompile(`^(\d+)\s+values?\s+hashing\s+to\s+([0-9A-Fa-f]+)$`)
	errorRE   = regexp.MustCompile(`^(?:statement|query)\s+error\s+(?:pgcode\s+([[:alnum:]]+)\s+)?(.*)$`)

	// Input selection
	logictestdata = flag.String("d", "testdata/[^.]*", "test data glob")
	bigtest       = flag.Bool("bigtest", false, "use the big set of logic test files (overrides testdata)")

	// Testing mode
	tryHarder = flag.Bool("try-harder", false,
		"do not stop the test at the first query error encountered")
	maxErrsPerInput = flag.Int("max-errors", 10,
		"stop processing each input file after this number of errors (set to 0 for no limit)")
	softTest  = flag.Bool("soft", false, "tolerate unexpected errors when preparing a query")
	flexTypes = flag.Bool("flex-types", false,
		"do not fail when a test expects a column of a number type but the query provides another type")

	// Output parameters
	showSQL = flag.Bool("show-sql", false,
		"print the individual SQL statement/queries before processing")
	errorSummary = flag.Bool("error-summary", false,
		"print a per-error summary of failing queries at the end of testing, when -soft is set")
	fullMessages = flag.Bool("full-messages", false,
		"do not shorten the error or SQL strings when printing the summary for -soft or -flex-types.")
)

// logicMaxOffset is the value of the MaxOffset parameter used for
// each test database. This value is smaller than the default so as to
// make the tests run faster.
const logicMaxOffset = 50 * time.Millisecond

// lineScanner handles reading from input test files.
type lineScanner struct {
	*bufio.Scanner
	line int
	skip bool
}

func newLineScanner(r io.Reader) *lineScanner {
	return &lineScanner{
		Scanner: bufio.NewScanner(r),
		line:    0,
	}
}

func (l *lineScanner) Scan() bool {
	ok := l.Scanner.Scan()
	if ok {
		l.line++
	}
	return ok
}

// logicStatement represents a single statement test in Test-Script.
type logicStatement struct {
	// file and line number of the test.
	pos string
	// SQL string to be sent to the database.
	sql string
	// expected error, if any. "" indicates the statement should
	// succeed.
	expectErr string
	// expected pgcode for the error, if any. "" indicates no particular
	// pgwire error code is expected.
	expectErrCode string
}

// logicSorter is the common type for result sorter functions.
// Sorting occurs (or not) depending on Test-Script's sorting option
// for a "query" test. See the implementation of the "query" directive
// below for details.
type logicSorter func(numCols int, values []string)

type rowSorter struct {
	numCols int
	numRows int
	values  []string
}

func (r rowSorter) row(i int) []string {
	return r.values[i*r.numCols : (i+1)*r.numCols]
}

func (r rowSorter) Len() int {
	return r.numRows
}

func (r rowSorter) Less(i, j int) bool {
	a := r.row(i)
	b := r.row(j)
	for k := range a {
		if a[k] < b[k] {
			return true
		}
		if a[k] > b[k] {
			return false
		}
	}
	return false
}

func (r rowSorter) Swap(i, j int) {
	a := r.row(i)
	b := r.row(j)
	for i := range a {
		a[i], b[i] = b[i], a[i]
	}
}

func rowSort(numCols int, values []string) {
	sort.Sort(rowSorter{
		numCols: numCols,
		numRows: len(values) / numCols,
		values:  values,
	})
}

func valueSort(numCols int, values []string) {
	sort.Strings(values)
}

// logicQuery represents a single query test in Test-Script.
type logicQuery struct {
	// pos and sql are as in logicStatement.
	pos string
	sql string
	// colTypes indicate the expected result column types.
	colTypes string
	// colNames tells to check whether the result columns have the same
	// names as indicated in the test input.
	colNames bool
	// label annotates queries so as to check that queries referring to
	// the same label produce the same output.
	label string
	// some tests require the output to match modulo sorting.
	sorter logicSorter
	// expectedErr and expectedErrCode are as in logicStatement.
	expectErr     string
	expectErrCode string
	// expectedResults indicate the expected result values, indicated as
	// the sequence of output text words that would be produced by
	// printing all rows in plain text as in a SQL shell. Note, this
	// does not specify row separators.
	expectedResults []string
	// expectedResultsRaw indicate the same as expectedResults, but
	// retaining the original formatting (whitespace, indentation) as
	// the test input file. This is used for pretty-printing unexpected
	// results.
	expectedResultsRaw []string
	// expectedHash indicates the expected hash of all result rows
	// combined. "" indicates hash checking is disabled.
	expectedHash string
	// expectedValues indicate how many rows are expected when
	// expectedHash is set.
	expectedValues int
}

// logicTest executes the test cases specified in a file. The file format is
// taken from the sqllogictest tool
// (http://www.sqlite.org/sqllogictest/doc/trunk/about.wiki) with various
// extensions to allow specifying errors and additional options. See
// https://github.com/gregrahn/sqllogictest/ for a github mirror of the
// sqllogictest source.
type logicTest struct {
	*testing.T
	// the database server instantiated for this input file.
	srv *testServer
	// map of built clients. Needs to be persisted so that we can
	// re-use them and close them all on exit.
	clients map[string]*gosql.DB
	// client currently in use. This can change during processing
	// of a test input file when encountering the "user" directive.
	// see setUser() for details.
	user            string
	db              *gosql.DB
	cleanupRootUser func()
	// progress holds the number of tests executed so far.
	progress int
	// failures holds the number of tests failed so far, when
	// -try-harder is set.
	failures int
	// unsupported holds the number of queries ignored due
	// to prepare errors, when -soft is set.
	unsupported int
	// lastProgress is used for the progress indicator message.
	lastProgress time.Time
	// traceFile holds the current trace file between "traceon"
	// and "traceoff" directives.
	traceFile *os.File
	// verbose indicate whether -v was passed.
	verbose bool
	// softSummary retains the per-error list of failing queries when
	// -error-summary is set.
	softSummary map[string][]string
}

func (t *logicTest) close() {
	if t.cleanupRootUser != nil {
		t.cleanupRootUser()
	}
	if t.srv != nil {
		cleanupTestServer(t.srv)
		t.srv = nil
	}
	if t.clients != nil {
		for _, c := range t.clients {
			c.Close()
		}
		t.clients = nil
	}
	t.db = nil
}

// setUser sets the DB client to the specified user.
// It returns a cleanup function to be run when the credentials
// are no longer needed.
func (t *logicTest) setUser(user string) func() {
	var outDBName string

	if t.db != nil {
		var inDBName string

		if err := t.db.QueryRow("SHOW DATABASE").Scan(&inDBName); err != nil {
			t.Fatal(err)
		}

		defer func() {
			if inDBName != outDBName {
				// Propagate the DATABASE setting to the newly-live connection.
				if _, err := t.db.Exec(fmt.Sprintf("SET DATABASE = %s", inDBName)); err != nil {
					t.Fatal(err)
				}
			}
		}()
	}

	if t.clients == nil {
		t.clients = map[string]*gosql.DB{}
	}
	if db, ok := t.clients[user]; ok {
		t.db = db
		t.user = user

		if err := t.db.QueryRow("SHOW DATABASE").Scan(&outDBName); err != nil {
			t.Fatal(err)
		}

		// No cleanup necessary, but return a no-op func to avoid nil pointer dereference.
		return func() {}
	}

	pgURL, cleanupFunc := sqlutils.PGUrl(t.T, t.srv.TestServer.ServingAddr(), user, "TestLogic")
	db, err := gosql.Open("postgres", pgURL.String())
	if err != nil {
		t.Fatal(err)
	}
	t.clients[user] = db
	t.db = db
	t.user = user

	if t.verbose {
		fmt.Printf("--- new user: %s\n", user)
	}

	return cleanupFunc
}

func (t *logicTest) run(path string) {
	defer t.close()
	t.setup()
	if err := t.processTestFile(path); err != nil {
		t.Error(err)
	}
}

func (t *logicTest) setup() {
	// TODO(pmattis): Add a flag to make it easy to run the tests against a local
	// MySQL or Postgres instance.
	ctx := server.MakeTestContext()
	ctx.MaxOffset = logicMaxOffset
	ctx.TestingKnobs.SQLExecutor = &sql.ExecutorTestingKnobs{
		WaitForGossipUpdate:   true,
		CheckStmtStringChange: true,
	}
	t.srv = setupTestServerWithContext(t.T, &ctx)

	// db may change over the lifetime of this function, with intermediate
	// values cached in t.clients and finally closed in t.close().
	t.cleanupRootUser = t.setUser(security.RootUser)

	if _, err := t.db.Exec(`
CREATE DATABASE test;
SET DATABASE = test;
`); err != nil {
		t.Fatal(err)
	}
}

// TODO(tschottdorf): some logic tests currently take a long time to run.
// Probably a case of heartbeats timing out or many restarts in some tests.
// Need to investigate when all moving parts are in place.
func (t *logicTest) processTestFile(path string) error {
	file, err := os.Open(path)
	if err != nil {
		return err
	}
	defer file.Close()
	defer t.traceStop()

	if t.verbose {
		fmt.Println("--- queries start here")
		defer t.printCompletion(path)
		if *errorSummary {
			defer t.printErrorSummary()
		}
	}

	t.lastProgress = timeutil.Now()

	execKnobs := t.srv.Ctx.TestingKnobs.SQLExecutor.(*sql.ExecutorTestingKnobs)

	lastFailures := t.failures

	repeat := 1
	s := newLineScanner(file)
	for s.Scan() {
		if *maxErrsPerInput > 0 && t.failures > lastFailures+*maxErrsPerInput {
			return fmt.Errorf("%s: too many errors; skipping to next file", path)
		}

		fields := strings.Fields(s.Text())
		if len(fields) == 0 {
			continue
		}
		cmd := fields[0]
		if strings.HasPrefix(cmd, "#") {
			// Skip comment lines.
			continue
		}
		switch cmd {
		case "repeat":
			// A line "repeat X" makes the test repeat the following statement or query X times.
			var err error
			count := 0
			if len(fields) != 2 {
				err = errors.New("invalid line format")
			} else if count, err = strconv.Atoi(fields[1]); err == nil && count < 2 {
				err = errors.New("invalid count")
			}
			if err != nil {
				return fmt.Errorf("%s:%d invalid repeat line: %s", path, s.line, err)
			}
			repeat = count

		case "statement":
			stmt := logicStatement{pos: fmt.Sprintf("%s:%d", path, s.line)}
			// Parse "query error <regexp>"
			if m := errorRE.FindStringSubmatch(s.Text()); m != nil {
				stmt.expectErrCode = m[1]
				stmt.expectErr = m[2]
			}
			var buf bytes.Buffer
			for s.Scan() {
				line := s.Text()
				if line == "" {
					break
				}
				fmt.Fprintln(&buf, line)
			}
			stmt.sql = strings.TrimSpace(buf.String())
			if !s.skip {
				for i := 0; i < repeat; i++ {
					if ok := t.execStatement(stmt); !ok {
						return fmt.Errorf("%s: error in statement, skipping to next file", stmt.pos)
					}
				}
			} else {
				s.skip = false
			}
			repeat = 1
			t.success(path)

		case "query":
			query := logicQuery{pos: fmt.Sprintf("%s:%d", path, s.line)}
			// Parse "query error <regexp>"
			if m := errorRE.FindStringSubmatch(s.Text()); m != nil {
				query.expectErrCode = m[1]
				query.expectErr = m[2]
			} else if len(fields) < 2 {
				return fmt.Errorf("%s: invalid test statement: %s", query.pos, s.Text())
			} else {
				// Parse "query <type-string> <sort-mode> <label>"
				// The type string specifies the number of columns and their types:
				//   - T for text
				//   - I for integer
				//   - R for floating point or decimal
				//   - B for boolean
				// The sort mode is one of:
				//   - "nosort" (default)
				//   - "rowsort"
				//   - "valuesort"
				//   - "colnames"
				//
				// The label is optional. If specified, the test runner stores a hash
				// of the results of the query under the given label. If the label is
				// reused, the test runner verifies that the results are the
				// same. This can be used to verify that two or more queries in the
				// same test script that are logically equivalent always generate the
				// same output.
				query.colTypes = fields[1]
				if len(fields) >= 3 {
					for _, opt := range strings.Split(fields[2], ",") {
						switch opt {
						case "nosort":
							query.sorter = nil

						case "rowsort":
							query.sorter = rowSort

						case "valuesort":
							query.sorter = valueSort

						case "colnames":
							query.colNames = true

						default:
							return fmt.Errorf("%s: unknown sort mode: %s", query.pos, opt)
						}
					}
				}
				if len(fields) >= 4 {
					query.label = fields[3]
				}
			}

			var buf bytes.Buffer
			for s.Scan() {
				line := s.Text()
				if line == "----" {
					if query.expectErr != "" {
						return fmt.Errorf("%s: invalid ---- delimiter after a query expecting an error: %s", query.pos, query.expectErr)
					}
					break
				}
				if strings.TrimSpace(s.Text()) == "" {
					break
				}
				fmt.Fprintln(&buf, line)
			}
			query.sql = strings.TrimSpace(buf.String())

			if query.expectErr == "" {
				// Query results are either a space separated list of values up to a
				// blank line or a line of the form "xx values hashing to yyy". The
				// latter format is used by sqllogictest when a large number of results
				// match the query.
				if s.Scan() {
					if m := resultsRE.FindStringSubmatch(s.Text()); m != nil {
						var err error
						query.expectedValues, err = strconv.Atoi(m[1])
						if err != nil {
							return err
						}
						query.expectedHash = m[2]
					} else {
						for {
							query.expectedResultsRaw = append(query.expectedResultsRaw, s.Text())
							results := strings.Fields(s.Text())
							if len(results) == 0 {
								break
							}
							query.expectedResults = append(query.expectedResults, results...)
							if !s.Scan() {
								break
							}
						}
						query.expectedValues = len(query.expectedResults)
					}
				}
			}

			if !s.skip {
				for i := 0; i < repeat; i++ {
					t.execQuery(query)
				}
			} else {
				s.skip = false
			}
			repeat = 1
			t.success(path)

		case "halt", "hash-threshold":
			break

		case "user":
			if len(fields) < 2 {
				return fmt.Errorf("user command requires one argument, found: %v", fields)
			}
			if len(fields[1]) == 0 {
				return errors.New("user command requires a non-blank argument")
			}
			cleanupUserFunc := t.setUser(fields[1])
			defer cleanupUserFunc()

		case "skipif":
			if len(fields) < 2 {
				return fmt.Errorf("skipif command requires one argument, found: %v", fields)
			}
			switch fields[1] {
			case "":
				return errors.New("skipif command requires a non-blank argument")
			case "mysql":
			case "postgresql", "cockroachdb":
				s.skip = true
				continue
			default:
				return fmt.Errorf("unimplemented test statement: %s", s.Text())
			}

		case "onlyif":
			if len(fields) < 2 {
				return fmt.Errorf("onlyif command requires one argument, found: %v", fields)
			}
			switch fields[1] {
			case "":
				return errors.New("onlyif command requires a non-blank argument")
			case "cockroachdb":
			case "mysql":
				s.skip = true
				continue
			default:
				return fmt.Errorf("unimplemented test statement: %s", s.Text())
			}

		case "traceon":
			if len(fields) != 2 {
				return fmt.Errorf("traceon requires a filename argument, found: %v", fields)
			}
			t.traceStart(fields[1])

		case "traceoff":
			if t.traceFile == nil {
				return errors.New("no trace active")
			}
			t.traceStop()

		case "fix-txn-priorities":
			// fix-txn-priorities causes future transactions to have hardcoded
			// priority values (based on the priority level), (replacing the
			// probabilistic generation).
			// The change stays in effect for the duration of that particular
			// test file.
			if len(fields) != 1 {
				return fmt.Errorf("fix-txn-priority takes no arguments, found: %v", fields[1:])
			}
			fmt.Println("Setting deterministic priorities.")

			execKnobs.FixTxnPriority = true
			defer func() { execKnobs.FixTxnPriority = false }()
		default:
			return fmt.Errorf("%s:%d: unknown command: %s", path, s.line, cmd)
		}
	}

	if err := s.Err(); err != nil {
		return err
	}

	return nil
}

// verifyError checks that either no error was found where none was
// expected, or that an error was found when one was expected. Returns
// "true" to indicate the behavior was as expected.
func (t *logicTest) verifyError(
	sql string, pos string, expectErr string, expectErrCode string, err error) bool {
	if expectErr == "" && expectErrCode == "" && err != nil {
		return t.unexpectedError(sql, pos, err)
	}
	if expectErr != "" && !testutils.IsError(err, expectErr) {
		if err != nil {
			t.Errorf("%s: expected %q, but found\n%s", pos, expectErr, err)
		} else {
			t.Errorf("%s: expected %q, but found success", pos, expectErr)
		}
		return false
	}
	if expectErrCode != "" {
		if err != nil {
			pqErr, ok := err.(*pq.Error)
			if !ok {
				t.Errorf("%s: expected error code %q, but the error we found is not "+
					"a libpq error: %s", pos, expectErrCode, err)
				return false
			}
			if pqErr.Code != pq.ErrorCode(expectErrCode) {
				t.Errorf("%s: expected error code %q, but found code %q (%s)",
					pos, expectErrCode, pqErr.Code, pqErr.Code.Name())
				return false
			}
		} else {
			t.Errorf("%s: expected error code %q, but found success",
				pos, expectErrCode)
			return false
		}
	}
	return true
}

// unexpectedError handles ignoring queries that fail during prepare
// when -soft is specified. The argument "sql" is "" to indicate the
// work is done on behalf of a statement, which always fail upon an
// unexpected error.
func (t *logicTest) unexpectedError(sql string, pos string, err error) bool {
	if *softTest && sql != "" {
		// This is a query and -soft is set.  Try to prepare the query. If
		// prepare fails, this means we (probably) do not support the
		// input syntax, and -soft instructs us to ignore the unexpected
		// error.
		stmt, err := t.db.Prepare(sql)
		if err != nil {
			if *showSQL {
				fmt.Printf("\t-- fails prepare: %s", err)
			}
			t.signalSoftError(err, pos, sql)
			return true
		}
		if err := stmt.Close(); err != nil {
			t.Errorf("%s: error when closing prepared statement: %s", pos, err)
		}
	}
	t.Errorf("%s: expected success, but found\n%s", pos, err)
	return false
}

func (t *logicTest) execStatement(stmt logicStatement) bool {
	if *showSQL {
		fmt.Printf("%s;", stmt.sql)
	}
	_, err := t.db.Exec(stmt.sql)

	// General policy for failing vs. continuing:
	// - we want to do as much work as possible;
	// - however, a statement that fails when it should succeed or
	//   a statement that succeeds when it should fail may have left
	//   the database in an improper state, so we stop there;
	// - error on expected error is worth going further, even
	//   if the obtained error does not match the expected error.
	if ok := t.verifyError("", stmt.pos, stmt.expectErr, stmt.expectErrCode, err); !ok {
		return false
	}
	t.finishOne("OK")
	return true
}

func (t *logicTest) execQuery(query logicQuery) {
	if *showSQL {
		fmt.Printf("%s;", query.sql)
	}
	rows, err := t.db.Query(query.sql)
	if ok := t.verifyError(query.sql, query.pos, query.expectErr, query.expectErrCode, err); !ok {
		return
	}
	if err != nil {
		// An error occurred, but it was expected.
		t.finishOne("XFAIL")
		return
	}
	defer rows.Close()

	cols, err := rows.Columns()
	if err != nil {
		t.Error(err)
		return
	}
	if len(query.colTypes) != len(cols) {
		t.Errorf("%s: expected %d columns based on type-string, but found %d",
			query.pos, len(query.colTypes), len(cols))
		return
	}
	vals := make([]interface{}, len(cols))
	for i := range vals {
		vals[i] = new(interface{})
	}

	var results []string
	var resultLines [][]string
	if query.colNames {
		for _, col := range cols {
			// We split column names on whitespace and append a separate "result"
			// for each string. A bit unusual, but otherwise we can't match strings
			// containing whitespace.
			results = append(results, strings.Fields(col)...)
		}
		resultLines = append(resultLines, cols)
	}
	for rows.Next() {
		var resultLine []string
		if err := rows.Scan(vals...); err != nil {
			t.Error(err)
			return
		}
		for i, v := range vals {
			if val := *v.(*interface{}); val != nil {
				valT := reflect.TypeOf(val).Kind()
				colT := query.colTypes[i]
				switch colT {
				case 'T':
					if valT != reflect.String && valT != reflect.Slice && valT != reflect.Struct {
						t.Errorf("%s: expected text value for column %d, but found %T: %#v", query.pos, i, val, val)
						return
					}
				case 'I':
					if valT != reflect.Int64 {
						if *flexTypes && (valT == reflect.Float64 || valT == reflect.Slice) {
							t.signalSoftError(fmt.Errorf("result type mismatch: expected I, got %T", val), query.pos, query.sql)
						} else {
							t.Errorf("%s: expected int value for column %d, but found %T: %#v", query.pos, i, val, val)
						}
						return
					}
				case 'R':
					if valT != reflect.Float64 && valT != reflect.Slice {
						if *flexTypes && (valT == reflect.Int64) {
							t.signalSoftError(fmt.Errorf("result type mismatch: expected R, got %T", val), query.pos, query.sql)
						} else {
							t.Errorf("%s: expected float/decimal value for column %d, but found %T: %#v", query.pos, i, val, val)
						}
						return
					}
				case 'B':
					if valT != reflect.Bool {
						t.Errorf("%s: expected boolean value for column %d, but found %T: %#v", query.pos, i, val, val)
						return
					}
				default:
					t.Errorf("%s: unknown type in type string: %c in %s", query.pos, colT, query.colTypes)
					return
				}

				if byteArray, ok := val.([]byte); ok {
					// The postgres wire protocol does not distinguish between
					// strings and byte arrays, but our tests do. In order to do
					// The Right Thing™, we replace byte arrays which are valid
					// UTF-8 with strings. This allows byte arrays which are not
					// valid UTF-8 to print as a list of bytes (e.g. `[124 107]`)
					// while printing valid strings naturally.
					if str := string(byteArray); utf8.ValidString(str) {
						val = str
					}
				}
				// We split string results on whitespace and append a separate result
				// for each string. A bit unusual, but otherwise we can't match strings
				// containing whitespace.
				valStr := fmt.Sprint(val)
				results = append(results, strings.Fields(valStr)...)
				resultLine = append(resultLine, valStr)
			} else {
				results = append(results, "NULL")
				resultLine = append(resultLine, "NULL")
			}
		}
		resultLines = append(resultLines, resultLine)
	}
	if err := rows.Err(); err != nil {
		t.Error(err)
		return
	}

	if query.sorter != nil {
		query.sorter(len(cols), results)
	}

	if query.expectedHash != "" {
		n := len(results)
		if query.expectedValues != n {
			t.Errorf("%s: expected %d results, but found %d", query.pos, query.expectedValues, n)
			return
		}
		// Hash the values using MD5. This hashing precisely matches the hashing in
		// sqllogictest.c.
		h := md5.New()
		for _, r := range results {
			if _, err := h.Write(append([]byte(r), byte('\n'))); err != nil {
				t.Error(err)
				return
			}
		}
		hash := fmt.Sprintf("%x", h.Sum(nil))
		if query.expectedHash != hash {
			t.Errorf("%s: expected %s, but found %s", query.pos, query.expectedHash, hash)
			return
		}
	} else if !reflect.DeepEqual(query.expectedResults, results) {
		var buf bytes.Buffer
		tw := tabwriter.NewWriter(&buf, 2, 1, 2, ' ', 0)

		fmt.Fprintf(&buf, "%s: \nexpected:\n", query.pos)
		for _, expectedResultRaw := range query.expectedResultsRaw {
			fmt.Fprintf(tw, "    %s\n", expectedResultRaw)
		}
		fmt.Fprint(&buf, "but found:\n")
		for _, resultLine := range resultLines {
			fmt.Fprint(tw, "    ")
			for _, value := range resultLine {
				fmt.Fprintf(tw, "%s\t", value)
			}
			fmt.Fprint(tw, "\n")
		}
		_ = tw.Flush()
		t.Error(buf.String())
		return
	}

	t.finishOne("OK")
}

func (t *logicTest) success(file string) {
	t.progress++
	now := timeutil.Now()
	if now.Sub(t.lastProgress) >= 2*time.Second {
		t.lastProgress = now
		fmt.Printf("--- progress: %s: %d statements/queries\n", file, t.progress)
	}
}

func (t *logicTest) traceStart(filename string) {
	if t.traceFile != nil {
		t.Fatalf("tracing already active")
	}
	var err error
	t.traceFile, err = os.Create(filename)
	if err != nil {
		t.Fatalf("unable to open trace output file: %s", err)
	}
	if err := trace.Start(t.traceFile); err != nil {
		t.Fatalf("unable to start tracing: %s", err)
	}
}

func (t *logicTest) traceStop() {
	if t.traceFile != nil {
		trace.Stop()
		t.traceFile.Close()
		t.traceFile = nil
	}
}

func TestLogic(t *testing.T) {
	defer leaktest.AfterTest(t)()

	// TODO(marc): splitting ranges at table boundaries causes
	// a blocked task and won't drain. Investigate and fix.
	defer config.TestingDisableTableSplits()()

	var globs []string
	if *bigtest {
		const logicTestPath = "../../sqllogictest"
		if _, err := os.Stat(logicTestPath); os.IsNotExist(err) {
			fullPath, err := filepath.Abs(logicTestPath)
			if err != nil {
				t.Fatal(err)
			}
			t.Fatalf("unable to find sqllogictest repo: %s\n"+
				"git clone https://github.com/cockroachdb/sqllogictest %s",
				logicTestPath, fullPath)
			return
		}
		globs = []string{
			logicTestPath + "/test/index/between/*/*.test",
			logicTestPath + "/test/index/commute/*/*.test",
			logicTestPath + "/test/index/delete/*/*.test",
			logicTestPath + "/test/index/in/*/*.test",
			logicTestPath + "/test/index/orderby/*/*.test",
			logicTestPath + "/test/index/orderby_nosort/*/*.test",

			// TODO(pmattis): We don't support aggregate functions.
			// logicTestPath + "/test/random/expr/*.test",

			// TODO(pmattis): We don't support tables without primary keys.
			// logicTestPath + "/test/select*.test",

			// TODO(pmattis): We don't support views.
			// logicTestPath + "/test/index/view/*/*.test",

			// TODO(pmattis): We don't support joins.
			// [uses joins] logicTestPath + "/test/index/random/*/*.test",
			// [uses joins] logicTestPath + "/test/random/aggregates/*.test",
			// [uses joins] logicTestPath + "/test/random/groupby/*.test",
			// [uses joins] logicTestPath + "/test/random/select/*.test",
		}
	} else {
		globs = []string{*logictestdata}
	}

	var paths []string
	for _, g := range globs {
		match, err := filepath.Glob(g)
		if err != nil {
			t.Fatal(err)
		}
		paths = append(paths, match...)
	}

	if len(paths) == 0 {
		t.Fatalf("No testfiles found (globs: %v)", globs)
	}

	total := 0
	totalFail := 0
	totalUnsupported := 0
	verbose := testing.Verbose() || log.V(1)
	lastProgress := timeutil.Now()
	for _, p := range paths {
		if verbose {
			fmt.Printf("--- input: %s\n", p)
		}
		l := logicTest{
			T:           t,
			verbose:     verbose,
			softSummary: make(map[string][]string),
		}
		l.run(p)
		total += l.progress
		totalFail += l.failures
		totalUnsupported += l.unsupported

		now := timeutil.Now()
		if now.Sub(lastProgress) >= 2*time.Second {
			lastProgress = now
			fmt.Printf("--- total progress: %d statements/queries\n", total)
		}
	}

	unsupportedMsg := ""
	if totalUnsupported > 0 {
		unsupportedMsg = fmt.Sprintf(", ignored %d unsupported queries", totalUnsupported)
	}

	fmt.Printf("--- total: %d tests, %d failures%s\n", total, totalFail, unsupportedMsg)
}

// printErrorSummary shows the final per-error list of failing queries when
// -soft or -flex-types are specified.
func (t *logicTest) printErrorSummary() {
	if t.unsupported > 0 {
		fmt.Println("--- summary of ignored errors:")
		for errmsg, sql := range t.softSummary {
			fmt.Println(errmsg)
			for _, q := range sql {
				fmt.Println("\t", strings.Replace(q, "\n", "\n\t", -1))
			}
			fmt.Println()
		}
	}
}

// splitLastMessagePart separates, for messages containing multiple
// parts separated by ":", all parts but the last and the last part
// into separate strings.
func splitLastMessagePart(msg string) (string, string) {
	expIdx := strings.LastIndex(msg, ": ")
	if expIdx != -1 {
		return msg[:expIdx] + ": ...", msg[expIdx+2:]
	}
	return msg, ""
}

// shortenString cuts its argument on the right so that it more likely
// fits onto the developer's screen.  The behavior can be disabled by
// the command-line flag "-full-messages".
func shortenString(msg string) string {
	if *fullMessages {
		return msg
	}

	shortened := false

	nlPos := strings.IndexRune(msg, '\n')
	if nlPos >= 0 {
		shortened = true
		msg = msg[:nlPos]
	}

	if len(msg) > 80 {
		shortened = true
		msg = msg[:80]
	}

	if shortened {
		msg = msg + "..."
	}

	return msg
}

// signalSoftError registers a failing but ignored query.
func (t *logicTest) signalSoftError(err error, pos string, sql string) {
	if *errorSummary {
		errmsg, expected := splitLastMessagePart(shortenString(fmt.Sprintf("%s", err)))
		var buf bytes.Buffer
		fmt.Fprintf(&buf, "-- %s (%s)\n%s", pos, expected, shortenString(sql+";"))
		t.softSummary[errmsg] = append(t.softSummary[errmsg], buf.String())
	}
	t.unsupported++
}

// Errorf overloads testing.T.Errorf so that the test can fail early
// if -try-harder was not specified. It also handles printing the
// per-query "FAIL" marker when -show-sql is set.
func (t *logicTest) Errorf(format string, args ...interface{}) {
	if *showSQL {
		fmt.Println("\t-- FAIL")
	}
	t.T.Errorf(format, args...)
	if !*tryHarder {
		t.FailNow()
	}
	t.failures++
}

// Fatalf overloads testing.T.Fatalf to ensure the fatal error message
// is printed on its own line when -show-sql is set.
func (t *logicTest) Fatalf(format string, args ...interface{}) {
	if *showSQL {
		fmt.Println()
	}
	t.T.Fatalf(format, args...)
}

// finishOne marks completion of a single test. It handles
// printing the success marker then -show-sql is set.
func (t *logicTest) finishOne(msg string) {
	if *showSQL {
		fmt.Printf("\t-- %s;\n", msg)
	}
}

// printCompletion reports on the completion of all tests in a given
// input file.
func (t *logicTest) printCompletion(path string) {
	unsupportedMsg := ""
	if t.unsupported > 0 {
		unsupportedMsg = fmt.Sprintf(", ignored %d unsupported queries", t.unsupported)
	}
	fmt.Printf("--- done: %s: %d tests, %d failures%s\n", path, t.progress, t.failures,
		unsupportedMsg)
}