Skip to content

Commit

Permalink
sctest: Add a test that runs comparator testing from logictest stmts
Browse files Browse the repository at this point in the history
This commit introduced a new test that takes as input a path to a corpus
file (of stmts collected from logic tests) and feed them into the
comparator testing framework.

It also edit the existing nightly such that we now collect the corpus
and immediately feed them into the comparator testing framework, without
having to store them in the cloud in between.

Release note: None
  • Loading branch information
Xiang-Gu committed Oct 5, 2023
1 parent 617ef0a commit f26c1bf
Showing 5 changed files with 74 additions and 25 deletions.
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
source "$dir/teamcity-support.sh" # For $root
source "$dir/teamcity-bazel-support.sh" # For run_bazel

tc_start_block "Collect SQL Logic Tests Statements"
tc_start_block "Schema Changer Comparator Testing"
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILD_BRANCH -e GITHUB_API_TOKEN -e GOOGLE_EPHEMERAL_CREDENTIALS -e BUILD_VCS_NUMBER -e TC_BUILD_ID -e TC_SERVER_URL -e TC_BUILDTYPE_ID -e GITHUB_REPO" \
run_bazel build/teamcity/cockroach/nightlies/sqllogic_statements_corpus_nightly_impl.sh
tc_end_block "Collect SQL Logic Tests Statements"
run_bazel build/teamcity/cockroach/nightlies/schema_changer_comparator_nightly_impl.sh
tc_end_block "Schema Changer Comparator Testing"
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash

set -xeuo pipefail

dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
source "$dir/teamcity-support.sh"

CORPUS_DIR=/artifacts/logictest-stmts-corpus-dir # dir to store all collected corpus file(s)
exit_status=0

# Collect sql logic tests statements corpus.
bazel run -- //pkg/cmd/generate-logictest-corpus:generate-logictest-corpus \
-out-dir=$CORPUS_DIR \
|| exit_status=$?

bazel build //pkg/cmd/bazci --config=ci
BAZEL_BIN=$(bazel info bazel-bin --config=ci)

# Run schema changer comparator test with statements from the collected corpus file(s).
for CORPUS_FILE in "$CORPUS_DIR"/*
do
$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci test -- --config=ci \
//pkg/sql/schemachanger:schemachanger_test \
--test_arg=--logictest-stmt-corpus-path="$CORPUS_FILE" \
--test_filter='^TestComparatorFromLogicTests$' \
--test_env=GO_TEST_WRAP_TESTV=1 \
--test_env=GO_TEST_WRAP=1 \
--test_timeout=7200 \
|| exit_status=$?
done

exit $exit_status

This file was deleted.

1 change: 1 addition & 0 deletions pkg/sql/schemachanger/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -54,6 +54,7 @@ go_test(
"//pkg/sql/rowenc",
"//pkg/sql/schemachanger/scexec",
"//pkg/sql/schemachanger/scop",
"//pkg/sql/schemachanger/scpb",
"//pkg/sql/schemachanger/scplan",
"//pkg/sql/schemachanger/sctest", # keep
"//pkg/sql/sessiondatapb",
38 changes: 38 additions & 0 deletions pkg/sql/schemachanger/schemachanger_test.go
Original file line number Diff line number Diff line change
@@ -13,8 +13,10 @@ package schemachanger_test
import (
"context"
"encoding/hex"
"flag"
"fmt"
"math/rand"
"os"
"regexp"
"strings"
"sync"
@@ -36,6 +38,7 @@ import (
"github.com/cockroachdb/cockroach/pkg/sql/rowenc"
"github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scexec"
"github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scop"
"github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scpb"
"github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scplan"
"github.com/cockroachdb/cockroach/pkg/sql/schemachanger/sctest"
"github.com/cockroachdb/cockroach/pkg/sql/sessiondatapb"
@@ -885,3 +888,38 @@ func TestCompareLegacyAndDeclarative(t *testing.T) {

sctest.CompareLegacyAndDeclarative(t, ss)
}

var logictestStmtsCorpusFile = flag.String("logictest-stmt-corpus-path", "", "path to logictest stmts corpus")

func TestComparatorFromLogicTests(t *testing.T) {
defer leaktest.AfterTest(t)()
defer log.Scope(t).Close(t)
if *logictestStmtsCorpusFile == "" {
skip.IgnoreLint(t, "require `--logictest-stmt-corpus-path` to be set")
}

bytes, err := os.ReadFile(*logictestStmtsCorpusFile)
require.NoError(t, err)
corpus := scpb.LogicTestStmtsCorpus{}
err = protoutil.Unmarshal(bytes, &corpus)
require.NoError(t, err)

// Shuffle entries so the order of execution is different each time. It might
// speed up finding bugs.
rand.Shuffle(len(corpus.Entries), func(i, j int) {
corpus.Entries[i], corpus.Entries[j] = corpus.Entries[j], corpus.Entries[i]
})

for _, entry := range corpus.Entries {
subtestName := entry.Name
subtestStatements := entry.Statements
t.Run(entry.Name, func(t *testing.T) {
t.Logf("running schema changer comparator testing on statements collected from logic test %q\n", subtestName)
ss := &staticSQLStmtLineProvider{
stmts: subtestStatements,
}
sctest.CompareLegacyAndDeclarative(t, ss)
t.Logf("schema changer comparator testing succeeded on logic test %q\n", subtestName)
})
}
}

0 comments on commit f26c1bf

Please sign in to comment.