Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding random query generation for endtoend testing of the Gen4 planner #13260

Merged
merged 29 commits into from
Jun 29, 2023
Merged

Adding random query generation for endtoend testing of the Gen4 planner #13260

Show file tree
Hide file tree
Changes from 24 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
65c8ca5
group by queries failing with limit and distinct
arvind-murty Jun 7, 2023
f575d69
added some failing test cases and a toggle to generate known failing …
arvind-murty Jun 8, 2023
e0736a3
fix WaitForAuthoritative function call
arvind-murty Jun 9, 2023
ffe4df2
moved randomly generated buggy queries to aggregation
arvind-murty Jun 9, 2023
9bff9e0
added left/right joins
arvind-murty Jun 9, 2023
1f440f3
Refactor sqlparser/random_expr.go to be public
arvind-murty Jun 12, 2023
40298d5
added random expressions to random query generation
arvind-murty Jun 12, 2023
8e80022
fixed syntax error in random query generation group by
arvind-murty Jun 12, 2023
2ec528d
added column aliases
arvind-murty Jun 13, 2023
6fb6f4f
added derived tables
arvind-murty Jun 13, 2023
f093992
fixed random_expr_test formatting
arvind-murty Jun 13, 2023
f3f00c5
fixed infinite loop in predicate generation
arvind-murty Jun 14, 2023
7eb939d
added column aliases
arvind-murty Jun 16, 2023
3ce4f20
renamed TableT and Col methods
arvind-murty Jun 16, 2023
a59380e
reorder failing queries
arvind-murty Jun 16, 2023
34a46cd
refactor random query generation to use the ast
arvind-murty Jun 18, 2023
e925343
separated failures in must-fix and known failures
arvind-murty Jun 19, 2023
2a30aea
added two must-fix queries
arvind-murty Jun 19, 2023
5aabd0e
added having to random query generation
arvind-murty Jun 21, 2023
9de596c
removed one passing must-fix query
arvind-murty Jun 21, 2023
8b80cd9
created interface for random expression generation with a schema
arvind-murty Jun 22, 2023
4e07f06
added toggle to fail on EOF and mismatched results errors
arvind-murty Jun 23, 2023
9ef5861
deleted aggregation/fuzz_test.go and minor fixes/changes to random
arvind-murty Jun 23, 2023
656f638
limited query generation to make TestRandom consistently pass
arvind-murty Jun 26, 2023
8e4a1da
rename random/query_gen_test.go to query_gen.go
arvind-murty Jun 28, 2023
db14cd4
fixed reviewed comments
arvind-murty Jun 28, 2023
c1b1f59
added support for non-aggregate queries and a few more known failures
arvind-murty Jun 28, 2023
51c1431
updated tableT.name comment
arvind-murty Jun 29, 2023
455a7e0
added vtgate/queries/random to CI
arvind-murty Jun 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion go/test/endtoend/utils/cmp.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,8 @@ func (mcmp *MySQLCompare) ExecWithColumnCompare(query string) *sqltypes.Result {
// - MySQL and Vitess did not find an error, but their results are matching
//
// The result set and error produced by Vitess are returned to the caller.
// If the Vitess and MySQL error are both nil, but the results do not match,
// the mismatched results are instead returned as an error, as well as the Vitess result set
func (mcmp *MySQLCompare) ExecAllowAndCompareError(query string) (*sqltypes.Result, error) {
mcmp.t.Helper()
vtQr, vtErr := mcmp.VtConn.ExecuteFetch(query, 1000, true)
Expand All @@ -242,7 +244,7 @@ func (mcmp *MySQLCompare) ExecAllowAndCompareError(query string) (*sqltypes.Resu
// Since we allow errors, we don't want to compare results if one of the client failed.
// Vitess and MySQL should always be agreeing whether the query returns an error or not.
if vtErr == nil && mysqlErr == nil {
compareVitessAndMySQLResults(mcmp.t, query, mcmp.VtConn, vtQr, mysqlQr, false)
vtErr = compareVitessAndMySQLResults(mcmp.t, query, mcmp.VtConn, vtQr, mysqlQr, false)
}
return vtQr, vtErr
}
Expand Down
26 changes: 17 additions & 9 deletions go/test/endtoend/utils/mysql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package utils

import (
"context"
"errors"
"fmt"
"os"
"path"
Expand Down Expand Up @@ -154,24 +155,27 @@ func prepareMySQLWithSchema(params mysql.ConnParams, sql string) error {
return nil
}

func compareVitessAndMySQLResults(t *testing.T, query string, vtConn *mysql.Conn, vtQr, mysqlQr *sqltypes.Result, compareColumns bool) {
func compareVitessAndMySQLResults(t *testing.T, query string, vtConn *mysql.Conn, vtQr, mysqlQr *sqltypes.Result, compareColumns bool) error {
if vtQr == nil && mysqlQr == nil {
return
return nil
}
if vtQr == nil {
t.Error("Vitess result is 'nil' while MySQL's is not.")
return
return errors.New("Vitess result is 'nil' while MySQL's is not.\n")
}
if mysqlQr == nil {
t.Error("MySQL result is 'nil' while Vitess' is not.")
return
return errors.New("MySQL result is 'nil' while Vitess' is not.\n")
}

var errStr string
if compareColumns {
vtColCount := len(vtQr.Fields)
myColCount := len(mysqlQr.Fields)
if vtColCount > 0 && myColCount > 0 {
if vtColCount != myColCount {
t.Errorf("column count does not match: %d vs %d", vtColCount, myColCount)
errStr += fmt.Sprintf("column count does not match: %d vs %d\n", vtColCount, myColCount)
}

var vtCols []string
Expand All @@ -180,26 +184,29 @@ func compareVitessAndMySQLResults(t *testing.T, query string, vtConn *mysql.Conn
vtCols = append(vtCols, vtField.Name)
myCols = append(myCols, mysqlQr.Fields[i].Name)
}
assert.Equal(t, myCols, vtCols, "column names do not match - the expected values are what mysql produced")
if !assert.Equal(t, myCols, vtCols, "column names do not match - the expected values are what mysql produced") {
errStr += "column names do not match - the expected values are what mysql produced\n"
errStr += fmt.Sprintf("Not equal: \nexpected: %v\nactual: %v\n", myCols, vtCols)
}
}
}
stmt, err := sqlparser.Parse(query)
if err != nil {
t.Error(err)
return
return err
}
orderBy := false
if selStmt, isSelStmt := stmt.(sqlparser.SelectStatement); isSelStmt {
orderBy = selStmt.GetOrderBy() != nil
}

if orderBy && sqltypes.ResultsEqual([]sqltypes.Result{*vtQr}, []sqltypes.Result{*mysqlQr}) {
return
return nil
} else if sqltypes.ResultsEqualUnordered([]sqltypes.Result{*vtQr}, []sqltypes.Result{*mysqlQr}) {
return
return nil
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can combine this if/else if into a single if


errStr := "Query (" + query + ") results mismatched.\nVitess Results:\n"
errStr += "Query (" + query + ") results mismatched.\nVitess Results:\n"
for _, row := range vtQr.Rows {
errStr += fmt.Sprintf("%s\n", row)
}
Expand All @@ -212,6 +219,7 @@ func compareVitessAndMySQLResults(t *testing.T, query string, vtConn *mysql.Conn
errStr += fmt.Sprintf("query plan: \n%s\n", qr.Rows[0][0].ToString())
}
t.Error(errStr)
return errors.New(errStr)
}

func compareVitessAndMySQLErrors(t *testing.T, vtErr, mysqlErr error) {
Expand Down
48 changes: 48 additions & 0 deletions go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,54 @@ func TestBuggyQueries(t *testing.T) {

mcmp.Exec("select /*vt+ PLANNER=gen4 */sum(tbl1.a), min(tbl0.b) from t10 as tbl0, t10 as tbl1 left join t10 as tbl2 on tbl1.a = tbl2.a and tbl1.b = tbl2.k")
mcmp.Exec("select /*vt+ PLANNER=gen4 */count(*) from t10 left join t10 as t11 on t10.a = t11.b where t11.a")

// from random/random_test.go
mcmp.Exec("INSERT INTO emp(empno, ename, job, mgr, hiredate, sal, comm, deptno) VALUES (7369,'SMITH','CLERK',7902,'1980-12-17',800,NULL,20), (7499,'ALLEN','SALESMAN',7698,'1981-02-20',1600,300,30), (7521,'WARD','SALESMAN',7698,'1981-02-22',1250,500,30), (7566,'JONES','MANAGER',7839,'1981-04-02',2975,NULL,20), (7654,'MARTIN','SALESMAN',7698,'1981-09-28',1250,1400,30), (7698,'BLAKE','MANAGER',7839,'1981-05-01',2850,NULL,30), (7782,'CLARK','MANAGER',7839,'1981-06-09',2450,NULL,10), (7788,'SCOTT','ANALYST',7566,'1982-12-09',3000,NULL,20), (7839,'KING','PRESIDENT',NULL,'1981-11-17',5000,NULL,10), (7844,'TURNER','SALESMAN',7698,'1981-09-08',1500,0,30), (7876,'ADAMS','CLERK',7788,'1983-01-12',1100,NULL,20), (7900,'JAMES','CLERK',7698,'1981-12-03',950,NULL,30), (7902,'FORD','ANALYST',7566,'1981-12-03',3000,NULL,20), (7934,'MILLER','CLERK',7782,'1982-01-23',1300,NULL,10)")
mcmp.Exec("INSERT INTO dept(deptno, dname, loc) VALUES ('10','ACCOUNTING','NEW YORK'), ('20','RESEARCH','DALLAS'), ('30','SALES','CHICAGO'), ('40','OPERATIONS','BOSTON')")

mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ count(*), count(*), count(*) from dept as tbl0, emp as tbl1 where tbl0.deptno = tbl1.deptno group by tbl1.empno order by tbl1.empno",
`[[INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)] [INT64(1) INT64(1) INT64(1)]]`)
//mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ count(tbl0.deptno) from dept as tbl0, emp as tbl1 group by tbl1.job order by tbl1.job limit 3",
// `[[INT64(8)] [INT64(16)] [INT64(12)]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ count(*), count(*) from emp as tbl0 group by tbl0.empno order by tbl0.empno",
`[[INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)] [INT64(1) INT64(1)]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ distinct count(*), tbl0.loc from dept as tbl0 group by tbl0.loc",
`[[INT64(1) VARCHAR("BOSTON")] [INT64(1) VARCHAR("CHICAGO")] [INT64(1) VARCHAR("DALLAS")] [INT64(1) VARCHAR("NEW YORK")]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ distinct count(*) from dept as tbl0 group by tbl0.loc",
`[[INT64(1)]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ sum(tbl1.comm) from emp as tbl0, emp as tbl1",
`[[DECIMAL(30800)]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ tbl1.mgr, tbl1.mgr, count(*) from emp as tbl1 group by tbl1.mgr",
`[[NULL NULL INT64(1)] [INT64(7566) INT64(7566) INT64(2)] [INT64(7698) INT64(7698) INT64(5)] [INT64(7782) INT64(7782) INT64(1)] [INT64(7788) INT64(7788) INT64(1)] [INT64(7839) INT64(7839) INT64(3)] [INT64(7902) INT64(7902) INT64(1)]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ tbl1.mgr, tbl1.mgr, count(*) from emp as tbl0, emp as tbl1 group by tbl1.mgr",
`[[NULL NULL INT64(14)] [INT64(7566) INT64(7566) INT64(28)] [INT64(7698) INT64(7698) INT64(70)] [INT64(7782) INT64(7782) INT64(14)] [INT64(7788) INT64(7788) INT64(14)] [INT64(7839) INT64(7839) INT64(42)] [INT64(7902) INT64(7902) INT64(14)]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ count(*), count(*), count(tbl0.comm) from emp as tbl0, emp as tbl1 join dept as tbl2",
`[[INT64(784) INT64(784) INT64(224)]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ count(*), count(*) from (select count(*) from dept as tbl0 group by tbl0.deptno) as tbl0, dept as tbl1",
`[[INT64(16) INT64(16)]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ count(*) from (select count(*) from dept as tbl0 group by tbl0.deptno) as tbl0",
`[[INT64(4)]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ min(tbl0.loc) from dept as tbl0",
`[[VARCHAR("BOSTON")]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ tbl1.empno, max(tbl1.job) from dept as tbl0, emp as tbl1 group by tbl1.empno",
`[[INT64(7369) VARCHAR("CLERK")] [INT64(7499) VARCHAR("SALESMAN")] [INT64(7521) VARCHAR("SALESMAN")] [INT64(7566) VARCHAR("MANAGER")] [INT64(7654) VARCHAR("SALESMAN")] [INT64(7698) VARCHAR("MANAGER")] [INT64(7782) VARCHAR("MANAGER")] [INT64(7788) VARCHAR("ANALYST")] [INT64(7839) VARCHAR("PRESIDENT")] [INT64(7844) VARCHAR("SALESMAN")] [INT64(7876) VARCHAR("CLERK")] [INT64(7900) VARCHAR("CLERK")] [INT64(7902) VARCHAR("ANALYST")] [INT64(7934) VARCHAR("CLERK")]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ tbl1.ename, max(tbl0.comm) from emp as tbl0, emp as tbl1 group by tbl1.ename",
`[[VARCHAR("ADAMS") INT64(1400)] [VARCHAR("ALLEN") INT64(1400)] [VARCHAR("BLAKE") INT64(1400)] [VARCHAR("CLARK") INT64(1400)] [VARCHAR("FORD") INT64(1400)] [VARCHAR("JAMES") INT64(1400)] [VARCHAR("JONES") INT64(1400)] [VARCHAR("KING") INT64(1400)] [VARCHAR("MARTIN") INT64(1400)] [VARCHAR("MILLER") INT64(1400)] [VARCHAR("SCOTT") INT64(1400)] [VARCHAR("SMITH") INT64(1400)] [VARCHAR("TURNER") INT64(1400)] [VARCHAR("WARD") INT64(1400)]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ tbl0.dname, tbl0.dname, min(tbl0.deptno) from dept as tbl0, dept as tbl1 group by tbl0.dname, tbl0.dname",
`[[VARCHAR("ACCOUNTING") VARCHAR("ACCOUNTING") INT64(10)] [VARCHAR("OPERATIONS") VARCHAR("OPERATIONS") INT64(40)] [VARCHAR("RESEARCH") VARCHAR("RESEARCH") INT64(20)] [VARCHAR("SALES") VARCHAR("SALES") INT64(30)]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ tbl0.dname, min(tbl1.deptno) from dept as tbl0, dept as tbl1 group by tbl0.dname, tbl1.dname",
`[[VARCHAR("ACCOUNTING") INT64(10)] [VARCHAR("ACCOUNTING") INT64(40)] [VARCHAR("ACCOUNTING") INT64(20)] [VARCHAR("ACCOUNTING") INT64(30)] [VARCHAR("OPERATIONS") INT64(10)] [VARCHAR("OPERATIONS") INT64(40)] [VARCHAR("OPERATIONS") INT64(20)] [VARCHAR("OPERATIONS") INT64(30)] [VARCHAR("RESEARCH") INT64(10)] [VARCHAR("RESEARCH") INT64(40)] [VARCHAR("RESEARCH") INT64(20)] [VARCHAR("RESEARCH") INT64(30)] [VARCHAR("SALES") INT64(10)] [VARCHAR("SALES") INT64(40)] [VARCHAR("SALES") INT64(20)] [VARCHAR("SALES") INT64(30)]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ max(tbl0.hiredate) from emp as tbl0",
`[[DATE("1983-01-12")]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ min(tbl0.deptno) as caggr0, count(*) as caggr1 from dept as tbl0 left join dept as tbl1 on tbl1.loc = tbl1.dname",
`[[INT64(10) INT64(4)]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ count(tbl1.loc) as caggr0 from dept as tbl1 left join dept as tbl2 on tbl1.loc = tbl2.loc where (tbl2.deptno)",
`[[INT64(4)]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ sum(tbl1.ename), min(tbl0.empno) from emp as tbl0, emp as tbl1 left join dept as tbl2 on tbl1.job = tbl2.loc and tbl1.comm = tbl2.deptno where ('trout') and tbl0.deptno = tbl1.comm",
`[[NULL NULL]]`)
mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ distinct max(tbl0.deptno), count(tbl0.job) from emp as tbl0, dept as tbl1 left join dept as tbl2 on tbl1.dname = tbl2.loc and tbl1.dname = tbl2.loc where (tbl2.loc) and tbl0.deptno = tbl1.deptno",
`[[NULL INT64(0)]]`)

}

func TestMinMaxAcrossJoins(t *testing.T) {
Expand Down
213 changes: 0 additions & 213 deletions go/test/endtoend/vtgate/queries/aggregation/fuzz_test.go
View file
Open in desktop

This file was deleted.

Loading