From fe57780cd3c7ac85be39cfd5bd9f8c796f33c50f Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Sat, 21 Sep 2024 10:04:33 +0900 Subject: [PATCH 01/35] Implement CREATE/DROP/ALTER SEARCH INDEX --- ast/ast.go | 50 ++++++++- parser.go | 103 ++++++++++++++++++ .../alter_search_index_add_stored_column.sql | 1 + .../alter_search_index_drop_stored_column.sql | 1 + .../ddl/create_search_index_null_filtered.sql | 4 + .../input/ddl/create_search_index_simple.sql | 3 + testdata/input/ddl/drop_search_index.sql | 1 + ...ter_search_index_add_stored_column.sql.txt | 22 ++++ ...er_search_index_drop_stored_column.sql.txt | 22 ++++ .../create_search_index_null_filtered.sql.txt | 57 ++++++++++ .../ddl/create_search_index_simple.sql.txt | 35 ++++++ testdata/result/ddl/drop_search_index.sql.txt | 15 +++ ...ter_search_index_add_stored_column.sql.txt | 22 ++++ ...er_search_index_drop_stored_column.sql.txt | 22 ++++ .../create_search_index_null_filtered.sql.txt | 57 ++++++++++ .../create_search_index_simple.sql.txt | 35 ++++++ .../statement/drop_search_index.sql.txt | 15 +++ 17 files changed, 464 insertions(+), 1 deletion(-) create mode 100644 testdata/input/ddl/alter_search_index_add_stored_column.sql create mode 100644 testdata/input/ddl/alter_search_index_drop_stored_column.sql create mode 100644 testdata/input/ddl/create_search_index_null_filtered.sql create mode 100644 testdata/input/ddl/create_search_index_simple.sql create mode 100644 testdata/input/ddl/drop_search_index.sql create mode 100644 testdata/result/ddl/alter_search_index_add_stored_column.sql.txt create mode 100644 testdata/result/ddl/alter_search_index_drop_stored_column.sql.txt create mode 100644 testdata/result/ddl/create_search_index_null_filtered.sql.txt create mode 100644 testdata/result/ddl/create_search_index_simple.sql.txt create mode 100644 testdata/result/ddl/drop_search_index.sql.txt create mode 100644 testdata/result/statement/alter_search_index_add_stored_column.sql.txt create mode 100644 testdata/result/statement/alter_search_index_drop_stored_column.sql.txt create mode 100644 testdata/result/statement/create_search_index_null_filtered.sql.txt create mode 100644 testdata/result/statement/create_search_index_simple.sql.txt create mode 100644 testdata/result/statement/drop_search_index.sql.txt diff --git a/ast/ast.go b/ast/ast.go index 56e8bf1a..7bc16bef 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -934,7 +934,7 @@ type SelectorExpr struct { // IndexExpr is array item access expression node. // -// {{.Expr | sql}}[{{if .Ordinal}}ORDINAL{{else}}OFFSET{{end}}({{.Index | sql}})] +// {{.Expr | sql}}[{{if .Ordinal}}ORDINAL{{else}}OFFSET{{end}}({{.Name | sql}})] type IndexExpr struct { // pos = Expr.pos // end = Rbrack + 1 @@ -2469,3 +2469,51 @@ type SequenceOptions struct { Records []*SequenceOption // len(Records) > 0 } + +// GenericOptions is generic OPTIONS clause node without key and value checking. +// +// OPTIONS ({{.Records | sqlJoin ","}}) +type GenericOptions struct { + // pos = Options + // end = Rparen + 1 + + Options token.Pos // position of "OPTIONS" keyword + Rparen token.Pos // position of ")" + + Records []*GenericOption // len(Records) > 0 +} + +func (g *GenericOptions) Pos() token.Pos { + return g.Options +} + +func (g *GenericOptions) End() token.Pos { + return g.Rparen + 1 +} + +func (g *GenericOptions) SQL() string { + return "OPTIONS " + sqlJoin(g.Records, ", ") +} + +// GenericOption is generic option for CREATE statements. +// +// {{.Name | sql}} = {{.Value | sql}} +type GenericOption struct { + // pos = Name.pos + // end = Value.end + + Name *Ident + Value Expr +} + +func (g *GenericOption) Pos() token.Pos { + return g.Name.Pos() +} + +func (g *GenericOption) End() token.Pos { + return g.Value.End() +} + +func (g *GenericOption) SQL() string { + return g.Name.SQL() + " = " + g.Value.SQL() +} diff --git a/parser.go b/parser.go index 2252041b..37969b37 100644 --- a/parser.go +++ b/parser.go @@ -2066,6 +2066,8 @@ func (p *Parser) parseDDL() ast.DDL { return p.parseCreateIndex(pos) case p.Token.IsKeywordLike("VECTOR"): return p.parseCreateVectorIndex(pos) + case p.Token.IsKeywordLike("SEARCH"): + return p.parseCreateSearchIndex(pos) case p.Token.IsKeywordLike("ROLE"): return p.parseCreateRole(pos) case p.Token.IsKeywordLike("CHANGE"): @@ -2079,6 +2081,8 @@ func (p *Parser) parseDDL() ast.DDL { return p.parseAlterTable(pos) case p.Token.IsKeywordLike("INDEX"): return p.parseAlterIndex(pos) + case p.Token.IsKeywordLike("Search"): + return p.parseAlterSearchIndex(pos) case p.Token.IsKeywordLike("SEQUENCE"): return p.parseAlterSequence(pos) case p.Token.IsKeywordLike("CHANGE"): @@ -2094,6 +2098,8 @@ func (p *Parser) parseDDL() ast.DDL { return p.parseDropIndex(pos) case p.Token.IsKeywordLike("VECTOR"): return p.parseDropVectorIndex(pos) + case p.Token.IsKeywordLike("SEARCH"): + return p.parseDropSearchIndex(pos) case p.Token.IsKeywordLike("SEQUENCE"): return p.parseDropSequence(pos) case p.Token.IsKeywordLike("ROLE"): @@ -2595,6 +2601,103 @@ func (p *Parser) parseCreateVectorIndex(pos token.Pos) *ast.CreateVectorIndex { } } +func (p *Parser) parseCreateSearchIndex(pos token.Pos) *ast.CreateSearchIndex { + p.expectKeywordLike("SEARCH") + p.expectKeywordLike("INDEX") + name := p.parseIdent() + p.expect("ON") + tableName := p.parseIdent() + p.expect("(") + columnName := parseSeparatedList(p, ",", p.parseIdent) + rparen := p.expect(")").Pos + + storing := p.tryParseStoring() + + var partitionColumns []*ast.Ident + if p.tryExpect("PARTITION") != nil { + p.expect("BY") + partitionColumns = parseSeparatedList(p, ",", p.parseIdent) + } + orderBy := p.tryParseOrderBy() + where := p.tryParseWhere() + interleave := p.tryParseInterleaveIn() + options := p.tryParseGenericOptions() + + return &ast.CreateSearchIndex{ + Create: pos, + Name: name, + TableName: tableName, + TokenListPart: columnName, + Rparen: rparen, + Storing: storing, + PartitionColumns: partitionColumns, + OrderBy: orderBy, + Where: where, + Interleave: interleave, + Options: options, + } +} + +func (p *Parser) tryParseGenericOptions() *ast.GenericOptions { + options := p.tryExpectKeywordLike("OPTIONS") + if options == nil { + return nil + } + p.expect("(") + records := parseSeparatedList(p, ",", func() *ast.GenericOption { + name := p.parseIdent() + p.expect("=") + value := p.parseExpr() + return &ast.GenericOption{ + Name: name, + Value: value, + } + }) + rparen := p.expect(")").Pos + + return &ast.GenericOptions{ + Options: options.Pos, + Rparen: rparen, + Records: records, + } +} + +func (p *Parser) parseDropSearchIndex(pos token.Pos) *ast.DropSearchIndex { + p.expectKeywordLike("SEARCH") + p.expectKeywordLike("INDEX") + ifExists := p.parseIfExists() + name := p.parseIdent() + return &ast.DropSearchIndex{ + Drop: pos, + IfExists: ifExists, + Name: name, + } + +} + +func (p *Parser) parseAlterSearchIndex(pos token.Pos) *ast.AlterSearchIndex { + p.expectKeywordLike("SEARCH") + p.expectKeywordLike("INDEX") + + name := p.parseIdent() + + var alteration ast.IndexAlteration + switch { + case p.Token.IsKeywordLike("ADD"): + alteration = p.parseAddStoredColumn() + case p.Token.IsKeywordLike("DROP"): + alteration = p.parseDropStoredColumn() + default: + p.panicfAtToken(&p.Token, "expected pseudo keyword: ADD, DROP, but: %s", p.Token.AsString) + } + + return &ast.AlterSearchIndex{ + Alter: pos, + Name: name, + IndexAlteration: alteration, + } +} + func (p *Parser) parseCreateIndex(pos token.Pos) *ast.CreateIndex { unique := false if p.Token.IsKeywordLike("UNIQUE") { diff --git a/testdata/input/ddl/alter_search_index_add_stored_column.sql b/testdata/input/ddl/alter_search_index_add_stored_column.sql new file mode 100644 index 00000000..680bb9d9 --- /dev/null +++ b/testdata/input/ddl/alter_search_index_add_stored_column.sql @@ -0,0 +1 @@ +ALTER SEARCH INDEX AlbumsIndex ADD STORED COLUMN Genre \ No newline at end of file diff --git a/testdata/input/ddl/alter_search_index_drop_stored_column.sql b/testdata/input/ddl/alter_search_index_drop_stored_column.sql new file mode 100644 index 00000000..a9dcd601 --- /dev/null +++ b/testdata/input/ddl/alter_search_index_drop_stored_column.sql @@ -0,0 +1 @@ +ALTER SEARCH INDEX AlbumsIndex DROP STORED COLUMN Genre \ No newline at end of file diff --git a/testdata/input/ddl/create_search_index_null_filtered.sql b/testdata/input/ddl/create_search_index_null_filtered.sql new file mode 100644 index 00000000..68ad3f9c --- /dev/null +++ b/testdata/input/ddl/create_search_index_null_filtered.sql @@ -0,0 +1,4 @@ +CREATE SEARCH INDEX AlbumsIndex +ON Albums(AlbumTitle_Tokens) +STORING(Genre) +WHERE Genre IS NOT NULL \ No newline at end of file diff --git a/testdata/input/ddl/create_search_index_simple.sql b/testdata/input/ddl/create_search_index_simple.sql new file mode 100644 index 00000000..6414cc26 --- /dev/null +++ b/testdata/input/ddl/create_search_index_simple.sql @@ -0,0 +1,3 @@ +-- no optional clauses +CREATE SEARCH INDEX AlbumsIndex + ON Albums(AlbumTitle_Tokens) \ No newline at end of file diff --git a/testdata/input/ddl/drop_search_index.sql b/testdata/input/ddl/drop_search_index.sql new file mode 100644 index 00000000..101ff3fc --- /dev/null +++ b/testdata/input/ddl/drop_search_index.sql @@ -0,0 +1 @@ +DROP SEARCH INDEX IF EXISTS AlbumsIndex \ No newline at end of file diff --git a/testdata/result/ddl/alter_search_index_add_stored_column.sql.txt b/testdata/result/ddl/alter_search_index_add_stored_column.sql.txt new file mode 100644 index 00000000..7f13c100 --- /dev/null +++ b/testdata/result/ddl/alter_search_index_add_stored_column.sql.txt @@ -0,0 +1,22 @@ +--- alter_search_index_add_stored_column.sql +ALTER SEARCH INDEX AlbumsIndex ADD STORED COLUMN Genre +--- AST +&ast.AlterSearchIndex{ + Alter: 0, + Name: &ast.Ident{ + NamePos: 19, + NameEnd: 30, + Name: "AlbumsIndex", + }, + IndexAlteration: &ast.AddStoredColumn{ + Add: 31, + Name: &ast.Ident{ + NamePos: 49, + NameEnd: 54, + Name: "Genre", + }, + }, +} + +--- SQL +ALTER INDEX AlbumsIndex ADD STORED COLUMN Genre diff --git a/testdata/result/ddl/alter_search_index_drop_stored_column.sql.txt b/testdata/result/ddl/alter_search_index_drop_stored_column.sql.txt new file mode 100644 index 00000000..6a39d7d2 --- /dev/null +++ b/testdata/result/ddl/alter_search_index_drop_stored_column.sql.txt @@ -0,0 +1,22 @@ +--- alter_search_index_drop_stored_column.sql +ALTER SEARCH INDEX AlbumsIndex DROP STORED COLUMN Genre +--- AST +&ast.AlterSearchIndex{ + Alter: 0, + Name: &ast.Ident{ + NamePos: 19, + NameEnd: 30, + Name: "AlbumsIndex", + }, + IndexAlteration: &ast.DropStoredColumn{ + Drop: 31, + Name: &ast.Ident{ + NamePos: 50, + NameEnd: 55, + Name: "Genre", + }, + }, +} + +--- SQL +ALTER INDEX AlbumsIndex DROP STORED COLUMN Genre diff --git a/testdata/result/ddl/create_search_index_null_filtered.sql.txt b/testdata/result/ddl/create_search_index_null_filtered.sql.txt new file mode 100644 index 00000000..56e116ce --- /dev/null +++ b/testdata/result/ddl/create_search_index_null_filtered.sql.txt @@ -0,0 +1,57 @@ +--- create_search_index_null_filtered.sql +CREATE SEARCH INDEX AlbumsIndex +ON Albums(AlbumTitle_Tokens) +STORING(Genre) +WHERE Genre IS NOT NULL +--- AST +&ast.CreateSearchIndex{ + Create: 0, + Name: &ast.Ident{ + NamePos: 20, + NameEnd: 31, + Name: "AlbumsIndex", + }, + TableName: &ast.Ident{ + NamePos: 35, + NameEnd: 41, + Name: "Albums", + }, + TokenListPart: []*ast.Ident{ + &ast.Ident{ + NamePos: 42, + NameEnd: 59, + Name: "AlbumTitle_Tokens", + }, + }, + Rparen: 59, + Storing: &ast.Storing{ + Storing: 61, + Rparen: 74, + Columns: []*ast.Ident{ + &ast.Ident{ + NamePos: 69, + NameEnd: 74, + Name: "Genre", + }, + }, + }, + PartitionColumns: []*ast.Ident(nil), + OrderBy: (*ast.OrderBy)(nil), + Where: &ast.Where{ + Where: 76, + Expr: &ast.IsNullExpr{ + Null: 95, + Not: true, + Left: &ast.Ident{ + NamePos: 82, + NameEnd: 87, + Name: "Genre", + }, + }, + }, + Interleave: (*ast.InterleaveIn)(nil), + Options: (*ast.GenericOptions)(nil), +} + +--- SQL +CREATE SEARCH INDEX AlbumsIndex ON Albums(AlbumTitle_Tokens) STORING (Genre) WHERE Genre IS NOT NULL diff --git a/testdata/result/ddl/create_search_index_simple.sql.txt b/testdata/result/ddl/create_search_index_simple.sql.txt new file mode 100644 index 00000000..5020feb6 --- /dev/null +++ b/testdata/result/ddl/create_search_index_simple.sql.txt @@ -0,0 +1,35 @@ +--- create_search_index_simple.sql +-- no optional clauses +CREATE SEARCH INDEX AlbumsIndex + ON Albums(AlbumTitle_Tokens) +--- AST +&ast.CreateSearchIndex{ + Create: 23, + Name: &ast.Ident{ + NamePos: 43, + NameEnd: 54, + Name: "AlbumsIndex", + }, + TableName: &ast.Ident{ + NamePos: 60, + NameEnd: 66, + Name: "Albums", + }, + TokenListPart: []*ast.Ident{ + &ast.Ident{ + NamePos: 67, + NameEnd: 84, + Name: "AlbumTitle_Tokens", + }, + }, + Rparen: 84, + Storing: (*ast.Storing)(nil), + PartitionColumns: []*ast.Ident(nil), + OrderBy: (*ast.OrderBy)(nil), + Where: (*ast.Where)(nil), + Interleave: (*ast.InterleaveIn)(nil), + Options: (*ast.GenericOptions)(nil), +} + +--- SQL +CREATE SEARCH INDEX AlbumsIndex ON Albums(AlbumTitle_Tokens) diff --git a/testdata/result/ddl/drop_search_index.sql.txt b/testdata/result/ddl/drop_search_index.sql.txt new file mode 100644 index 00000000..fe6d374c --- /dev/null +++ b/testdata/result/ddl/drop_search_index.sql.txt @@ -0,0 +1,15 @@ +--- drop_search_index.sql +DROP SEARCH INDEX IF EXISTS AlbumsIndex +--- AST +&ast.DropSearchIndex{ + Drop: 0, + IfExists: true, + Name: &ast.Ident{ + NamePos: 28, + NameEnd: 39, + Name: "AlbumsIndex", + }, +} + +--- SQL +DROP SEARCH INDEX IF EXISTS AlbumsIndex diff --git a/testdata/result/statement/alter_search_index_add_stored_column.sql.txt b/testdata/result/statement/alter_search_index_add_stored_column.sql.txt new file mode 100644 index 00000000..7f13c100 --- /dev/null +++ b/testdata/result/statement/alter_search_index_add_stored_column.sql.txt @@ -0,0 +1,22 @@ +--- alter_search_index_add_stored_column.sql +ALTER SEARCH INDEX AlbumsIndex ADD STORED COLUMN Genre +--- AST +&ast.AlterSearchIndex{ + Alter: 0, + Name: &ast.Ident{ + NamePos: 19, + NameEnd: 30, + Name: "AlbumsIndex", + }, + IndexAlteration: &ast.AddStoredColumn{ + Add: 31, + Name: &ast.Ident{ + NamePos: 49, + NameEnd: 54, + Name: "Genre", + }, + }, +} + +--- SQL +ALTER INDEX AlbumsIndex ADD STORED COLUMN Genre diff --git a/testdata/result/statement/alter_search_index_drop_stored_column.sql.txt b/testdata/result/statement/alter_search_index_drop_stored_column.sql.txt new file mode 100644 index 00000000..6a39d7d2 --- /dev/null +++ b/testdata/result/statement/alter_search_index_drop_stored_column.sql.txt @@ -0,0 +1,22 @@ +--- alter_search_index_drop_stored_column.sql +ALTER SEARCH INDEX AlbumsIndex DROP STORED COLUMN Genre +--- AST +&ast.AlterSearchIndex{ + Alter: 0, + Name: &ast.Ident{ + NamePos: 19, + NameEnd: 30, + Name: "AlbumsIndex", + }, + IndexAlteration: &ast.DropStoredColumn{ + Drop: 31, + Name: &ast.Ident{ + NamePos: 50, + NameEnd: 55, + Name: "Genre", + }, + }, +} + +--- SQL +ALTER INDEX AlbumsIndex DROP STORED COLUMN Genre diff --git a/testdata/result/statement/create_search_index_null_filtered.sql.txt b/testdata/result/statement/create_search_index_null_filtered.sql.txt new file mode 100644 index 00000000..56e116ce --- /dev/null +++ b/testdata/result/statement/create_search_index_null_filtered.sql.txt @@ -0,0 +1,57 @@ +--- create_search_index_null_filtered.sql +CREATE SEARCH INDEX AlbumsIndex +ON Albums(AlbumTitle_Tokens) +STORING(Genre) +WHERE Genre IS NOT NULL +--- AST +&ast.CreateSearchIndex{ + Create: 0, + Name: &ast.Ident{ + NamePos: 20, + NameEnd: 31, + Name: "AlbumsIndex", + }, + TableName: &ast.Ident{ + NamePos: 35, + NameEnd: 41, + Name: "Albums", + }, + TokenListPart: []*ast.Ident{ + &ast.Ident{ + NamePos: 42, + NameEnd: 59, + Name: "AlbumTitle_Tokens", + }, + }, + Rparen: 59, + Storing: &ast.Storing{ + Storing: 61, + Rparen: 74, + Columns: []*ast.Ident{ + &ast.Ident{ + NamePos: 69, + NameEnd: 74, + Name: "Genre", + }, + }, + }, + PartitionColumns: []*ast.Ident(nil), + OrderBy: (*ast.OrderBy)(nil), + Where: &ast.Where{ + Where: 76, + Expr: &ast.IsNullExpr{ + Null: 95, + Not: true, + Left: &ast.Ident{ + NamePos: 82, + NameEnd: 87, + Name: "Genre", + }, + }, + }, + Interleave: (*ast.InterleaveIn)(nil), + Options: (*ast.GenericOptions)(nil), +} + +--- SQL +CREATE SEARCH INDEX AlbumsIndex ON Albums(AlbumTitle_Tokens) STORING (Genre) WHERE Genre IS NOT NULL diff --git a/testdata/result/statement/create_search_index_simple.sql.txt b/testdata/result/statement/create_search_index_simple.sql.txt new file mode 100644 index 00000000..5020feb6 --- /dev/null +++ b/testdata/result/statement/create_search_index_simple.sql.txt @@ -0,0 +1,35 @@ +--- create_search_index_simple.sql +-- no optional clauses +CREATE SEARCH INDEX AlbumsIndex + ON Albums(AlbumTitle_Tokens) +--- AST +&ast.CreateSearchIndex{ + Create: 23, + Name: &ast.Ident{ + NamePos: 43, + NameEnd: 54, + Name: "AlbumsIndex", + }, + TableName: &ast.Ident{ + NamePos: 60, + NameEnd: 66, + Name: "Albums", + }, + TokenListPart: []*ast.Ident{ + &ast.Ident{ + NamePos: 67, + NameEnd: 84, + Name: "AlbumTitle_Tokens", + }, + }, + Rparen: 84, + Storing: (*ast.Storing)(nil), + PartitionColumns: []*ast.Ident(nil), + OrderBy: (*ast.OrderBy)(nil), + Where: (*ast.Where)(nil), + Interleave: (*ast.InterleaveIn)(nil), + Options: (*ast.GenericOptions)(nil), +} + +--- SQL +CREATE SEARCH INDEX AlbumsIndex ON Albums(AlbumTitle_Tokens) diff --git a/testdata/result/statement/drop_search_index.sql.txt b/testdata/result/statement/drop_search_index.sql.txt new file mode 100644 index 00000000..fe6d374c --- /dev/null +++ b/testdata/result/statement/drop_search_index.sql.txt @@ -0,0 +1,15 @@ +--- drop_search_index.sql +DROP SEARCH INDEX IF EXISTS AlbumsIndex +--- AST +&ast.DropSearchIndex{ + Drop: 0, + IfExists: true, + Name: &ast.Ident{ + NamePos: 28, + NameEnd: 39, + Name: "AlbumsIndex", + }, +} + +--- SQL +DROP SEARCH INDEX IF EXISTS AlbumsIndex From 17228a12136c3f2520817c118cc796e17b40da8e Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Sat, 21 Sep 2024 13:04:41 +0900 Subject: [PATCH 02/35] Make STORED optional --- ast/ast.go | 9 ++++++--- ast/pos.go | 7 ++++++- ast/sql.go | 2 +- parser.go | 17 ++++++++++++----- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 7bc16bef..22d91db5 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -1536,13 +1536,16 @@ type ColumnDefaultExpr struct { // GeneratedColumnExpr is generated column expression. // -// AS ({{.Expr | sql}}) STORED +// AS ({{.Expr | sql}}) {{if .IsStored}}STORED{{end}} type GeneratedColumnExpr struct { // pos = As - // end = Stored + // end = Stored + 6 || Rparen + 1 As token.Pos // position of "AS" keyword - Stored token.Pos // position of "STORED" keyword + Stored token.Pos // position of "STORED" keyword, InvalidPos if not stored + Rparen token.Pos // position of ")" + + IsStored bool Expr Expr } diff --git a/ast/pos.go b/ast/pos.go index c817c775..7908b860 100644 --- a/ast/pos.go +++ b/ast/pos.go @@ -571,7 +571,12 @@ func (g *ColumnDefaultExpr) Pos() token.Pos { return g.Default } func (g *ColumnDefaultExpr) End() token.Pos { return g.Rparen } func (g *GeneratedColumnExpr) Pos() token.Pos { return g.As } -func (g *GeneratedColumnExpr) End() token.Pos { return g.Stored } +func (g *GeneratedColumnExpr) End() token.Pos { + if g.IsStored { + return g.Stored + 6 + } + return g.Rparen + 1 +} func (c *ColumnDefOptions) Pos() token.Pos { return c.Options } func (c *ColumnDefOptions) End() token.Pos { return c.Rparen + 1 } diff --git a/ast/sql.go b/ast/sql.go index 2cb2c744..74c801ba 100644 --- a/ast/sql.go +++ b/ast/sql.go @@ -880,7 +880,7 @@ func (c *ColumnDefaultExpr) SQL() string { } func (g *GeneratedColumnExpr) SQL() string { - return "AS (" + g.Expr.SQL() + ") STORED" + return "AS (" + g.Expr.SQL() + ")" + strOpt(g.IsStored, " STORED") } func (c *ColumnDefOptions) SQL() string { diff --git a/parser.go b/parser.go index 37969b37..7572fc57 100644 --- a/parser.go +++ b/parser.go @@ -2411,13 +2411,20 @@ func (p *Parser) tryParseGeneratedColumnExpr() *ast.GeneratedColumnExpr { posAs := p.expect("AS").Pos p.expect("(") expr := p.parseExpr() - p.expect(")") - posEnd := p.expectKeywordLike("STORED").End + rparen := p.expect(")").Pos + stored := p.tryExpectKeywordLike("STORED") + + var storedPos token.Pos + if stored != nil { + storedPos = stored.Pos + } return &ast.GeneratedColumnExpr{ - As: posAs, - Stored: posEnd, - Expr: expr, + As: posAs, + Stored: storedPos, + Rparen: rparen, + IsStored: stored != nil, + Expr: expr, } } From a9c66b124138834c9c8b960d1707c776d11a3959 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Sat, 21 Sep 2024 13:05:10 +0900 Subject: [PATCH 03/35] Support HIDDEN columns --- ast/ast.go | 7 +++++-- ast/sql.go | 3 +++ parser.go | 7 +++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 22d91db5..7273f28f 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -1506,10 +1506,11 @@ type CreateSequence struct { // {{.Type | sql}} {{if .NotNull}}NOT NULL{{end}} // {{.DefaultExpr | sqlOpt}} // {{.GeneratedExpr | sqlOpt}} +// {{if .Hidden}}HIDDEN{{end}} // {{.Options | sqlOpt}} type ColumnDef struct { // pos = Name.pos - // end = Options.end || GeneratedExpr.end || DefaultExpr.end || Null + 4 || Type.end + // end = Options.end || Hidden + 6 || GeneratedExpr.end || DefaultExpr.end || Null + 4 || Type.end Null token.Pos // position of "NULL" @@ -1518,7 +1519,9 @@ type ColumnDef struct { NotNull bool DefaultExpr *ColumnDefaultExpr // optional GeneratedExpr *GeneratedColumnExpr // optional - Options *ColumnDefOptions // optional + Hidden token.Pos // InvalidPos if not hidden + IsHidden bool + Options *ColumnDefOptions // optional } // ColumnDefaultExpr is a default value expression for the column. diff --git a/ast/sql.go b/ast/sql.go index 74c801ba..f2a1c776 100644 --- a/ast/sql.go +++ b/ast/sql.go @@ -832,6 +832,9 @@ func (c *ColumnDef) SQL() string { if c.GeneratedExpr != nil { sql += " " + c.GeneratedExpr.SQL() } + if c.IsHidden { + sql += " HIDDEN" + } if c.Options != nil { sql += " " + c.Options.SQL() } diff --git a/parser.go b/parser.go index 7572fc57..25ff8fdd 100644 --- a/parser.go +++ b/parser.go @@ -2295,6 +2295,11 @@ func (p *Parser) parseColumnDef() *ast.ColumnDef { t, notNull, null := p.parseTypeNotNull() defaultExpr := p.tryParseColumnDefaultExpr() generated := p.tryParseGeneratedColumnExpr() + hidden := p.tryExpectKeywordLike("HIDDEN") + var hiddenPos token.Pos + if hidden != nil { + hiddenPos = hidden.Pos + } options := p.tryParseColumnDefOptions() return &ast.ColumnDef{ @@ -2304,6 +2309,8 @@ func (p *Parser) parseColumnDef() *ast.ColumnDef { NotNull: notNull, DefaultExpr: defaultExpr, GeneratedExpr: generated, + Hidden: hiddenPos, + IsHidden: hidden != nil, Options: options, } } From 3d8958c5bd2e9b024f48f215ff86c1d1aa1b488d Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Sat, 21 Sep 2024 13:11:23 +0900 Subject: [PATCH 04/35] Add TOKENLIST type --- ast/const.go | 1 + parser.go | 1 + 2 files changed, 2 insertions(+) diff --git a/ast/const.go b/ast/const.go index 9ea04b10..5c564515 100644 --- a/ast/const.go +++ b/ast/const.go @@ -109,6 +109,7 @@ const ( TimestampTypeName ScalarTypeName = "TIMESTAMP" NumericTypeName ScalarTypeName = "NUMERIC" JSONTypeName ScalarTypeName = "JSON" + TokenListTypeName ScalarTypeName = "TOKENLIST" ) type OnDeleteAction string diff --git a/parser.go b/parser.go index 25ff8fdd..3d4f2ceb 100644 --- a/parser.go +++ b/parser.go @@ -3448,6 +3448,7 @@ var scalarSchemaTypes = []string{ "TIMESTAMP", "NUMERIC", "JSON", + "TOKENLIST", } var sizedSchemaTypes = []string{ From 41d9fdc0746e6df44144aac5905a5a2ebc267129 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Sat, 21 Sep 2024 13:11:45 +0900 Subject: [PATCH 05/35] Update testdata --- testdata/input/ddl/create_table.sql | 2 +- .../ddl/create_table_fulltext_albums.sql | 10 + .../result/ddl/alter_table_add_column.sql.txt | 2 + ...ter_table_add_column_if_not_exists.sql.txt | 2 + ...able_add_column_with_if_expression.sql.txt | 12 +- testdata/result/ddl/create_table.sql.txt | 34 ++- .../result/ddl/create_table_cluster.sql.txt | 4 + ...le_cluster_and_row_deletion_policy.sql.txt | 6 + ..._table_cluster_on_delete_no_action.sql.txt | 2 + ...create_table_cluster_set_on_delete.sql.txt | 2 + .../ddl/create_table_fulltext_albums.sql.txt | 219 ++++++++++++++++++ .../ddl/create_table_if_not_exists.sql.txt | 4 + .../create_table_row_deletion_policy.sql.txt | 6 + .../ddl/create_table_trailing_comma.sql.txt | 4 + .../result/ddl/create_table_types.sql.txt | 24 ++ ...reate_table_with_sequence_function.sql.txt | 6 + .../statement/alter_table_add_column.sql.txt | 2 + ...ter_table_add_column_if_not_exists.sql.txt | 2 + ...able_add_column_with_if_expression.sql.txt | 12 +- .../result/statement/create_table.sql.txt | 34 ++- .../statement/create_table_cluster.sql.txt | 4 + ...le_cluster_and_row_deletion_policy.sql.txt | 6 + ..._table_cluster_on_delete_no_action.sql.txt | 2 + ...create_table_cluster_set_on_delete.sql.txt | 2 + .../create_table_fulltext_albums.sql.txt | 219 ++++++++++++++++++ .../create_table_if_not_exists.sql.txt | 4 + .../create_table_row_deletion_policy.sql.txt | 6 + .../create_table_trailing_comma.sql.txt | 4 + .../statement/create_table_types.sql.txt | 24 ++ ...reate_table_with_sequence_function.sql.txt | 6 + 30 files changed, 637 insertions(+), 29 deletions(-) create mode 100644 testdata/input/ddl/create_table_fulltext_albums.sql create mode 100644 testdata/result/ddl/create_table_fulltext_albums.sql.txt create mode 100644 testdata/result/statement/create_table_fulltext_albums.sql.txt diff --git a/testdata/input/ddl/create_table.sql b/testdata/input/ddl/create_table.sql index 4bae8db4..acf8118a 100644 --- a/testdata/input/ddl/create_table.sql +++ b/testdata/input/ddl/create_table.sql @@ -10,5 +10,5 @@ create table foo ( check (foo > 0), constraint cname check (bar > 0), quux json, - corge timestamp not null default (current_timestamp()) + corge timestamp not null default (current_timestamp()), ) primary key (foo, bar) diff --git a/testdata/input/ddl/create_table_fulltext_albums.sql b/testdata/input/ddl/create_table_fulltext_albums.sql new file mode 100644 index 00000000..4dbd2f86 --- /dev/null +++ b/testdata/input/ddl/create_table_fulltext_albums.sql @@ -0,0 +1,10 @@ +-- https://cloud.google.com/spanner/docs/full-text-search/search-indexes#search-index-schema-definitions +CREATE TABLE Albums ( + AlbumId STRING(MAX) NOT NULL, + SingerId INT64 NOT NULL, + ReleaseTimestamp INT64 NOT NULL, + AlbumTitle STRING(MAX), + Rating FLOAT64, + AlbumTitle_Tokens TOKENLIST AS (TOKENIZE_FULLTEXT(AlbumTitle)) HIDDEN, + Rating_Tokens TOKENLIST AS (TOKENIZE_NUMBER(Rating)) HIDDEN +) PRIMARY KEY(AlbumId) \ No newline at end of file diff --git a/testdata/result/ddl/alter_table_add_column.sql.txt b/testdata/result/ddl/alter_table_add_column.sql.txt index b3003ef7..739db75f 100644 --- a/testdata/result/ddl/alter_table_add_column.sql.txt +++ b/testdata/result/ddl/alter_table_add_column.sql.txt @@ -28,6 +28,8 @@ alter table foo add column baz string(max) not null NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/alter_table_add_column_if_not_exists.sql.txt b/testdata/result/ddl/alter_table_add_column_if_not_exists.sql.txt index 3d0c2760..45a65bcc 100644 --- a/testdata/result/ddl/alter_table_add_column_if_not_exists.sql.txt +++ b/testdata/result/ddl/alter_table_add_column_if_not_exists.sql.txt @@ -29,6 +29,8 @@ alter table foo add column if not exists baz string(max) not null NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/alter_table_add_column_with_if_expression.sql.txt b/testdata/result/ddl/alter_table_add_column_with_if_expression.sql.txt index 296ec2a0..db18fc62 100644 --- a/testdata/result/ddl/alter_table_add_column_with_if_expression.sql.txt +++ b/testdata/result/ddl/alter_table_add_column_with_if_expression.sql.txt @@ -25,9 +25,11 @@ ALTER TABLE foo ADD COLUMN expired_at TIMESTAMP AS (IF (status != "OPEN" AND sta NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: &ast.GeneratedColumnExpr{ - As: 48, - Stored: 159, - Expr: &ast.CallExpr{ + As: 48, + Stored: 153, + Rparen: 151, + IsStored: true, + Expr: &ast.CallExpr{ Rparen: 150, Func: &ast.Ident{ NamePos: 52, @@ -111,7 +113,9 @@ ALTER TABLE foo ADD COLUMN expired_at TIMESTAMP AS (IF (status != "OPEN" AND sta NamedArgs: []*ast.NamedArg(nil), }, }, - Options: (*ast.ColumnDefOptions)(nil), + Hidden: 0, + IsHidden: false, + Options: (*ast.ColumnDefOptions)(nil), }, }, } diff --git a/testdata/result/ddl/create_table.sql.txt b/testdata/result/ddl/create_table.sql.txt index 09e7fed4..2d3b6d55 100644 --- a/testdata/result/ddl/create_table.sql.txt +++ b/testdata/result/ddl/create_table.sql.txt @@ -11,13 +11,13 @@ create table foo ( check (foo > 0), constraint cname check (bar > 0), quux json, - corge timestamp not null default (current_timestamp()) + corge timestamp not null default (current_timestamp()), ) primary key (foo, bar) --- AST &ast.CreateTable{ Create: 0, - Rparen: 573, + Rparen: 574, IfNotExists: false, Name: &ast.Ident{ NamePos: 13, @@ -39,6 +39,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -55,6 +57,8 @@ create table foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -79,6 +83,8 @@ create table foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: &ast.ColumnDefOptions{ Options: 83, Rparen: 121, @@ -107,9 +113,11 @@ create table foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: &ast.GeneratedColumnExpr{ - As: 150, - Stored: 178, - Expr: &ast.CallExpr{ + As: 150, + Stored: 172, + Rparen: 170, + IsStored: true, + Expr: &ast.CallExpr{ Rparen: 169, Func: &ast.Ident{ NamePos: 154, @@ -136,7 +144,9 @@ create table foo ( NamedArgs: []*ast.NamedArg(nil), }, }, - Options: (*ast.ColumnDefOptions)(nil), + Hidden: 0, + IsHidden: false, + Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ Null: -1, @@ -152,6 +162,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -182,6 +194,8 @@ create table foo ( }, }, GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, @@ -369,8 +383,8 @@ create table foo ( &ast.IndexKey{ DirPos: -1, Name: &ast.Ident{ - NamePos: 565, - NameEnd: 568, + NamePos: 566, + NameEnd: 569, Name: "foo", }, Dir: "", @@ -378,8 +392,8 @@ create table foo ( &ast.IndexKey{ DirPos: -1, Name: &ast.Ident{ - NamePos: 570, - NameEnd: 573, + NamePos: 571, + NameEnd: 574, Name: "bar", }, Dir: "", diff --git a/testdata/result/ddl/create_table_cluster.sql.txt b/testdata/result/ddl/create_table_cluster.sql.txt index 8eb83c31..e5db3e43 100644 --- a/testdata/result/ddl/create_table_cluster.sql.txt +++ b/testdata/result/ddl/create_table_cluster.sql.txt @@ -30,6 +30,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -46,6 +48,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_cluster_and_row_deletion_policy.sql.txt b/testdata/result/ddl/create_table_cluster_and_row_deletion_policy.sql.txt index 26aa3fb5..0953ba37 100644 --- a/testdata/result/ddl/create_table_cluster_and_row_deletion_policy.sql.txt +++ b/testdata/result/ddl/create_table_cluster_and_row_deletion_policy.sql.txt @@ -32,6 +32,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -48,6 +50,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -64,6 +68,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_cluster_on_delete_no_action.sql.txt b/testdata/result/ddl/create_table_cluster_on_delete_no_action.sql.txt index c2caa5ca..d2b99719 100644 --- a/testdata/result/ddl/create_table_cluster_on_delete_no_action.sql.txt +++ b/testdata/result/ddl/create_table_cluster_on_delete_no_action.sql.txt @@ -29,6 +29,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_cluster_set_on_delete.sql.txt b/testdata/result/ddl/create_table_cluster_set_on_delete.sql.txt index 4de2bccd..b05a71e3 100644 --- a/testdata/result/ddl/create_table_cluster_set_on_delete.sql.txt +++ b/testdata/result/ddl/create_table_cluster_set_on_delete.sql.txt @@ -29,6 +29,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_fulltext_albums.sql.txt b/testdata/result/ddl/create_table_fulltext_albums.sql.txt new file mode 100644 index 00000000..5f170208 --- /dev/null +++ b/testdata/result/ddl/create_table_fulltext_albums.sql.txt @@ -0,0 +1,219 @@ +--- create_table_fulltext_albums.sql +-- https://cloud.google.com/spanner/docs/full-text-search/search-indexes#search-index-schema-definitions +CREATE TABLE Albums ( + AlbumId STRING(MAX) NOT NULL, + SingerId INT64 NOT NULL, + ReleaseTimestamp INT64 NOT NULL, + AlbumTitle STRING(MAX), + Rating FLOAT64, + AlbumTitle_Tokens TOKENLIST AS (TOKENIZE_FULLTEXT(AlbumTitle)) HIDDEN, + Rating_Tokens TOKENLIST AS (TOKENIZE_NUMBER(Rating)) HIDDEN +) PRIMARY KEY(AlbumId) +--- AST +&ast.CreateTable{ + Create: 105, + Rparen: 575, + IfNotExists: false, + Name: &ast.Ident{ + NamePos: 118, + NameEnd: 124, + Name: "Albums", + }, + Columns: []*ast.ColumnDef{ + &ast.ColumnDef{ + Null: 175, + Name: &ast.Ident{ + NamePos: 151, + NameEnd: 158, + Name: "AlbumId", + }, + Type: &ast.SizedSchemaType{ + NamePos: 159, + Rparen: 169, + Name: "STRING", + Max: true, + Size: nil, + }, + NotNull: true, + DefaultExpr: (*ast.ColumnDefaultExpr)(nil), + GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, + Options: (*ast.ColumnDefOptions)(nil), + }, + &ast.ColumnDef{ + Null: 224, + Name: &ast.Ident{ + NamePos: 205, + NameEnd: 213, + Name: "SingerId", + }, + Type: &ast.ScalarSchemaType{ + NamePos: 214, + Name: "INT64", + }, + NotNull: true, + DefaultExpr: (*ast.ColumnDefaultExpr)(nil), + GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, + Options: (*ast.ColumnDefOptions)(nil), + }, + &ast.ColumnDef{ + Null: 281, + Name: &ast.Ident{ + NamePos: 254, + NameEnd: 270, + Name: "ReleaseTimestamp", + }, + Type: &ast.ScalarSchemaType{ + NamePos: 271, + Name: "INT64", + }, + NotNull: true, + DefaultExpr: (*ast.ColumnDefaultExpr)(nil), + GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, + Options: (*ast.ColumnDefOptions)(nil), + }, + &ast.ColumnDef{ + Null: -1, + Name: &ast.Ident{ + NamePos: 311, + NameEnd: 321, + Name: "AlbumTitle", + }, + Type: &ast.SizedSchemaType{ + NamePos: 322, + Rparen: 332, + Name: "STRING", + Max: true, + Size: nil, + }, + NotNull: false, + DefaultExpr: (*ast.ColumnDefaultExpr)(nil), + GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, + Options: (*ast.ColumnDefOptions)(nil), + }, + &ast.ColumnDef{ + Null: -1, + Name: &ast.Ident{ + NamePos: 359, + NameEnd: 365, + Name: "Rating", + }, + Type: &ast.ScalarSchemaType{ + NamePos: 366, + Name: "FLOAT64", + }, + NotNull: false, + DefaultExpr: (*ast.ColumnDefaultExpr)(nil), + GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, + Options: (*ast.ColumnDefOptions)(nil), + }, + &ast.ColumnDef{ + Null: -1, + Name: &ast.Ident{ + NamePos: 399, + NameEnd: 416, + Name: "AlbumTitle_Tokens", + }, + Type: &ast.ScalarSchemaType{ + NamePos: 417, + Name: "TOKENLIST", + }, + NotNull: false, + DefaultExpr: (*ast.ColumnDefaultExpr)(nil), + GeneratedExpr: &ast.GeneratedColumnExpr{ + As: 427, + Stored: 0, + Rparen: 460, + IsStored: false, + Expr: &ast.CallExpr{ + Rparen: 459, + Func: &ast.Ident{ + NamePos: 431, + NameEnd: 448, + Name: "TOKENIZE_FULLTEXT", + }, + Distinct: false, + Args: []ast.Arg{ + &ast.ExprArg{ + Expr: &ast.Ident{ + NamePos: 449, + NameEnd: 459, + Name: "AlbumTitle", + }, + }, + }, + }, + }, + Hidden: 462, + IsHidden: true, + Options: (*ast.ColumnDefOptions)(nil), + }, + &ast.ColumnDef{ + Null: -1, + Name: &ast.Ident{ + NamePos: 494, + NameEnd: 507, + Name: "Rating_Tokens", + }, + Type: &ast.ScalarSchemaType{ + NamePos: 508, + Name: "TOKENLIST", + }, + NotNull: false, + DefaultExpr: (*ast.ColumnDefaultExpr)(nil), + GeneratedExpr: &ast.GeneratedColumnExpr{ + As: 518, + Stored: 0, + Rparen: 545, + IsStored: false, + Expr: &ast.CallExpr{ + Rparen: 544, + Func: &ast.Ident{ + NamePos: 522, + NameEnd: 537, + Name: "TOKENIZE_NUMBER", + }, + Distinct: false, + Args: []ast.Arg{ + &ast.ExprArg{ + Expr: &ast.Ident{ + NamePos: 538, + NameEnd: 544, + Name: "Rating", + }, + }, + }, + }, + }, + Hidden: 547, + IsHidden: true, + Options: (*ast.ColumnDefOptions)(nil), + }, + }, + TableConstraints: []*ast.TableConstraint(nil), + PrimaryKeys: []*ast.IndexKey{ + &ast.IndexKey{ + DirPos: -1, + Name: &ast.Ident{ + NamePos: 568, + NameEnd: 575, + Name: "AlbumId", + }, + Dir: "", + }, + }, + Cluster: (*ast.Cluster)(nil), + RowDeletionPolicy: (*ast.CreateRowDeletionPolicy)(nil), +} + +--- SQL +CREATE TABLE Albums (AlbumId STRING(MAX) NOT NULL, SingerId INT64 NOT NULL, ReleaseTimestamp INT64 NOT NULL, AlbumTitle STRING(MAX), Rating FLOAT64, AlbumTitle_Tokens TOKENLIST AS (TOKENIZE_FULLTEXT(AlbumTitle)) HIDDEN, Rating_Tokens TOKENLIST AS (TOKENIZE_NUMBER(Rating)) HIDDEN) PRIMARY KEY (AlbumId) diff --git a/testdata/result/ddl/create_table_if_not_exists.sql.txt b/testdata/result/ddl/create_table_if_not_exists.sql.txt index 1677b5c3..07d0f784 100644 --- a/testdata/result/ddl/create_table_if_not_exists.sql.txt +++ b/testdata/result/ddl/create_table_if_not_exists.sql.txt @@ -29,6 +29,8 @@ create table if not exists foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -45,6 +47,8 @@ create table if not exists foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_row_deletion_policy.sql.txt b/testdata/result/ddl/create_table_row_deletion_policy.sql.txt index 4a49748b..4223209f 100644 --- a/testdata/result/ddl/create_table_row_deletion_policy.sql.txt +++ b/testdata/result/ddl/create_table_row_deletion_policy.sql.txt @@ -31,6 +31,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -47,6 +49,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -63,6 +67,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_trailing_comma.sql.txt b/testdata/result/ddl/create_table_trailing_comma.sql.txt index 0ff16b9c..13a9b167 100644 --- a/testdata/result/ddl/create_table_trailing_comma.sql.txt +++ b/testdata/result/ddl/create_table_trailing_comma.sql.txt @@ -32,6 +32,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -48,6 +50,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_types.sql.txt b/testdata/result/ddl/create_table_types.sql.txt index f7f536db..f8f793b8 100644 --- a/testdata/result/ddl/create_table_types.sql.txt +++ b/testdata/result/ddl/create_table_types.sql.txt @@ -40,6 +40,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -56,6 +58,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -72,6 +76,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -88,6 +94,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -104,6 +112,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -120,6 +130,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -144,6 +156,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -163,6 +177,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -187,6 +203,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -206,6 +224,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -226,6 +246,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -249,6 +271,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ diff --git a/testdata/result/ddl/create_table_with_sequence_function.sql.txt b/testdata/result/ddl/create_table_with_sequence_function.sql.txt index d3fac4a3..0469da83 100644 --- a/testdata/result/ddl/create_table_with_sequence_function.sql.txt +++ b/testdata/result/ddl/create_table_with_sequence_function.sql.txt @@ -54,6 +54,8 @@ CREATE TABLE foo }, }, GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -73,6 +75,8 @@ CREATE TABLE foo NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -89,6 +93,8 @@ CREATE TABLE foo NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/alter_table_add_column.sql.txt b/testdata/result/statement/alter_table_add_column.sql.txt index b3003ef7..739db75f 100644 --- a/testdata/result/statement/alter_table_add_column.sql.txt +++ b/testdata/result/statement/alter_table_add_column.sql.txt @@ -28,6 +28,8 @@ alter table foo add column baz string(max) not null NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/alter_table_add_column_if_not_exists.sql.txt b/testdata/result/statement/alter_table_add_column_if_not_exists.sql.txt index 3d0c2760..45a65bcc 100644 --- a/testdata/result/statement/alter_table_add_column_if_not_exists.sql.txt +++ b/testdata/result/statement/alter_table_add_column_if_not_exists.sql.txt @@ -29,6 +29,8 @@ alter table foo add column if not exists baz string(max) not null NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/alter_table_add_column_with_if_expression.sql.txt b/testdata/result/statement/alter_table_add_column_with_if_expression.sql.txt index 296ec2a0..db18fc62 100644 --- a/testdata/result/statement/alter_table_add_column_with_if_expression.sql.txt +++ b/testdata/result/statement/alter_table_add_column_with_if_expression.sql.txt @@ -25,9 +25,11 @@ ALTER TABLE foo ADD COLUMN expired_at TIMESTAMP AS (IF (status != "OPEN" AND sta NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: &ast.GeneratedColumnExpr{ - As: 48, - Stored: 159, - Expr: &ast.CallExpr{ + As: 48, + Stored: 153, + Rparen: 151, + IsStored: true, + Expr: &ast.CallExpr{ Rparen: 150, Func: &ast.Ident{ NamePos: 52, @@ -111,7 +113,9 @@ ALTER TABLE foo ADD COLUMN expired_at TIMESTAMP AS (IF (status != "OPEN" AND sta NamedArgs: []*ast.NamedArg(nil), }, }, - Options: (*ast.ColumnDefOptions)(nil), + Hidden: 0, + IsHidden: false, + Options: (*ast.ColumnDefOptions)(nil), }, }, } diff --git a/testdata/result/statement/create_table.sql.txt b/testdata/result/statement/create_table.sql.txt index 09e7fed4..2d3b6d55 100644 --- a/testdata/result/statement/create_table.sql.txt +++ b/testdata/result/statement/create_table.sql.txt @@ -11,13 +11,13 @@ create table foo ( check (foo > 0), constraint cname check (bar > 0), quux json, - corge timestamp not null default (current_timestamp()) + corge timestamp not null default (current_timestamp()), ) primary key (foo, bar) --- AST &ast.CreateTable{ Create: 0, - Rparen: 573, + Rparen: 574, IfNotExists: false, Name: &ast.Ident{ NamePos: 13, @@ -39,6 +39,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -55,6 +57,8 @@ create table foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -79,6 +83,8 @@ create table foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: &ast.ColumnDefOptions{ Options: 83, Rparen: 121, @@ -107,9 +113,11 @@ create table foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: &ast.GeneratedColumnExpr{ - As: 150, - Stored: 178, - Expr: &ast.CallExpr{ + As: 150, + Stored: 172, + Rparen: 170, + IsStored: true, + Expr: &ast.CallExpr{ Rparen: 169, Func: &ast.Ident{ NamePos: 154, @@ -136,7 +144,9 @@ create table foo ( NamedArgs: []*ast.NamedArg(nil), }, }, - Options: (*ast.ColumnDefOptions)(nil), + Hidden: 0, + IsHidden: false, + Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ Null: -1, @@ -152,6 +162,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -182,6 +194,8 @@ create table foo ( }, }, GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, @@ -369,8 +383,8 @@ create table foo ( &ast.IndexKey{ DirPos: -1, Name: &ast.Ident{ - NamePos: 565, - NameEnd: 568, + NamePos: 566, + NameEnd: 569, Name: "foo", }, Dir: "", @@ -378,8 +392,8 @@ create table foo ( &ast.IndexKey{ DirPos: -1, Name: &ast.Ident{ - NamePos: 570, - NameEnd: 573, + NamePos: 571, + NameEnd: 574, Name: "bar", }, Dir: "", diff --git a/testdata/result/statement/create_table_cluster.sql.txt b/testdata/result/statement/create_table_cluster.sql.txt index 8eb83c31..e5db3e43 100644 --- a/testdata/result/statement/create_table_cluster.sql.txt +++ b/testdata/result/statement/create_table_cluster.sql.txt @@ -30,6 +30,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -46,6 +48,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_cluster_and_row_deletion_policy.sql.txt b/testdata/result/statement/create_table_cluster_and_row_deletion_policy.sql.txt index 26aa3fb5..0953ba37 100644 --- a/testdata/result/statement/create_table_cluster_and_row_deletion_policy.sql.txt +++ b/testdata/result/statement/create_table_cluster_and_row_deletion_policy.sql.txt @@ -32,6 +32,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -48,6 +50,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -64,6 +68,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_cluster_on_delete_no_action.sql.txt b/testdata/result/statement/create_table_cluster_on_delete_no_action.sql.txt index c2caa5ca..d2b99719 100644 --- a/testdata/result/statement/create_table_cluster_on_delete_no_action.sql.txt +++ b/testdata/result/statement/create_table_cluster_on_delete_no_action.sql.txt @@ -29,6 +29,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_cluster_set_on_delete.sql.txt b/testdata/result/statement/create_table_cluster_set_on_delete.sql.txt index 4de2bccd..b05a71e3 100644 --- a/testdata/result/statement/create_table_cluster_set_on_delete.sql.txt +++ b/testdata/result/statement/create_table_cluster_set_on_delete.sql.txt @@ -29,6 +29,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_fulltext_albums.sql.txt b/testdata/result/statement/create_table_fulltext_albums.sql.txt new file mode 100644 index 00000000..5f170208 --- /dev/null +++ b/testdata/result/statement/create_table_fulltext_albums.sql.txt @@ -0,0 +1,219 @@ +--- create_table_fulltext_albums.sql +-- https://cloud.google.com/spanner/docs/full-text-search/search-indexes#search-index-schema-definitions +CREATE TABLE Albums ( + AlbumId STRING(MAX) NOT NULL, + SingerId INT64 NOT NULL, + ReleaseTimestamp INT64 NOT NULL, + AlbumTitle STRING(MAX), + Rating FLOAT64, + AlbumTitle_Tokens TOKENLIST AS (TOKENIZE_FULLTEXT(AlbumTitle)) HIDDEN, + Rating_Tokens TOKENLIST AS (TOKENIZE_NUMBER(Rating)) HIDDEN +) PRIMARY KEY(AlbumId) +--- AST +&ast.CreateTable{ + Create: 105, + Rparen: 575, + IfNotExists: false, + Name: &ast.Ident{ + NamePos: 118, + NameEnd: 124, + Name: "Albums", + }, + Columns: []*ast.ColumnDef{ + &ast.ColumnDef{ + Null: 175, + Name: &ast.Ident{ + NamePos: 151, + NameEnd: 158, + Name: "AlbumId", + }, + Type: &ast.SizedSchemaType{ + NamePos: 159, + Rparen: 169, + Name: "STRING", + Max: true, + Size: nil, + }, + NotNull: true, + DefaultExpr: (*ast.ColumnDefaultExpr)(nil), + GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, + Options: (*ast.ColumnDefOptions)(nil), + }, + &ast.ColumnDef{ + Null: 224, + Name: &ast.Ident{ + NamePos: 205, + NameEnd: 213, + Name: "SingerId", + }, + Type: &ast.ScalarSchemaType{ + NamePos: 214, + Name: "INT64", + }, + NotNull: true, + DefaultExpr: (*ast.ColumnDefaultExpr)(nil), + GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, + Options: (*ast.ColumnDefOptions)(nil), + }, + &ast.ColumnDef{ + Null: 281, + Name: &ast.Ident{ + NamePos: 254, + NameEnd: 270, + Name: "ReleaseTimestamp", + }, + Type: &ast.ScalarSchemaType{ + NamePos: 271, + Name: "INT64", + }, + NotNull: true, + DefaultExpr: (*ast.ColumnDefaultExpr)(nil), + GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, + Options: (*ast.ColumnDefOptions)(nil), + }, + &ast.ColumnDef{ + Null: -1, + Name: &ast.Ident{ + NamePos: 311, + NameEnd: 321, + Name: "AlbumTitle", + }, + Type: &ast.SizedSchemaType{ + NamePos: 322, + Rparen: 332, + Name: "STRING", + Max: true, + Size: nil, + }, + NotNull: false, + DefaultExpr: (*ast.ColumnDefaultExpr)(nil), + GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, + Options: (*ast.ColumnDefOptions)(nil), + }, + &ast.ColumnDef{ + Null: -1, + Name: &ast.Ident{ + NamePos: 359, + NameEnd: 365, + Name: "Rating", + }, + Type: &ast.ScalarSchemaType{ + NamePos: 366, + Name: "FLOAT64", + }, + NotNull: false, + DefaultExpr: (*ast.ColumnDefaultExpr)(nil), + GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, + Options: (*ast.ColumnDefOptions)(nil), + }, + &ast.ColumnDef{ + Null: -1, + Name: &ast.Ident{ + NamePos: 399, + NameEnd: 416, + Name: "AlbumTitle_Tokens", + }, + Type: &ast.ScalarSchemaType{ + NamePos: 417, + Name: "TOKENLIST", + }, + NotNull: false, + DefaultExpr: (*ast.ColumnDefaultExpr)(nil), + GeneratedExpr: &ast.GeneratedColumnExpr{ + As: 427, + Stored: 0, + Rparen: 460, + IsStored: false, + Expr: &ast.CallExpr{ + Rparen: 459, + Func: &ast.Ident{ + NamePos: 431, + NameEnd: 448, + Name: "TOKENIZE_FULLTEXT", + }, + Distinct: false, + Args: []ast.Arg{ + &ast.ExprArg{ + Expr: &ast.Ident{ + NamePos: 449, + NameEnd: 459, + Name: "AlbumTitle", + }, + }, + }, + }, + }, + Hidden: 462, + IsHidden: true, + Options: (*ast.ColumnDefOptions)(nil), + }, + &ast.ColumnDef{ + Null: -1, + Name: &ast.Ident{ + NamePos: 494, + NameEnd: 507, + Name: "Rating_Tokens", + }, + Type: &ast.ScalarSchemaType{ + NamePos: 508, + Name: "TOKENLIST", + }, + NotNull: false, + DefaultExpr: (*ast.ColumnDefaultExpr)(nil), + GeneratedExpr: &ast.GeneratedColumnExpr{ + As: 518, + Stored: 0, + Rparen: 545, + IsStored: false, + Expr: &ast.CallExpr{ + Rparen: 544, + Func: &ast.Ident{ + NamePos: 522, + NameEnd: 537, + Name: "TOKENIZE_NUMBER", + }, + Distinct: false, + Args: []ast.Arg{ + &ast.ExprArg{ + Expr: &ast.Ident{ + NamePos: 538, + NameEnd: 544, + Name: "Rating", + }, + }, + }, + }, + }, + Hidden: 547, + IsHidden: true, + Options: (*ast.ColumnDefOptions)(nil), + }, + }, + TableConstraints: []*ast.TableConstraint(nil), + PrimaryKeys: []*ast.IndexKey{ + &ast.IndexKey{ + DirPos: -1, + Name: &ast.Ident{ + NamePos: 568, + NameEnd: 575, + Name: "AlbumId", + }, + Dir: "", + }, + }, + Cluster: (*ast.Cluster)(nil), + RowDeletionPolicy: (*ast.CreateRowDeletionPolicy)(nil), +} + +--- SQL +CREATE TABLE Albums (AlbumId STRING(MAX) NOT NULL, SingerId INT64 NOT NULL, ReleaseTimestamp INT64 NOT NULL, AlbumTitle STRING(MAX), Rating FLOAT64, AlbumTitle_Tokens TOKENLIST AS (TOKENIZE_FULLTEXT(AlbumTitle)) HIDDEN, Rating_Tokens TOKENLIST AS (TOKENIZE_NUMBER(Rating)) HIDDEN) PRIMARY KEY (AlbumId) diff --git a/testdata/result/statement/create_table_if_not_exists.sql.txt b/testdata/result/statement/create_table_if_not_exists.sql.txt index 1677b5c3..07d0f784 100644 --- a/testdata/result/statement/create_table_if_not_exists.sql.txt +++ b/testdata/result/statement/create_table_if_not_exists.sql.txt @@ -29,6 +29,8 @@ create table if not exists foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -45,6 +47,8 @@ create table if not exists foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_row_deletion_policy.sql.txt b/testdata/result/statement/create_table_row_deletion_policy.sql.txt index 4a49748b..4223209f 100644 --- a/testdata/result/statement/create_table_row_deletion_policy.sql.txt +++ b/testdata/result/statement/create_table_row_deletion_policy.sql.txt @@ -31,6 +31,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -47,6 +49,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -63,6 +67,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_trailing_comma.sql.txt b/testdata/result/statement/create_table_trailing_comma.sql.txt index 0ff16b9c..13a9b167 100644 --- a/testdata/result/statement/create_table_trailing_comma.sql.txt +++ b/testdata/result/statement/create_table_trailing_comma.sql.txt @@ -32,6 +32,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -48,6 +50,8 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_types.sql.txt b/testdata/result/statement/create_table_types.sql.txt index f7f536db..f8f793b8 100644 --- a/testdata/result/statement/create_table_types.sql.txt +++ b/testdata/result/statement/create_table_types.sql.txt @@ -40,6 +40,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -56,6 +58,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -72,6 +76,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -88,6 +94,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -104,6 +112,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -120,6 +130,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -144,6 +156,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -163,6 +177,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -187,6 +203,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -206,6 +224,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -226,6 +246,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -249,6 +271,8 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ diff --git a/testdata/result/statement/create_table_with_sequence_function.sql.txt b/testdata/result/statement/create_table_with_sequence_function.sql.txt index d3fac4a3..0469da83 100644 --- a/testdata/result/statement/create_table_with_sequence_function.sql.txt +++ b/testdata/result/statement/create_table_with_sequence_function.sql.txt @@ -54,6 +54,8 @@ CREATE TABLE foo }, }, GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -73,6 +75,8 @@ CREATE TABLE foo NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -89,6 +93,8 @@ CREATE TABLE foo NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: 0, + IsHidden: false, Options: (*ast.ColumnDefOptions)(nil), }, }, From 403348ad05d02882111e34ee7f215c5e44f6670d Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Sat, 21 Sep 2024 10:04:33 +0900 Subject: [PATCH 06/35] Implement CREATE/DROP/ALTER SEARCH INDEX --- ast/ast.go | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/ast/ast.go b/ast/ast.go index 7273f28f..eb6542a1 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -2342,6 +2342,121 @@ type ArraySchemaType struct { Item SchemaType // ScalarSchemaType or SizedSchemaType } +// ================================================================================ +// +// Search Index DDL +// +// ================================================================================ + +// CreateSearchIndex +// +// CREATE SEARCH INDEX {{.Name | sql}} +// ON {{.TableName | sql}} +// ({{.TokenListPart | sqlJoin ", "}}) +// {{.Storing | sqlOpt}} +// {{if not(.PartitionColumns | isnil)}}PARTITION BY {{.PartitionColumns | sqlJoin ", "}}{{end}} +// {{.OrderBy | sqlOpt}} +// {{.Where | sqlOpt}} +// {{.Interleave | sqlOpt}} +// {{.Options | sqlOpt}} +type CreateSearchIndex struct { + // pos = Create + // end = (Options ?? Interleave ?? Where ?? OrderBy ?? PartitionColumns[$] ?? Storing).end || Rparen + 1 + + Create token.Pos + + Name *Ident + TableName *Ident + TokenListPart []*Ident + Rparen token.Pos // position of ")" after TokenListPart + Storing *Storing // optional + PartitionColumns []*Ident // optional + OrderBy *OrderBy // optional + Where *Where // optional + Interleave *InterleaveIn //optional + Options *GenericOptions // optional +} + +func (c *CreateSearchIndex) Pos() token.Pos { + return c.Create +} + +func (c *CreateSearchIndex) End() token.Pos { + if e := firstValidEnd(c.Options, c.Interleave, c.Where, c.OrderBy, lastNode(c.PartitionColumns), c.Storing); e != token.InvalidPos { + return e + } + return c.Rparen + 1 +} + +func (c *CreateSearchIndex) SQL() string { + return "CREATE SEARCH INDEX " + c.Name.SQL() + " ON " + c.TableName.SQL() + + "(" + sqlJoin(c.TokenListPart, ", ") + ")" + + sqlOpt(" ", c.Storing, "") + + strOpt(len(c.PartitionColumns) > 0, " PARTITION BY "+sqlJoin(c.PartitionColumns, ", ")) + + sqlOpt(" ", c.OrderBy, "") + + sqlOpt(" ", c.Where, "") + + sqlOpt(" ", c.Interleave, "") + + sqlOpt(" ", c.Options, "") +} + +func (CreateSearchIndex) isStatement() {} +func (CreateSearchIndex) isDDL() {} + +// DropSearchIndex +// +// DROP SEARCH INDEX {{.IfExists | strOpt "IF EXISTS "}}{{Name | sql}} +type DropSearchIndex struct { + // pos = Drop + // end = Name.end + + Drop token.Pos + IfExists bool + Name *Ident +} + +func (d *DropSearchIndex) Pos() token.Pos { + return d.Drop +} + +func (d *DropSearchIndex) End() token.Pos { + return d.Name.End() +} + +func (d *DropSearchIndex) SQL() string { + return "DROP SEARCH INDEX " + strOpt(d.IfExists, "IF EXISTS ") + d.Name.SQL() +} + +func (DropSearchIndex) isStatement() {} +func (DropSearchIndex) isDDL() {} + +// AlterSearchIndex +// +// ALTER INDEX {{.Name | sql}} {{.IndexAlteration | sql}} +type AlterSearchIndex struct { + // pos = Alter + // end = IndexAlteration.end + + Alter token.Pos + Name *Ident + IndexAlteration IndexAlteration +} + +func (AlterSearchIndex) isStatement() {} + +func (AlterSearchIndex) isDDL() {} + +func (a *AlterSearchIndex) Pos() token.Pos { + return a.Alter +} + +func (a *AlterSearchIndex) End() token.Pos { + return a.IndexAlteration.End() +} + +func (a *AlterSearchIndex) SQL() string { + return "ALTER INDEX " + a.Name.SQL() + " " + a.IndexAlteration.SQL() +} + // ================================================================================ // // DML From 6987a82ab810be8a3eef5f2ae66c99ba48dd7add Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Tue, 15 Oct 2024 23:07:41 +0900 Subject: [PATCH 07/35] Add parser functions/methods --- parser.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/parser.go b/parser.go index 3d4f2ceb..2b61fae5 100644 --- a/parser.go +++ b/parser.go @@ -3904,6 +3904,20 @@ func (p *Parser) expectKeywordLike(s string) *token.Token { return id } +func (p *Parser) tryExpect(s token.TokenKind) *token.Token { + if p.Token.Kind != s { + return nil + } + return p.expect(s) +} + +func (p *Parser) tryExpectKeywordLike(s string) *token.Token { + if !p.Token.IsKeywordLike(s) { + return nil + } + return p.expectKeywordLike(s) +} + func (p *Parser) errorfAtToken(tok *token.Token, msg string, params ...interface{}) *Error { return &Error{ Message: fmt.Sprintf(msg, params...), @@ -3914,3 +3928,31 @@ func (p *Parser) errorfAtToken(tok *token.Token, msg string, params ...interface func (p *Parser) panicfAtToken(tok *token.Token, msg string, params ...interface{}) { panic(p.errorfAtToken(tok, msg, params...)) } + +// This function can't be a method because Go haven't yet supported generic methods. +func parseSeparatedList[T interface { + ast.Node + comparable +}](p *Parser, sep token.TokenKind, doParse func() T) []T { + var zero T + + first := doParse() + if first == zero { + return nil + } + + list := []T{first} + for { + if p.Token.Kind != sep { + break + } + p.nextToken() + + elem := doParse() + if elem == zero { + return list + } + list = append(list, elem) + } + return list +} From c159518336327c1c54cc2b278b8c1ef8f374c4b2 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Tue, 15 Oct 2024 23:07:58 +0900 Subject: [PATCH 08/35] Add TOKENLIST as simple type --- parser.go | 1 + 1 file changed, 1 insertion(+) diff --git a/parser.go b/parser.go index 2b61fae5..2a2a5956 100644 --- a/parser.go +++ b/parser.go @@ -1920,6 +1920,7 @@ var simpleTypes = []string{ "STRING", "BYTES", "JSON", + "TOKENLIST", } func (p *Parser) parseSimpleType() *ast.SimpleType { From 50c47ee58033372fea68d701c55167f482767075 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Tue, 15 Oct 2024 23:14:35 +0900 Subject: [PATCH 09/35] Fix HIDDEN implementation --- ast/ast.go | 3 +-- ast/sql.go | 23 ++++++----------------- parser.go | 3 +-- 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index eb6542a1..5eff1a53 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -1520,8 +1520,7 @@ type ColumnDef struct { DefaultExpr *ColumnDefaultExpr // optional GeneratedExpr *GeneratedColumnExpr // optional Hidden token.Pos // InvalidPos if not hidden - IsHidden bool - Options *ColumnDefOptions // optional + Options *ColumnDefOptions // optional } // ColumnDefaultExpr is a default value expression for the column. diff --git a/ast/sql.go b/ast/sql.go index f2a1c776..19f471c0 100644 --- a/ast/sql.go +++ b/ast/sql.go @@ -822,23 +822,12 @@ func (c *CreateView) SQL() string { } func (c *ColumnDef) SQL() string { - sql := c.Name.SQL() + " " + c.Type.SQL() - if c.NotNull { - sql += " NOT NULL" - } - if c.DefaultExpr != nil { - sql += " " + c.DefaultExpr.SQL() - } - if c.GeneratedExpr != nil { - sql += " " + c.GeneratedExpr.SQL() - } - if c.IsHidden { - sql += " HIDDEN" - } - if c.Options != nil { - sql += " " + c.Options.SQL() - } - return sql + return c.Name.SQL() + " " + c.Type.SQL() + + strOpt(c.NotNull, " NOT NULL") + + sqlOpt(" ", c.DefaultExpr, "") + + sqlOpt(" ", c.GeneratedExpr, "") + + strOpt(!c.Hidden.Invalid(), " HIDDEN") + + sqlOpt(" ", c.Options, "") } func (c *TableConstraint) SQL() string { diff --git a/parser.go b/parser.go index 2a2a5956..9900fcef 100644 --- a/parser.go +++ b/parser.go @@ -2297,7 +2297,7 @@ func (p *Parser) parseColumnDef() *ast.ColumnDef { defaultExpr := p.tryParseColumnDefaultExpr() generated := p.tryParseGeneratedColumnExpr() hidden := p.tryExpectKeywordLike("HIDDEN") - var hiddenPos token.Pos + hiddenPos := token.InvalidPos if hidden != nil { hiddenPos = hidden.Pos } @@ -2311,7 +2311,6 @@ func (p *Parser) parseColumnDef() *ast.ColumnDef { DefaultExpr: defaultExpr, GeneratedExpr: generated, Hidden: hiddenPos, - IsHidden: hidden != nil, Options: options, } } From d3d9be2b82b86ac34abb49f634c611c806a008e3 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Tue, 15 Oct 2024 23:14:54 +0900 Subject: [PATCH 10/35] Update testdata --- .../result/ddl/alter_table_add_column.sql.txt | 3 +- ...ter_table_add_column_if_not_exists.sql.txt | 3 +- ...able_add_column_with_if_expression.sql.txt | 5 +-- testdata/result/ddl/create_table.sql.txt | 20 ++++------ .../result/ddl/create_table_cluster.sql.txt | 6 +-- ...le_cluster_and_row_deletion_policy.sql.txt | 9 ++--- ..._table_cluster_on_delete_no_action.sql.txt | 3 +- ...create_table_cluster_set_on_delete.sql.txt | 3 +- .../ddl/create_table_fulltext_albums.sql.txt | 27 ++++++-------- .../ddl/create_table_if_not_exists.sql.txt | 6 +-- .../create_table_row_deletion_policy.sql.txt | 9 ++--- .../ddl/create_table_trailing_comma.sql.txt | 6 +-- .../result/ddl/create_table_types.sql.txt | 37 +++++++------------ ...reate_table_with_sequence_function.sql.txt | 9 ++--- .../statement/alter_table_add_column.sql.txt | 3 +- ...ter_table_add_column_if_not_exists.sql.txt | 3 +- ...able_add_column_with_if_expression.sql.txt | 5 +-- .../result/statement/create_table.sql.txt | 20 ++++------ .../statement/create_table_cluster.sql.txt | 6 +-- ...le_cluster_and_row_deletion_policy.sql.txt | 9 ++--- ..._table_cluster_on_delete_no_action.sql.txt | 3 +- ...create_table_cluster_set_on_delete.sql.txt | 3 +- .../create_table_fulltext_albums.sql.txt | 27 ++++++-------- .../create_table_if_not_exists.sql.txt | 6 +-- .../create_table_row_deletion_policy.sql.txt | 9 ++--- .../create_table_trailing_comma.sql.txt | 6 +-- .../statement/create_table_types.sql.txt | 37 +++++++------------ ...reate_table_with_sequence_function.sql.txt | 9 ++--- 28 files changed, 104 insertions(+), 188 deletions(-) diff --git a/testdata/result/ddl/alter_table_add_column.sql.txt b/testdata/result/ddl/alter_table_add_column.sql.txt index 739db75f..facb64c1 100644 --- a/testdata/result/ddl/alter_table_add_column.sql.txt +++ b/testdata/result/ddl/alter_table_add_column.sql.txt @@ -28,8 +28,7 @@ alter table foo add column baz string(max) not null NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/alter_table_add_column_if_not_exists.sql.txt b/testdata/result/ddl/alter_table_add_column_if_not_exists.sql.txt index 45a65bcc..407ca7dc 100644 --- a/testdata/result/ddl/alter_table_add_column_if_not_exists.sql.txt +++ b/testdata/result/ddl/alter_table_add_column_if_not_exists.sql.txt @@ -29,8 +29,7 @@ alter table foo add column if not exists baz string(max) not null NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/alter_table_add_column_with_if_expression.sql.txt b/testdata/result/ddl/alter_table_add_column_with_if_expression.sql.txt index db18fc62..472d04e2 100644 --- a/testdata/result/ddl/alter_table_add_column_with_if_expression.sql.txt +++ b/testdata/result/ddl/alter_table_add_column_with_if_expression.sql.txt @@ -113,9 +113,8 @@ ALTER TABLE foo ADD COLUMN expired_at TIMESTAMP AS (IF (status != "OPEN" AND sta NamedArgs: []*ast.NamedArg(nil), }, }, - Hidden: 0, - IsHidden: false, - Options: (*ast.ColumnDefOptions)(nil), + Hidden: -1, + Options: (*ast.ColumnDefOptions)(nil), }, }, } diff --git a/testdata/result/ddl/create_table.sql.txt b/testdata/result/ddl/create_table.sql.txt index 2d3b6d55..ffc2b1a1 100644 --- a/testdata/result/ddl/create_table.sql.txt +++ b/testdata/result/ddl/create_table.sql.txt @@ -39,8 +39,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -57,8 +56,7 @@ create table foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -83,8 +81,7 @@ create table foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: &ast.ColumnDefOptions{ Options: 83, Rparen: 121, @@ -144,9 +141,8 @@ create table foo ( NamedArgs: []*ast.NamedArg(nil), }, }, - Hidden: 0, - IsHidden: false, - Options: (*ast.ColumnDefOptions)(nil), + Hidden: -1, + Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ Null: -1, @@ -162,8 +158,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -194,8 +189,7 @@ create table foo ( }, }, GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_cluster.sql.txt b/testdata/result/ddl/create_table_cluster.sql.txt index e5db3e43..b9aabc0f 100644 --- a/testdata/result/ddl/create_table_cluster.sql.txt +++ b/testdata/result/ddl/create_table_cluster.sql.txt @@ -30,8 +30,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -48,8 +47,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_cluster_and_row_deletion_policy.sql.txt b/testdata/result/ddl/create_table_cluster_and_row_deletion_policy.sql.txt index 0953ba37..24e4d575 100644 --- a/testdata/result/ddl/create_table_cluster_and_row_deletion_policy.sql.txt +++ b/testdata/result/ddl/create_table_cluster_and_row_deletion_policy.sql.txt @@ -32,8 +32,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -50,8 +49,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -68,8 +66,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_cluster_on_delete_no_action.sql.txt b/testdata/result/ddl/create_table_cluster_on_delete_no_action.sql.txt index d2b99719..f255195b 100644 --- a/testdata/result/ddl/create_table_cluster_on_delete_no_action.sql.txt +++ b/testdata/result/ddl/create_table_cluster_on_delete_no_action.sql.txt @@ -29,8 +29,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_cluster_set_on_delete.sql.txt b/testdata/result/ddl/create_table_cluster_set_on_delete.sql.txt index b05a71e3..4fcfa8af 100644 --- a/testdata/result/ddl/create_table_cluster_set_on_delete.sql.txt +++ b/testdata/result/ddl/create_table_cluster_set_on_delete.sql.txt @@ -29,8 +29,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_fulltext_albums.sql.txt b/testdata/result/ddl/create_table_fulltext_albums.sql.txt index 5f170208..0522a699 100644 --- a/testdata/result/ddl/create_table_fulltext_albums.sql.txt +++ b/testdata/result/ddl/create_table_fulltext_albums.sql.txt @@ -37,8 +37,7 @@ CREATE TABLE Albums ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -55,8 +54,7 @@ CREATE TABLE Albums ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -73,8 +71,7 @@ CREATE TABLE Albums ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -94,8 +91,7 @@ CREATE TABLE Albums ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -112,8 +108,7 @@ CREATE TABLE Albums ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -151,11 +146,11 @@ CREATE TABLE Albums ( }, }, }, + NamedArgs: []*ast.NamedArg(nil), }, }, - Hidden: 462, - IsHidden: true, - Options: (*ast.ColumnDefOptions)(nil), + Hidden: 462, + Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ Null: -1, @@ -192,11 +187,11 @@ CREATE TABLE Albums ( }, }, }, + NamedArgs: []*ast.NamedArg(nil), }, }, - Hidden: 547, - IsHidden: true, - Options: (*ast.ColumnDefOptions)(nil), + Hidden: 547, + Options: (*ast.ColumnDefOptions)(nil), }, }, TableConstraints: []*ast.TableConstraint(nil), diff --git a/testdata/result/ddl/create_table_if_not_exists.sql.txt b/testdata/result/ddl/create_table_if_not_exists.sql.txt index 07d0f784..418af068 100644 --- a/testdata/result/ddl/create_table_if_not_exists.sql.txt +++ b/testdata/result/ddl/create_table_if_not_exists.sql.txt @@ -29,8 +29,7 @@ create table if not exists foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -47,8 +46,7 @@ create table if not exists foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_row_deletion_policy.sql.txt b/testdata/result/ddl/create_table_row_deletion_policy.sql.txt index 4223209f..8ec6eb63 100644 --- a/testdata/result/ddl/create_table_row_deletion_policy.sql.txt +++ b/testdata/result/ddl/create_table_row_deletion_policy.sql.txt @@ -31,8 +31,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -49,8 +48,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -67,8 +65,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_trailing_comma.sql.txt b/testdata/result/ddl/create_table_trailing_comma.sql.txt index 13a9b167..4c897f98 100644 --- a/testdata/result/ddl/create_table_trailing_comma.sql.txt +++ b/testdata/result/ddl/create_table_trailing_comma.sql.txt @@ -32,8 +32,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -50,8 +49,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_types.sql.txt b/testdata/result/ddl/create_table_types.sql.txt index f8f793b8..b680afd7 100644 --- a/testdata/result/ddl/create_table_types.sql.txt +++ b/testdata/result/ddl/create_table_types.sql.txt @@ -40,8 +40,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -58,8 +57,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -76,8 +74,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -94,8 +91,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -112,8 +108,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -130,8 +125,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -156,8 +150,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -177,8 +170,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -203,8 +195,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -224,8 +215,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -246,8 +236,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -271,8 +260,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -299,6 +287,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/ddl/create_table_with_sequence_function.sql.txt b/testdata/result/ddl/create_table_with_sequence_function.sql.txt index 0469da83..9ba17f0e 100644 --- a/testdata/result/ddl/create_table_with_sequence_function.sql.txt +++ b/testdata/result/ddl/create_table_with_sequence_function.sql.txt @@ -54,8 +54,7 @@ CREATE TABLE foo }, }, GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -75,8 +74,7 @@ CREATE TABLE foo NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -93,8 +91,7 @@ CREATE TABLE foo NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/alter_table_add_column.sql.txt b/testdata/result/statement/alter_table_add_column.sql.txt index 739db75f..facb64c1 100644 --- a/testdata/result/statement/alter_table_add_column.sql.txt +++ b/testdata/result/statement/alter_table_add_column.sql.txt @@ -28,8 +28,7 @@ alter table foo add column baz string(max) not null NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/alter_table_add_column_if_not_exists.sql.txt b/testdata/result/statement/alter_table_add_column_if_not_exists.sql.txt index 45a65bcc..407ca7dc 100644 --- a/testdata/result/statement/alter_table_add_column_if_not_exists.sql.txt +++ b/testdata/result/statement/alter_table_add_column_if_not_exists.sql.txt @@ -29,8 +29,7 @@ alter table foo add column if not exists baz string(max) not null NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/alter_table_add_column_with_if_expression.sql.txt b/testdata/result/statement/alter_table_add_column_with_if_expression.sql.txt index db18fc62..472d04e2 100644 --- a/testdata/result/statement/alter_table_add_column_with_if_expression.sql.txt +++ b/testdata/result/statement/alter_table_add_column_with_if_expression.sql.txt @@ -113,9 +113,8 @@ ALTER TABLE foo ADD COLUMN expired_at TIMESTAMP AS (IF (status != "OPEN" AND sta NamedArgs: []*ast.NamedArg(nil), }, }, - Hidden: 0, - IsHidden: false, - Options: (*ast.ColumnDefOptions)(nil), + Hidden: -1, + Options: (*ast.ColumnDefOptions)(nil), }, }, } diff --git a/testdata/result/statement/create_table.sql.txt b/testdata/result/statement/create_table.sql.txt index 2d3b6d55..ffc2b1a1 100644 --- a/testdata/result/statement/create_table.sql.txt +++ b/testdata/result/statement/create_table.sql.txt @@ -39,8 +39,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -57,8 +56,7 @@ create table foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -83,8 +81,7 @@ create table foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: &ast.ColumnDefOptions{ Options: 83, Rparen: 121, @@ -144,9 +141,8 @@ create table foo ( NamedArgs: []*ast.NamedArg(nil), }, }, - Hidden: 0, - IsHidden: false, - Options: (*ast.ColumnDefOptions)(nil), + Hidden: -1, + Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ Null: -1, @@ -162,8 +158,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -194,8 +189,7 @@ create table foo ( }, }, GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_cluster.sql.txt b/testdata/result/statement/create_table_cluster.sql.txt index e5db3e43..b9aabc0f 100644 --- a/testdata/result/statement/create_table_cluster.sql.txt +++ b/testdata/result/statement/create_table_cluster.sql.txt @@ -30,8 +30,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -48,8 +47,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_cluster_and_row_deletion_policy.sql.txt b/testdata/result/statement/create_table_cluster_and_row_deletion_policy.sql.txt index 0953ba37..24e4d575 100644 --- a/testdata/result/statement/create_table_cluster_and_row_deletion_policy.sql.txt +++ b/testdata/result/statement/create_table_cluster_and_row_deletion_policy.sql.txt @@ -32,8 +32,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -50,8 +49,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -68,8 +66,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_cluster_on_delete_no_action.sql.txt b/testdata/result/statement/create_table_cluster_on_delete_no_action.sql.txt index d2b99719..f255195b 100644 --- a/testdata/result/statement/create_table_cluster_on_delete_no_action.sql.txt +++ b/testdata/result/statement/create_table_cluster_on_delete_no_action.sql.txt @@ -29,8 +29,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_cluster_set_on_delete.sql.txt b/testdata/result/statement/create_table_cluster_set_on_delete.sql.txt index b05a71e3..4fcfa8af 100644 --- a/testdata/result/statement/create_table_cluster_set_on_delete.sql.txt +++ b/testdata/result/statement/create_table_cluster_set_on_delete.sql.txt @@ -29,8 +29,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_fulltext_albums.sql.txt b/testdata/result/statement/create_table_fulltext_albums.sql.txt index 5f170208..0522a699 100644 --- a/testdata/result/statement/create_table_fulltext_albums.sql.txt +++ b/testdata/result/statement/create_table_fulltext_albums.sql.txt @@ -37,8 +37,7 @@ CREATE TABLE Albums ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -55,8 +54,7 @@ CREATE TABLE Albums ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -73,8 +71,7 @@ CREATE TABLE Albums ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -94,8 +91,7 @@ CREATE TABLE Albums ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -112,8 +108,7 @@ CREATE TABLE Albums ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -151,11 +146,11 @@ CREATE TABLE Albums ( }, }, }, + NamedArgs: []*ast.NamedArg(nil), }, }, - Hidden: 462, - IsHidden: true, - Options: (*ast.ColumnDefOptions)(nil), + Hidden: 462, + Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ Null: -1, @@ -192,11 +187,11 @@ CREATE TABLE Albums ( }, }, }, + NamedArgs: []*ast.NamedArg(nil), }, }, - Hidden: 547, - IsHidden: true, - Options: (*ast.ColumnDefOptions)(nil), + Hidden: 547, + Options: (*ast.ColumnDefOptions)(nil), }, }, TableConstraints: []*ast.TableConstraint(nil), diff --git a/testdata/result/statement/create_table_if_not_exists.sql.txt b/testdata/result/statement/create_table_if_not_exists.sql.txt index 07d0f784..418af068 100644 --- a/testdata/result/statement/create_table_if_not_exists.sql.txt +++ b/testdata/result/statement/create_table_if_not_exists.sql.txt @@ -29,8 +29,7 @@ create table if not exists foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -47,8 +46,7 @@ create table if not exists foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_row_deletion_policy.sql.txt b/testdata/result/statement/create_table_row_deletion_policy.sql.txt index 4223209f..8ec6eb63 100644 --- a/testdata/result/statement/create_table_row_deletion_policy.sql.txt +++ b/testdata/result/statement/create_table_row_deletion_policy.sql.txt @@ -31,8 +31,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -49,8 +48,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -67,8 +65,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_trailing_comma.sql.txt b/testdata/result/statement/create_table_trailing_comma.sql.txt index 13a9b167..4c897f98 100644 --- a/testdata/result/statement/create_table_trailing_comma.sql.txt +++ b/testdata/result/statement/create_table_trailing_comma.sql.txt @@ -32,8 +32,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -50,8 +49,7 @@ create table foo ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_types.sql.txt b/testdata/result/statement/create_table_types.sql.txt index f8f793b8..b680afd7 100644 --- a/testdata/result/statement/create_table_types.sql.txt +++ b/testdata/result/statement/create_table_types.sql.txt @@ -40,8 +40,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -58,8 +57,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -76,8 +74,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -94,8 +91,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -112,8 +108,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -130,8 +125,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -156,8 +150,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -177,8 +170,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -203,8 +195,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -224,8 +215,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -246,8 +236,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -271,8 +260,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -299,6 +287,7 @@ create table types ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, diff --git a/testdata/result/statement/create_table_with_sequence_function.sql.txt b/testdata/result/statement/create_table_with_sequence_function.sql.txt index 0469da83..9ba17f0e 100644 --- a/testdata/result/statement/create_table_with_sequence_function.sql.txt +++ b/testdata/result/statement/create_table_with_sequence_function.sql.txt @@ -54,8 +54,7 @@ CREATE TABLE foo }, }, GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -75,8 +74,7 @@ CREATE TABLE foo NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, &ast.ColumnDef{ @@ -93,8 +91,7 @@ CREATE TABLE foo NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), - Hidden: 0, - IsHidden: false, + Hidden: -1, Options: (*ast.ColumnDefOptions)(nil), }, }, From 758b42281ba949517c0e5f928b876e26f9d045ba Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Tue, 15 Oct 2024 23:31:55 +0900 Subject: [PATCH 11/35] Fix ast of search index DDLs --- ast/ast.go | 74 +++++++++--------------------------------------------- ast/pos.go | 35 +++++++++++++++++++++++--- ast/sql.go | 26 +++++++++++++++++++ 3 files changed, 69 insertions(+), 66 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 5eff1a53..5be61448 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -74,6 +74,9 @@ func (Revoke) isStatement() {} func (CreateChangeStream) isStatement() {} func (AlterChangeStream) isStatement() {} func (DropChangeStream) isStatement() {} +func (CreateSearchIndex) isStatement() {} +func (DropSearchIndex) isStatement() {} +func (AlterSearchIndex) isStatement() {} // QueryExpr represents set operator operands. type QueryExpr interface { @@ -245,6 +248,9 @@ func (Revoke) isDDL() {} func (CreateChangeStream) isDDL() {} func (AlterChangeStream) isDDL() {} func (DropChangeStream) isDDL() {} +func (CreateSearchIndex) isDDL() {} +func (DropSearchIndex) isDDL() {} +func (AlterSearchIndex) isDDL() {} // Constraint represents table constraint of CONSTARINT clause. type Constraint interface { @@ -1506,7 +1512,7 @@ type CreateSequence struct { // {{.Type | sql}} {{if .NotNull}}NOT NULL{{end}} // {{.DefaultExpr | sqlOpt}} // {{.GeneratedExpr | sqlOpt}} -// {{if .Hidden}}HIDDEN{{end}} +// {{if not(.Hidden.Invalid())}}HIDDEN{{end}} // {{.Options | sqlOpt}} type ColumnDef struct { // pos = Name.pos @@ -2347,7 +2353,7 @@ type ArraySchemaType struct { // // ================================================================================ -// CreateSearchIndex +// CreateSearchIndex represents CREATE SEARCH INDEX statement // // CREATE SEARCH INDEX {{.Name | sql}} // ON {{.TableName | sql}} @@ -2372,36 +2378,11 @@ type CreateSearchIndex struct { PartitionColumns []*Ident // optional OrderBy *OrderBy // optional Where *Where // optional - Interleave *InterleaveIn //optional + Interleave *InterleaveIn // optional Options *GenericOptions // optional } -func (c *CreateSearchIndex) Pos() token.Pos { - return c.Create -} - -func (c *CreateSearchIndex) End() token.Pos { - if e := firstValidEnd(c.Options, c.Interleave, c.Where, c.OrderBy, lastNode(c.PartitionColumns), c.Storing); e != token.InvalidPos { - return e - } - return c.Rparen + 1 -} - -func (c *CreateSearchIndex) SQL() string { - return "CREATE SEARCH INDEX " + c.Name.SQL() + " ON " + c.TableName.SQL() + - "(" + sqlJoin(c.TokenListPart, ", ") + ")" + - sqlOpt(" ", c.Storing, "") + - strOpt(len(c.PartitionColumns) > 0, " PARTITION BY "+sqlJoin(c.PartitionColumns, ", ")) + - sqlOpt(" ", c.OrderBy, "") + - sqlOpt(" ", c.Where, "") + - sqlOpt(" ", c.Interleave, "") + - sqlOpt(" ", c.Options, "") -} - -func (CreateSearchIndex) isStatement() {} -func (CreateSearchIndex) isDDL() {} - -// DropSearchIndex +// DropSearchIndex represents DROP SEARCH INDEX statement. // // DROP SEARCH INDEX {{.IfExists | strOpt "IF EXISTS "}}{{Name | sql}} type DropSearchIndex struct { @@ -2413,24 +2394,9 @@ type DropSearchIndex struct { Name *Ident } -func (d *DropSearchIndex) Pos() token.Pos { - return d.Drop -} - -func (d *DropSearchIndex) End() token.Pos { - return d.Name.End() -} - -func (d *DropSearchIndex) SQL() string { - return "DROP SEARCH INDEX " + strOpt(d.IfExists, "IF EXISTS ") + d.Name.SQL() -} - -func (DropSearchIndex) isStatement() {} -func (DropSearchIndex) isDDL() {} - -// AlterSearchIndex +// AlterSearchIndex represents ALTER SEARCH INDEX statement. // -// ALTER INDEX {{.Name | sql}} {{.IndexAlteration | sql}} +// ALTER SEARCH INDEX {{.Name | sql}} {{.IndexAlteration | sql}} type AlterSearchIndex struct { // pos = Alter // end = IndexAlteration.end @@ -2440,22 +2406,6 @@ type AlterSearchIndex struct { IndexAlteration IndexAlteration } -func (AlterSearchIndex) isStatement() {} - -func (AlterSearchIndex) isDDL() {} - -func (a *AlterSearchIndex) Pos() token.Pos { - return a.Alter -} - -func (a *AlterSearchIndex) End() token.Pos { - return a.IndexAlteration.End() -} - -func (a *AlterSearchIndex) SQL() string { - return "ALTER INDEX " + a.Name.SQL() + " " + a.IndexAlteration.SQL() -} - // ================================================================================ // // DML diff --git a/ast/pos.go b/ast/pos.go index 7908b860..5a0a6f0c 100644 --- a/ast/pos.go +++ b/ast/pos.go @@ -552,14 +552,15 @@ func (c *ColumnDef) Pos() token.Pos { } func (c *ColumnDef) End() token.Pos { + // TODO: It may able to be refactored using Pos arithmetic like InvalidPos + n = InvalidPos. if c.Options != nil { return c.Options.End() } - if c.GeneratedExpr != nil { - return c.GeneratedExpr.End() + if !c.Hidden.Invalid() { + return c.Hidden + 6 } - if c.DefaultExpr != nil { - return c.DefaultExpr.End() + if end := firstValidEnd(c.GeneratedExpr, c.DefaultExpr); !end.Invalid() { + return end } if !c.Null.Invalid() { return c.Null + 4 @@ -859,6 +860,32 @@ func (c *ChangeStreamForTable) End() token.Pos { return c.Rparen + 1 } +// ================================================================================ +// +// Search Index DDL +// +// ================================================================================ + +func (c *CreateSearchIndex) Pos() token.Pos { return c.Create } + +func (c *CreateSearchIndex) End() token.Pos { + if end := firstValidEnd(c.Options, c.Interleave, c.Where, c.OrderBy, lastNode(c.PartitionColumns), c.Storing); end != token.InvalidPos { + return end + } + return c.Rparen + 1 +} + +func (d *DropSearchIndex) Pos() token.Pos { + return d.Drop +} + +func (d *DropSearchIndex) End() token.Pos { + return d.Name.End() +} + +func (a *AlterSearchIndex) Pos() token.Pos { return a.Alter } +func (a *AlterSearchIndex) End() token.Pos { return a.IndexAlteration.End() } + // ================================================================================ // // DML diff --git a/ast/sql.go b/ast/sql.go index 19f471c0..6850168c 100644 --- a/ast/sql.go +++ b/ast/sql.go @@ -1300,6 +1300,32 @@ func (a *ArraySchemaType) SQL() string { return "ARRAY<" + a.Item.SQL() + ">" } +// ================================================================================ +// +// Search Index DDL +// +// ================================================================================ + +func (c *CreateSearchIndex) SQL() string { + return "CREATE SEARCH INDEX " + c.Name.SQL() + " ON " + c.TableName.SQL() + + "(" + sqlJoin(c.TokenListPart, ", ") + ")" + + sqlOpt(" ", c.Storing, "") + + strOpt(len(c.PartitionColumns) > 0, " PARTITION BY "+sqlJoin(c.PartitionColumns, ", ")) + + sqlOpt(" ", c.OrderBy, "") + + sqlOpt(" ", c.Where, "") + + sqlOpt(" ", c.Interleave, "") + + sqlOpt(" ", c.Options, "") +} + + +func (d *DropSearchIndex) SQL() string { + return "DROP SEARCH INDEX " + strOpt(d.IfExists, "IF EXISTS ") + d.Name.SQL() +} + +func (a *AlterSearchIndex) SQL() string { + return "ALTER INDEX " + a.Name.SQL() + " " + a.IndexAlteration.SQL() +} + // ================================================================================ // // DML From 48751bfaa0d064b5521ea83761d7a4feb8a3a720 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 16 Oct 2024 00:58:02 +0900 Subject: [PATCH 12/35] Fix some SQL() --- ast/ast.go | 2 +- ast/sql.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 5be61448..64219711 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -2562,7 +2562,7 @@ func (g *GenericOptions) End() token.Pos { } func (g *GenericOptions) SQL() string { - return "OPTIONS " + sqlJoin(g.Records, ", ") + return "OPTIONS (" + sqlJoin(g.Records, ", ") + ")" } // GenericOption is generic option for CREATE statements. diff --git a/ast/sql.go b/ast/sql.go index 6850168c..05c806c3 100644 --- a/ast/sql.go +++ b/ast/sql.go @@ -1313,11 +1313,10 @@ func (c *CreateSearchIndex) SQL() string { strOpt(len(c.PartitionColumns) > 0, " PARTITION BY "+sqlJoin(c.PartitionColumns, ", ")) + sqlOpt(" ", c.OrderBy, "") + sqlOpt(" ", c.Where, "") + - sqlOpt(" ", c.Interleave, "") + + sqlOpt("", c.Interleave, "") + sqlOpt(" ", c.Options, "") } - func (d *DropSearchIndex) SQL() string { return "DROP SEARCH INDEX " + strOpt(d.IfExists, "IF EXISTS ") + d.Name.SQL() } From f1c69e88bbd452032da68b78ad3fb71ceb26fc33 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 16 Oct 2024 00:58:31 +0900 Subject: [PATCH 13/35] Add create_search_index_full.sql --- .../input/ddl/create_search_index_full.sql | 9 ++ .../ddl/create_search_index_full.sql.txt | 122 ++++++++++++++++++ .../create_search_index_full.sql.txt | 122 ++++++++++++++++++ 3 files changed, 253 insertions(+) create mode 100644 testdata/input/ddl/create_search_index_full.sql create mode 100644 testdata/result/ddl/create_search_index_full.sql.txt create mode 100644 testdata/result/statement/create_search_index_full.sql.txt diff --git a/testdata/input/ddl/create_search_index_full.sql b/testdata/input/ddl/create_search_index_full.sql new file mode 100644 index 00000000..db67d674 --- /dev/null +++ b/testdata/input/ddl/create_search_index_full.sql @@ -0,0 +1,9 @@ +-- Should +CREATE SEARCH INDEX AlbumsIndexFull +ON Albums(Title_Tokens, Studio_Tokens) +STORING(Genre) +PARTITION BY SingerId +ORDER BY ReleaseTimestamp DESC +WHERE Genre IS NOT NULL AND ReleaseTimestamp IS NOT NULL +, INTERLEAVE IN Singers +OPTIONS(sort_order_sharding=true) \ No newline at end of file diff --git a/testdata/result/ddl/create_search_index_full.sql.txt b/testdata/result/ddl/create_search_index_full.sql.txt new file mode 100644 index 00000000..767fc744 --- /dev/null +++ b/testdata/result/ddl/create_search_index_full.sql.txt @@ -0,0 +1,122 @@ +--- create_search_index_full.sql +-- Should +CREATE SEARCH INDEX AlbumsIndexFull +ON Albums(Title_Tokens, Studio_Tokens) +STORING(Genre) +PARTITION BY SingerId +ORDER BY ReleaseTimestamp DESC +WHERE Genre IS NOT NULL AND ReleaseTimestamp IS NOT NULL +, INTERLEAVE IN Singers +OPTIONS(sort_order_sharding=true) +--- AST +&ast.CreateSearchIndex{ + Create: 10, + Name: &ast.Ident{ + NamePos: 30, + NameEnd: 45, + Name: "AlbumsIndexFull", + }, + TableName: &ast.Ident{ + NamePos: 49, + NameEnd: 55, + Name: "Albums", + }, + TokenListPart: []*ast.Ident{ + &ast.Ident{ + NamePos: 56, + NameEnd: 68, + Name: "Title_Tokens", + }, + &ast.Ident{ + NamePos: 70, + NameEnd: 83, + Name: "Studio_Tokens", + }, + }, + Rparen: 83, + Storing: &ast.Storing{ + Storing: 85, + Rparen: 98, + Columns: []*ast.Ident{ + &ast.Ident{ + NamePos: 93, + NameEnd: 98, + Name: "Genre", + }, + }, + }, + PartitionColumns: []*ast.Ident{ + &ast.Ident{ + NamePos: 113, + NameEnd: 121, + Name: "SingerId", + }, + }, + OrderBy: &ast.OrderBy{ + Order: 122, + Items: []*ast.OrderByItem{ + &ast.OrderByItem{ + DirPos: 148, + Expr: &ast.Ident{ + NamePos: 131, + NameEnd: 147, + Name: "ReleaseTimestamp", + }, + Collate: (*ast.Collate)(nil), + Dir: "DESC", + }, + }, + }, + Where: &ast.Where{ + Where: 153, + Expr: &ast.BinaryExpr{ + Op: "AND", + Left: &ast.IsNullExpr{ + Null: 172, + Not: true, + Left: &ast.Ident{ + NamePos: 159, + NameEnd: 164, + Name: "Genre", + }, + }, + Right: &ast.IsNullExpr{ + Null: 205, + Not: true, + Left: &ast.Ident{ + NamePos: 181, + NameEnd: 197, + Name: "ReleaseTimestamp", + }, + }, + }, + }, + Interleave: &ast.InterleaveIn{ + Comma: 210, + TableName: &ast.Ident{ + NamePos: 226, + NameEnd: 233, + Name: "Singers", + }, + }, + Options: &ast.GenericOptions{ + Options: 234, + Rparen: 266, + Records: []*ast.GenericOption{ + &ast.GenericOption{ + Name: &ast.Ident{ + NamePos: 242, + NameEnd: 261, + Name: "sort_order_sharding", + }, + Value: &ast.BoolLiteral{ + ValuePos: 262, + Value: true, + }, + }, + }, + }, +} + +--- SQL +CREATE SEARCH INDEX AlbumsIndexFull ON Albums(Title_Tokens, Studio_Tokens) STORING (Genre) PARTITION BY SingerId ORDER BY ReleaseTimestamp DESC WHERE Genre IS NOT NULL AND ReleaseTimestamp IS NOT NULL, INTERLEAVE IN Singers OPTIONS (sort_order_sharding = TRUE) diff --git a/testdata/result/statement/create_search_index_full.sql.txt b/testdata/result/statement/create_search_index_full.sql.txt new file mode 100644 index 00000000..767fc744 --- /dev/null +++ b/testdata/result/statement/create_search_index_full.sql.txt @@ -0,0 +1,122 @@ +--- create_search_index_full.sql +-- Should +CREATE SEARCH INDEX AlbumsIndexFull +ON Albums(Title_Tokens, Studio_Tokens) +STORING(Genre) +PARTITION BY SingerId +ORDER BY ReleaseTimestamp DESC +WHERE Genre IS NOT NULL AND ReleaseTimestamp IS NOT NULL +, INTERLEAVE IN Singers +OPTIONS(sort_order_sharding=true) +--- AST +&ast.CreateSearchIndex{ + Create: 10, + Name: &ast.Ident{ + NamePos: 30, + NameEnd: 45, + Name: "AlbumsIndexFull", + }, + TableName: &ast.Ident{ + NamePos: 49, + NameEnd: 55, + Name: "Albums", + }, + TokenListPart: []*ast.Ident{ + &ast.Ident{ + NamePos: 56, + NameEnd: 68, + Name: "Title_Tokens", + }, + &ast.Ident{ + NamePos: 70, + NameEnd: 83, + Name: "Studio_Tokens", + }, + }, + Rparen: 83, + Storing: &ast.Storing{ + Storing: 85, + Rparen: 98, + Columns: []*ast.Ident{ + &ast.Ident{ + NamePos: 93, + NameEnd: 98, + Name: "Genre", + }, + }, + }, + PartitionColumns: []*ast.Ident{ + &ast.Ident{ + NamePos: 113, + NameEnd: 121, + Name: "SingerId", + }, + }, + OrderBy: &ast.OrderBy{ + Order: 122, + Items: []*ast.OrderByItem{ + &ast.OrderByItem{ + DirPos: 148, + Expr: &ast.Ident{ + NamePos: 131, + NameEnd: 147, + Name: "ReleaseTimestamp", + }, + Collate: (*ast.Collate)(nil), + Dir: "DESC", + }, + }, + }, + Where: &ast.Where{ + Where: 153, + Expr: &ast.BinaryExpr{ + Op: "AND", + Left: &ast.IsNullExpr{ + Null: 172, + Not: true, + Left: &ast.Ident{ + NamePos: 159, + NameEnd: 164, + Name: "Genre", + }, + }, + Right: &ast.IsNullExpr{ + Null: 205, + Not: true, + Left: &ast.Ident{ + NamePos: 181, + NameEnd: 197, + Name: "ReleaseTimestamp", + }, + }, + }, + }, + Interleave: &ast.InterleaveIn{ + Comma: 210, + TableName: &ast.Ident{ + NamePos: 226, + NameEnd: 233, + Name: "Singers", + }, + }, + Options: &ast.GenericOptions{ + Options: 234, + Rparen: 266, + Records: []*ast.GenericOption{ + &ast.GenericOption{ + Name: &ast.Ident{ + NamePos: 242, + NameEnd: 261, + Name: "sort_order_sharding", + }, + Value: &ast.BoolLiteral{ + ValuePos: 262, + Value: true, + }, + }, + }, + }, +} + +--- SQL +CREATE SEARCH INDEX AlbumsIndexFull ON Albums(Title_Tokens, Studio_Tokens) STORING (Genre) PARTITION BY SingerId ORDER BY ReleaseTimestamp DESC WHERE Genre IS NOT NULL AND ReleaseTimestamp IS NOT NULL, INTERLEAVE IN Singers OPTIONS (sort_order_sharding = TRUE) From 51314c42f0f4ba80d6aa7621e5612140654461ed Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 16 Oct 2024 01:57:41 +0900 Subject: [PATCH 14/35] Implement SearchIndexOptions on GenericOptions --- ast/ast.go | 84 +++++++++++++------ ast/ast_test.go | 82 ++++++++++++++++++ parser.go | 8 +- .../ddl/create_search_index_full.sql.txt | 2 +- .../create_search_index_null_filtered.sql.txt | 2 +- .../ddl/create_search_index_simple.sql.txt | 2 +- .../create_search_index_full.sql.txt | 2 +- .../create_search_index_null_filtered.sql.txt | 2 +- .../create_search_index_simple.sql.txt | 2 +- 9 files changed, 154 insertions(+), 32 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 64219711..49acaec4 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -32,6 +32,7 @@ package ast import ( + "fmt" "github.com/cloudspannerecosystem/memefish/token" ) @@ -2373,13 +2374,36 @@ type CreateSearchIndex struct { Name *Ident TableName *Ident TokenListPart []*Ident - Rparen token.Pos // position of ")" after TokenListPart - Storing *Storing // optional - PartitionColumns []*Ident // optional - OrderBy *OrderBy // optional - Where *Where // optional - Interleave *InterleaveIn // optional - Options *GenericOptions // optional + Rparen token.Pos // position of ")" after TokenListPart + Storing *Storing // optional + PartitionColumns []*Ident // optional + OrderBy *OrderBy // optional + Where *Where // optional + Interleave *InterleaveIn // optional + Options *SearchIndexOptions // optional +} + +// SearchIndexOptions represents OPTIONS for CREATE SEARCH INDEX statement. +type SearchIndexOptions GenericOptions + +func (o *SearchIndexOptions) Pos() token.Pos { + return (*GenericOptions)(o).Pos() +} + +func (o *SearchIndexOptions) End() token.Pos { + return (*GenericOptions)(o).End() +} + +func (o *SearchIndexOptions) SQL() string { + return (*GenericOptions)(o).SQL() +} + +func (o *SearchIndexOptions) SortOrderSharding() (*bool, error) { + return (*GenericOptions)(o).GetBool("sort_order_sharding") +} + +func (o *SearchIndexOptions) DisableAutomaticUIDColumn() (*bool, error) { + return (*GenericOptions)(o).GetBool("disable_automatic_uid_column") } // DropSearchIndex represents DROP SEARCH INDEX statement. @@ -2553,17 +2577,10 @@ type GenericOptions struct { Records []*GenericOption // len(Records) > 0 } -func (g *GenericOptions) Pos() token.Pos { - return g.Options -} +func (g *GenericOptions) Pos() token.Pos { return g.Options } +func (g *GenericOptions) End() token.Pos { return g.Rparen + 1 } -func (g *GenericOptions) End() token.Pos { - return g.Rparen + 1 -} - -func (g *GenericOptions) SQL() string { - return "OPTIONS (" + sqlJoin(g.Records, ", ") + ")" -} +func (g *GenericOptions) SQL() string { return "OPTIONS (" + sqlJoin(g.Records, ", ") + ")" } // GenericOption is generic option for CREATE statements. // @@ -2576,14 +2593,33 @@ type GenericOption struct { Value Expr } -func (g *GenericOption) Pos() token.Pos { - return g.Name.Pos() +func (o *GenericOptions) FindName(name string) (Expr, bool) { + for _, r := range o.Records { + if r.Name.Name != name { + continue + } + return r.Value, true + } + return nil, false } -func (g *GenericOption) End() token.Pos { - return g.Value.End() +func (o *GenericOptions) GetBool(name string) (*bool, error) { + v, ok := o.FindName(name) + if !ok { + return nil, nil + } + switch v := v.(type) { + case *BoolLiteral: + return &v.Value, nil + case *NullLiteral: + return nil, nil + default: + return nil, fmt.Errorf("expect bool or null, but have unknown type %T", v) + } } -func (g *GenericOption) SQL() string { - return g.Name.SQL() + " = " + g.Value.SQL() -} +func (g *GenericOption) Pos() token.Pos { return g.Name.Pos() } + +func (g *GenericOption) End() token.Pos { return g.Value.End() } + +func (g *GenericOption) SQL() string { return g.Name.SQL() + " = " + g.Value.SQL() } diff --git a/ast/ast_test.go b/ast/ast_test.go index e3ab0b70..5fa96bdd 100644 --- a/ast/ast_test.go +++ b/ast/ast_test.go @@ -1,6 +1,7 @@ package ast import ( + "github.com/google/go-cmp/cmp" "testing" ) @@ -192,3 +193,84 @@ func TestInsertInput(t *testing.T) { InsertInput(&ValuesInput{}).isInsertInput() InsertInput(&SubQueryInput{}).isInsertInput() } + +func ptr[T any](v T) *T { + return &v +} + +func TestGenericOptionsGetBool(t *testing.T) { + tt := []struct { + desc string + input *GenericOptions + name string + want *bool + wantErr string + }{ + { + desc: "true", + input: &GenericOptions{ + Records: []*GenericOption{ + {Name: &Ident{Name: "sort_order_sharding"}, Value: &BoolLiteral{Value: true}}, + }, + }, + name: "sort_order_sharding", + want: ptr(true), + }, + { + desc: "false", + input: &GenericOptions{ + Records: []*GenericOption{ + {Name: &Ident{Name: "sort_order_sharding"}, Value: &BoolLiteral{Value: false}}, + }, + }, + name: "sort_order_sharding", + want: ptr(false), + }, + { + desc: "explicit null", + input: &GenericOptions{ + Records: []*GenericOption{ + {Name: &Ident{Name: "sort_order_sharding"}, Value: &NullLiteral{}}, + }, + }, + name: "sort_order_sharding", + want: nil, + }, + { + desc: "implicit null", + input: &GenericOptions{ + Records: []*GenericOption{ + {Name: &Ident{Name: "disable_automatic_uid_column"}, Value: &BoolLiteral{Value: true}}, + }, + }, + name: "sort_order_sharding", + want: nil, + }, + { + desc: "invalid value", + input: &GenericOptions{ + Records: []*GenericOption{ + {Name: &Ident{Name: "sort_order_sharding"}, Value: &StringLiteral{Value: "foo"}}, + }, + }, + name: "sort_order_sharding", + wantErr: "expect bool or null, but have unknown type *ast.StringLiteral", + }, + } + + for _, tcase := range tt { + t.Run(tcase.desc, func(t *testing.T) { + b, err := tcase.input.GetBool(tcase.name) + if tcase.wantErr == "" && err != nil { + t.Errorf("should not fail, but: %v", err) + } + if tcase.wantErr != "" && err.Error() != tcase.wantErr { + t.Errorf("should fail, want: %v, got: %v", tcase.wantErr, err) + } + if diff := cmp.Diff(tcase.want, b); diff != "" { + t.Errorf("differ: %v", diff) + } + }) + } + +} diff --git a/parser.go b/parser.go index 9900fcef..e4e051e3 100644 --- a/parser.go +++ b/parser.go @@ -2635,7 +2635,11 @@ func (p *Parser) parseCreateSearchIndex(pos token.Pos) *ast.CreateSearchIndex { orderBy := p.tryParseOrderBy() where := p.tryParseWhere() interleave := p.tryParseInterleaveIn() - options := p.tryParseGenericOptions() + + var searchIndexOptions *ast.SearchIndexOptions + if options := p.tryParseGenericOptions(); options != nil { + searchIndexOptions = (*ast.SearchIndexOptions)(options) + } return &ast.CreateSearchIndex{ Create: pos, @@ -2648,7 +2652,7 @@ func (p *Parser) parseCreateSearchIndex(pos token.Pos) *ast.CreateSearchIndex { OrderBy: orderBy, Where: where, Interleave: interleave, - Options: options, + Options: searchIndexOptions, } } diff --git a/testdata/result/ddl/create_search_index_full.sql.txt b/testdata/result/ddl/create_search_index_full.sql.txt index 767fc744..548040e8 100644 --- a/testdata/result/ddl/create_search_index_full.sql.txt +++ b/testdata/result/ddl/create_search_index_full.sql.txt @@ -99,7 +99,7 @@ OPTIONS(sort_order_sharding=true) Name: "Singers", }, }, - Options: &ast.GenericOptions{ + Options: &ast.SearchIndexOptions{ Options: 234, Rparen: 266, Records: []*ast.GenericOption{ diff --git a/testdata/result/ddl/create_search_index_null_filtered.sql.txt b/testdata/result/ddl/create_search_index_null_filtered.sql.txt index 56e116ce..bdb8bc53 100644 --- a/testdata/result/ddl/create_search_index_null_filtered.sql.txt +++ b/testdata/result/ddl/create_search_index_null_filtered.sql.txt @@ -50,7 +50,7 @@ WHERE Genre IS NOT NULL }, }, Interleave: (*ast.InterleaveIn)(nil), - Options: (*ast.GenericOptions)(nil), + Options: (*ast.SearchIndexOptions)(nil), } --- SQL diff --git a/testdata/result/ddl/create_search_index_simple.sql.txt b/testdata/result/ddl/create_search_index_simple.sql.txt index 5020feb6..b29adfe0 100644 --- a/testdata/result/ddl/create_search_index_simple.sql.txt +++ b/testdata/result/ddl/create_search_index_simple.sql.txt @@ -28,7 +28,7 @@ CREATE SEARCH INDEX AlbumsIndex OrderBy: (*ast.OrderBy)(nil), Where: (*ast.Where)(nil), Interleave: (*ast.InterleaveIn)(nil), - Options: (*ast.GenericOptions)(nil), + Options: (*ast.SearchIndexOptions)(nil), } --- SQL diff --git a/testdata/result/statement/create_search_index_full.sql.txt b/testdata/result/statement/create_search_index_full.sql.txt index 767fc744..548040e8 100644 --- a/testdata/result/statement/create_search_index_full.sql.txt +++ b/testdata/result/statement/create_search_index_full.sql.txt @@ -99,7 +99,7 @@ OPTIONS(sort_order_sharding=true) Name: "Singers", }, }, - Options: &ast.GenericOptions{ + Options: &ast.SearchIndexOptions{ Options: 234, Rparen: 266, Records: []*ast.GenericOption{ diff --git a/testdata/result/statement/create_search_index_null_filtered.sql.txt b/testdata/result/statement/create_search_index_null_filtered.sql.txt index 56e116ce..bdb8bc53 100644 --- a/testdata/result/statement/create_search_index_null_filtered.sql.txt +++ b/testdata/result/statement/create_search_index_null_filtered.sql.txt @@ -50,7 +50,7 @@ WHERE Genre IS NOT NULL }, }, Interleave: (*ast.InterleaveIn)(nil), - Options: (*ast.GenericOptions)(nil), + Options: (*ast.SearchIndexOptions)(nil), } --- SQL diff --git a/testdata/result/statement/create_search_index_simple.sql.txt b/testdata/result/statement/create_search_index_simple.sql.txt index 5020feb6..b29adfe0 100644 --- a/testdata/result/statement/create_search_index_simple.sql.txt +++ b/testdata/result/statement/create_search_index_simple.sql.txt @@ -28,7 +28,7 @@ CREATE SEARCH INDEX AlbumsIndex OrderBy: (*ast.OrderBy)(nil), Where: (*ast.Where)(nil), Interleave: (*ast.InterleaveIn)(nil), - Options: (*ast.GenericOptions)(nil), + Options: (*ast.SearchIndexOptions)(nil), } --- SQL From 161e1daecdb3f34a4dec271376a83ec6b85a6bc7 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 16 Oct 2024 02:00:38 +0900 Subject: [PATCH 15/35] Fix ast_test.go --- ast/ast_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ast/ast_test.go b/ast/ast_test.go index 5fa96bdd..bb7aaff8 100644 --- a/ast/ast_test.go +++ b/ast/ast_test.go @@ -199,7 +199,7 @@ func ptr[T any](v T) *T { } func TestGenericOptionsGetBool(t *testing.T) { - tt := []struct { + tcases := []struct { desc string input *GenericOptions name string @@ -258,16 +258,16 @@ func TestGenericOptionsGetBool(t *testing.T) { }, } - for _, tcase := range tt { + for _, tcase := range tcases { t.Run(tcase.desc, func(t *testing.T) { - b, err := tcase.input.GetBool(tcase.name) + got, err := tcase.input.GetBool(tcase.name) if tcase.wantErr == "" && err != nil { t.Errorf("should not fail, but: %v", err) } if tcase.wantErr != "" && err.Error() != tcase.wantErr { t.Errorf("should fail, want: %v, got: %v", tcase.wantErr, err) } - if diff := cmp.Diff(tcase.want, b); diff != "" { + if diff := cmp.Diff(tcase.want, got); diff != "" { t.Errorf("differ: %v", diff) } }) From fb67c7a6bf5ca8ee38c6646f376c2492c99385e9 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 16 Oct 2024 02:03:24 +0900 Subject: [PATCH 16/35] Update TestDDL --- ast/ast_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ast/ast_test.go b/ast/ast_test.go index bb7aaff8..b60d8b16 100644 --- a/ast/ast_test.go +++ b/ast/ast_test.go @@ -141,6 +141,9 @@ func TestDDL(t *testing.T) { DDL(&DropRole{}).isDDL() DDL(&Grant{}).isDDL() DDL(&Revoke{}).isDDL() + DDL(&CreateSearchIndex{}).isDDL() + DDL(&DropSearchIndex{}).isDDL() + DDL(&AlterSearchIndex{}).isDDL() } func TestConstraint(t *testing.T) { From 5a8477485c5b34cf0f025ab5814e386b90b9e91f Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 16 Oct 2024 02:08:31 +0900 Subject: [PATCH 17/35] Improve placement of GenericOptions/GenericOption --- ast/ast.go | 108 ++++++++++++++++++++++++----------------------------- ast/pos.go | 6 +++ ast/sql.go | 4 ++ 3 files changed, 59 insertions(+), 59 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 49acaec4..df59f2ab 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -1452,6 +1452,55 @@ type CastNumValue struct { // // ================================================================================ +// GenericOptions is generic OPTIONS clause node without key and value checking. +// +// OPTIONS ({{.Records | sqlJoin ","}}) +type GenericOptions struct { + // pos = Options + // end = Rparen + 1 + + Options token.Pos // position of "OPTIONS" keyword + Rparen token.Pos // position of ")" + + Records []*GenericOption // len(Records) > 0 +} + +func (o *GenericOptions) Find(name string) (Expr, bool) { + for _, r := range o.Records { + if r.Name.Name != name { + continue + } + return r.Value, true + } + return nil, false +} + +func (o *GenericOptions) GetBool(name string) (*bool, error) { + v, ok := o.Find(name) + if !ok { + return nil, nil + } + switch v := v.(type) { + case *BoolLiteral: + return &v.Value, nil + case *NullLiteral: + return nil, nil + default: + return nil, fmt.Errorf("expect bool or null, but have unknown type %T", v) + } +} + +// GenericOption is generic option for CREATE statements. +// +// {{.Name | sql}} = {{.Value | sql}} +type GenericOption struct { + // pos = Name.pos + // end = Value.end + + Name *Ident + Value Expr +} + // CreateDatabase is CREATE DATABASE statement node. // // CREATE DATABASE {{.Name | sql}} @@ -2564,62 +2613,3 @@ type SequenceOptions struct { Records []*SequenceOption // len(Records) > 0 } -// GenericOptions is generic OPTIONS clause node without key and value checking. -// -// OPTIONS ({{.Records | sqlJoin ","}}) -type GenericOptions struct { - // pos = Options - // end = Rparen + 1 - - Options token.Pos // position of "OPTIONS" keyword - Rparen token.Pos // position of ")" - - Records []*GenericOption // len(Records) > 0 -} - -func (g *GenericOptions) Pos() token.Pos { return g.Options } -func (g *GenericOptions) End() token.Pos { return g.Rparen + 1 } - -func (g *GenericOptions) SQL() string { return "OPTIONS (" + sqlJoin(g.Records, ", ") + ")" } - -// GenericOption is generic option for CREATE statements. -// -// {{.Name | sql}} = {{.Value | sql}} -type GenericOption struct { - // pos = Name.pos - // end = Value.end - - Name *Ident - Value Expr -} - -func (o *GenericOptions) FindName(name string) (Expr, bool) { - for _, r := range o.Records { - if r.Name.Name != name { - continue - } - return r.Value, true - } - return nil, false -} - -func (o *GenericOptions) GetBool(name string) (*bool, error) { - v, ok := o.FindName(name) - if !ok { - return nil, nil - } - switch v := v.(type) { - case *BoolLiteral: - return &v.Value, nil - case *NullLiteral: - return nil, nil - default: - return nil, fmt.Errorf("expect bool or null, but have unknown type %T", v) - } -} - -func (g *GenericOption) Pos() token.Pos { return g.Name.Pos() } - -func (g *GenericOption) End() token.Pos { return g.Value.End() } - -func (g *GenericOption) SQL() string { return g.Name.SQL() + " = " + g.Value.SQL() } diff --git a/ast/pos.go b/ast/pos.go index 5a0a6f0c..7bba2ee6 100644 --- a/ast/pos.go +++ b/ast/pos.go @@ -514,6 +514,12 @@ func (c *CastNumValue) End() token.Pos { return c.Rparen + 1 } // // ================================================================================ +func (g *GenericOptions) Pos() token.Pos { return g.Options } +func (g *GenericOptions) End() token.Pos { return g.Rparen + 1 } + +func (g *GenericOption) Pos() token.Pos { return g.Name.Pos() } +func (g *GenericOption) End() token.Pos { return g.Value.End() } + func (c *CreateDatabase) Pos() token.Pos { return c.Create } func (c *CreateDatabase) End() token.Pos { return c.Name.End() } diff --git a/ast/sql.go b/ast/sql.go index 05c806c3..a13296b1 100644 --- a/ast/sql.go +++ b/ast/sql.go @@ -762,6 +762,10 @@ func (c *CastNumValue) SQL() string { // // ================================================================================ +func (g *GenericOptions) SQL() string { return "OPTIONS (" + sqlJoin(g.Records, ", ") + ")" } + +func (g *GenericOption) SQL() string { return g.Name.SQL() + " = " + g.Value.SQL() } + func (c *CreateDatabase) SQL() string { return "CREATE DATABASE " + c.Name.SQL() } From c3f3a977be098f8ff5cbc527a544db8570129d71 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 16 Oct 2024 02:27:42 +0900 Subject: [PATCH 18/35] Refine Find/FindBool --- ast/ast.go | 13 ++++++++----- ast/ast_test.go | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index df59f2ab..7d613959 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -1465,7 +1465,9 @@ type GenericOptions struct { Records []*GenericOption // len(Records) > 0 } -func (o *GenericOptions) Find(name string) (Expr, bool) { +// Find finds name in Records, and return its value as Expr. +// The second return value indicates that the name was found in Records. +func (o *GenericOptions) Find(name string) (expr Expr, found bool) { for _, r := range o.Records { if r.Name.Name != name { continue @@ -1475,7 +1477,9 @@ func (o *GenericOptions) Find(name string) (Expr, bool) { return nil, false } -func (o *GenericOptions) GetBool(name string) (*bool, error) { +// FindBool finds name in Records, and return its value as *bool. +// If the value is neither bool nor null, it returns error. +func (o *GenericOptions) FindBool(name string) (*bool, error) { v, ok := o.Find(name) if !ok { return nil, nil @@ -2448,11 +2452,11 @@ func (o *SearchIndexOptions) SQL() string { } func (o *SearchIndexOptions) SortOrderSharding() (*bool, error) { - return (*GenericOptions)(o).GetBool("sort_order_sharding") + return (*GenericOptions)(o).FindBool("sort_order_sharding") } func (o *SearchIndexOptions) DisableAutomaticUIDColumn() (*bool, error) { - return (*GenericOptions)(o).GetBool("disable_automatic_uid_column") + return (*GenericOptions)(o).FindBool("disable_automatic_uid_column") } // DropSearchIndex represents DROP SEARCH INDEX statement. @@ -2612,4 +2616,3 @@ type SequenceOptions struct { Records []*SequenceOption // len(Records) > 0 } - diff --git a/ast/ast_test.go b/ast/ast_test.go index b60d8b16..83da326f 100644 --- a/ast/ast_test.go +++ b/ast/ast_test.go @@ -263,7 +263,7 @@ func TestGenericOptionsGetBool(t *testing.T) { for _, tcase := range tcases { t.Run(tcase.desc, func(t *testing.T) { - got, err := tcase.input.GetBool(tcase.name) + got, err := tcase.input.FindBool(tcase.name) if tcase.wantErr == "" && err != nil { t.Errorf("should not fail, but: %v", err) } From aecd271b0e9e4d10136f001127f36b96c17f9f8e Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 16 Oct 2024 02:32:59 +0900 Subject: [PATCH 19/35] Remove GeneratedColumnExpr.IsStored --- ast/ast.go | 2 -- ast/pos.go | 2 +- ast/sql.go | 2 +- parser.go | 11 +++++------ ...table_add_column_with_if_expression.sql.txt | 9 ++++----- testdata/result/ddl/create_table.sql.txt | 9 ++++----- .../ddl/create_table_fulltext_albums.sql.txt | 18 ++++++++---------- ...table_add_column_with_if_expression.sql.txt | 9 ++++----- testdata/result/statement/create_table.sql.txt | 9 ++++----- .../create_table_fulltext_albums.sql.txt | 18 ++++++++---------- 10 files changed, 39 insertions(+), 50 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 7d613959..83d67140 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -1607,8 +1607,6 @@ type GeneratedColumnExpr struct { Stored token.Pos // position of "STORED" keyword, InvalidPos if not stored Rparen token.Pos // position of ")" - IsStored bool - Expr Expr } diff --git a/ast/pos.go b/ast/pos.go index 7bba2ee6..f2b76bcd 100644 --- a/ast/pos.go +++ b/ast/pos.go @@ -579,7 +579,7 @@ func (g *ColumnDefaultExpr) End() token.Pos { return g.Rparen } func (g *GeneratedColumnExpr) Pos() token.Pos { return g.As } func (g *GeneratedColumnExpr) End() token.Pos { - if g.IsStored { + if !g.Stored.Invalid() { return g.Stored + 6 } return g.Rparen + 1 diff --git a/ast/sql.go b/ast/sql.go index a13296b1..71ae6080 100644 --- a/ast/sql.go +++ b/ast/sql.go @@ -876,7 +876,7 @@ func (c *ColumnDefaultExpr) SQL() string { } func (g *GeneratedColumnExpr) SQL() string { - return "AS (" + g.Expr.SQL() + ")" + strOpt(g.IsStored, " STORED") + return "AS (" + g.Expr.SQL() + ")" + strOpt(!g.Stored.Invalid(), " STORED") } func (c *ColumnDefOptions) SQL() string { diff --git a/parser.go b/parser.go index e4e051e3..37a80fc3 100644 --- a/parser.go +++ b/parser.go @@ -2421,17 +2421,16 @@ func (p *Parser) tryParseGeneratedColumnExpr() *ast.GeneratedColumnExpr { rparen := p.expect(")").Pos stored := p.tryExpectKeywordLike("STORED") - var storedPos token.Pos + storedPos := token.InvalidPos if stored != nil { storedPos = stored.Pos } return &ast.GeneratedColumnExpr{ - As: posAs, - Stored: storedPos, - Rparen: rparen, - IsStored: stored != nil, - Expr: expr, + As: posAs, + Stored: storedPos, + Rparen: rparen, + Expr: expr, } } diff --git a/testdata/result/ddl/alter_table_add_column_with_if_expression.sql.txt b/testdata/result/ddl/alter_table_add_column_with_if_expression.sql.txt index 472d04e2..1e2a9b7c 100644 --- a/testdata/result/ddl/alter_table_add_column_with_if_expression.sql.txt +++ b/testdata/result/ddl/alter_table_add_column_with_if_expression.sql.txt @@ -25,11 +25,10 @@ ALTER TABLE foo ADD COLUMN expired_at TIMESTAMP AS (IF (status != "OPEN" AND sta NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: &ast.GeneratedColumnExpr{ - As: 48, - Stored: 153, - Rparen: 151, - IsStored: true, - Expr: &ast.CallExpr{ + As: 48, + Stored: 153, + Rparen: 151, + Expr: &ast.CallExpr{ Rparen: 150, Func: &ast.Ident{ NamePos: 52, diff --git a/testdata/result/ddl/create_table.sql.txt b/testdata/result/ddl/create_table.sql.txt index ffc2b1a1..21ed883f 100644 --- a/testdata/result/ddl/create_table.sql.txt +++ b/testdata/result/ddl/create_table.sql.txt @@ -110,11 +110,10 @@ create table foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: &ast.GeneratedColumnExpr{ - As: 150, - Stored: 172, - Rparen: 170, - IsStored: true, - Expr: &ast.CallExpr{ + As: 150, + Stored: 172, + Rparen: 170, + Expr: &ast.CallExpr{ Rparen: 169, Func: &ast.Ident{ NamePos: 154, diff --git a/testdata/result/ddl/create_table_fulltext_albums.sql.txt b/testdata/result/ddl/create_table_fulltext_albums.sql.txt index 0522a699..ef74700c 100644 --- a/testdata/result/ddl/create_table_fulltext_albums.sql.txt +++ b/testdata/result/ddl/create_table_fulltext_albums.sql.txt @@ -125,11 +125,10 @@ CREATE TABLE Albums ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: &ast.GeneratedColumnExpr{ - As: 427, - Stored: 0, - Rparen: 460, - IsStored: false, - Expr: &ast.CallExpr{ + As: 427, + Stored: -1, + Rparen: 460, + Expr: &ast.CallExpr{ Rparen: 459, Func: &ast.Ident{ NamePos: 431, @@ -166,11 +165,10 @@ CREATE TABLE Albums ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: &ast.GeneratedColumnExpr{ - As: 518, - Stored: 0, - Rparen: 545, - IsStored: false, - Expr: &ast.CallExpr{ + As: 518, + Stored: -1, + Rparen: 545, + Expr: &ast.CallExpr{ Rparen: 544, Func: &ast.Ident{ NamePos: 522, diff --git a/testdata/result/statement/alter_table_add_column_with_if_expression.sql.txt b/testdata/result/statement/alter_table_add_column_with_if_expression.sql.txt index 472d04e2..1e2a9b7c 100644 --- a/testdata/result/statement/alter_table_add_column_with_if_expression.sql.txt +++ b/testdata/result/statement/alter_table_add_column_with_if_expression.sql.txt @@ -25,11 +25,10 @@ ALTER TABLE foo ADD COLUMN expired_at TIMESTAMP AS (IF (status != "OPEN" AND sta NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: &ast.GeneratedColumnExpr{ - As: 48, - Stored: 153, - Rparen: 151, - IsStored: true, - Expr: &ast.CallExpr{ + As: 48, + Stored: 153, + Rparen: 151, + Expr: &ast.CallExpr{ Rparen: 150, Func: &ast.Ident{ NamePos: 52, diff --git a/testdata/result/statement/create_table.sql.txt b/testdata/result/statement/create_table.sql.txt index ffc2b1a1..21ed883f 100644 --- a/testdata/result/statement/create_table.sql.txt +++ b/testdata/result/statement/create_table.sql.txt @@ -110,11 +110,10 @@ create table foo ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: &ast.GeneratedColumnExpr{ - As: 150, - Stored: 172, - Rparen: 170, - IsStored: true, - Expr: &ast.CallExpr{ + As: 150, + Stored: 172, + Rparen: 170, + Expr: &ast.CallExpr{ Rparen: 169, Func: &ast.Ident{ NamePos: 154, diff --git a/testdata/result/statement/create_table_fulltext_albums.sql.txt b/testdata/result/statement/create_table_fulltext_albums.sql.txt index 0522a699..ef74700c 100644 --- a/testdata/result/statement/create_table_fulltext_albums.sql.txt +++ b/testdata/result/statement/create_table_fulltext_albums.sql.txt @@ -125,11 +125,10 @@ CREATE TABLE Albums ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: &ast.GeneratedColumnExpr{ - As: 427, - Stored: 0, - Rparen: 460, - IsStored: false, - Expr: &ast.CallExpr{ + As: 427, + Stored: -1, + Rparen: 460, + Expr: &ast.CallExpr{ Rparen: 459, Func: &ast.Ident{ NamePos: 431, @@ -166,11 +165,10 @@ CREATE TABLE Albums ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: &ast.GeneratedColumnExpr{ - As: 518, - Stored: 0, - Rparen: 545, - IsStored: false, - Expr: &ast.CallExpr{ + As: 518, + Stored: -1, + Rparen: 545, + Expr: &ast.CallExpr{ Rparen: 544, Func: &ast.Ident{ NamePos: 522, From 2b459ade9b7b12a96ce1fb2da5bd3faf40f49d15 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 16 Oct 2024 02:35:22 +0900 Subject: [PATCH 20/35] Move SearchIndexOptions methods to correct place --- ast/ast.go | 12 ------------ ast/pos.go | 3 +++ ast/sql.go | 2 ++ 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 83d67140..d7c15827 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -2437,18 +2437,6 @@ type CreateSearchIndex struct { // SearchIndexOptions represents OPTIONS for CREATE SEARCH INDEX statement. type SearchIndexOptions GenericOptions -func (o *SearchIndexOptions) Pos() token.Pos { - return (*GenericOptions)(o).Pos() -} - -func (o *SearchIndexOptions) End() token.Pos { - return (*GenericOptions)(o).End() -} - -func (o *SearchIndexOptions) SQL() string { - return (*GenericOptions)(o).SQL() -} - func (o *SearchIndexOptions) SortOrderSharding() (*bool, error) { return (*GenericOptions)(o).FindBool("sort_order_sharding") } diff --git a/ast/pos.go b/ast/pos.go index f2b76bcd..892565d0 100644 --- a/ast/pos.go +++ b/ast/pos.go @@ -881,6 +881,9 @@ func (c *CreateSearchIndex) End() token.Pos { return c.Rparen + 1 } +func (o *SearchIndexOptions) Pos() token.Pos { return (*GenericOptions)(o).Pos() } +func (o *SearchIndexOptions) End() token.Pos { return (*GenericOptions)(o).End() } + func (d *DropSearchIndex) Pos() token.Pos { return d.Drop } diff --git a/ast/sql.go b/ast/sql.go index 71ae6080..fb35cd4d 100644 --- a/ast/sql.go +++ b/ast/sql.go @@ -1321,6 +1321,8 @@ func (c *CreateSearchIndex) SQL() string { sqlOpt(" ", c.Options, "") } +func (o *SearchIndexOptions) SQL() string { return (*GenericOptions)(o).SQL() } + func (d *DropSearchIndex) SQL() string { return "DROP SEARCH INDEX " + strOpt(d.IfExists, "IF EXISTS ") + d.Name.SQL() } From 57b212ae0250079bd6eefcaab1d19ad93e187ed1 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 16 Oct 2024 02:37:07 +0900 Subject: [PATCH 21/35] Fix comment --- ast/pos.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ast/pos.go b/ast/pos.go index 892565d0..e1a272b2 100644 --- a/ast/pos.go +++ b/ast/pos.go @@ -558,7 +558,7 @@ func (c *ColumnDef) Pos() token.Pos { } func (c *ColumnDef) End() token.Pos { - // TODO: It may able to be refactored using Pos arithmetic like InvalidPos + n = InvalidPos. + // TODO: It may be able to be refactored using Pos arithmetic like InvalidPos + n = InvalidPos. if c.Options != nil { return c.Options.End() } From 2127e0eb5495c130deb192f1ebe3d4930bd5d728 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 16 Oct 2024 02:56:08 +0900 Subject: [PATCH 22/35] Refactor using Pos arithmetic --- ast/pos.go | 49 +++++++++++++++++++++++++++---------------------- token/file.go | 8 ++++++++ 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/ast/pos.go b/ast/pos.go index e1a272b2..554224c1 100644 --- a/ast/pos.go +++ b/ast/pos.go @@ -49,6 +49,17 @@ func lastEnd[T Node](s []T) token.Pos { return lastNode(s).End() } +// firstValid returns the first valid token.Pos in arguments. +// This function corresponds to PosExpr ("||" PosExpr)* in ast.go +func firstValid(ps ...token.Pos) token.Pos { + for _, p := range ps { + if !p.Invalid() { + return p + } + } + return token.InvalidPos +} + // ================================================================================ // // SELECT @@ -558,20 +569,13 @@ func (c *ColumnDef) Pos() token.Pos { } func (c *ColumnDef) End() token.Pos { - // TODO: It may be able to be refactored using Pos arithmetic like InvalidPos + n = InvalidPos. - if c.Options != nil { - return c.Options.End() - } - if !c.Hidden.Invalid() { - return c.Hidden + 6 - } - if end := firstValidEnd(c.GeneratedExpr, c.DefaultExpr); !end.Invalid() { - return end - } - if !c.Null.Invalid() { - return c.Null + 4 - } - return c.Type.End() + return firstValid( + firstValidEnd(c.Options), + c.Hidden.Add(6), + firstValidEnd(c.GeneratedExpr, c.DefaultExpr), + c.Null.Add(4), + c.Type.End(), + ) } func (g *ColumnDefaultExpr) Pos() token.Pos { return g.Default } @@ -579,10 +583,7 @@ func (g *ColumnDefaultExpr) End() token.Pos { return g.Rparen } func (g *GeneratedColumnExpr) Pos() token.Pos { return g.As } func (g *GeneratedColumnExpr) End() token.Pos { - if !g.Stored.Invalid() { - return g.Stored + 6 - } - return g.Rparen + 1 + return firstValid(g.Stored.Add(6), g.Rparen.Add(1)) } func (c *ColumnDefOptions) Pos() token.Pos { return c.Options } @@ -875,10 +876,14 @@ func (c *ChangeStreamForTable) End() token.Pos { func (c *CreateSearchIndex) Pos() token.Pos { return c.Create } func (c *CreateSearchIndex) End() token.Pos { - if end := firstValidEnd(c.Options, c.Interleave, c.Where, c.OrderBy, lastNode(c.PartitionColumns), c.Storing); end != token.InvalidPos { - return end - } - return c.Rparen + 1 + return firstValid( + firstValidEnd(c.Options, + c.Interleave, + c.Where, + c.OrderBy, + lastNode(c.PartitionColumns), + c.Storing), + c.Rparen.Add(1)) } func (o *SearchIndexOptions) Pos() token.Pos { return (*GenericOptions)(o).Pos() } diff --git a/token/file.go b/token/file.go index 0d08fe5f..e4dd17be 100644 --- a/token/file.go +++ b/token/file.go @@ -18,6 +18,14 @@ func (p Pos) Invalid() bool { return p < 0 } +// Add returns Pos + offset, or InvalidPos if Pos.Invalid(). +func (p Pos) Add(offset int) Pos { + if p.Invalid() { + return InvalidPos + } + return p + Pos(offset) +} + // Position is source code position with file path // and source code around this position. type Position struct { From 49f5dccd874c68ee9fc6ab643f949a3ef73460c7 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 16 Oct 2024 03:08:33 +0900 Subject: [PATCH 23/35] Remove unnecessary comment --- .../input/ddl/create_search_index_full.sql | 1 - .../ddl/create_search_index_full.sql.txt | 71 +++++++++---------- .../create_search_index_full.sql.txt | 71 +++++++++---------- 3 files changed, 70 insertions(+), 73 deletions(-) diff --git a/testdata/input/ddl/create_search_index_full.sql b/testdata/input/ddl/create_search_index_full.sql index db67d674..f9a98cfb 100644 --- a/testdata/input/ddl/create_search_index_full.sql +++ b/testdata/input/ddl/create_search_index_full.sql @@ -1,4 +1,3 @@ --- Should CREATE SEARCH INDEX AlbumsIndexFull ON Albums(Title_Tokens, Studio_Tokens) STORING(Genre) diff --git a/testdata/result/ddl/create_search_index_full.sql.txt b/testdata/result/ddl/create_search_index_full.sql.txt index 548040e8..f7957b94 100644 --- a/testdata/result/ddl/create_search_index_full.sql.txt +++ b/testdata/result/ddl/create_search_index_full.sql.txt @@ -1,5 +1,4 @@ --- create_search_index_full.sql --- Should CREATE SEARCH INDEX AlbumsIndexFull ON Albums(Title_Tokens, Studio_Tokens) STORING(Genre) @@ -10,56 +9,56 @@ WHERE Genre IS NOT NULL AND ReleaseTimestamp IS NOT NULL OPTIONS(sort_order_sharding=true) --- AST &ast.CreateSearchIndex{ - Create: 10, + Create: 0, Name: &ast.Ident{ - NamePos: 30, - NameEnd: 45, + NamePos: 20, + NameEnd: 35, Name: "AlbumsIndexFull", }, TableName: &ast.Ident{ - NamePos: 49, - NameEnd: 55, + NamePos: 39, + NameEnd: 45, Name: "Albums", }, TokenListPart: []*ast.Ident{ &ast.Ident{ - NamePos: 56, - NameEnd: 68, + NamePos: 46, + NameEnd: 58, Name: "Title_Tokens", }, &ast.Ident{ - NamePos: 70, - NameEnd: 83, + NamePos: 60, + NameEnd: 73, Name: "Studio_Tokens", }, }, - Rparen: 83, + Rparen: 73, Storing: &ast.Storing{ - Storing: 85, - Rparen: 98, + Storing: 75, + Rparen: 88, Columns: []*ast.Ident{ &ast.Ident{ - NamePos: 93, - NameEnd: 98, + NamePos: 83, + NameEnd: 88, Name: "Genre", }, }, }, PartitionColumns: []*ast.Ident{ &ast.Ident{ - NamePos: 113, - NameEnd: 121, + NamePos: 103, + NameEnd: 111, Name: "SingerId", }, }, OrderBy: &ast.OrderBy{ - Order: 122, + Order: 112, Items: []*ast.OrderByItem{ &ast.OrderByItem{ - DirPos: 148, + DirPos: 138, Expr: &ast.Ident{ - NamePos: 131, - NameEnd: 147, + NamePos: 121, + NameEnd: 137, Name: "ReleaseTimestamp", }, Collate: (*ast.Collate)(nil), @@ -68,49 +67,49 @@ OPTIONS(sort_order_sharding=true) }, }, Where: &ast.Where{ - Where: 153, + Where: 143, Expr: &ast.BinaryExpr{ Op: "AND", Left: &ast.IsNullExpr{ - Null: 172, + Null: 162, Not: true, Left: &ast.Ident{ - NamePos: 159, - NameEnd: 164, + NamePos: 149, + NameEnd: 154, Name: "Genre", }, }, Right: &ast.IsNullExpr{ - Null: 205, + Null: 195, Not: true, Left: &ast.Ident{ - NamePos: 181, - NameEnd: 197, + NamePos: 171, + NameEnd: 187, Name: "ReleaseTimestamp", }, }, }, }, Interleave: &ast.InterleaveIn{ - Comma: 210, + Comma: 200, TableName: &ast.Ident{ - NamePos: 226, - NameEnd: 233, + NamePos: 216, + NameEnd: 223, Name: "Singers", }, }, Options: &ast.SearchIndexOptions{ - Options: 234, - Rparen: 266, + Options: 224, + Rparen: 256, Records: []*ast.GenericOption{ &ast.GenericOption{ Name: &ast.Ident{ - NamePos: 242, - NameEnd: 261, + NamePos: 232, + NameEnd: 251, Name: "sort_order_sharding", }, Value: &ast.BoolLiteral{ - ValuePos: 262, + ValuePos: 252, Value: true, }, }, diff --git a/testdata/result/statement/create_search_index_full.sql.txt b/testdata/result/statement/create_search_index_full.sql.txt index 548040e8..f7957b94 100644 --- a/testdata/result/statement/create_search_index_full.sql.txt +++ b/testdata/result/statement/create_search_index_full.sql.txt @@ -1,5 +1,4 @@ --- create_search_index_full.sql --- Should CREATE SEARCH INDEX AlbumsIndexFull ON Albums(Title_Tokens, Studio_Tokens) STORING(Genre) @@ -10,56 +9,56 @@ WHERE Genre IS NOT NULL AND ReleaseTimestamp IS NOT NULL OPTIONS(sort_order_sharding=true) --- AST &ast.CreateSearchIndex{ - Create: 10, + Create: 0, Name: &ast.Ident{ - NamePos: 30, - NameEnd: 45, + NamePos: 20, + NameEnd: 35, Name: "AlbumsIndexFull", }, TableName: &ast.Ident{ - NamePos: 49, - NameEnd: 55, + NamePos: 39, + NameEnd: 45, Name: "Albums", }, TokenListPart: []*ast.Ident{ &ast.Ident{ - NamePos: 56, - NameEnd: 68, + NamePos: 46, + NameEnd: 58, Name: "Title_Tokens", }, &ast.Ident{ - NamePos: 70, - NameEnd: 83, + NamePos: 60, + NameEnd: 73, Name: "Studio_Tokens", }, }, - Rparen: 83, + Rparen: 73, Storing: &ast.Storing{ - Storing: 85, - Rparen: 98, + Storing: 75, + Rparen: 88, Columns: []*ast.Ident{ &ast.Ident{ - NamePos: 93, - NameEnd: 98, + NamePos: 83, + NameEnd: 88, Name: "Genre", }, }, }, PartitionColumns: []*ast.Ident{ &ast.Ident{ - NamePos: 113, - NameEnd: 121, + NamePos: 103, + NameEnd: 111, Name: "SingerId", }, }, OrderBy: &ast.OrderBy{ - Order: 122, + Order: 112, Items: []*ast.OrderByItem{ &ast.OrderByItem{ - DirPos: 148, + DirPos: 138, Expr: &ast.Ident{ - NamePos: 131, - NameEnd: 147, + NamePos: 121, + NameEnd: 137, Name: "ReleaseTimestamp", }, Collate: (*ast.Collate)(nil), @@ -68,49 +67,49 @@ OPTIONS(sort_order_sharding=true) }, }, Where: &ast.Where{ - Where: 153, + Where: 143, Expr: &ast.BinaryExpr{ Op: "AND", Left: &ast.IsNullExpr{ - Null: 172, + Null: 162, Not: true, Left: &ast.Ident{ - NamePos: 159, - NameEnd: 164, + NamePos: 149, + NameEnd: 154, Name: "Genre", }, }, Right: &ast.IsNullExpr{ - Null: 205, + Null: 195, Not: true, Left: &ast.Ident{ - NamePos: 181, - NameEnd: 197, + NamePos: 171, + NameEnd: 187, Name: "ReleaseTimestamp", }, }, }, }, Interleave: &ast.InterleaveIn{ - Comma: 210, + Comma: 200, TableName: &ast.Ident{ - NamePos: 226, - NameEnd: 233, + NamePos: 216, + NameEnd: 223, Name: "Singers", }, }, Options: &ast.SearchIndexOptions{ - Options: 234, - Rparen: 266, + Options: 224, + Rparen: 256, Records: []*ast.GenericOption{ &ast.GenericOption{ Name: &ast.Ident{ - NamePos: 242, - NameEnd: 261, + NamePos: 232, + NameEnd: 251, Name: "sort_order_sharding", }, Value: &ast.BoolLiteral{ - ValuePos: 262, + ValuePos: 252, Value: true, }, }, From fdefffb4229337749296a510b8b525fe389e52ae Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 16 Oct 2024 03:12:55 +0900 Subject: [PATCH 24/35] Revert unneeded change --- testdata/input/ddl/create_table.sql | 2 +- testdata/result/ddl/create_table.sql.txt | 12 ++++++------ testdata/result/statement/create_table.sql.txt | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/testdata/input/ddl/create_table.sql b/testdata/input/ddl/create_table.sql index acf8118a..4bae8db4 100644 --- a/testdata/input/ddl/create_table.sql +++ b/testdata/input/ddl/create_table.sql @@ -10,5 +10,5 @@ create table foo ( check (foo > 0), constraint cname check (bar > 0), quux json, - corge timestamp not null default (current_timestamp()), + corge timestamp not null default (current_timestamp()) ) primary key (foo, bar) diff --git a/testdata/result/ddl/create_table.sql.txt b/testdata/result/ddl/create_table.sql.txt index 21ed883f..7d492386 100644 --- a/testdata/result/ddl/create_table.sql.txt +++ b/testdata/result/ddl/create_table.sql.txt @@ -11,13 +11,13 @@ create table foo ( check (foo > 0), constraint cname check (bar > 0), quux json, - corge timestamp not null default (current_timestamp()), + corge timestamp not null default (current_timestamp()) ) primary key (foo, bar) --- AST &ast.CreateTable{ Create: 0, - Rparen: 574, + Rparen: 573, IfNotExists: false, Name: &ast.Ident{ NamePos: 13, @@ -376,8 +376,8 @@ create table foo ( &ast.IndexKey{ DirPos: -1, Name: &ast.Ident{ - NamePos: 566, - NameEnd: 569, + NamePos: 565, + NameEnd: 568, Name: "foo", }, Dir: "", @@ -385,8 +385,8 @@ create table foo ( &ast.IndexKey{ DirPos: -1, Name: &ast.Ident{ - NamePos: 571, - NameEnd: 574, + NamePos: 570, + NameEnd: 573, Name: "bar", }, Dir: "", diff --git a/testdata/result/statement/create_table.sql.txt b/testdata/result/statement/create_table.sql.txt index 21ed883f..7d492386 100644 --- a/testdata/result/statement/create_table.sql.txt +++ b/testdata/result/statement/create_table.sql.txt @@ -11,13 +11,13 @@ create table foo ( check (foo > 0), constraint cname check (bar > 0), quux json, - corge timestamp not null default (current_timestamp()), + corge timestamp not null default (current_timestamp()) ) primary key (foo, bar) --- AST &ast.CreateTable{ Create: 0, - Rparen: 574, + Rparen: 573, IfNotExists: false, Name: &ast.Ident{ NamePos: 13, @@ -376,8 +376,8 @@ create table foo ( &ast.IndexKey{ DirPos: -1, Name: &ast.Ident{ - NamePos: 566, - NameEnd: 569, + NamePos: 565, + NameEnd: 568, Name: "foo", }, Dir: "", @@ -385,8 +385,8 @@ create table foo ( &ast.IndexKey{ DirPos: -1, Name: &ast.Ident{ - NamePos: 571, - NameEnd: 574, + NamePos: 570, + NameEnd: 573, Name: "bar", }, Dir: "", From 8671d6bd879534add7db27c609d7b541d937d147 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Fri, 18 Oct 2024 20:19:30 +0900 Subject: [PATCH 25/35] Add Options --- ast/ast.go | 56 +++++++++++++++++++++++-------------------------- ast/ast_test.go | 24 ++++++++++----------- ast/pos.go | 11 ++++------ ast/sql.go | 6 ++---- parser.go | 41 ++++++++++++++++++------------------ 5 files changed, 65 insertions(+), 73 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index d7c15827..baf2795c 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -32,6 +32,7 @@ package ast import ( + "errors" "fmt" "github.com/cloudspannerecosystem/memefish/token" ) @@ -1452,22 +1453,23 @@ type CastNumValue struct { // // ================================================================================ -// GenericOptions is generic OPTIONS clause node without key and value checking. +// Options is generic OPTIONS clause node without key and value checking. // // OPTIONS ({{.Records | sqlJoin ","}}) -type GenericOptions struct { +type Options struct { // pos = Options // end = Rparen + 1 Options token.Pos // position of "OPTIONS" keyword Rparen token.Pos // position of ")" - Records []*GenericOption // len(Records) > 0 + Records []*OptionsRecord // len(Records) > 0 } -// Find finds name in Records, and return its value as Expr. +// field finds name in Records, and return its value as Expr. // The second return value indicates that the name was found in Records. -func (o *GenericOptions) Find(name string) (expr Expr, found bool) { +// It is not exposed because you can use *Field methods. +func (o *Options) field(name string) (expr Expr, found bool) { for _, r := range o.Records { if r.Name.Name != name { continue @@ -1477,12 +1479,17 @@ func (o *GenericOptions) Find(name string) (expr Expr, found bool) { return nil, false } -// FindBool finds name in Records, and return its value as *bool. -// If the value is neither bool nor null, it returns error. -func (o *GenericOptions) FindBool(name string) (*bool, error) { - v, ok := o.Find(name) +var FieldNotFound = errors.New("field not found") + +// BoolField finds name in records, and return its value as *bool. +// If Options doesn't have a record with name, it returns FieldNotFound error. +// If record have NullLiteral value, it returns nil. +// If record have BoolLiteral value, it returns pointer of bool value. +// If record have value which is neither NullLiteral nor BoolLiteral, it returns error. +func (o *Options) BoolField(name string) (*bool, error) { + v, ok := o.field(name) if !ok { - return nil, nil + return nil, FieldNotFound } switch v := v.(type) { case *BoolLiteral: @@ -1494,10 +1501,10 @@ func (o *GenericOptions) FindBool(name string) (*bool, error) { } } -// GenericOption is generic option for CREATE statements. +// OptionsRecord is generic option for CREATE statements. // // {{.Name | sql}} = {{.Value | sql}} -type GenericOption struct { +type OptionsRecord struct { // pos = Name.pos // end = Value.end @@ -2425,24 +2432,13 @@ type CreateSearchIndex struct { Name *Ident TableName *Ident TokenListPart []*Ident - Rparen token.Pos // position of ")" after TokenListPart - Storing *Storing // optional - PartitionColumns []*Ident // optional - OrderBy *OrderBy // optional - Where *Where // optional - Interleave *InterleaveIn // optional - Options *SearchIndexOptions // optional -} - -// SearchIndexOptions represents OPTIONS for CREATE SEARCH INDEX statement. -type SearchIndexOptions GenericOptions - -func (o *SearchIndexOptions) SortOrderSharding() (*bool, error) { - return (*GenericOptions)(o).FindBool("sort_order_sharding") -} - -func (o *SearchIndexOptions) DisableAutomaticUIDColumn() (*bool, error) { - return (*GenericOptions)(o).FindBool("disable_automatic_uid_column") + Rparen token.Pos // position of ")" after TokenListPart + Storing *Storing // optional + PartitionColumns []*Ident // optional + OrderBy *OrderBy // optional + Where *Where // optional + Interleave *InterleaveIn // optional + Options *Options // optional } // DropSearchIndex represents DROP SEARCH INDEX statement. diff --git a/ast/ast_test.go b/ast/ast_test.go index 83da326f..e0687f9a 100644 --- a/ast/ast_test.go +++ b/ast/ast_test.go @@ -204,15 +204,15 @@ func ptr[T any](v T) *T { func TestGenericOptionsGetBool(t *testing.T) { tcases := []struct { desc string - input *GenericOptions + input *Options name string want *bool wantErr string }{ { desc: "true", - input: &GenericOptions{ - Records: []*GenericOption{ + input: &Options{ + Records: []*OptionsRecord{ {Name: &Ident{Name: "sort_order_sharding"}, Value: &BoolLiteral{Value: true}}, }, }, @@ -221,8 +221,8 @@ func TestGenericOptionsGetBool(t *testing.T) { }, { desc: "false", - input: &GenericOptions{ - Records: []*GenericOption{ + input: &Options{ + Records: []*OptionsRecord{ {Name: &Ident{Name: "sort_order_sharding"}, Value: &BoolLiteral{Value: false}}, }, }, @@ -231,8 +231,8 @@ func TestGenericOptionsGetBool(t *testing.T) { }, { desc: "explicit null", - input: &GenericOptions{ - Records: []*GenericOption{ + input: &Options{ + Records: []*OptionsRecord{ {Name: &Ident{Name: "sort_order_sharding"}, Value: &NullLiteral{}}, }, }, @@ -241,8 +241,8 @@ func TestGenericOptionsGetBool(t *testing.T) { }, { desc: "implicit null", - input: &GenericOptions{ - Records: []*GenericOption{ + input: &Options{ + Records: []*OptionsRecord{ {Name: &Ident{Name: "disable_automatic_uid_column"}, Value: &BoolLiteral{Value: true}}, }, }, @@ -251,8 +251,8 @@ func TestGenericOptionsGetBool(t *testing.T) { }, { desc: "invalid value", - input: &GenericOptions{ - Records: []*GenericOption{ + input: &Options{ + Records: []*OptionsRecord{ {Name: &Ident{Name: "sort_order_sharding"}, Value: &StringLiteral{Value: "foo"}}, }, }, @@ -263,7 +263,7 @@ func TestGenericOptionsGetBool(t *testing.T) { for _, tcase := range tcases { t.Run(tcase.desc, func(t *testing.T) { - got, err := tcase.input.FindBool(tcase.name) + got, err := tcase.input.BoolField(tcase.name) if tcase.wantErr == "" && err != nil { t.Errorf("should not fail, but: %v", err) } diff --git a/ast/pos.go b/ast/pos.go index 554224c1..4b41787c 100644 --- a/ast/pos.go +++ b/ast/pos.go @@ -525,11 +525,11 @@ func (c *CastNumValue) End() token.Pos { return c.Rparen + 1 } // // ================================================================================ -func (g *GenericOptions) Pos() token.Pos { return g.Options } -func (g *GenericOptions) End() token.Pos { return g.Rparen + 1 } +func (g *Options) Pos() token.Pos { return g.Options } +func (g *Options) End() token.Pos { return g.Rparen + 1 } -func (g *GenericOption) Pos() token.Pos { return g.Name.Pos() } -func (g *GenericOption) End() token.Pos { return g.Value.End() } +func (g *OptionsRecord) Pos() token.Pos { return g.Name.Pos() } +func (g *OptionsRecord) End() token.Pos { return g.Value.End() } func (c *CreateDatabase) Pos() token.Pos { return c.Create } func (c *CreateDatabase) End() token.Pos { return c.Name.End() } @@ -886,9 +886,6 @@ func (c *CreateSearchIndex) End() token.Pos { c.Rparen.Add(1)) } -func (o *SearchIndexOptions) Pos() token.Pos { return (*GenericOptions)(o).Pos() } -func (o *SearchIndexOptions) End() token.Pos { return (*GenericOptions)(o).End() } - func (d *DropSearchIndex) Pos() token.Pos { return d.Drop } diff --git a/ast/sql.go b/ast/sql.go index fb35cd4d..b7a1453b 100644 --- a/ast/sql.go +++ b/ast/sql.go @@ -762,9 +762,9 @@ func (c *CastNumValue) SQL() string { // // ================================================================================ -func (g *GenericOptions) SQL() string { return "OPTIONS (" + sqlJoin(g.Records, ", ") + ")" } +func (g *Options) SQL() string { return "OPTIONS (" + sqlJoin(g.Records, ", ") + ")" } -func (g *GenericOption) SQL() string { return g.Name.SQL() + " = " + g.Value.SQL() } +func (g *OptionsRecord) SQL() string { return g.Name.SQL() + " = " + g.Value.SQL() } func (c *CreateDatabase) SQL() string { return "CREATE DATABASE " + c.Name.SQL() @@ -1321,8 +1321,6 @@ func (c *CreateSearchIndex) SQL() string { sqlOpt(" ", c.Options, "") } -func (o *SearchIndexOptions) SQL() string { return (*GenericOptions)(o).SQL() } - func (d *DropSearchIndex) SQL() string { return "DROP SEARCH INDEX " + strOpt(d.IfExists, "IF EXISTS ") + d.Name.SQL() } diff --git a/parser.go b/parser.go index 3208c4b1..9e992299 100644 --- a/parser.go +++ b/parser.go @@ -2605,16 +2605,13 @@ func (p *Parser) parseCreateSearchIndex(pos token.Pos) *ast.CreateSearchIndex { var partitionColumns []*ast.Ident if p.tryExpect("PARTITION") != nil { p.expect("BY") - partitionColumns = parseSeparatedList(p, ",", p.parseIdent) + partitionColumns = parseCommaSeparatedList(p, p.parseIdent) } orderBy := p.tryParseOrderBy() where := p.tryParseWhere() interleave := p.tryParseInterleaveIn() - var searchIndexOptions *ast.SearchIndexOptions - if options := p.tryParseGenericOptions(); options != nil { - searchIndexOptions = (*ast.SearchIndexOptions)(options) - } + options := p.tryParseOptions() return &ast.CreateSearchIndex{ Create: pos, @@ -2627,34 +2624,38 @@ func (p *Parser) parseCreateSearchIndex(pos token.Pos) *ast.CreateSearchIndex { OrderBy: orderBy, Where: where, Interleave: interleave, - Options: searchIndexOptions, + Options: options, } } -func (p *Parser) tryParseGenericOptions() *ast.GenericOptions { - options := p.tryExpectKeywordLike("OPTIONS") - if options == nil { +func (p *Parser) tryParseOptions() *ast.Options { + if !p.Token.IsKeywordLike("OPTIONS") { return nil } + + optionsPos := p.expectKeywordLike("OPTIONS").Pos + p.expect("(") - records := parseSeparatedList(p, ",", func() *ast.GenericOption { - name := p.parseIdent() - p.expect("=") - value := p.parseExpr() - return &ast.GenericOption{ - Name: name, - Value: value, - } - }) + records := parseCommaSeparatedList(p, p.parseOptionsRecord) rparen := p.expect(")").Pos - return &ast.GenericOptions{ - Options: options.Pos, + return &ast.Options{ + Options: optionsPos, Rparen: rparen, Records: records, } } +func (p *Parser) parseOptionsRecord() *ast.OptionsRecord { + name := p.parseIdent() + p.expect("=") + value := p.parseExpr() + return &ast.OptionsRecord{ + Name: name, + Value: value, + } +} + func (p *Parser) parseDropSearchIndex(pos token.Pos) *ast.DropSearchIndex { p.expectKeywordLike("SEARCH") p.expectKeywordLike("INDEX") From a90745cba4b9eb574f5e1383688d7f69ea865c45 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Fri, 18 Oct 2024 20:20:22 +0900 Subject: [PATCH 26/35] Migrate to parseCommaSeparatedList --- parser.go | 32 +++----------------------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/parser.go b/parser.go index 9e992299..a97cb959 100644 --- a/parser.go +++ b/parser.go @@ -2594,10 +2594,12 @@ func (p *Parser) parseCreateSearchIndex(pos token.Pos) *ast.CreateSearchIndex { p.expectKeywordLike("SEARCH") p.expectKeywordLike("INDEX") name := p.parseIdent() + p.expect("ON") tableName := p.parseIdent() + p.expect("(") - columnName := parseSeparatedList(p, ",", p.parseIdent) + columnName := parseCommaSeparatedList(p, p.parseIdent) rparen := p.expect(")").Pos storing := p.tryParseStoring() @@ -3876,31 +3878,3 @@ func (p *Parser) errorfAtToken(tok *token.Token, msg string, params ...interface func (p *Parser) panicfAtToken(tok *token.Token, msg string, params ...interface{}) { panic(p.errorfAtToken(tok, msg, params...)) } - -// This function can't be a method because Go haven't yet supported generic methods. -func parseSeparatedList[T interface { - ast.Node - comparable -}](p *Parser, sep token.TokenKind, doParse func() T) []T { - var zero T - - first := doParse() - if first == zero { - return nil - } - - list := []T{first} - for { - if p.Token.Kind != sep { - break - } - p.nextToken() - - elem := doParse() - if elem == zero { - return list - } - list = append(list, elem) - } - return list -} From 4969ac556a2f07ef27fdb5785fd17e1ae397f122 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Fri, 18 Oct 2024 20:24:37 +0900 Subject: [PATCH 27/35] Remove p.tryExpect() and p.tryExpectKeywordLike() --- parser.go | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/parser.go b/parser.go index a97cb959..01a6611c 100644 --- a/parser.go +++ b/parser.go @@ -2280,11 +2280,12 @@ func (p *Parser) parseColumnDef() *ast.ColumnDef { t, notNull, null := p.parseTypeNotNull() defaultExpr := p.tryParseColumnDefaultExpr() generated := p.tryParseGeneratedColumnExpr() - hidden := p.tryExpectKeywordLike("HIDDEN") + hiddenPos := token.InvalidPos - if hidden != nil { - hiddenPos = hidden.Pos + if p.Token.IsKeywordLike("HIDDEN") { + hiddenPos = p.expectKeywordLike("HIDDEN").Pos } + options := p.tryParseColumnDefOptions() return &ast.ColumnDef{ @@ -2395,10 +2396,10 @@ func (p *Parser) tryParseGeneratedColumnExpr() *ast.GeneratedColumnExpr { p.expect("(") expr := p.parseExpr() rparen := p.expect(")").Pos - stored := p.tryExpectKeywordLike("STORED") storedPos := token.InvalidPos - if stored != nil { + if p.Token.IsKeywordLike("STORED") { + stored := p.expectKeywordLike("STORED") storedPos = stored.Pos } @@ -2605,10 +2606,12 @@ func (p *Parser) parseCreateSearchIndex(pos token.Pos) *ast.CreateSearchIndex { storing := p.tryParseStoring() var partitionColumns []*ast.Ident - if p.tryExpect("PARTITION") != nil { + if p.Token.Kind == "PARTITION" { + p.nextToken() p.expect("BY") partitionColumns = parseCommaSeparatedList(p, p.parseIdent) } + orderBy := p.tryParseOrderBy() where := p.tryParseWhere() interleave := p.tryParseInterleaveIn() @@ -3854,20 +3857,6 @@ func (p *Parser) expectKeywordLike(s string) *token.Token { return id } -func (p *Parser) tryExpect(s token.TokenKind) *token.Token { - if p.Token.Kind != s { - return nil - } - return p.expect(s) -} - -func (p *Parser) tryExpectKeywordLike(s string) *token.Token { - if !p.Token.IsKeywordLike(s) { - return nil - } - return p.expectKeywordLike(s) -} - func (p *Parser) errorfAtToken(tok *token.Token, msg string, params ...interface{}) *Error { return &Error{ Message: fmt.Sprintf(msg, params...), From 358d86cb528e37f2ba55d065acb5246b553e1010 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Fri, 18 Oct 2024 21:07:34 +0900 Subject: [PATCH 28/35] Add *Field methods --- ast/ast.go | 53 +++++++++++++++++++++++++++++++++++++++++++++---- ast/ast_test.go | 5 +++-- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index baf2795c..72d78695 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -35,6 +35,7 @@ import ( "errors" "fmt" "github.com/cloudspannerecosystem/memefish/token" + "strconv" ) // Node is base interface of Spanner SQL AST nodes. @@ -1466,10 +1467,10 @@ type Options struct { Records []*OptionsRecord // len(Records) > 0 } -// field finds name in Records, and return its value as Expr. +// Field finds name in Records, and return its value as Expr. // The second return value indicates that the name was found in Records. // It is not exposed because you can use *Field methods. -func (o *Options) field(name string) (expr Expr, found bool) { +func (o *Options) Field(name string) (expr Expr, found bool) { for _, r := range o.Records { if r.Name.Name != name { continue @@ -1487,7 +1488,7 @@ var FieldNotFound = errors.New("field not found") // If record have BoolLiteral value, it returns pointer of bool value. // If record have value which is neither NullLiteral nor BoolLiteral, it returns error. func (o *Options) BoolField(name string) (*bool, error) { - v, ok := o.field(name) + v, ok := o.Field(name) if !ok { return nil, FieldNotFound } @@ -1497,7 +1498,51 @@ func (o *Options) BoolField(name string) (*bool, error) { case *NullLiteral: return nil, nil default: - return nil, fmt.Errorf("expect bool or null, but have unknown type %T", v) + return nil, fmt.Errorf("expect BoolLiteral or NullLiteral, but have unknown type %T", v) + } +} + +// IntegerField finds name in records, and return its value as *int64. +// If Options doesn't have a record with name, it returns FieldNotFound error. +// If record have NullLiteral value, it returns nil. +// If record have IntegerLiteral value, it returns pointer of int64 value. +// If record have value which is neither NullLiteral nor BoolLiteral, it returns error. +func (o *Options) IntegerField(name string) (*int64, error) { + v, ok := o.Field(name) + if !ok { + return nil, FieldNotFound + } + switch v := v.(type) { + case *IntLiteral: + n, err := strconv.ParseInt(v.Value, 10, 64) + if err != nil { + return nil, err + } + return &n, nil + case *NullLiteral: + return nil, nil + default: + return nil, fmt.Errorf("expect IntLiteral or NullLiteral, but have unknown type %T", v) + } +} + +// StringField finds name in records, and return its value as *string. +// If Options doesn't have a record with name, it returns FieldNotFound error. +// If record have NullLiteral value, it returns nil. +// If record have StringLiteral value, it returns pointer of string value. +// If record have value which is neither NullLiteral nor StringLiteral, it returns error. +func (o *Options) StringField(name string) (*string, error) { + v, ok := o.Field(name) + if !ok { + return nil, FieldNotFound + } + switch v := v.(type) { + case *StringLiteral: + return &v.Value, nil + case *NullLiteral: + return nil, nil + default: + return nil, fmt.Errorf("expect StringLiteral or NullLiteral, but have unknown type %T", v) } } diff --git a/ast/ast_test.go b/ast/ast_test.go index e0687f9a..f7877a56 100644 --- a/ast/ast_test.go +++ b/ast/ast_test.go @@ -246,8 +246,9 @@ func TestGenericOptionsGetBool(t *testing.T) { {Name: &Ident{Name: "disable_automatic_uid_column"}, Value: &BoolLiteral{Value: true}}, }, }, - name: "sort_order_sharding", - want: nil, + name: "sort_order_sharding", + want: nil, + wantErr: "field not found", }, { desc: "invalid value", From 86f449b3cadb4f3653291dd8eda7db1680300f4f Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Fri, 18 Oct 2024 21:07:50 +0900 Subject: [PATCH 29/35] Update testdata --- testdata/result/ddl/create_search_index_full.sql.txt | 6 +++--- .../result/ddl/create_search_index_null_filtered.sql.txt | 2 +- testdata/result/ddl/create_search_index_simple.sql.txt | 2 +- testdata/result/statement/create_search_index_full.sql.txt | 6 +++--- .../statement/create_search_index_null_filtered.sql.txt | 2 +- .../result/statement/create_search_index_simple.sql.txt | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/testdata/result/ddl/create_search_index_full.sql.txt b/testdata/result/ddl/create_search_index_full.sql.txt index f7957b94..f876e718 100644 --- a/testdata/result/ddl/create_search_index_full.sql.txt +++ b/testdata/result/ddl/create_search_index_full.sql.txt @@ -98,11 +98,11 @@ OPTIONS(sort_order_sharding=true) Name: "Singers", }, }, - Options: &ast.SearchIndexOptions{ + Options: &ast.Options{ Options: 224, Rparen: 256, - Records: []*ast.GenericOption{ - &ast.GenericOption{ + Records: []*ast.OptionsRecord{ + &ast.OptionsRecord{ Name: &ast.Ident{ NamePos: 232, NameEnd: 251, diff --git a/testdata/result/ddl/create_search_index_null_filtered.sql.txt b/testdata/result/ddl/create_search_index_null_filtered.sql.txt index bdb8bc53..f1fab7c1 100644 --- a/testdata/result/ddl/create_search_index_null_filtered.sql.txt +++ b/testdata/result/ddl/create_search_index_null_filtered.sql.txt @@ -50,7 +50,7 @@ WHERE Genre IS NOT NULL }, }, Interleave: (*ast.InterleaveIn)(nil), - Options: (*ast.SearchIndexOptions)(nil), + Options: (*ast.Options)(nil), } --- SQL diff --git a/testdata/result/ddl/create_search_index_simple.sql.txt b/testdata/result/ddl/create_search_index_simple.sql.txt index b29adfe0..60072aa5 100644 --- a/testdata/result/ddl/create_search_index_simple.sql.txt +++ b/testdata/result/ddl/create_search_index_simple.sql.txt @@ -28,7 +28,7 @@ CREATE SEARCH INDEX AlbumsIndex OrderBy: (*ast.OrderBy)(nil), Where: (*ast.Where)(nil), Interleave: (*ast.InterleaveIn)(nil), - Options: (*ast.SearchIndexOptions)(nil), + Options: (*ast.Options)(nil), } --- SQL diff --git a/testdata/result/statement/create_search_index_full.sql.txt b/testdata/result/statement/create_search_index_full.sql.txt index f7957b94..f876e718 100644 --- a/testdata/result/statement/create_search_index_full.sql.txt +++ b/testdata/result/statement/create_search_index_full.sql.txt @@ -98,11 +98,11 @@ OPTIONS(sort_order_sharding=true) Name: "Singers", }, }, - Options: &ast.SearchIndexOptions{ + Options: &ast.Options{ Options: 224, Rparen: 256, - Records: []*ast.GenericOption{ - &ast.GenericOption{ + Records: []*ast.OptionsRecord{ + &ast.OptionsRecord{ Name: &ast.Ident{ NamePos: 232, NameEnd: 251, diff --git a/testdata/result/statement/create_search_index_null_filtered.sql.txt b/testdata/result/statement/create_search_index_null_filtered.sql.txt index bdb8bc53..f1fab7c1 100644 --- a/testdata/result/statement/create_search_index_null_filtered.sql.txt +++ b/testdata/result/statement/create_search_index_null_filtered.sql.txt @@ -50,7 +50,7 @@ WHERE Genre IS NOT NULL }, }, Interleave: (*ast.InterleaveIn)(nil), - Options: (*ast.SearchIndexOptions)(nil), + Options: (*ast.Options)(nil), } --- SQL diff --git a/testdata/result/statement/create_search_index_simple.sql.txt b/testdata/result/statement/create_search_index_simple.sql.txt index b29adfe0..60072aa5 100644 --- a/testdata/result/statement/create_search_index_simple.sql.txt +++ b/testdata/result/statement/create_search_index_simple.sql.txt @@ -28,7 +28,7 @@ CREATE SEARCH INDEX AlbumsIndex OrderBy: (*ast.OrderBy)(nil), Where: (*ast.Where)(nil), Interleave: (*ast.InterleaveIn)(nil), - Options: (*ast.SearchIndexOptions)(nil), + Options: (*ast.Options)(nil), } --- SQL From 79842b16302aa7ba7e0844cdd9feabb0d33f5602 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 23 Oct 2024 00:29:13 +0900 Subject: [PATCH 30/35] Fix some wording and ordering --- parser.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/parser.go b/parser.go index 556bed15..c72e6e02 100644 --- a/parser.go +++ b/parser.go @@ -2163,7 +2163,7 @@ func (p *Parser) parseDDL() ast.DDL { return p.parseAlterTable(pos) case p.Token.IsKeywordLike("INDEX"): return p.parseAlterIndex(pos) - case p.Token.IsKeywordLike("Search"): + case p.Token.IsKeywordLike("SEARCH"): return p.parseAlterSearchIndex(pos) case p.Token.IsKeywordLike("SEQUENCE"): return p.parseAlterSequence(pos) @@ -2178,10 +2178,10 @@ func (p *Parser) parseDDL() ast.DDL { return p.parseDropTable(pos) case p.Token.IsKeywordLike("INDEX"): return p.parseDropIndex(pos) - case p.Token.IsKeywordLike("VECTOR"): - return p.parseDropVectorIndex(pos) case p.Token.IsKeywordLike("SEARCH"): return p.parseDropSearchIndex(pos) + case p.Token.IsKeywordLike("VECTOR"): + return p.parseDropVectorIndex(pos) case p.Token.IsKeywordLike("SEQUENCE"): return p.parseDropSequence(pos) case p.Token.IsKeywordLike("VIEW"): From beb79fb64ac38e1fd4fdd1aa0047ff3df245c3db Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 23 Oct 2024 00:32:05 +0900 Subject: [PATCH 31/35] Add parseIndexAlteration method --- parser.go | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/parser.go b/parser.go index c72e6e02..266f7925 100644 --- a/parser.go +++ b/parser.go @@ -2698,15 +2698,7 @@ func (p *Parser) parseAlterSearchIndex(pos token.Pos) *ast.AlterSearchIndex { name := p.parseIdent() - var alteration ast.IndexAlteration - switch { - case p.Token.IsKeywordLike("ADD"): - alteration = p.parseAddStoredColumn() - case p.Token.IsKeywordLike("DROP"): - alteration = p.parseDropStoredColumn() - default: - p.panicfAtToken(&p.Token, "expected pseudo keyword: ADD, DROP, but: %s", p.Token.AsString) - } + alteration := p.parseIndexAlteration() return &ast.AlterSearchIndex{ Alter: pos, @@ -2715,6 +2707,17 @@ func (p *Parser) parseAlterSearchIndex(pos token.Pos) *ast.AlterSearchIndex { } } +func (p *Parser) parseIndexAlteration() ast.IndexAlteration { + switch { + case p.Token.IsKeywordLike("ADD"): + return p.parseAddStoredColumn() + case p.Token.IsKeywordLike("DROP"): + return p.parseDropStoredColumn() + default: + panic(p.errorfAtToken(&p.Token, "expected pseudo keyword: ADD, DROP, but: %s", p.Token.AsString)) + } +} + func (p *Parser) parseCreateIndex(pos token.Pos) *ast.CreateIndex { unique := false if p.Token.IsKeywordLike("UNIQUE") { @@ -3097,15 +3100,7 @@ func (p *Parser) parseAlterIndex(pos token.Pos) *ast.AlterIndex { name := p.parseIdent() - var alteration ast.IndexAlteration - switch { - case p.Token.IsKeywordLike("ADD"): - alteration = p.parseAddStoredColumn() - case p.Token.IsKeywordLike("DROP"): - alteration = p.parseDropStoredColumn() - default: - p.panicfAtToken(&p.Token, "expected pseudo keyword: ADD, DROP, but: %s", p.Token.AsString) - } + alteration := p.parseIndexAlteration() return &ast.AlterIndex{ Alter: pos, From 1e8cfe480559fa99938da6ecb44292a5f63a9ab1 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Wed, 23 Oct 2024 00:36:58 +0900 Subject: [PATCH 32/35] Remove unneeded newlines --- parser.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/parser.go b/parser.go index 266f7925..e364949d 100644 --- a/parser.go +++ b/parser.go @@ -2697,7 +2697,6 @@ func (p *Parser) parseAlterSearchIndex(pos token.Pos) *ast.AlterSearchIndex { p.expectKeywordLike("INDEX") name := p.parseIdent() - alteration := p.parseIndexAlteration() return &ast.AlterSearchIndex{ @@ -3099,7 +3098,6 @@ func (p *Parser) parseAlterIndex(pos token.Pos) *ast.AlterIndex { p.expectKeywordLike("INDEX") name := p.parseIdent() - alteration := p.parseIndexAlteration() return &ast.AlterIndex{ From ca9a1b6e32131dc04f4f461a68824a58969bcb60 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:28:52 +0900 Subject: [PATCH 33/35] Update ast/ast.go Co-authored-by: Hiroya Fujinami --- ast/ast.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ast/ast.go b/ast/ast.go index 4335245d..92fc718d 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -2550,7 +2550,7 @@ type CreateSearchIndex struct { // DropSearchIndex represents DROP SEARCH INDEX statement. // -// DROP SEARCH INDEX {{.IfExists | strOpt "IF EXISTS "}}{{Name | sql}} +// DROP SEARCH INDEX{{if .IfExists}}IF EXISTS{{end}} {{Name | sql}} type DropSearchIndex struct { // pos = Drop // end = Name.end From e7f319187f81d6cf3c233d7eab8a174cc34cee46 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:30:20 +0900 Subject: [PATCH 34/35] Remove useless use of isnil --- ast/ast.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ast/ast.go b/ast/ast.go index 92fc718d..cc79779b 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -2525,7 +2525,7 @@ type ArraySchemaType struct { // ON {{.TableName | sql}} // ({{.TokenListPart | sqlJoin ", "}}) // {{.Storing | sqlOpt}} -// {{if not(.PartitionColumns | isnil)}}PARTITION BY {{.PartitionColumns | sqlJoin ", "}}{{end}} +// {{if .PartitionColumns}}PARTITION BY {{.PartitionColumns | sqlJoin ", "}}{{end}} // {{.OrderBy | sqlOpt}} // {{.Where | sqlOpt}} // {{.Interleave | sqlOpt}} From ebd86bd8c211af53fcf85db461c2f20113f72314 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:36:03 +0900 Subject: [PATCH 35/35] Update testdata --- testdata/result/ddl/create_table_fulltext_albums.sql.txt | 1 + testdata/result/ddl/create_table_synonyms.sql.txt | 2 ++ testdata/result/ddl/create_table_synonyms_abnormal.sql.txt | 2 ++ testdata/result/statement/create_table_fulltext_albums.sql.txt | 1 + testdata/result/statement/create_table_synonyms.sql.txt | 2 ++ .../result/statement/create_table_synonyms_abnormal.sql.txt | 2 ++ 6 files changed, 10 insertions(+) diff --git a/testdata/result/ddl/create_table_fulltext_albums.sql.txt b/testdata/result/ddl/create_table_fulltext_albums.sql.txt index f75c92db..26d125f3 100644 --- a/testdata/result/ddl/create_table_fulltext_albums.sql.txt +++ b/testdata/result/ddl/create_table_fulltext_albums.sql.txt @@ -208,6 +208,7 @@ CREATE TABLE Albums ( Dir: "", }, }, + Synonyms: []*ast.Synonym(nil), Cluster: (*ast.Cluster)(nil), RowDeletionPolicy: (*ast.CreateRowDeletionPolicy)(nil), } diff --git a/testdata/result/ddl/create_table_synonyms.sql.txt b/testdata/result/ddl/create_table_synonyms.sql.txt index 3a34aa0c..9264d6f8 100644 --- a/testdata/result/ddl/create_table_synonyms.sql.txt +++ b/testdata/result/ddl/create_table_synonyms.sql.txt @@ -29,6 +29,7 @@ CREATE TABLE Singers ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: -1, Options: (*ast.Options)(nil), }, &ast.ColumnDef{ @@ -53,6 +54,7 @@ CREATE TABLE Singers ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: -1, Options: (*ast.Options)(nil), }, }, diff --git a/testdata/result/ddl/create_table_synonyms_abnormal.sql.txt b/testdata/result/ddl/create_table_synonyms_abnormal.sql.txt index 887f578a..412d563e 100644 --- a/testdata/result/ddl/create_table_synonyms_abnormal.sql.txt +++ b/testdata/result/ddl/create_table_synonyms_abnormal.sql.txt @@ -31,6 +31,7 @@ CREATE TABLE Singers ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: -1, Options: (*ast.Options)(nil), }, &ast.ColumnDef{ @@ -55,6 +56,7 @@ CREATE TABLE Singers ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: -1, Options: (*ast.Options)(nil), }, }, diff --git a/testdata/result/statement/create_table_fulltext_albums.sql.txt b/testdata/result/statement/create_table_fulltext_albums.sql.txt index f75c92db..26d125f3 100644 --- a/testdata/result/statement/create_table_fulltext_albums.sql.txt +++ b/testdata/result/statement/create_table_fulltext_albums.sql.txt @@ -208,6 +208,7 @@ CREATE TABLE Albums ( Dir: "", }, }, + Synonyms: []*ast.Synonym(nil), Cluster: (*ast.Cluster)(nil), RowDeletionPolicy: (*ast.CreateRowDeletionPolicy)(nil), } diff --git a/testdata/result/statement/create_table_synonyms.sql.txt b/testdata/result/statement/create_table_synonyms.sql.txt index 3a34aa0c..9264d6f8 100644 --- a/testdata/result/statement/create_table_synonyms.sql.txt +++ b/testdata/result/statement/create_table_synonyms.sql.txt @@ -29,6 +29,7 @@ CREATE TABLE Singers ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: -1, Options: (*ast.Options)(nil), }, &ast.ColumnDef{ @@ -53,6 +54,7 @@ CREATE TABLE Singers ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: -1, Options: (*ast.Options)(nil), }, }, diff --git a/testdata/result/statement/create_table_synonyms_abnormal.sql.txt b/testdata/result/statement/create_table_synonyms_abnormal.sql.txt index 887f578a..412d563e 100644 --- a/testdata/result/statement/create_table_synonyms_abnormal.sql.txt +++ b/testdata/result/statement/create_table_synonyms_abnormal.sql.txt @@ -31,6 +31,7 @@ CREATE TABLE Singers ( NotNull: true, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: -1, Options: (*ast.Options)(nil), }, &ast.ColumnDef{ @@ -55,6 +56,7 @@ CREATE TABLE Singers ( NotNull: false, DefaultExpr: (*ast.ColumnDefaultExpr)(nil), GeneratedExpr: (*ast.GeneratedColumnExpr)(nil), + Hidden: -1, Options: (*ast.Options)(nil), }, },