From d6261608ab15111271554d235940cb4e930016d6 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Sat, 23 Nov 2024 08:25:43 +0900 Subject: [PATCH 1/7] Support TVF --- ast/ast.go | 56 ++++++++++++++++++++++++++++++++++++++++++ ast/pos.go | 24 ++++++++++++++++++ ast/sql.go | 17 +++++++++++++ parser.go | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 165 insertions(+), 4 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 1de8aac0..ea7697ce 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -144,6 +144,7 @@ func (PathTableExpr) isTableExpr() {} func (SubQueryTableExpr) isTableExpr() {} func (ParenTableExpr) isTableExpr() {} func (Join) isTableExpr() {} +func (TVFCallExpr) isTableExpr() {} // JoinCondition represents condition part of JOIN expression. type JoinCondition interface { @@ -218,6 +219,15 @@ func (ExprArg) isArg() {} func (IntervalArg) isArg() {} func (SequenceArg) isArg() {} +type TVFArg interface { + Node + isTVFArg() +} + +func (ExprArg) isTVFArg() {} +func (ModelArg) isTVFArg() {} +func (TableArg) isTVFArg() {} + // NullHandlingModifier represents IGNORE/RESPECT NULLS of aggregate function calls type NullHandlingModifier interface { Node @@ -1157,6 +1167,28 @@ type CallExpr struct { Having HavingModifier // optional } +// TVFCallExpr is table-valued function call expression node. +// +// {{.Name | sql}}( +// {{.Args | sqlJoin ", "}} +// {{if len(.Args) > 0 && len(.NamedArgs) > 0}}, {{end}} +// {{.NamedArgs | sqlJoin ", "}} +// ) +// {{.Hint | sqlOpt}} +// {{.Sample | sqlOpt}} +type TVFCallExpr struct { + // pos = Name.pos + // end = (Sample ?? Hint).end || Rparen + 1 + + Rparen token.Pos // position of ")" + + Name *Path + Args []TVFArg + NamedArgs []*NamedArg + Hint *Hint // optional + Sample *TableSample // optional +} + // ExprArg is argument of the generic function call. // // {{.Expr | sql}} @@ -1192,6 +1224,30 @@ type SequenceArg struct { Expr Expr } +// ModelArg is argument of model function call. +// +// MODEL {{.Name | sql}} +type ModelArg struct { + // pos = Model + // end = Name.end + + Model token.Pos // position of "MODEL" keyword + + Name *Path +} + +// TableArg is TABLE table_name argument of table valued function call. +// +// TABLE {{.Path | sql}} +type TableArg struct { + // pos = Table + // end = Name.end + + Table token.Pos // position of "TABLE" keyword + + Name *Path +} + // NamedArg represents a name and value pair in named arguments // // {{.Name | sql}} => {{.Value | sql}} diff --git a/ast/pos.go b/ast/pos.go index dd0cd882..334128e4 100644 --- a/ast/pos.go +++ b/ast/pos.go @@ -398,6 +398,14 @@ func (c *CallExpr) End() token.Pos { return posAdd(c.Rparen, 1) } +func (t *TVFCallExpr) Pos() token.Pos { + return nodePos(wrapNode(t.Name)) +} + +func (t *TVFCallExpr) End() token.Pos { + return posAdd(t.Rparen, 1) +} + func (e *ExprArg) Pos() token.Pos { return nodePos(wrapNode(e.Expr)) } @@ -422,6 +430,22 @@ func (s *SequenceArg) End() token.Pos { return nodeEnd(wrapNode(s.Expr)) } +func (m *ModelArg) Pos() token.Pos { + return m.Model +} + +func (m *ModelArg) End() token.Pos { + return nodeEnd(wrapNode(m.Name)) +} + +func (t *TableArg) Pos() token.Pos { + return t.Table +} + +func (t *TableArg) End() token.Pos { + return nodeEnd(wrapNode(t.Name)) +} + func (n *NamedArg) Pos() token.Pos { return nodePos(wrapNode(n.Name)) } diff --git a/ast/sql.go b/ast/sql.go index 12f864db..18ed22bd 100644 --- a/ast/sql.go +++ b/ast/sql.go @@ -506,6 +506,15 @@ func (c *CallExpr) SQL() string { ")" } +func (c *TVFCallExpr) SQL() string { + return c.Name.SQL() + "(" + + sqlJoin(c.Args, ", ") + + strOpt(len(c.Args) > 0 && len(c.NamedArgs) > 0, ", ") + + sqlJoin(c.NamedArgs, ", ") + + ")" + + sqlOpt(" ", c.Hint, "") +} + func (n *NamedArg) SQL() string { return n.Name.SQL() + " => " + n.Value.SQL() } func (i *IgnoreNulls) SQL() string { return "IGNORE NULLS" } @@ -532,6 +541,14 @@ func (s *SequenceArg) SQL() string { return "SEQUENCE " + s.Expr.SQL() } +func (s *ModelArg) SQL() string { + return "MODEL " + s.Name.SQL() +} + +func (s *TableArg) SQL() string { + return "TABLE " + s.Name.SQL() +} + func (*CountStarExpr) SQL() string { return "COUNT(*)" } diff --git a/parser.go b/parser.go index c2a2595f..32b86815 100644 --- a/parser.go +++ b/parser.go @@ -821,6 +821,9 @@ func (p *Parser) parseSimpleTableExpr() ast.TableExpr { if p.Token.Kind == token.TokenIdent { ids := p.parseIdentOrPath() + if p.Token.Kind == "(" { + return p.parseTVFCallExpr(ids) + } if len(ids) == 1 { return p.parseTableNameSuffix(ids[0]) } @@ -830,6 +833,63 @@ func (p *Parser) parseSimpleTableExpr() ast.TableExpr { panic(p.errorfAtToken(&p.Token, "expected token: (, UNNEST, , but: %s", p.Token.Kind)) } +func (p *Parser) parseTVFCallExpr(ids []*ast.Ident) *ast.TVFCallExpr { + p.expect("(") + + var args []ast.TVFArg + if p.Token.Kind != ")" { + for p.Token.Kind != token.TokenEOF && !p.lookaheadNamedArg() { + args = append(args, p.parseTVFArg()) + if p.Token.Kind != "," { + break + } + p.nextToken() + } + } + + var namedArgs []*ast.NamedArg + if p.lookaheadNamedArg() { + namedArgs = parseCommaSeparatedList(p, p.parseNamedArg) + } + + rparen := p.expect(")").Pos + hint := p.tryParseHint() + sample := p.tryParseTableSample() + + return &ast.TVFCallExpr{ + Rparen: rparen, + Name: &ast.Path{Idents: ids}, + Args: args, + NamedArgs: namedArgs, + Hint: hint, + Sample: sample, + } +} + +func (p *Parser) parseTVFArg() ast.TVFArg { + pos := p.Token.Pos + switch { + case p.Token.IsKeywordLike("TABLE"): + p.nextToken() + path := p.parsePath() + + return &ast.TableArg{ + Table: pos, + Name: path, + } + case p.Token.IsKeywordLike("MODEL"): + p.nextToken() + path := p.parsePath() + + return &ast.ModelArg{ + Model: pos, + Name: path, + } + default: + return p.parseExprArg() + } +} + func (p *Parser) parseIdentOrPath() []*ast.Ident { ids := []*ast.Ident{p.parseIdent()} for p.Token.Kind == "." { @@ -1579,10 +1639,7 @@ func (p *Parser) lookaheadNamedArg() bool { return p.Token.Kind == "=>" } -func (p *Parser) tryParseNamedArg() *ast.NamedArg { - if !p.lookaheadNamedArg() { - return nil - } +func (p *Parser) parseNamedArg() *ast.NamedArg { name := p.parseIdent() p.expect("=>") value := p.parseExpr() @@ -1592,6 +1649,13 @@ func (p *Parser) tryParseNamedArg() *ast.NamedArg { } } +func (p *Parser) tryParseNamedArg() *ast.NamedArg { + if !p.lookaheadNamedArg() { + return nil + } + return p.parseNamedArg() +} + func (p *Parser) parseArg() ast.Arg { if i := p.tryParseIntervalArg(); i != nil { return i From 5b0657a288f7124416f8a053fe697a25b888cf4e Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Sat, 23 Nov 2024 08:26:15 +0900 Subject: [PATCH 2/7] Update testdata --- testdata/input/dml/update_with_ml_predict.sql | 12 + .../input/query/select_from_change_stream.sql | 6 + .../query/select_from_ml_predict_hint.sql | 7 + .../query/select_from_ml_predict_simple.sql | 2 + .../select_from_ml_predict_textbison.sql | 15 + .../result/dml/update_with_ml_predict.sql.txt | 226 ++++++++++ .../query/select_from_change_stream.sql.txt | 99 +++++ .../query/select_from_ml_predict_hint.sql.txt | 194 +++++++++ .../select_from_ml_predict_simple.sql.txt | 93 ++++ .../select_from_ml_predict_textbison.sql.txt | 399 ++++++++++++++++++ .../select_from_change_stream.sql.txt | 99 +++++ .../select_from_ml_predict_hint.sql.txt | 194 +++++++++ .../select_from_ml_predict_simple.sql.txt | 93 ++++ .../select_from_ml_predict_textbison.sql.txt | 399 ++++++++++++++++++ .../statement/update_with_ml_predict.sql.txt | 226 ++++++++++ 15 files changed, 2064 insertions(+) create mode 100644 testdata/input/dml/update_with_ml_predict.sql create mode 100644 testdata/input/query/select_from_change_stream.sql create mode 100644 testdata/input/query/select_from_ml_predict_hint.sql create mode 100644 testdata/input/query/select_from_ml_predict_simple.sql create mode 100644 testdata/input/query/select_from_ml_predict_textbison.sql create mode 100644 testdata/result/dml/update_with_ml_predict.sql.txt create mode 100644 testdata/result/query/select_from_change_stream.sql.txt create mode 100644 testdata/result/query/select_from_ml_predict_hint.sql.txt create mode 100644 testdata/result/query/select_from_ml_predict_simple.sql.txt create mode 100644 testdata/result/query/select_from_ml_predict_textbison.sql.txt create mode 100644 testdata/result/statement/select_from_change_stream.sql.txt create mode 100644 testdata/result/statement/select_from_ml_predict_hint.sql.txt create mode 100644 testdata/result/statement/select_from_ml_predict_simple.sql.txt create mode 100644 testdata/result/statement/select_from_ml_predict_textbison.sql.txt create mode 100644 testdata/result/statement/update_with_ml_predict.sql.txt diff --git a/testdata/input/dml/update_with_ml_predict.sql b/testdata/input/dml/update_with_ml_predict.sql new file mode 100644 index 00000000..67f25010 --- /dev/null +++ b/testdata/input/dml/update_with_ml_predict.sql @@ -0,0 +1,12 @@ +-- https://cloud.google.com/spanner/docs/backfill-embeddings?hl=en#backfill +UPDATE products +SET + products.desc_embed = ( + SELECT embeddings.values + FROM SAFE.ML.PREDICT( + MODEL gecko_model, + (SELECT products.description AS content) + ) @{remote_udf_max_rows_per_rpc=200} + ), + products.desc_embed_model_version = 3 +WHERE products.desc_embed IS NULL \ No newline at end of file diff --git a/testdata/input/query/select_from_change_stream.sql b/testdata/input/query/select_from_change_stream.sql new file mode 100644 index 00000000..a649601a --- /dev/null +++ b/testdata/input/query/select_from_change_stream.sql @@ -0,0 +1,6 @@ +SELECT ChangeRecord FROM READ_SingersNameStream ( + start_timestamp => "2022-05-01T09:00:00Z", + end_timestamp => NULL, + partition_token => NULL, + heartbeat_milliseconds => 10000 +) \ No newline at end of file diff --git a/testdata/input/query/select_from_ml_predict_hint.sql b/testdata/input/query/select_from_ml_predict_hint.sql new file mode 100644 index 00000000..8f2a0f0c --- /dev/null +++ b/testdata/input/query/select_from_ml_predict_hint.sql @@ -0,0 +1,7 @@ +-- https://cloud.google.com/spanner/docs/ml-tutorial-generative-ai?hl=en#register_a_generative_ai_model_in_a_schema +SELECT content +FROM ML.PREDICT( + MODEL TextBison, + (SELECT "Is 13 prime?" AS prompt), + STRUCT(256 AS maxOutputTokens, 0.2 AS temperature, 40 as topK, 0.95 AS topP) + ) @{remote_udf_max_rows_per_rpc=1} \ No newline at end of file diff --git a/testdata/input/query/select_from_ml_predict_simple.sql b/testdata/input/query/select_from_ml_predict_simple.sql new file mode 100644 index 00000000..8220177e --- /dev/null +++ b/testdata/input/query/select_from_ml_predict_simple.sql @@ -0,0 +1,2 @@ +SELECT id, color, value +FROM ML.PREDICT(MODEL DiamondAppraise, TABLE Diamonds) \ No newline at end of file diff --git a/testdata/input/query/select_from_ml_predict_textbison.sql b/testdata/input/query/select_from_ml_predict_textbison.sql new file mode 100644 index 00000000..4c47c225 --- /dev/null +++ b/testdata/input/query/select_from_ml_predict_textbison.sql @@ -0,0 +1,15 @@ +SELECT product_id, product_name, content +FROM ML.PREDICT( + MODEL TextBison, + (SELECT + product.id as product_id, + product.name as product_name, + CONCAT("Is this product safe for infants?", "\n", + "Product Name: ", product.name, "\n", + "Category Name: ", category.name, "\n", + "Product Description:", product.description) AS prompt + FROM + Products AS product JOIN Categories AS category + ON product.category_id = category.id), + STRUCT(100 AS maxOutputTokens) + ) @{remote_udf_max_rows_per_rpc=1} \ No newline at end of file diff --git a/testdata/result/dml/update_with_ml_predict.sql.txt b/testdata/result/dml/update_with_ml_predict.sql.txt new file mode 100644 index 00000000..cb5ab0ee --- /dev/null +++ b/testdata/result/dml/update_with_ml_predict.sql.txt @@ -0,0 +1,226 @@ +--- update_with_ml_predict.sql +-- https://cloud.google.com/spanner/docs/backfill-embeddings?hl=en#backfill +UPDATE products +SET + products.desc_embed = ( + SELECT embeddings.values + FROM SAFE.ML.PREDICT( + MODEL gecko_model, + (SELECT products.description AS content) + ) @{remote_udf_max_rows_per_rpc=200} + ), + products.desc_embed_model_version = 3 +WHERE products.desc_embed IS NULL +--- AST +&ast.Update{ + Update: 76, + TableName: &ast.Ident{ + NamePos: 83, + NameEnd: 91, + Name: "products", + }, + As: (*ast.AsAlias)(nil), + Updates: []*ast.UpdateItem{ + &ast.UpdateItem{ + Path: []*ast.Ident{ + &ast.Ident{ + NamePos: 100, + NameEnd: 108, + Name: "products", + }, + &ast.Ident{ + NamePos: 109, + NameEnd: 119, + Name: "desc_embed", + }, + }, + DefaultExpr: &ast.DefaultExpr{ + DefaultPos: -1, + Default: false, + Expr: &ast.ScalarSubQuery{ + Lparen: 122, + Rparen: 333, + Query: &ast.Select{ + Select: 132, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.ExprSelectItem{ + Expr: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 139, + NameEnd: 149, + Name: "embeddings", + }, + &ast.Ident{ + NamePos: 150, + NameEnd: 156, + Name: "values", + }, + }, + }, + }, + }, + From: &ast.From{ + From: 165, + Source: &ast.TVFCallExpr{ + Rparen: 292, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 170, + NameEnd: 174, + Name: "SAFE", + }, + &ast.Ident{ + NamePos: 175, + NameEnd: 177, + Name: "ML", + }, + &ast.Ident{ + NamePos: 178, + NameEnd: 185, + Name: "PREDICT", + }, + }, + }, + Args: []ast.TVFArg{ + &ast.ModelArg{ + Model: 203, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 209, + NameEnd: 220, + Name: "gecko_model", + }, + }, + }, + }, + &ast.ExprArg{ + Expr: &ast.ScalarSubQuery{ + Lparen: 238, + Rparen: 277, + Query: &ast.Select{ + Select: 239, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.Alias{ + Expr: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 246, + NameEnd: 254, + Name: "products", + }, + &ast.Ident{ + NamePos: 255, + NameEnd: 266, + Name: "description", + }, + }, + }, + As: &ast.AsAlias{ + As: 267, + Alias: &ast.Ident{ + NamePos: 270, + NameEnd: 277, + Name: "content", + }, + }, + }, + }, + From: (*ast.From)(nil), + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, + }, + }, + }, + NamedArgs: []*ast.NamedArg(nil), + Hint: &ast.Hint{ + Atmark: 294, + Rbrace: 327, + Records: []*ast.HintRecord{ + &ast.HintRecord{ + Key: &ast.Ident{ + NamePos: 296, + NameEnd: 323, + Name: "remote_udf_max_rows_per_rpc", + }, + Value: &ast.IntLiteral{ + ValuePos: 324, + ValueEnd: 327, + Base: 10, + Value: "200", + }, + }, + }, + }, + Sample: (*ast.TableSample)(nil), + }, + }, + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, + }, + }, + }, + &ast.UpdateItem{ + Path: []*ast.Ident{ + &ast.Ident{ + NamePos: 340, + NameEnd: 348, + Name: "products", + }, + &ast.Ident{ + NamePos: 349, + NameEnd: 373, + Name: "desc_embed_model_version", + }, + }, + DefaultExpr: &ast.DefaultExpr{ + DefaultPos: -1, + Default: false, + Expr: &ast.IntLiteral{ + ValuePos: 376, + ValueEnd: 377, + Base: 10, + Value: "3", + }, + }, + }, + }, + Where: &ast.Where{ + Where: 378, + Expr: &ast.IsNullExpr{ + Null: 407, + Not: false, + Left: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 384, + NameEnd: 392, + Name: "products", + }, + &ast.Ident{ + NamePos: 393, + NameEnd: 403, + Name: "desc_embed", + }, + }, + }, + }, + }, +} + +--- SQL +UPDATE products SET products.desc_embed = (SELECT embeddings.values FROM SAFE.ML.PREDICT(MODEL gecko_model, (SELECT products.description AS content)) @{remote_udf_max_rows_per_rpc=200}), products.desc_embed_model_version = 3 WHERE products.desc_embed IS NULL diff --git a/testdata/result/query/select_from_change_stream.sql.txt b/testdata/result/query/select_from_change_stream.sql.txt new file mode 100644 index 00000000..86d71125 --- /dev/null +++ b/testdata/result/query/select_from_change_stream.sql.txt @@ -0,0 +1,99 @@ +--- select_from_change_stream.sql +SELECT ChangeRecord FROM READ_SingersNameStream ( + start_timestamp => "2022-05-01T09:00:00Z", + end_timestamp => NULL, + partition_token => NULL, + heartbeat_milliseconds => 10000 +) +--- AST +&ast.QueryStatement{ + Hint: (*ast.Hint)(nil), + With: (*ast.With)(nil), + Query: &ast.Select{ + Select: 0, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 7, + NameEnd: 19, + Name: "ChangeRecord", + }, + }, + }, + From: &ast.From{ + From: 20, + Source: &ast.TVFCallExpr{ + Rparen: 181, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 25, + NameEnd: 47, + Name: "READ_SingersNameStream", + }, + }, + }, + Args: []ast.TVFArg(nil), + NamedArgs: []*ast.NamedArg{ + &ast.NamedArg{ + Name: &ast.Ident{ + NamePos: 52, + NameEnd: 67, + Name: "start_timestamp", + }, + Value: &ast.StringLiteral{ + ValuePos: 71, + ValueEnd: 93, + Value: "2022-05-01T09:00:00Z", + }, + }, + &ast.NamedArg{ + Name: &ast.Ident{ + NamePos: 97, + NameEnd: 110, + Name: "end_timestamp", + }, + Value: &ast.NullLiteral{ + Null: 114, + }, + }, + &ast.NamedArg{ + Name: &ast.Ident{ + NamePos: 122, + NameEnd: 137, + Name: "partition_token", + }, + Value: &ast.NullLiteral{ + Null: 141, + }, + }, + &ast.NamedArg{ + Name: &ast.Ident{ + NamePos: 149, + NameEnd: 171, + Name: "heartbeat_milliseconds", + }, + Value: &ast.IntLiteral{ + ValuePos: 175, + ValueEnd: 180, + Base: 10, + Value: "10000", + }, + }, + }, + Hint: (*ast.Hint)(nil), + Sample: (*ast.TableSample)(nil), + }, + }, + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, +} + +--- SQL +SELECT ChangeRecord FROM READ_SingersNameStream(start_timestamp => "2022-05-01T09:00:00Z", end_timestamp => NULL, partition_token => NULL, heartbeat_milliseconds => 10000) diff --git a/testdata/result/query/select_from_ml_predict_hint.sql.txt b/testdata/result/query/select_from_ml_predict_hint.sql.txt new file mode 100644 index 00000000..19165cb0 --- /dev/null +++ b/testdata/result/query/select_from_ml_predict_hint.sql.txt @@ -0,0 +1,194 @@ +--- select_from_ml_predict_hint.sql +-- https://cloud.google.com/spanner/docs/ml-tutorial-generative-ai?hl=en#register_a_generative_ai_model_in_a_schema +SELECT content +FROM ML.PREDICT( + MODEL TextBison, + (SELECT "Is 13 prime?" AS prompt), + STRUCT(256 AS maxOutputTokens, 0.2 AS temperature, 40 as topK, 0.95 AS topP) + ) @{remote_udf_max_rows_per_rpc=1} +--- AST +&ast.QueryStatement{ + Hint: (*ast.Hint)(nil), + With: (*ast.With)(nil), + Query: &ast.Select{ + Select: 116, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 123, + NameEnd: 130, + Name: "content", + }, + }, + }, + From: &ast.From{ + From: 131, + Source: &ast.TVFCallExpr{ + Rparen: 306, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 136, + NameEnd: 138, + Name: "ML", + }, + &ast.Ident{ + NamePos: 139, + NameEnd: 146, + Name: "PREDICT", + }, + }, + }, + Args: []ast.TVFArg{ + &ast.ModelArg{ + Model: 156, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 162, + NameEnd: 171, + Name: "TextBison", + }, + }, + }, + }, + &ast.ExprArg{ + Expr: &ast.ScalarSubQuery{ + Lparen: 181, + Rparen: 213, + Query: &ast.Select{ + Select: 182, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.Alias{ + Expr: &ast.StringLiteral{ + ValuePos: 189, + ValueEnd: 203, + Value: "Is 13 prime?", + }, + As: &ast.AsAlias{ + As: 204, + Alias: &ast.Ident{ + NamePos: 207, + NameEnd: 213, + Name: "prompt", + }, + }, + }, + }, + From: (*ast.From)(nil), + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, + }, + }, + &ast.ExprArg{ + Expr: &ast.TypelessStructLiteral{ + Struct: 224, + Rparen: 299, + Values: []ast.TypelessStructLiteralArg{ + &ast.Alias{ + Expr: &ast.IntLiteral{ + ValuePos: 231, + ValueEnd: 234, + Base: 10, + Value: "256", + }, + As: &ast.AsAlias{ + As: 235, + Alias: &ast.Ident{ + NamePos: 238, + NameEnd: 253, + Name: "maxOutputTokens", + }, + }, + }, + &ast.Alias{ + Expr: &ast.FloatLiteral{ + ValuePos: 255, + ValueEnd: 258, + Value: "0.2", + }, + As: &ast.AsAlias{ + As: 259, + Alias: &ast.Ident{ + NamePos: 262, + NameEnd: 273, + Name: "temperature", + }, + }, + }, + &ast.Alias{ + Expr: &ast.IntLiteral{ + ValuePos: 275, + ValueEnd: 277, + Base: 10, + Value: "40", + }, + As: &ast.AsAlias{ + As: 278, + Alias: &ast.Ident{ + NamePos: 281, + NameEnd: 285, + Name: "topK", + }, + }, + }, + &ast.Alias{ + Expr: &ast.FloatLiteral{ + ValuePos: 287, + ValueEnd: 291, + Value: "0.95", + }, + As: &ast.AsAlias{ + As: 292, + Alias: &ast.Ident{ + NamePos: 295, + NameEnd: 299, + Name: "topP", + }, + }, + }, + }, + }, + }, + }, + NamedArgs: []*ast.NamedArg(nil), + Hint: &ast.Hint{ + Atmark: 308, + Rbrace: 339, + Records: []*ast.HintRecord{ + &ast.HintRecord{ + Key: &ast.Ident{ + NamePos: 310, + NameEnd: 337, + Name: "remote_udf_max_rows_per_rpc", + }, + Value: &ast.IntLiteral{ + ValuePos: 338, + ValueEnd: 339, + Base: 10, + Value: "1", + }, + }, + }, + }, + Sample: (*ast.TableSample)(nil), + }, + }, + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, +} + +--- SQL +SELECT content FROM ML.PREDICT(MODEL TextBison, (SELECT "Is 13 prime\?" AS prompt), STRUCT(256 AS maxOutputTokens, 0.2 AS temperature, 40 AS topK, 0.95 AS topP)) @{remote_udf_max_rows_per_rpc=1} diff --git a/testdata/result/query/select_from_ml_predict_simple.sql.txt b/testdata/result/query/select_from_ml_predict_simple.sql.txt new file mode 100644 index 00000000..62b8e238 --- /dev/null +++ b/testdata/result/query/select_from_ml_predict_simple.sql.txt @@ -0,0 +1,93 @@ +--- select_from_ml_predict_simple.sql +SELECT id, color, value +FROM ML.PREDICT(MODEL DiamondAppraise, TABLE Diamonds) +--- AST +&ast.QueryStatement{ + Hint: (*ast.Hint)(nil), + With: (*ast.With)(nil), + Query: &ast.Select{ + Select: 0, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 7, + NameEnd: 9, + Name: "id", + }, + }, + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 11, + NameEnd: 16, + Name: "color", + }, + }, + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 18, + NameEnd: 23, + Name: "value", + }, + }, + }, + From: &ast.From{ + From: 24, + Source: &ast.TVFCallExpr{ + Rparen: 77, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 29, + NameEnd: 31, + Name: "ML", + }, + &ast.Ident{ + NamePos: 32, + NameEnd: 39, + Name: "PREDICT", + }, + }, + }, + Args: []ast.TVFArg{ + &ast.ModelArg{ + Model: 40, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 46, + NameEnd: 61, + Name: "DiamondAppraise", + }, + }, + }, + }, + &ast.TableArg{ + Table: 63, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 69, + NameEnd: 77, + Name: "Diamonds", + }, + }, + }, + }, + }, + NamedArgs: []*ast.NamedArg(nil), + Hint: (*ast.Hint)(nil), + Sample: (*ast.TableSample)(nil), + }, + }, + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, +} + +--- SQL +SELECT id, color, value FROM ML.PREDICT(MODEL DiamondAppraise, TABLE Diamonds) diff --git a/testdata/result/query/select_from_ml_predict_textbison.sql.txt b/testdata/result/query/select_from_ml_predict_textbison.sql.txt new file mode 100644 index 00000000..f89b83e9 --- /dev/null +++ b/testdata/result/query/select_from_ml_predict_textbison.sql.txt @@ -0,0 +1,399 @@ +--- select_from_ml_predict_textbison.sql +SELECT product_id, product_name, content +FROM ML.PREDICT( + MODEL TextBison, + (SELECT + product.id as product_id, + product.name as product_name, + CONCAT("Is this product safe for infants?", "\n", + "Product Name: ", product.name, "\n", + "Category Name: ", category.name, "\n", + "Product Description:", product.description) AS prompt + FROM + Products AS product JOIN Categories AS category + ON product.category_id = category.id), + STRUCT(100 AS maxOutputTokens) + ) @{remote_udf_max_rows_per_rpc=1} +--- AST +&ast.QueryStatement{ + Hint: (*ast.Hint)(nil), + With: (*ast.With)(nil), + Query: &ast.Select{ + Select: 0, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 7, + NameEnd: 17, + Name: "product_id", + }, + }, + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 19, + NameEnd: 31, + Name: "product_name", + }, + }, + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 33, + NameEnd: 40, + Name: "content", + }, + }, + }, + From: &ast.From{ + From: 41, + Source: &ast.TVFCallExpr{ + Rparen: 633, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 46, + NameEnd: 48, + Name: "ML", + }, + &ast.Ident{ + NamePos: 49, + NameEnd: 56, + Name: "PREDICT", + }, + }, + }, + Args: []ast.TVFArg{ + &ast.ModelArg{ + Model: 66, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 72, + NameEnd: 81, + Name: "TextBison", + }, + }, + }, + }, + &ast.ExprArg{ + Expr: &ast.ScalarSubQuery{ + Lparen: 91, + Rparen: 586, + Query: &ast.Select{ + Select: 92, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.Alias{ + Expr: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 112, + NameEnd: 119, + Name: "product", + }, + &ast.Ident{ + NamePos: 120, + NameEnd: 122, + Name: "id", + }, + }, + }, + As: &ast.AsAlias{ + As: 123, + Alias: &ast.Ident{ + NamePos: 126, + NameEnd: 136, + Name: "product_id", + }, + }, + }, + &ast.Alias{ + Expr: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 151, + NameEnd: 158, + Name: "product", + }, + &ast.Ident{ + NamePos: 159, + NameEnd: 163, + Name: "name", + }, + }, + }, + As: &ast.AsAlias{ + As: 164, + Alias: &ast.Ident{ + NamePos: 167, + NameEnd: 179, + Name: "product_name", + }, + }, + }, + &ast.Alias{ + Expr: &ast.CallExpr{ + Rparen: 425, + Func: &ast.Ident{ + NamePos: 194, + NameEnd: 200, + Name: "CONCAT", + }, + Distinct: false, + Args: []ast.Arg{ + &ast.ExprArg{ + Expr: &ast.StringLiteral{ + ValuePos: 201, + ValueEnd: 236, + Value: "Is this product safe for infants?", + }, + }, + &ast.ExprArg{ + Expr: &ast.StringLiteral{ + ValuePos: 238, + ValueEnd: 242, + Value: "\n", + }, + }, + &ast.ExprArg{ + Expr: &ast.StringLiteral{ + ValuePos: 264, + ValueEnd: 280, + Value: "Product Name: ", + }, + }, + &ast.ExprArg{ + Expr: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 282, + NameEnd: 289, + Name: "product", + }, + &ast.Ident{ + NamePos: 290, + NameEnd: 294, + Name: "name", + }, + }, + }, + }, + &ast.ExprArg{ + Expr: &ast.StringLiteral{ + ValuePos: 296, + ValueEnd: 300, + Value: "\n", + }, + }, + &ast.ExprArg{ + Expr: &ast.StringLiteral{ + ValuePos: 322, + ValueEnd: 339, + Value: "Category Name: ", + }, + }, + &ast.ExprArg{ + Expr: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 341, + NameEnd: 349, + Name: "category", + }, + &ast.Ident{ + NamePos: 350, + NameEnd: 354, + Name: "name", + }, + }, + }, + }, + &ast.ExprArg{ + Expr: &ast.StringLiteral{ + ValuePos: 356, + ValueEnd: 360, + Value: "\n", + }, + }, + &ast.ExprArg{ + Expr: &ast.StringLiteral{ + ValuePos: 382, + ValueEnd: 404, + Value: "Product Description:", + }, + }, + &ast.ExprArg{ + Expr: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 406, + NameEnd: 413, + Name: "product", + }, + &ast.Ident{ + NamePos: 414, + NameEnd: 425, + Name: "description", + }, + }, + }, + }, + }, + NamedArgs: []*ast.NamedArg(nil), + NullHandling: nil, + Having: nil, + }, + As: &ast.AsAlias{ + As: 427, + Alias: &ast.Ident{ + NamePos: 430, + NameEnd: 436, + Name: "prompt", + }, + }, + }, + }, + From: &ast.From{ + From: 446, + Source: &ast.Join{ + Op: "INNER JOIN", + Method: "", + Hint: (*ast.Hint)(nil), + Left: &ast.TableName{ + Table: &ast.Ident{ + NamePos: 464, + NameEnd: 472, + Name: "Products", + }, + Hint: (*ast.Hint)(nil), + As: &ast.AsAlias{ + As: 473, + Alias: &ast.Ident{ + NamePos: 476, + NameEnd: 483, + Name: "product", + }, + }, + Sample: (*ast.TableSample)(nil), + }, + Right: &ast.TableName{ + Table: &ast.Ident{ + NamePos: 489, + NameEnd: 499, + Name: "Categories", + }, + Hint: (*ast.Hint)(nil), + As: &ast.AsAlias{ + As: 500, + Alias: &ast.Ident{ + NamePos: 503, + NameEnd: 511, + Name: "category", + }, + }, + Sample: (*ast.TableSample)(nil), + }, + Cond: &ast.On{ + On: 550, + Expr: &ast.BinaryExpr{ + Op: "=", + Left: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 553, + NameEnd: 560, + Name: "product", + }, + &ast.Ident{ + NamePos: 561, + NameEnd: 572, + Name: "category_id", + }, + }, + }, + Right: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 575, + NameEnd: 583, + Name: "category", + }, + &ast.Ident{ + NamePos: 584, + NameEnd: 586, + Name: "id", + }, + }, + }, + }, + }, + }, + }, + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, + }, + }, + &ast.ExprArg{ + Expr: &ast.TypelessStructLiteral{ + Struct: 597, + Rparen: 626, + Values: []ast.TypelessStructLiteralArg{ + &ast.Alias{ + Expr: &ast.IntLiteral{ + ValuePos: 604, + ValueEnd: 607, + Base: 10, + Value: "100", + }, + As: &ast.AsAlias{ + As: 608, + Alias: &ast.Ident{ + NamePos: 611, + NameEnd: 626, + Name: "maxOutputTokens", + }, + }, + }, + }, + }, + }, + }, + NamedArgs: []*ast.NamedArg(nil), + Hint: &ast.Hint{ + Atmark: 635, + Rbrace: 666, + Records: []*ast.HintRecord{ + &ast.HintRecord{ + Key: &ast.Ident{ + NamePos: 637, + NameEnd: 664, + Name: "remote_udf_max_rows_per_rpc", + }, + Value: &ast.IntLiteral{ + ValuePos: 665, + ValueEnd: 666, + Base: 10, + Value: "1", + }, + }, + }, + }, + Sample: (*ast.TableSample)(nil), + }, + }, + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, +} + +--- SQL +SELECT product_id, product_name, content FROM ML.PREDICT(MODEL TextBison, (SELECT product.id AS product_id, product.name AS product_name, CONCAT("Is this product safe for infants\?", "\n", "Product Name: ", product.name, "\n", "Category Name: ", category.name, "\n", "Product Description:", product.description) AS prompt FROM Products AS product INNER JOIN Categories AS category ON product.category_id = category.id), STRUCT(100 AS maxOutputTokens)) @{remote_udf_max_rows_per_rpc=1} diff --git a/testdata/result/statement/select_from_change_stream.sql.txt b/testdata/result/statement/select_from_change_stream.sql.txt new file mode 100644 index 00000000..86d71125 --- /dev/null +++ b/testdata/result/statement/select_from_change_stream.sql.txt @@ -0,0 +1,99 @@ +--- select_from_change_stream.sql +SELECT ChangeRecord FROM READ_SingersNameStream ( + start_timestamp => "2022-05-01T09:00:00Z", + end_timestamp => NULL, + partition_token => NULL, + heartbeat_milliseconds => 10000 +) +--- AST +&ast.QueryStatement{ + Hint: (*ast.Hint)(nil), + With: (*ast.With)(nil), + Query: &ast.Select{ + Select: 0, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 7, + NameEnd: 19, + Name: "ChangeRecord", + }, + }, + }, + From: &ast.From{ + From: 20, + Source: &ast.TVFCallExpr{ + Rparen: 181, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 25, + NameEnd: 47, + Name: "READ_SingersNameStream", + }, + }, + }, + Args: []ast.TVFArg(nil), + NamedArgs: []*ast.NamedArg{ + &ast.NamedArg{ + Name: &ast.Ident{ + NamePos: 52, + NameEnd: 67, + Name: "start_timestamp", + }, + Value: &ast.StringLiteral{ + ValuePos: 71, + ValueEnd: 93, + Value: "2022-05-01T09:00:00Z", + }, + }, + &ast.NamedArg{ + Name: &ast.Ident{ + NamePos: 97, + NameEnd: 110, + Name: "end_timestamp", + }, + Value: &ast.NullLiteral{ + Null: 114, + }, + }, + &ast.NamedArg{ + Name: &ast.Ident{ + NamePos: 122, + NameEnd: 137, + Name: "partition_token", + }, + Value: &ast.NullLiteral{ + Null: 141, + }, + }, + &ast.NamedArg{ + Name: &ast.Ident{ + NamePos: 149, + NameEnd: 171, + Name: "heartbeat_milliseconds", + }, + Value: &ast.IntLiteral{ + ValuePos: 175, + ValueEnd: 180, + Base: 10, + Value: "10000", + }, + }, + }, + Hint: (*ast.Hint)(nil), + Sample: (*ast.TableSample)(nil), + }, + }, + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, +} + +--- SQL +SELECT ChangeRecord FROM READ_SingersNameStream(start_timestamp => "2022-05-01T09:00:00Z", end_timestamp => NULL, partition_token => NULL, heartbeat_milliseconds => 10000) diff --git a/testdata/result/statement/select_from_ml_predict_hint.sql.txt b/testdata/result/statement/select_from_ml_predict_hint.sql.txt new file mode 100644 index 00000000..19165cb0 --- /dev/null +++ b/testdata/result/statement/select_from_ml_predict_hint.sql.txt @@ -0,0 +1,194 @@ +--- select_from_ml_predict_hint.sql +-- https://cloud.google.com/spanner/docs/ml-tutorial-generative-ai?hl=en#register_a_generative_ai_model_in_a_schema +SELECT content +FROM ML.PREDICT( + MODEL TextBison, + (SELECT "Is 13 prime?" AS prompt), + STRUCT(256 AS maxOutputTokens, 0.2 AS temperature, 40 as topK, 0.95 AS topP) + ) @{remote_udf_max_rows_per_rpc=1} +--- AST +&ast.QueryStatement{ + Hint: (*ast.Hint)(nil), + With: (*ast.With)(nil), + Query: &ast.Select{ + Select: 116, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 123, + NameEnd: 130, + Name: "content", + }, + }, + }, + From: &ast.From{ + From: 131, + Source: &ast.TVFCallExpr{ + Rparen: 306, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 136, + NameEnd: 138, + Name: "ML", + }, + &ast.Ident{ + NamePos: 139, + NameEnd: 146, + Name: "PREDICT", + }, + }, + }, + Args: []ast.TVFArg{ + &ast.ModelArg{ + Model: 156, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 162, + NameEnd: 171, + Name: "TextBison", + }, + }, + }, + }, + &ast.ExprArg{ + Expr: &ast.ScalarSubQuery{ + Lparen: 181, + Rparen: 213, + Query: &ast.Select{ + Select: 182, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.Alias{ + Expr: &ast.StringLiteral{ + ValuePos: 189, + ValueEnd: 203, + Value: "Is 13 prime?", + }, + As: &ast.AsAlias{ + As: 204, + Alias: &ast.Ident{ + NamePos: 207, + NameEnd: 213, + Name: "prompt", + }, + }, + }, + }, + From: (*ast.From)(nil), + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, + }, + }, + &ast.ExprArg{ + Expr: &ast.TypelessStructLiteral{ + Struct: 224, + Rparen: 299, + Values: []ast.TypelessStructLiteralArg{ + &ast.Alias{ + Expr: &ast.IntLiteral{ + ValuePos: 231, + ValueEnd: 234, + Base: 10, + Value: "256", + }, + As: &ast.AsAlias{ + As: 235, + Alias: &ast.Ident{ + NamePos: 238, + NameEnd: 253, + Name: "maxOutputTokens", + }, + }, + }, + &ast.Alias{ + Expr: &ast.FloatLiteral{ + ValuePos: 255, + ValueEnd: 258, + Value: "0.2", + }, + As: &ast.AsAlias{ + As: 259, + Alias: &ast.Ident{ + NamePos: 262, + NameEnd: 273, + Name: "temperature", + }, + }, + }, + &ast.Alias{ + Expr: &ast.IntLiteral{ + ValuePos: 275, + ValueEnd: 277, + Base: 10, + Value: "40", + }, + As: &ast.AsAlias{ + As: 278, + Alias: &ast.Ident{ + NamePos: 281, + NameEnd: 285, + Name: "topK", + }, + }, + }, + &ast.Alias{ + Expr: &ast.FloatLiteral{ + ValuePos: 287, + ValueEnd: 291, + Value: "0.95", + }, + As: &ast.AsAlias{ + As: 292, + Alias: &ast.Ident{ + NamePos: 295, + NameEnd: 299, + Name: "topP", + }, + }, + }, + }, + }, + }, + }, + NamedArgs: []*ast.NamedArg(nil), + Hint: &ast.Hint{ + Atmark: 308, + Rbrace: 339, + Records: []*ast.HintRecord{ + &ast.HintRecord{ + Key: &ast.Ident{ + NamePos: 310, + NameEnd: 337, + Name: "remote_udf_max_rows_per_rpc", + }, + Value: &ast.IntLiteral{ + ValuePos: 338, + ValueEnd: 339, + Base: 10, + Value: "1", + }, + }, + }, + }, + Sample: (*ast.TableSample)(nil), + }, + }, + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, +} + +--- SQL +SELECT content FROM ML.PREDICT(MODEL TextBison, (SELECT "Is 13 prime\?" AS prompt), STRUCT(256 AS maxOutputTokens, 0.2 AS temperature, 40 AS topK, 0.95 AS topP)) @{remote_udf_max_rows_per_rpc=1} diff --git a/testdata/result/statement/select_from_ml_predict_simple.sql.txt b/testdata/result/statement/select_from_ml_predict_simple.sql.txt new file mode 100644 index 00000000..62b8e238 --- /dev/null +++ b/testdata/result/statement/select_from_ml_predict_simple.sql.txt @@ -0,0 +1,93 @@ +--- select_from_ml_predict_simple.sql +SELECT id, color, value +FROM ML.PREDICT(MODEL DiamondAppraise, TABLE Diamonds) +--- AST +&ast.QueryStatement{ + Hint: (*ast.Hint)(nil), + With: (*ast.With)(nil), + Query: &ast.Select{ + Select: 0, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 7, + NameEnd: 9, + Name: "id", + }, + }, + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 11, + NameEnd: 16, + Name: "color", + }, + }, + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 18, + NameEnd: 23, + Name: "value", + }, + }, + }, + From: &ast.From{ + From: 24, + Source: &ast.TVFCallExpr{ + Rparen: 77, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 29, + NameEnd: 31, + Name: "ML", + }, + &ast.Ident{ + NamePos: 32, + NameEnd: 39, + Name: "PREDICT", + }, + }, + }, + Args: []ast.TVFArg{ + &ast.ModelArg{ + Model: 40, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 46, + NameEnd: 61, + Name: "DiamondAppraise", + }, + }, + }, + }, + &ast.TableArg{ + Table: 63, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 69, + NameEnd: 77, + Name: "Diamonds", + }, + }, + }, + }, + }, + NamedArgs: []*ast.NamedArg(nil), + Hint: (*ast.Hint)(nil), + Sample: (*ast.TableSample)(nil), + }, + }, + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, +} + +--- SQL +SELECT id, color, value FROM ML.PREDICT(MODEL DiamondAppraise, TABLE Diamonds) diff --git a/testdata/result/statement/select_from_ml_predict_textbison.sql.txt b/testdata/result/statement/select_from_ml_predict_textbison.sql.txt new file mode 100644 index 00000000..f89b83e9 --- /dev/null +++ b/testdata/result/statement/select_from_ml_predict_textbison.sql.txt @@ -0,0 +1,399 @@ +--- select_from_ml_predict_textbison.sql +SELECT product_id, product_name, content +FROM ML.PREDICT( + MODEL TextBison, + (SELECT + product.id as product_id, + product.name as product_name, + CONCAT("Is this product safe for infants?", "\n", + "Product Name: ", product.name, "\n", + "Category Name: ", category.name, "\n", + "Product Description:", product.description) AS prompt + FROM + Products AS product JOIN Categories AS category + ON product.category_id = category.id), + STRUCT(100 AS maxOutputTokens) + ) @{remote_udf_max_rows_per_rpc=1} +--- AST +&ast.QueryStatement{ + Hint: (*ast.Hint)(nil), + With: (*ast.With)(nil), + Query: &ast.Select{ + Select: 0, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 7, + NameEnd: 17, + Name: "product_id", + }, + }, + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 19, + NameEnd: 31, + Name: "product_name", + }, + }, + &ast.ExprSelectItem{ + Expr: &ast.Ident{ + NamePos: 33, + NameEnd: 40, + Name: "content", + }, + }, + }, + From: &ast.From{ + From: 41, + Source: &ast.TVFCallExpr{ + Rparen: 633, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 46, + NameEnd: 48, + Name: "ML", + }, + &ast.Ident{ + NamePos: 49, + NameEnd: 56, + Name: "PREDICT", + }, + }, + }, + Args: []ast.TVFArg{ + &ast.ModelArg{ + Model: 66, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 72, + NameEnd: 81, + Name: "TextBison", + }, + }, + }, + }, + &ast.ExprArg{ + Expr: &ast.ScalarSubQuery{ + Lparen: 91, + Rparen: 586, + Query: &ast.Select{ + Select: 92, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.Alias{ + Expr: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 112, + NameEnd: 119, + Name: "product", + }, + &ast.Ident{ + NamePos: 120, + NameEnd: 122, + Name: "id", + }, + }, + }, + As: &ast.AsAlias{ + As: 123, + Alias: &ast.Ident{ + NamePos: 126, + NameEnd: 136, + Name: "product_id", + }, + }, + }, + &ast.Alias{ + Expr: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 151, + NameEnd: 158, + Name: "product", + }, + &ast.Ident{ + NamePos: 159, + NameEnd: 163, + Name: "name", + }, + }, + }, + As: &ast.AsAlias{ + As: 164, + Alias: &ast.Ident{ + NamePos: 167, + NameEnd: 179, + Name: "product_name", + }, + }, + }, + &ast.Alias{ + Expr: &ast.CallExpr{ + Rparen: 425, + Func: &ast.Ident{ + NamePos: 194, + NameEnd: 200, + Name: "CONCAT", + }, + Distinct: false, + Args: []ast.Arg{ + &ast.ExprArg{ + Expr: &ast.StringLiteral{ + ValuePos: 201, + ValueEnd: 236, + Value: "Is this product safe for infants?", + }, + }, + &ast.ExprArg{ + Expr: &ast.StringLiteral{ + ValuePos: 238, + ValueEnd: 242, + Value: "\n", + }, + }, + &ast.ExprArg{ + Expr: &ast.StringLiteral{ + ValuePos: 264, + ValueEnd: 280, + Value: "Product Name: ", + }, + }, + &ast.ExprArg{ + Expr: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 282, + NameEnd: 289, + Name: "product", + }, + &ast.Ident{ + NamePos: 290, + NameEnd: 294, + Name: "name", + }, + }, + }, + }, + &ast.ExprArg{ + Expr: &ast.StringLiteral{ + ValuePos: 296, + ValueEnd: 300, + Value: "\n", + }, + }, + &ast.ExprArg{ + Expr: &ast.StringLiteral{ + ValuePos: 322, + ValueEnd: 339, + Value: "Category Name: ", + }, + }, + &ast.ExprArg{ + Expr: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 341, + NameEnd: 349, + Name: "category", + }, + &ast.Ident{ + NamePos: 350, + NameEnd: 354, + Name: "name", + }, + }, + }, + }, + &ast.ExprArg{ + Expr: &ast.StringLiteral{ + ValuePos: 356, + ValueEnd: 360, + Value: "\n", + }, + }, + &ast.ExprArg{ + Expr: &ast.StringLiteral{ + ValuePos: 382, + ValueEnd: 404, + Value: "Product Description:", + }, + }, + &ast.ExprArg{ + Expr: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 406, + NameEnd: 413, + Name: "product", + }, + &ast.Ident{ + NamePos: 414, + NameEnd: 425, + Name: "description", + }, + }, + }, + }, + }, + NamedArgs: []*ast.NamedArg(nil), + NullHandling: nil, + Having: nil, + }, + As: &ast.AsAlias{ + As: 427, + Alias: &ast.Ident{ + NamePos: 430, + NameEnd: 436, + Name: "prompt", + }, + }, + }, + }, + From: &ast.From{ + From: 446, + Source: &ast.Join{ + Op: "INNER JOIN", + Method: "", + Hint: (*ast.Hint)(nil), + Left: &ast.TableName{ + Table: &ast.Ident{ + NamePos: 464, + NameEnd: 472, + Name: "Products", + }, + Hint: (*ast.Hint)(nil), + As: &ast.AsAlias{ + As: 473, + Alias: &ast.Ident{ + NamePos: 476, + NameEnd: 483, + Name: "product", + }, + }, + Sample: (*ast.TableSample)(nil), + }, + Right: &ast.TableName{ + Table: &ast.Ident{ + NamePos: 489, + NameEnd: 499, + Name: "Categories", + }, + Hint: (*ast.Hint)(nil), + As: &ast.AsAlias{ + As: 500, + Alias: &ast.Ident{ + NamePos: 503, + NameEnd: 511, + Name: "category", + }, + }, + Sample: (*ast.TableSample)(nil), + }, + Cond: &ast.On{ + On: 550, + Expr: &ast.BinaryExpr{ + Op: "=", + Left: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 553, + NameEnd: 560, + Name: "product", + }, + &ast.Ident{ + NamePos: 561, + NameEnd: 572, + Name: "category_id", + }, + }, + }, + Right: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 575, + NameEnd: 583, + Name: "category", + }, + &ast.Ident{ + NamePos: 584, + NameEnd: 586, + Name: "id", + }, + }, + }, + }, + }, + }, + }, + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, + }, + }, + &ast.ExprArg{ + Expr: &ast.TypelessStructLiteral{ + Struct: 597, + Rparen: 626, + Values: []ast.TypelessStructLiteralArg{ + &ast.Alias{ + Expr: &ast.IntLiteral{ + ValuePos: 604, + ValueEnd: 607, + Base: 10, + Value: "100", + }, + As: &ast.AsAlias{ + As: 608, + Alias: &ast.Ident{ + NamePos: 611, + NameEnd: 626, + Name: "maxOutputTokens", + }, + }, + }, + }, + }, + }, + }, + NamedArgs: []*ast.NamedArg(nil), + Hint: &ast.Hint{ + Atmark: 635, + Rbrace: 666, + Records: []*ast.HintRecord{ + &ast.HintRecord{ + Key: &ast.Ident{ + NamePos: 637, + NameEnd: 664, + Name: "remote_udf_max_rows_per_rpc", + }, + Value: &ast.IntLiteral{ + ValuePos: 665, + ValueEnd: 666, + Base: 10, + Value: "1", + }, + }, + }, + }, + Sample: (*ast.TableSample)(nil), + }, + }, + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, +} + +--- SQL +SELECT product_id, product_name, content FROM ML.PREDICT(MODEL TextBison, (SELECT product.id AS product_id, product.name AS product_name, CONCAT("Is this product safe for infants\?", "\n", "Product Name: ", product.name, "\n", "Category Name: ", category.name, "\n", "Product Description:", product.description) AS prompt FROM Products AS product INNER JOIN Categories AS category ON product.category_id = category.id), STRUCT(100 AS maxOutputTokens)) @{remote_udf_max_rows_per_rpc=1} diff --git a/testdata/result/statement/update_with_ml_predict.sql.txt b/testdata/result/statement/update_with_ml_predict.sql.txt new file mode 100644 index 00000000..cb5ab0ee --- /dev/null +++ b/testdata/result/statement/update_with_ml_predict.sql.txt @@ -0,0 +1,226 @@ +--- update_with_ml_predict.sql +-- https://cloud.google.com/spanner/docs/backfill-embeddings?hl=en#backfill +UPDATE products +SET + products.desc_embed = ( + SELECT embeddings.values + FROM SAFE.ML.PREDICT( + MODEL gecko_model, + (SELECT products.description AS content) + ) @{remote_udf_max_rows_per_rpc=200} + ), + products.desc_embed_model_version = 3 +WHERE products.desc_embed IS NULL +--- AST +&ast.Update{ + Update: 76, + TableName: &ast.Ident{ + NamePos: 83, + NameEnd: 91, + Name: "products", + }, + As: (*ast.AsAlias)(nil), + Updates: []*ast.UpdateItem{ + &ast.UpdateItem{ + Path: []*ast.Ident{ + &ast.Ident{ + NamePos: 100, + NameEnd: 108, + Name: "products", + }, + &ast.Ident{ + NamePos: 109, + NameEnd: 119, + Name: "desc_embed", + }, + }, + DefaultExpr: &ast.DefaultExpr{ + DefaultPos: -1, + Default: false, + Expr: &ast.ScalarSubQuery{ + Lparen: 122, + Rparen: 333, + Query: &ast.Select{ + Select: 132, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.ExprSelectItem{ + Expr: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 139, + NameEnd: 149, + Name: "embeddings", + }, + &ast.Ident{ + NamePos: 150, + NameEnd: 156, + Name: "values", + }, + }, + }, + }, + }, + From: &ast.From{ + From: 165, + Source: &ast.TVFCallExpr{ + Rparen: 292, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 170, + NameEnd: 174, + Name: "SAFE", + }, + &ast.Ident{ + NamePos: 175, + NameEnd: 177, + Name: "ML", + }, + &ast.Ident{ + NamePos: 178, + NameEnd: 185, + Name: "PREDICT", + }, + }, + }, + Args: []ast.TVFArg{ + &ast.ModelArg{ + Model: 203, + Name: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 209, + NameEnd: 220, + Name: "gecko_model", + }, + }, + }, + }, + &ast.ExprArg{ + Expr: &ast.ScalarSubQuery{ + Lparen: 238, + Rparen: 277, + Query: &ast.Select{ + Select: 239, + Distinct: false, + As: nil, + Results: []ast.SelectItem{ + &ast.Alias{ + Expr: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 246, + NameEnd: 254, + Name: "products", + }, + &ast.Ident{ + NamePos: 255, + NameEnd: 266, + Name: "description", + }, + }, + }, + As: &ast.AsAlias{ + As: 267, + Alias: &ast.Ident{ + NamePos: 270, + NameEnd: 277, + Name: "content", + }, + }, + }, + }, + From: (*ast.From)(nil), + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, + }, + }, + }, + NamedArgs: []*ast.NamedArg(nil), + Hint: &ast.Hint{ + Atmark: 294, + Rbrace: 327, + Records: []*ast.HintRecord{ + &ast.HintRecord{ + Key: &ast.Ident{ + NamePos: 296, + NameEnd: 323, + Name: "remote_udf_max_rows_per_rpc", + }, + Value: &ast.IntLiteral{ + ValuePos: 324, + ValueEnd: 327, + Base: 10, + Value: "200", + }, + }, + }, + }, + Sample: (*ast.TableSample)(nil), + }, + }, + Where: (*ast.Where)(nil), + GroupBy: (*ast.GroupBy)(nil), + Having: (*ast.Having)(nil), + OrderBy: (*ast.OrderBy)(nil), + Limit: (*ast.Limit)(nil), + }, + }, + }, + }, + &ast.UpdateItem{ + Path: []*ast.Ident{ + &ast.Ident{ + NamePos: 340, + NameEnd: 348, + Name: "products", + }, + &ast.Ident{ + NamePos: 349, + NameEnd: 373, + Name: "desc_embed_model_version", + }, + }, + DefaultExpr: &ast.DefaultExpr{ + DefaultPos: -1, + Default: false, + Expr: &ast.IntLiteral{ + ValuePos: 376, + ValueEnd: 377, + Base: 10, + Value: "3", + }, + }, + }, + }, + Where: &ast.Where{ + Where: 378, + Expr: &ast.IsNullExpr{ + Null: 407, + Not: false, + Left: &ast.Path{ + Idents: []*ast.Ident{ + &ast.Ident{ + NamePos: 384, + NameEnd: 392, + Name: "products", + }, + &ast.Ident{ + NamePos: 393, + NameEnd: 403, + Name: "desc_embed", + }, + }, + }, + }, + }, +} + +--- SQL +UPDATE products SET products.desc_embed = (SELECT embeddings.values FROM SAFE.ML.PREDICT(MODEL gecko_model, (SELECT products.description AS content)) @{remote_udf_max_rows_per_rpc=200}), products.desc_embed_model_version = 3 WHERE products.desc_embed IS NULL From bd2fd12e67653ea7cd6112e8aea0ea749890d764 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Sat, 23 Nov 2024 08:32:57 +0900 Subject: [PATCH 3/7] Do go test --update --- ast/pos.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ast/pos.go b/ast/pos.go index 334128e4..9cbc725c 100644 --- a/ast/pos.go +++ b/ast/pos.go @@ -403,7 +403,7 @@ func (t *TVFCallExpr) Pos() token.Pos { } func (t *TVFCallExpr) End() token.Pos { - return posAdd(t.Rparen, 1) + return posChoice(nodeEnd(nodeChoice(wrapNode(t.Sample), wrapNode(t.Hint))), posAdd(t.Rparen, 1)) } func (e *ExprArg) Pos() token.Pos { From cdf97d98b940bac9e3243e5183d3786e2d460128 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Sat, 23 Nov 2024 08:34:27 +0900 Subject: [PATCH 4/7] Fix indentation --- .../query/select_from_ml_predict_hint.sql | 8 +- .../select_from_ml_predict_textbison.sql | 26 +-- .../query/select_from_ml_predict_hint.sql.txt | 88 ++++---- .../select_from_ml_predict_textbison.sql.txt | 198 +++++++++--------- .../select_from_ml_predict_hint.sql.txt | 88 ++++---- .../select_from_ml_predict_textbison.sql.txt | 198 +++++++++--------- 6 files changed, 303 insertions(+), 303 deletions(-) diff --git a/testdata/input/query/select_from_ml_predict_hint.sql b/testdata/input/query/select_from_ml_predict_hint.sql index 8f2a0f0c..2d17a97c 100644 --- a/testdata/input/query/select_from_ml_predict_hint.sql +++ b/testdata/input/query/select_from_ml_predict_hint.sql @@ -1,7 +1,7 @@ -- https://cloud.google.com/spanner/docs/ml-tutorial-generative-ai?hl=en#register_a_generative_ai_model_in_a_schema SELECT content FROM ML.PREDICT( - MODEL TextBison, - (SELECT "Is 13 prime?" AS prompt), - STRUCT(256 AS maxOutputTokens, 0.2 AS temperature, 40 as topK, 0.95 AS topP) - ) @{remote_udf_max_rows_per_rpc=1} \ No newline at end of file + MODEL TextBison, + (SELECT "Is 13 prime?" AS prompt), + STRUCT(256 AS maxOutputTokens, 0.2 AS temperature, 40 as topK, 0.95 AS topP) +) @{remote_udf_max_rows_per_rpc=1} \ No newline at end of file diff --git a/testdata/input/query/select_from_ml_predict_textbison.sql b/testdata/input/query/select_from_ml_predict_textbison.sql index 4c47c225..6352ba90 100644 --- a/testdata/input/query/select_from_ml_predict_textbison.sql +++ b/testdata/input/query/select_from_ml_predict_textbison.sql @@ -1,15 +1,15 @@ SELECT product_id, product_name, content FROM ML.PREDICT( - MODEL TextBison, - (SELECT - product.id as product_id, - product.name as product_name, - CONCAT("Is this product safe for infants?", "\n", - "Product Name: ", product.name, "\n", - "Category Name: ", category.name, "\n", - "Product Description:", product.description) AS prompt - FROM - Products AS product JOIN Categories AS category - ON product.category_id = category.id), - STRUCT(100 AS maxOutputTokens) - ) @{remote_udf_max_rows_per_rpc=1} \ No newline at end of file + MODEL TextBison, + (SELECT + product.id as product_id, + product.name as product_name, + CONCAT("Is this product safe for infants?", "\n", + "Product Name: ", product.name, "\n", + "Category Name: ", category.name, "\n", + "Product Description:", product.description) AS prompt + FROM + Products AS product JOIN Categories AS category + ON product.category_id = category.id), + STRUCT(100 AS maxOutputTokens) +) @{remote_udf_max_rows_per_rpc=1} \ No newline at end of file diff --git a/testdata/result/query/select_from_ml_predict_hint.sql.txt b/testdata/result/query/select_from_ml_predict_hint.sql.txt index 19165cb0..b3c5d118 100644 --- a/testdata/result/query/select_from_ml_predict_hint.sql.txt +++ b/testdata/result/query/select_from_ml_predict_hint.sql.txt @@ -2,10 +2,10 @@ -- https://cloud.google.com/spanner/docs/ml-tutorial-generative-ai?hl=en#register_a_generative_ai_model_in_a_schema SELECT content FROM ML.PREDICT( - MODEL TextBison, - (SELECT "Is 13 prime?" AS prompt), - STRUCT(256 AS maxOutputTokens, 0.2 AS temperature, 40 as topK, 0.95 AS topP) - ) @{remote_udf_max_rows_per_rpc=1} + MODEL TextBison, + (SELECT "Is 13 prime?" AS prompt), + STRUCT(256 AS maxOutputTokens, 0.2 AS temperature, 40 as topK, 0.95 AS topP) +) @{remote_udf_max_rows_per_rpc=1} --- AST &ast.QueryStatement{ Hint: (*ast.Hint)(nil), @@ -26,7 +26,7 @@ FROM ML.PREDICT( From: &ast.From{ From: 131, Source: &ast.TVFCallExpr{ - Rparen: 306, + Rparen: 289, Name: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ @@ -43,12 +43,12 @@ FROM ML.PREDICT( }, Args: []ast.TVFArg{ &ast.ModelArg{ - Model: 156, + Model: 152, Name: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 162, - NameEnd: 171, + NamePos: 158, + NameEnd: 167, Name: "TextBison", }, }, @@ -56,24 +56,24 @@ FROM ML.PREDICT( }, &ast.ExprArg{ Expr: &ast.ScalarSubQuery{ - Lparen: 181, - Rparen: 213, + Lparen: 173, + Rparen: 205, Query: &ast.Select{ - Select: 182, + Select: 174, Distinct: false, As: nil, Results: []ast.SelectItem{ &ast.Alias{ Expr: &ast.StringLiteral{ - ValuePos: 189, - ValueEnd: 203, + ValuePos: 181, + ValueEnd: 195, Value: "Is 13 prime?", }, As: &ast.AsAlias{ - As: 204, + As: 196, Alias: &ast.Ident{ - NamePos: 207, - NameEnd: 213, + NamePos: 199, + NameEnd: 205, Name: "prompt", }, }, @@ -90,67 +90,67 @@ FROM ML.PREDICT( }, &ast.ExprArg{ Expr: &ast.TypelessStructLiteral{ - Struct: 224, - Rparen: 299, + Struct: 212, + Rparen: 287, Values: []ast.TypelessStructLiteralArg{ &ast.Alias{ Expr: &ast.IntLiteral{ - ValuePos: 231, - ValueEnd: 234, + ValuePos: 219, + ValueEnd: 222, Base: 10, Value: "256", }, As: &ast.AsAlias{ - As: 235, + As: 223, Alias: &ast.Ident{ - NamePos: 238, - NameEnd: 253, + NamePos: 226, + NameEnd: 241, Name: "maxOutputTokens", }, }, }, &ast.Alias{ Expr: &ast.FloatLiteral{ - ValuePos: 255, - ValueEnd: 258, + ValuePos: 243, + ValueEnd: 246, Value: "0.2", }, As: &ast.AsAlias{ - As: 259, + As: 247, Alias: &ast.Ident{ - NamePos: 262, - NameEnd: 273, + NamePos: 250, + NameEnd: 261, Name: "temperature", }, }, }, &ast.Alias{ Expr: &ast.IntLiteral{ - ValuePos: 275, - ValueEnd: 277, + ValuePos: 263, + ValueEnd: 265, Base: 10, Value: "40", }, As: &ast.AsAlias{ - As: 278, + As: 266, Alias: &ast.Ident{ - NamePos: 281, - NameEnd: 285, + NamePos: 269, + NameEnd: 273, Name: "topK", }, }, }, &ast.Alias{ Expr: &ast.FloatLiteral{ - ValuePos: 287, - ValueEnd: 291, + ValuePos: 275, + ValueEnd: 279, Value: "0.95", }, As: &ast.AsAlias{ - As: 292, + As: 280, Alias: &ast.Ident{ - NamePos: 295, - NameEnd: 299, + NamePos: 283, + NameEnd: 287, Name: "topP", }, }, @@ -161,18 +161,18 @@ FROM ML.PREDICT( }, NamedArgs: []*ast.NamedArg(nil), Hint: &ast.Hint{ - Atmark: 308, - Rbrace: 339, + Atmark: 291, + Rbrace: 322, Records: []*ast.HintRecord{ &ast.HintRecord{ Key: &ast.Ident{ - NamePos: 310, - NameEnd: 337, + NamePos: 293, + NameEnd: 320, Name: "remote_udf_max_rows_per_rpc", }, Value: &ast.IntLiteral{ - ValuePos: 338, - ValueEnd: 339, + ValuePos: 321, + ValueEnd: 322, Base: 10, Value: "1", }, diff --git a/testdata/result/query/select_from_ml_predict_textbison.sql.txt b/testdata/result/query/select_from_ml_predict_textbison.sql.txt index f89b83e9..de4d7761 100644 --- a/testdata/result/query/select_from_ml_predict_textbison.sql.txt +++ b/testdata/result/query/select_from_ml_predict_textbison.sql.txt @@ -1,19 +1,19 @@ --- select_from_ml_predict_textbison.sql SELECT product_id, product_name, content FROM ML.PREDICT( - MODEL TextBison, - (SELECT - product.id as product_id, - product.name as product_name, - CONCAT("Is this product safe for infants?", "\n", - "Product Name: ", product.name, "\n", - "Category Name: ", category.name, "\n", - "Product Description:", product.description) AS prompt - FROM - Products AS product JOIN Categories AS category - ON product.category_id = category.id), - STRUCT(100 AS maxOutputTokens) - ) @{remote_udf_max_rows_per_rpc=1} + MODEL TextBison, + (SELECT + product.id as product_id, + product.name as product_name, + CONCAT("Is this product safe for infants?", "\n", + "Product Name: ", product.name, "\n", + "Category Name: ", category.name, "\n", + "Product Description:", product.description) AS prompt + FROM + Products AS product JOIN Categories AS category + ON product.category_id = category.id), + STRUCT(100 AS maxOutputTokens) +) @{remote_udf_max_rows_per_rpc=1} --- AST &ast.QueryStatement{ Hint: (*ast.Hint)(nil), @@ -48,7 +48,7 @@ FROM ML.PREDICT( From: &ast.From{ From: 41, Source: &ast.TVFCallExpr{ - Rparen: 633, + Rparen: 580, Name: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ @@ -65,12 +65,12 @@ FROM ML.PREDICT( }, Args: []ast.TVFArg{ &ast.ModelArg{ - Model: 66, + Model: 62, Name: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 72, - NameEnd: 81, + NamePos: 68, + NameEnd: 77, Name: "TextBison", }, }, @@ -78,10 +78,10 @@ FROM ML.PREDICT( }, &ast.ExprArg{ Expr: &ast.ScalarSubQuery{ - Lparen: 91, - Rparen: 586, + Lparen: 83, + Rparen: 542, Query: &ast.Select{ - Select: 92, + Select: 84, Distinct: false, As: nil, Results: []ast.SelectItem{ @@ -89,22 +89,22 @@ FROM ML.PREDICT( Expr: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 112, - NameEnd: 119, + NamePos: 100, + NameEnd: 107, Name: "product", }, &ast.Ident{ - NamePos: 120, - NameEnd: 122, + NamePos: 108, + NameEnd: 110, Name: "id", }, }, }, As: &ast.AsAlias{ - As: 123, + As: 111, Alias: &ast.Ident{ - NamePos: 126, - NameEnd: 136, + NamePos: 114, + NameEnd: 124, Name: "product_id", }, }, @@ -113,54 +113,54 @@ FROM ML.PREDICT( Expr: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 151, - NameEnd: 158, + NamePos: 135, + NameEnd: 142, Name: "product", }, &ast.Ident{ - NamePos: 159, - NameEnd: 163, + NamePos: 143, + NameEnd: 147, Name: "name", }, }, }, As: &ast.AsAlias{ - As: 164, + As: 148, Alias: &ast.Ident{ - NamePos: 167, - NameEnd: 179, + NamePos: 151, + NameEnd: 163, Name: "product_name", }, }, }, &ast.Alias{ Expr: &ast.CallExpr{ - Rparen: 425, + Rparen: 393, Func: &ast.Ident{ - NamePos: 194, - NameEnd: 200, + NamePos: 174, + NameEnd: 180, Name: "CONCAT", }, Distinct: false, Args: []ast.Arg{ &ast.ExprArg{ Expr: &ast.StringLiteral{ - ValuePos: 201, - ValueEnd: 236, + ValuePos: 181, + ValueEnd: 216, Value: "Is this product safe for infants?", }, }, &ast.ExprArg{ Expr: &ast.StringLiteral{ - ValuePos: 238, - ValueEnd: 242, + ValuePos: 218, + ValueEnd: 222, Value: "\n", }, }, &ast.ExprArg{ Expr: &ast.StringLiteral{ - ValuePos: 264, - ValueEnd: 280, + ValuePos: 240, + ValueEnd: 256, Value: "Product Name: ", }, }, @@ -168,13 +168,13 @@ FROM ML.PREDICT( Expr: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 282, - NameEnd: 289, + NamePos: 258, + NameEnd: 265, Name: "product", }, &ast.Ident{ - NamePos: 290, - NameEnd: 294, + NamePos: 266, + NameEnd: 270, Name: "name", }, }, @@ -182,15 +182,15 @@ FROM ML.PREDICT( }, &ast.ExprArg{ Expr: &ast.StringLiteral{ - ValuePos: 296, - ValueEnd: 300, + ValuePos: 272, + ValueEnd: 276, Value: "\n", }, }, &ast.ExprArg{ Expr: &ast.StringLiteral{ - ValuePos: 322, - ValueEnd: 339, + ValuePos: 294, + ValueEnd: 311, Value: "Category Name: ", }, }, @@ -198,13 +198,13 @@ FROM ML.PREDICT( Expr: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 341, - NameEnd: 349, + NamePos: 313, + NameEnd: 321, Name: "category", }, &ast.Ident{ - NamePos: 350, - NameEnd: 354, + NamePos: 322, + NameEnd: 326, Name: "name", }, }, @@ -212,15 +212,15 @@ FROM ML.PREDICT( }, &ast.ExprArg{ Expr: &ast.StringLiteral{ - ValuePos: 356, - ValueEnd: 360, + ValuePos: 328, + ValueEnd: 332, Value: "\n", }, }, &ast.ExprArg{ Expr: &ast.StringLiteral{ - ValuePos: 382, - ValueEnd: 404, + ValuePos: 350, + ValueEnd: 372, Value: "Product Description:", }, }, @@ -228,13 +228,13 @@ FROM ML.PREDICT( Expr: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 406, - NameEnd: 413, + NamePos: 374, + NameEnd: 381, Name: "product", }, &ast.Ident{ - NamePos: 414, - NameEnd: 425, + NamePos: 382, + NameEnd: 393, Name: "description", }, }, @@ -246,33 +246,33 @@ FROM ML.PREDICT( Having: nil, }, As: &ast.AsAlias{ - As: 427, + As: 395, Alias: &ast.Ident{ - NamePos: 430, - NameEnd: 436, + NamePos: 398, + NameEnd: 404, Name: "prompt", }, }, }, }, From: &ast.From{ - From: 446, + From: 410, Source: &ast.Join{ Op: "INNER JOIN", Method: "", Hint: (*ast.Hint)(nil), Left: &ast.TableName{ Table: &ast.Ident{ - NamePos: 464, - NameEnd: 472, + NamePos: 424, + NameEnd: 432, Name: "Products", }, Hint: (*ast.Hint)(nil), As: &ast.AsAlias{ - As: 473, + As: 433, Alias: &ast.Ident{ - NamePos: 476, - NameEnd: 483, + NamePos: 436, + NameEnd: 443, Name: "product", }, }, @@ -280,35 +280,35 @@ FROM ML.PREDICT( }, Right: &ast.TableName{ Table: &ast.Ident{ - NamePos: 489, - NameEnd: 499, + NamePos: 449, + NameEnd: 459, Name: "Categories", }, Hint: (*ast.Hint)(nil), As: &ast.AsAlias{ - As: 500, + As: 460, Alias: &ast.Ident{ - NamePos: 503, - NameEnd: 511, + NamePos: 463, + NameEnd: 471, Name: "category", }, }, Sample: (*ast.TableSample)(nil), }, Cond: &ast.On{ - On: 550, + On: 506, Expr: &ast.BinaryExpr{ Op: "=", Left: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 553, - NameEnd: 560, + NamePos: 509, + NameEnd: 516, Name: "product", }, &ast.Ident{ - NamePos: 561, - NameEnd: 572, + NamePos: 517, + NameEnd: 528, Name: "category_id", }, }, @@ -316,13 +316,13 @@ FROM ML.PREDICT( Right: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 575, - NameEnd: 583, + NamePos: 531, + NameEnd: 539, Name: "category", }, &ast.Ident{ - NamePos: 584, - NameEnd: 586, + NamePos: 540, + NameEnd: 542, Name: "id", }, }, @@ -341,21 +341,21 @@ FROM ML.PREDICT( }, &ast.ExprArg{ Expr: &ast.TypelessStructLiteral{ - Struct: 597, - Rparen: 626, + Struct: 549, + Rparen: 578, Values: []ast.TypelessStructLiteralArg{ &ast.Alias{ Expr: &ast.IntLiteral{ - ValuePos: 604, - ValueEnd: 607, + ValuePos: 556, + ValueEnd: 559, Base: 10, Value: "100", }, As: &ast.AsAlias{ - As: 608, + As: 560, Alias: &ast.Ident{ - NamePos: 611, - NameEnd: 626, + NamePos: 563, + NameEnd: 578, Name: "maxOutputTokens", }, }, @@ -366,18 +366,18 @@ FROM ML.PREDICT( }, NamedArgs: []*ast.NamedArg(nil), Hint: &ast.Hint{ - Atmark: 635, - Rbrace: 666, + Atmark: 582, + Rbrace: 613, Records: []*ast.HintRecord{ &ast.HintRecord{ Key: &ast.Ident{ - NamePos: 637, - NameEnd: 664, + NamePos: 584, + NameEnd: 611, Name: "remote_udf_max_rows_per_rpc", }, Value: &ast.IntLiteral{ - ValuePos: 665, - ValueEnd: 666, + ValuePos: 612, + ValueEnd: 613, Base: 10, Value: "1", }, diff --git a/testdata/result/statement/select_from_ml_predict_hint.sql.txt b/testdata/result/statement/select_from_ml_predict_hint.sql.txt index 19165cb0..b3c5d118 100644 --- a/testdata/result/statement/select_from_ml_predict_hint.sql.txt +++ b/testdata/result/statement/select_from_ml_predict_hint.sql.txt @@ -2,10 +2,10 @@ -- https://cloud.google.com/spanner/docs/ml-tutorial-generative-ai?hl=en#register_a_generative_ai_model_in_a_schema SELECT content FROM ML.PREDICT( - MODEL TextBison, - (SELECT "Is 13 prime?" AS prompt), - STRUCT(256 AS maxOutputTokens, 0.2 AS temperature, 40 as topK, 0.95 AS topP) - ) @{remote_udf_max_rows_per_rpc=1} + MODEL TextBison, + (SELECT "Is 13 prime?" AS prompt), + STRUCT(256 AS maxOutputTokens, 0.2 AS temperature, 40 as topK, 0.95 AS topP) +) @{remote_udf_max_rows_per_rpc=1} --- AST &ast.QueryStatement{ Hint: (*ast.Hint)(nil), @@ -26,7 +26,7 @@ FROM ML.PREDICT( From: &ast.From{ From: 131, Source: &ast.TVFCallExpr{ - Rparen: 306, + Rparen: 289, Name: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ @@ -43,12 +43,12 @@ FROM ML.PREDICT( }, Args: []ast.TVFArg{ &ast.ModelArg{ - Model: 156, + Model: 152, Name: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 162, - NameEnd: 171, + NamePos: 158, + NameEnd: 167, Name: "TextBison", }, }, @@ -56,24 +56,24 @@ FROM ML.PREDICT( }, &ast.ExprArg{ Expr: &ast.ScalarSubQuery{ - Lparen: 181, - Rparen: 213, + Lparen: 173, + Rparen: 205, Query: &ast.Select{ - Select: 182, + Select: 174, Distinct: false, As: nil, Results: []ast.SelectItem{ &ast.Alias{ Expr: &ast.StringLiteral{ - ValuePos: 189, - ValueEnd: 203, + ValuePos: 181, + ValueEnd: 195, Value: "Is 13 prime?", }, As: &ast.AsAlias{ - As: 204, + As: 196, Alias: &ast.Ident{ - NamePos: 207, - NameEnd: 213, + NamePos: 199, + NameEnd: 205, Name: "prompt", }, }, @@ -90,67 +90,67 @@ FROM ML.PREDICT( }, &ast.ExprArg{ Expr: &ast.TypelessStructLiteral{ - Struct: 224, - Rparen: 299, + Struct: 212, + Rparen: 287, Values: []ast.TypelessStructLiteralArg{ &ast.Alias{ Expr: &ast.IntLiteral{ - ValuePos: 231, - ValueEnd: 234, + ValuePos: 219, + ValueEnd: 222, Base: 10, Value: "256", }, As: &ast.AsAlias{ - As: 235, + As: 223, Alias: &ast.Ident{ - NamePos: 238, - NameEnd: 253, + NamePos: 226, + NameEnd: 241, Name: "maxOutputTokens", }, }, }, &ast.Alias{ Expr: &ast.FloatLiteral{ - ValuePos: 255, - ValueEnd: 258, + ValuePos: 243, + ValueEnd: 246, Value: "0.2", }, As: &ast.AsAlias{ - As: 259, + As: 247, Alias: &ast.Ident{ - NamePos: 262, - NameEnd: 273, + NamePos: 250, + NameEnd: 261, Name: "temperature", }, }, }, &ast.Alias{ Expr: &ast.IntLiteral{ - ValuePos: 275, - ValueEnd: 277, + ValuePos: 263, + ValueEnd: 265, Base: 10, Value: "40", }, As: &ast.AsAlias{ - As: 278, + As: 266, Alias: &ast.Ident{ - NamePos: 281, - NameEnd: 285, + NamePos: 269, + NameEnd: 273, Name: "topK", }, }, }, &ast.Alias{ Expr: &ast.FloatLiteral{ - ValuePos: 287, - ValueEnd: 291, + ValuePos: 275, + ValueEnd: 279, Value: "0.95", }, As: &ast.AsAlias{ - As: 292, + As: 280, Alias: &ast.Ident{ - NamePos: 295, - NameEnd: 299, + NamePos: 283, + NameEnd: 287, Name: "topP", }, }, @@ -161,18 +161,18 @@ FROM ML.PREDICT( }, NamedArgs: []*ast.NamedArg(nil), Hint: &ast.Hint{ - Atmark: 308, - Rbrace: 339, + Atmark: 291, + Rbrace: 322, Records: []*ast.HintRecord{ &ast.HintRecord{ Key: &ast.Ident{ - NamePos: 310, - NameEnd: 337, + NamePos: 293, + NameEnd: 320, Name: "remote_udf_max_rows_per_rpc", }, Value: &ast.IntLiteral{ - ValuePos: 338, - ValueEnd: 339, + ValuePos: 321, + ValueEnd: 322, Base: 10, Value: "1", }, diff --git a/testdata/result/statement/select_from_ml_predict_textbison.sql.txt b/testdata/result/statement/select_from_ml_predict_textbison.sql.txt index f89b83e9..de4d7761 100644 --- a/testdata/result/statement/select_from_ml_predict_textbison.sql.txt +++ b/testdata/result/statement/select_from_ml_predict_textbison.sql.txt @@ -1,19 +1,19 @@ --- select_from_ml_predict_textbison.sql SELECT product_id, product_name, content FROM ML.PREDICT( - MODEL TextBison, - (SELECT - product.id as product_id, - product.name as product_name, - CONCAT("Is this product safe for infants?", "\n", - "Product Name: ", product.name, "\n", - "Category Name: ", category.name, "\n", - "Product Description:", product.description) AS prompt - FROM - Products AS product JOIN Categories AS category - ON product.category_id = category.id), - STRUCT(100 AS maxOutputTokens) - ) @{remote_udf_max_rows_per_rpc=1} + MODEL TextBison, + (SELECT + product.id as product_id, + product.name as product_name, + CONCAT("Is this product safe for infants?", "\n", + "Product Name: ", product.name, "\n", + "Category Name: ", category.name, "\n", + "Product Description:", product.description) AS prompt + FROM + Products AS product JOIN Categories AS category + ON product.category_id = category.id), + STRUCT(100 AS maxOutputTokens) +) @{remote_udf_max_rows_per_rpc=1} --- AST &ast.QueryStatement{ Hint: (*ast.Hint)(nil), @@ -48,7 +48,7 @@ FROM ML.PREDICT( From: &ast.From{ From: 41, Source: &ast.TVFCallExpr{ - Rparen: 633, + Rparen: 580, Name: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ @@ -65,12 +65,12 @@ FROM ML.PREDICT( }, Args: []ast.TVFArg{ &ast.ModelArg{ - Model: 66, + Model: 62, Name: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 72, - NameEnd: 81, + NamePos: 68, + NameEnd: 77, Name: "TextBison", }, }, @@ -78,10 +78,10 @@ FROM ML.PREDICT( }, &ast.ExprArg{ Expr: &ast.ScalarSubQuery{ - Lparen: 91, - Rparen: 586, + Lparen: 83, + Rparen: 542, Query: &ast.Select{ - Select: 92, + Select: 84, Distinct: false, As: nil, Results: []ast.SelectItem{ @@ -89,22 +89,22 @@ FROM ML.PREDICT( Expr: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 112, - NameEnd: 119, + NamePos: 100, + NameEnd: 107, Name: "product", }, &ast.Ident{ - NamePos: 120, - NameEnd: 122, + NamePos: 108, + NameEnd: 110, Name: "id", }, }, }, As: &ast.AsAlias{ - As: 123, + As: 111, Alias: &ast.Ident{ - NamePos: 126, - NameEnd: 136, + NamePos: 114, + NameEnd: 124, Name: "product_id", }, }, @@ -113,54 +113,54 @@ FROM ML.PREDICT( Expr: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 151, - NameEnd: 158, + NamePos: 135, + NameEnd: 142, Name: "product", }, &ast.Ident{ - NamePos: 159, - NameEnd: 163, + NamePos: 143, + NameEnd: 147, Name: "name", }, }, }, As: &ast.AsAlias{ - As: 164, + As: 148, Alias: &ast.Ident{ - NamePos: 167, - NameEnd: 179, + NamePos: 151, + NameEnd: 163, Name: "product_name", }, }, }, &ast.Alias{ Expr: &ast.CallExpr{ - Rparen: 425, + Rparen: 393, Func: &ast.Ident{ - NamePos: 194, - NameEnd: 200, + NamePos: 174, + NameEnd: 180, Name: "CONCAT", }, Distinct: false, Args: []ast.Arg{ &ast.ExprArg{ Expr: &ast.StringLiteral{ - ValuePos: 201, - ValueEnd: 236, + ValuePos: 181, + ValueEnd: 216, Value: "Is this product safe for infants?", }, }, &ast.ExprArg{ Expr: &ast.StringLiteral{ - ValuePos: 238, - ValueEnd: 242, + ValuePos: 218, + ValueEnd: 222, Value: "\n", }, }, &ast.ExprArg{ Expr: &ast.StringLiteral{ - ValuePos: 264, - ValueEnd: 280, + ValuePos: 240, + ValueEnd: 256, Value: "Product Name: ", }, }, @@ -168,13 +168,13 @@ FROM ML.PREDICT( Expr: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 282, - NameEnd: 289, + NamePos: 258, + NameEnd: 265, Name: "product", }, &ast.Ident{ - NamePos: 290, - NameEnd: 294, + NamePos: 266, + NameEnd: 270, Name: "name", }, }, @@ -182,15 +182,15 @@ FROM ML.PREDICT( }, &ast.ExprArg{ Expr: &ast.StringLiteral{ - ValuePos: 296, - ValueEnd: 300, + ValuePos: 272, + ValueEnd: 276, Value: "\n", }, }, &ast.ExprArg{ Expr: &ast.StringLiteral{ - ValuePos: 322, - ValueEnd: 339, + ValuePos: 294, + ValueEnd: 311, Value: "Category Name: ", }, }, @@ -198,13 +198,13 @@ FROM ML.PREDICT( Expr: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 341, - NameEnd: 349, + NamePos: 313, + NameEnd: 321, Name: "category", }, &ast.Ident{ - NamePos: 350, - NameEnd: 354, + NamePos: 322, + NameEnd: 326, Name: "name", }, }, @@ -212,15 +212,15 @@ FROM ML.PREDICT( }, &ast.ExprArg{ Expr: &ast.StringLiteral{ - ValuePos: 356, - ValueEnd: 360, + ValuePos: 328, + ValueEnd: 332, Value: "\n", }, }, &ast.ExprArg{ Expr: &ast.StringLiteral{ - ValuePos: 382, - ValueEnd: 404, + ValuePos: 350, + ValueEnd: 372, Value: "Product Description:", }, }, @@ -228,13 +228,13 @@ FROM ML.PREDICT( Expr: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 406, - NameEnd: 413, + NamePos: 374, + NameEnd: 381, Name: "product", }, &ast.Ident{ - NamePos: 414, - NameEnd: 425, + NamePos: 382, + NameEnd: 393, Name: "description", }, }, @@ -246,33 +246,33 @@ FROM ML.PREDICT( Having: nil, }, As: &ast.AsAlias{ - As: 427, + As: 395, Alias: &ast.Ident{ - NamePos: 430, - NameEnd: 436, + NamePos: 398, + NameEnd: 404, Name: "prompt", }, }, }, }, From: &ast.From{ - From: 446, + From: 410, Source: &ast.Join{ Op: "INNER JOIN", Method: "", Hint: (*ast.Hint)(nil), Left: &ast.TableName{ Table: &ast.Ident{ - NamePos: 464, - NameEnd: 472, + NamePos: 424, + NameEnd: 432, Name: "Products", }, Hint: (*ast.Hint)(nil), As: &ast.AsAlias{ - As: 473, + As: 433, Alias: &ast.Ident{ - NamePos: 476, - NameEnd: 483, + NamePos: 436, + NameEnd: 443, Name: "product", }, }, @@ -280,35 +280,35 @@ FROM ML.PREDICT( }, Right: &ast.TableName{ Table: &ast.Ident{ - NamePos: 489, - NameEnd: 499, + NamePos: 449, + NameEnd: 459, Name: "Categories", }, Hint: (*ast.Hint)(nil), As: &ast.AsAlias{ - As: 500, + As: 460, Alias: &ast.Ident{ - NamePos: 503, - NameEnd: 511, + NamePos: 463, + NameEnd: 471, Name: "category", }, }, Sample: (*ast.TableSample)(nil), }, Cond: &ast.On{ - On: 550, + On: 506, Expr: &ast.BinaryExpr{ Op: "=", Left: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 553, - NameEnd: 560, + NamePos: 509, + NameEnd: 516, Name: "product", }, &ast.Ident{ - NamePos: 561, - NameEnd: 572, + NamePos: 517, + NameEnd: 528, Name: "category_id", }, }, @@ -316,13 +316,13 @@ FROM ML.PREDICT( Right: &ast.Path{ Idents: []*ast.Ident{ &ast.Ident{ - NamePos: 575, - NameEnd: 583, + NamePos: 531, + NameEnd: 539, Name: "category", }, &ast.Ident{ - NamePos: 584, - NameEnd: 586, + NamePos: 540, + NameEnd: 542, Name: "id", }, }, @@ -341,21 +341,21 @@ FROM ML.PREDICT( }, &ast.ExprArg{ Expr: &ast.TypelessStructLiteral{ - Struct: 597, - Rparen: 626, + Struct: 549, + Rparen: 578, Values: []ast.TypelessStructLiteralArg{ &ast.Alias{ Expr: &ast.IntLiteral{ - ValuePos: 604, - ValueEnd: 607, + ValuePos: 556, + ValueEnd: 559, Base: 10, Value: "100", }, As: &ast.AsAlias{ - As: 608, + As: 560, Alias: &ast.Ident{ - NamePos: 611, - NameEnd: 626, + NamePos: 563, + NameEnd: 578, Name: "maxOutputTokens", }, }, @@ -366,18 +366,18 @@ FROM ML.PREDICT( }, NamedArgs: []*ast.NamedArg(nil), Hint: &ast.Hint{ - Atmark: 635, - Rbrace: 666, + Atmark: 582, + Rbrace: 613, Records: []*ast.HintRecord{ &ast.HintRecord{ Key: &ast.Ident{ - NamePos: 637, - NameEnd: 664, + NamePos: 584, + NameEnd: 611, Name: "remote_udf_max_rows_per_rpc", }, Value: &ast.IntLiteral{ - ValuePos: 665, - ValueEnd: 666, + ValuePos: 612, + ValueEnd: 613, Base: 10, Value: "1", }, From 8beb6475c129fe18fe524e60d9afc037786bdcec Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Sat, 23 Nov 2024 08:37:14 +0900 Subject: [PATCH 5/7] Rename to update_with_safe_ml_predict.sql --- ...date_with_ml_predict.sql => update_with_safe_ml_predict.sql} | 0 ...h_ml_predict.sql.txt => update_with_safe_ml_predict.sql.txt} | 2 +- ...h_ml_predict.sql.txt => update_with_safe_ml_predict.sql.txt} | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename testdata/input/dml/{update_with_ml_predict.sql => update_with_safe_ml_predict.sql} (100%) rename testdata/result/dml/{update_with_ml_predict.sql.txt => update_with_safe_ml_predict.sql.txt} (99%) rename testdata/result/statement/{update_with_ml_predict.sql.txt => update_with_safe_ml_predict.sql.txt} (99%) diff --git a/testdata/input/dml/update_with_ml_predict.sql b/testdata/input/dml/update_with_safe_ml_predict.sql similarity index 100% rename from testdata/input/dml/update_with_ml_predict.sql rename to testdata/input/dml/update_with_safe_ml_predict.sql diff --git a/testdata/result/dml/update_with_ml_predict.sql.txt b/testdata/result/dml/update_with_safe_ml_predict.sql.txt similarity index 99% rename from testdata/result/dml/update_with_ml_predict.sql.txt rename to testdata/result/dml/update_with_safe_ml_predict.sql.txt index cb5ab0ee..1debc9d9 100644 --- a/testdata/result/dml/update_with_ml_predict.sql.txt +++ b/testdata/result/dml/update_with_safe_ml_predict.sql.txt @@ -1,4 +1,4 @@ ---- update_with_ml_predict.sql +--- update_with_safe_ml_predict.sql -- https://cloud.google.com/spanner/docs/backfill-embeddings?hl=en#backfill UPDATE products SET diff --git a/testdata/result/statement/update_with_ml_predict.sql.txt b/testdata/result/statement/update_with_safe_ml_predict.sql.txt similarity index 99% rename from testdata/result/statement/update_with_ml_predict.sql.txt rename to testdata/result/statement/update_with_safe_ml_predict.sql.txt index cb5ab0ee..1debc9d9 100644 --- a/testdata/result/statement/update_with_ml_predict.sql.txt +++ b/testdata/result/statement/update_with_safe_ml_predict.sql.txt @@ -1,4 +1,4 @@ ---- update_with_ml_predict.sql +--- update_with_safe_ml_predict.sql -- https://cloud.google.com/spanner/docs/backfill-embeddings?hl=en#backfill UPDATE products SET From 6f124194d595ee188e1c96bc0ac80eb2cdf73f77 Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Sat, 23 Nov 2024 08:46:45 +0900 Subject: [PATCH 6/7] Fix doc comment --- ast/ast.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ast/ast.go b/ast/ast.go index ea7697ce..32ab288e 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -1238,7 +1238,7 @@ type ModelArg struct { // TableArg is TABLE table_name argument of table valued function call. // -// TABLE {{.Path | sql}} +// TABLE {{.Name | sql}} type TableArg struct { // pos = Table // end = Name.end From 00cba6ad246e3d04ccd78c89ac95c7c8b50247ad Mon Sep 17 00:00:00 2001 From: apstndb <803393+apstndb@users.noreply.github.com> Date: Sat, 23 Nov 2024 09:10:14 +0900 Subject: [PATCH 7/7] Simplify implementation --- parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser.go b/parser.go index 32b86815..f8c69e73 100644 --- a/parser.go +++ b/parser.go @@ -838,7 +838,7 @@ func (p *Parser) parseTVFCallExpr(ids []*ast.Ident) *ast.TVFCallExpr { var args []ast.TVFArg if p.Token.Kind != ")" { - for p.Token.Kind != token.TokenEOF && !p.lookaheadNamedArg() { + for !p.lookaheadNamedArg() { args = append(args, p.parseTVFArg()) if p.Token.Kind != "," { break