diff --git a/go.mod b/go.mod index f9468f5..b0fa802 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,12 @@ module github.com/cloudspannerecosystem/wrench -go 1.21 +go 1.23.1 + +toolchain go1.23.2 require ( cloud.google.com/go/spanner v1.64.0 + github.com/apstndb/gsqlutils v0.0.0-20241110070457-b8882df29778 github.com/google/uuid v1.6.0 github.com/hashicorp/go-multierror v1.1.1 github.com/spf13/cobra v1.7.0 @@ -21,6 +24,7 @@ require ( github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cloudspannerecosystem/memefish v0.0.0-20241106111047-2b2b4b23a1e7 // indirect github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 // indirect github.com/envoyproxy/go-control-plane v0.12.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect @@ -34,6 +38,7 @@ require ( github.com/googleapis/gax-go/v2 v2.12.5 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/samber/lo v1.47.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect @@ -45,7 +50,7 @@ require ( golang.org/x/net v0.26.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect @@ -53,4 +58,5 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect google.golang.org/protobuf v1.34.2 // indirect + spheric.cloud/xiter v0.0.0-20240904151420-c999f37a46b2 // indirect ) diff --git a/go.sum b/go.sum index e43f5f9..a422bad 100644 --- a/go.sum +++ b/go.sum @@ -614,6 +614,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 h1:oVLqHXhnYtUwM89y9T1fXGaK9wTkXHgNp8/ZNMQzUxE= github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0/go.mod h1:dppbR7CwXD4pgtV9t3wD1812RaLDcBjtblcDF5f1vI0= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZYIR/J6A= +github.com/MakeNowJust/heredoc/v2 v2.0.1/go.mod h1:6/2Abh5s+hc3g9nbWLe9ObDIOhaRrqsyY9MWy+4JdRM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= @@ -624,6 +626,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/apstndb/gsqlutils v0.0.0-20241110070457-b8882df29778 h1:eFzpW6G5NzTytQtSnsEIOfH2aAYGnNN5zmbdh2jQFCY= +github.com/apstndb/gsqlutils v0.0.0-20241110070457-b8882df29778/go.mod h1:iy/NytOejQT/jaXcRaYYHthvkTpgWNZxggLVEecd7C4= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -638,6 +642,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudspannerecosystem/memefish v0.0.0-20241106111047-2b2b4b23a1e7 h1:vmS9Nvh7ij5EVnMwvkWsL84xzx5cGM3j/SJlM1zfVLE= +github.com/cloudspannerecosystem/memefish v0.0.0-20241106111047-2b2b4b23a1e7/go.mod h1:iYAaNZfVIn4QYfUmXt+3EeHAok/kqpN/fp/8kgDHjx8= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -831,6 +837,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/k0kubun/pp v1.3.1-0.20200204103551-99835366d1cc h1:XLjmW07gT7cG/wb6mavIrvAIWBYaTacPo8UOnxGSspA= +github.com/k0kubun/pp v1.3.1-0.20200204103551-99835366d1cc/go.mod h1:qK2ivXw91omfE1uXcpR5kWbAMZRdDOnGbqWlZ7reRFk= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= @@ -846,7 +854,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= @@ -873,6 +884,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= +github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= +github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= @@ -1181,8 +1194,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -1618,3 +1631,5 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +spheric.cloud/xiter v0.0.0-20240904151420-c999f37a46b2 h1:/2/LZep2TpsdSwYhdMsGRh+OVun1UMYlMEip4EEsgg8= +spheric.cloud/xiter v0.0.0-20240904151420-c999f37a46b2/go.mod h1:i4SlkNfFrn1974zbGZWg8FYXAWLnrS6cYAXtSfmIDhU= diff --git a/pkg/spanner/client_test.go b/pkg/spanner/client_test.go index 612adb2..89f5c2a 100644 --- a/pkg/spanner/client_test.go +++ b/pkg/spanner/client_test.go @@ -73,7 +73,7 @@ func TestLoadDDL(t *testing.T) { t.Fatalf("failed to load ddl: %v", err) } - wantDDL, err := os.ReadFile("testdata/schema.sql") + wantDDL, err := os.ReadFile("testdata/schema_loaded.sql") if err != nil { t.Fatalf("failed to read ddl file: %v", err) } diff --git a/pkg/spanner/memefish.go b/pkg/spanner/memefish.go new file mode 100644 index 0000000..06df701 --- /dev/null +++ b/pkg/spanner/memefish.go @@ -0,0 +1,42 @@ +package spanner + +import ( + "github.com/apstndb/gsqlutils" + "github.com/cloudspannerecosystem/memefish" +) + +// Directly use of memefish/gsqlutils is permitted only in this file. +func toStatements(filename string, data []byte) ([]string, error) { + rawStmts, err := memefish.SplitRawStatements(filename, string(data)) + if err != nil { + return nil, err + } + + // need to strip comments because memefish.SplitRawStatements preserve comments, but UpdateDDL doesn't support comments. + var result []string + for _, rawStmt := range rawStmts { + stripped, err := gsqlutils.SimpleStripComments("", rawStmt.Statement) + if err != nil { + return nil, err + } + result = append(result, stripped) + } + return result, nil +} + +func isDML(statement string) bool { + token, err := gsqlutils.FirstNonHintToken("", statement) + if err != nil { + return false + } + return token.IsKeywordLike("INSERT") +} + +func isPartitionedDML(statement string) bool { + // It is better than regular expression because PDML can be prefixed by statement hints. + token, err := gsqlutils.FirstNonHintToken("", statement) + if err != nil { + return false + } + return token.IsKeywordLike("UPDATE") || token.IsKeywordLike("DELETE") +} diff --git a/pkg/spanner/migration.go b/pkg/spanner/migration.go index 2cfc3dc..0cc6aa0 100644 --- a/pkg/spanner/migration.go +++ b/pkg/spanner/migration.go @@ -26,8 +26,6 @@ import ( "path/filepath" "regexp" "strconv" - - "cloud.google.com/go/spanner/spansql" ) var ( @@ -38,9 +36,6 @@ var ( migrationFileRegex = regexp.MustCompile(`^([0-9]+)(?:_([a-zA-Z0-9_\-]+))?(\.up)?\.sql$`) MigrationNameRegex = regexp.MustCompile(`[a-zA-Z0-9_\-]+`) - - dmlRegex = regexp.MustCompile("^(INSERT)[\t\n\f\r ].*") - partitionedDmlRegex = regexp.MustCompile("^(UPDATE|DELETE)[\t\n\f\r ].*") ) const ( @@ -144,31 +139,11 @@ func LoadMigrations(dir string) (Migrations, error) { } func ddlToStatements(filename string, data []byte) ([]string, error) { - ddl, err := spansql.ParseDDL(filename, string(data)) - if err != nil { - return nil, err - } - - var statements []string - for _, stmt := range ddl.List { - statements = append(statements, stmt.SQL()) - } - - return statements, nil + return toStatements(filename, data) } func dmlToStatements(filename string, data []byte) ([]string, error) { - dml, err := spansql.ParseDML(filename, string(data)) - if err != nil { - return nil, err - } - - var statements []string - for _, stmt := range dml.List { - statements = append(statements, stmt.SQL()) - } - - return statements, nil + return toStatements(filename, data) } func inspectStatementsKind(statements []string) (statementKind, error) { @@ -200,10 +175,3 @@ func inspectStatementsKind(statements []string) (statementKind, error) { } } -func isDML(statement string) bool { - return dmlRegex.Match([]byte(statement)) -} - -func isPartitionedDML(statement string) bool { - return partitionedDmlRegex.Match([]byte(statement)) -} diff --git a/pkg/spanner/testdata/ddl.sql b/pkg/spanner/testdata/ddl.sql index 8565107..f2008c6 100644 --- a/pkg/spanner/testdata/ddl.sql +++ b/pkg/spanner/testdata/ddl.sql @@ -1 +1,2 @@ +-- Comments must be permitted ALTER TABLE Singers ADD COLUMN Foo STRING(MAX); diff --git a/pkg/spanner/testdata/migrations/000002_test.sql b/pkg/spanner/testdata/migrations/000002_test.sql index ce86ce6..f6cb2fc 100644 --- a/pkg/spanner/testdata/migrations/000002_test.sql +++ b/pkg/spanner/testdata/migrations/000002_test.sql @@ -1 +1,2 @@ +-- Comments must be ignored ALTER TABLE Singers ADD COLUMN LastName STRING(MAX); diff --git a/pkg/spanner/testdata/migrations/000003.sql b/pkg/spanner/testdata/migrations/000003.sql index 51101af..8ed6bc3 100644 --- a/pkg/spanner/testdata/migrations/000003.sql +++ b/pkg/spanner/testdata/migrations/000003.sql @@ -1 +1,2 @@ +-- Comments must be permitted UPDATE Singers SET LastName = "" WHERE LastName IS NULL; diff --git a/pkg/spanner/testdata/schema.sql b/pkg/spanner/testdata/schema.sql index 921e523..a5c3ed8 100644 --- a/pkg/spanner/testdata/schema.sql +++ b/pkg/spanner/testdata/schema.sql @@ -1,3 +1,4 @@ +-- Comments must be permitted CREATE TABLE SchemaMigrations ( Version INT64 NOT NULL, Dirty BOOL NOT NULL, diff --git a/pkg/spanner/testdata/schema_loaded.sql b/pkg/spanner/testdata/schema_loaded.sql new file mode 100644 index 0000000..921e523 --- /dev/null +++ b/pkg/spanner/testdata/schema_loaded.sql @@ -0,0 +1,9 @@ +CREATE TABLE SchemaMigrations ( + Version INT64 NOT NULL, + Dirty BOOL NOT NULL, +) PRIMARY KEY(Version); + +CREATE TABLE Singers ( + SingerID STRING(36) NOT NULL, + FirstName STRING(1024), +) PRIMARY KEY(SingerID);