Migrate to memefish

go.mod

@@ -1,9 +1,12 @@
 module github.com/cloudspannerecosystem/wrench
 
-go 1.21
+go 1.23.1
+
+toolchain go1.23.2
 
 require (
 	cloud.google.com/go/spanner v1.64.0
+	github.com/apstndb/gsqlutils v0.0.0-20241110070457-b8882df29778
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/spf13/cobra v1.7.0
@@ -21,6 +24,7 @@ require (
 	github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cloudspannerecosystem/memefish v0.0.0-20241106111047-2b2b4b23a1e7 // indirect
 	github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 // indirect
 	github.com/envoyproxy/go-control-plane v0.12.0 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect
@@ -34,6 +38,7 @@ require (
 	github.com/googleapis/gax-go/v2 v2.12.5 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/samber/lo v1.47.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
@@ -45,12 +50,13 @@ require (
 	golang.org/x/net v0.26.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/genproto v0.0.0-20240617180043-68d350f18fd4 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
+	spheric.cloud/xiter v0.0.0-20240904151420-c999f37a46b2 // indirect
 )

go.sum

@@ -614,6 +614,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 h1:oVLqHXhnYtUwM89y9T1fXGaK9wTkXHgNp8/ZNMQzUxE=
 github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0/go.mod h1:dppbR7CwXD4pgtV9t3wD1812RaLDcBjtblcDF5f1vI0=
 github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
+github.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZYIR/J6A=
+github.com/MakeNowJust/heredoc/v2 v2.0.1/go.mod h1:6/2Abh5s+hc3g9nbWLe9ObDIOhaRrqsyY9MWy+4JdRM=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
 github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
@@ -624,6 +626,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
 github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
 github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI=
 github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
+github.com/apstndb/gsqlutils v0.0.0-20241110070457-b8882df29778 h1:eFzpW6G5NzTytQtSnsEIOfH2aAYGnNN5zmbdh2jQFCY=
+github.com/apstndb/gsqlutils v0.0.0-20241110070457-b8882df29778/go.mod h1:iy/NytOejQT/jaXcRaYYHthvkTpgWNZxggLVEecd7C4=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -638,6 +642,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudspannerecosystem/memefish v0.0.0-20241106111047-2b2b4b23a1e7 h1:vmS9Nvh7ij5EVnMwvkWsL84xzx5cGM3j/SJlM1zfVLE=
+github.com/cloudspannerecosystem/memefish v0.0.0-20241106111047-2b2b4b23a1e7/go.mod h1:iYAaNZfVIn4QYfUmXt+3EeHAok/kqpN/fp/8kgDHjx8=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -831,6 +837,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
+github.com/k0kubun/pp v1.3.1-0.20200204103551-99835366d1cc h1:XLjmW07gT7cG/wb6mavIrvAIWBYaTacPo8UOnxGSspA=
+github.com/k0kubun/pp v1.3.1-0.20200204103551-99835366d1cc/go.mod h1:qK2ivXw91omfE1uXcpR5kWbAMZRdDOnGbqWlZ7reRFk=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
@@ -846,7 +854,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
 github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
 github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o=
+github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
@@ -873,6 +884,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w=
 github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk=
+github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc=
+github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
@@ -1181,8 +1194,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
@@ -1618,3 +1631,5 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+spheric.cloud/xiter v0.0.0-20240904151420-c999f37a46b2 h1:/2/LZep2TpsdSwYhdMsGRh+OVun1UMYlMEip4EEsgg8=
+spheric.cloud/xiter v0.0.0-20240904151420-c999f37a46b2/go.mod h1:i4SlkNfFrn1974zbGZWg8FYXAWLnrS6cYAXtSfmIDhU=

pkg/spanner/client_test.go

@@ -73,7 +73,7 @@ func TestLoadDDL(t *testing.T) {
 		t.Fatalf("failed to load ddl: %v", err)
 	}
 
-	wantDDL, err := os.ReadFile("testdata/schema.sql")
+	wantDDL, err := os.ReadFile("testdata/schema_loaded.sql")
 	if err != nil {
 		t.Fatalf("failed to read ddl file: %v", err)
 	}

pkg/spanner/memefish.go

@@ -0,0 +1,42 @@
+package spanner
+
+import (
+	"github.com/apstndb/gsqlutils"
+	"github.com/cloudspannerecosystem/memefish"
+)
+
+// Directly use of memefish/gsqlutils is permitted only in this file.
+func toStatements(filename string, data []byte) ([]string, error) {
+	rawStmts, err := memefish.SplitRawStatements(filename, string(data))
+	if err != nil {
+		return nil, err
+	}
+
+	// need to strip comments because memefish.SplitRawStatements preserve comments, but UpdateDDL doesn't support comments.
+	var result []string
+	for _, rawStmt := range rawStmts {
+		stripped, err := gsqlutils.SimpleStripComments("", rawStmt.Statement)
+		if err != nil {
+			return nil, err
+		}
+		result = append(result, stripped)
+	}
+	return result, nil
+}
+
+func isDML(statement string) bool {
+	token, err := gsqlutils.FirstNonHintToken("", statement)
+	if err != nil {
+		return false
+	}
+	return token.IsKeywordLike("INSERT")
+}
+
+func isPartitionedDML(statement string) bool {
+	// It is better than regular expression because PDML can be prefixed by statement hints.
+	token, err := gsqlutils.FirstNonHintToken("", statement)
+	if err != nil {
+		return false
+	}
+	return token.IsKeywordLike("UPDATE") || token.IsKeywordLike("DELETE")
+}

pkg/spanner/migration.go

@@ -26,8 +26,6 @@ import (
 	"path/filepath"
 	"regexp"
 	"strconv"
-
-	"cloud.google.com/go/spanner/spansql"
 )
 
 var (
@@ -38,9 +36,6 @@ var (
 	migrationFileRegex = regexp.MustCompile(`^([0-9]+)(?:_([a-zA-Z0-9_\-]+))?(\.up)?\.sql$`)
 
 	MigrationNameRegex = regexp.MustCompile(`[a-zA-Z0-9_\-]+`)
-
-	dmlRegex            = regexp.MustCompile("^(INSERT)[\t\n\f\r ].*")
-	partitionedDmlRegex = regexp.MustCompile("^(UPDATE|DELETE)[\t\n\f\r ].*")
 )
 
 const (
@@ -144,31 +139,11 @@ func LoadMigrations(dir string) (Migrations, error) {
 }
 
 func ddlToStatements(filename string, data []byte) ([]string, error) {
-	ddl, err := spansql.ParseDDL(filename, string(data))
-	if err != nil {
-		return nil, err
-	}
-
-	var statements []string
-	for _, stmt := range ddl.List {
-		statements = append(statements, stmt.SQL())
-	}
-
-	return statements, nil
+	return toStatements(filename, data)
 }
 
 func dmlToStatements(filename string, data []byte) ([]string, error) {
-	dml, err := spansql.ParseDML(filename, string(data))
-	if err != nil {
-		return nil, err
-	}
-
-	var statements []string
-	for _, stmt := range dml.List {
-		statements = append(statements, stmt.SQL())
-	}
-
-	return statements, nil
+	return toStatements(filename, data)
 }
 
 func inspectStatementsKind(statements []string) (statementKind, error) {
@@ -200,10 +175,3 @@ func inspectStatementsKind(statements []string) (statementKind, error) {
 	}
 }
 
-func isDML(statement string) bool {
-	return dmlRegex.Match([]byte(statement))
-}
-
-func isPartitionedDML(statement string) bool {
-	return partitionedDmlRegex.Match([]byte(statement))
-}

pkg/spanner/testdata/ddl.sql

@@ -1 +1,2 @@
+-- Comments must be permitted
 ALTER TABLE Singers ADD COLUMN Foo STRING(MAX);

pkg/spanner/testdata/migrations/000002_test.sql

@@ -1 +1,2 @@
+-- Comments must be ignored
 ALTER TABLE Singers ADD COLUMN LastName STRING(MAX);

pkg/spanner/testdata/migrations/000003.sql

@@ -1 +1,2 @@
+-- Comments must be permitted
 UPDATE Singers SET LastName = "" WHERE LastName IS NULL;

pkg/spanner/testdata/schema.sql

@@ -1,3 +1,4 @@
+-- Comments must be permitted
 CREATE TABLE SchemaMigrations (
   Version INT64 NOT NULL,
   Dirty BOOL NOT NULL,

pkg/spanner/testdata/schema_loaded.sql

@@ -0,0 +1,9 @@
+CREATE TABLE SchemaMigrations (
+  Version INT64 NOT NULL,
+  Dirty BOOL NOT NULL,
+) PRIMARY KEY(Version);
+
+CREATE TABLE Singers (
+  SingerID STRING(36) NOT NULL,
+  FirstName STRING(1024),
+) PRIMARY KEY(SingerID);