From 9b551a1c104d49e3c158dba9cc56e18ddb50f331 Mon Sep 17 00:00:00 2001 From: Brian Wigginton Date: Fri, 15 Feb 2019 16:08:38 -0600 Subject: [PATCH 01/12] initial mssql support --- Dockerfile | 2 +- Makefile | 2 +- database/mssql/mssql.go | 421 ++++++++++++++++++++++++++++++++++++ go.mod | 1 + internal/cli/build_mssql.go | 7 + 5 files changed, 431 insertions(+), 2 deletions(-) create mode 100644 database/mssql/mssql.go create mode 100644 internal/cli/build_mssql.go diff --git a/Dockerfile b/Dockerfile index 8e2c27248..8031e23e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ WORKDIR /go/src/github.com/golang-migrate/migrate COPY . ./ ENV GO111MODULE=on -ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse" +ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse mssql" ENV SOURCES="file go_bindata github aws_s3 google_cloud_storage" RUN go build -a -o build/migrate.linux-386 -ldflags="-X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cmd/migrate diff --git a/Makefile b/Makefile index 0cb226e8e..2e96fdf65 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SOURCE ?= file go_bindata github aws_s3 google_cloud_storage godoc_vfs -DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb +DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb mssql VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") diff --git a/database/mssql/mssql.go b/database/mssql/mssql.go new file mode 100644 index 000000000..387e9a6ca --- /dev/null +++ b/database/mssql/mssql.go @@ -0,0 +1,421 @@ +package mssql + +import ( + "context" + "database/sql" + "fmt" + "io" + "io/ioutil" + nurl "net/url" + + _ "github.com/denisenkom/go-mssqldb" // mssql support + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" +) + +func init() { + db := MSSQL{} + database.Register("mssql", &db) + database.Register("sqlserver", &db) +} + +// DefaultMigrationsTable is the name of the migrations table in the database +var DefaultMigrationsTable = "SchemaMigrations" + +var ( + ErrNilConfig = fmt.Errorf("no config") + ErrNoDatabaseName = fmt.Errorf("no database name") + ErrNoSchema = fmt.Errorf("no schema") + ErrDatabaseDirty = fmt.Errorf("database is dirty") +) + +// Config for database +type Config struct { + MigrationsTable string + DatabaseName string + SchemaName string +} + +// MSSQL connection +type MSSQL struct { + // Locking and unlocking need to use the same connection + conn *sql.Conn + db *sql.DB + isLocked bool + + // Open and WithInstance need to garantuee that config is never nil + config *Config +} + +// WithInstance returns a database instance from an already create database connection +// TODO: WithInstance double check that docs are correct for this function +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + if err := instance.Ping(); err != nil { + return nil, err + } + + query := `SELECT DB_NAME()` + var databaseName string + if err := instance.QueryRow(query).Scan(&databaseName); err != nil { + return nil, &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(databaseName) == 0 { + return nil, ErrNoDatabaseName + } + + config.DatabaseName = databaseName + + query = `SELECT SCHEMA_NAME()` + var schemaName string + if err := instance.QueryRow(query).Scan(&schemaName); err != nil { + return nil, &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(schemaName) == 0 { + return nil, ErrNoSchema + } + + config.SchemaName = schemaName + + if len(config.MigrationsTable) == 0 { + config.MigrationsTable = DefaultMigrationsTable + } + + conn, err := instance.Conn(context.Background()) + + if err != nil { + return nil, err + } + + ss := &MSSQL{ + conn: conn, + db: instance, + config: config, + } + + if err := ss.ensureVersionTable(); err != nil { + return nil, err + } + + return ss, nil +} + +func dbConnectionString(host, port string) string { + return fmt.Sprintf("postgres://postgres@%s:%s/postgres?sslmode=disable", host, port) +} + +// Open a connection to the database +func (ss *MSSQL) Open(url string) (database.Driver, error) { + purl, err := nurl.Parse(url) + if err != nil { + return nil, err + } + + db, err := sql.Open("mssql", migrate.FilterCustomQuery(purl).String()) + if err != nil { + return nil, err + } + + migrationsTable := purl.Query().Get("x-migrations-table") + if len(migrationsTable) == 0 { + migrationsTable = DefaultMigrationsTable + } + + px, err := WithInstance(db, &Config{ + DatabaseName: purl.Path, + MigrationsTable: migrationsTable, + }) + if err != nil { + return nil, err + } + + return px, nil +} + +// Close the database connection +func (ss *MSSQL) Close() error { + connErr := ss.conn.Close() + dbErr := ss.db.Close() + if connErr != nil || dbErr != nil { + return fmt.Errorf("conn: %v, db: %v", connErr, dbErr) + } + return nil +} + +// Lock creates an advisory local on the database to prevent multiple migrations from running at the same time. +func (ss *MSSQL) Lock() error { + if ss.isLocked { + return database.ErrLocked + } + + aid, err := database.GenerateAdvisoryLockId(ss.config.DatabaseName, ss.config.SchemaName) + if err != nil { + return err + } + + // start a transaction + query := "BEGIN TRANSACTION" + if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Err: "get lock transaction", Query: []byte(query)} + } + + // This will either obtain the lock immediately and return true, + // or return false if the lock cannot be acquired immediately. + // MS Docs: sp_getapplock: https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-getapplock-transact-sql?view=sql-server-2017 + query = `EXEC sp_getapplock @Resource = ?, @LockMode = 'Update'` + if _, err := ss.conn.ExecContext(context.Background(), query, aid); err != nil { + return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)} + } + + ss.isLocked = true + return nil +} + +// Unlock froms the migration lock from the database +func (ss *MSSQL) Unlock() error { + if !ss.isLocked { + return nil + } + + aid, err := database.GenerateAdvisoryLockId(ss.config.DatabaseName, ss.config.SchemaName) + if err != nil { + return err + } + + // MS Docs: sp_releaseapplock: https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-releaseapplock-transact-sql?view=sql-server-2017 + query := `EXEC sp_releaseapplock @Resource = ?` + if _, err := ss.conn.ExecContext(context.Background(), query, aid); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + ss.isLocked = false + + // end lock transaction + query = "COMMIT" + if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Err: "commit lock transaction", Query: []byte(query)} + } + + return nil +} + +// Run the migrations for the database +func (ss *MSSQL) Run(migration io.Reader) error { + migr, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + + // run migration + query := string(migr[:]) + if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { + // // FIXME: check for mssql error here + // if pgErr, ok := err.(*pq.Error); ok { + // var line uint + // var col uint + // var lineColOK bool + // if pgErr.Position != "" { + // if pos, err := strconv.ParseUint(pgErr.Position, 10, 64); err == nil { + // line, col, lineColOK = computeLineFromPos(query, int(pos)) + // } + // } + // message := fmt.Sprintf("migration failed: %s", pgErr.Message) + // if lineColOK { + // message = fmt.Sprintf("%s (column %d)", message, col) + // } + // if pgErr.Detail != "" { + // message = fmt.Sprintf("%s, %s", message, pgErr.Detail) + // } + // return database.Error{OrigErr: err, Err: message, Query: migr, Line: line} + // } + return database.Error{OrigErr: err, Err: "migration failed", Query: migr} + } + + return nil +} + +// func computeLineFromPos(s string, pos int) (line uint, col uint, ok bool) { +// // replace crlf with lf +// s = strings.Replace(s, "\r\n", "\n", -1) +// // pg docs: pos uses index 1 for the first character, and positions are measured in characters not bytes +// runes := []rune(s) +// if pos > len(runes) { +// return 0, 0, false +// } +// sel := runes[:pos] +// line = uint(runesCount(sel, newLine) + 1) +// col = uint(pos - 1 - runesLastIndex(sel, newLine)) +// return line, col, true +// } + +const newLine = '\n' + +func runesCount(input []rune, target rune) int { + var count int + for _, r := range input { + if r == target { + count++ + } + } + return count +} + +func runesLastIndex(input []rune, target rune) int { + for i := len(input) - 1; i >= 0; i-- { + if input[i] == target { + return i + } + } + return -1 +} + +// SetVersion for the current database +func (ss *MSSQL) SetVersion(version int, dirty bool) error { + + tx := ss.db + // tx, err := ss.conn.BeginTx(context.Background(), &sql.TxOptions{}) + // if err != nil { + // return &database.Error{OrigErr: err, Err: "transaction start failed"} + // } + + query := `TRUNCATE TABLE "` + ss.config.MigrationsTable + `"` + if _, err := tx.Exec(query); err != nil { + // tx.Rollback() + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + if version >= 0 { + var dirtyBit int + if dirty { + dirtyBit = 1 + } + query = `INSERT INTO "` + ss.config.MigrationsTable + `" (version, dirty) VALUES ($1, $2)` + if _, err := tx.Exec(query, version, dirtyBit); err != nil { + // tx.Rollback() + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + // if err := tx.Commit(); err != nil { + // return &database.Error{OrigErr: err, Err: "transaction commit failed"} + // } + + return nil +} + +// Version of the current database state +func (ss *MSSQL) Version() (version int, dirty bool, err error) { + query := `SELECT TOP 1 version, dirty FROM "` + ss.config.MigrationsTable + `"` + err = ss.conn.QueryRowContext(context.Background(), query).Scan(&version, &dirty) + switch { + case err == sql.ErrNoRows: + return database.NilVersion, false, nil + + case err != nil: + // FIXME: convert to MSSQL error + // if e, ok := err.(*pq.Error); ok { + // if e.Code.Name() == "undefined_table" { + // return database.NilVersion, false, nil + // } + // } + return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} + + default: + return version, dirty, nil + } +} + +// Drop all tables from the database. +func (ss *MSSQL) Drop() error { + + // drop all referential integrity constraints + query := ` + DECLARE @Sql NVARCHAR(500) DECLARE @Cursor CURSOR + + SET @Cursor = CURSOR FAST_FORWARD FOR + SELECT DISTINCT sql = 'ALTER TABLE [' + tc2.TABLE_NAME + '] DROP [' + rc1.CONSTRAINT_NAME + ']' + FROM INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS rc1 + LEFT JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc2 ON tc2.CONSTRAINT_NAME =rc1.CONSTRAINT_NAME + + OPEN @Cursor FETCH NEXT FROM @Cursor INTO @Sql + + WHILE (@@FETCH_STATUS = 0) + BEGIN + Exec sp_executesql @Sql + FETCH NEXT FROM @Cursor INTO @Sql + END + + CLOSE @Cursor DEALLOCATE @Cursor` + + if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + // drop the tables + query = `EXEC sp_MSforeachtable 'DROP TABLE ?'` + if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + if err := ss.ensureVersionTable(); err != nil { + return err + } + + return nil + + // // select all tables in current schema + // query := `SELECT t.name FROM sys.tables AS t INNER JOIN sys.schemas AS s ON t.[schema_id] = s.[schema_id] WHERE s.name = N'dbo';` + // tables, err := ss.conn.QueryContext(context.Background(), query) + // if err != nil { + // return &database.Error{OrigErr: err, Query: []byte(query)} + // } + // defer tables.Close() + + // // delete one table after another + // tableNames := make([]string, 0) + // for tables.Next() { + // var tableName string + // if err := tables.Scan(&tableName); err != nil { + // return err + // } + // if len(tableName) > 0 { + // tableNames = append(tableNames, tableName) + // } + // } + + // if len(tableNames) > 0 { + // // delete one by one ... + // for _, t := range tableNames { + // query = `DROP TABLE IF EXISTS "` + t + `"` + // if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { + // return &database.Error{OrigErr: err, Query: []byte(query)} + // } + // } + + // if err := ss.ensureVersionTable(); err != nil { + // return err + // } + // } + + // return nil +} + +func (ss *MSSQL) ensureVersionTable() (err error) { + query := `IF NOT EXISTS + (SELECT * + FROM sysobjects + WHERE id = object_id(N'[dbo].[` + ss.config.MigrationsTable + `]') + AND OBJECTPROPERTY(id, N'IsUserTable') = 1 + ) + CREATE TABLE ` + ss.config.MigrationsTable + ` ( version BIGINT PRIMARY KEY NOT NULL, dirty BIT NOT NULL );` + + if _, err = ss.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + return nil +} diff --git a/go.mod b/go.mod index 981e9487b..ac8901e12 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/cockroachdb/apd v1.1.0 // indirect github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c github.com/cznic/ql v1.2.0 + github.com/denisenkom/go-mssqldb v0.0.0-20190121005146-b04fd42d9952 github.com/dhui/dktest v0.3.0 github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf github.com/fsouza/fake-gcs-server v1.3.0 diff --git a/internal/cli/build_mssql.go b/internal/cli/build_mssql.go new file mode 100644 index 000000000..2de4ad1b0 --- /dev/null +++ b/internal/cli/build_mssql.go @@ -0,0 +1,7 @@ +// +build postgres + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/database/mssql" +) From 407d1e39823cf4d92d5e62be8776875136d025d5 Mon Sep 17 00:00:00 2001 From: Brian Wigginton Date: Fri, 15 Feb 2019 16:09:43 -0600 Subject: [PATCH 02/12] bump go.sum --- go.sum | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go.sum b/go.sum index 536508a85..d0b524a30 100644 --- a/go.sum +++ b/go.sum @@ -46,13 +46,15 @@ github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc/go.mod h1:Y1SNZ4dRUOKX github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.0.0-20190121005146-b04fd42d9952 h1:b5OnbZD49x9g+/FcYbs/vukEt8C/jUbGhCJ3uduQmu8= +github.com/denisenkom/go-mssqldb v0.0.0-20190121005146-b04fd42d9952/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= github.com/dhui/dktest v0.3.0 h1:kwX5a7EkLcjo7VpsPQSYJcKGbXBXdjI9FGjuUj1jn6I= github.com/dhui/dktest v0.3.0/go.mod h1:cyzIUfGsBEbZ6BT7tnXqAShHSXCZhSNmFl70sZ7c1yc= github.com/docker/distribution v2.7.0+incompatible h1:neUDAlf3wX6Ml4HdqTrbcOHXtfRN0TFIwt6YFL7N9RU= github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b h1:Y0C03XhDDcak1Ow6em58mBJmUJjxaMfB5sFttITXE0Q= github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf h1:2v/98rHzs3v6X0AHtoCH9u+e56SdnpogB1Z2fFe1KqQ= +github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf h1:ZjoOm2c/ckflZVEbHBOwuf4FoUajM5Jx9dDTVBHK9vc= github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= From ae500dbe45c0e72ffe8f4c6c5aad8d2a5c3e2fb2 Mon Sep 17 00:00:00 2001 From: nathan-c Date: Sun, 19 May 2019 13:37:40 +0100 Subject: [PATCH 03/12] finish implementing mssql db and add tests --- .gitignore | 3 +- .../1085649617_create_users_table.down.sql | 1 + .../1085649617_create_users_table.up.sql | 5 + .../1185749658_add_city_to_users.down.sql | 1 + .../1185749658_add_city_to_users.up.sql | 3 + ...85849751_add_index_on_user_emails.down.sql | 1 + ...1285849751_add_index_on_user_emails.up.sql | 3 + .../1385949617_create_books_table.down.sql | 1 + .../1385949617_create_books_table.up.sql | 5 + .../1485949617_create_movies_table.down.sql | 1 + .../1485949617_create_movies_table.up.sql | 5 + .../1585849751_just_a_comment.up.sql | 1 + .../1685849751_another_comment.up.sql | 1 + .../1785849751_another_comment.up.sql | 1 + .../1885849751_another_comment.up.sql | 1 + database/mssql/mssql.go | 185 +++++------------- database/mssql/mssql_test.go | 157 +++++++++++++++ go.sum | 2 +- 18 files changed, 238 insertions(+), 139 deletions(-) create mode 100644 database/mssql/examples/migrations/1085649617_create_users_table.down.sql create mode 100644 database/mssql/examples/migrations/1085649617_create_users_table.up.sql create mode 100644 database/mssql/examples/migrations/1185749658_add_city_to_users.down.sql create mode 100644 database/mssql/examples/migrations/1185749658_add_city_to_users.up.sql create mode 100644 database/mssql/examples/migrations/1285849751_add_index_on_user_emails.down.sql create mode 100644 database/mssql/examples/migrations/1285849751_add_index_on_user_emails.up.sql create mode 100644 database/mssql/examples/migrations/1385949617_create_books_table.down.sql create mode 100644 database/mssql/examples/migrations/1385949617_create_books_table.up.sql create mode 100644 database/mssql/examples/migrations/1485949617_create_movies_table.down.sql create mode 100644 database/mssql/examples/migrations/1485949617_create_movies_table.up.sql create mode 100644 database/mssql/examples/migrations/1585849751_just_a_comment.up.sql create mode 100644 database/mssql/examples/migrations/1685849751_another_comment.up.sql create mode 100644 database/mssql/examples/migrations/1785849751_another_comment.up.sql create mode 100644 database/mssql/examples/migrations/1885849751_another_comment.up.sql create mode 100755 database/mssql/mssql_test.go diff --git a/.gitignore b/.gitignore index 00b92cb33..acb4088f2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ cli/cli cli/migrate .coverage .godoc.pid -vendor/ \ No newline at end of file +vendor/ +.vscode/ diff --git a/database/mssql/examples/migrations/1085649617_create_users_table.down.sql b/database/mssql/examples/migrations/1085649617_create_users_table.down.sql new file mode 100644 index 000000000..c99ddcdc8 --- /dev/null +++ b/database/mssql/examples/migrations/1085649617_create_users_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS users; diff --git a/database/mssql/examples/migrations/1085649617_create_users_table.up.sql b/database/mssql/examples/migrations/1085649617_create_users_table.up.sql new file mode 100644 index 000000000..92897dcab --- /dev/null +++ b/database/mssql/examples/migrations/1085649617_create_users_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE users ( + user_id integer unique, + name varchar(40), + email varchar(40) +); diff --git a/database/mssql/examples/migrations/1185749658_add_city_to_users.down.sql b/database/mssql/examples/migrations/1185749658_add_city_to_users.down.sql new file mode 100644 index 000000000..940c60712 --- /dev/null +++ b/database/mssql/examples/migrations/1185749658_add_city_to_users.down.sql @@ -0,0 +1 @@ +ALTER TABLE users DROP COLUMN IF EXISTS city; diff --git a/database/mssql/examples/migrations/1185749658_add_city_to_users.up.sql b/database/mssql/examples/migrations/1185749658_add_city_to_users.up.sql new file mode 100644 index 000000000..2add820be --- /dev/null +++ b/database/mssql/examples/migrations/1185749658_add_city_to_users.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE users ADD city varchar(100); + + diff --git a/database/mssql/examples/migrations/1285849751_add_index_on_user_emails.down.sql b/database/mssql/examples/migrations/1285849751_add_index_on_user_emails.down.sql new file mode 100644 index 000000000..3e87dd229 --- /dev/null +++ b/database/mssql/examples/migrations/1285849751_add_index_on_user_emails.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS users_email_index; diff --git a/database/mssql/examples/migrations/1285849751_add_index_on_user_emails.up.sql b/database/mssql/examples/migrations/1285849751_add_index_on_user_emails.up.sql new file mode 100644 index 000000000..03a04639c --- /dev/null +++ b/database/mssql/examples/migrations/1285849751_add_index_on_user_emails.up.sql @@ -0,0 +1,3 @@ +CREATE UNIQUE INDEX users_email_index ON users (email); + +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/mssql/examples/migrations/1385949617_create_books_table.down.sql b/database/mssql/examples/migrations/1385949617_create_books_table.down.sql new file mode 100644 index 000000000..1a0b1a214 --- /dev/null +++ b/database/mssql/examples/migrations/1385949617_create_books_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS books; diff --git a/database/mssql/examples/migrations/1385949617_create_books_table.up.sql b/database/mssql/examples/migrations/1385949617_create_books_table.up.sql new file mode 100644 index 000000000..f1503b518 --- /dev/null +++ b/database/mssql/examples/migrations/1385949617_create_books_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE books ( + user_id integer, + name varchar(40), + author varchar(40) +); diff --git a/database/mssql/examples/migrations/1485949617_create_movies_table.down.sql b/database/mssql/examples/migrations/1485949617_create_movies_table.down.sql new file mode 100644 index 000000000..3a5187689 --- /dev/null +++ b/database/mssql/examples/migrations/1485949617_create_movies_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS movies; diff --git a/database/mssql/examples/migrations/1485949617_create_movies_table.up.sql b/database/mssql/examples/migrations/1485949617_create_movies_table.up.sql new file mode 100644 index 000000000..f0ef5943b --- /dev/null +++ b/database/mssql/examples/migrations/1485949617_create_movies_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE movies ( + user_id integer, + name varchar(40), + director varchar(40) +); diff --git a/database/mssql/examples/migrations/1585849751_just_a_comment.up.sql b/database/mssql/examples/migrations/1585849751_just_a_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/mssql/examples/migrations/1585849751_just_a_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/mssql/examples/migrations/1685849751_another_comment.up.sql b/database/mssql/examples/migrations/1685849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/mssql/examples/migrations/1685849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/mssql/examples/migrations/1785849751_another_comment.up.sql b/database/mssql/examples/migrations/1785849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/mssql/examples/migrations/1785849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/mssql/examples/migrations/1885849751_another_comment.up.sql b/database/mssql/examples/migrations/1885849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/mssql/examples/migrations/1885849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/mssql/mssql.go b/database/mssql/mssql.go index 387e9a6ca..ff06d70e1 100644 --- a/database/mssql/mssql.go +++ b/database/mssql/mssql.go @@ -8,9 +8,10 @@ import ( "io/ioutil" nurl "net/url" - _ "github.com/denisenkom/go-mssqldb" // mssql support + mssql "github.com/denisenkom/go-mssqldb" // mssql support "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" ) func init() { @@ -29,6 +30,13 @@ var ( ErrDatabaseDirty = fmt.Errorf("database is dirty") ) +var lockErrorMap = map[int]string{ + -1: "The lock request timed out.", + -2: "The lock request was canceled.", + -3: "The lock request was chosen as a deadlock victim.", + -999: "Parameter validation or other call error.", +} + // Config for database type Config struct { MigrationsTable string @@ -47,8 +55,7 @@ type MSSQL struct { config *Config } -// WithInstance returns a database instance from an already create database connection -// TODO: WithInstance double check that docs are correct for this function +// WithInstance returns a database instance from an already created database connection func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { if config == nil { return nil, ErrNilConfig @@ -105,10 +112,6 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { return ss, nil } -func dbConnectionString(host, port string) string { - return fmt.Sprintf("postgres://postgres@%s:%s/postgres?sslmode=disable", host, port) -} - // Open a connection to the database func (ss *MSSQL) Open(url string) (database.Driver, error) { purl, err := nurl.Parse(url) @@ -122,14 +125,12 @@ func (ss *MSSQL) Open(url string) (database.Driver, error) { } migrationsTable := purl.Query().Get("x-migrations-table") - if len(migrationsTable) == 0 { - migrationsTable = DefaultMigrationsTable - } px, err := WithInstance(db, &Config{ DatabaseName: purl.Path, MigrationsTable: migrationsTable, }) + if err != nil { return nil, err } @@ -158,22 +159,20 @@ func (ss *MSSQL) Lock() error { return err } - // start a transaction - query := "BEGIN TRANSACTION" - if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { - return &database.Error{OrigErr: err, Err: "get lock transaction", Query: []byte(query)} - } - // This will either obtain the lock immediately and return true, // or return false if the lock cannot be acquired immediately. // MS Docs: sp_getapplock: https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-getapplock-transact-sql?view=sql-server-2017 - query = `EXEC sp_getapplock @Resource = ?, @LockMode = 'Update'` - if _, err := ss.conn.ExecContext(context.Background(), query, aid); err != nil { + query := `EXEC sp_getapplock @Resource = ?, @LockMode = 'Update', @LockOwner = 'Session', @LockTimeout = 0` + + var status mssql.ReturnStatus + if _, err = ss.conn.ExecContext(context.Background(), query, aid, &status); err == nil && status > -1 { + ss.isLocked = true + return nil + } else if err != nil { return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)} + } else { + return &database.Error{Err: fmt.Sprintf("try lock failed with error %v", lockErrorMap[int(status)]), Query: []byte(query)} } - - ss.isLocked = true - return nil } // Unlock froms the migration lock from the database @@ -188,18 +187,12 @@ func (ss *MSSQL) Unlock() error { } // MS Docs: sp_releaseapplock: https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-releaseapplock-transact-sql?view=sql-server-2017 - query := `EXEC sp_releaseapplock @Resource = ?` + query := `EXEC sp_releaseapplock @Resource = ?, @LockOwner = 'Session'` if _, err := ss.conn.ExecContext(context.Background(), query, aid); err != nil { return &database.Error{OrigErr: err, Query: []byte(query)} } ss.isLocked = false - // end lock transaction - query = "COMMIT" - if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { - return &database.Error{OrigErr: err, Err: "commit lock transaction", Query: []byte(query)} - } - return nil } @@ -213,78 +206,25 @@ func (ss *MSSQL) Run(migration io.Reader) error { // run migration query := string(migr[:]) if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { - // // FIXME: check for mssql error here - // if pgErr, ok := err.(*pq.Error); ok { - // var line uint - // var col uint - // var lineColOK bool - // if pgErr.Position != "" { - // if pos, err := strconv.ParseUint(pgErr.Position, 10, 64); err == nil { - // line, col, lineColOK = computeLineFromPos(query, int(pos)) - // } - // } - // message := fmt.Sprintf("migration failed: %s", pgErr.Message) - // if lineColOK { - // message = fmt.Sprintf("%s (column %d)", message, col) - // } - // if pgErr.Detail != "" { - // message = fmt.Sprintf("%s, %s", message, pgErr.Detail) - // } - // return database.Error{OrigErr: err, Err: message, Query: migr, Line: line} - // } return database.Error{OrigErr: err, Err: "migration failed", Query: migr} } return nil } -// func computeLineFromPos(s string, pos int) (line uint, col uint, ok bool) { -// // replace crlf with lf -// s = strings.Replace(s, "\r\n", "\n", -1) -// // pg docs: pos uses index 1 for the first character, and positions are measured in characters not bytes -// runes := []rune(s) -// if pos > len(runes) { -// return 0, 0, false -// } -// sel := runes[:pos] -// line = uint(runesCount(sel, newLine) + 1) -// col = uint(pos - 1 - runesLastIndex(sel, newLine)) -// return line, col, true -// } - -const newLine = '\n' - -func runesCount(input []rune, target rune) int { - var count int - for _, r := range input { - if r == target { - count++ - } - } - return count -} - -func runesLastIndex(input []rune, target rune) int { - for i := len(input) - 1; i >= 0; i-- { - if input[i] == target { - return i - } - } - return -1 -} - // SetVersion for the current database func (ss *MSSQL) SetVersion(version int, dirty bool) error { - tx := ss.db - // tx, err := ss.conn.BeginTx(context.Background(), &sql.TxOptions{}) - // if err != nil { - // return &database.Error{OrigErr: err, Err: "transaction start failed"} - // } + tx, err := ss.conn.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } query := `TRUNCATE TABLE "` + ss.config.MigrationsTable + `"` if _, err := tx.Exec(query); err != nil { - // tx.Rollback() + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } return &database.Error{OrigErr: err, Query: []byte(query)} } @@ -295,14 +235,16 @@ func (ss *MSSQL) SetVersion(version int, dirty bool) error { } query = `INSERT INTO "` + ss.config.MigrationsTable + `" (version, dirty) VALUES ($1, $2)` if _, err := tx.Exec(query, version, dirtyBit); err != nil { - // tx.Rollback() + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } return &database.Error{OrigErr: err, Query: []byte(query)} } } - // if err := tx.Commit(); err != nil { - // return &database.Error{OrigErr: err, Err: "transaction commit failed"} - // } + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } return nil } @@ -317,11 +259,6 @@ func (ss *MSSQL) Version() (version int, dirty bool, err error) { case err != nil: // FIXME: convert to MSSQL error - // if e, ok := err.(*pq.Error); ok { - // if e.Code.Name() == "undefined_table" { - // return database.NilVersion, false, nil - // } - // } return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} default: @@ -361,50 +298,24 @@ func (ss *MSSQL) Drop() error { return &database.Error{OrigErr: err, Query: []byte(query)} } - if err := ss.ensureVersionTable(); err != nil { - return err - } - return nil - - // // select all tables in current schema - // query := `SELECT t.name FROM sys.tables AS t INNER JOIN sys.schemas AS s ON t.[schema_id] = s.[schema_id] WHERE s.name = N'dbo';` - // tables, err := ss.conn.QueryContext(context.Background(), query) - // if err != nil { - // return &database.Error{OrigErr: err, Query: []byte(query)} - // } - // defer tables.Close() - - // // delete one table after another - // tableNames := make([]string, 0) - // for tables.Next() { - // var tableName string - // if err := tables.Scan(&tableName); err != nil { - // return err - // } - // if len(tableName) > 0 { - // tableNames = append(tableNames, tableName) - // } - // } - - // if len(tableNames) > 0 { - // // delete one by one ... - // for _, t := range tableNames { - // query = `DROP TABLE IF EXISTS "` + t + `"` - // if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { - // return &database.Error{OrigErr: err, Query: []byte(query)} - // } - // } - - // if err := ss.ensureVersionTable(); err != nil { - // return err - // } - // } - - // return nil } func (ss *MSSQL) ensureVersionTable() (err error) { + if err = ss.Lock(); err != nil { + return err + } + + defer func() { + if e := ss.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + query := `IF NOT EXISTS (SELECT * FROM sysobjects diff --git a/database/mssql/mssql_test.go b/database/mssql/mssql_test.go new file mode 100755 index 000000000..d350b9554 --- /dev/null +++ b/database/mssql/mssql_test.go @@ -0,0 +1,157 @@ +package mssql + +import ( + "context" + "database/sql" + sqldriver "database/sql/driver" + "fmt" + "testing" + + "github.com/dhui/dktest" + "github.com/golang-migrate/migrate/v4" + + dt "github.com/golang-migrate/migrate/v4/database/testing" + "github.com/golang-migrate/migrate/v4/dktesting" + + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +const defaultPort = 1433 +const saPassword = "Root1234" + +var ( + opts = dktest.Options{ + Env: map[string]string{"ACCEPT_EULA": "Y", "SA_PASSWORD": saPassword, "MSSQL_PID": "Express"}, + PortRequired: true, ReadyFunc: isReady, + } + // Supported versions: https://www.mysql.com/support/supportedplatforms/database.html + specs = []dktesting.ContainerSpec{ + {ImageName: "mcr.microsoft.com/mssql/server:2017-latest-ubuntu", Options: opts}, + {ImageName: "mcr.microsoft.com/mssql/server:2019-latest", Options: opts}, + } +) + +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + ip, port, err := c.Port(defaultPort) + if err != nil { + return false + } + uri := fmt.Sprintf("sqlserver://sa:%v@%v:%v?database=master", saPassword, ip, port) + db, err := sql.Open("sqlserver", uri) + if err != nil { + return false + } + defer db.Close() + if err = db.PingContext(ctx); err != nil { + switch err { + case sqldriver.ErrBadConn: + return false + default: + fmt.Println(err) + } + return false + } + + return true +} + +func Test(t *testing.T) { + // mysql.SetLogger(mysql.Logger(log.New(ioutil.Discard, "", log.Ltime))) + + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("sqlserver://sa:%v@%v:%v?master", saPassword, ip, port) + p := &MSSQL{} + d, err := p.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + defer d.Close() + dt.Test(t, d, []byte("SELECT 1")) + + // check ensureVersionTable + if err := d.(*MSSQL).ensureVersionTable(); err != nil { + t.Fatal(err) + } + // check again + if err := d.(*MSSQL).ensureVersionTable(); err != nil { + t.Fatal(err) + } + }) +} + +func TestMigrate(t *testing.T) { + // mysql.SetLogger(mysql.Logger(log.New(ioutil.Discard, "", log.Ltime))) + + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("sqlserver://sa:%v@%v:%v?master", saPassword, ip, port) + p := &MSSQL{} + d, err := p.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + defer d.Close() + + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "public", d) + if err != nil { + t.Fatalf("%v", err) + } + dt.TestMigrate(t, m, []byte("SELECT 1")) + + // check ensureVersionTable + if err := d.(*MSSQL).ensureVersionTable(); err != nil { + t.Fatal(err) + } + // check again + if err := d.(*MSSQL).ensureVersionTable(); err != nil { + t.Fatal(err) + } + }) +} + +func TestLockWorks(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("sqlserver://sa:%v@%v:%v?master", saPassword, ip, port) + p := &MSSQL{} + d, err := p.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + dt.Test(t, d, []byte("SELECT 1")) + + ms := d.(*MSSQL) + + err = ms.Lock() + if err != nil { + t.Fatal(err) + } + err = ms.Unlock() + if err != nil { + t.Fatal(err) + } + + // make sure the 2nd lock works (RELEASE_LOCK is very finicky) + err = ms.Lock() + if err != nil { + t.Fatal(err) + } + err = ms.Unlock() + if err != nil { + t.Fatal(err) + } + }) +} diff --git a/go.sum b/go.sum index c6a7a9c57..9b6b62c7d 100644 --- a/go.sum +++ b/go.sum @@ -56,7 +56,7 @@ github.com/dhui/dktest v0.3.0/go.mod h1:cyzIUfGsBEbZ6BT7tnXqAShHSXCZhSNmFl70sZ7c github.com/docker/distribution v2.7.0+incompatible h1:neUDAlf3wX6Ml4HdqTrbcOHXtfRN0TFIwt6YFL7N9RU= github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf h1:ZjoOm2c/ckflZVEbHBOwuf4FoUajM5Jx9dDTVBHK9vc= +github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf h1:2v/98rHzs3v6X0AHtoCH9u+e56SdnpogB1Z2fFe1KqQ= github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= From 2fbec2f7608983b3589a2d57ac250c9f255a4e06 Mon Sep 17 00:00:00 2001 From: nathan-c Date: Sun, 19 May 2019 13:43:47 +0100 Subject: [PATCH 04/12] update mssql driver module --- go.mod | 3 +-- go.sum | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index afeea521e..215985713 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/cockroachdb/apd v1.1.0 // indirect github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c github.com/cznic/ql v1.2.0 - github.com/denisenkom/go-mssqldb v0.0.0-20190121005146-b04fd42d9952 + github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 github.com/dhui/dktest v0.3.0 github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf github.com/fsouza/fake-gcs-server v1.7.0 @@ -51,5 +51,4 @@ require ( google.golang.org/appengine v1.5.0 // indirect google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb google.golang.org/grpc v1.20.1 // indirect - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) diff --git a/go.sum b/go.sum index 9b6b62c7d..55002b321 100644 --- a/go.sum +++ b/go.sum @@ -49,8 +49,8 @@ github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc/go.mod h1:Y1SNZ4dRUOKX github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisenkom/go-mssqldb v0.0.0-20190121005146-b04fd42d9952 h1:b5OnbZD49x9g+/FcYbs/vukEt8C/jUbGhCJ3uduQmu8= -github.com/denisenkom/go-mssqldb v0.0.0-20190121005146-b04fd42d9952/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= +github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 h1:tkum0XDgfR0jcVVXuTsYv/erY2NnEDqwRojbxR1rBYA= +github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/dhui/dktest v0.3.0 h1:kwX5a7EkLcjo7VpsPQSYJcKGbXBXdjI9FGjuUj1jn6I= github.com/dhui/dktest v0.3.0/go.mod h1:cyzIUfGsBEbZ6BT7tnXqAShHSXCZhSNmFl70sZ7c1yc= github.com/docker/distribution v2.7.0+incompatible h1:neUDAlf3wX6Ml4HdqTrbcOHXtfRN0TFIwt6YFL7N9RU= @@ -202,6 +202,7 @@ go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -295,6 +296,7 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From c273636efe14ce8f47671ee1742d6994e1f7c6ec Mon Sep 17 00:00:00 2001 From: nathan-c Date: Sun, 19 May 2019 14:08:10 +0100 Subject: [PATCH 05/12] parse mssql errors --- database/mssql/mssql.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/database/mssql/mssql.go b/database/mssql/mssql.go index ff06d70e1..e6b6ed5e0 100644 --- a/database/mssql/mssql.go +++ b/database/mssql/mssql.go @@ -206,6 +206,13 @@ func (ss *MSSQL) Run(migration io.Reader) error { // run migration query := string(migr[:]) if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { + if msErr, ok := err.(*mssql.Error); ok { + message := fmt.Sprintf("migration failed: %s", msErr.Message) + if msErr.ProcName != "" { + message = fmt.Sprintf("%s (proc name %s)", msErr.Message, msErr.ProcName) + } + return database.Error{OrigErr: err, Err: message, Query: migr, Line: uint(msErr.LineNo)} + } return database.Error{OrigErr: err, Err: "migration failed", Query: migr} } From 14e6c50d2efac0d4cee44038ce6d2e8cce0e3f4e Mon Sep 17 00:00:00 2001 From: nathan-c Date: Sun, 19 May 2019 14:47:09 +0100 Subject: [PATCH 06/12] fix mssql build flag --- internal/cli/build_mssql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cli/build_mssql.go b/internal/cli/build_mssql.go index 2de4ad1b0..7d8f83ec2 100644 --- a/internal/cli/build_mssql.go +++ b/internal/cli/build_mssql.go @@ -1,4 +1,4 @@ -// +build postgres +// +build mssql package cli From 95290f2fbd9b930f70d1a6ac1735efbf5a7ce21e Mon Sep 17 00:00:00 2001 From: nathan-c Date: Sun, 19 May 2019 15:09:56 +0100 Subject: [PATCH 07/12] mssql: change migration table name and add readme --- database/mssql/README.md | 21 +++++++++++++++++++++ database/mssql/mssql.go | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 database/mssql/README.md diff --git a/database/mssql/README.md b/database/mssql/README.md new file mode 100644 index 000000000..bddb62584 --- /dev/null +++ b/database/mssql/README.md @@ -0,0 +1,21 @@ +# Microsoft SQL Server + +`sqlserver://username:password@host/instance?param1=value¶m2=value` +`sqlserver://username:password@host:port?param1=value¶m2=value` + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| `x-migrations-table` | `MigrationsTable` | Name of the migrations table | +| `database` | `DatabaseName` | The name of the database to connect to | +| `username` | | enter the SQL Server Authentication user id or the Windows Authentication user id in the DOMAIN\User format. On Windows, if user id is empty or missing Single-Sign-On is used. | +| `password` | | The user's password. | +| `host` | | The host to connect to. | +| `port` | | The port to connect to. | +| `instance` | | SQL Server instance name. | +| `connection+timeout` | | in seconds (default is 0 for no timeout), set to 0 for no timeout. | +| `dial+timeout` | | in seconds (default is 15), set to 0 for no timeout. | +| `encrypt` | | `disable` - Data send between client and server is not encrypted. `false` - Data sent between client and server is not encrypted beyond the login packet (Default). `true` - Data sent between client and server is encrypted. | +| `app+name` || The application name (default is go-mssqldb). | + +See https://github.com/denisenkom/go-mssqldb for full parameter list. + diff --git a/database/mssql/mssql.go b/database/mssql/mssql.go index e6b6ed5e0..64f8eb1e8 100644 --- a/database/mssql/mssql.go +++ b/database/mssql/mssql.go @@ -21,7 +21,7 @@ func init() { } // DefaultMigrationsTable is the name of the migrations table in the database -var DefaultMigrationsTable = "SchemaMigrations" +var DefaultMigrationsTable = "schema_migrations" var ( ErrNilConfig = fmt.Errorf("no config") From 3bd91a774e30f573a417a782c16406366d682db7 Mon Sep 17 00:00:00 2001 From: nathan-c Date: Sun, 19 May 2019 15:15:07 +0100 Subject: [PATCH 08/12] mssql: reorder readme --- database/mssql/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/mssql/README.md b/database/mssql/README.md index bddb62584..060f03ae6 100644 --- a/database/mssql/README.md +++ b/database/mssql/README.md @@ -6,12 +6,12 @@ | URL Query | WithInstance Config | Description | |------------|---------------------|-------------| | `x-migrations-table` | `MigrationsTable` | Name of the migrations table | -| `database` | `DatabaseName` | The name of the database to connect to | | `username` | | enter the SQL Server Authentication user id or the Windows Authentication user id in the DOMAIN\User format. On Windows, if user id is empty or missing Single-Sign-On is used. | | `password` | | The user's password. | | `host` | | The host to connect to. | | `port` | | The port to connect to. | | `instance` | | SQL Server instance name. | +| `database` | `DatabaseName` | The name of the database to connect to | | `connection+timeout` | | in seconds (default is 0 for no timeout), set to 0 for no timeout. | | `dial+timeout` | | in seconds (default is 15), set to 0 for no timeout. | | `encrypt` | | `disable` - Data send between client and server is not encrypted. `false` - Data sent between client and server is not encrypted beyond the login packet (Default). `true` - Data sent between client and server is encrypted. | From 5ac583ba7bc795a7a46d87a00b09b78f3fd69702 Mon Sep 17 00:00:00 2001 From: nathan-c Date: Sun, 19 May 2019 15:21:25 +0100 Subject: [PATCH 09/12] mssql: fix linter errors --- database/mssql/mssql_test.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/database/mssql/mssql_test.go b/database/mssql/mssql_test.go index d350b9554..817fa58d2 100755 --- a/database/mssql/mssql_test.go +++ b/database/mssql/mssql_test.go @@ -5,6 +5,7 @@ import ( "database/sql" sqldriver "database/sql/driver" "fmt" + "log" "testing" "github.com/dhui/dktest" @@ -41,7 +42,11 @@ func isReady(ctx context.Context, c dktest.ContainerInfo) bool { if err != nil { return false } - defer db.Close() + defer func() { + if err := db.Close(); err != nil { + log.Println("close error:", err) + } + }() if err = db.PingContext(ctx); err != nil { switch err { case sqldriver.ErrBadConn: @@ -70,7 +75,13 @@ func Test(t *testing.T) { if err != nil { t.Fatalf("%v", err) } - defer d.Close() + + defer func() { + if err := d.Close(); err != nil { + log.Println("close error:", err) + } + }() + dt.Test(t, d, []byte("SELECT 1")) // check ensureVersionTable @@ -99,7 +110,12 @@ func TestMigrate(t *testing.T) { if err != nil { t.Fatalf("%v", err) } - defer d.Close() + + defer func() { + if err := d.Close(); err != nil { + log.Println("close error:", err) + } + }() m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "public", d) if err != nil { From 98e5f88b9fa47837524890c56b5bb06323aedf59 Mon Sep 17 00:00:00 2001 From: nathan-c Date: Sun, 19 May 2019 16:08:15 +0100 Subject: [PATCH 10/12] mssql: fix error parsing and add tests --- database/mssql/mssql.go | 2 +- database/mssql/mssql_test.go | 94 +++++++++++++++++++++++++++--------- 2 files changed, 71 insertions(+), 25 deletions(-) diff --git a/database/mssql/mssql.go b/database/mssql/mssql.go index 64f8eb1e8..fd12630e0 100644 --- a/database/mssql/mssql.go +++ b/database/mssql/mssql.go @@ -206,7 +206,7 @@ func (ss *MSSQL) Run(migration io.Reader) error { // run migration query := string(migr[:]) if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { - if msErr, ok := err.(*mssql.Error); ok { + if msErr, ok := err.(mssql.Error); ok { message := fmt.Sprintf("migration failed: %s", msErr.Message) if msErr.ProcName != "" { message = fmt.Sprintf("%s (proc name %s)", msErr.Message, msErr.ProcName) diff --git a/database/mssql/mssql_test.go b/database/mssql/mssql_test.go index 817fa58d2..013a234c0 100755 --- a/database/mssql/mssql_test.go +++ b/database/mssql/mssql_test.go @@ -6,6 +6,7 @@ import ( sqldriver "database/sql/driver" "fmt" "log" + "strings" "testing" "github.com/dhui/dktest" @@ -32,12 +33,16 @@ var ( } ) +func msConnectionString(host, port string) string { + return fmt.Sprintf("sqlserver://sa:%v@%v:%v?database=master", saPassword, host, port) +} + func isReady(ctx context.Context, c dktest.ContainerInfo) bool { ip, port, err := c.Port(defaultPort) if err != nil { return false } - uri := fmt.Sprintf("sqlserver://sa:%v@%v:%v?database=master", saPassword, ip, port) + uri := msConnectionString(ip, port) db, err := sql.Open("sqlserver", uri) if err != nil { return false @@ -61,15 +66,13 @@ func isReady(ctx context.Context, c dktest.ContainerInfo) bool { } func Test(t *testing.T) { - // mysql.SetLogger(mysql.Logger(log.New(ioutil.Discard, "", log.Ltime))) - dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { ip, port, err := c.Port(defaultPort) if err != nil { t.Fatal(err) } - addr := fmt.Sprintf("sqlserver://sa:%v@%v:%v?master", saPassword, ip, port) + addr := msConnectionString(ip, port) p := &MSSQL{} d, err := p.Open(addr) if err != nil { @@ -78,33 +81,22 @@ func Test(t *testing.T) { defer func() { if err := d.Close(); err != nil { - log.Println("close error:", err) + t.Error(err) } }() dt.Test(t, d, []byte("SELECT 1")) - - // check ensureVersionTable - if err := d.(*MSSQL).ensureVersionTable(); err != nil { - t.Fatal(err) - } - // check again - if err := d.(*MSSQL).ensureVersionTable(); err != nil { - t.Fatal(err) - } }) } func TestMigrate(t *testing.T) { - // mysql.SetLogger(mysql.Logger(log.New(ioutil.Discard, "", log.Ltime))) - dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { ip, port, err := c.Port(defaultPort) if err != nil { t.Fatal(err) } - addr := fmt.Sprintf("sqlserver://sa:%v@%v:%v?master", saPassword, ip, port) + addr := msConnectionString(ip, port) p := &MSSQL{} d, err := p.Open(addr) if err != nil { @@ -113,24 +105,78 @@ func TestMigrate(t *testing.T) { defer func() { if err := d.Close(); err != nil { - log.Println("close error:", err) + t.Error(err) } }() - m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "public", d) + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "master", d) if err != nil { - t.Fatalf("%v", err) + t.Fatal(err) } dt.TestMigrate(t, m, []byte("SELECT 1")) + }) +} - // check ensureVersionTable - if err := d.(*MSSQL).ensureVersionTable(); err != nil { +func TestMultiStatement(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { t.Fatal(err) } - // check again - if err := d.(*MSSQL).ensureVersionTable(); err != nil { + + addr := msConnectionString(ip, port) + ms := &MSSQL{} + d, err := ms.Open(addr) + if err != nil { t.Fatal(err) } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);")); err != nil { + t.Fatalf("expected err to be nil, got %v", err) + } + + // make sure second table exists + var exists int + if err := d.(*MSSQL).conn.QueryRowContext(context.Background(), "SELECT COUNT(1) FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT schema_name()) AND table_catalog = (SELECT db_name())").Scan(&exists); err != nil { + t.Fatal(err) + } + if exists != 1 { + t.Fatalf("expected table bar to exist") + } + }) +} + +func TestErrorParsing(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := msConnectionString(ip, port) + p := &MSSQL{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + wantErr := `migration failed: Unknown object type 'TABLEE' used in a CREATE, DROP, or ALTER statement. in line 1:` + + ` CREATE TABLE foo (foo text); CREATE TABLEE bar (bar text); (details: mssql: Unknown object type ` + + `'TABLEE' used in a CREATE, DROP, or ALTER statement.)` + if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLEE bar (bar text);")); err == nil { + t.Fatal("expected err but got nil") + } else if err.Error() != wantErr { + t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error()) + } }) } From e211c5b5f51766962186d45a623ff53878173f19 Mon Sep 17 00:00:00 2001 From: nathan-c Date: Sun, 19 May 2019 21:36:50 +0100 Subject: [PATCH 11/12] mssql: increase pull timeout --- database/mssql/mssql_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/database/mssql/mssql_test.go b/database/mssql/mssql_test.go index 013a234c0..341c0b991 100755 --- a/database/mssql/mssql_test.go +++ b/database/mssql/mssql_test.go @@ -8,6 +8,7 @@ import ( "log" "strings" "testing" + "time" "github.com/dhui/dktest" "github.com/golang-migrate/migrate/v4" @@ -24,7 +25,7 @@ const saPassword = "Root1234" var ( opts = dktest.Options{ Env: map[string]string{"ACCEPT_EULA": "Y", "SA_PASSWORD": saPassword, "MSSQL_PID": "Express"}, - PortRequired: true, ReadyFunc: isReady, + PortRequired: true, ReadyFunc: isReady, PullTimeout: 2 * time.Minute, } // Supported versions: https://www.mysql.com/support/supportedplatforms/database.html specs = []dktesting.ContainerSpec{ From aeb7f633f3b8699f81402037e9881115d280f966 Mon Sep 17 00:00:00 2001 From: Nathan Collard Date: Tue, 21 May 2019 08:44:17 +0100 Subject: [PATCH 12/12] mssql: fix code review comments --- README.md | 1 + database/mssql/mssql.go | 4 ++-- database/mssql/mssql_test.go | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d6f089fd2..ad006b6b8 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ Database drivers run migrations. [Add a new database?](database/driver.go) * [CockroachDB](database/cockroachdb) * [ClickHouse](database/clickhouse) * [Firebird](database/firebird) ([todo #49](https://github.com/golang-migrate/migrate/issues/49)) + * [Postgres](database/postgres) ### Database URLs diff --git a/database/mssql/mssql.go b/database/mssql/mssql.go index fd12630e0..a7ca8ce25 100644 --- a/database/mssql/mssql.go +++ b/database/mssql/mssql.go @@ -30,7 +30,7 @@ var ( ErrDatabaseDirty = fmt.Errorf("database is dirty") ) -var lockErrorMap = map[int]string{ +var lockErrorMap = map[mssql.ReturnStatus]string{ -1: "The lock request timed out.", -2: "The lock request was canceled.", -3: "The lock request was chosen as a deadlock victim.", @@ -171,7 +171,7 @@ func (ss *MSSQL) Lock() error { } else if err != nil { return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)} } else { - return &database.Error{Err: fmt.Sprintf("try lock failed with error %v", lockErrorMap[int(status)]), Query: []byte(query)} + return &database.Error{Err: fmt.Sprintf("try lock failed with error %v: %v", status, lockErrorMap[status]), Query: []byte(query)} } } diff --git a/database/mssql/mssql_test.go b/database/mssql/mssql_test.go index 341c0b991..d26341d21 100755 --- a/database/mssql/mssql_test.go +++ b/database/mssql/mssql_test.go @@ -27,7 +27,7 @@ var ( Env: map[string]string{"ACCEPT_EULA": "Y", "SA_PASSWORD": saPassword, "MSSQL_PID": "Express"}, PortRequired: true, ReadyFunc: isReady, PullTimeout: 2 * time.Minute, } - // Supported versions: https://www.mysql.com/support/supportedplatforms/database.html + // Container versions: https://mcr.microsoft.com/v2/mssql/server/tags/list specs = []dktesting.ContainerSpec{ {ImageName: "mcr.microsoft.com/mssql/server:2017-latest-ubuntu", Options: opts}, {ImageName: "mcr.microsoft.com/mssql/server:2019-latest", Options: opts},