From 729c440ad19db2cd59fa6c487724edf42d5bf141 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Mon, 6 Jan 2020 15:53:45 +0530 Subject: [PATCH 01/24] initial pass --- Dockerfile | 8 ++ Makefile | 2 +- database/driver.go | 2 +- database/neo4j/neo4j.go | 161 +++++++++++++++++++++++++++++++++++ database/neo4j/neo4j_test.go | 46 ++++++++++ go.mod | 2 + go.sum | 13 +++ 7 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 database/neo4j/neo4j.go create mode 100644 database/neo4j/neo4j_test.go diff --git a/Dockerfile b/Dockerfile index cfab35537..716a96495 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,14 @@ ARG VERSION RUN apk add --no-cache git gcc musl-dev +# dependencies +RUN apk add --update --no-cache ca-certificates cmake make g++ openssl-dev git curl pkgconfig + +# build seabolt +RUN git clone -b 1.7 https://github.com/neo4j-drivers/seabolt.git /seabolt +WORKDIR /seabolt/build +RUN cmake -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_LIBDIR=lib .. && cmake --build . --target install + WORKDIR /go/src/github.com/golang-migrate/migrate COPY . ./ diff --git a/Makefile b/Makefile index 291c179cc..425f4b822 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SOURCE ?= file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab -DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver firebird +DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver firebird neo4j VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") diff --git a/database/driver.go b/database/driver.go index 2c673caaf..c26a71e73 100644 --- a/database/driver.go +++ b/database/driver.go @@ -61,7 +61,7 @@ type Driver interface { // all migrations have been run. Unlock() error - // Run applies a migration to the database. migration is garantueed to be not nil. + // Run applies a migration to the database. migration is guaranteed to be not nil. Run(migration io.Reader) error // SetVersion saves version and dirty state. diff --git a/database/neo4j/neo4j.go b/database/neo4j/neo4j.go new file mode 100644 index 000000000..75433b4ff --- /dev/null +++ b/database/neo4j/neo4j.go @@ -0,0 +1,161 @@ +package neo4j + +import ( + "fmt" + "io" + "io/ioutil" + neturl "net/url" + + "github.com/golang-migrate/migrate/v4/database" + "github.com/neo4j/neo4j-go-driver/neo4j" +) + +func init() { + db := Neo4J{} + database.Register("bolt", &db) + database.Register("neo4j", &db) +} + +var DefaultMigrationsLabel = "SchemaMigration" + +var ( + ErrNilConfig = fmt.Errorf("no config") +) + +type Config struct { + AuthToken neo4j.AuthToken + URL string + MigrationsLabel string +} + +type Neo4J struct { + driver neo4j.Driver + + // Open and WithInstance need to guarantee that config is never nil + config *Config +} + +func WithInstance(config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + neoDriver, err := neo4j.NewDriver(config.URL, config.AuthToken) + if err != nil { + return nil, err + } + + driver := &Neo4J{ + driver: neoDriver, + config: config, + } + + if err := driver.ensureVersionConstraint(); err != nil { + return nil, err + } + + return driver, nil +} + +func (n *Neo4J) Open(url string) (database.Driver, error) { + uri, err := neturl.Parse(url) + if err != nil { + return nil, err + } + password, _ := uri.User.Password() + authToken := neo4j.BasicAuth(uri.User.Username(), password, "") + + return WithInstance(&Config{ + URL: url, + AuthToken: authToken, + MigrationsLabel: DefaultMigrationsLabel, + }) +} + +func (n *Neo4J) Close() error { + return n.driver.Close() +} + +func (n *Neo4J) Lock() error { + return nil +} + +func (n *Neo4J) Unlock() error { + return nil +} + +func (n *Neo4J) Run(migration io.Reader) error { + body, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + + session, err := n.driver.Session(neo4j.AccessModeWrite) + if err != nil { + return err + } + defer session.Close() + + _, err = session.Run(string(body[:]), nil) + return err +} + +func (n *Neo4J) SetVersion(version int, dirty bool) error { + session, err := n.driver.Session(neo4j.AccessModeRead) + if err != nil { + return err + } + defer session.Close() + + _, err = session.Run("MERGE (sm:$migration {version: $version, dirty: $dirty})", + map[string]interface{}{"migration": n.config.MigrationsLabel, "version": version, "dirty": dirty}) + return err +} + +func (n *Neo4J) Version() (version int, dirty bool, err error) { + session, err := n.driver.Session(neo4j.AccessModeRead) + if err != nil { + return -1, false, err + } + defer session.Close() + + result, err := session.Run("MATCH (sm:$migration) RETURN sm.version, sm.dirty ORDER BY sm.version DESC LIMIT 1", + map[string]interface{}{"migration": n.config.MigrationsLabel}) + if err != nil { + return -1, false, err + } + if result.Next() { + versionResult, ok := result.Record().Get("version") + if !ok { + version = -1 + } else { + version = versionResult.(int) + } + } else { + version = -1 + } + + return +} + +func (n *Neo4J) Drop() error { + session, err := n.driver.Session(neo4j.AccessModeWrite); if err != nil { + return err + } + defer session.Close() + + _, err = session.Run("MATCH (n) DETACH DELETE n", map[string]interface{}{}) + return err +} + +func (n *Neo4J) ensureVersionConstraint() (err error) { + session, err := n.driver.Session(neo4j.AccessModeWrite); if err != nil { + return err + } + defer session.Close() + + _, err = session.Run("CREATE CONSTRAINT ON (a:$migration) ASSERT a.version IS UNIQUE", + map[string]interface{}{"migration": n.config.MigrationsLabel}) + return err +} + diff --git a/database/neo4j/neo4j_test.go b/database/neo4j/neo4j_test.go new file mode 100644 index 000000000..0c3482228 --- /dev/null +++ b/database/neo4j/neo4j_test.go @@ -0,0 +1,46 @@ +package neo4j + +import ( + "fmt" + "testing" + + "github.com/dhui/dktest" + "github.com/golang-migrate/migrate/v4/dktesting" + + dt "github.com/golang-migrate/migrate/v4/database/testing" + +) + +var ( + opts = dktest.Options{PortRequired: true, ReadyFunc: nil} + specs = []dktesting.ContainerSpec{ + {ImageName: "neo4j:3.5", Options: opts}, + {ImageName: "neo4j:3.5-enterprise", Options: opts}, + } +) + +func neoConnectionString(host, port string) string { + return fmt.Sprintf("neo4j://neo4j:neo4j@%s:%s", host, port) +} + +func Test(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := neoConnectionString(ip, port) + n := &Neo4J{} + d, err := n.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + dt.Test(t, d, []byte("MATCH a RETURN a")) + }) +} \ No newline at end of file diff --git a/go.mod b/go.mod index b7ad4b1cf..c2fc9d191 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,8 @@ require ( github.com/mattn/go-sqlite3 v1.10.0 github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c // indirect github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 + github.com/neo4j-drivers/gobolt v1.7.4 // indirect + github.com/neo4j/neo4j-go-driver v1.7.4 github.com/pkg/errors v0.8.1 // indirect github.com/satori/go.uuid v1.2.0 // indirect github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 // indirect diff --git a/go.sum b/go.sum index 086241c5d..cf388667c 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,7 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 h1:aaQcKT9WumO6JEJcRyTqFVq4XUZiUcKR2/GI31TOcz8= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsouza/fake-gcs-server v1.7.0 h1:Un0BXUXrRWYSmYyC1Rqm2e2WJfTPyDy/HGMz31emTi8= github.com/fsouza/fake-gcs-server v1.7.0/go.mod h1:5XIRs4YvwNbNoz+1JF8j6KLAyDh7RHGAyAK3EP2EsNk= @@ -94,6 +95,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= @@ -127,6 +129,7 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 h1:vr3AYkKovP8uR8AvSGGUK1IDqRa5lAAvEkZG1LKaCRc= github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= @@ -161,8 +164,14 @@ github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 h1:P48LjvUQpTReR3TQRbxSeSBsMXzfK0uol7eRcr7VBYQ= github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA= +github.com/neo4j-drivers/gobolt v1.7.4 h1:80c7W+vtw39ES9Q85q9GZh4tJo+1MpQGpFTuo28CP+Y= +github.com/neo4j-drivers/gobolt v1.7.4/go.mod h1:O9AUbip4Dgre+CD3p40dnMD4a4r52QBIfblg5k7CTbE= +github.com/neo4j/neo4j-go-driver v1.7.4 h1:BgVVwYkG3DWcZGiOPUOkwkd54sSg+UHDaLYz3aiNCek= +github.com/neo4j/neo4j-go-driver v1.7.4/go.mod h1:aPO0vVr+WnhEJne+FgFjfsjzAnssPFLucHgGZ76Zb/U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -192,6 +201,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= @@ -303,11 +313,14 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= From 5a1b3510712acd257be2c8bfd3896034560384ba Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Mon, 6 Jan 2020 16:03:54 +0530 Subject: [PATCH 02/24] new cli build file --- database/driver.go | 2 +- internal/cli/build_neo4j.go | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 internal/cli/build_neo4j.go diff --git a/database/driver.go b/database/driver.go index c26a71e73..7f2ea57b5 100644 --- a/database/driver.go +++ b/database/driver.go @@ -33,7 +33,7 @@ var drivers = make(map[string]Driver) // All other functions are tested by tests in database/testing. // Saves you some time and makes sure all database drivers behave the same way. // 5. Call Register in init(). -// 6. Create a migrate/cli/build_.go file +// 6. Create a internal/cli/build_.go file // 7. Add driver name in 'DATABASE' variable in Makefile // // Guidelines: diff --git a/internal/cli/build_neo4j.go b/internal/cli/build_neo4j.go new file mode 100644 index 000000000..1f6b83a70 --- /dev/null +++ b/internal/cli/build_neo4j.go @@ -0,0 +1,7 @@ +// +build neo4j + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/database/neo4j" +) \ No newline at end of file From e67feb18a62805a5ade11c63876f8cd044640ed5 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Mon, 6 Jan 2020 17:19:33 +0530 Subject: [PATCH 03/24] add ready function for docker start --- database/neo4j/neo4j_test.go | 45 +++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/database/neo4j/neo4j_test.go b/database/neo4j/neo4j_test.go index 0c3482228..9baebfade 100644 --- a/database/neo4j/neo4j_test.go +++ b/database/neo4j/neo4j_test.go @@ -1,38 +1,67 @@ package neo4j import ( + "context" "fmt" + "log" "testing" "github.com/dhui/dktest" - "github.com/golang-migrate/migrate/v4/dktesting" + "github.com/neo4j/neo4j-go-driver/neo4j" dt "github.com/golang-migrate/migrate/v4/database/testing" - + "github.com/golang-migrate/migrate/v4/dktesting" + _ "github.com/golang-migrate/migrate/v4/source/file" ) var ( - opts = dktest.Options{PortRequired: true, ReadyFunc: nil} + opts = dktest.Options{PortRequired: true, ReadyFunc: isReady} specs = []dktesting.ContainerSpec{ {ImageName: "neo4j:3.5", Options: opts}, - {ImageName: "neo4j:3.5-enterprise", Options: opts}, + //{ImageName: "neo4j:3.5-enterprise", Options: opts}, } ) func neoConnectionString(host, port string) string { - return fmt.Sprintf("neo4j://neo4j:neo4j@%s:%s", host, port) + return fmt.Sprintf("bolt://neo4j:neo4j@%s:%s", host, port) +} + +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + ip, port, err := c.Port(7687) + if err != nil { + return false + } + + driver, err := neo4j.NewDriver(neoConnectionString(ip, port), neo4j.BasicAuth("neo4j", "neo4j", "")) + if err != nil { + return false + } + defer func() { + if err := driver.Close(); err != nil { + log.Println("close error:", err) + } + }() + session, err := driver.Session(neo4j.AccessModeRead) + if err != nil { + return false + } + _, err = session.Run("RETURN 1", nil) + if err != nil { + return false + } + + return true } func Test(t *testing.T) { dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { - ip, port, err := c.FirstPort() + ip, port, err := c.Port(7687) if err != nil { t.Fatal(err) } - addr := neoConnectionString(ip, port) n := &Neo4J{} - d, err := n.Open(addr) + d, err := n.Open(neoConnectionString(ip, port)) if err != nil { t.Fatal(err) } From d5340b95bcf8dd34c8732630ebbec9bb79b0285f Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Tue, 7 Jan 2020 09:52:51 +0530 Subject: [PATCH 04/24] lowercase j --- database/neo4j/neo4j.go | 24 ++++++++++++------------ database/neo4j/neo4j_test.go | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/database/neo4j/neo4j.go b/database/neo4j/neo4j.go index 75433b4ff..b150c6cb0 100644 --- a/database/neo4j/neo4j.go +++ b/database/neo4j/neo4j.go @@ -11,7 +11,7 @@ import ( ) func init() { - db := Neo4J{} + db := Neo4j{} database.Register("bolt", &db) database.Register("neo4j", &db) } @@ -28,7 +28,7 @@ type Config struct { MigrationsLabel string } -type Neo4J struct { +type Neo4j struct { driver neo4j.Driver // Open and WithInstance need to guarantee that config is never nil @@ -45,7 +45,7 @@ func WithInstance(config *Config) (database.Driver, error) { return nil, err } - driver := &Neo4J{ + driver := &Neo4j{ driver: neoDriver, config: config, } @@ -57,7 +57,7 @@ func WithInstance(config *Config) (database.Driver, error) { return driver, nil } -func (n *Neo4J) Open(url string) (database.Driver, error) { +func (n *Neo4j) Open(url string) (database.Driver, error) { uri, err := neturl.Parse(url) if err != nil { return nil, err @@ -72,19 +72,19 @@ func (n *Neo4J) Open(url string) (database.Driver, error) { }) } -func (n *Neo4J) Close() error { +func (n *Neo4j) Close() error { return n.driver.Close() } -func (n *Neo4J) Lock() error { +func (n *Neo4j) Lock() error { return nil } -func (n *Neo4J) Unlock() error { +func (n *Neo4j) Unlock() error { return nil } -func (n *Neo4J) Run(migration io.Reader) error { +func (n *Neo4j) Run(migration io.Reader) error { body, err := ioutil.ReadAll(migration) if err != nil { return err @@ -100,7 +100,7 @@ func (n *Neo4J) Run(migration io.Reader) error { return err } -func (n *Neo4J) SetVersion(version int, dirty bool) error { +func (n *Neo4j) SetVersion(version int, dirty bool) error { session, err := n.driver.Session(neo4j.AccessModeRead) if err != nil { return err @@ -112,7 +112,7 @@ func (n *Neo4J) SetVersion(version int, dirty bool) error { return err } -func (n *Neo4J) Version() (version int, dirty bool, err error) { +func (n *Neo4j) Version() (version int, dirty bool, err error) { session, err := n.driver.Session(neo4j.AccessModeRead) if err != nil { return -1, false, err @@ -138,7 +138,7 @@ func (n *Neo4J) Version() (version int, dirty bool, err error) { return } -func (n *Neo4J) Drop() error { +func (n *Neo4j) Drop() error { session, err := n.driver.Session(neo4j.AccessModeWrite); if err != nil { return err } @@ -148,7 +148,7 @@ func (n *Neo4J) Drop() error { return err } -func (n *Neo4J) ensureVersionConstraint() (err error) { +func (n *Neo4j) ensureVersionConstraint() (err error) { session, err := n.driver.Session(neo4j.AccessModeWrite); if err != nil { return err } diff --git a/database/neo4j/neo4j_test.go b/database/neo4j/neo4j_test.go index 9baebfade..6e6fc8509 100644 --- a/database/neo4j/neo4j_test.go +++ b/database/neo4j/neo4j_test.go @@ -60,7 +60,7 @@ func Test(t *testing.T) { t.Fatal(err) } - n := &Neo4J{} + n := &Neo4j{} d, err := n.Open(neoConnectionString(ip, port)) if err != nil { t.Fatal(err) From 79946b102cd8a41c5c19b0ff827d91d8bb679378 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Tue, 7 Jan 2020 09:58:47 +0530 Subject: [PATCH 05/24] explicit return --- database/neo4j/neo4j.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/neo4j/neo4j.go b/database/neo4j/neo4j.go index b150c6cb0..126952607 100644 --- a/database/neo4j/neo4j.go +++ b/database/neo4j/neo4j.go @@ -135,7 +135,7 @@ func (n *Neo4j) Version() (version int, dirty bool, err error) { version = -1 } - return + return version, dirty, nil } func (n *Neo4j) Drop() error { From a4e737011176178ff36a53feb70eeb3857440a84 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Wed, 8 Jan 2020 00:26:58 +0530 Subject: [PATCH 06/24] test migration --- .gitignore | 1 + ...21040_create_movies_constraint.down.cypher | 1 + ...8421040_create_movies_constraint.up.cypher | 1 + .../1578421725_create_movies.down.cypher | 2 + .../1578421725_create_movies.up.cypher | 2 + database/neo4j/neo4j.go | 95 +++++++++++++------ database/neo4j/neo4j_test.go | 41 +++++++- 7 files changed, 111 insertions(+), 32 deletions(-) create mode 100644 database/neo4j/examples/migrations/1578421040_create_movies_constraint.down.cypher create mode 100644 database/neo4j/examples/migrations/1578421040_create_movies_constraint.up.cypher create mode 100644 database/neo4j/examples/migrations/1578421725_create_movies.down.cypher create mode 100644 database/neo4j/examples/migrations/1578421725_create_movies.up.cypher diff --git a/.gitignore b/.gitignore index acb4088f2..1fd5860f8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ cli/migrate .godoc.pid vendor/ .vscode/ +.idea/ \ No newline at end of file diff --git a/database/neo4j/examples/migrations/1578421040_create_movies_constraint.down.cypher b/database/neo4j/examples/migrations/1578421040_create_movies_constraint.down.cypher new file mode 100644 index 000000000..8e65d5aef --- /dev/null +++ b/database/neo4j/examples/migrations/1578421040_create_movies_constraint.down.cypher @@ -0,0 +1 @@ +DROP CONSTRAINT ON (m:Movie) ASSERT m.Name IS UNIQUE \ No newline at end of file diff --git a/database/neo4j/examples/migrations/1578421040_create_movies_constraint.up.cypher b/database/neo4j/examples/migrations/1578421040_create_movies_constraint.up.cypher new file mode 100644 index 000000000..4ca81fed1 --- /dev/null +++ b/database/neo4j/examples/migrations/1578421040_create_movies_constraint.up.cypher @@ -0,0 +1 @@ +CREATE CONSTRAINT ON (m:Movie) ASSERT m.Name IS UNIQUE \ No newline at end of file diff --git a/database/neo4j/examples/migrations/1578421725_create_movies.down.cypher b/database/neo4j/examples/migrations/1578421725_create_movies.down.cypher new file mode 100644 index 000000000..110dd68de --- /dev/null +++ b/database/neo4j/examples/migrations/1578421725_create_movies.down.cypher @@ -0,0 +1,2 @@ +MATCH (m:Movie) +DELETE m \ No newline at end of file diff --git a/database/neo4j/examples/migrations/1578421725_create_movies.up.cypher b/database/neo4j/examples/migrations/1578421725_create_movies.up.cypher new file mode 100644 index 000000000..5283d85ac --- /dev/null +++ b/database/neo4j/examples/migrations/1578421725_create_movies.up.cypher @@ -0,0 +1,2 @@ +CREATE (:Movie {name: "Footloose"}) +CREATE (:Movie {name: "Ghost"}) \ No newline at end of file diff --git a/database/neo4j/neo4j.go b/database/neo4j/neo4j.go index 126952607..10282f538 100644 --- a/database/neo4j/neo4j.go +++ b/database/neo4j/neo4j.go @@ -5,6 +5,7 @@ import ( "io" "io/ioutil" neturl "net/url" + "sync/atomic" "github.com/golang-migrate/migrate/v4/database" "github.com/neo4j/neo4j-go-driver/neo4j" @@ -19,17 +20,18 @@ func init() { var DefaultMigrationsLabel = "SchemaMigration" var ( - ErrNilConfig = fmt.Errorf("no config") + ErrNilConfig = fmt.Errorf("no config") ) type Config struct { - AuthToken neo4j.AuthToken - URL string + AuthToken neo4j.AuthToken + URL string MigrationsLabel string } type Neo4j struct { driver neo4j.Driver + lock uint32 // Open and WithInstance need to guarantee that config is never nil config *Config @@ -76,11 +78,17 @@ func (n *Neo4j) Close() error { return n.driver.Close() } +// local locking in order to pass tests, Neo doesn't support database locking func (n *Neo4j) Lock() error { + if !atomic.CompareAndSwapUint32(&n.lock, 0, 1) { + return database.ErrLocked + } + return nil } func (n *Neo4j) Unlock() error { + atomic.StoreUint32(&n.lock, 0) return nil } @@ -96,8 +104,11 @@ func (n *Neo4j) Run(migration io.Reader) error { } defer session.Close() - _, err = session.Run(string(body[:]), nil) - return err + result, err := session.Run(string(body[:]), nil) + if err != nil { + return err + } + return result.Err() } func (n *Neo4j) SetVersion(version int, dirty bool) error { @@ -107,9 +118,18 @@ func (n *Neo4j) SetVersion(version int, dirty bool) error { } defer session.Close() - _, err = session.Run("MERGE (sm:$migration {version: $version, dirty: $dirty})", - map[string]interface{}{"migration": n.config.MigrationsLabel, "version": version, "dirty": dirty}) - return err + query := fmt.Sprintf("MERGE (sm:%s {version: $version, dirty: $dirty})", + n.config.MigrationsLabel) + result, err := session.Run(query, map[string]interface{}{"version": version, "dirty": dirty}) + if err != nil { + return err + } + return result.Err() +} + +type MigrationRecord struct { + Version int + Dirty bool } func (n *Neo4j) Version() (version int, dirty bool, err error) { @@ -119,27 +139,45 @@ func (n *Neo4j) Version() (version int, dirty bool, err error) { } defer session.Close() - result, err := session.Run("MATCH (sm:$migration) RETURN sm.version, sm.dirty ORDER BY sm.version DESC LIMIT 1", - map[string]interface{}{"migration": n.config.MigrationsLabel}) + query := fmt.Sprintf("MATCH (sm:%s) RETURN sm.version AS version, sm.dirty AS dirty ORDER BY sm.version DESC LIMIT 1", + n.config.MigrationsLabel) + result, err := session.ReadTransaction(func(transaction neo4j.Transaction) (interface{}, error) { + result, err := transaction.Run(query, nil) + if err != nil { + return nil, err + } + if result.Next() { + record := result.Record() + mr := MigrationRecord{} + versionResult, ok := record.Get("version") + if !ok { + mr.Version = -1 + } else { + mr.Version = int(versionResult.(int64)) + } + + dirtyResult, ok := record.Get("dirty") + if ok { + mr.Dirty = dirtyResult.(bool) + } + + return mr, nil + } + return nil, result.Err() + }) if err != nil { return -1, false, err } - if result.Next() { - versionResult, ok := result.Record().Get("version") - if !ok { - version = -1 - } else { - version = versionResult.(int) - } - } else { - version = -1 + if result == nil { + return -1, false, nil } - - return version, dirty, nil + mr := result.(MigrationRecord) + return mr.Version, mr.Dirty, nil } func (n *Neo4j) Drop() error { - session, err := n.driver.Session(neo4j.AccessModeWrite); if err != nil { + session, err := n.driver.Session(neo4j.AccessModeWrite); + if err != nil { return err } defer session.Close() @@ -149,13 +187,16 @@ func (n *Neo4j) Drop() error { } func (n *Neo4j) ensureVersionConstraint() (err error) { - session, err := n.driver.Session(neo4j.AccessModeWrite); if err != nil { + session, err := n.driver.Session(neo4j.AccessModeWrite); + if err != nil { return err } defer session.Close() - _, err = session.Run("CREATE CONSTRAINT ON (a:$migration) ASSERT a.version IS UNIQUE", - map[string]interface{}{"migration": n.config.MigrationsLabel}) - return err + query := fmt.Sprintf("CREATE CONSTRAINT ON (a:%s) ASSERT a.version IS UNIQUE", n.config.MigrationsLabel) + result, err := session.Run(query, nil) + if err != nil { + return err + } + return result.Err() } - diff --git a/database/neo4j/neo4j_test.go b/database/neo4j/neo4j_test.go index 6e6fc8509..296ae7064 100644 --- a/database/neo4j/neo4j_test.go +++ b/database/neo4j/neo4j_test.go @@ -9,21 +9,25 @@ import ( "github.com/dhui/dktest" "github.com/neo4j/neo4j-go-driver/neo4j" + "github.com/golang-migrate/migrate/v4" dt "github.com/golang-migrate/migrate/v4/database/testing" "github.com/golang-migrate/migrate/v4/dktesting" _ "github.com/golang-migrate/migrate/v4/source/file" ) var ( - opts = dktest.Options{PortRequired: true, ReadyFunc: isReady} + opts = dktest.Options{PortRequired: true, ReadyFunc: isReady, + Env: map[string]string{"NEO4J_AUTH": "neo4j/migratetest", "NEO4J_ACCEPT_LICENSE_AGREEMENT": "yes"}} specs = []dktesting.ContainerSpec{ {ImageName: "neo4j:3.5", Options: opts}, - //{ImageName: "neo4j:3.5-enterprise", Options: opts}, + {ImageName: "neo4j:3.5-enterprise", Options: opts}, + {ImageName: "neo4j:3.4", Options: opts}, + {ImageName: "neo4j:3.4-enterprise", Options: opts}, } ) func neoConnectionString(host, port string) string { - return fmt.Sprintf("bolt://neo4j:neo4j@%s:%s", host, port) + return fmt.Sprintf("bolt://neo4j:migratetest@%s:%s", host, port) } func isReady(ctx context.Context, c dktest.ContainerInfo) bool { @@ -32,7 +36,7 @@ func isReady(ctx context.Context, c dktest.ContainerInfo) bool { return false } - driver, err := neo4j.NewDriver(neoConnectionString(ip, port), neo4j.BasicAuth("neo4j", "neo4j", "")) + driver, err := neo4j.NewDriver(neoConnectionString(ip, port), neo4j.BasicAuth("neo4j", "migratetest", "")) if err != nil { return false } @@ -45,9 +49,11 @@ func isReady(ctx context.Context, c dktest.ContainerInfo) bool { if err != nil { return false } - _, err = session.Run("RETURN 1", nil) + result, err := session.Run("RETURN 1", nil) if err != nil { return false + } else if result.Err() != nil { + return false } return true @@ -72,4 +78,29 @@ func Test(t *testing.T) { }() dt.Test(t, d, []byte("MATCH a RETURN a")) }) +} + +func TestMigrate(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(7687) + if err != nil { + t.Fatal(err) + } + + n := &Neo4j{} + d, err := n.Open(neoConnectionString(ip, port)) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "neo4j", d) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m, []byte("MATCH a RETURN a")) + }) } \ No newline at end of file From f1f063f2ab2ffec86e5e29aefb5508f5e78da096 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Wed, 8 Jan 2020 00:28:15 +0530 Subject: [PATCH 07/24] undo dockerfile change --- Dockerfile | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 716a96495..cfab35537 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,14 +3,6 @@ ARG VERSION RUN apk add --no-cache git gcc musl-dev -# dependencies -RUN apk add --update --no-cache ca-certificates cmake make g++ openssl-dev git curl pkgconfig - -# build seabolt -RUN git clone -b 1.7 https://github.com/neo4j-drivers/seabolt.git /seabolt -WORKDIR /seabolt/build -RUN cmake -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_LIBDIR=lib .. && cmake --build . --target install - WORKDIR /go/src/github.com/golang-migrate/migrate COPY . ./ From 2866d25dd9f31df3afe7987c3f0988cb88052242 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Thu, 9 Jan 2020 02:11:56 +0530 Subject: [PATCH 08/24] neo4j in dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cfab35537..6a3cb863f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ WORKDIR /go/src/github.com/golang-migrate/migrate COPY . ./ ENV GO111MODULE=on -ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver firebird" +ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver firebird neo4j" ENV SOURCES="file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab" RUN go build -a -o build/migrate.linux-386 -ldflags="-s -w -X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cmd/migrate From d1cc5cded1c9d3a92efd4f69e8d5550d01a2b2eb Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Thu, 9 Jan 2020 02:15:24 +0530 Subject: [PATCH 09/24] remove auth from url once transferred to auth token --- database/neo4j/neo4j.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/neo4j/neo4j.go b/database/neo4j/neo4j.go index 10282f538..c6fd365cb 100644 --- a/database/neo4j/neo4j.go +++ b/database/neo4j/neo4j.go @@ -13,7 +13,6 @@ import ( func init() { db := Neo4j{} - database.Register("bolt", &db) database.Register("neo4j", &db) } @@ -66,9 +65,10 @@ func (n *Neo4j) Open(url string) (database.Driver, error) { } password, _ := uri.User.Password() authToken := neo4j.BasicAuth(uri.User.Username(), password, "") + uri.User = nil return WithInstance(&Config{ - URL: url, + URL: uri.String(), AuthToken: authToken, MigrationsLabel: DefaultMigrationsLabel, }) From 441db85cf0faa083e2f5927c130936df3de941e1 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Thu, 9 Jan 2020 02:23:03 +0530 Subject: [PATCH 10/24] assorted review notes --- database/neo4j/neo4j.go | 6 ++++-- database/neo4j/neo4j_test.go | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/database/neo4j/neo4j.go b/database/neo4j/neo4j.go index c6fd365cb..36a9f61f5 100644 --- a/database/neo4j/neo4j.go +++ b/database/neo4j/neo4j.go @@ -1,3 +1,5 @@ +// +build cgo + package neo4j import ( @@ -24,7 +26,7 @@ var ( type Config struct { AuthToken neo4j.AuthToken - URL string + URL string // if using WithInstance, don't provide auth in the URL, it will be ignored MigrationsLabel string } @@ -182,7 +184,7 @@ func (n *Neo4j) Drop() error { } defer session.Close() - _, err = session.Run("MATCH (n) DETACH DELETE n", map[string]interface{}{}) + _, err = session.Run("MATCH (n) DETACH DELETE n", nil) return err } diff --git a/database/neo4j/neo4j_test.go b/database/neo4j/neo4j_test.go index 296ae7064..ba1e4ec7d 100644 --- a/database/neo4j/neo4j_test.go +++ b/database/neo4j/neo4j_test.go @@ -1,3 +1,5 @@ +// +build cgo + package neo4j import ( @@ -101,6 +103,6 @@ func TestMigrate(t *testing.T) { if err != nil { t.Fatal(err) } - dt.TestMigrate(t, m, []byte("MATCH a RETURN a")) + dt.TestMigrate(t, m) }) } \ No newline at end of file From 46a6265db98b9dede4a2ba5c3acc6cb486cb51fb Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Thu, 9 Jan 2020 02:45:44 +0530 Subject: [PATCH 11/24] initial docs attempt --- database/neo4j/README.md | 11 +++++ database/neo4j/TUTORIAL.md | 97 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 database/neo4j/TUTORIAL.md diff --git a/database/neo4j/README.md b/database/neo4j/README.md index e69de29bb..e01a11970 100644 --- a/database/neo4j/README.md +++ b/database/neo4j/README.md @@ -0,0 +1,11 @@ +# neo4j + +`bolt://user:password@host:port/` + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| `user` | Contained within `AuthConfig` | The user to sign in as | +| `password` | Contained within `AuthConfig` | The user's password | +| `host` | | The host to connect to. Values that start with / are for unix domain sockets. (default is localhost) | +| `port` | | The port to bind to. (default is 7687) | +| | `MigrationsLabel` | Name of the migrations node label | diff --git a/database/neo4j/TUTORIAL.md b/database/neo4j/TUTORIAL.md new file mode 100644 index 000000000..216a12106 --- /dev/null +++ b/database/neo4j/TUTORIAL.md @@ -0,0 +1,97 @@ +## Create migrations +Let's create nodes called `Users`: +``` +migrate create -ext cypher -dir db/migrations -seq create_user_nodes +``` +If there were no errors, we should have two files available under `db/migrations` folder: +- 000001_create_user_nodes.down.cypher +- 000001_create_user_nodes.up.cypher + +Note the `cypher` extension that we provided. + +In the `.up.cypher` file let's create the table: +``` +CREATE (u1:User {name: "Peter"}) +CREATE (u2:User {name: "Paul"}) +CREATE (u3:User {name: "Mary"}) +``` +And in the `.down.sql` let's delete it: +``` +MATCH (u:User) WHERE u.name IN ["Peter", "Paul", "Mary"] DELETE u +``` +Ideally your migrations should be idempotent. You can read more about idempotency in [getting started](GETTING_STARTED.md#create-migrations) + +## Run migrations +``` +migrate -database ${NEO4J_URL} -path db/migrations up +``` +Let's check if the table was created properly by running `bin/cypher-shell -u neo4j -p password`, then `neo4j> MATCH (u:User)` +The output you are supposed to see: +``` ++-----------------------------------------------------------------+ +| u | ++-----------------------------------------------------------------+ +| (:User {name: "Peter") | +| (:User {name: "Paul") | +| (:User {name: "Mary") | ++-----------------------------------------------------------------+ +``` +Great! Now let's check if running reverse migration also works: +``` +migrate -database ${NEO4J_URL} -path db/migrations down +``` +Make sure to check if your database changed as expected in this case as well. + +## Database transactions + +To show database transactions usage, let's create another set of migrations by running: +``` +migrate create -ext cypher -dir db/migrations -seq add_mood_to_users +``` +Again, it should create for us two migrations files: +- 000002_add_mood_to_users.down.cypher +- 000002_add_mood_to_users.up.cypher + +In Neo4j, when we want our queries to be done in a transaction, we need to wrap it with `:BEGIN` and `:COMMIT` commands. +Migration up: +``` +:BEGIN + +MATCH (u:User) +SET u.mood = "Cheery" + +:COMMIT +``` +Migration down: +``` +:BEGIN + +MATCH (u:User) +SET u.mood = null + +:COMMIT +``` + +## Optional: Run migrations within your Go app +Here is a very simple app running migrations for the above configuration: +``` +import ( + "log" + + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/neo4j" + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +func main() { + m, err := migrate.New( + "file://db/migrations", + "bolt://neo4j:password@localhost:7687/") + if err != nil { + log.Fatal(err) + } + if err := m.Up(); err != nil { + log.Fatal(err) + } +} +``` \ No newline at end of file From 2050d0f8f7e66b5646432f78ced9ad82b12fadb7 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Thu, 9 Jan 2020 11:27:05 +0530 Subject: [PATCH 12/24] unlock behavior more in line with lock behavior --- database/driver.go | 1 + database/neo4j/neo4j.go | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/database/driver.go b/database/driver.go index 7f2ea57b5..64789e9d8 100644 --- a/database/driver.go +++ b/database/driver.go @@ -14,6 +14,7 @@ import ( var ( ErrLocked = fmt.Errorf("can't acquire lock") + ErrNotLocked = fmt.Errorf("can't unlock, as not currently locked") ) const NilVersion int = -1 diff --git a/database/neo4j/neo4j.go b/database/neo4j/neo4j.go index 36a9f61f5..f9233297b 100644 --- a/database/neo4j/neo4j.go +++ b/database/neo4j/neo4j.go @@ -90,7 +90,9 @@ func (n *Neo4j) Lock() error { } func (n *Neo4j) Unlock() error { - atomic.StoreUint32(&n.lock, 0) + if !atomic.CompareAndSwapUint32(&n.lock, 1, 0) { + return database.ErrNotLocked + } return nil } From ea75d4b689a06c849b781ad07e6ef2d79e929b63 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Thu, 9 Jan 2020 11:50:41 +0530 Subject: [PATCH 13/24] consistent naming --- database/neo4j/README.md | 2 +- database/neo4j/TUTORIAL.md | 2 +- database/neo4j/neo4j.go | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/database/neo4j/README.md b/database/neo4j/README.md index e01a11970..4b93cb040 100644 --- a/database/neo4j/README.md +++ b/database/neo4j/README.md @@ -1,6 +1,6 @@ # neo4j -`bolt://user:password@host:port/` +`neo4j://user:password@host:port/` | URL Query | WithInstance Config | Description | |------------|---------------------|-------------| diff --git a/database/neo4j/TUTORIAL.md b/database/neo4j/TUTORIAL.md index 216a12106..14bb1986c 100644 --- a/database/neo4j/TUTORIAL.md +++ b/database/neo4j/TUTORIAL.md @@ -86,7 +86,7 @@ import ( func main() { m, err := migrate.New( "file://db/migrations", - "bolt://neo4j:password@localhost:7687/") + "neo4j://neo4j:password@localhost:7687/") if err != nil { log.Fatal(err) } diff --git a/database/neo4j/neo4j.go b/database/neo4j/neo4j.go index f9233297b..be9947e3d 100644 --- a/database/neo4j/neo4j.go +++ b/database/neo4j/neo4j.go @@ -68,6 +68,7 @@ func (n *Neo4j) Open(url string) (database.Driver, error) { password, _ := uri.User.Password() authToken := neo4j.BasicAuth(uri.User.Username(), password, "") uri.User = nil + uri.Scheme = "bolt" return WithInstance(&Config{ URL: uri.String(), From e994a5ddd5746095e19387bc95293323e883846a Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Thu, 9 Jan 2020 12:12:16 +0530 Subject: [PATCH 14/24] linting --- Dockerfile | 8 +++++++ database/driver.go | 2 +- database/neo4j/neo4j.go | 44 ++++++++++++++++++++++++++++-------- database/neo4j/neo4j_test.go | 4 +--- go.mod | 1 + go.sum | 4 ++++ 6 files changed, 49 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 975cebcfa..96978c06b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,14 @@ ARG VERSION RUN apk add --no-cache git gcc musl-dev +# dependencies for neo4j +RUN apk add --update --no-cache ca-certificates cmake make g++ openssl-dev git curl pkgconfig + +# build seabolt for neo4j driver +RUN git clone -b 1.7 https://github.com/neo4j-drivers/seabolt.git /seabolt +WORKDIR /seabolt/build +RUN cmake -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_LIBDIR=lib .. && cmake --build . --target install + WORKDIR /go/src/github.com/golang-migrate/migrate COPY . ./ diff --git a/database/driver.go b/database/driver.go index 64789e9d8..fa80f455c 100644 --- a/database/driver.go +++ b/database/driver.go @@ -13,7 +13,7 @@ import ( ) var ( - ErrLocked = fmt.Errorf("can't acquire lock") + ErrLocked = fmt.Errorf("can't acquire lock") ErrNotLocked = fmt.Errorf("can't unlock, as not currently locked") ) diff --git a/database/neo4j/neo4j.go b/database/neo4j/neo4j.go index be9947e3d..85ea8c115 100644 --- a/database/neo4j/neo4j.go +++ b/database/neo4j/neo4j.go @@ -1,9 +1,8 @@ -// +build cgo - package neo4j import ( "fmt" + "github.com/prometheus/common/log" "io" "io/ioutil" neturl "net/url" @@ -107,7 +106,12 @@ func (n *Neo4j) Run(migration io.Reader) error { if err != nil { return err } - defer session.Close() + defer func() { + err := session.Close() + if err != nil { + log.Error(err) + } + }() result, err := session.Run(string(body[:]), nil) if err != nil { @@ -121,7 +125,12 @@ func (n *Neo4j) SetVersion(version int, dirty bool) error { if err != nil { return err } - defer session.Close() + defer func() { + err := session.Close() + if err != nil { + log.Error(err) + } + }() query := fmt.Sprintf("MERGE (sm:%s {version: $version, dirty: $dirty})", n.config.MigrationsLabel) @@ -134,7 +143,7 @@ func (n *Neo4j) SetVersion(version int, dirty bool) error { type MigrationRecord struct { Version int - Dirty bool + Dirty bool } func (n *Neo4j) Version() (version int, dirty bool, err error) { @@ -142,7 +151,12 @@ func (n *Neo4j) Version() (version int, dirty bool, err error) { if err != nil { return -1, false, err } - defer session.Close() + defer func() { + err := session.Close() + if err != nil { + log.Error(err) + } + }() query := fmt.Sprintf("MATCH (sm:%s) RETURN sm.version AS version, sm.dirty AS dirty ORDER BY sm.version DESC LIMIT 1", n.config.MigrationsLabel) @@ -181,22 +195,32 @@ func (n *Neo4j) Version() (version int, dirty bool, err error) { } func (n *Neo4j) Drop() error { - session, err := n.driver.Session(neo4j.AccessModeWrite); + session, err := n.driver.Session(neo4j.AccessModeWrite) if err != nil { return err } - defer session.Close() + defer func() { + err := session.Close() + if err != nil { + log.Error(err) + } + }() _, err = session.Run("MATCH (n) DETACH DELETE n", nil) return err } func (n *Neo4j) ensureVersionConstraint() (err error) { - session, err := n.driver.Session(neo4j.AccessModeWrite); + session, err := n.driver.Session(neo4j.AccessModeWrite) if err != nil { return err } - defer session.Close() + defer func() { + err := session.Close() + if err != nil { + log.Error(err) + } + }() query := fmt.Sprintf("CREATE CONSTRAINT ON (a:%s) ASSERT a.version IS UNIQUE", n.config.MigrationsLabel) result, err := session.Run(query, nil) diff --git a/database/neo4j/neo4j_test.go b/database/neo4j/neo4j_test.go index ba1e4ec7d..5e1ebe554 100644 --- a/database/neo4j/neo4j_test.go +++ b/database/neo4j/neo4j_test.go @@ -1,5 +1,3 @@ -// +build cgo - package neo4j import ( @@ -105,4 +103,4 @@ func TestMigrate(t *testing.T) { } dt.TestMigrate(t, m) }) -} \ No newline at end of file +} diff --git a/go.mod b/go.mod index 037b68310..d9f199857 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/neo4j-drivers/gobolt v1.7.4 // indirect github.com/neo4j/neo4j-go-driver v1.7.4 github.com/pkg/errors v0.8.1 // indirect + github.com/prometheus/common v0.2.0 github.com/satori/go.uuid v1.2.0 // indirect github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 // indirect github.com/sirupsen/logrus v1.4.1 // indirect diff --git a/go.sum b/go.sum index 2b7c03a15..6e70bfefd 100644 --- a/go.sum +++ b/go.sum @@ -13,7 +13,9 @@ github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEV github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/aws/aws-sdk-go v1.17.7 h1:/4+rDPe0W95KBmNGYCG+NUvdL8ssPYBMxL+aSCg6nIA= @@ -171,6 +173,7 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -293,6 +296,7 @@ google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= From 4d636f7447542318eb79da419e47a3d13a266479 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Thu, 9 Jan 2020 12:26:28 +0530 Subject: [PATCH 15/24] seabolt in travis ci --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index c2367b3e1..c74400b1f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,11 @@ before_install: # Install golangci-lint - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.22.2 - echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}" + # Install seabolt for Neo4j + - sudo apt-get install -y libssl1.0.0 + - wget https://github.com/neo4j-drivers/seabolt/releases/download/v1.7.4/seabolt-1.7.4-Linux-ubuntu-$(lsb_release -rs).deb + - dpkg -i seabolt-1.7.3-Linux-ubuntu-$(lsb_release -rs).deb + - rm seabolt-1.7.3-Linux-ubuntu-$(lsb_release -rs).deb install: - go get github.com/mattn/goveralls From a0f6bfe8854f98f5c7f861f5e53304aef4394241 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Thu, 9 Jan 2020 12:38:19 +0530 Subject: [PATCH 16/24] stdlib logger --- database/neo4j/neo4j.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/database/neo4j/neo4j.go b/database/neo4j/neo4j.go index 85ea8c115..432e96af3 100644 --- a/database/neo4j/neo4j.go +++ b/database/neo4j/neo4j.go @@ -2,9 +2,9 @@ package neo4j import ( "fmt" - "github.com/prometheus/common/log" "io" "io/ioutil" + "log" neturl "net/url" "sync/atomic" @@ -109,7 +109,7 @@ func (n *Neo4j) Run(migration io.Reader) error { defer func() { err := session.Close() if err != nil { - log.Error(err) + log.Printf("error: %s", err) } }() @@ -128,7 +128,7 @@ func (n *Neo4j) SetVersion(version int, dirty bool) error { defer func() { err := session.Close() if err != nil { - log.Error(err) + log.Printf("error: %s", err) } }() @@ -154,7 +154,7 @@ func (n *Neo4j) Version() (version int, dirty bool, err error) { defer func() { err := session.Close() if err != nil { - log.Error(err) + log.Printf("error: %s", err) } }() @@ -202,7 +202,7 @@ func (n *Neo4j) Drop() error { defer func() { err := session.Close() if err != nil { - log.Error(err) + log.Printf("error: %s", err) } }() @@ -218,7 +218,7 @@ func (n *Neo4j) ensureVersionConstraint() (err error) { defer func() { err := session.Close() if err != nil { - log.Error(err) + log.Printf("error: %s", err) } }() From 712409211d59e01cd5e3bc70b3c40a91397e5d6a Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Thu, 9 Jan 2020 12:40:28 +0530 Subject: [PATCH 17/24] install seabolt as sudo --- .travis.yml | 2 +- database/neo4j/neo4j.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c74400b1f..beea970d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ before_install: # Install seabolt for Neo4j - sudo apt-get install -y libssl1.0.0 - wget https://github.com/neo4j-drivers/seabolt/releases/download/v1.7.4/seabolt-1.7.4-Linux-ubuntu-$(lsb_release -rs).deb - - dpkg -i seabolt-1.7.3-Linux-ubuntu-$(lsb_release -rs).deb + - sudo dpkg -i seabolt-1.7.3-Linux-ubuntu-$(lsb_release -rs).deb - rm seabolt-1.7.3-Linux-ubuntu-$(lsb_release -rs).deb install: diff --git a/database/neo4j/neo4j.go b/database/neo4j/neo4j.go index 432e96af3..1d905c6da 100644 --- a/database/neo4j/neo4j.go +++ b/database/neo4j/neo4j.go @@ -1,6 +1,7 @@ package neo4j import ( + "C" // import C so that we can't compile with CGO_ENABLED=0 "fmt" "io" "io/ioutil" From d969cad018514ccc59dad4f2126e90d2ce58168c Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Thu, 9 Jan 2020 12:43:09 +0530 Subject: [PATCH 18/24] enabled cgo --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 31dbac2bf..705d5876b 100644 --- a/Makefile +++ b/Makefile @@ -8,11 +8,11 @@ COVERAGE_DIR ?= .coverage build-cli: clean -mkdir ./cli/build - cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o ../../cli/build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . - cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -a -o ../../cli/build/migrate.linux-armv7 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . - cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -a -o ../../cli/build/migrate.linux-arm64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . - cd ./cmd/migrate && CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -a -o ../../cli/build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . - cd ./cmd/migrate && CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -a -o ../../cli/build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -a -o ../../cli/build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=1 GOOS=linux GOARCH=arm GOARM=7 go build -a -o ../../cli/build/migrate.linux-armv7 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=1 GOOS=linux GOARCH=arm64 go build -a -o ../../cli/build/migrate.linux-arm64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -a -o ../../cli/build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=1 GOOS=windows GOARCH=amd64 go build -a -o ../../cli/build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . cd ./cli/build && find . -name 'migrate*' | xargs -I{} tar czf {}.tar.gz {} cd ./cli/build && shasum -a 256 * > sha256sum.txt cat ./cli/build/sha256sum.txt From c468895e6aa03fcf41cfb9741920759773b81184 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Thu, 9 Jan 2020 12:46:52 +0530 Subject: [PATCH 19/24] correct version # --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index beea970d8..ec44d4a02 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,8 +39,8 @@ before_install: # Install seabolt for Neo4j - sudo apt-get install -y libssl1.0.0 - wget https://github.com/neo4j-drivers/seabolt/releases/download/v1.7.4/seabolt-1.7.4-Linux-ubuntu-$(lsb_release -rs).deb - - sudo dpkg -i seabolt-1.7.3-Linux-ubuntu-$(lsb_release -rs).deb - - rm seabolt-1.7.3-Linux-ubuntu-$(lsb_release -rs).deb + - sudo dpkg -i seabolt-1.7.4-Linux-ubuntu-$(lsb_release -rs).deb + - rm seabolt-1.7.4-Linux-ubuntu-$(lsb_release -rs).deb install: - go get github.com/mattn/goveralls From af4be16bf24cce4119818ad0e2f6147bc984f63b Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Thu, 9 Jan 2020 13:19:31 +0530 Subject: [PATCH 20/24] remove neo4j todo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c9076cb3e..4f3da48a6 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Database drivers run migrations. [Add a new database?](database/driver.go) * [Cassandra](database/cassandra) * [SQLite](database/sqlite3) ([todo #165](https://github.com/mattes/migrate/issues/165)) * [MySQL/ MariaDB](database/mysql) -* [Neo4j](database/neo4j) ([todo #167](https://github.com/mattes/migrate/issues/167)) +* [Neo4j](database/neo4j) * [MongoDB](database/mongodb) * [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170)) * [Shell](database/shell) ([todo #171](https://github.com/mattes/migrate/issues/171)) From ce35a581f0e1325f24ba4cea180c0a9b71e90a4c Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Fri, 10 Jan 2020 11:20:32 +0530 Subject: [PATCH 21/24] named return values for errors --- database/neo4j/neo4j.go | 51 ++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/database/neo4j/neo4j.go b/database/neo4j/neo4j.go index 1d905c6da..9c8d117ae 100644 --- a/database/neo4j/neo4j.go +++ b/database/neo4j/neo4j.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "io/ioutil" - "log" neturl "net/url" "sync/atomic" @@ -97,7 +96,7 @@ func (n *Neo4j) Unlock() error { return nil } -func (n *Neo4j) Run(migration io.Reader) error { +func (n *Neo4j) Run(migration io.Reader) (err error) { body, err := ioutil.ReadAll(migration) if err != nil { return err @@ -108,9 +107,8 @@ func (n *Neo4j) Run(migration io.Reader) error { return err } defer func() { - err := session.Close() - if err != nil { - log.Printf("error: %s", err) + if cerr := session.Close(); cerr != nil { + err = cerr } }() @@ -118,18 +116,20 @@ func (n *Neo4j) Run(migration io.Reader) error { if err != nil { return err } - return result.Err() + if result.Err() != nil { + err = result.Err() + } + return err } -func (n *Neo4j) SetVersion(version int, dirty bool) error { +func (n *Neo4j) SetVersion(version int, dirty bool) (err error) { session, err := n.driver.Session(neo4j.AccessModeRead) if err != nil { return err } defer func() { - err := session.Close() - if err != nil { - log.Printf("error: %s", err) + if cerr := session.Close(); cerr != nil { + err = cerr } }() @@ -139,7 +139,10 @@ func (n *Neo4j) SetVersion(version int, dirty bool) error { if err != nil { return err } - return result.Err() + if result.Err() != nil { + err = result.Err() + } + return err } type MigrationRecord struct { @@ -153,9 +156,8 @@ func (n *Neo4j) Version() (version int, dirty bool, err error) { return -1, false, err } defer func() { - err := session.Close() - if err != nil { - log.Printf("error: %s", err) + if cerr := session.Close(); cerr != nil { + err = cerr } }() @@ -189,21 +191,20 @@ func (n *Neo4j) Version() (version int, dirty bool, err error) { return -1, false, err } if result == nil { - return -1, false, nil + return -1, false, err } mr := result.(MigrationRecord) - return mr.Version, mr.Dirty, nil + return mr.Version, mr.Dirty, err } -func (n *Neo4j) Drop() error { +func (n *Neo4j) Drop() (err error) { session, err := n.driver.Session(neo4j.AccessModeWrite) if err != nil { return err } defer func() { - err := session.Close() - if err != nil { - log.Printf("error: %s", err) + if cerr := session.Close(); cerr != nil { + err = cerr } }() @@ -217,9 +218,8 @@ func (n *Neo4j) ensureVersionConstraint() (err error) { return err } defer func() { - err := session.Close() - if err != nil { - log.Printf("error: %s", err) + if cerr := session.Close(); cerr != nil { + err = cerr } }() @@ -228,5 +228,8 @@ func (n *Neo4j) ensureVersionConstraint() (err error) { if err != nil { return err } - return result.Err() + if result.Err() != nil { + err = result.Err() + } + return err } From 7931609a11bd6fe46ffdef51993e39415e2f9d69 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Sat, 11 Jan 2020 16:22:19 +0300 Subject: [PATCH 22/24] review notes --- Makefile | 13 +++++++------ database/neo4j/neo4j.go | 29 +++++++++++++++-------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 705d5876b..28a184b0b 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ SOURCE ?= file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab -DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver firebird neo4j +DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver firebird +DATABASE_TEST ?= $(DATABASE) sqlite neo4j VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") @@ -8,11 +9,11 @@ COVERAGE_DIR ?= .coverage build-cli: clean -mkdir ./cli/build - cd ./cmd/migrate && CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -a -o ../../cli/build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . - cd ./cmd/migrate && CGO_ENABLED=1 GOOS=linux GOARCH=arm GOARM=7 go build -a -o ../../cli/build/migrate.linux-armv7 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . - cd ./cmd/migrate && CGO_ENABLED=1 GOOS=linux GOARCH=arm64 go build -a -o ../../cli/build/migrate.linux-arm64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . +# cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o ../../cli/build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . +# cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -a -o ../../cli/build/migrate.linux-armv7 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . +# cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -a -o ../../cli/build/migrate.linux-arm64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . cd ./cmd/migrate && CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -a -o ../../cli/build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . - cd ./cmd/migrate && CGO_ENABLED=1 GOOS=windows GOARCH=amd64 go build -a -o ../../cli/build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . +# cd ./cmd/migrate && CGO_ENABLED=1 GOOS=windows GOARCH=amd64 go build -a -o ../../cli/build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . cd ./cli/build && find . -name 'migrate*' | xargs -I{} tar czf {}.tar.gz {} cd ./cli/build && shasum -a 256 * > sha256sum.txt cat ./cli/build/sha256sum.txt @@ -34,7 +35,7 @@ test: test-with-flags: @echo SOURCE: $(SOURCE) - @echo DATABASE: $(DATABASE) + @echo DATABASE_TEST: $(DATABASE_TEST) @go test $(TEST_FLAGS) ./... diff --git a/database/neo4j/neo4j.go b/database/neo4j/neo4j.go index 9c8d117ae..f89338239 100644 --- a/database/neo4j/neo4j.go +++ b/database/neo4j/neo4j.go @@ -3,6 +3,7 @@ package neo4j import ( "C" // import C so that we can't compile with CGO_ENABLED=0 "fmt" + "github.com/hashicorp/go-multierror" "io" "io/ioutil" neturl "net/url" @@ -108,7 +109,7 @@ func (n *Neo4j) Run(migration io.Reader) (err error) { } defer func() { if cerr := session.Close(); cerr != nil { - err = cerr + err = multierror.Append(err, cerr) } }() @@ -116,10 +117,10 @@ func (n *Neo4j) Run(migration io.Reader) (err error) { if err != nil { return err } - if result.Err() != nil { - err = result.Err() + if err = result.Err(); err != nil { + return err } - return err + return nil } func (n *Neo4j) SetVersion(version int, dirty bool) (err error) { @@ -129,7 +130,7 @@ func (n *Neo4j) SetVersion(version int, dirty bool) (err error) { } defer func() { if cerr := session.Close(); cerr != nil { - err = cerr + err = multierror.Append(err, cerr) } }() @@ -139,10 +140,10 @@ func (n *Neo4j) SetVersion(version int, dirty bool) (err error) { if err != nil { return err } - if result.Err() != nil { - err = result.Err() + if err = result.Err(); err != nil { + return err } - return err + return nil } type MigrationRecord struct { @@ -157,7 +158,7 @@ func (n *Neo4j) Version() (version int, dirty bool, err error) { } defer func() { if cerr := session.Close(); cerr != nil { - err = cerr + err = multierror.Append(err, cerr) } }() @@ -204,7 +205,7 @@ func (n *Neo4j) Drop() (err error) { } defer func() { if cerr := session.Close(); cerr != nil { - err = cerr + err = multierror.Append(err, cerr) } }() @@ -219,7 +220,7 @@ func (n *Neo4j) ensureVersionConstraint() (err error) { } defer func() { if cerr := session.Close(); cerr != nil { - err = cerr + err = multierror.Append(err, cerr) } }() @@ -228,8 +229,8 @@ func (n *Neo4j) ensureVersionConstraint() (err error) { if err != nil { return err } - if result.Err() != nil { - err = result.Err() + if err = result.Err(); err != nil { + return err } - return err + return nil } From 9936ded61e3f592ed52885e622f875b06bc677e9 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Sat, 11 Jan 2020 17:00:09 +0300 Subject: [PATCH 23/24] add libssl for alpine --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 96978c06b..4d38960aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ARG VERSION RUN apk add --no-cache git gcc musl-dev # dependencies for neo4j -RUN apk add --update --no-cache ca-certificates cmake make g++ openssl-dev git curl pkgconfig +RUN apk add --update --no-cache ca-certificates cmake make g++ libressl-dev openssl-dev git curl pkgconfig # build seabolt for neo4j driver RUN git clone -b 1.7 https://github.com/neo4j-drivers/seabolt.git /seabolt From ff92b049d59f711eb63fa5ba22fdb9e6aba8aec4 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Mon, 13 Jan 2020 10:31:38 -0800 Subject: [PATCH 24/24] updates for review --- Dockerfile | 2 +- Makefile | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4d38960aa..029c4c9d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ARG VERSION RUN apk add --no-cache git gcc musl-dev # dependencies for neo4j -RUN apk add --update --no-cache ca-certificates cmake make g++ libressl-dev openssl-dev git curl pkgconfig +RUN apk add --update --no-cache ca-certificates cmake make g++ openssl-libs-static openssl-dev git curl pkgconfig # build seabolt for neo4j driver RUN git clone -b 1.7 https://github.com/neo4j-drivers/seabolt.git /seabolt diff --git a/Makefile b/Makefile index 28a184b0b..19c8e386d 100644 --- a/Makefile +++ b/Makefile @@ -9,11 +9,11 @@ COVERAGE_DIR ?= .coverage build-cli: clean -mkdir ./cli/build -# cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o ../../cli/build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . -# cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -a -o ../../cli/build/migrate.linux-armv7 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . -# cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -a -o ../../cli/build/migrate.linux-arm64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . - cd ./cmd/migrate && CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -a -o ../../cli/build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . -# cd ./cmd/migrate && CGO_ENABLED=1 GOOS=windows GOARCH=amd64 go build -a -o ../../cli/build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o ../../cli/build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -a -o ../../cli/build/migrate.linux-armv7 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -a -o ../../cli/build/migrate.linux-arm64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -a -o ../../cli/build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -a -o ../../cli/build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . cd ./cli/build && find . -name 'migrate*' | xargs -I{} tar czf {}.tar.gz {} cd ./cli/build && shasum -a 256 * > sha256sum.txt cat ./cli/build/sha256sum.txt