Fix lint issue, and reconfigure build scripts so db test and integrat…

…ion test are separate.

Signed-off-by: Ying Li <[email protected]>

Makefile

@@ -130,16 +130,13 @@ test:
 	@echo
 	go test -tags "${NOTARY_BUILDTAGS}" $(TESTOPTS) $(PKGS)
 
-test-full: TESTOPTS =
-test-full: vet lint
-	@echo Note: when testing with a yubikey plugged in, make sure to include 'TESTOPTS="-p 1"'
-	@echo "+ $@"
-	@echo
-	go test -tags "${NOTARY_BUILDTAGS}" $(TESTOPTS) -v $(PKGS)
-
 integration: TESTDB = mysql
 integration:
-	buildscripts/integrationtest.sh development.$(TESTDB).yml
+	buildscripts/integrationtest.sh $(TESTDB)
+
+testdb: TESTDB = mysql
+testdb:
+	buildscripts/dbtests.sh $(TESTDB)
 
 protos:
 	@protoc --go_out=plugins=grpc:. proto/*.proto

buildscripts/circle_parallelism.sh

@@ -1,19 +1,17 @@
 #!/usr/bin/env bash
 
 set -e
-set +x
 case $CIRCLE_NODE_INDEX in
 0) docker run --rm -e NOTARY_BUILDTAGS=pkcs11 --env-file buildscripts/env.list --user notary notary_client bash -c "make ci && codecov"
    ;;
 1) docker run --rm -e NOTARY_BUILDTAGS=none --env-file buildscripts/env.list --user notary notary_client bash -c "make ci && codecov"
    ;;
-2) SKIPENVCHECK=1 make TESTDB=mysql integration
+2) SKIPENVCHECK=1 make TESTDB=mysql testdb
+   SKIPENVCHECK=1 make TESTDB=mysql integration
    ;;
-3) SKIPENVCHECK=1 make TESTDB=rethink integration
+3) SKIPENVCHECK=1 make TESTDB=rethink testdb
+   SKIPENVCHECK=1 make TESTDB=rethink integration
    ;;
 4) docker run --rm -e NOTARY_BUILDTAGS=pkcs11 notary_client make vet lint fmt misspell
    ;;
 esac
-
-set +e
-set +x

buildscripts/dbtests.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+db="$1"
+
+case ${db} in
+  mysql*)
+    db="mysql"
+    dbContainerOpts="--name mysql_tests mysql mysqld --innodb_file_per_table"
+    DBURL="server@tcp(mysql_tests:3306)/notaryserver?parseTime=True"
+    ;;
+  rethink*)
+    db="rethink"
+    dbContainerOpts="--name rethinkdb_tests rdb-01 --bind all --driver-tls-key /tls/key.pem --driver-tls-cert /tls/cert.pem"
+    DBURL="rethinkdb_tests"
+    ;;
+esac
+
+composeFile="development.${db}.yml"
+project=dbtests
+
+function cleanup {
+    rm -f bin/notary
+    docker-compose -p "${project}_${db}" -f "${composeFile}" kill
+    # if we're in CircleCI, we cannot remove any containers
+    if [[ -z "${CIRCLECI}" ]]; then
+        docker-compose -p "${project}_${db}" -f "${composeFile}" down -v --remove-orphans
+    fi
+}
+
+clientCmd="make test"
+if [[ -z "${CIRCLECI}" ]]; then
+    BUILDOPTS="--force-rm"
+else
+    clientCmd="make ci && codecov"
+fi
+
+set -e
+set -x
+
+cleanup
+
+docker-compose -p "${project}_${db}" -f ${composeFile} build ${BUILDOPTS} client
+
+trap cleanup SIGINT SIGTERM EXIT
+
+# run the unit tests that require a DB
+
+docker-compose -p "${project}_${db}" -f "${composeFile}" run --no-deps -d ${dbContainerOpts}
+docker-compose -p "${project}_${db}" -f "${composeFile}" run --no-deps \
+    -e NOTARY_BUILDTAGS="${db}db" -e DBURL="${DBURL}" \
+    -e PKGS="github.com/docker/notary/server/storage" \
+    client bash -c "${clientCmd}"

buildscripts/env.list

@@ -38,3 +38,5 @@ WORKSPACE
 # TRAVIS_REPO_SLUG
 # TRAVIS_COMMIT
 # TRAVIS_BUILD_DIR
+
+SKIPENVCHECK=1

buildscripts/integrationtest.sh

@@ -1,13 +1,15 @@
 #!/usr/bin/env bash
 
-composeFile="$1"
+db="$1"
+composeFile="development.${db}.yml"
+project=integration
 
 function cleanup {
     rm -f bin/notary
-	docker-compose -f $composeFile kill
+	docker-compose -p "${project}_${db}" -f ${composeFile} kill
 	# if we're in CircleCI, we cannot remove any containers
 	if [[ -z "${CIRCLECI}" ]]; then
-		docker-compose -f $composeFile down -v --remove-orphans
+		docker-compose -p "${project}_${db}" -f ${composeFile} down -v --remove-orphans
 	fi
 }
 
@@ -32,35 +34,9 @@ set -x
 
 cleanup
 
-docker-compose -f $composeFile config
-docker-compose -f $composeFile build ${BUILDOPTS} --pull | tee
+docker-compose -p "${project}_${db}" -f ${composeFile} config
+docker-compose -p "${project}_${db}" -f ${composeFile} build ${BUILDOPTS} --pull | tee
 
 trap cleanupAndExit SIGINT SIGTERM EXIT
 
-# run the unit tests that require a DB
-case $composeFile in
-  development.mysql.yml)
-    docker-compose -f $composeFile run --no-deps -d --name "mysql_tests" mysql mysqld --innodb_file_per_table
-    docker-compose -f $composeFile run --no-deps \
-        -e NOTARY_BUILDTAGS=mysqldb \
-        -e PKGS="github.com/docker/notary/server/storage" \
-        -e MYSQL="server@tcp(mysql_tests:3306)/notaryserver?parseTime=True" \
-        --user notary \
-        client bash -c "make ci && codecov"
-    ;;
-  development.rethink.yml)
-    docker-compose -f $composeFile run --no-deps -d --name "rethinkdb_tests" \
-        rdb-01 --bind all --no-http-admin --server-name rdb_01 \
-        --driver-tls-key /tls/key.pem --driver-tls-cert /tls/cert.pem
-    docker-compose -f $composeFile run --no-deps \
-        -e NOTARY_BUILDTAGS=rethinkdb \
-        -e PKGS="github.com/docker/notary/server/storage" \
-        -e RETHINK="rethinkdb_tests" \
-        --user notary \
-        client bash -c "make ci && codecov"
-    ;;
-esac
-
-docker-compose -f $composeFile down -v
-
-docker-compose -f $composeFile up --abort-on-container-exit
+docker-compose -p "${project}_${db}" -f ${composeFile} up --abort-on-container-exit

server/storage/memory_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func assertExpectedMemoryTUFMeta(t *testing.T, expected []tufMeta, s *MemStorage) {
+func assertExpectedMemoryTUFMeta(t *testing.T, expected []StoredTUFMeta, s *MemStorage) {
 	counter := make(map[string]int)
 	for _, tufObj := range expected {
 		k := entryKey(tufObj.Gun, tufObj.Role)

server/storage/mysql_test.go

@@ -17,7 +17,7 @@ import (
 
 func init() {
 	// Get the MYSQL connection string from an environment variable
-	dburl := os.Getenv("MYSQL")
+	dburl := os.Getenv("DBURL")
 	if dburl == "" {
 		logrus.Fatal("MYSQL environment variable not set")
 	}

server/storage/rethink_realdb_test.go

@@ -19,7 +19,7 @@ var tlsOpts = tlsconfig.Options{InsecureSkipVerify: true}
 
 func rethinkSessionSetup(t *testing.T) (*gorethink.Session, string) {
 	// Get the MYSQL connection string from an environment variable
-	rethinkSource := os.Getenv("RETHINK")
+	rethinkSource := os.Getenv("DBURL")
 	require.NotEqual(t, "", rethinkSource)
 
 	sess, err := rethinkdb.AdminConnection(tlsOpts, rethinkSource)

server/storage/sqldb_test.go

@@ -64,7 +64,7 @@ func SampleUpdate(version int) MetaUpdate {
 	}
 }
 
-func assertExpectedGormTUFMeta(t *testing.T, expected []tufMeta, gormDB gorm.DB) {
+func assertExpectedGormTUFMeta(t *testing.T, expected []StoredTUFMeta, gormDB gorm.DB) {
 	expectedGorm := make([]TUFFile, len(expected))
 	for i, tufObj := range expected {
 		expectedGorm[i] = TUFFile{

server/storage/storage_test.go

@@ -10,18 +10,18 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-type tufMeta struct {
+type StoredTUFMeta struct {
 	Gun     string
 	Role    string
 	Sha256  string
 	Data    []byte
 	Version int
 }
 
-func SampleCustomTUFObj(role, gun string, tufdata []byte, version int) tufMeta {
+func SampleCustomTUFObj(role, gun string, tufdata []byte, version int) StoredTUFMeta {
 	checksum := sha256.Sum256(tufdata)
 	hexChecksum := hex.EncodeToString(checksum[:])
-	return tufMeta{
+	return StoredTUFMeta{
 		Gun:     gun,
 		Role:    role,
 		Version: version,
@@ -38,7 +38,7 @@ func SampleCustomUpdate(role string, tufdata []byte, version int) MetaUpdate {
 	}
 }
 
-func assertExpectedTUFMetaInStore(t *testing.T, s MetaStore, expected []tufMeta) {
+func assertExpectedTUFMetaInStore(t *testing.T, s MetaStore, expected []StoredTUFMeta) {
 	for _, tufObj := range expected {
 		_, tufdata, err := s.GetCurrent(tufObj.Gun, tufObj.Role)
 		require.NoError(t, err)
@@ -55,8 +55,8 @@ func assertExpectedTUFMetaInStore(t *testing.T, s MetaStore, expected []tufMeta)
 
 // UpdateCurrent should succeed if there was no previous metadata of the same
 // gun and role.  They should be gettable.
-func testUpdateCurrentEmptyStore(t *testing.T, s MetaStore) []tufMeta {
-	expected := make([]tufMeta, 0, 10)
+func testUpdateCurrentEmptyStore(t *testing.T, s MetaStore) []StoredTUFMeta {
+	expected := make([]StoredTUFMeta, 0, 10)
 	for _, role := range append(data.BaseRoles, "targets/a") {
 		for _, gun := range []string{"gun1", "gun2"} {
 			// Adding a new TUF file should succeed
@@ -72,7 +72,7 @@ func testUpdateCurrentEmptyStore(t *testing.T, s MetaStore) []tufMeta {
 
 // UpdateCurrent will successfully add a new (higher) version of an existing TUF file,
 // but will return an error if there is an older version of a TUF file.
-func testUpdateCurrentVersionCheck(t *testing.T, s MetaStore) []tufMeta {
+func testUpdateCurrentVersionCheck(t *testing.T, s MetaStore) []StoredTUFMeta {
 	role, gun, tufdata := data.CanonicalRootRole, "testGUN", []byte("1")
 
 	// starting meta is version 1
@@ -90,7 +90,7 @@ func testUpdateCurrentVersionCheck(t *testing.T, s MetaStore) []tufMeta {
 			"Expected ErrOldVersion error type, got: %v", err)
 	}
 
-	expected := []tufMeta{
+	expected := []StoredTUFMeta{
 		SampleCustomTUFObj(role, gun, tufdata, 1),
 		SampleCustomTUFObj(role, gun, tufdata, 2),
 		SampleCustomTUFObj(role, gun, tufdata, 4),
@@ -101,9 +101,9 @@ func testUpdateCurrentVersionCheck(t *testing.T, s MetaStore) []tufMeta {
 
 // UpdateMany succeeds if the updates do not conflict with each other or with what's
 // already in the DB
-func testUpdateManyNoConflicts(t *testing.T, s MetaStore) []tufMeta {
+func testUpdateManyNoConflicts(t *testing.T, s MetaStore) []StoredTUFMeta {
 	gun, tufdata := "testGUN", []byte("many")
-	expected := make([]tufMeta, 4)
+	expected := make([]StoredTUFMeta, 4)
 	updates := make([]MetaUpdate, 4)
 	for i, role := range data.BaseRoles {
 		expected[i] = SampleCustomTUFObj(role, gun, tufdata, 1)
@@ -137,9 +137,9 @@ func testUpdateManyNoConflicts(t *testing.T, s MetaStore) []tufMeta {
 
 // UpdateMany does not insert any rows (or at least rolls them back) if there
 // are any conflicts.
-func testUpdateManyConflictRollback(t *testing.T, s MetaStore) []tufMeta {
+func testUpdateManyConflictRollback(t *testing.T, s MetaStore) []StoredTUFMeta {
 	gun := "testGUN"
-	successBatch := make([]tufMeta, 4)
+	successBatch := make([]StoredTUFMeta, 4)
 	updates := make([]MetaUpdate, 4)
 	for i, role := range data.BaseRoles {
 		tufdata := []byte(gun + "_" + role + "_1")
@@ -150,7 +150,7 @@ func testUpdateManyConflictRollback(t *testing.T, s MetaStore) []tufMeta {
 	require.NoError(t, s.UpdateMany(gun, updates))
 
 	// conflicts with what's in DB
-	badBatch := make([]tufMeta, 4)
+	badBatch := make([]StoredTUFMeta, 4)
 	for i, role := range data.BaseRoles {
 		version := 2
 		if role == data.CanonicalTargetsRole {
@@ -193,7 +193,7 @@ func testDeleteSuccess(t *testing.T, s MetaStore) {
 	require.NoError(t, s.Delete(gun))
 
 	// If there is data in the DB, all versions are deleted
-	unexpected := make([]tufMeta, 0, 10)
+	unexpected := make([]StoredTUFMeta, 0, 10)
 	updates := make([]MetaUpdate, 0, 10)
 	for _, role := range append(data.BaseRoles, "targets/a") {
 		for version := 1; version < 3; version++ {