diff --git a/.github/workflows/end-to-end-test.yml b/.github/workflows/end-to-end-test.yml index 1037d7b2..f974efa5 100644 --- a/.github/workflows/end-to-end-test.yml +++ b/.github/workflows/end-to-end-test.yml @@ -58,3 +58,9 @@ jobs: hostname -i >> localip ssh-keygen -b 2048 -t rsa -f ./sshKey -q -N "" java -jar vexrun.jar -f ./src/endtoend-test/remotes/ssh/sshWorkflowTests.yml + - name: Run DB Matrix Tests + run: java -jar vexrun.jar -f ./src/endtoend-test/db-matrix/databases.yml + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} diff --git a/src/endtoend-test/db-matrix/databases.yml b/src/endtoend-test/db-matrix/databases.yml new file mode 100644 index 00000000..497ae98b --- /dev/null +++ b/src/endtoend-test/db-matrix/databases.yml @@ -0,0 +1,46 @@ +tests: + - "aws configure access_key_id": + command: aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + env: + get: + - AWS_ACCESS_KEY_ID + - "aws configure access_secret_key": + command: aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + env: + get: + - AWS_SECRET_ACCESS_KEY + - "aws configure region": + command: aws configure set region $AWS_REGION + env: + get: + - AWS_REGION + - "can install titan": + command: titan install + wait: 10 + stdout: + contains: Titan cli successfully installed, happy data versioning :) +files: + - matrix-tests.yml: + parameters: + DB: postgres + VERSION: 12.0 + PORT: 5432:5432 + URI: s3://titan-data-testdata/e2etest + - matrix-tests.yml: + parameters: + DB: postgres + VERSION: 11.5 + PORT: 5432:5432 + URI: s3://titan-data-testdata/e2etest + - matrix-tests.yml: + parameters: + DB: mongo + VERSION: 4 + PORT: 27017:27017 + URI: s3://titan-data-testdata/e2etest + - matrix-tests.yml: + parameters: + DB: mongo + VERSION: 3.6.14 + PORT: 27017:27017 + URI: s3://titan-data-testdata/e2etest diff --git a/src/endtoend-test/db-matrix/matrix-tests.yml b/src/endtoend-test/db-matrix/matrix-tests.yml new file mode 100644 index 00000000..7f029ec1 --- /dev/null +++ b/src/endtoend-test/db-matrix/matrix-tests.yml @@ -0,0 +1,174 @@ +tests: + - "$DB > $VERSION > run": + command: titan run -- --name $DB-test -p $PORT -d $DB:$VERSION + stdout: + contains: + - Creating repository $DB-test + - Running controlled container $DB-test + env: + get: + - DB + - VERSION + - PORT + - "$DB > $VERSION > commit": + command: [titan, commit, -m, Test Commit, $DB-test] + stdout: + contains: Commit + env: + get: + - DB + - VERSION + set: + - COMMIT_GUID: + split: + delimiter: " " + position: 1 + - "$DB > $VERSION > checkout": + command: [titan, checkout, --commit, $COMMIT_GUID, $DB-test] + wait: 5 + stdout: |- + Stopping container $DB-test + Checkout $COMMIT_GUID + Starting container $DB-test + $COMMIT_GUID checked out + env: + get: + - COMMIT_GUID + - DB + - VERSION + - "$DB > $VERSION > remote > add > s3": + command: titan remote add -r s3 $URI $DB-test + env: + get: + - DB + - VERSION + - URI + - "$DB > $VERSION > remote > ls": + command: titan remote ls $DB-test + stdout: + contains: $URI + env: + get: + - DB + - VERSION + - URI + - "$DB > $VERSION > remote > log > missing commit": + command: titan remote log $DB-test + stdout: + excludes: Commit $COMMIT_GUID + env: + get: + - DB + - VERSION + - COMMIT_GUID + - "$DB > $VERSION > push": + command: [titan, push, -r, s3, -c, $COMMIT_GUID, $DB-test] + stdout: + contains: + - Pushing $COMMIT_GUID to 's3' + - Push completed successfully + env: + get: + - DB + - VERSION + - COMMIT_GUID + - "$DB > $VERSION > remote > log > has commit": + command: titan remote log $DB-test + stdout: + contains: Commit $COMMIT_GUID + env: + get: + - DB + - VERSION + - COMMIT_GUID + - "$DB > $VERSION > delete > commit": + command: titan delete -c $COMMIT_GUID $DB-test + stdout: $COMMIT_GUID deleted + env: + get: + - DB + - VERSION + - COMMIT_GUID + - "$DB > $VERSION > log > missing commit": + command: titan log $DB-test + stdout: + excludes: + - $COMMIT_GUID + env: + get: + - DB + - VERSION + - COMMIT_GUID + - "$DB > $VERSION > pull": + command: [titan, pull, -c, $COMMIT_GUID, -r, s3, $DB-test] + stdout: + contains: + - Pulling $COMMIT_GUID from 's3' + - Pull completed successfully + env: + get: + - DB + - VERSION + - COMMIT_GUID + - "$DB > $VERSION > log > has commit": + command: titan log $DB-test + stdout: + contains: commit $COMMIT_GUID + env: + get: + - DB + - VERSION + - COMMIT_GUID + - "$DB > $VERSION > rm": + command: titan rm -f $DB-test + wait: 5 + stdout: + contains: + - Removing container $DB-test + - $DB-test removed + env: + get: + - DB + - VERSION + - "$DB > $VERSION > clone": + command: titan clone $URI $DB-test + stdout: + contains: + - Creating repository $DB-test + - Running controlled container $DB-test + - Pull completed successfully + - Starting container $DB-test + env: + get: + - DB + - VERSION + - URI + - "$DB > $VERSION > log > has commit": + command: titan log $DB-test + stdout: + contains: commit $COMMIT_GUID + env: + get: + - DB + - VERSION + - COMMIT_GUID + - "$DB > $VERSION > remove s3 assets": + command: aws s3 rm $URI --recursive + env: + get: + - DB + - VERSION + - URI + - "$DB > $VERSION > rm": + command: titan rm -f $DB-test + wait: 5 + stdout: + contains: + - Removing container $DB-test + - $DB-test removed + env: + get: + - DB + - VERSION +after: + clearVars: true \ No newline at end of file diff --git a/src/endtoend-test/remotes/s3/s3Cleanup.yml b/src/endtoend-test/remotes/s3/s3Cleanup.yml index 378868a8..682c65d0 100644 --- a/src/endtoend-test/remotes/s3/s3Cleanup.yml +++ b/src/endtoend-test/remotes/s3/s3Cleanup.yml @@ -1,6 +1,6 @@ tests: - "remove $REMOTE assets": - command: aws s3 rm $URI + command: aws s3 rm $URI --recursive env: get: - URI diff --git a/src/main/kotlin/io/titandata/titan/clients/Docker.kt b/src/main/kotlin/io/titandata/titan/clients/Docker.kt index ee6f1391..aa34ecd3 100644 --- a/src/main/kotlin/io/titandata/titan/clients/Docker.kt +++ b/src/main/kotlin/io/titandata/titan/clients/Docker.kt @@ -179,12 +179,14 @@ class Docker(private val executor: CommandExecutor) { } fun String.runtimeToArguments(): List { - val arguments = this.removePrefix("[").removeSuffix("]").toList(", ").toMutableList() - if (arguments.contains("--mount")) { - arguments.removeAt((arguments.indexOf("--mount") + 1)) - arguments.removeAt(arguments.indexOf("--mount")) + val rawArguments = this.removePrefix("[").removeSuffix("]").toList(", ") + val returnArgs = mutableListOf() + for (arg in rawArguments) { + if (arg != "--mount" && !arg.contains("type=volume")) { + returnArgs.add(arg) + } } - return arguments + return returnArgs } fun List.fetchName(): String { diff --git a/src/main/kotlin/io/titandata/titan/providers/local/Clone.kt b/src/main/kotlin/io/titandata/titan/providers/local/Clone.kt index 526d9daf..a7be821f 100644 --- a/src/main/kotlin/io/titandata/titan/providers/local/Clone.kt +++ b/src/main/kotlin/io/titandata/titan/providers/local/Clone.kt @@ -13,6 +13,7 @@ import io.titandata.models.Repository import io.titandata.titan.utils.CommandExecutor import io.titandata.serialization.RemoteUtil import io.titandata.titan.exceptions.CommandException +import kotlin.system.exitProcess class Clone ( private val remoteAdd: (container:String, uri: String, remoteName: String?, params: Map) -> Unit, @@ -72,6 +73,7 @@ class Clone ( println(e.message) println(e.output) remove(repository.name, true) + exitProcess(1) } } } \ No newline at end of file