Merge pull request #28 from titan-data/master

Feature/mongo clone testing (titan-data#102)

.github/workflows/end-to-end-test.yml

@@ -58,3 +58,9 @@ jobs:
           hostname -i >> localip
           ssh-keygen -b 2048 -t rsa -f ./sshKey -q -N ""
           java -jar vexrun.jar -f ./src/endtoend-test/remotes/ssh/sshWorkflowTests.yml
+      - name: Run DB Matrix Tests
+        run: java -jar vexrun.jar -f ./src/endtoend-test/db-matrix/databases.yml
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: ${{ secrets.AWS_REGION }}

src/endtoend-test/db-matrix/databases.yml

@@ -0,0 +1,46 @@
+tests:
+  - "aws configure access_key_id":
+      command: aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
+      env:
+        get:
+          - AWS_ACCESS_KEY_ID
+  - "aws configure access_secret_key":
+      command: aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
+      env:
+        get:
+          - AWS_SECRET_ACCESS_KEY
+  - "aws configure region":
+      command: aws configure set region $AWS_REGION
+      env:
+        get:
+          - AWS_REGION
+  - "can install titan":
+      command: titan install
+      wait: 10
+      stdout:
+        contains: Titan cli successfully installed, happy data versioning :)
+files:
+  - matrix-tests.yml:
+      parameters:
+        DB: postgres
+        VERSION: 12.0
+        PORT: 5432:5432
+        URI: s3://titan-data-testdata/e2etest
+  - matrix-tests.yml:
+      parameters:
+        DB: postgres
+        VERSION: 11.5
+        PORT: 5432:5432
+        URI: s3://titan-data-testdata/e2etest
+  - matrix-tests.yml:
+      parameters:
+        DB: mongo
+        VERSION: 4
+        PORT: 27017:27017
+        URI: s3://titan-data-testdata/e2etest
+  - matrix-tests.yml:
+      parameters:
+        DB: mongo
+        VERSION: 3.6.14
+        PORT: 27017:27017
+        URI: s3://titan-data-testdata/e2etest

src/endtoend-test/db-matrix/matrix-tests.yml

@@ -0,0 +1,174 @@
+tests:
+  - "$DB > $VERSION > run":
+      command: titan run -- --name $DB-test -p $PORT -d $DB:$VERSION
+      stdout:
+        contains:
+          - Creating repository $DB-test
+          - Running controlled container $DB-test
+      env:
+        get:
+          - DB
+          - VERSION
+          - PORT
+  - "$DB > $VERSION > commit":
+      command: [titan, commit, -m, Test Commit, $DB-test]
+      stdout:
+        contains: Commit
+      env:
+        get:
+          - DB
+          - VERSION
+        set:
+          - COMMIT_GUID:
+              split:
+                delimiter: " "
+                position: 1
+  - "$DB > $VERSION > checkout":
+      command: [titan, checkout, --commit, $COMMIT_GUID, $DB-test]
+      wait: 5
+      stdout: |-
+        Stopping container $DB-test
+        Checkout $COMMIT_GUID
+        Starting container $DB-test
+        $COMMIT_GUID checked out
+      env:
+        get:
+          - COMMIT_GUID
+          - DB
+          - VERSION
+  - "$DB > $VERSION > remote > add > s3":
+      command: titan remote add -r s3 $URI $DB-test
+      env:
+        get:
+          - DB
+          - VERSION
+          - URI
+  - "$DB > $VERSION > remote > ls":
+      command: titan remote ls $DB-test
+      stdout:
+        contains: $URI
+      env:
+        get:
+          - DB
+          - VERSION
+          - URI
+  - "$DB > $VERSION > remote > log > missing commit":
+      command: titan remote log $DB-test
+      stdout:
+        excludes: Commit $COMMIT_GUID
+      env:
+        get:
+          - DB
+          - VERSION
+          - COMMIT_GUID
+  - "$DB > $VERSION > push":
+      command: [titan, push, -r, s3, -c, $COMMIT_GUID, $DB-test]
+      stdout:
+        contains:
+          - Pushing $COMMIT_GUID to 's3'
+          - Push completed successfully
+      env:
+        get:
+          - DB
+          - VERSION
+          - COMMIT_GUID
+  - "$DB > $VERSION > remote > log > has commit":
+      command: titan remote log $DB-test
+      stdout:
+        contains: Commit $COMMIT_GUID
+      env:
+        get:
+          - DB
+          - VERSION
+          - COMMIT_GUID
+  - "$DB > $VERSION > delete > commit":
+      command: titan delete -c $COMMIT_GUID $DB-test
+      stdout: $COMMIT_GUID deleted
+      env:
+        get:
+          - DB
+          - VERSION
+          - COMMIT_GUID
+  - "$DB > $VERSION > log > missing commit":
+      command: titan log $DB-test
+      stdout:
+        excludes:
+          - $COMMIT_GUID
+      env:
+        get:
+          - DB
+          - VERSION
+          - COMMIT_GUID
+  - "$DB > $VERSION > pull":
+      command: [titan, pull, -c, $COMMIT_GUID, -r, s3, $DB-test]
+      stdout:
+        contains:
+          - Pulling $COMMIT_GUID from 's3'
+          - Pull completed successfully
+      env:
+        get:
+          - DB
+          - VERSION
+          - COMMIT_GUID
+  - "$DB > $VERSION > log > has commit":
+      command: titan log $DB-test
+      stdout:
+        contains: commit $COMMIT_GUID
+      env:
+        get:
+          - DB
+          - VERSION
+          - COMMIT_GUID
+  - "$DB > $VERSION > rm":
+      command: titan rm -f $DB-test
+      wait: 5
+      stdout:
+        contains:
+          - Removing container $DB-test
+          - $DB-test removed
+      env:
+        get:
+          - DB
+          - VERSION
+  - "$DB > $VERSION > clone":
+      command: titan clone $URI $DB-test
+      stdout:
+        contains:
+          - Creating repository $DB-test
+          - Running controlled container $DB-test
+          - Pull completed successfully
+          - Starting container $DB-test
+      env:
+        get:
+          - DB
+          - VERSION
+          - URI
+  - "$DB > $VERSION > log > has commit":
+      command: titan log $DB-test
+      stdout:
+        contains: commit $COMMIT_GUID
+      env:
+        get:
+          - DB
+          - VERSION
+          - COMMIT_GUID
+  - "$DB > $VERSION > remove s3 assets":
+      command: aws s3 rm $URI --recursive
+      env:
+        get:
+          - DB
+          - VERSION
+          - URI
+  - "$DB > $VERSION > rm":
+      command: titan rm -f $DB-test
+      wait: 5
+      stdout:
+        contains:
+          - Removing container $DB-test
+          - $DB-test removed
+      env:
+        get:
+          - DB
+          - VERSION
+after:
+  clearVars: true

src/endtoend-test/remotes/s3/s3Cleanup.yml

@@ -1,6 +1,6 @@
 tests:
   - "remove $REMOTE assets":
-      command: aws s3 rm $URI
+      command: aws s3 rm $URI --recursive
       env:
         get:
           - URI

src/main/kotlin/io/titandata/titan/clients/Docker.kt

@@ -179,12 +179,14 @@ class Docker(private val executor: CommandExecutor) {
         }
 
         fun String.runtimeToArguments(): List<String> {
-            val arguments = this.removePrefix("[").removeSuffix("]").toList(", ").toMutableList()
-            if (arguments.contains("--mount")) {
-                arguments.removeAt((arguments.indexOf("--mount") + 1))
-                arguments.removeAt(arguments.indexOf("--mount"))
+            val rawArguments = this.removePrefix("[").removeSuffix("]").toList(", ")
+            val returnArgs = mutableListOf<String>()
+            for (arg in rawArguments) {
+               if (arg != "--mount" && !arg.contains("type=volume")) {
+                   returnArgs.add(arg)
+               }
             }
-            return arguments
+            return returnArgs
         }
 
         fun List<String>.fetchName(): String {

src/main/kotlin/io/titandata/titan/providers/local/Clone.kt

@@ -13,6 +13,7 @@ import io.titandata.models.Repository
 import io.titandata.titan.utils.CommandExecutor
 import io.titandata.serialization.RemoteUtil
 import io.titandata.titan.exceptions.CommandException
+import kotlin.system.exitProcess
 
 class Clone (
     private val remoteAdd: (container:String, uri: String, remoteName: String?, params: Map<String, String>) -> Unit,
@@ -72,6 +73,7 @@ class Clone (
             println(e.message)
             println(e.output)
             remove(repository.name, true)
+            exitProcess(1)
         }
     }
 }