Feature/clone tag (#136)

Fixes #97

.github/workflows/end-to-end-test.yml

@@ -59,6 +59,12 @@ jobs:
         run: |
           ssh-keygen -b 2048 -t rsa -f ./sshKey -q -N ""
           java -jar vexrun.jar -f ./src/endtoend-test/remotes/ssh/sshWorkflowTests.yml
+      - name: Run Tag Tests
+        run: java -jar vexrun.jar -d ./src/endtoend-test/tags
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: ${{ secrets.AWS_REGION }}
       - name: Run DB Matrix Tests
         run: java -jar vexrun.jar -f ./src/endtoend-test/db-matrix/databases.yml
         env:

docs/src/cli/cmd/clone.rst

@@ -15,15 +15,17 @@ Syntax
 
 ::
 
-    titan clone [-c id] [-p key=value ...] [-n repository] <uri> -- [additional context specific arguments]...
+    titan clone [-c id] [-p key=value ...] [-t key=value ...] [-n repository] <uri> -- [additional context specific arguments]...
 
 Arguments
 ---------
 
 uri
     *Required*. The URI of the remote to clone from. For more information on
     remotes, the URI format, and different remote providers, see the
-    :ref:`remote` section.
+    :ref:`remote` section. Note that due to current limitations, when using a URL
+    with query parameters (for tags), it must be passed after the "--" delimeter,
+    otherwise you will get an error of "no such option".
 
 
 Options
@@ -37,6 +39,11 @@ Options
                                 port mapping of exposed ports from the container
                                 to localhost.
 
+-t, --tag       key=value       Optional tags to filter commits when finding the
+                                latest commit. Generates an error if no matching
+                                commit is found, or if an explicit commit ID
+                                is specified.
+
 -n, --name      TEXT            Optional. Name of the new repository to create.
                                 If not specified, then the name of the original
                                 repository is used (which may or may not match

docs/src/remote/clone.rst

@@ -16,7 +16,16 @@ for more details.
 
 .. note::
 
-   The clone command currently always uses the latest commit by default. To clone a specific
+   The clone command uses the latest commit by default. To clone a specific
    commit, add the commit GUID to the URI with a `#` tag. Example::
 
     $ titan clone -n hello-world s3web://demo.titan-data.io/hello-world/postgres#0f53a6a4-90ff-4f8c-843a-a6cce36f4f4f
+
+.. note::
+
+   The clone command supports filtering the latest commit by tag, which can be done
+   via the command line or as part of the URL. To specify tags in the URL, provide
+   them as one or more "tag" query parameter. Note that due to a current limitation,
+   this must be provided after the "--" delimiteer.
+
+   $ titan clone -- s3://my-bucket/hello-world?tag=label=nightly

src/endtoend-test/getting-started/GettingStartedTests.yml

@@ -17,8 +17,8 @@ tests:
   - "can list hello-world/postgres":
       command: titan ls
       stdout: |-
-        CONTEXT      REPOSITORY            STATUS
-        docker       hello-world           running
+        CONTEXT       REPOSITORY            STATUS
+        docker        hello-world           running
   - "can get contents of hello-world/postgres":
       command: [docker, exec, hello-world, psql, postgres://postgres:postgres@localhost/postgres, -t, -c, SELECT * FROM messages;]
       stdout: Hello, World!

src/endtoend-test/tags/clone-tags.yml

@@ -0,0 +1,63 @@
+tests:
+  - "can launch postgres":
+      command: titan run postgres
+  - "can create commit with tag=one":
+      command: titan commit -t tag=one postgres
+      stdout:
+        contains: Commit
+      env:
+        set:
+          - COMMIT_ONE:
+              split:
+                delimiter: " "
+                position: 1
+  - "can create commit with tag=two":
+      command: titan commit -t tag=two postgres
+      stdout:
+        contains: Commit
+      env:
+        set:
+          - COMMIT_TWO:
+              split:
+                delimiter: " "
+                position: 1
+  - "can add s3 remote":
+      command: titan remote add s3://titan-data-testdata/e2etest postgres
+  - "can push tag=two":
+      command: titan push -t tag=two postgres
+  - "commit two exists in remote":
+      command: titan remote log postgres
+      stdout:
+        contains: $COMMIT_TWO
+        excludes: $COMMIT_ONE
+      env:
+        get:
+          - COMMIT_ONE
+          - COMMIT_TWO
+  - "can push tag=one":
+      command: titan push -t tag=one postgres
+  - "commit one exists in remote":
+      command: titan remote log postgres
+      stdout:
+        contains: $COMMIT_ONE
+      env:
+        get:
+          - COMMIT_ONE
+  - "can remove postgres":
+      command: titan rm -f postgres
+  - "can clone tag=one":
+      command: titan clone -n postgres -- s3://titan-data-testdata/e2etest?tag=tag=one
+  - "commit one exists locally":
+      command: titan log postgres
+      stdout:
+        contains: $COMMIT_ONE
+      env:
+        get:
+          - COMMIT_ONE
+  - "can remove cloned postgres":
+      command: titan rm -f postgres
+  - "clone of non-existent tag fails":
+      command: titan clone -n postgres2 -- s3://titan-data-testdata/e2etest?tag=tag=three
+      exitValue: 1
+  - "can cleanup S3 assets":
+      command: aws s3 rm s3://titan-data-testdata/e2etest --recursive

src/main/kotlin/io/titandata/titan/commands/Clone.kt

@@ -25,6 +25,7 @@ class Clone : CliktCommand(help = "Clone a remote repository to local repository
     private val commit by option("-c", "--commit", help = "Commit GUID to pull from, defaults to latest")
     private val parameters by option("-p", "--parameters", help = "Provider specific parameters. key=value format.").multiple()
     private val disablePortMapping by option("-P", "--disable-port-mapping", help = "Disable default port mapping from container to localhost.").flag(default = false)
+    private val tags by option("-t", "--tag", help = "Filter latest commit by tags").multiple()
     private val arguments by argument().multiple()
 
     override fun run() {
@@ -37,7 +38,7 @@ class Clone : CliktCommand(help = "Clone a remote repository to local repository
             }
             params[split[0]] = split[1]
         }
-        provider.clone(uri, repoName, commit, params, arguments, disablePortMapping)
+        provider.clone(uri, repoName, commit, params, arguments, disablePortMapping, tags)
     }
 }
 

src/main/kotlin/io/titandata/titan/providers/Kubernetes.kt

@@ -239,10 +239,10 @@ class Kubernetes(val contextName: String = "kubernetes", val host: String = "loc
         throw NotImplementedError("cp is not supported in kubernetes context")
     }
 
-    override fun clone(uri: String, container: String?, commit: String?, params: Map<String, String>, arguments: List<String>, disablePortMapping: Boolean) {
+    override fun clone(uri: String, container: String?, commit: String?, params: Map<String, String>, arguments: List<String>, disablePortMapping: Boolean, tags: List<String>) {
         val runCommand = Run(::exit, commandExecutor, docker, kubernetes, repositoriesApi, volumesApi)
         val cloneCommand = Clone(::remoteAdd, ::pull, ::checkout, runCommand::run, ::remove, commandExecutor, docker,
                 remotesApi, repositoriesApi)
-        return cloneCommand.clone(uri, container, commit, params, arguments, disablePortMapping)
+        return cloneCommand.clone(uri, container, commit, params, arguments, disablePortMapping, tags)
     }
 }

src/main/kotlin/io/titandata/titan/providers/Local.kt

@@ -229,10 +229,10 @@ class Local(val contextName: String = "docker", val host: String = "localhost",
         return cpCommand.cp(container, driver, source, path)
     }
 
-    override fun clone(uri: String, container: String?, commit: String?, params: Map<String, String>, arguments: List<String>, disablePortMapping: Boolean) {
+    override fun clone(uri: String, container: String?, commit: String?, params: Map<String, String>, arguments: List<String>, disablePortMapping: Boolean, tags: List<String>) {
         val runCommand = Run(::exit, commandExecutor, docker, repositoriesApi)
         val cloneCommand = Clone(::remoteAdd, ::pull, ::checkout, runCommand::run, ::remove, commandExecutor, docker, remotesApi, repositoriesApi)
-        return cloneCommand.clone(uri, container, commit, params, arguments, disablePortMapping)
+        return cloneCommand.clone(uri, container, commit, params, arguments, disablePortMapping, tags)
     }
 
     override fun delete(repository: String, commit: String?, tags: List<String>) {

src/main/kotlin/io/titandata/titan/providers/Mock.kt

@@ -105,7 +105,7 @@ class Mock : Provider {
         println("copying data into $container with $driver from $source")
     }
 
-    override fun clone(uri: String, container: String?, commit: String?, params: Map<String, String>, arguments: List<String>, disablePortMapping: Boolean) {
+    override fun clone(uri: String, container: String?, commit: String?, params: Map<String, String>, arguments: List<String>, disablePortMapping: Boolean, tags: List<String>) {
         println("cloning $container from $uri")
     }
 

src/main/kotlin/io/titandata/titan/providers/Provider.kt

@@ -30,5 +30,5 @@ interface Provider {
     fun start(container: String)
     fun remove(container: String, force: Boolean)
     fun cp(container: String, driver: String, source: String, path: String)
-    fun clone(uri: String, container: String?, commit: String?, params: Map<String, String>, arguments: List<String>, disablePortMapping: Boolean)
+    fun clone(uri: String, container: String?, commit: String?, params: Map<String, String>, arguments: List<String>, disablePortMapping: Boolean, tags: List<String>)
 }

src/main/kotlin/io/titandata/titan/providers/generic/Clone.kt

@@ -13,7 +13,9 @@ import io.titandata.titan.clients.Docker
 import io.titandata.titan.exceptions.CommandException
 import io.titandata.titan.providers.Metadata
 import io.titandata.titan.utils.CommandExecutor
+import java.net.URI
 import kotlin.system.exitProcess
+import okhttp3.HttpUrl
 
 class Clone(
     private val remoteAdd: (container: String, uri: String, remoteName: String?, params: Map<String, String>) -> Unit,
@@ -27,25 +29,47 @@ class Clone(
     private val repositoriesApi: RepositoriesApi = RepositoriesApi(),
     private val remoteUtil: RemoteUtil = RemoteUtil()
 ) {
-    fun clone(uri: String, container: String?, guid: String?, params: Map<String, String>, arguments: List<String>, disablePortMapping: Boolean) {
+    fun clone(
+        uri: String,
+        container: String?,
+        guid: String?,
+        params: Map<String, String>,
+        arguments: List<String>,
+        disablePortMapping: Boolean,
+        tags: List<String>
+    ) {
+        val parsedUri = URI(uri)
         val repoName = when (container) {
-            null -> uri.split("/").last().substringBefore('#')
+            null -> parsedUri.path.split("/").last()
             else -> container
         }
         val commitId = when {
-            guid.isNullOrEmpty() && uri.contains('#') -> uri.split("#").last()
+            guid.isNullOrEmpty() && parsedUri.fragment != null -> parsedUri.fragment
             else -> guid
         }
         val repository = Repository(repoName, emptyMap())
+        val plainUri = "${parsedUri.scheme}://${parsedUri.authority}${parsedUri.path}"
+        val allTags = tags.toMutableList()
+        if (parsedUri.query != null) {
+            allTags.addAll(HttpUrl.parse("http://host?${parsedUri.query}")?.queryParameterValues("tag") ?: emptyList())
+        }
+        var cleanup = false
         try {
             repositoriesApi.createRepository(repository)
-            remoteAdd(repoName, uri.substringBefore('#'), null, params)
+            cleanup = true
+            remoteAdd(repoName, plainUri, null, params)
             val remote = remotesApi.getRemote(repoName, "origin")
             var commit = Commit("id", emptyMap())
             if (commitId.isNullOrEmpty()) {
-                val remoteCommits = remotesApi.listRemoteCommits(repoName, remote.name, remoteUtil.getParameters(remote))
+                val remoteCommits = remotesApi.listRemoteCommits(repoName, remote.name, remoteUtil.getParameters(remote), allTags)
+                if (remoteCommits.isEmpty()) {
+                    error("unable to find any matching commits in remote repository")
+                }
                 commit = remoteCommits.first()
             } else {
+                if (!tags.isEmpty()) {
+                    error("tags cannot be specified with commit ID")
+                }
                 commit = remotesApi.getRemoteCommit(repoName, remote.name, commitId, remoteUtil.getParameters(remote))
             }
             val metadata = Metadata.load(commit.properties)
@@ -66,10 +90,20 @@ class Clone(
             run(metadata.image.digest, repoName, metadata.environment, arguments, disablePortMapping, false)
             pull(repoName, commit.id, null, listOf(), false)
             checkout(repoName, commit.id, listOf())
-        } catch (e: CommandException) {
-            println(e.message)
-            println(e.output)
-            remove(repository.name, true)
+            cleanup = false
+        } catch (t: Throwable) {
+            // We explicitly handle the exception so that the error message appears before the remove messages
+            println(t.message)
+            if (t is CommandException) {
+                println(t.output)
+            }
+            if (cleanup) {
+                try {
+                    remove(repository.name, true)
+                } catch (t: Throwable) {
+                    // Ignore
+                }
+            }
             exitProcess(1)
         }
     }

src/test/kotlin/io/titandata/titan/providers/MockTest.kt

@@ -204,7 +204,7 @@ class MockTest {
     fun `can clone`() {
         val byteStream = ByteArrayOutputStream()
         System.setOut(PrintStream(byteStream))
-        mockProvider.clone("http://user:pass@path", "container", null, emptyMap(), emptyList(), false)
+        mockProvider.clone("http://user:pass@path", "container", null, emptyMap(), emptyList(), false, emptyList())
         byteStream.flush()
         val expected = String(byteStream.toByteArray()).trim()
         assertEquals(expected, "cloning container from http://user:pass@path")