diff --git a/.github/workflows/end-to-end-test.yml b/.github/workflows/end-to-end-test.yml index a25e7e40..e69d4acc 100644 --- a/.github/workflows/end-to-end-test.yml +++ b/.github/workflows/end-to-end-test.yml @@ -59,6 +59,12 @@ jobs: run: | ssh-keygen -b 2048 -t rsa -f ./sshKey -q -N "" java -jar vexrun.jar -f ./src/endtoend-test/remotes/ssh/sshWorkflowTests.yml + - name: Run Tag Tests + run: java -jar vexrun.jar -d ./src/endtoend-test/tags + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} - name: Run DB Matrix Tests run: java -jar vexrun.jar -f ./src/endtoend-test/db-matrix/databases.yml env: diff --git a/docs/src/cli/cmd/clone.rst b/docs/src/cli/cmd/clone.rst index 0d4ae046..284bbdf8 100644 --- a/docs/src/cli/cmd/clone.rst +++ b/docs/src/cli/cmd/clone.rst @@ -15,7 +15,7 @@ Syntax :: - titan clone [-c id] [-p key=value ...] [-n repository] -- [additional context specific arguments]... + titan clone [-c id] [-p key=value ...] [-t key=value ...] [-n repository] -- [additional context specific arguments]... Arguments --------- @@ -23,7 +23,9 @@ Arguments uri *Required*. The URI of the remote to clone from. For more information on remotes, the URI format, and different remote providers, see the - :ref:`remote` section. + :ref:`remote` section. Note that due to current limitations, when using a URL + with query parameters (for tags), it must be passed after the "--" delimeter, + otherwise you will get an error of "no such option". Options @@ -37,6 +39,11 @@ Options port mapping of exposed ports from the container to localhost. +-t, --tag key=value Optional tags to filter commits when finding the + latest commit. Generates an error if no matching + commit is found, or if an explicit commit ID + is specified. + -n, --name TEXT Optional. Name of the new repository to create. If not specified, then the name of the original repository is used (which may or may not match diff --git a/docs/src/remote/clone.rst b/docs/src/remote/clone.rst index cd061561..6184318c 100644 --- a/docs/src/remote/clone.rst +++ b/docs/src/remote/clone.rst @@ -16,7 +16,16 @@ for more details. .. note:: - The clone command currently always uses the latest commit by default. To clone a specific + The clone command uses the latest commit by default. To clone a specific commit, add the commit GUID to the URI with a `#` tag. Example:: $ titan clone -n hello-world s3web://demo.titan-data.io/hello-world/postgres#0f53a6a4-90ff-4f8c-843a-a6cce36f4f4f + +.. note:: + + The clone command supports filtering the latest commit by tag, which can be done + via the command line or as part of the URL. To specify tags in the URL, provide + them as one or more "tag" query parameter. Note that due to a current limitation, + this must be provided after the "--" delimiteer. + + $ titan clone -- s3://my-bucket/hello-world?tag=label=nightly diff --git a/src/endtoend-test/getting-started/GettingStartedTests.yml b/src/endtoend-test/getting-started/GettingStartedTests.yml index aa7b3c3c..7a6c1045 100644 --- a/src/endtoend-test/getting-started/GettingStartedTests.yml +++ b/src/endtoend-test/getting-started/GettingStartedTests.yml @@ -17,8 +17,8 @@ tests: - "can list hello-world/postgres": command: titan ls stdout: |- - CONTEXT REPOSITORY STATUS - docker hello-world running + CONTEXT REPOSITORY STATUS + docker hello-world running - "can get contents of hello-world/postgres": command: [docker, exec, hello-world, psql, postgres://postgres:postgres@localhost/postgres, -t, -c, SELECT * FROM messages;] stdout: Hello, World! diff --git a/src/endtoend-test/tags/clone-tags.yml b/src/endtoend-test/tags/clone-tags.yml new file mode 100644 index 00000000..397d41c7 --- /dev/null +++ b/src/endtoend-test/tags/clone-tags.yml @@ -0,0 +1,63 @@ +tests: + - "can launch postgres": + command: titan run postgres + - "can create commit with tag=one": + command: titan commit -t tag=one postgres + stdout: + contains: Commit + env: + set: + - COMMIT_ONE: + split: + delimiter: " " + position: 1 + - "can create commit with tag=two": + command: titan commit -t tag=two postgres + stdout: + contains: Commit + env: + set: + - COMMIT_TWO: + split: + delimiter: " " + position: 1 + - "can add s3 remote": + command: titan remote add s3://titan-data-testdata/e2etest postgres + - "can push tag=two": + command: titan push -t tag=two postgres + - "commit two exists in remote": + command: titan remote log postgres + stdout: + contains: $COMMIT_TWO + excludes: $COMMIT_ONE + env: + get: + - COMMIT_ONE + - COMMIT_TWO + - "can push tag=one": + command: titan push -t tag=one postgres + - "commit one exists in remote": + command: titan remote log postgres + stdout: + contains: $COMMIT_ONE + env: + get: + - COMMIT_ONE + - "can remove postgres": + command: titan rm -f postgres + - "can clone tag=one": + command: titan clone -n postgres -- s3://titan-data-testdata/e2etest?tag=tag=one + - "commit one exists locally": + command: titan log postgres + stdout: + contains: $COMMIT_ONE + env: + get: + - COMMIT_ONE + - "can remove cloned postgres": + command: titan rm -f postgres + - "clone of non-existent tag fails": + command: titan clone -n postgres2 -- s3://titan-data-testdata/e2etest?tag=tag=three + exitValue: 1 + - "can cleanup S3 assets": + command: aws s3 rm s3://titan-data-testdata/e2etest --recursive diff --git a/src/main/kotlin/io/titandata/titan/commands/Clone.kt b/src/main/kotlin/io/titandata/titan/commands/Clone.kt index fc5597cc..bf9c7e91 100644 --- a/src/main/kotlin/io/titandata/titan/commands/Clone.kt +++ b/src/main/kotlin/io/titandata/titan/commands/Clone.kt @@ -25,6 +25,7 @@ class Clone : CliktCommand(help = "Clone a remote repository to local repository private val commit by option("-c", "--commit", help = "Commit GUID to pull from, defaults to latest") private val parameters by option("-p", "--parameters", help = "Provider specific parameters. key=value format.").multiple() private val disablePortMapping by option("-P", "--disable-port-mapping", help = "Disable default port mapping from container to localhost.").flag(default = false) + private val tags by option("-t", "--tag", help = "Filter latest commit by tags").multiple() private val arguments by argument().multiple() override fun run() { @@ -37,7 +38,7 @@ class Clone : CliktCommand(help = "Clone a remote repository to local repository } params[split[0]] = split[1] } - provider.clone(uri, repoName, commit, params, arguments, disablePortMapping) + provider.clone(uri, repoName, commit, params, arguments, disablePortMapping, tags) } } diff --git a/src/main/kotlin/io/titandata/titan/providers/Kubernetes.kt b/src/main/kotlin/io/titandata/titan/providers/Kubernetes.kt index 7086b548..9dcb07d4 100644 --- a/src/main/kotlin/io/titandata/titan/providers/Kubernetes.kt +++ b/src/main/kotlin/io/titandata/titan/providers/Kubernetes.kt @@ -239,10 +239,10 @@ class Kubernetes(val contextName: String = "kubernetes", val host: String = "loc throw NotImplementedError("cp is not supported in kubernetes context") } - override fun clone(uri: String, container: String?, commit: String?, params: Map, arguments: List, disablePortMapping: Boolean) { + override fun clone(uri: String, container: String?, commit: String?, params: Map, arguments: List, disablePortMapping: Boolean, tags: List) { val runCommand = Run(::exit, commandExecutor, docker, kubernetes, repositoriesApi, volumesApi) val cloneCommand = Clone(::remoteAdd, ::pull, ::checkout, runCommand::run, ::remove, commandExecutor, docker, remotesApi, repositoriesApi) - return cloneCommand.clone(uri, container, commit, params, arguments, disablePortMapping) + return cloneCommand.clone(uri, container, commit, params, arguments, disablePortMapping, tags) } } diff --git a/src/main/kotlin/io/titandata/titan/providers/Local.kt b/src/main/kotlin/io/titandata/titan/providers/Local.kt index 9751bcd3..f8667d28 100644 --- a/src/main/kotlin/io/titandata/titan/providers/Local.kt +++ b/src/main/kotlin/io/titandata/titan/providers/Local.kt @@ -229,10 +229,10 @@ class Local(val contextName: String = "docker", val host: String = "localhost", return cpCommand.cp(container, driver, source, path) } - override fun clone(uri: String, container: String?, commit: String?, params: Map, arguments: List, disablePortMapping: Boolean) { + override fun clone(uri: String, container: String?, commit: String?, params: Map, arguments: List, disablePortMapping: Boolean, tags: List) { val runCommand = Run(::exit, commandExecutor, docker, repositoriesApi) val cloneCommand = Clone(::remoteAdd, ::pull, ::checkout, runCommand::run, ::remove, commandExecutor, docker, remotesApi, repositoriesApi) - return cloneCommand.clone(uri, container, commit, params, arguments, disablePortMapping) + return cloneCommand.clone(uri, container, commit, params, arguments, disablePortMapping, tags) } override fun delete(repository: String, commit: String?, tags: List) { diff --git a/src/main/kotlin/io/titandata/titan/providers/Mock.kt b/src/main/kotlin/io/titandata/titan/providers/Mock.kt index e656b311..1cb26a2a 100644 --- a/src/main/kotlin/io/titandata/titan/providers/Mock.kt +++ b/src/main/kotlin/io/titandata/titan/providers/Mock.kt @@ -105,7 +105,7 @@ class Mock : Provider { println("copying data into $container with $driver from $source") } - override fun clone(uri: String, container: String?, commit: String?, params: Map, arguments: List, disablePortMapping: Boolean) { + override fun clone(uri: String, container: String?, commit: String?, params: Map, arguments: List, disablePortMapping: Boolean, tags: List) { println("cloning $container from $uri") } diff --git a/src/main/kotlin/io/titandata/titan/providers/Provider.kt b/src/main/kotlin/io/titandata/titan/providers/Provider.kt index 6c24137f..eaba23f3 100644 --- a/src/main/kotlin/io/titandata/titan/providers/Provider.kt +++ b/src/main/kotlin/io/titandata/titan/providers/Provider.kt @@ -30,5 +30,5 @@ interface Provider { fun start(container: String) fun remove(container: String, force: Boolean) fun cp(container: String, driver: String, source: String, path: String) - fun clone(uri: String, container: String?, commit: String?, params: Map, arguments: List, disablePortMapping: Boolean) + fun clone(uri: String, container: String?, commit: String?, params: Map, arguments: List, disablePortMapping: Boolean, tags: List) } diff --git a/src/main/kotlin/io/titandata/titan/providers/generic/Clone.kt b/src/main/kotlin/io/titandata/titan/providers/generic/Clone.kt index d1aa5219..fa00952a 100644 --- a/src/main/kotlin/io/titandata/titan/providers/generic/Clone.kt +++ b/src/main/kotlin/io/titandata/titan/providers/generic/Clone.kt @@ -13,7 +13,9 @@ import io.titandata.titan.clients.Docker import io.titandata.titan.exceptions.CommandException import io.titandata.titan.providers.Metadata import io.titandata.titan.utils.CommandExecutor +import java.net.URI import kotlin.system.exitProcess +import okhttp3.HttpUrl class Clone( private val remoteAdd: (container: String, uri: String, remoteName: String?, params: Map) -> Unit, @@ -27,25 +29,47 @@ class Clone( private val repositoriesApi: RepositoriesApi = RepositoriesApi(), private val remoteUtil: RemoteUtil = RemoteUtil() ) { - fun clone(uri: String, container: String?, guid: String?, params: Map, arguments: List, disablePortMapping: Boolean) { + fun clone( + uri: String, + container: String?, + guid: String?, + params: Map, + arguments: List, + disablePortMapping: Boolean, + tags: List + ) { + val parsedUri = URI(uri) val repoName = when (container) { - null -> uri.split("/").last().substringBefore('#') + null -> parsedUri.path.split("/").last() else -> container } val commitId = when { - guid.isNullOrEmpty() && uri.contains('#') -> uri.split("#").last() + guid.isNullOrEmpty() && parsedUri.fragment != null -> parsedUri.fragment else -> guid } val repository = Repository(repoName, emptyMap()) + val plainUri = "${parsedUri.scheme}://${parsedUri.authority}${parsedUri.path}" + val allTags = tags.toMutableList() + if (parsedUri.query != null) { + allTags.addAll(HttpUrl.parse("http://host?${parsedUri.query}")?.queryParameterValues("tag") ?: emptyList()) + } + var cleanup = false try { repositoriesApi.createRepository(repository) - remoteAdd(repoName, uri.substringBefore('#'), null, params) + cleanup = true + remoteAdd(repoName, plainUri, null, params) val remote = remotesApi.getRemote(repoName, "origin") var commit = Commit("id", emptyMap()) if (commitId.isNullOrEmpty()) { - val remoteCommits = remotesApi.listRemoteCommits(repoName, remote.name, remoteUtil.getParameters(remote)) + val remoteCommits = remotesApi.listRemoteCommits(repoName, remote.name, remoteUtil.getParameters(remote), allTags) + if (remoteCommits.isEmpty()) { + error("unable to find any matching commits in remote repository") + } commit = remoteCommits.first() } else { + if (!tags.isEmpty()) { + error("tags cannot be specified with commit ID") + } commit = remotesApi.getRemoteCommit(repoName, remote.name, commitId, remoteUtil.getParameters(remote)) } val metadata = Metadata.load(commit.properties) @@ -66,10 +90,20 @@ class Clone( run(metadata.image.digest, repoName, metadata.environment, arguments, disablePortMapping, false) pull(repoName, commit.id, null, listOf(), false) checkout(repoName, commit.id, listOf()) - } catch (e: CommandException) { - println(e.message) - println(e.output) - remove(repository.name, true) + cleanup = false + } catch (t: Throwable) { + // We explicitly handle the exception so that the error message appears before the remove messages + println(t.message) + if (t is CommandException) { + println(t.output) + } + if (cleanup) { + try { + remove(repository.name, true) + } catch (t: Throwable) { + // Ignore + } + } exitProcess(1) } } diff --git a/src/test/kotlin/io/titandata/titan/providers/MockTest.kt b/src/test/kotlin/io/titandata/titan/providers/MockTest.kt index e99b0c6f..329adee3 100644 --- a/src/test/kotlin/io/titandata/titan/providers/MockTest.kt +++ b/src/test/kotlin/io/titandata/titan/providers/MockTest.kt @@ -204,7 +204,7 @@ class MockTest { fun `can clone`() { val byteStream = ByteArrayOutputStream() System.setOut(PrintStream(byteStream)) - mockProvider.clone("http://user:pass@path", "container", null, emptyMap(), emptyList(), false) + mockProvider.clone("http://user:pass@path", "container", null, emptyMap(), emptyList(), false, emptyList()) byteStream.flush() val expected = String(byteStream.toByteArray()).trim() assertEquals(expected, "cloning container from http://user:pass@path")