From da8ec2583f152e51c78d8da0e97b8984ab0585c0 Mon Sep 17 00:00:00 2001 From: Carl Johnson Date: Thu, 3 Sep 2020 12:37:33 +0100 Subject: [PATCH 01/10] tests --- python/test_perforce.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/python/test_perforce.py b/python/test_perforce.py index e1deafa..fe6f2ea 100644 --- a/python/test_perforce.py +++ b/python/test_perforce.py @@ -204,6 +204,24 @@ def test_checkout(server, tmpdir): with open(os.path.join(tmpdir, "p4config")) as content: assert "P4PORT=%s\n" % repo.perforce.port in content.readlines(), "Unexpected p4config content" +def test_checkout_partial_path(server, tmpdir): + """Test checking out a subset of view with one path""" + repo = P4Repo(root=tmpdir, sync='//depot/file.txt') + repo.sync() + assert 'file.txt' in os.listdir(tmpdir) + +def test_checkout_partial_dir(server, tmpdir): + """Test checking out a subset of view with one directory""" + repo = P4Repo(root=tmpdir, sync=['//depot/...']) + repo.sync() + assert 'file.txt' in os.listdir(tmpdir) + +def test_checkout_partial_multiple(server, tmpdir): + """Test checking out a subset of view with multiple paths""" + repo = P4Repo(root=tmpdir, sync=['//depot/fake-dir/...', '//depot/file.txt']) + repo.sync() + assert 'file.txt' in os.listdir(tmpdir) + def test_checkout_stream(server, tmpdir): """Test checking out a stream depot""" repo = P4Repo(root=tmpdir, stream='//stream-depot/main') From 1e86681b0fd1c12c055bbff2d0dbaf1048f177d2 Mon Sep 17 00:00:00 2001 From: Carl Johnson Date: Thu, 3 Sep 2020 12:38:29 +0100 Subject: [PATCH 02/10] Update test_perforce.py --- python/test_perforce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/test_perforce.py b/python/test_perforce.py index fe6f2ea..b4ebac1 100644 --- a/python/test_perforce.py +++ b/python/test_perforce.py @@ -212,7 +212,7 @@ def test_checkout_partial_path(server, tmpdir): def test_checkout_partial_dir(server, tmpdir): """Test checking out a subset of view with one directory""" - repo = P4Repo(root=tmpdir, sync=['//depot/...']) + repo = P4Repo(root=tmpdir, sync='//depot/...') repo.sync() assert 'file.txt' in os.listdir(tmpdir) From 10e571f8b211da49b4d639bf8efaf4e6ad08b012 Mon Sep 17 00:00:00 2001 From: Carl Johnson Date: Thu, 3 Sep 2020 13:22:31 +0100 Subject: [PATCH 03/10] unshelve coverage --- python/test_perforce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/test_perforce.py b/python/test_perforce.py index b4ebac1..87ab43d 100644 --- a/python/test_perforce.py +++ b/python/test_perforce.py @@ -316,7 +316,7 @@ def test_p4print_unshelve(server, tmpdir): assert not os.path.exists(os.path.join(tmpdir, "newfile.txt")), "File unshelved for add was not deleted" # Shelved changes containing files not selected for sync are skipped - repo = P4Repo(root=tmpdir, sync='//depot/fake-dir/...') + repo = P4Repo(root=tmpdir, sync=['//depot/fake-dir/...']) repo.sync() repo.p4print_unshelve('3') # Modify file.txt assert not os.path.exists(os.path.join(tmpdir, "file.txt")) From 7061a6b0958b35bdccc1c70def9b218729c73153 Mon Sep 17 00:00:00 2001 From: Carl Johnson Date: Thu, 3 Sep 2020 13:24:33 +0100 Subject: [PATCH 04/10] support list of paths --- python/perforce.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/perforce.py b/python/perforce.py index 5b40321..9c38ac0 100644 --- a/python/perforce.py +++ b/python/perforce.py @@ -25,7 +25,7 @@ def __init__(self, root=None, view=None, stream=None, self.root = os.path.abspath(root or '') self.stream = stream self.view = self._localize_view(view or []) - self.sync_paths = sync or '//...' + self.sync_paths = sync if isinstance(sync, list) else [sync or '//...'] # Support list of paths or single path as input self.client_opts = client_opts or '' self.parallel = parallel @@ -176,9 +176,10 @@ def sync(self, revision=None): """Sync the workspace""" self._setup_client() self.revert() + sync_files = ['%s%s' % (path, revision or '') for path in self.sync_paths] result = self.perforce.run_sync( '--parallel=threads=%s' % self.parallel, - '%s%s' % (self.sync_paths, revision or ''), + *sync_files, handler=SyncOutput(self.perforce.logger), ) if result: @@ -248,14 +249,14 @@ def p4print_unshelve(self, changelist): # Turn sync spec info a prefix to filter out unwanted files # e.g. //my-depot/dir/... => //my-depot/dir/ - sync_prefix = self.sync_paths.rstrip('.') + sync_prefixes = [prefix.rstrip('.') for prefix in self.sync_paths] cmds = [] for depotfile, localfile in depot_to_local.items(): if os.path.isfile(localfile): os.chmod(localfile, stat.S_IWRITE) os.unlink(localfile) - if depotfile.startswith(sync_prefix): + if any(depotfile.startswith(prefix) for prefix in sync_prefixes): cmds.append(('print', '-o', localfile, '%s@=%s' % (depotfile, changelist))) self.run_parallel_cmds(cmds) From 2ae09e09d3b1b47eb1f4dfeaf8477a2267270d90 Mon Sep 17 00:00:00 2001 From: Carl Johnson Date: Thu, 3 Sep 2020 13:24:40 +0100 Subject: [PATCH 05/10] driveby doc --- python/perforce.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/perforce.py b/python/perforce.py index 9c38ac0..9bbbfa1 100644 --- a/python/perforce.py +++ b/python/perforce.py @@ -21,6 +21,9 @@ def __init__(self, root=None, view=None, stream=None, root: Directory in which to create the client workspace view: Client workspace mapping stream: Client workspace stream. Overrides view parameter. + sync: Single path or list of paths to sync. Defaults to entire view. + client_opts: Additional options to add to client. (e.g. allwrite) + parallel: How many threads to use for parallel sync. """ self.root = os.path.abspath(root or '') self.stream = stream From 3f200a0fcfef0ea597f3a1804b3286f8d17b4793 Mon Sep 17 00:00:00 2001 From: Carl Johnson Date: Thu, 3 Sep 2020 13:39:54 +0100 Subject: [PATCH 06/10] parse plugin cfg properly --- python/buildkite.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/python/buildkite.py b/python/buildkite.py index fba5c48..b48cbf3 100644 --- a/python/buildkite.py +++ b/python/buildkite.py @@ -32,7 +32,18 @@ def get_config(): conf = {} conf['view'] = os.environ.get('BUILDKITE_PLUGIN_PERFORCE_VIEW') or '//... ...' conf['stream'] = os.environ.get('BUILDKITE_PLUGIN_PERFORCE_STREAM') - conf['sync'] = os.environ.get('BUILDKITE_PLUGIN_PERFORCE_SYNC') + + conf['sync'] = os.environ.get('BUILDKITE_PLUGIN_PERFORCE_SYNC', []) + if not conf['sync']: + # Read from array instead of single item + i = 0 + while True: + path = os.environ.get('BUILDKITE_PLUGIN_PERFORCE_SYNC_%d' % i) + if not path: + break + conf['sync'].append(path) + i += 1 + conf['parallel'] = os.environ.get('BUILDKITE_PLUGIN_PERFORCE_PARALLEL') or 0 conf['client_opts'] = os.environ.get('BUILDKITE_PLUGIN_PERFORCE_CLIENT_OPTIONS') From d8fa0bf7dbe72ba9168c0684779e2fe6baef4cf6 Mon Sep 17 00:00:00 2001 From: Carl Johnson Date: Thu, 3 Sep 2020 13:40:02 +0100 Subject: [PATCH 07/10] assume list --- python/perforce.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/perforce.py b/python/perforce.py index 9bbbfa1..d0d5a4f 100644 --- a/python/perforce.py +++ b/python/perforce.py @@ -21,14 +21,14 @@ def __init__(self, root=None, view=None, stream=None, root: Directory in which to create the client workspace view: Client workspace mapping stream: Client workspace stream. Overrides view parameter. - sync: Single path or list of paths to sync. Defaults to entire view. + sync: List of paths to sync. Defaults to entire view. client_opts: Additional options to add to client. (e.g. allwrite) parallel: How many threads to use for parallel sync. """ self.root = os.path.abspath(root or '') self.stream = stream self.view = self._localize_view(view or []) - self.sync_paths = sync if isinstance(sync, list) else [sync or '//...'] # Support list of paths or single path as input + self.sync_paths = sync or ['//...'] self.client_opts = client_opts or '' self.parallel = parallel From c42522f6d649c7a4a6b7c622e619baf97820e2de Mon Sep 17 00:00:00 2001 From: Carl Johnson Date: Thu, 3 Sep 2020 13:53:05 +0100 Subject: [PATCH 08/10] update unit tests --- plugin.yml | 2 +- python/perforce.py | 1 + python/test_perforce.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/plugin.yml b/plugin.yml index ba25f79..554da48 100644 --- a/plugin.yml +++ b/plugin.yml @@ -16,7 +16,7 @@ configuration: stream: type: string sync: - type: string + type: array parallel: type: string client_opts: diff --git a/python/perforce.py b/python/perforce.py index d0d5a4f..6df9677 100644 --- a/python/perforce.py +++ b/python/perforce.py @@ -29,6 +29,7 @@ def __init__(self, root=None, view=None, stream=None, self.stream = stream self.view = self._localize_view(view or []) self.sync_paths = sync or ['//...'] + assert isinstance(self.sync_paths, list) self.client_opts = client_opts or '' self.parallel = parallel diff --git a/python/test_perforce.py b/python/test_perforce.py index 87ab43d..0e9b03a 100644 --- a/python/test_perforce.py +++ b/python/test_perforce.py @@ -206,13 +206,13 @@ def test_checkout(server, tmpdir): def test_checkout_partial_path(server, tmpdir): """Test checking out a subset of view with one path""" - repo = P4Repo(root=tmpdir, sync='//depot/file.txt') + repo = P4Repo(root=tmpdir, sync=['//depot/file.txt']) repo.sync() assert 'file.txt' in os.listdir(tmpdir) def test_checkout_partial_dir(server, tmpdir): """Test checking out a subset of view with one directory""" - repo = P4Repo(root=tmpdir, sync='//depot/...') + repo = P4Repo(root=tmpdir, sync=['//depot/...']) repo.sync() assert 'file.txt' in os.listdir(tmpdir) From 1f3f8912db66556f7613c823b923257e048aaf43 Mon Sep 17 00:00:00 2001 From: Carl Johnson Date: Thu, 3 Sep 2020 17:03:33 +0100 Subject: [PATCH 09/10] from arr --- python/buildkite.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/python/buildkite.py b/python/buildkite.py index b48cbf3..1e80e4b 100644 --- a/python/buildkite.py +++ b/python/buildkite.py @@ -27,23 +27,28 @@ def get_env(): env[p4var] = plugin_value return env +def list_from_env_array(var): + """Read list of values from either VAR or VAR_1, VAR_2 etc""" + result = os.environ.get(var, []) + if result: + return [result] # convert single value to list + + i = 0 + while True: + elem = os.environ.get("%s_%d" % (var, i)) + if not elem: + break + result.append(elem) + i += 1 + + return result + def get_config(): """Get configuration which will be passed directly to perforce.P4Repo as kwargs""" conf = {} conf['view'] = os.environ.get('BUILDKITE_PLUGIN_PERFORCE_VIEW') or '//... ...' conf['stream'] = os.environ.get('BUILDKITE_PLUGIN_PERFORCE_STREAM') - - conf['sync'] = os.environ.get('BUILDKITE_PLUGIN_PERFORCE_SYNC', []) - if not conf['sync']: - # Read from array instead of single item - i = 0 - while True: - path = os.environ.get('BUILDKITE_PLUGIN_PERFORCE_SYNC_%d' % i) - if not path: - break - conf['sync'].append(path) - i += 1 - + conf['sync'] = list_from_env_array('BUILDKITE_PLUGIN_PERFORCE_SYNC') conf['parallel'] = os.environ.get('BUILDKITE_PLUGIN_PERFORCE_PARALLEL') or 0 conf['client_opts'] = os.environ.get('BUILDKITE_PLUGIN_PERFORCE_CLIENT_OPTIONS') From ab7ac56517679696c55eb7651a46f117b412c8dc Mon Sep 17 00:00:00 2001 From: Carl Johnson Date: Thu, 3 Sep 2020 17:10:15 +0100 Subject: [PATCH 10/10] Update python/buildkite.py --- python/buildkite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/buildkite.py b/python/buildkite.py index 1e80e4b..f994086 100644 --- a/python/buildkite.py +++ b/python/buildkite.py @@ -28,7 +28,7 @@ def get_env(): return env def list_from_env_array(var): - """Read list of values from either VAR or VAR_1, VAR_2 etc""" + """Read list of values from either VAR or VAR_0, VAR_1 etc""" result = os.environ.get(var, []) if result: return [result] # convert single value to list