From 4cae86b07606bf5ca883ec1b8856bdff5abc5faf Mon Sep 17 00:00:00 2001
From: Carl Johnson <carljohnson@improbable.io>
Date: Thu, 13 Aug 2020 13:01:08 +0100
Subject: [PATCH 1/6] ENG-3523 Create virtual env once for unique
 requirements.txt (#167)

---
 .gitignore     |  4 ++--
 hooks/checkout | 24 +++++++++++++++++-------
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/.gitignore b/.gitignore
index 6d7f1d0..9f53f1a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,7 +5,7 @@ __pycache__/
 .vscode
 .pytest_cache
 .dev-venv
-.perforce-plugin-venv
+.perforce-plugin-venv*
 
 local-pipeline
-python/fixture/server
\ No newline at end of file
+python/fixture/server
diff --git a/hooks/checkout b/hooks/checkout
index 0df4479..c54599c 100755
--- a/hooks/checkout
+++ b/hooks/checkout
@@ -3,7 +3,6 @@ set -eo pipefail
 
 plugin_root="${BASH_SOURCE%/*}/.."
 
-venv_dir="${BUILDKITE_BUILD_CHECKOUT_PATH}/../.perforce-plugin-venv"
 
 # Try to use explicit python3 if its installed
 python_bin="python3"
@@ -16,16 +15,27 @@ fi
 if [[ ! $(${python_bin} -m pip freeze) =~ "virtualenv==" ]]; then
     ${python_bin} -m pip install "virtualenv==16.7.7"
 fi
-${python_bin} -m virtualenv "${venv_dir}"
+
+# Unique virtualenv for requirements.txt
+venv_md5=$(md5sum "${plugin_root}/python/requirements.txt" | awk '{print $1}')
+venv_dir="${BUILDKITE_BUILD_CHECKOUT_PATH}/../.perforce-plugin-venv-${venv_md5}"
 
 platform=$(${python_bin} -c "import platform; print(platform.system())")
 if [[ "${platform}" == "Windows" ]]; then
-    venv_bin="${venv_dir}/Scripts"
+    venv_python_bin="/Scripts/python"
 else
-    venv_bin="${venv_dir}/bin"
+    venv_python_bin="/bin/python"
 fi
 
+if ! [[ -d "${venv_dir}" ]]; then
+    temp_venv_dir=$(mktemp -d -t perforce-plugin-venv-XXXXXX)
+    trap "rm -rf ${temp_venv_dir}" EXIT
+    ${python_bin} -m virtualenv "${temp_venv_dir}"
+    ${temp_venv_dir}${venv_python_bin} -m pip install -r "${plugin_root}/python/requirements.txt"
+    if ! [[ -d "${venv_dir}" ]]; then # second check to minimize venv init race
+        mv "${temp_venv_dir}" "${venv_dir}"
+    fi
+    echo "virtualenv created at ${venv_dir}"
+fi
 
-"${venv_bin}/python" -m pip install -r "${plugin_root}/python/requirements.txt"
-
-"${venv_bin}/python" "${plugin_root}/python/checkout.py"
+${venv_dir}${venv_python_bin} "${plugin_root}/python/checkout.py"

From d5a6e3602e15a9b14c0bee26de4bea9eb06e5405 Mon Sep 17 00:00:00 2001
From: Matt Church <mattchurch@improbable.io>
Date: Tue, 1 Sep 2020 15:30:13 +0000
Subject: [PATCH 2/6] Force in case the client has files open (#176)

* Force in case the client has files open

* extra dot

* force save client
---
 python/perforce.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/python/perforce.py b/python/perforce.py
index dd64da7..3fa3a74 100644
--- a/python/perforce.py
+++ b/python/perforce.py
@@ -90,7 +90,8 @@ def _setup_client(self):
         # (e.g. interrupted syncs, artefacts that have been checked-in)
         client._options = self.client_opts + ' clobber'
 
-        self.perforce.save_client(client)
+        # force in case the client has files open
+        self.perforce.save_client(client, '-f')
 
         if not os.path.isfile(self.p4config):
             self.perforce.logger.warning("p4config missing, flushing workspace to revision zero")

From f395fde8ce41c16e39d019bfc7ebdf008066254e Mon Sep 17 00:00:00 2001
From: Matt Church <mattchurch@improbable.io>
Date: Tue, 1 Sep 2020 17:40:44 +0000
Subject: [PATCH 3/6] Revert "Force in case the client has files open (#176)"
 (#177)

This reverts commit d5a6e3602e15a9b14c0bee26de4bea9eb06e5405.
---
 python/perforce.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/python/perforce.py b/python/perforce.py
index 3fa3a74..dd64da7 100644
--- a/python/perforce.py
+++ b/python/perforce.py
@@ -90,8 +90,7 @@ def _setup_client(self):
         # (e.g. interrupted syncs, artefacts that have been checked-in)
         client._options = self.client_opts + ' clobber'
 
-        # force in case the client has files open
-        self.perforce.save_client(client, '-f')
+        self.perforce.save_client(client)
 
         if not os.path.isfile(self.p4config):
             self.perforce.logger.warning("p4config missing, flushing workspace to revision zero")

From 6b9229d2e953843189a1200b8dcc0ab4e6a3df5c Mon Sep 17 00:00:00 2001
From: Carl Johnson <carljohnson@improbable.io>
Date: Wed, 2 Sep 2020 10:12:12 +0100
Subject: [PATCH 4/6] Add example workspace cleanup script (#174)

---
 examples/cleanup-unused-workspaces.py | 47 +++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 examples/cleanup-unused-workspaces.py

diff --git a/examples/cleanup-unused-workspaces.py b/examples/cleanup-unused-workspaces.py
new file mode 100644
index 0000000..be172de
--- /dev/null
+++ b/examples/cleanup-unused-workspaces.py
@@ -0,0 +1,47 @@
+import sys
+import logging
+
+# Recommended reference: https://www.perforce.com/manuals/p4python/p4python.pdf
+from P4 import P4
+from datetime import datetime, timedelta
+from pprint import pprint
+
+# delete workspaces where last access time > N days ago
+__days_unused__ = 30
+
+p4 = P4()
+logger = logging.getLogger("p4python")
+logger.setLevel(logging.INFO)
+handler = logging.StreamHandler(sys.stdout)
+formatter = logging.Formatter(
+    '%(asctime)s %(name)s %(levelname)s: %(message)s',
+    '%H:%M:%S',
+)
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+p4.logger = logger
+
+p4.connect()
+
+clients = p4.run_clients()
+
+# Filter by basic prefix matching.
+# May want to include filtering by user and other fields to avoid false positives.
+bk_clients = [client for client in clients 
+              if client.get('client', '').startswith('bk-p4-')]
+
+now = datetime.now()
+n_days_ago = (now - timedelta(days=__days_unused__)).timestamp()
+unused_clients = [client for client in bk_clients
+                  if int(client.get('Access')) < n_days_ago]
+
+pprint(unused_clients)
+proceed = input("Will delete %d/%d Buildkite clients. Continue? (y/n) " % (len(unused_clients),len(bk_clients))).lower() == 'y'
+
+if proceed:
+    for client in unused_clients:
+        clientname = client.get('client')
+        try:
+            p4.run_client('-d', clientname)
+        except:
+            pass

From d5091892a855ea33b5fb98d890da0c10c36948b9 Mon Sep 17 00:00:00 2001
From: Carl Johnson <carljohnson@improbable.io>
Date: Wed, 2 Sep 2020 16:35:20 +0100
Subject: [PATCH 5/6] ENG-3659 Disable tmp cleanup for P4Python connections
 (#178)

---
 python/perforce.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/python/perforce.py b/python/perforce.py
index dd64da7..5b40321 100644
--- a/python/perforce.py
+++ b/python/perforce.py
@@ -34,6 +34,7 @@ def __init__(self, root=None, view=None, stream=None,
         self.p4config = os.path.join(self.root, 'p4config')
 
         self.perforce = P4()
+        self.perforce.disable_tmp_cleanup() # Required to use multiple P4 connections in parallel safely
         self.perforce.exception_level = 1  # Only errors are raised as exceptions
         logger = logging.getLogger("p4python")
         logger.setLevel(logging.INFO)

From c482e6051e5dc263d66a03bfbf9e50cfc73cafc4 Mon Sep 17 00:00:00 2001
From: Carl Johnson <carljohnson@improbable.io>
Date: Thu, 3 Sep 2020 10:29:30 +0100
Subject: [PATCH 6/6] Upgrade p4python to 2020.1 (#179)

---
 .travis.yml             |  1 +
 ci/install_linux.sh     | 11 -----------
 ci/requirements.txt     |  4 ++--
 python/requirements.txt |  2 +-
 4 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index dbb3492..7e8652d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,6 +8,7 @@ before_install:
   - pip install -U pip
 install: 
   - pip install -r ./ci/requirements.txt
+  - pip install -r ./python/requirements.txt
   - ./ci/install_$TRAVIS_OS_NAME.sh
 script:
   - ./ci/test.sh
diff --git a/ci/install_linux.sh b/ci/install_linux.sh
index 693da9d..b3d74fd 100755
--- a/ci/install_linux.sh
+++ b/ci/install_linux.sh
@@ -7,17 +7,6 @@ wget http://www.perforce.com/downloads/perforce/r18.2/bin.linux26x86_64/p4d
 sudo chmod +x p4d
 sudo mv p4d /usr/local/bin/p4d
 
-# Build P4Python from source, pip install fails as we cannot connect to ftp.perforce.com from travis agents
-wget http://www.perforce.com/downloads/perforce/r18.2/bin.linux26x86_64/p4api.tgz
-# Detect concrete version number inside p4api.tgz, changes as new versions are published
-P4API_VERSION=$(tar -tf p4api.tgz | head -1 || true)
-tar xzf p4api.tgz --directory /tmp/
-wget https://files.pythonhosted.org/packages/36/5a/0a1b192cdecd31cb8bc0d0ba39c73ffd84ce823053d0004823a1fdbe1440/p4python-2018.2.1743033.tar.gz
-tar xfz p4python-2018.2.1743033.tar.gz --directory /tmp/
-pushd /tmp/p4python-2018.2.1743033/
-python setup.py install --apidir /tmp/${P4API_VERSION}
-popd
-
 # bk cli for integration tests
 wget https://github.com/buildkite/cli/releases/download/v1.0.0/bk-linux-amd64-1.0.0 -O bk
 sudo chmod +x bk
diff --git a/ci/requirements.txt b/ci/requirements.txt
index f2ef027..97c9348 100644
--- a/ci/requirements.txt
+++ b/ci/requirements.txt
@@ -1,2 +1,2 @@
-pytest==4.5.0;
-pylint==2.3.1;
\ No newline at end of file
+pytest==4.5.0
+pylint==2.3.1
diff --git a/python/requirements.txt b/python/requirements.txt
index 62be5ca..d35cf46 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -1 +1 @@
-p4python==2018.2.1743033
\ No newline at end of file
+p4python==2020.1.1983437