Merge branch 'master' into label-unittest

* master:
  Upgrade p4python to 2020.1 (#179)
  ENG-3659 Disable tmp cleanup for P4Python connections (#178)
  Add example workspace cleanup script (#174)
  Revert "Force in case the client has files open (#176)" (#177)
  Force in case the client has files open (#176)
  ENG-3523 Create virtual env once for unique requirements.txt (#167)

.gitignore

@@ -5,7 +5,7 @@ __pycache__/
 .vscode
 .pytest_cache
 .dev-venv
-.perforce-plugin-venv
+.perforce-plugin-venv*
 
 local-pipeline
-python/fixture/server
+python/fixture/server

.travis.yml

@@ -8,6 +8,7 @@ before_install:
   - pip install -U pip
 install: 
   - pip install -r ./ci/requirements.txt
+  - pip install -r ./python/requirements.txt
   - ./ci/install_$TRAVIS_OS_NAME.sh
 script:
   - ./ci/test.sh

ci/install_linux.sh

@@ -7,17 +7,6 @@ wget http://www.perforce.com/downloads/perforce/r18.2/bin.linux26x86_64/p4d
 sudo chmod +x p4d
 sudo mv p4d /usr/local/bin/p4d
 
-# Build P4Python from source, pip install fails as we cannot connect to ftp.perforce.com from travis agents
-wget http://www.perforce.com/downloads/perforce/r18.2/bin.linux26x86_64/p4api.tgz
-# Detect concrete version number inside p4api.tgz, changes as new versions are published
-P4API_VERSION=$(tar -tf p4api.tgz | head -1 || true)
-tar xzf p4api.tgz --directory /tmp/
-wget https://files.pythonhosted.org/packages/36/5a/0a1b192cdecd31cb8bc0d0ba39c73ffd84ce823053d0004823a1fdbe1440/p4python-2018.2.1743033.tar.gz
-tar xfz p4python-2018.2.1743033.tar.gz --directory /tmp/
-pushd /tmp/p4python-2018.2.1743033/
-python setup.py install --apidir /tmp/${P4API_VERSION}
-popd
-
 # bk cli for integration tests
 wget https://github.com/buildkite/cli/releases/download/v1.0.0/bk-linux-amd64-1.0.0 -O bk
 sudo chmod +x bk

ci/requirements.txt

@@ -1,2 +1,2 @@
-pytest==4.5.0;
-pylint==2.3.1;
+pytest==4.5.0
+pylint==2.3.1

examples/cleanup-unused-workspaces.py

@@ -0,0 +1,47 @@
+import sys
+import logging
+
+# Recommended reference: https://www.perforce.com/manuals/p4python/p4python.pdf
+from P4 import P4
+from datetime import datetime, timedelta
+from pprint import pprint
+
+# delete workspaces where last access time > N days ago
+__days_unused__ = 30
+
+p4 = P4()
+logger = logging.getLogger("p4python")
+logger.setLevel(logging.INFO)
+handler = logging.StreamHandler(sys.stdout)
+formatter = logging.Formatter(
+    '%(asctime)s %(name)s %(levelname)s: %(message)s',
+    '%H:%M:%S',
+)
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+p4.logger = logger
+
+p4.connect()
+
+clients = p4.run_clients()
+
+# Filter by basic prefix matching.
+# May want to include filtering by user and other fields to avoid false positives.
+bk_clients = [client for client in clients 
+              if client.get('client', '').startswith('bk-p4-')]
+
+now = datetime.now()
+n_days_ago = (now - timedelta(days=__days_unused__)).timestamp()
+unused_clients = [client for client in bk_clients
+                  if int(client.get('Access')) < n_days_ago]
+
+pprint(unused_clients)
+proceed = input("Will delete %d/%d Buildkite clients. Continue? (y/n) " % (len(unused_clients),len(bk_clients))).lower() == 'y'
+
+if proceed:
+    for client in unused_clients:
+        clientname = client.get('client')
+        try:
+            p4.run_client('-d', clientname)
+        except:
+            pass

hooks/checkout

@@ -3,7 +3,6 @@ set -eo pipefail
 
 plugin_root="${BASH_SOURCE%/*}/.."
 
-venv_dir="${BUILDKITE_BUILD_CHECKOUT_PATH}/../.perforce-plugin-venv"
 
 # Try to use explicit python3 if its installed
 python_bin="python3"
@@ -16,16 +15,27 @@ fi
 if [[ ! $(${python_bin} -m pip freeze) =~ "virtualenv==" ]]; then
     ${python_bin} -m pip install "virtualenv==16.7.7"
 fi
-${python_bin} -m virtualenv "${venv_dir}"
+
+# Unique virtualenv for requirements.txt
+venv_md5=$(md5sum "${plugin_root}/python/requirements.txt" | awk '{print $1}')
+venv_dir="${BUILDKITE_BUILD_CHECKOUT_PATH}/../.perforce-plugin-venv-${venv_md5}"
 
 platform=$(${python_bin} -c "import platform; print(platform.system())")
 if [[ "${platform}" == "Windows" ]]; then
-    venv_bin="${venv_dir}/Scripts"
+    venv_python_bin="/Scripts/python"
 else
-    venv_bin="${venv_dir}/bin"
+    venv_python_bin="/bin/python"
 fi
 
+if ! [[ -d "${venv_dir}" ]]; then
+    temp_venv_dir=$(mktemp -d -t perforce-plugin-venv-XXXXXX)
+    trap "rm -rf ${temp_venv_dir}" EXIT
+    ${python_bin} -m virtualenv "${temp_venv_dir}"
+    ${temp_venv_dir}${venv_python_bin} -m pip install -r "${plugin_root}/python/requirements.txt"
+    if ! [[ -d "${venv_dir}" ]]; then # second check to minimize venv init race
+        mv "${temp_venv_dir}" "${venv_dir}"
+    fi
+    echo "virtualenv created at ${venv_dir}"
+fi
 
-"${venv_bin}/python" -m pip install -r "${plugin_root}/python/requirements.txt"
-
-"${venv_bin}/python" "${plugin_root}/python/checkout.py"
+${venv_dir}${venv_python_bin} "${plugin_root}/python/checkout.py"

python/perforce.py

@@ -34,6 +34,7 @@ def __init__(self, root=None, view=None, stream=None,
         self.p4config = os.path.join(self.root, 'p4config')
 
         self.perforce = P4()
+        self.perforce.disable_tmp_cleanup() # Required to use multiple P4 connections in parallel safely
         self.perforce.exception_level = 1  # Only errors are raised as exceptions
         logger = logging.getLogger("p4python")
         logger.setLevel(logging.INFO)

python/requirements.txt

@@ -1 +1 @@
-p4python==2018.2.1743033
+p4python==2020.1.1983437