diff --git a/.gitignore b/.gitignore index 6d7f1d0..9f53f1a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ __pycache__/ .vscode .pytest_cache .dev-venv -.perforce-plugin-venv +.perforce-plugin-venv* local-pipeline -python/fixture/server \ No newline at end of file +python/fixture/server diff --git a/.travis.yml b/.travis.yml index dbb3492..7e8652d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ before_install: - pip install -U pip install: - pip install -r ./ci/requirements.txt + - pip install -r ./python/requirements.txt - ./ci/install_$TRAVIS_OS_NAME.sh script: - ./ci/test.sh diff --git a/ci/install_linux.sh b/ci/install_linux.sh index 693da9d..b3d74fd 100755 --- a/ci/install_linux.sh +++ b/ci/install_linux.sh @@ -7,17 +7,6 @@ wget http://www.perforce.com/downloads/perforce/r18.2/bin.linux26x86_64/p4d sudo chmod +x p4d sudo mv p4d /usr/local/bin/p4d -# Build P4Python from source, pip install fails as we cannot connect to ftp.perforce.com from travis agents -wget http://www.perforce.com/downloads/perforce/r18.2/bin.linux26x86_64/p4api.tgz -# Detect concrete version number inside p4api.tgz, changes as new versions are published -P4API_VERSION=$(tar -tf p4api.tgz | head -1 || true) -tar xzf p4api.tgz --directory /tmp/ -wget https://files.pythonhosted.org/packages/36/5a/0a1b192cdecd31cb8bc0d0ba39c73ffd84ce823053d0004823a1fdbe1440/p4python-2018.2.1743033.tar.gz -tar xfz p4python-2018.2.1743033.tar.gz --directory /tmp/ -pushd /tmp/p4python-2018.2.1743033/ -python setup.py install --apidir /tmp/${P4API_VERSION} -popd - # bk cli for integration tests wget https://github.com/buildkite/cli/releases/download/v1.0.0/bk-linux-amd64-1.0.0 -O bk sudo chmod +x bk diff --git a/ci/requirements.txt b/ci/requirements.txt index f2ef027..97c9348 100644 --- a/ci/requirements.txt +++ b/ci/requirements.txt @@ -1,2 +1,2 @@ -pytest==4.5.0; -pylint==2.3.1; \ No newline at end of file +pytest==4.5.0 +pylint==2.3.1 diff --git a/examples/cleanup-unused-workspaces.py b/examples/cleanup-unused-workspaces.py new file mode 100644 index 0000000..be172de --- /dev/null +++ b/examples/cleanup-unused-workspaces.py @@ -0,0 +1,47 @@ +import sys +import logging + +# Recommended reference: https://www.perforce.com/manuals/p4python/p4python.pdf +from P4 import P4 +from datetime import datetime, timedelta +from pprint import pprint + +# delete workspaces where last access time > N days ago +__days_unused__ = 30 + +p4 = P4() +logger = logging.getLogger("p4python") +logger.setLevel(logging.INFO) +handler = logging.StreamHandler(sys.stdout) +formatter = logging.Formatter( + '%(asctime)s %(name)s %(levelname)s: %(message)s', + '%H:%M:%S', +) +handler.setFormatter(formatter) +logger.addHandler(handler) +p4.logger = logger + +p4.connect() + +clients = p4.run_clients() + +# Filter by basic prefix matching. +# May want to include filtering by user and other fields to avoid false positives. +bk_clients = [client for client in clients + if client.get('client', '').startswith('bk-p4-')] + +now = datetime.now() +n_days_ago = (now - timedelta(days=__days_unused__)).timestamp() +unused_clients = [client for client in bk_clients + if int(client.get('Access')) < n_days_ago] + +pprint(unused_clients) +proceed = input("Will delete %d/%d Buildkite clients. Continue? (y/n) " % (len(unused_clients),len(bk_clients))).lower() == 'y' + +if proceed: + for client in unused_clients: + clientname = client.get('client') + try: + p4.run_client('-d', clientname) + except: + pass diff --git a/hooks/checkout b/hooks/checkout index 0df4479..c54599c 100755 --- a/hooks/checkout +++ b/hooks/checkout @@ -3,7 +3,6 @@ set -eo pipefail plugin_root="${BASH_SOURCE%/*}/.." -venv_dir="${BUILDKITE_BUILD_CHECKOUT_PATH}/../.perforce-plugin-venv" # Try to use explicit python3 if its installed python_bin="python3" @@ -16,16 +15,27 @@ fi if [[ ! $(${python_bin} -m pip freeze) =~ "virtualenv==" ]]; then ${python_bin} -m pip install "virtualenv==16.7.7" fi -${python_bin} -m virtualenv "${venv_dir}" + +# Unique virtualenv for requirements.txt +venv_md5=$(md5sum "${plugin_root}/python/requirements.txt" | awk '{print $1}') +venv_dir="${BUILDKITE_BUILD_CHECKOUT_PATH}/../.perforce-plugin-venv-${venv_md5}" platform=$(${python_bin} -c "import platform; print(platform.system())") if [[ "${platform}" == "Windows" ]]; then - venv_bin="${venv_dir}/Scripts" + venv_python_bin="/Scripts/python" else - venv_bin="${venv_dir}/bin" + venv_python_bin="/bin/python" fi +if ! [[ -d "${venv_dir}" ]]; then + temp_venv_dir=$(mktemp -d -t perforce-plugin-venv-XXXXXX) + trap "rm -rf ${temp_venv_dir}" EXIT + ${python_bin} -m virtualenv "${temp_venv_dir}" + ${temp_venv_dir}${venv_python_bin} -m pip install -r "${plugin_root}/python/requirements.txt" + if ! [[ -d "${venv_dir}" ]]; then # second check to minimize venv init race + mv "${temp_venv_dir}" "${venv_dir}" + fi + echo "virtualenv created at ${venv_dir}" +fi -"${venv_bin}/python" -m pip install -r "${plugin_root}/python/requirements.txt" - -"${venv_bin}/python" "${plugin_root}/python/checkout.py" +${venv_dir}${venv_python_bin} "${plugin_root}/python/checkout.py" diff --git a/python/perforce.py b/python/perforce.py index dd64da7..5b40321 100644 --- a/python/perforce.py +++ b/python/perforce.py @@ -34,6 +34,7 @@ def __init__(self, root=None, view=None, stream=None, self.p4config = os.path.join(self.root, 'p4config') self.perforce = P4() + self.perforce.disable_tmp_cleanup() # Required to use multiple P4 connections in parallel safely self.perforce.exception_level = 1 # Only errors are raised as exceptions logger = logging.getLogger("p4python") logger.setLevel(logging.INFO) diff --git a/python/requirements.txt b/python/requirements.txt index 62be5ca..d35cf46 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1 +1 @@ -p4python==2018.2.1743033 \ No newline at end of file +p4python==2020.1.1983437