Skip to content

Commit

Permalink
Merge pull request #2969 from vector-im/rav/keep_old_bundles
Browse files Browse the repository at this point in the history
Update redeploy script to keep old bundles
  • Loading branch information
richvdh authored Jan 17, 2017
2 parents 6257019 + 8371006 commit 641a5c2
Showing 1 changed file with 118 additions and 24 deletions.
142 changes: 118 additions & 24 deletions scripts/redeploy.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,32 @@
#!/usr/bin/env python
#
# auto-deploy script for https://riot.im/develop
#
# Listens for HTTP hits. When it gets one, downloads the artifact from jenkins
# and deploys it as the new version.
#
# Requires the following python packages:
#
# - requests
# - flask
#
from __future__ import print_function
import json, requests, tarfile, argparse, os, errno
import time
from urlparse import urljoin
from flask import Flask, jsonify, request, abort

app = Flask(__name__)

arg_jenkins_url, arg_extract_path, arg_should_clean, arg_symlink, arg_config_location = (
None, None, None, None, None
)
arg_jenkins_url = None
arg_extract_path = None
arg_bundles_path = None
arg_should_clean = None
arg_symlink = None
arg_config_location = None

class DeployException(Exception):
pass

def download_file(url):
local_filename = url.split('/')[-1]
Expand Down Expand Up @@ -57,6 +76,9 @@ def on_receive_jenkins_poke():
abort(400, "Missing or bad build number")
return

return fetch_jenkins_build(job_name, build_num)

def fetch_jenkins_build(job_name, build_num):
artifact_url = urljoin(
arg_jenkins_url, "job/%s/%s/api/json" % (job_name, build_num)
)
Expand Down Expand Up @@ -106,27 +128,38 @@ def on_receive_jenkins_poke():
arg_jenkins_url, "job/%s/%s/artifact/%s" % (job_name, build_num, tar_gz_path)
)

print("Retrieving .tar.gz file: %s" % tar_gz_url)
# we extract into a directory based on the build number. This avoids the
# problem of multiple builds building the same git version and thus having
# the same tarball name. That would lead to two potential problems:
# (a) sometimes jenkins serves corrupted artifacts; we would replace
# a good deploy with a bad one
# (b) we'll be overwriting the live deployment, which means people might
# see half-written files.
build_dir = os.path.join(arg_extract_path, "%s-#%s" % (job_name, build_num))
try:
deploy_tarball(tar_gz_url, build_dir)
except DeployException as e:
abort(400, e.message)

return jsonify({})

def deploy_tarball(tar_gz_url, build_dir):
"""Download a tarball from jenkins and deploy it as the new version
"""
print("Deploying %s to %s" % (tar_gz_url, build_dir))

if os.path.exists(build_dir):
raise DeployException(
"Not deploying. We have previously deployed this build."
)
os.mkdir(build_dir)

# we rely on the fact that flask only serves one request at a time to
# ensure that we do not overwrite a tarball from a concurrent request.
filename = download_file(tar_gz_url)
print("Downloaded file: %s" % filename)

try:
# we extract into a directory based on the build number. This avoids the
# problem of multiple builds building the same git version and thus having
# the same tarball name. That would lead to two potential problems:
# (a) sometimes jenkins serves corrupted artifacts; we would replace
# a good deploy with a bad one
# (b) we'll be overwriting the live deployment, which means people might
# see half-written files.
build_dir = os.path.join(arg_extract_path, "%s-#%s" % (job_name, build_num))
if os.path.exists(build_dir):
abort(400, "Not deploying. We have previously deployed this build.")
return
os.mkdir(build_dir)

untar_to(filename, build_dir)
print("Extracted to: %s" % build_dir)
finally:
Expand All @@ -139,9 +172,47 @@ def on_receive_jenkins_poke():
if arg_config_location:
create_symlink(source=arg_config_location, linkname=os.path.join(extracted_dir, 'config.json'))

if arg_bundles_path:
extracted_bundles = os.path.join(extracted_dir, 'bundles')
move_bundles(source=extracted_bundles, dest=arg_bundles_path)

# replace the (hopefully now empty) extracted_bundles dir with a
# symlink to the common dir.
relpath = os.path.relpath(arg_bundles_path, extracted_dir)
os.rmdir(extracted_bundles)
print ("Symlink %s -> %s" % (extracted_bundles, relpath))
os.symlink(relpath, extracted_bundles)

create_symlink(source=extracted_dir, linkname=arg_symlink)

return jsonify({})
def move_bundles(source, dest):
"""Move the contents of the 'bundles' directory to a common dir
We check that we will not be overwriting anything before we proceed.
Args:
source (str): path to 'bundles' within the extracted tarball
dest (str): target common directory
"""

if not os.path.isdir(dest):
os.mkdir(dest)

# build a map from source to destination, checking for non-existence as we go.
renames = {}
for f in os.listdir(source):
dst = os.path.join(dest, f)
if os.path.exists(dst):
raise DeployException(
"Not deploying. The bundle includes '%s' which we have previously deployed."
% f
)
renames[os.path.join(source, f)] = dst

for (src, dst) in renames.iteritems():
print ("Move %s -> %s" % (src, dst))
os.rename(src, dst)


if __name__ == "__main__":
parser = argparse.ArgumentParser("Runs a Vector redeployment server.")
Expand All @@ -161,6 +232,13 @@ def on_receive_jenkins_poke():
"The location to extract .tar.gz files to."
)
)
parser.add_argument(
"-b", "--bundles-dir", dest="bundles_dir", help=(
"A directory to move the contents of the 'bundles' directory to. A \
symlink to the bundles directory will also be written inside the \
extracted tarball. Example: './bundles'."
)
)
parser.add_argument(
"-c", "--clean", dest="clean", action="store_true", default=False, help=(
"Remove .tar.gz files after they have been downloaded and extracted."
Expand All @@ -179,18 +257,34 @@ def on_receive_jenkins_poke():
To this location."
)
)
parser.add_argument(
"--test", dest="tarball_uri", help=(
"Don't start an HTTP listener. Instead download a build from Jenkins \
immediately."
),
)

args = parser.parse_args()
if args.jenkins.endswith("/"): # important for urljoin
arg_jenkins_url = args.jenkins
else:
arg_jenkins_url = args.jenkins + "/"
arg_extract_path = args.extract
arg_bundles_path = args.bundles_dir
arg_should_clean = args.clean
arg_symlink = args.symlink
arg_config_location = args.config
print(
"Listening on port %s. Extracting to %s%s. Symlinking to %s. Jenkins URL: %s. Config location: %s" %
(args.port, arg_extract_path,
" (clean after)" if arg_should_clean else "", arg_symlink, arg_jenkins_url, arg_config_location)
)
app.run(host="0.0.0.0", port=args.port, debug=True)

if not os.path.isdir(arg_extract_path):
os.mkdir(arg_extract_path)

if args.tarball_uri is not None:
build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time()))
deploy_tarball(args.tarball_uri, build_dir)
else:
print(
"Listening on port %s. Extracting to %s%s. Symlinking to %s. Jenkins URL: %s. Config location: %s" %
(args.port, arg_extract_path,
" (clean after)" if arg_should_clean else "", arg_symlink, arg_jenkins_url, arg_config_location)
)
app.run(host="0.0.0.0", port=args.port, debug=True)

0 comments on commit 641a5c2

Please sign in to comment.