Skip to content

Commit

Permalink
Jenkins WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
khk-globus committed Apr 28, 2024
1 parent e5e2212 commit cd689c6
Show file tree
Hide file tree
Showing 6 changed files with 172 additions and 44 deletions.
81 changes: 81 additions & 0 deletions compute_endpoint/packaging/JenkinsFile
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
#! groovy

@Library('gcs-build-scripts') _

pipeline {
agent none
options {
buildDiscarder(
logRotator(
numToKeepStr: '5',
artifactNumToKeepStr: '1',
artifactDaysToKeepStr: '7'
)
)
timeout(time: 3, unit: 'HOURS')
disableConcurrentBuilds()
}
stages {
stage ("Prep source") {
agent {label "package_creator"}
steps {
checkout scm
script {
env.SOURCE_STASH_NAME = "${UUID.randomUUID()}";
def py_full_version = sh(script: "/opt/globus-python/bin/python3 -c 'import sys; print(\"{}.{}.{}\".format(*sys.version_info))'", returnStdout: true).trim()
def (py_epoch_version, py_major_version, py_minor_version) = py_full_version.tokenize(".")
def pkg_version = "not yet set"

def venv_py = "${env.WORKSPACE}/compute_endpoint/packaging/venv-${py_epoch_version}${py_major_version}/bin/python"
sh "git clean -fdx"
sh "git checkout kevin_package_test"

dir("compute_endpoint/packaging/") {
sh(script: "make show_vars")
sh(script: "make setup_dist_for_deb setup_dist_for_rpm")
}
dir("compute_endpoint/") {
pkg_version = sh(script: "${venv_py} setup.py --version", returnStdout: true).trim().replace("-", "~")
echo "pkg_version = ${pkg_version}"
}
env.PKG_TARBALL = "globus_compute_endpoint-${pkg_version.replace("~", "")}.tar.gz"
env.PREREQS_TARBALL = "globus_compute_endpoint-prereqs-py${py_epoch_version}${py_major_version}-${pkg_version}.tar.gz"
dir("compute_endpoint/packaging/") {
sh "mv dist/ ${env.SOURCE_STASH_NAME}"
stash(name: env.SOURCE_STASH_NAME, includes: "${env.SOURCE_STASH_NAME}/**/*")
}
}
}
}
stage ("Build packages") {
steps {
script {
lock(resource: 'globus-compute-agent-build') {
parallel "debian": {
def extra_tarball_map = [
prereqs: env.PREREQS_TARBALL
]
echo "extra_tarball_map = ${extra_tarball_map}"
env.DEB_ARTIFACTS_STASH = buildDebian(
env.SOURCE_STASH_NAME,
env.PKG_TARBALL,
require_gcs5_repo: true,
epic: null,
exclude: null,
exclude_arch: "arm64",
extra_tarball_map: extra_tarball_map
)
}, "rpm": {
env.RPM_ARTIFACTS_STASH = buildMock(
env.SOURCE_STASH_NAME,
env.PKG_TARBALL,
true,
null
)
}, "failFast": true
}
}
}
}
}
}
98 changes: 71 additions & 27 deletions compute_endpoint/packaging/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,11 @@ PIP_NAME_D := $(shell cd ../; "$(VENV_PY)" setup.py --name)
PIP_NAME_U := $(shell echo $(PIP_NAME_D) | tr '-' '_')

PKG_VERSION := $(shell cd ../; "$(VENV_PY)" setup.py --version | tr '-' '~')
PKG_WHEEL = $(PIP_NAME_U)-$(PKG_VERSION)-py$(PY_MAJOR_VERSION)-none-any.whl
PREREQS_TARBALL_NAME = $(PIP_NAME_U)-prereqs-py$(PY_VERSION)-$(PKG_VERSION).tar.xz
PKG_WHEEL := $(PIP_NAME_U)-$(PKG_VERSION)-py$(PY_MAJOR_VERSION)-none-any.whl
PKG_SOURCE_DIR := $(PIP_NAME_U)-$(PKG_VERSION)
PKG_TARBALL := $(PKG_SOURCE_DIR).tar.gz
PREREQS_DIR := $(PIP_NAME_U)-prereqs-py$(PY_VERSION)-$(PKG_VERSION)
PREREQS_TARBALL_NAME = $(PREREQS_DIR).tar.gz

OS_CODENAME := $(shell test -f /etc/os-release && . /etc/os-release; echo $${VERSION_CODENAME:-focal})

Expand Down Expand Up @@ -62,7 +65,10 @@ show_vars: ##-For debugging, show the Makefile variables; will install a venv
@echo "PIP_NAME_U : $(PIP_NAME_U)"
@echo "PKG_NAME : $(PKG_NAME)"
@echo "PKG_VERSION : $(PKG_VERSION)"
@echo "PREREQS_DIR : $(PREREQS_DIR)"
@echo "PREREQS_TARBALL_NAME : $(PREREQS_TARBALL_NAME)"
@echo "PKG_SOURCE_DIR : $(PKG_SOURCE_DIR)"
@echo "PKG_TARBALL : $(PKG_TARBALL)"
@echo "PKG_WHEEL : $(PKG_WHEEL)"
@echo
@echo " Override python path with PYTHON3 variable:"
Expand Down Expand Up @@ -107,7 +113,7 @@ $(PKG_WHEEL): $(VENV_PY)
{ /bin/echo -e "\nBUILD COMPUTE FROM A RELEASE TAG (current branch: $$(git branch --show-current))"; exit 1; } \
fi \
&& rm -rf tests/ \
&& "$(VENV_PY)" -m build --wheel -o ../../ \
&& "$(VENV_PY)" -m build -o ../../ \
)

wheel: $(PKG_WHEEL) ##-Make the wheel (note that this does *not* include dependencies)
Expand All @@ -117,24 +123,32 @@ $(PREREQS_TARBALL_NAME): $(VENV_PY) $(PKG_WHEEL)

prereq_tarball: $(PREREQS_TARBALL_NAME) ##-Make a tarball of wheel dependencies

dist: $(PREREQS_TARBALL_NAME) ##-Make the dist/ directory with prereqs and wheel ready for packaging step
rm -rf dist/ \
&& mkdir dist/ \
&& cp $(PREREQS_TARBALL_NAME) $(PKG_WHEEL) dist/
_dist:
rm -rf dist/ && mkdir dist/
cp "$(PKG_WHEEL)" dist/

# For CI, which does things differently than this Makefile:
# - append PKG_WHEEL to *prereqs* tarball
# - append package_shim.sh to *package* tarball
( _prereq_dir="$$(tar -tf "$(PREREQS_TARBALL_NAME)" | head -1)" \
&& _pkg_dir="$$(tar -tf "$(PKG_TARBALL)" | head -1)" \
&& cd dist/ \
&& tar -xf "../$(PREREQS_TARBALL_NAME)" \
&& cp "$(PKG_WHEEL)" "$${_prereq_dir}" \
&& tar -czf "$(PREREQS_TARBALL_NAME)" "$${_prereq_dir}" \
&& rm -r "$${_prereq_dir}" \
&& tar -xf "../$(PKG_TARBALL)" \
&& cp "../package_shim.sh" "$${_pkg_dir}" \
&& tar -czf "$(PKG_TARBALL)" "$${_pkg_dir}" \
&& rm -r "$${_pkg_dir}" \
)

deb_build_needs: ##-Check that necessary executables are available before starting the DEB build.
@[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; }
@dpkg-checkbuilddeps
dist: $(PREREQS_TARBALL_NAME) _dist ##-Make the dist/ directory with prereqs and wheel ready for packaging step
rm "$(PKG_WHEEL)" "$(PKG_TARBALL)" "$(PREREQS_TARBALL_NAME)"

deb: deb_build_needs dist ##-Build a Debian package of the Globus Compute Endpoint (.deb)
_setup_dist_for_deb:
( cd dist/ \
&& rm -rf debbuild/ \
&& mkdir -p debbuild/$(PKG_NAME)/wheels/ \
&& tar -C debbuild/$(PKG_NAME)/wheels/ -xf "$(PREREQS_TARBALL_NAME)" \
&& cp $(PKG_WHEEL) debbuild/$(PKG_NAME)/wheels/ \
&& cp -R ../debian debbuild/$(PKG_NAME)/ \
&& cp ../package_shim.sh debbuild/$(PKG_NAME)/ \
&& cd debbuild/$(PKG_NAME)/ \
&& cp -R ../debian ./ \
&& mv debian/changelog.in.in debian/changelog \
&& sed -i debian/changelog \
-e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \
Expand All @@ -148,29 +162,59 @@ deb: deb_build_needs dist ##-Build a Debian package of the Globus Compute Endpo
-e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \
-e "s/@PACKAGE_VERSION@/$(PKG_VERSION)/g" \
-e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \
)

setup_dist_for_deb: dist _setup_dist_for_deb ##-Place Debian-build required files in dist/

deb_build_needs: ##-Check that necessary executables are available before starting the DEB build.
@[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; }
@dpkg-checkbuilddeps

_deb:
( cd dist/ \
&& rm -rf debbuild/ \
&& mkdir -p debbuild/$(PKG_NAME)/prereqs/ \
&& cp -R debian debbuild/$(PKG_NAME)/ \
&& tar -C debbuild/$(PKG_NAME)/prereqs/ --strip 1 -xf "$(PREREQS_TARBALL_NAME)" \
&& cp $(PKG_WHEEL) debbuild/$(PKG_NAME)/prereqs/ \
&& cp ../package_shim.sh debbuild/$(PKG_NAME)/ \
&& cd debbuild/$(PKG_NAME)/ \
&& dpkg-buildpackage -uc -us \
)
@echo "\nDEB package successfully built:"
@ls -lh dist/debbuild/*deb

rpm_build_needs: ##-Check that necessary executables are available before starting the RPM build.
@[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; }
# Reminder: deb rule is not invoked by CI; this is for one-off needs
deb: deb_build_needs setup_dist_for_deb _deb ##-Build a Debian package of the Globus Compute Endpoint (.deb)

rpm: rpm_build_needs dist ##-Build an RPM package of the Globus Compute Endpoint (.rpm)
_setup_dist_for_rpm:
( cd dist/ \
&& pwd && ls \
&& rm -rf rpmbuild/ \
&& mkdir -p rpmbuild/{BUILD,BUILDROOT,RPMS,SPECS,SRPMS} rpmbuild/SOURCES/$(PKG_NAME)/wheels \
&& tar -C rpmbuild/SOURCES/$(PKG_NAME)/wheels -xf "$(PREREQS_TARBALL_NAME)" \
&& cp $(PKG_WHEEL) rpmbuild/SOURCES/$(PKG_NAME)/wheels/ \
&& cp ../package_shim.sh rpmbuild/SOURCES/$(PKG_NAME)/ \
&& sed \
-e "s/@PACKAGE_VERSION@/$(PKG_VERSION)/g" \
-e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \
-e "s/@PACKAGE_WHEEL@/$(PKG_WHEEL)/g" \
-e "s/@PACKAGE_TARBALL_NAME@/$(PKG_TARBALL)/g" \
-e "s/@PREREQS_TARBALL_NAME@/$(PREREQS_TARBALL_NAME)/g" \
-e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \
-e "s/@PACKAGE_SOURCE_DIR@/$(PKG_SOURCE_DIR)/" \
-e "s/@PREREQS_DIR@/$(PREREQS_DIR)/" \
< ../fedora/$(PKG_NAME).spec.in > ./$(PKG_NAME).spec \
)

setup_dist_for_rpm: dist _setup_dist_for_rpm ##-Place RPM-build required files in dist/ (separated for CI building purposes)

rpm_build_needs: ##-Check that necessary executables are available before starting the RPM build.
@[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; }

# Reminder: rpm rule is not invoked by CI; this is for one-off needs
_rpm:
( cd dist/ \
&& rm -rf rpmbuild/ \
&& mkdir -p rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS}/ \
&& cp *.tar.gz rpmbuild/SOURCES/ \
&& HOME="$$(pwd)" rpmbuild --define "_topdir $$(pwd)/rpmbuild" -ba ./$(PKG_NAME).spec \
)
@echo -e "\nRPM package successfully built:"
@ls -lh dist/rpmbuild/RPMS/**/*rpm

rpm: rpm_build_needs setup_dist_for_rpm _rpm ##-Build an RPM package of the Globus Compute Endpoint (.rpm)
6 changes: 3 additions & 3 deletions compute_endpoint/packaging/create-prereqs-tarball.sh
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,12 @@ done > dependent-prereqs.txt

modified_time="$(TZ=UTC0 date --rfc-3339=seconds)"

TZ=UTC0 tar -C "$download_dir/" --format=posix \
TZ=UTC0 tar --format=posix \
--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime,delete=mtime \
--mtime="$modified_time" \
--numeric-owner \
--owner=0 \
--group=0 \
--mode="go-rwx,u+rw" \
-cf - . \
| xz -9
-cf - "$download_dir/" \
| gzip -9
2 changes: 1 addition & 1 deletion compute_endpoint/packaging/debian/control
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Source: @PACKAGE_NAME@
Priority: optional
Section: contrib/science
Maintainer: Globus Support <[email protected]>
Build-Depends: debhelper (>= 13.6), globus-python (>= 3.9), dh-exec, dh-python
Build-Depends: debhelper (>= 12), globus-python (>= 3.9), dh-exec, dh-python
Standards-Version: 4.6.0
Homepage: https://globus-compute.readthedocs.io/

Expand Down
12 changes: 9 additions & 3 deletions compute_endpoint/packaging/debian/rules
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,13 @@ _unitdir=/lib/systemd/system
.PHONY: override_dh_auto_configure
override_dh_auto_configure:
$(PYTHON3) -mvenv $${PWD}/$(TMP_VIRTUAL_ENV)
. "$(TMP_VIRTUAL_ENV)/bin/activate"; \
. "$${PWD}/$(TMP_VIRTUAL_ENV)/bin/activate"; \
set -x; \
python3 -mpip install --no-index --no-cache-dir -I --compile -U wheels/pip-*.whl; \
python3 -mpip install --no-index --no-cache-dir -I --pre --compile --find-links=file://$${PWD}/wheels/ "@PIP_NAME@"
pwd; \
ls -lR; \
pwd; \
python -mpip install --no-index --no-cache-dir -I --compile -U prereqs/pip-*.whl; \
python -mpip install --no-index --no-cache-dir -I --pre --compile --find-links=file://$${PWD}/prereqs/ "@PIP_NAME@"

.PHONY: override_dh_auto_clean
override_dh_auto_clean:
Expand Down Expand Up @@ -78,5 +81,8 @@ override_dh_python3:
override_dh_builddeb:
dh_builddeb -- -Zxz

.PHONY: override_dh_auto_build
override_dh_auto_build: # empty == we do it ourselves in auto_install (above)

%:
dh $@ --with python3 --with systemd
17 changes: 7 additions & 10 deletions compute_endpoint/packaging/fedora/globus-compute-agent.spec.in
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ Name: @PACKAGE_NAME@
%global pythonversion py39
%global VIRTUAL_ENV /opt/%{name}/venv-%{pythonversion}
%global src_dir %{_sourcedir}/%{name}
%global wheels %{src_dir}/wheels
%global package_executable @PIP_NAME@
%global globus_python3_version 3.9
%global __python /opt/globus-python/bin/python3
Expand Down Expand Up @@ -33,6 +32,8 @@ Requires(pre): /usr/sbin/useradd, /usr/bin/getent
Requires(postun): /usr/sbin/userdel

Group: System Environment/Daemons
Source: @PACKAGE_TARBALL_NAME@
Source1: @PREREQS_TARBALL_NAME@

License: Apache 2.0
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}
Expand All @@ -48,24 +49,20 @@ want to install the `@PIP_NAME@` directly from PyPI.
%prep
%global DEST_VIRTUAL_ENV "$PWD/TMP_RPM_BUILD_ROOT%{VIRTUAL_ENV}"

[ -f %{wheels}/@PACKAGE_WHEEL@ ] || {
C="" R=""
[ -t 2 ] && { C="\033[91;1;40m"; R="\033[m"; }
>&2 echo -e "${C}Main source wheel not found. Reminder: the Makefile sets that up. (Hint: make rpm)${R}";
exit 1
}
%setup -n "@PACKAGE_SOURCE_DIR@"
%setup -n "@PREREQS_DIR@" -T -D -b 1

rm -rf "%{DEST_VIRTUAL_ENV}"
%__python -mvenv "%{DEST_VIRTUAL_ENV}"

%build
. "%{DEST_VIRTUAL_ENV}/bin/activate"
python3 -mpip install --no-index --no-cache-dir -I --compile -U %{wheels}/pip-*.whl
python3 -mpip install --no-index --no-cache-dir -I --compile -U pip-*.whl
python3 -mpip install --pre --compile --no-index --no-cache-dir -I --find-links=./ "@PIP_NAME@"
deactivate

%install
. "%{DEST_VIRTUAL_ENV}/bin/activate"
python3 -mpip install --pre --compile --no-index --no-cache-dir -I --find-links=file://%{wheels} "@PIP_NAME@"

tar -C "${PWD}/TMP_RPM_BUILD_ROOT" -cf - . | tar -C "${RPM_BUILD_ROOT}" -xf -

Expand All @@ -84,7 +81,7 @@ sed -i "${RPM_BUILD_ROOT}%{VIRTUAL_ENV}/bin/activate" \
-e "s|^VIRTUAL_ENV=.*|VIRTUAL_ENV=%{VIRTUAL_ENV}|"

install -d -m 755 ${RPM_BUILD_ROOT}%{_sbindir}
install -m 755 %{src_dir}/package_shim.sh "${RPM_BUILD_ROOT}%{_sbindir}/%{package_executable}"
install -m 755 "%{_builddir}/@PACKAGE_SOURCE_DIR@"/package_shim.sh "${RPM_BUILD_ROOT}%{_sbindir}/%{package_executable}"
sed -i "${RPM_BUILD_ROOT}%{_sbindir}/%{package_executable}" -e "s|@VIRTUAL_ENV@|%{VIRTUAL_ENV}|"

%pre
Expand Down

0 comments on commit cd689c6

Please sign in to comment.