diff --git a/compute_endpoint/packaging/JenkinsFile b/compute_endpoint/packaging/JenkinsFile new file mode 100644 index 000000000..f6165abd6 --- /dev/null +++ b/compute_endpoint/packaging/JenkinsFile @@ -0,0 +1,75 @@ +#! groovy + +@Library('gcs-build-scripts') _ + +pipeline { + agent none + options { + buildDiscarder( + logRotator( + numToKeepStr: '5', + artifactNumToKeepStr: '1', + artifactDaysToKeepStr: '7' + ) + ) + timeout(time: 3, unit: 'HOURS') + disableConcurrentBuilds() + } + stages { + stage ("Prep source") { + agent {label "package_creator"} + steps { + checkout scm + script { + env.SOURCE_STASH_NAME = "${UUID.randomUUID()}"; + sh "git clean -fdx" + sh "git checkout kevin_package_test" + + dir("compute_endpoint/packaging/") { + sh(script: "make show_vars") + sh(script: "make setup_dist_for_deb setup_dist_for_rpm") + sh "mv dist/ ${env.SOURCE_STASH_NAME}" + stash(name: env.SOURCE_STASH_NAME, includes: "${env.SOURCE_STASH_NAME}/**/*") + } + + def venv_py = "${env.WORKSPACE}/compute_endpoint/packaging/venv/bin/python" + def py_full_version = sh(script: "'${venv_py}' -c 'import sys; print(\"{}.{}\".format(*sys.version_info))'", returnStdout: true).trim() + def (py_epoch_version, py_major_version) = py_full_version.tokenize(".") + def pkg_version = "not yet set" + + dir("compute_endpoint/") { + pkg_version = sh(script: "${venv_py} setup.py --version", returnStdout: true).trim().replace("-", "~") + echo "pkg_version = ${pkg_version}" + } + env.PKG_TARBALL = "globus_compute_endpoint-${pkg_version.replace("~", "")}.tar.gz" + env.PREREQS_TARBALL = "globus_compute_endpoint-prereqs-py${py_epoch_version}${py_major_version}-${pkg_version}.tar.gz" + } + } + } + stage ("Build packages") { + steps { + script { + lock(resource: 'globus-compute-agent-build') { + parallel "debian": { + def extra_tarball_map = [ + prereqs: env.PREREQS_TARBALL + ] + env.DEB_ARTIFACTS_STASH = buildDebian( + env.SOURCE_STASH_NAME, + env.PKG_TARBALL, + require_gcs5_repo: true, + extra_tarball_map: extra_tarball_map + ) + }, "rpm": { + env.RPM_ARTIFACTS_STASH = buildMock( + env.SOURCE_STASH_NAME, + env.PKG_TARBALL, + true + ) + }, "failFast": false + } + } + } + } + } +} diff --git a/compute_endpoint/packaging/Makefile b/compute_endpoint/packaging/Makefile index ff842d4ab..ef6484fa7 100644 --- a/compute_endpoint/packaging/Makefile +++ b/compute_endpoint/packaging/Makefile @@ -16,7 +16,7 @@ VENV_PY := $(shell pwd)/$(VIRTUALENV)/bin/python # This if conditional before the PKG_ variables, so as to ensure we have setuptools; # not so ideal, but at the moment, I don't know how to do better. Hmm. ifeq ($(wildcard $(VENV_PY)),) - _DUMMY := $(shell "$(PYTHON3)" -mvenv "$(VIRTUALENV)") + _DUMMY := $(shell "$(PYTHON3)" -mvenv "$(VIRTUALENV)"; ln -sf "$(VIRTUALENV)" venv) _DUMMY := $(shell "$(VENV_PY)" -mpip install -U pip -U setuptools) endif @@ -26,8 +26,11 @@ PIP_NAME_D := $(shell cd ../; "$(VENV_PY)" setup.py --name) PIP_NAME_U := $(shell echo $(PIP_NAME_D) | tr '-' '_') PKG_VERSION := $(shell cd ../; "$(VENV_PY)" setup.py --version | tr '-' '~') -PKG_WHEEL = $(PIP_NAME_U)-$(PKG_VERSION)-py$(PY_MAJOR_VERSION)-none-any.whl -PREREQS_TARBALL_NAME = $(PIP_NAME_U)-prereqs-py$(PY_VERSION)-$(PKG_VERSION).tar.xz +PKG_WHEEL := $(PIP_NAME_U)-$(PKG_VERSION)-py$(PY_MAJOR_VERSION)-none-any.whl +PKG_SOURCE_DIR := $(PIP_NAME_U)-$(PKG_VERSION) +PKG_TARBALL := $(PKG_SOURCE_DIR).tar.gz +PREREQS_DIR := $(PIP_NAME_U)-prereqs-py$(PY_VERSION)-$(PKG_VERSION) +PREREQS_TARBALL_NAME = $(PREREQS_DIR).tar.gz OS_CODENAME := $(shell test -f /etc/os-release && . /etc/os-release; echo $${VERSION_CODENAME:-focal}) @@ -62,7 +65,10 @@ show_vars: ##-For debugging, show the Makefile variables; will install a venv @echo "PIP_NAME_U : $(PIP_NAME_U)" @echo "PKG_NAME : $(PKG_NAME)" @echo "PKG_VERSION : $(PKG_VERSION)" + @echo "PREREQS_DIR : $(PREREQS_DIR)" @echo "PREREQS_TARBALL_NAME : $(PREREQS_TARBALL_NAME)" + @echo "PKG_SOURCE_DIR : $(PKG_SOURCE_DIR)" + @echo "PKG_TARBALL : $(PKG_TARBALL)" @echo "PKG_WHEEL : $(PKG_WHEEL)" @echo @echo " Override python path with PYTHON3 variable:" @@ -70,7 +76,7 @@ show_vars: ##-For debugging, show the Makefile variables; will install a venv .PHONY: clean clean: ##-Remove the venv, build/ dist/, prereqs tarball, and the package wheel - rm -rf -- "$(VIRTUALENV)" build/ dist/ "$(PREREQS_TARBALL_NAME)" "$(PKG_WHEEL)" + rm -rf -- "$(VIRTUALENV)" venv build/ dist/ "$(PREREQS_TARBALL_NAME)" "$(PKG_WHEEL)" .PHONY: distclean distclean: clean ##-Run `clean` target, then additionally remove venv-*, *tar.xz, *whl @@ -95,19 +101,20 @@ $(VENV_PY): _build_needs exit 1; \ fi "$(PYTHON3)" -mvenv "$(VIRTUALENV)" - . "$(VIRTUALENV)/bin/activate" + ln -sf "$(VIRTUALENV)" venv + . venv/bin/activate @"$(VENV_PY)" -m pip install -U pip -U setuptools -U build $(PKG_WHEEL): $(VENV_PY) ( rm -rf build/ \ && git clone ../../ build/ \ && cd build/compute_endpoint/ \ - && echo -n " Git Tag: " \ - && if ! git describe --tags --exact-match; then \ - { /bin/echo -e "\nBUILD COMPUTE FROM A RELEASE TAG (current branch: $$(git branch --show-current))"; exit 1; } \ + && if git describe --tags --exact-match 2> /dev/null; \ + then { echo " Git Tag: $$(git describe --tags --exact-match)"; }; \ + else { echo " Git Branch: $$(git rev-parse HEAD) ($$(git describe --tags))"; }; \ fi \ && rm -rf tests/ \ - && "$(VENV_PY)" -m build --wheel -o ../../ \ + && "$(VENV_PY)" -m build -o ../../ \ ) wheel: $(PKG_WHEEL) ##-Make the wheel (note that this does *not* include dependencies) @@ -117,24 +124,32 @@ $(PREREQS_TARBALL_NAME): $(VENV_PY) $(PKG_WHEEL) prereq_tarball: $(PREREQS_TARBALL_NAME) ##-Make a tarball of wheel dependencies -dist: $(PREREQS_TARBALL_NAME) ##-Make the dist/ directory with prereqs and wheel ready for packaging step - rm -rf dist/ \ - && mkdir dist/ \ - && cp $(PREREQS_TARBALL_NAME) $(PKG_WHEEL) dist/ +_dist: + rm -rf dist/ && mkdir dist/ + cp "$(PKG_WHEEL)" dist/ + + # For CI, which does things differently than this Makefile: + # - append PKG_WHEEL to *prereqs* tarball + # - append package_shim.sh to *package* tarball + ( _prereq_dir="$$(tar -tf "$(PREREQS_TARBALL_NAME)" | head -1)" \ + && _pkg_dir="$$(tar -tf "$(PKG_TARBALL)" | head -1)" \ + && cd dist/ \ + && tar -xf "../$(PREREQS_TARBALL_NAME)" \ + && cp "$(PKG_WHEEL)" "$${_prereq_dir}" \ + && tar -czf "$(PREREQS_TARBALL_NAME)" "$${_prereq_dir}" \ + && rm -r "$${_prereq_dir}" \ + && tar -xf "../$(PKG_TARBALL)" \ + && cp "../package_shim.sh" "$${_pkg_dir}" \ + && tar -czf "$(PKG_TARBALL)" "$${_pkg_dir}" \ + && rm -r "$${_pkg_dir}" \ + ) -deb_build_needs: ##-Check that necessary executables are available before starting the DEB build. - @[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; } - @dpkg-checkbuilddeps +dist: $(PREREQS_TARBALL_NAME) _dist ##-Make the dist/ directory with prereqs and wheel ready for packaging step + rm "$(PKG_WHEEL)" "$(PKG_TARBALL)" "$(PREREQS_TARBALL_NAME)" -deb: deb_build_needs dist ##-Build a Debian package of the Globus Compute Endpoint (.deb) +_setup_dist_for_deb: ( cd dist/ \ - && rm -rf debbuild/ \ - && mkdir -p debbuild/$(PKG_NAME)/wheels/ \ - && tar -C debbuild/$(PKG_NAME)/wheels/ -xf "$(PREREQS_TARBALL_NAME)" \ - && cp $(PKG_WHEEL) debbuild/$(PKG_NAME)/wheels/ \ - && cp -R ../debian debbuild/$(PKG_NAME)/ \ - && cp ../package_shim.sh debbuild/$(PKG_NAME)/ \ - && cd debbuild/$(PKG_NAME)/ \ + && cp -R ../debian ./ \ && mv debian/changelog.in.in debian/changelog \ && sed -i debian/changelog \ -e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \ @@ -148,29 +163,59 @@ deb: deb_build_needs dist ##-Build a Debian package of the Globus Compute Endpo -e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \ -e "s/@PACKAGE_VERSION@/$(PKG_VERSION)/g" \ -e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \ + ) + +setup_dist_for_deb: dist _setup_dist_for_deb ##-Place Debian-build required files in dist/ + +deb_build_needs: ##-Check that necessary executables are available before starting the DEB build. + @[ -x "$$(command -v dpkg-checkbuilddeps)" ] || { echo "'dpkg-checkbuilddeps' not found; missing 'dpkg-dev' package?"; exit 1; } + @dpkg-checkbuilddeps + +_deb: + ( cd dist/ \ + && rm -rf debbuild/ \ + && mkdir -p debbuild/$(PKG_NAME)/prereqs/ \ + && cp -R debian debbuild/$(PKG_NAME)/ \ + && tar -C debbuild/$(PKG_NAME)/prereqs/ --strip 1 -xf "$(PREREQS_TARBALL_NAME)" \ + && cp $(PKG_WHEEL) debbuild/$(PKG_NAME)/prereqs/ \ + && cp ../package_shim.sh debbuild/$(PKG_NAME)/ \ + && cd debbuild/$(PKG_NAME)/ \ && dpkg-buildpackage -uc -us \ ) @echo "\nDEB package successfully built:" @ls -lh dist/debbuild/*deb -rpm_build_needs: ##-Check that necessary executables are available before starting the RPM build. - @[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; } +# Reminder: deb rule is not invoked by CI; this is for one-off needs +deb: deb_build_needs setup_dist_for_deb _deb ##-Build a Debian package of the Globus Compute Endpoint (.deb) -rpm: rpm_build_needs dist ##-Build an RPM package of the Globus Compute Endpoint (.rpm) +_setup_dist_for_rpm: ( cd dist/ \ - && pwd && ls \ - && rm -rf rpmbuild/ \ - && mkdir -p rpmbuild/{BUILD,BUILDROOT,RPMS,SPECS,SRPMS} rpmbuild/SOURCES/$(PKG_NAME)/wheels \ - && tar -C rpmbuild/SOURCES/$(PKG_NAME)/wheels -xf "$(PREREQS_TARBALL_NAME)" \ - && cp $(PKG_WHEEL) rpmbuild/SOURCES/$(PKG_NAME)/wheels/ \ - && cp ../package_shim.sh rpmbuild/SOURCES/$(PKG_NAME)/ \ && sed \ -e "s/@PACKAGE_VERSION@/$(PKG_VERSION)/g" \ -e "s/@PACKAGE_NAME@/$(PKG_NAME)/g" \ -e "s/@PACKAGE_WHEEL@/$(PKG_WHEEL)/g" \ + -e "s/@PACKAGE_TARBALL_NAME@/$(PKG_TARBALL)/g" \ + -e "s/@PREREQS_TARBALL_NAME@/$(PREREQS_TARBALL_NAME)/g" \ -e "s/@PIP_NAME@/$(PIP_NAME_D)/g" \ + -e "s/@PACKAGE_SOURCE_DIR@/$(PKG_SOURCE_DIR)/" \ + -e "s/@PREREQS_DIR@/$(PREREQS_DIR)/" \ < ../fedora/$(PKG_NAME).spec.in > ./$(PKG_NAME).spec \ + ) + +setup_dist_for_rpm: dist _setup_dist_for_rpm ##-Place RPM-build required files in dist/ (separated for CI building purposes) + +rpm_build_needs: ##-Check that necessary executables are available before starting the RPM build. + @[ -x "$$(command -v rpmbuild)" ] || { echo "'rpmbuild' not found; missing 'rpmdevtools' or 'rpm-build' package(s)?"; exit 1; } + +# Reminder: rpm rule is not invoked by CI; this is for one-off needs +_rpm: + ( cd dist/ \ + && rm -rf rpmbuild/ \ + && mkdir -p rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS}/ \ + && cp *.tar.gz rpmbuild/SOURCES/ \ && HOME="$$(pwd)" rpmbuild --define "_topdir $$(pwd)/rpmbuild" -ba ./$(PKG_NAME).spec \ ) @echo -e "\nRPM package successfully built:" @ls -lh dist/rpmbuild/RPMS/**/*rpm + +rpm: rpm_build_needs setup_dist_for_rpm _rpm ##-Build an RPM package of the Globus Compute Endpoint (.rpm) diff --git a/compute_endpoint/packaging/create-prereqs-tarball.sh b/compute_endpoint/packaging/create-prereqs-tarball.sh index 4fccbd594..402f20ad2 100755 --- a/compute_endpoint/packaging/create-prereqs-tarball.sh +++ b/compute_endpoint/packaging/create-prereqs-tarball.sh @@ -90,12 +90,12 @@ done > dependent-prereqs.txt modified_time="$(TZ=UTC0 date --rfc-3339=seconds)" -TZ=UTC0 tar -C "$download_dir/" --format=posix \ +TZ=UTC0 tar --format=posix \ --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime,delete=mtime \ --mtime="$modified_time" \ --numeric-owner \ --owner=0 \ --group=0 \ --mode="go-rwx,u+rw" \ - -cf - . \ - | xz -9 + -cf - "$download_dir/" \ + | gzip -9 diff --git a/compute_endpoint/packaging/debian/changelog.in.in b/compute_endpoint/packaging/debian/changelog.in.in index b08e94b34..7e3214c07 100644 --- a/compute_endpoint/packaging/debian/changelog.in.in +++ b/compute_endpoint/packaging/debian/changelog.in.in @@ -1,8 +1,8 @@ -@PACKAGE_NAME@ (@PACKAGE_VERSION@.@distro@) @distro@; urgency=low +@PACKAGE_NAME@ (@PACKAGE_VERSION@-1.@distro@) @distro@; urgency=low * Initial release as a Debian package * Includes support for multi-user Globus Compute Endpoints; see https://globus-compute.readthedocs.io/ for more information. - -- Maintainer: Globus Support Mon, 08 Apr 2024 11:01:30 -0500 + -- Maintainer: Globus Toolkit Mon, 08 Apr 2024 11:01:30 -0500 diff --git a/compute_endpoint/packaging/debian/control b/compute_endpoint/packaging/debian/control index 642c7fa1d..145fc418a 100644 --- a/compute_endpoint/packaging/debian/control +++ b/compute_endpoint/packaging/debian/control @@ -1,8 +1,8 @@ Source: @PACKAGE_NAME@ Priority: optional Section: contrib/science -Maintainer: Globus Support -Build-Depends: debhelper (>= 13.6), globus-python (>= 3.9), dh-exec, dh-python +Maintainer: Globus Toolkit +Build-Depends: debhelper (>= 12), globus-python (>= 3.9), dh-exec, dh-python Standards-Version: 4.6.0 Homepage: https://globus-compute.readthedocs.io/ diff --git a/compute_endpoint/packaging/debian/rules b/compute_endpoint/packaging/debian/rules index 5391a1679..fcce70efb 100755 --- a/compute_endpoint/packaging/debian/rules +++ b/compute_endpoint/packaging/debian/rules @@ -30,10 +30,13 @@ _unitdir=/lib/systemd/system .PHONY: override_dh_auto_configure override_dh_auto_configure: $(PYTHON3) -mvenv $${PWD}/$(TMP_VIRTUAL_ENV) - . "$(TMP_VIRTUAL_ENV)/bin/activate"; \ + . "$${PWD}/$(TMP_VIRTUAL_ENV)/bin/activate"; \ set -x; \ - python3 -mpip install --no-index --no-cache-dir -I --compile -U wheels/pip-*.whl; \ - python3 -mpip install --no-index --no-cache-dir -I --pre --compile --find-links=file://$${PWD}/wheels/ "@PIP_NAME@" + pwd; \ + ls -lR; \ + pwd; \ + python -mpip install --no-index --no-cache-dir -I --compile -U prereqs/pip-*.whl; \ + python -mpip install --no-index --no-cache-dir -I --pre --compile --find-links=file://$${PWD}/prereqs/ "@PIP_NAME@" .PHONY: override_dh_auto_clean override_dh_auto_clean: @@ -78,5 +81,8 @@ override_dh_python3: override_dh_builddeb: dh_builddeb -- -Zxz +.PHONY: override_dh_auto_build +override_dh_auto_build: # empty == we do it ourselves in auto_install (above) + %: dh $@ --with python3 --with systemd diff --git a/compute_endpoint/packaging/fedora/globus-compute-agent.spec.in b/compute_endpoint/packaging/fedora/globus-compute-agent.spec.in index 0d9837c77..2c078c4ea 100644 --- a/compute_endpoint/packaging/fedora/globus-compute-agent.spec.in +++ b/compute_endpoint/packaging/fedora/globus-compute-agent.spec.in @@ -5,7 +5,6 @@ Name: @PACKAGE_NAME@ %global pythonversion py39 %global VIRTUAL_ENV /opt/%{name}/venv-%{pythonversion} %global src_dir %{_sourcedir}/%{name} -%global wheels %{src_dir}/wheels %global package_executable @PIP_NAME@ %global globus_python3_version 3.9 %global __python /opt/globus-python/bin/python3 @@ -33,6 +32,8 @@ Requires(pre): /usr/sbin/useradd, /usr/bin/getent Requires(postun): /usr/sbin/userdel Group: System Environment/Daemons +Source: @PACKAGE_TARBALL_NAME@ +Source1: @PREREQS_TARBALL_NAME@ License: Apache 2.0 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release} @@ -48,24 +49,20 @@ want to install the `@PIP_NAME@` directly from PyPI. %prep %global DEST_VIRTUAL_ENV "$PWD/TMP_RPM_BUILD_ROOT%{VIRTUAL_ENV}" -[ -f %{wheels}/@PACKAGE_WHEEL@ ] || { - C="" R="" - [ -t 2 ] && { C="\033[91;1;40m"; R="\033[m"; } - >&2 echo -e "${C}Main source wheel not found. Reminder: the Makefile sets that up. (Hint: make rpm)${R}"; - exit 1 -} +%setup -n "@PACKAGE_SOURCE_DIR@" +%setup -n "@PREREQS_DIR@" -T -D -b 1 rm -rf "%{DEST_VIRTUAL_ENV}" %__python -mvenv "%{DEST_VIRTUAL_ENV}" %build . "%{DEST_VIRTUAL_ENV}/bin/activate" -python3 -mpip install --no-index --no-cache-dir -I --compile -U %{wheels}/pip-*.whl +python3 -mpip install --no-index --no-cache-dir -I --compile -U pip-*.whl +python3 -mpip install --pre --compile --no-index --no-cache-dir -I --find-links=./ "@PIP_NAME@" deactivate %install . "%{DEST_VIRTUAL_ENV}/bin/activate" -python3 -mpip install --pre --compile --no-index --no-cache-dir -I --find-links=file://%{wheels} "@PIP_NAME@" tar -C "${PWD}/TMP_RPM_BUILD_ROOT" -cf - . | tar -C "${RPM_BUILD_ROOT}" -xf - @@ -84,7 +81,7 @@ sed -i "${RPM_BUILD_ROOT}%{VIRTUAL_ENV}/bin/activate" \ -e "s|^VIRTUAL_ENV=.*|VIRTUAL_ENV=%{VIRTUAL_ENV}|" install -d -m 755 ${RPM_BUILD_ROOT}%{_sbindir} -install -m 755 %{src_dir}/package_shim.sh "${RPM_BUILD_ROOT}%{_sbindir}/%{package_executable}" +install -m 755 "%{_builddir}/@PACKAGE_SOURCE_DIR@"/package_shim.sh "${RPM_BUILD_ROOT}%{_sbindir}/%{package_executable}" sed -i "${RPM_BUILD_ROOT}%{_sbindir}/%{package_executable}" -e "s|@VIRTUAL_ENV@|%{VIRTUAL_ENV}|" %pre