Skip to content
This repository has been archived by the owner on Feb 23, 2023. It is now read-only.

Commit

Permalink
Update to gfal2.19 last RC and davix 0.7.6 devel (fixes segfault)
Browse files Browse the repository at this point in the history
  • Loading branch information
chaen committed Jan 29, 2021
1 parent 77a6b73 commit 67b3d6b
Show file tree
Hide file tree
Showing 6 changed files with 185 additions and 45 deletions.
14 changes: 8 additions & 6 deletions config/diracos.json
Original file line number Diff line number Diff line change
Expand Up @@ -472,14 +472,17 @@
"name": "davix",
"packages": [
{
"src": "https://diracos.web.cern.ch/diracos/SRPM/davix-0.7.5-1.el6.src.rpm",
"originalRepo": "https://dl.fedoraproject.org/pub/epel/6/SRPMS/Packages/",
"src": "https://diracos.web.cern.ch/diracos/SRPM/davix-0.7.6.200.3c48eab-1.el6.src.rpm",
"originalRepo": "https://storage-ci.web.cern.ch/storage-ci/davix/",
"name": "davix"
}
]
},
{
"name": "gfal2",
"RCRepos": "https://dmc-repo.web.cern.ch/dmc-repo/rc/",
"TestingRepos": "https://dmc-repo.web.cern.ch/dmc-repo/testing/",
"ProdRepos": "https://dmc-repo.web.cern.ch/dmc-repo/el6/x86_64/",
"packages": [
{
"src": "https://diracos.web.cern.ch/diracos/SRPM/epydoc-3.0.1-6.el6.src.rpm",
Expand Down Expand Up @@ -536,8 +539,7 @@
"buildOnly": true
},
{
"src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-2.17.1-1.el6_10.src.rpm",
"originalRepo": "https://dl.fedoraproject.org/pub/epel/6/SRPMS/Packages/",
"src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-2.19.0-1.el6.src.rpm",
"excludePatterns": [
".*-doc-.*",
".*-debuginfo-.*",
Expand All @@ -546,11 +548,11 @@
"name": "gfal2"
},
{
"src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-python-1.9.5-1.el6.src.rpm",
"src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-python-1.10.0-1.el6.src.rpm",
"name": "gfal2-python"
},
{
"src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-util-1.5.1-1.el6.src.rpm",
"src": "https://diracos.web.cern.ch/diracos/SRPM/gfal2-util-1.6.0-1.el6.src.rpm",
"name": "gfal2-util"
}
]
Expand Down
8 changes: 8 additions & 0 deletions docs/source/50_troubleshoot.rst
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,14 @@ prebuilt RPMs for an old ``golang`` version (split into
removed. Singularity can then be built from source as is the case for
other packages.

Davix (Cmake3)
--------------

Latest davix versions require ``CMake3``, which is available as RPM in the SLC6
environment in which we build. To solve this, we download in the ``davix`` routine
a binary of ``CMake3`` that we then inject in the ``mock`` environment. We may have
to do similar tricks for other packages in the future...

About manualDependencies
------------------------

Expand Down
75 changes: 50 additions & 25 deletions patches/davix.patch
Original file line number Diff line number Diff line change
@@ -1,36 +1,61 @@
Only in modified/: davix-0.7.6.200.3c48eab-1.el6.py27.usc4.src.rpm
diff -r -u original/davix.spec modified/davix.spec
--- original/davix.spec 2019-08-28 09:13:40.000000000 +0000
+++ modified/davix.spec 2020-02-13 14:16:43.513053440 +0000
@@ -22,10 +22,6 @@
--- original/davix.spec 2020-11-10 03:21:20.000000000 +0000
+++ modified/davix.spec 2021-01-28 08:09:13.207702045 +0000
@@ -1,5 +1,5 @@
%{!?_pkgdocdir: %global _pkgdocdir %{_docdir}/%{name}-%{version}}
-%global __cmake cmake3
+%global cmake /usr/bin/cmake

Name: davix
Version: 0.7.6.200.3c48eab
@@ -13,8 +13,6 @@
#main lib dependencies
BuildRequires: python2
BuildRequires: libuuid-devel
-BuildRequires: cmake
-BuildRequires: cmake3
BuildRequires: doxygen
BuildRequires: libxml2-devel
BuildRequires: openssl-devel
@@ -25,10 +23,6 @@
# davix-copy dependencies
BuildRequires: gsoap-devel
BuildRequires: libuuid-devel

-# unit tests and abi check
-%if %{?fedora}%{!?fedora:0} >= 10 || %{?rhel}%{!?rhel:0} >= 6
-BuildRequires: abi-compliance-checker
-%endif

Requires: %{name}-libs%{?_isa} = %{version}-%{release}
Requires: libuuid
@@ -83,18 +79,14 @@

@@ -66,9 +60,9 @@
with Http based protocols (WebDav, Amazon S3, ...).

%package doc
-Summary: Documentation for %{name}
+Summary: Documentation for %{name}
%if %{?fedora}%{!?fedora:0} >= 10 || %{?rhel}%{!?rhel:0} >= 6
-BuildArch: noarch
+BuildArch: noarch
%endif

%description doc
@@ -89,7 +83,7 @@
%cmake \
-DDOC_INSTALL_DIR=%{_pkgdocdir} \
-DENABLE_THIRD_PARTY_COPY=TRUE \
--DENABLE_HTML_DOCS=TRUE \
+-DCMAKE_INSTALL_PREFIX=/usr/ \
.
make %{?_smp_mflags}
make doc

-%check
-%if %{?fedora}%{!?fedora:0} >= 10 || %{?rhel}%{!?rhel:0} >= 6
-#make abi-check
-%endif
-ctest -V -T Test
-

%install
rm -rf %{buildroot}
make DESTDIR=%{buildroot} install

-%ldconfig_scriptlets libs
+%post libs -p /sbin/ldconfig
@@ -129,7 +123,7 @@
%files doc
%{_pkgdocdir}/LICENSE
%{_pkgdocdir}/RELEASE-NOTES.md
-%{_pkgdocdir}/html/
+
+%postun libs -p /sbin/ldconfig

%files
%{_bindir}/*

%files tests
%{_bindir}/davix-unit-tests
39 changes: 39 additions & 0 deletions patches/gfal2-util.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
--- original/gfal2-util.spec 2021-01-14 09:53:06.477321181 +0000
+++ modified/gfal2-util.spec 2021-01-14 09:51:10.169742718 +0000
@@ -14,7 +14,7 @@
%endif

%if %{with python2}
-%{!?python2_sitelib: %global python2_sitelib %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")}
+%global python2_sitelib /usr/lib/python2.7/site-packages
%endif

%if %{with python3}
@@ -83,13 +83,6 @@
python3 setup.py install --root=%{buildroot}
%endif

-%check
-%if %{with python2}
- python2 test/functional/test_all.py
-%endif
-%if %{with python3}
- python3 test/functional/test_all.py
-%endif

%clean
rm -rf %{buildroot}
@@ -117,12 +110,9 @@

BuildRequires: gfal2-python >= 1.9.0
BuildRequires: python2
-BuildRequires: python2-rpm-macros
-BuildRequires: python2-future
Requires: gfal2-python >= 1.9.0
Requires: gfal2-util-scripts = %{version}-%{release}
Requires: python2
-Requires: python2-future
%if %{?fedora}%{!?fedora:0} < 26 || %{?rhel}%{!?rhel:0} < 7
BuildRequires: python-argparse
Requires: python-argparse

14 changes: 0 additions & 14 deletions patches/gfal2.patch

This file was deleted.

80 changes: 80 additions & 0 deletions routines/davix.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import logging
import shutil
import subprocess
from diracos import diracoslib
import os
import glob
import tarfile
import tempfile
from yum import rpmUtils

# TODO: factorize with glite-ce-cream-cli.py, since it is essentialy the same thing

# Original URL: https://github.com/Kitware/CMake/releases/download/v3.19.3/cmake-3.19.3-Linux-x86_64.sh
CMAKE3_SLC6_INSTALLER = 'https://diracos.web.cern.ch/diracos/bootstrap/cmake-3.19.3-Linux-x86_64.sh'


def execute(**kwargs):
""" This requires also patching the tar.gz"""

logging.info("Executing xrootd routine")
logging.debug("Arguments %s", kwargs)

srpmFile = kwargs['src']
repository = kwargs['repo']
mockConfig = kwargs['mockConfig']
mockRoot = kwargs['mockRoot']
patchDir = kwargs['patchDir']
patchFile = None

workDir = kwargs.get('workDir', '/tmp')

logging.info("Building %s %s", srpmFile, "%s" % ("with mockConfig %s" % mockConfig
if mockConfig else ""))

if not os.path.isfile(srpmFile):
logging.debug("SRPM file is an URL, download it first")
srpmFile = diracoslib._downloadFile(srpmFile, '/tmp')

# get package name
pkgName, pkgVersion, _release, _epoch, _arch = rpmUtils.miscutils.splitFilename(
os.path.basename(srpmFile))

# If the src.rpm is already in the repo, do not rebuild it
#
existingBuild = glob.glob(os.path.join(repository, 'src', '%s-%s*' % (pkgName, pkgVersion)))
if existingBuild:
logging.info("The repo already contains a build, not rebuilding: %s", existingBuild)
return

if not patchFile and patchDir:

potentialPatchFile = os.path.join(patchDir, '%s.patch' % pkgName)
logging.debug("Checking existance of %s", potentialPatchFile)
if os.path.isfile(potentialPatchFile):
logging.debug("patch file found")
patchFile = potentialPatchFile

if patchFile:
newSRPM = diracoslib._mock_patchAndRecreateSRPM(srpmFile, patchFile, mockConfig, mockRoot=mockRoot)
logging.debug("New SRPM file %s", newSRPM)
srpmFile = newSRPM

# now comes the funky part !
# Davix decided to use cmake3, which is not easily available on SLC6 as RPM.
# Luckily for us, there is a binary distribution by cmake themselves which works fine.
# So we just extract it in the mock environment,
# and build the RPM without cleaning !

installer = diracoslib._downloadFile(CMAKE3_SLC6_INSTALLER, workDir)

mockUsrDir = os.path.join(mockRoot, 'root/usr/')
cmd = ['sh', installer, '--skip-license', '--prefix=%s' % mockUsrDir]
logging.info("Installing cmake3 in the mock environment %s", cmd)
subprocess.check_call(cmd)

diracoslib._mockRebuild(srpmFile, mockConfig, noClean=True)
mockResultDir = os.path.join(mockRoot, 'result/')
diracoslib._copyRPMs(mockResultDir, repository, byRPMType=True)
diracoslib._createRepo(repository)
logging.info('Finished')

0 comments on commit 67b3d6b

Please sign in to comment.