From ccd02e867c8d4c3a306d65e743004283dff4b4d4 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Wed, 29 May 2024 22:04:40 +0200 Subject: [PATCH 1/4] allow to pass extra ssh-opts for upload --- cmsBuild | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/cmsBuild b/cmsBuild index 9dc9060..79c766a 100755 --- a/cmsBuild +++ b/cmsBuild @@ -68,6 +68,12 @@ def fatal(message): def die(message): fatal(message) +def remoteRsync(opts): + return format("rsync -e 'ssh -o StrictHostKeyChecking=no %(sshOpts)s -p %(port)s' --rsync-path=/usr/bin/rsync --chmod=a+rX", + sshOpts = opts.sshOptions, + port = opts.uploadPort) + + def cmsdist_file(name, ext, options, pkgname): fname = 'spec' if ext == '.spec' else name + ext s = join(pkgname, fname) @@ -3218,6 +3224,10 @@ def parseOptions(): metavar="<0-65535>", help="The port on which the repository sshd is listening to (default 22).", default="22") + uploadGroup.add_option("--ssh-options", + dest="sshOptions", + help="Extra options to pass to ssh for uploading the newly build packages e.g. -J user@server", + default="") uploadGroup.add_option("--upload-root-directory", dest="uploadRootDirectory", metavar="PATH", @@ -3783,7 +3793,8 @@ def uploadStore(pkg, rpmfilename, scheduler): tmpStr = "%s-%s-%s" % (gethostname(), getpid(), getuser()) tmpFileName = join(opts.uploadRootDirectory, "store", "tmp", "%s.%s.tmp" % (uploadFile, sha256(tmpStr.encode()).hexdigest())) upload = format( - "scp -q -o StrictHostKeyChecking=no %(srcFile)s %(user)s@%(server)s:%(tmpFileName)s", + "%(rsync)s %(srcFile)s %(user)s@%(server)s:%(tmpFileName)s", + rsync=remoteRsync(opts), srcFile = join(abspath("RPMS/cache"), pkg.checksum, dirname(rpmfilename), uploadFile), user = opts.uploadUser, server = opts.uploadServer, @@ -4260,7 +4271,7 @@ def executeScript(opts, dumpedScript, local=True): if local: executionCmd = "sh -e %(debug)s %(filename)s" else: - executionCmd = "ssh < %(filename)s -q -o StrictHostKeyChecking=no -p %(cmdPort)s %(cmdUser)s@%(cmdServer)s sh -e %(debug)s" + executionCmd = "ssh < %(filename)s -q -o StrictHostKeyChecking=no -p %(cmdPort)s %(sshOpts)s %(cmdUser)s@%(cmdServer)s sh -e %(debug)s" # Run (or print-out) the script if not opts.pretend: fd, filename = mkstemp(".sh", "cmsBuildExecuted", opts.tempDirPrefix, True) @@ -4275,6 +4286,7 @@ def executeScript(opts, dumpedScript, local=True): filename=filename, cmdServer=opts.uploadServer, cmdUser=opts.uploadUser, + sshOpts=opts.sshOptions, cmdPort=opts.uploadPort) err, out = getstatusoutput(script) if opts.debug: log(out) @@ -4539,8 +4551,8 @@ def upload(opts, args, factory): repoInfo = {"uploadDir": uploadDir, "repository": opts.repository, - "rsync": format("rsync -e 'ssh -o StrictHostKeyChecking=no -p %(port)s' --rsync-path=/usr/bin/rsync --chmod=a+rX", - port=opts.uploadPort), + "rsync": format("%(rsync) --rsync-path=/usr/bin/rsync --chmod=a+rX", + rsync=remoteRsync(opts)), "server": opts.uploadServer, "user": opts.uploadUser} From ffb81d8632a2792b9528da83eb77edf6d9e587a2 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Wed, 29 May 2024 23:43:46 +0200 Subject: [PATCH 2/4] allow to take packages needed for built from system e.g. make,cmake,autoconf --- cmsBuild | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/cmsBuild b/cmsBuild index 79c766a..45d911f 100755 --- a/cmsBuild +++ b/cmsBuild @@ -74,6 +74,10 @@ def remoteRsync(opts): port = opts.uploadPort) +def getNonSystemPackages(pkgs): + return [p for p in pkgs if p not in opts.useSystemTools] + + def cmsdist_file(name, ext, options, pkgname): fname = 'spec' if ext == '.spec' else name + ext s = join(pkgname, fname) @@ -2363,8 +2367,8 @@ class Package(object): else: self.versionSuffix = not parseNoVersionSuffix(self.spec) self.noAutoDependency = parseNoAutoDependencyLine(self.spec) - self.installDependencies = parseInstallDependencies(self.spec) - self.uploadDependencies = parseUploadDependencies(self.spec) + self.installDependencies = getNonSystemPackages(parseInstallDependencies(self.spec)) + self.uploadDependencies = getNonSystemPackagesparseUploadDependencies(self.spec)) self.buildLogsToKeep = parseNoDeleteBuildLogs(self.spec) self.buildRequireToolfile = parseBuildRequireToolfileLine(self.spec) self.fullDependencies += self.dependencies @@ -2982,6 +2986,8 @@ class Package(object): for source in localSources]) patches.extend([cmsdist_file(patch, '.patch', self.options, self.name) for patch in localPatches]) + deps = getNonSystemPackages(deps) + buildDeps = getNonSystemPackages(buildDeps) PKGFactory.getRequiresCache()[self.name] = (deps, sources, patches, buildDeps, sourcesNumbers) return (deps, sources, patches, buildDeps, sourcesNumbers) @@ -3095,6 +3101,10 @@ def parseOptions(): action="store_true", help="""Use the system compiler rather than the one specified in CMSDIST""", default=False) + advancedBuildOptions.add_option("--use-system-tools", + dest="useSystemTools", + default="", + help="Comma separated list of package names which should be taken from system e.g. make,cmake,autoconf") advancedBuildOptions.add_option("--ignore-compile-errors", "-k", dest="ignoreCompileErrors", @@ -3280,6 +3290,7 @@ def parseOptions(): parser.add_option_group(uploadGroup) parser.add_option_group(debugGroup) opts, args = parser.parse_args(None, None) + opts.useSystemTools = opts.useSystemTools.split(",") if opts.enableMonitor and not run_monitor_on_command: opts.enableMonitor = False log ("Warning: Disabling monitoring. Failed to import run_monitor_on_command from monitor_build.") @@ -3338,6 +3349,7 @@ def parseOptions(): if len(args) < 2: parser.error("Please build or upload action. Use --help to see additional options.") + args = args[0] + [ for a in args if a not in opts.useSystemTools] opts.installDir = opts.workDir if (opts.repository == "cms") or opts.repository.startswith("cms."): opts.installDir = "/opt/cmssw" @@ -3365,6 +3377,10 @@ def parseOptions(): if opts.localSources and (not opts.forceUpload) and ('upload' in args): fatal("'upload' option can not be used together with '--source' flag.") + args = args[0] + getNonSystemPackages(args[1:]) + if len(args) < 2: + fatal("No non-system package build provided via command-line") + return opts, args From 0d11ebe5a69a06e3b8e9b9443eed2ede68e8b491 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Thu, 30 May 2024 00:34:41 +0200 Subject: [PATCH 3/4] typos --- cmsBuild | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/cmsBuild b/cmsBuild index 45d911f..eb932f4 100755 --- a/cmsBuild +++ b/cmsBuild @@ -74,7 +74,7 @@ def remoteRsync(opts): port = opts.uploadPort) -def getNonSystemPackages(pkgs): +def getNonSystemPackages(pkgs, opts): return [p for p in pkgs if p not in opts.useSystemTools] @@ -2367,8 +2367,8 @@ class Package(object): else: self.versionSuffix = not parseNoVersionSuffix(self.spec) self.noAutoDependency = parseNoAutoDependencyLine(self.spec) - self.installDependencies = getNonSystemPackages(parseInstallDependencies(self.spec)) - self.uploadDependencies = getNonSystemPackagesparseUploadDependencies(self.spec)) + self.installDependencies = getNonSystemPackages(parseInstallDependencies(self.spec), self.options) + self.uploadDependencies = getNonSystemPackages(parseUploadDependencies(self.spec), self.options) self.buildLogsToKeep = parseNoDeleteBuildLogs(self.spec) self.buildRequireToolfile = parseBuildRequireToolfileLine(self.spec) self.fullDependencies += self.dependencies @@ -2986,8 +2986,8 @@ class Package(object): for source in localSources]) patches.extend([cmsdist_file(patch, '.patch', self.options, self.name) for patch in localPatches]) - deps = getNonSystemPackages(deps) - buildDeps = getNonSystemPackages(buildDeps) + deps = getNonSystemPackages(deps, self.options) + buildDeps = getNonSystemPackages(buildDeps, self.options) PKGFactory.getRequiresCache()[self.name] = (deps, sources, patches, buildDeps, sourcesNumbers) return (deps, sources, patches, buildDeps, sourcesNumbers) @@ -3349,7 +3349,6 @@ def parseOptions(): if len(args) < 2: parser.error("Please build or upload action. Use --help to see additional options.") - args = args[0] + [ for a in args if a not in opts.useSystemTools] opts.installDir = opts.workDir if (opts.repository == "cms") or opts.repository.startswith("cms."): opts.installDir = "/opt/cmssw" @@ -3377,7 +3376,7 @@ def parseOptions(): if opts.localSources and (not opts.forceUpload) and ('upload' in args): fatal("'upload' option can not be used together with '--source' flag.") - args = args[0] + getNonSystemPackages(args[1:]) + args = [args[0]] + getNonSystemPackages(args[1:], opts) if len(args) < 2: fatal("No non-system package build provided via command-line") @@ -4567,8 +4566,7 @@ def upload(opts, args, factory): repoInfo = {"uploadDir": uploadDir, "repository": opts.repository, - "rsync": format("%(rsync) --rsync-path=/usr/bin/rsync --chmod=a+rX", - rsync=remoteRsync(opts)), + "rsync": remoteRsync(opts), "server": opts.uploadServer, "user": opts.uploadUser} From c2677a93972d257078ce8cac9d1caae63df020ae Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Thu, 30 May 2024 07:36:10 +0200 Subject: [PATCH 4/4] define rpm macros for each system package --- cmsBuild | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cmsBuild b/cmsBuild index eb932f4..8031d20 100755 --- a/cmsBuild +++ b/cmsBuild @@ -75,7 +75,7 @@ def remoteRsync(opts): def getNonSystemPackages(pkgs, opts): - return [p for p in pkgs if p not in opts.useSystemTools] + return [p for p in pkgs if p not in opts.useSystemTools] if opts.useSystemTools else pkgs def cmsdist_file(name, ext, options, pkgname): @@ -3290,7 +3290,12 @@ def parseOptions(): parser.add_option_group(uploadGroup) parser.add_option_group(debugGroup) opts, args = parser.parse_args(None, None) - opts.useSystemTools = opts.useSystemTools.split(",") + opts.useSystemTools = set(opts.useSystemTools.split(",")) + + if "gcc" in opts.useSystemTools: + opts.systemCompiler = True + opts.useSystemTools.remove("gcc") + if opts.enableMonitor and not run_monitor_on_command: opts.enableMonitor = False log ("Warning: Disabling monitoring. Failed to import run_monitor_on_command from monitor_build.") @@ -4825,6 +4830,8 @@ if __name__ == "__main__": opts.rpmQueryDefines = '--define "cmsdist_directory %s"' % opts.cmsdist if opts.systemCompiler: opts.rpmQueryDefines += ' --define "use_system_gcc 1"' + for pkg in opts.useSystemTools: + opts.rpmQueryDefines += ' --define "use_system_%s 1"' % pkg.replace("-","_") opts.rpmQueryDefines += ' --define "compilerv %s" --define "cmscompilerv %s" --define "cmsos %s"' % ( opts.compilerVersion.replace(".",""), arch_data[2],