From 7a1bbcf32d77cdb2784ca162fe5c7f789d535d27 Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Fri, 17 Dec 2021 20:21:41 +0100 Subject: [PATCH 1/9] Add process.options.accelerators option, and ProcessAccelerator configuration customization concept ProcessAccelerator objects - specify the valid values of process.options.accelerators parameter - tell if the accelerator is available in the system (or which accelerators are) - allow customizing the process based on the availability of the accelerators --- .../Framework/test/test_module_delete_cfg.py | 4 +- FWCore/Integration/test/BuildFile.xml | 10 + .../Integration/test/ProcessAccelerator_t.cpp | 120 ++++++++ FWCore/Integration/test/SwitchProducer_t.cpp | 4 +- .../test/testSwitchProducerAliasOutput_cfg.py | 4 +- ...estSwitchProducerAliasToNonExistent_cfg.py | 4 +- .../test/testSwitchProducerPathFilter_cfg.py | 4 +- .../testSwitchProducerPathWrongOrder_cfg.py | 4 +- .../test/testSwitchProducerPath_cfg.py | 4 +- .../test/testSwitchProducerTaskInput_cfg.py | 4 +- .../test/testSwitchProducerTask_cfg.py | 4 +- FWCore/ParameterSet/python/Config.py | 258 +++++++++++++++++- FWCore/ParameterSet/python/Modules.py | 41 +-- .../src/validateTopLevelParameterSets.cc | 5 + 14 files changed, 427 insertions(+), 43 deletions(-) create mode 100644 FWCore/Integration/test/ProcessAccelerator_t.cpp diff --git a/FWCore/Framework/test/test_module_delete_cfg.py b/FWCore/Framework/test/test_module_delete_cfg.py index c8304da748b40..6f6f04890fae9 100644 --- a/FWCore/Framework/test/test_module_delete_cfg.py +++ b/FWCore/Framework/test/test_module_delete_cfg.py @@ -118,8 +118,8 @@ class SwitchProducerTest(cms.SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest,self).__init__( dict( - test1 = lambda: (True, -10), - test2 = lambda: (True, -9) + test1 = lambda accelerators: (True, -10), + test2 = lambda accelerators: (True, -9) ), **kargs) process.producerEventSwitchProducerNotConsumed = cms.EDProducer("edmtest::TestModuleDeleteProducer") process.producerEventSwitchProducerConsumed = intEventProducerMustRun.clone() diff --git a/FWCore/Integration/test/BuildFile.xml b/FWCore/Integration/test/BuildFile.xml index 45067f0c6b6ab..be4ec7a9b723e 100644 --- a/FWCore/Integration/test/BuildFile.xml +++ b/FWCore/Integration/test/BuildFile.xml @@ -177,6 +177,16 @@ + + + + + + + + + + diff --git a/FWCore/Integration/test/ProcessAccelerator_t.cpp b/FWCore/Integration/test/ProcessAccelerator_t.cpp new file mode 100644 index 0000000000000..0d289d98c2b24 --- /dev/null +++ b/FWCore/Integration/test/ProcessAccelerator_t.cpp @@ -0,0 +1,120 @@ +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +#include "DataFormats/TestObjects/interface/ToyProducts.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSetReader/interface/ParameterSetReader.h" +#include "FWCore/TestProcessor/interface/TestProcessor.h" + +#include + +#include +#include + +static constexpr auto s_tag = "[ProcessAccelerator]"; + +namespace { + std::string makeConfig(bool test2Enabled, + std::string_view test1, + std::string_view test2, + std::string_view accelerator) { + const std::string appendTest2 = test2Enabled ? "self._enabled.append('test2')" : ""; + return fmt::format( + R"_(from FWCore.TestProcessor.TestProcess import * +import FWCore.ParameterSet.Config as cms + +class ProcessAcceleratorTest(cms.ProcessAccelerator): + def __init__(self): + super(ProcessAcceleratorTest,self).__init__() + self._labels = ["test1", "test2"] + self._enabled = ["test1"] + {} + def labels(self): + return self._labels + def enabledLabels(self): + return self._enabled + +class SwitchProducerTest(cms.SwitchProducer): + def __init__(self, **kargs): + super(SwitchProducerTest,self).__init__( + dict( + test1 = lambda accelerators: ("test1" in accelerators, -10), + test2 = lambda accelerators: ("test2" in accelerators, -9), + ), **kargs) + +process = TestProcess() +process.options.accelerators = ["{}"] +process.ProcessAcceleratorTest = ProcessAcceleratorTest() +process.s = SwitchProducerTest( + test1 = {}, + test2 = {} +) +process.moduleToTest(process.s) +)_", + appendTest2, + accelerator, + test1, + test2); + } +} // namespace + +TEST_CASE("Configuration", s_tag) { + const std::string test1{"cms.EDProducer('IntProducer', ivalue = cms.int32(1))"}; + const std::string test2{"cms.EDProducer('ManyIntProducer', ivalue = cms.int32(2), values = cms.VPSet())"}; + + const std::string baseConfig_auto = makeConfig(true, test1, test2, "auto"); + const std::string baseConfig_test1 = makeConfig(true, test1, test2, "test1"); + const std::string baseConfig_test2 = makeConfig(true, test1, test2, "test2"); + const std::string baseConfigTest2Disabled_auto = makeConfig(false, test1, test2, "auto"); + const std::string baseConfigTest2Disabled_test1 = makeConfig(false, test1, test2, "test1"); + const std::string baseConfigTest2Disabled_test2 = makeConfig(false, test1, test2, "test2"); + + SECTION("Configuration hash is not changed") { + auto pset_auto = edm::readConfig(baseConfig_auto); + auto pset_test1 = edm::readConfig(baseConfig_test1); + auto pset_test2 = edm::readConfig(baseConfig_test2); + auto psetTest2Disabled_auto = edm::readConfig(baseConfigTest2Disabled_auto); + auto psetTest2Disabled_test1 = edm::readConfig(baseConfigTest2Disabled_test1); + auto psetTest2Disabled_test2 = edm::readConfig(baseConfigTest2Disabled_test2); + pset_auto->registerIt(); + pset_test1->registerIt(); + pset_test2->registerIt(); + psetTest2Disabled_auto->registerIt(); + psetTest2Disabled_test1->registerIt(); + psetTest2Disabled_test2->registerIt(); + REQUIRE(pset_auto->id() == pset_test1->id()); + REQUIRE(pset_auto->id() == pset_test2->id()); + REQUIRE(pset_auto->id() == psetTest2Disabled_auto->id()); + REQUIRE(pset_auto->id() == psetTest2Disabled_test1->id()); + REQUIRE(pset_auto->id() == psetTest2Disabled_test2->id()); + } + + edm::test::TestProcessor::Config config_auto{baseConfig_auto}; + edm::test::TestProcessor::Config config_test1{baseConfig_test1}; + edm::test::TestProcessor::Config config_test2{baseConfig_test2}; + edm::test::TestProcessor::Config configTest2Disabled_auto{baseConfigTest2Disabled_auto}; + edm::test::TestProcessor::Config configTest2Disabled_test1{baseConfigTest2Disabled_test1}; + edm::test::TestProcessor::Config configTest2Disabled_test2{baseConfigTest2Disabled_test2}; + + SECTION("Base configuration is OK") { REQUIRE_NOTHROW(edm::test::TestProcessor(config_auto)); } + + SECTION("No event data") { + edm::test::TestProcessor tester(config_auto); + REQUIRE_NOTHROW(tester.test()); + } + + SECTION("beginJob and endJob only") { + edm::test::TestProcessor tester(config_auto); + REQUIRE_NOTHROW(tester.testBeginAndEndJobOnly()); + } + + SECTION("Run with no LuminosityBlocks") { + edm::test::TestProcessor tester(config_auto); + REQUIRE_NOTHROW(tester.testRunWithNoLuminosityBlocks()); + } + + SECTION("LuminosityBlock with no Events") { + edm::test::TestProcessor tester(config_auto); + REQUIRE_NOTHROW(tester.testLuminosityBlockWithNoEvents()); + } +} diff --git a/FWCore/Integration/test/SwitchProducer_t.cpp b/FWCore/Integration/test/SwitchProducer_t.cpp index 174367b4481aa..28b2449c99630 100644 --- a/FWCore/Integration/test/SwitchProducer_t.cpp +++ b/FWCore/Integration/test/SwitchProducer_t.cpp @@ -31,8 +31,8 @@ class SwitchProducerTest(cms.SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest,self).__init__( dict( - test1 = lambda: (True, -10), - test2 = lambda: ()_"} + + test1 = lambda accelerators: (True, -10), + test2 = lambda accelerators: ()_"} + (test2Enabled ? "True" : "False") + ", -9)\n" + R"_( ), **kargs) process = TestProcess() diff --git a/FWCore/Integration/test/testSwitchProducerAliasOutput_cfg.py b/FWCore/Integration/test/testSwitchProducerAliasOutput_cfg.py index c092c87c76844..40b69e0cc56ed 100644 --- a/FWCore/Integration/test/testSwitchProducerAliasOutput_cfg.py +++ b/FWCore/Integration/test/testSwitchProducerAliasOutput_cfg.py @@ -4,8 +4,8 @@ class SwitchProducerTest(cms.SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest,self).__init__( dict( - test1 = lambda: (True, -10), - test2 = lambda: (True, -9) + test1 = lambda accelerators: (True, -10), + test2 = lambda accelerators: (True, -9) ), **kargs) process = cms.Process("PROD1") diff --git a/FWCore/Integration/test/testSwitchProducerAliasToNonExistent_cfg.py b/FWCore/Integration/test/testSwitchProducerAliasToNonExistent_cfg.py index 157d5ee86ddb0..dab2a000620ad 100644 --- a/FWCore/Integration/test/testSwitchProducerAliasToNonExistent_cfg.py +++ b/FWCore/Integration/test/testSwitchProducerAliasToNonExistent_cfg.py @@ -7,8 +7,8 @@ class SwitchProducerTest(cms.SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest,self).__init__( dict( - test1 = lambda: (True, -10), - test2 = lambda: (True, -9) + test1 = lambda accelerators: (True, -10), + test2 = lambda accelerators: (True, -9) ), **kargs) process = cms.Process("PROD1") diff --git a/FWCore/Integration/test/testSwitchProducerPathFilter_cfg.py b/FWCore/Integration/test/testSwitchProducerPathFilter_cfg.py index 847725a1e9f80..5655c90705be7 100644 --- a/FWCore/Integration/test/testSwitchProducerPathFilter_cfg.py +++ b/FWCore/Integration/test/testSwitchProducerPathFilter_cfg.py @@ -6,8 +6,8 @@ class SwitchProducerTest(cms.SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest,self).__init__( dict( - test1 = lambda: (True, -10), - test2 = lambda: (enableTest2, -9) + test1 = lambda accelerators: (True, -10), + test2 = lambda accelerators: (enableTest2, -9) ), **kargs) process = cms.Process("PROD1") diff --git a/FWCore/Integration/test/testSwitchProducerPathWrongOrder_cfg.py b/FWCore/Integration/test/testSwitchProducerPathWrongOrder_cfg.py index 3cbc55a6305df..a0e5d0a057aff 100644 --- a/FWCore/Integration/test/testSwitchProducerPathWrongOrder_cfg.py +++ b/FWCore/Integration/test/testSwitchProducerPathWrongOrder_cfg.py @@ -4,8 +4,8 @@ class SwitchProducerTest(cms.SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest,self).__init__( dict( - test1 = lambda: (True, -10), - test2 = lambda: (True, -9) + test1 = lambda accelerators: (True, -10), + test2 = lambda accelerators: (True, -9) ), **kargs) process = cms.Process("PROD1") diff --git a/FWCore/Integration/test/testSwitchProducerPath_cfg.py b/FWCore/Integration/test/testSwitchProducerPath_cfg.py index 0a6b228011afa..652b22f555c15 100644 --- a/FWCore/Integration/test/testSwitchProducerPath_cfg.py +++ b/FWCore/Integration/test/testSwitchProducerPath_cfg.py @@ -6,8 +6,8 @@ class SwitchProducerTest(cms.SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest,self).__init__( dict( - test1 = lambda: (True, -10), - test2 = lambda: (enableTest2, -9) + test1 = lambda accelerators: (True, -10), + test2 = lambda accelerators: (enableTest2, -9) ), **kargs) process = cms.Process("PROD1") diff --git a/FWCore/Integration/test/testSwitchProducerTaskInput_cfg.py b/FWCore/Integration/test/testSwitchProducerTaskInput_cfg.py index 063dafe656914..55bf6288bfb54 100644 --- a/FWCore/Integration/test/testSwitchProducerTaskInput_cfg.py +++ b/FWCore/Integration/test/testSwitchProducerTaskInput_cfg.py @@ -6,8 +6,8 @@ class SwitchProducerTest(cms.SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest,self).__init__( dict( - test1 = lambda: (True, -10), - test2 = lambda: (enableTest2, -9) + test1 = lambda accelerators: (True, -10), + test2 = lambda accelerators: (enableTest2, -9) ), **kargs) process = cms.Process("PROD2") diff --git a/FWCore/Integration/test/testSwitchProducerTask_cfg.py b/FWCore/Integration/test/testSwitchProducerTask_cfg.py index ef5ebf4a55b72..69eea0af5d34d 100644 --- a/FWCore/Integration/test/testSwitchProducerTask_cfg.py +++ b/FWCore/Integration/test/testSwitchProducerTask_cfg.py @@ -6,8 +6,8 @@ class SwitchProducerTest(cms.SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest,self).__init__( dict( - test1 = lambda: (True, -10), - test2 = lambda: (enableTest2, -9) + test1 = lambda accelerators: (True, -10), + test2 = lambda accelerators: (enableTest2, -9) ), **kargs) process = cms.Process("PROD1") diff --git a/FWCore/ParameterSet/python/Config.py b/FWCore/ParameterSet/python/Config.py index 5875a6425bdd2..c71188595fa1e 100644 --- a/FWCore/ParameterSet/python/Config.py +++ b/FWCore/ParameterSet/python/Config.py @@ -136,6 +136,7 @@ def __init__(self,name,*Mods): self.__dict__['_Process__partialschedules'] = {} self.__isStrict = False self.__dict__['_Process__modifiers'] = Mods + self.__dict__['_Process__accelerators'] = {} self.options = Process.defaultOptions_() self.maxEvents = Process.defaultMaxEvents_() self.maxLuminosityBlocks = Process.defaultMaxLuminosityBlocks_() @@ -228,6 +229,7 @@ def defaultOptions_(): allowAnyLabel_ = required.untracked.uint32 ) ), + accelerators = untracked.vstring('auto'), wantSummary = untracked.bool(False), fileMode = untracked.string('FULLMERGE'), forceEventSetupCacheClearOnNewRun = untracked.bool(False), @@ -325,6 +327,10 @@ def services_(self): """returns a dict of the services that have been added to the Process""" return DictTypes.FixedKeysDict(self.__services) services = property(services_,doc="dictionary containing the services for the process") + def processAccelerators_(self): + """returns a dict of the ProcessAccelerators that have been added to the Process""" + return DictTypes.FixedKeysDict(self.__accelerators) + processAccelerators = property(processAccelerators_,doc="dictionary containing the ProcessAccelerators for the process") def es_producers_(self): """returns a dict of the esproducers that have been added to the Process""" return DictTypes.FixedKeysDict(self.__esproducers) @@ -708,6 +714,9 @@ def _placeService(self,typeName,mod): if typeName in self.__dict__: self.__dict__[typeName]._inProcess = False self.__dict__[typeName]=mod + def _placeAccelerator(self,typeName,mod): + self._place(typeName, mod, self.__accelerators) + self.__dict__[typeName]=mod def load(self, moduleName): moduleName = moduleName.replace("/",".") module = __import__(moduleName) @@ -1003,6 +1012,7 @@ def dumpPython(self, options=PrintOptions()): result+=self._dumpPythonList(self.analyzers_(), options) result+=self._dumpPythonList(self.outputModules_(), options) result+=self._dumpPythonList(self.services_(), options) + result+=self._dumpPythonList(self.processAccelerators_(), options) result+=self._dumpPythonList(self.es_producers_(), options) result+=self._dumpPythonList(self.es_sources_(), options) result+=self._dumpPython(self.es_prefers_(), options) @@ -1151,7 +1161,7 @@ def _insertSwitchProducersInto(self, parameterSet, labelModules, labelAliases, i aliases = parameterSet.getVString(tracked, labelAliases) for name,value in itemDict.items(): value.appendToProcessDescLists_(modules, aliases, name) - value.insertInto(parameterSet, name) + value.insertInto(parameterSet, name, self.options.accelerators) modules.sort() aliases.sort() parameterSet.addVString(tracked, labelModules, modules) @@ -1378,6 +1388,7 @@ def __extractPSet(self,pset): return pset self.validate() + self.handleProcessAccelerators() processPSet.addString(True, "@process_name", self.name_()) all_modules = self.producers_().copy() all_modules.update(self.filters_()) @@ -1431,6 +1442,33 @@ def validate(self): # raise RuntimeError("No input source was found for this process") pass + def handleProcessAccelerators(self): + # Sanity check + useSet = set(self.options.accelerators.value()) + accSet = set([label for acc in self.__dict__['_Process__accelerators'].values() for label in acc.labels()]) + accSet.add("auto") + diff = useSet.difference(accSet) + if len(diff) > 0: + invalid = ",".join(diff) + valid = ",".join(sorted(list(accSet))) + raise ValueError("Invalid value{} of {} in process.options.accelerators, valid values are {}".format("s" if len(diff) > 2 else "", + invalid, + valid)) + + # Resolve 'auto' + if "auto" in self.options.accelerators: + if len(self.options.accelerators) >= 2: + raise ValueError("process.options.accelerators may contain 'auto' only as the only element, now it has {} elements".format(len(self.options.accelerators))) + newValue = set() + for acc in self.__dict__['_Process__accelerators'].values(): + for l in acc.enabledLabels(): + newValue.add(l) + self.options.accelerators = list(newValue) + + # Customize + for acc in self.__dict__['_Process__accelerators'].values(): + acc.apply(self) + def prefer(self, esmodule,*args,**kargs): """Prefer this ES source or producer. The argument can either be an object label, e.g., @@ -1824,6 +1862,61 @@ def apply(self,process): self.__func(process) self.__seenProcesses.add(process) +class ProcessAccelerator(_ConfigureComponent,_Unlabelable): + """A class used to specify possible compute accelerators in a Process + instance. It is intended to be derived for any + accelerator/portability technology, and provides hooks such that a + specific customization can be applied to the Process on a worker + node at the point where the python configuration is serialized for C++. + + The customizations must touch only untracked parameters. Each + deriving class should have a specific unit test enabling all + combinations of accelerators and assert that the configuration + hash does not change.""" + def __init__(self): + pass + def _place(self, name, proc): + proc._placeAccelerator(self.type_(), self) + def type_(self): + return type(self).__name__ + def dumpPython(self, options=PrintOptions()): + specialImportRegistry.registerUse(self) + result = self.__class__.__name__+"(" # not including cms. since the deriving classes are not in cms "namespace" + options.indent() + res = self.dumpPythonImpl(options) + options.unindent() + if len(res) > 0: + result += "\n"+res+"\n" + result += ")\n" + return result + + # The following methods are hooks to be overridden (if needed) in the deriving class + def dumpPythonImpl(self, options): + """Override if need to add any 'body' content to dumpPython(). Returns a string.""" + return "" + def labels(self): + """Override to return a list of strings for the accelerator labels.""" + return [] + def enabledLabels(self): + """Override to return a list of strings for the accelerator labels + that are enabled in the system the job is being run on.""" + return [] + def apply(self, process): + """Override if need to customize the Process at worker node. The + available accelerator labels are available via + 'process.options.accelerators' (the 'auto' has been expanded + to concrete labels at this point). + + This function may touch only untracked parameters.""" + pass + +# Need to be a module-level function for the configuration with a +# SwitchProducer to be pickleable. +def _switchproducer_test2_case1(accelerators): + return ("test1" in accelerators, -10) +def _switchproducer_test2_case2(accelerators): + return ("test2" in accelerators, -9) + if __name__=="__main__": import unittest import copy @@ -1918,13 +2011,45 @@ class SwitchProducerTest(SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest,self).__init__( dict( - test1 = lambda: (True, -10), - test2 = lambda: (True, -9), - test3 = lambda: (True, -8), - test4 = lambda: (True, -7) + test1 = lambda accelerators: (True, -10), + test2 = lambda accelerators: (True, -9), + test3 = lambda accelerators: (True, -8), + test4 = lambda accelerators: (True, -7) ), **kargs) specialImportRegistry.registerSpecialImportForType(SwitchProducerTest, "from test import SwitchProducerTest") + class SwitchProducerTest2(SwitchProducer): + def __init__(self, **kargs): + super(SwitchProducerTest2,self).__init__( + dict( + test1 = _switchproducer_test2_case1, + test2 = _switchproducer_test2_case2, + ), **kargs) + specialImportRegistry.registerSpecialImportForType(SwitchProducerTest2, "from test import SwitchProducerTest2") + + class ProcessAcceleratorTest(ProcessAccelerator): + def __init__(self, enabled=["test1", "test2"]): + super(ProcessAcceleratorTest,self).__init__() + self._labels = ["test1", "test2"] + self.setEnabled(enabled) + def setEnabled(self, enabled): + invalid = set(enabled).difference(set(self._labels)) + if len(invalid) > 0: + raise Exception("Tried to enabled nonexistent test accelerators {}".format(",".join(invalid))) + self._enabled = enabled[:] + def dumpPythonImpl(self,options): + result = "{}enabled = [{}]".format(options.indentation(), + ", ".join(["'{}'".format(e) for e in self._enabled])) + return result + def labels(self): + return self._labels + def enabledLabels(self): + return self._enabled + def apply(self, process): + process.acceleratorTestProducer = EDProducer("AcceleratorTestProducer") + process.acceleratorTestPath = Path(process.acceleratorTestProducer) + specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorTest, "from test import ProcessAcceleratorTest") + class TestModuleCommand(unittest.TestCase): def setUp(self): """Nothing to do """ @@ -2101,6 +2226,7 @@ def testProcessDumpPython(self): IgnoreCompletely = cms.untracked.vstring(), Rethrow = cms.untracked.vstring(), SkipEvent = cms.untracked.vstring(), + accelerators = cms.untracked.vstring('auto'), allowUnscheduled = cms.obsolete.untracked.bool, canDeleteEarly = cms.untracked.vstring(), deleteNonConsumedUnscheduledModules = cms.untracked.bool(True), @@ -3781,5 +3907,127 @@ def testProcessFragment(self): p = Process('PROCESS') p.extend(f) self.assertTrue(hasattr(p,'fltr')) + def testProcessAccelerator(self): + proc = Process("TEST") + self.assertRaises(TypeError, setattr, proc, "processAcceleratorTest", ProcessAcceleratorTest()) + proc.ProcessAcceleratorTest = ProcessAcceleratorTest() + del proc.MessageLogger # remove boilerplate unnecessary for this test case + self.assertEqual(proc.dumpPython(), +"""import FWCore.ParameterSet.Config as cms +from test import ProcessAcceleratorTest + +process = cms.Process("TEST") + +process.maxEvents = cms.untracked.PSet( + input = cms.optional.untracked.int32, + output = cms.optional.untracked.allowed(cms.int32,cms.PSet) +) + +process.maxLuminosityBlocks = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.options = cms.untracked.PSet( + FailPath = cms.untracked.vstring(), + IgnoreCompletely = cms.untracked.vstring(), + Rethrow = cms.untracked.vstring(), + SkipEvent = cms.untracked.vstring(), + accelerators = cms.untracked.vstring('auto'), + allowUnscheduled = cms.obsolete.untracked.bool, + canDeleteEarly = cms.untracked.vstring(), + deleteNonConsumedUnscheduledModules = cms.untracked.bool(True), + dumpOptions = cms.untracked.bool(False), + emptyRunLumiMode = cms.obsolete.untracked.string, + eventSetup = cms.untracked.PSet( + forceNumberOfConcurrentIOVs = cms.untracked.PSet( + allowAnyLabel_=cms.required.untracked.uint32 + ), + numberOfConcurrentIOVs = cms.untracked.uint32(0) + ), + fileMode = cms.untracked.string('FULLMERGE'), + forceEventSetupCacheClearOnNewRun = cms.untracked.bool(False), + makeTriggerResults = cms.obsolete.untracked.bool, + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(0), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfStreams = cms.untracked.uint32(0), + numberOfThreads = cms.untracked.uint32(1), + printDependencies = cms.untracked.bool(False), + sizeOfStackForThreadsInKB = cms.optional.untracked.uint32, + throwIfIllegalParameter = cms.untracked.bool(True), + wantSummary = cms.untracked.bool(False) +) + +process.ProcessAcceleratorTest = ProcessAcceleratorTest( + enabled = ['test1', 'test2'] +) + + +""") + p = TestMakePSet() + proc.fillProcessDesc(p) + self.assertFalse(p.values["options"][1].values["accelerators"][0]) + accelerators = p.values["options"][1].values["accelerators"][1] + self.assertTrue("test1" in accelerators) + self.assertTrue("test2" in accelerators) + self.assertEqual((True, "AcceleratorTestProducer"), p.values["acceleratorTestProducer"][1].values["@module_type"]) + + proc = Process("TEST") + proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"]) + p = TestMakePSet() + proc.fillProcessDesc(p) + self.assertEqual(["test1"], p.values["options"][1].values["accelerators"][1]) + + proc = Process("TEST") + proc.ProcessAcceleratorTest = ProcessAcceleratorTest() + proc.options.accelerators = ["test2"] + p = TestMakePSet() + proc.fillProcessDesc(p) + self.assertEqual(["test2"], p.values["options"][1].values["accelerators"][1]) + + proc = Process("TEST") + proc.ProcessAcceleratorTest = ProcessAcceleratorTest() + proc.options.accelerators = ["test3"] + p = TestMakePSet() + self.assertRaises(ValueError, proc.fillProcessDesc, p) + + proc = Process("TEST") + proc.ProcessAcceleratorTest = ProcessAcceleratorTest() + proc.options.accelerators = ["auto", "test1"] + p = TestMakePSet() + self.assertRaises(ValueError, proc.fillProcessDesc, p) + + proc = Process("TEST") + proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"]) + proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo", + a = int32(1), + b = PSet(c = int32(2))), + test1 = EDProducer("Bar", + aa = int32(11), + bb = PSet(cc = int32(12)))) + proc.p = Path(proc.sp) + p = TestMakePSet() + proc.fillProcessDesc(p) + self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"]) + + import pickle + proc = Process("TEST") + proc.ProcessAcceleratorTest = ProcessAcceleratorTest() + proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo", + a = int32(1), + b = PSet(c = int32(2))), + test1 = EDProducer("Bar", + aa = int32(11), + bb = PSet(cc = int32(12)))) + proc.p = Path(proc.sp) + pkl = pickle.dumps(proc) + unpkl = pickle.loads(pkl) + p = TestMakePSet() + unpkl.fillProcessDesc(p) + self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"]) + unpkl = pickle.loads(pkl) + unpkl.ProcessAcceleratorTest.setEnabled(["test1"]) + p = TestMakePSet() + unpkl.fillProcessDesc(p) + self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"]) unittest.main() diff --git a/FWCore/ParameterSet/python/Modules.py b/FWCore/ParameterSet/python/Modules.py index fa0a9b899e066..9bfc25e95ae9d 100644 --- a/FWCore/ParameterSet/python/Modules.py +++ b/FWCore/ParameterSet/python/Modules.py @@ -214,7 +214,7 @@ def nameInProcessDesc_(self, myname): # Need to be a module-level function for the configuration with a # SwitchProducer to be pickleable. -def _switch_cpu(): +def _switch_cpu(accelerators): return (True, 1) class SwitchProducer(EDProducer): @@ -257,21 +257,21 @@ def getCpu(): """Returns a function that returns the priority for a CPU "computing device". Intended to be used by deriving classes.""" return _switch_cpu - def _chooseCase(self): + def _chooseCase(self, accelerators): """Returns the name of the chosen case.""" cases = self.parameterNames_() bestCase = None for case in cases: - (enabled, priority) = self._caseFunctionDict[case]() + (enabled, priority) = self._caseFunctionDict[case](accelerators) if enabled and (bestCase is None or bestCase[0] < priority): bestCase = (priority, case) if bestCase is None: raise RuntimeError("All cases '%s' were disabled" % (str(cases))) return bestCase[1] - def _getProducer(self): + def _getProducer(self, accelerators): """Returns the EDroducer of the chosen case""" - return self.__dict__[self._chooseCase()] + return self.__dict__[self._chooseCase(accelerators)] @staticmethod def __typeIsValid(typ): @@ -378,7 +378,7 @@ def appendToProcessDescLists_(self, modules, aliases, myname): else: modules.append(self.caseLabel_(myname, case)) - def insertInto(self, parameterSet, myname): + def insertInto(self, parameterSet, myname, accelerators): for case in self.parameterNames_(): producer = self.__dict__[case] producer.insertInto(parameterSet, self.caseLabel_(myname, case)) @@ -387,7 +387,7 @@ def insertInto(self, parameterSet, myname): newpset.addString(True, "@module_type", "SwitchProducer") newpset.addString(True, "@module_edm_type", "EDProducer") newpset.addVString(True, "@all_cases", [myname+"@"+p for p in self.parameterNames_()]) - newpset.addString(False, "@chosen_case", myname+"@"+self._chooseCase()) + newpset.addString(False, "@chosen_case", myname+"@"+self._chooseCase(accelerators)) parameterSet.addPSet(True, self.nameInProcessDesc_(myname), newpset) def _placeImpl(self,name,proc): @@ -423,23 +423,23 @@ class SwitchProducerTest(SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest,self).__init__( dict( - test1 = lambda: (True, -10), - test2 = lambda: (True, -9), - test3 = lambda: (True, -8) + test1 = lambda accelerators: (True, -10), + test2 = lambda accelerators: (True, -9), + test3 = lambda accelerators: (True, -8) ), **kargs) class SwitchProducerTest1Dis(SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest1Dis,self).__init__( dict( - test1 = lambda: (False, -10), - test2 = lambda: (True, -9) + test1 = lambda accelerators: (False, -10), + test2 = lambda accelerators: (True, -9) ), **kargs) class SwitchProducerTest2Dis(SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest2Dis,self).__init__( dict( - test1 = lambda: (True, -10), - test2 = lambda: (False, -9) + test1 = lambda accelerators: (True, -10), + test2 = lambda accelerators: (False, -9) ), **kargs) class SwitchProducerPickleable(SwitchProducer): def __init__(self, **kargs): @@ -566,16 +566,17 @@ def testSwitchProducer(self): self.assertRaises(TypeError, lambda: SwitchProducerTest(test1 = SwitchProducerTest(test1 = EDProducer("Foo")))) # Case decision + accelerators = [] sp = SwitchProducerTest(test1 = EDProducer("Foo"), test2 = EDProducer("Bar")) - self.assertEqual(sp._getProducer().type_(), "Bar") + self.assertEqual(sp._getProducer(accelerators).type_(), "Bar") sp = SwitchProducerTest1Dis(test1 = EDProducer("Foo"), test2 = EDProducer("Bar")) - self.assertEqual(sp._getProducer().type_(), "Bar") + self.assertEqual(sp._getProducer(accelerators).type_(), "Bar") sp = SwitchProducerTest2Dis(test1 = EDProducer("Foo"), test2 = EDProducer("Bar")) - self.assertEqual(sp._getProducer().type_(), "Foo") + self.assertEqual(sp._getProducer(accelerators).type_(), "Foo") sp = SwitchProducerTest(test1 = EDProducer("Bar")) - self.assertEqual(sp._getProducer().type_(), "Bar") + self.assertEqual(sp._getProducer(accelerators).type_(), "Bar") sp = SwitchProducerTest1Dis(test1 = EDProducer("Bar")) - self.assertRaises(RuntimeError, sp._getProducer) + self.assertRaises(RuntimeError, sp._getProducer, accelerators) # Mofications from .Types import int32, string, PSet @@ -593,7 +594,7 @@ def testSwitchProducer(self): self.assertEqual(cl.test2.type_(), "Bar") self.assertEqual(cl.test2.aa.value(), 11) self.assertEqual(cl.test2.bb.cc.value(), 12) - self.assertEqual(sp._getProducer().type_(), "Bar") + self.assertEqual(sp._getProducer(accelerators).type_(), "Bar") # Modify clone cl.test1.a = 3 self.assertEqual(cl.test1.a.value(), 3) diff --git a/FWCore/ParameterSet/src/validateTopLevelParameterSets.cc b/FWCore/ParameterSet/src/validateTopLevelParameterSets.cc index 3576021e6fcbf..1d04d84f1c681 100644 --- a/FWCore/ParameterSet/src/validateTopLevelParameterSets.cc +++ b/FWCore/ParameterSet/src/validateTopLevelParameterSets.cc @@ -50,6 +50,11 @@ namespace edm { eventSetupDescription.addUntracked("forceNumberOfConcurrentIOVs", nestedDescription); description.addUntracked("eventSetup", eventSetupDescription); + description.addUntracked>("accelerators", {"auto"}) + ->setComment( + "Specify the compute accelerator(s) the job should use. Empty value means that no accelerators will be " + "used. Special value 'auto', can be used for automatically deduce what accelerators to use based on the " + "available hardware."); description.addUntracked("wantSummary", false) ->setComment("Set true to print a report on the trigger decisions and timing of modules"); description.addUntracked("fileMode", "FULLMERGE") From d800249d2a87ba53458ce062f7c4f3d98e6dce58 Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Thu, 30 Dec 2021 15:45:19 +0100 Subject: [PATCH 2/9] Simplify SwitchProducer tests --- FWCore/ParameterSet/python/Modules.py | 35 +++++++-------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/FWCore/ParameterSet/python/Modules.py b/FWCore/ParameterSet/python/Modules.py index 9bfc25e95ae9d..57c208c7f0de7 100644 --- a/FWCore/ParameterSet/python/Modules.py +++ b/FWCore/ParameterSet/python/Modules.py @@ -423,23 +423,9 @@ class SwitchProducerTest(SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest,self).__init__( dict( - test1 = lambda accelerators: (True, -10), - test2 = lambda accelerators: (True, -9), - test3 = lambda accelerators: (True, -8) - ), **kargs) - class SwitchProducerTest1Dis(SwitchProducer): - def __init__(self, **kargs): - super(SwitchProducerTest1Dis,self).__init__( - dict( - test1 = lambda accelerators: (False, -10), - test2 = lambda accelerators: (True, -9) - ), **kargs) - class SwitchProducerTest2Dis(SwitchProducer): - def __init__(self, **kargs): - super(SwitchProducerTest2Dis,self).__init__( - dict( - test1 = lambda accelerators: (True, -10), - test2 = lambda accelerators: (False, -9) + test1 = lambda accelerators: ("test1" in accelerators, -10), + test2 = lambda accelerators: ("test2" in accelerators, -9), + test3 = lambda accelerators: ("test3" in accelerators, -8) ), **kargs) class SwitchProducerPickleable(SwitchProducer): def __init__(self, **kargs): @@ -566,17 +552,14 @@ def testSwitchProducer(self): self.assertRaises(TypeError, lambda: SwitchProducerTest(test1 = SwitchProducerTest(test1 = EDProducer("Foo")))) # Case decision - accelerators = [] + accelerators = ["test1", "test2", "test3"] sp = SwitchProducerTest(test1 = EDProducer("Foo"), test2 = EDProducer("Bar")) - self.assertEqual(sp._getProducer(accelerators).type_(), "Bar") - sp = SwitchProducerTest1Dis(test1 = EDProducer("Foo"), test2 = EDProducer("Bar")) - self.assertEqual(sp._getProducer(accelerators).type_(), "Bar") - sp = SwitchProducerTest2Dis(test1 = EDProducer("Foo"), test2 = EDProducer("Bar")) - self.assertEqual(sp._getProducer(accelerators).type_(), "Foo") + self.assertEqual(sp._getProducer(["test1", "test2", "test3"]).type_(), "Bar") + self.assertEqual(sp._getProducer(["test2", "test3"]).type_(), "Bar") + self.assertEqual(sp._getProducer(["test1", "test3"]).type_(), "Foo") sp = SwitchProducerTest(test1 = EDProducer("Bar")) - self.assertEqual(sp._getProducer(accelerators).type_(), "Bar") - sp = SwitchProducerTest1Dis(test1 = EDProducer("Bar")) - self.assertRaises(RuntimeError, sp._getProducer, accelerators) + self.assertEqual(sp._getProducer(["test1", "test2", "test3"]).type_(), "Bar") + self.assertRaises(RuntimeError, sp._getProducer, ["test2", "test3"]) # Mofications from .Types import int32, string, PSet From 010f6a3f2f2cac9d87dbe152e5a188aecd094a43 Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Thu, 30 Dec 2021 23:21:53 +0100 Subject: [PATCH 3/9] Throw a specific, UnavailableAccelerator, exception if requested accelerator is not available --- .../interface/ensureAvailableAccelerators.h | 5 ++ FWCore/Framework/src/EventProcessor.cc | 2 + .../src/ensureAvailableAccelerators.cc | 38 +++++++++++ FWCore/Integration/test/BuildFile.xml | 2 + .../Integration/test/ProcessAccelerator_t.cpp | 35 ++++++++++ .../test/run_TestProcessAccelerator.sh | 38 +++++++++++ .../test/testProcessAccelerator_cfg.py | 68 +++++++++++++++++++ FWCore/ParameterSet/python/Config.py | 36 ++++++++-- FWCore/TestProcessor/src/TestProcessor.cc | 3 + FWCore/Utilities/interface/EDMException.h | 2 + FWCore/Utilities/src/EDMException.cc | 1 + 11 files changed, 224 insertions(+), 6 deletions(-) create mode 100644 FWCore/Framework/interface/ensureAvailableAccelerators.h create mode 100644 FWCore/Framework/src/ensureAvailableAccelerators.cc create mode 100755 FWCore/Integration/test/run_TestProcessAccelerator.sh create mode 100644 FWCore/Integration/test/testProcessAccelerator_cfg.py diff --git a/FWCore/Framework/interface/ensureAvailableAccelerators.h b/FWCore/Framework/interface/ensureAvailableAccelerators.h new file mode 100644 index 0000000000000..b01c89de7aa80 --- /dev/null +++ b/FWCore/Framework/interface/ensureAvailableAccelerators.h @@ -0,0 +1,5 @@ +#include "FWCore/ParameterSet/interface/ParameterSet.h" + +namespace edm { + void ensureAvailableAccelerators(edm::ParameterSet const& parameterSet); +} diff --git a/FWCore/Framework/src/EventProcessor.cc b/FWCore/Framework/src/EventProcessor.cc index 9228ce0062cb5..394d5f3b8f645 100644 --- a/FWCore/Framework/src/EventProcessor.cc +++ b/FWCore/Framework/src/EventProcessor.cc @@ -37,6 +37,7 @@ #include "FWCore/Framework/interface/SharedResourcesRegistry.h" #include "FWCore/Framework/interface/streamTransitionAsync.h" #include "FWCore/Framework/interface/TransitionInfoTypes.h" +#include "FWCore/Framework/interface/ensureAvailableAccelerators.h" #include "FWCore/Framework/interface/globalTransitionAsync.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" @@ -372,6 +373,7 @@ namespace edm { fileModeNoMerge_ = (fileMode == "NOMERGE"); } forceESCacheClearOnNewRun_ = optionsPset.getUntrackedParameter("forceEventSetupCacheClearOnNewRun"); + ensureAvailableAccelerators(*parameterSet); //threading unsigned int nThreads = optionsPset.getUntrackedParameter("numberOfThreads"); diff --git a/FWCore/Framework/src/ensureAvailableAccelerators.cc b/FWCore/Framework/src/ensureAvailableAccelerators.cc new file mode 100644 index 0000000000000..8b4dc90492775 --- /dev/null +++ b/FWCore/Framework/src/ensureAvailableAccelerators.cc @@ -0,0 +1,38 @@ +#include "FWCore/Framework/interface/ensureAvailableAccelerators.h" +#include "FWCore/Utilities/interface/EDMException.h" + +#include +#include + +namespace edm { + void ensureAvailableAccelerators(edm::ParameterSet const& parameterSet) { + ParameterSet const& optionsPset(parameterSet.getUntrackedParameterSet("options")); + auto accelerators = optionsPset.getUntrackedParameter>("accelerators"); + if (not accelerators.empty()) { + auto const& availableAccelerators = + parameterSet.getUntrackedParameter>("@available_accelerators"); + std::sort(accelerators.begin(), accelerators.end()); + std::vector unavailableAccelerators; + std::set_difference(accelerators.begin(), + accelerators.end(), + availableAccelerators.begin(), + availableAccelerators.end(), + std::back_inserter(unavailableAccelerators)); + if (not unavailableAccelerators.empty()) { + Exception ex(errors::UnavailableAccelerator); + ex << "Compute accelerators "; + bool first = true; + for (auto const& acc : unavailableAccelerators) { + if (not first) { + ex << ", "; + } else { + first = true; + } + ex << acc; + } + ex << " were requested but are not available in this system."; + throw ex; + } + } + } +} // namespace edm diff --git a/FWCore/Integration/test/BuildFile.xml b/FWCore/Integration/test/BuildFile.xml index be4ec7a9b723e..2bcfc3d7548a5 100644 --- a/FWCore/Integration/test/BuildFile.xml +++ b/FWCore/Integration/test/BuildFile.xml @@ -147,6 +147,8 @@ + + diff --git a/FWCore/Integration/test/ProcessAccelerator_t.cpp b/FWCore/Integration/test/ProcessAccelerator_t.cpp index 0d289d98c2b24..b26ed1c7debb3 100644 --- a/FWCore/Integration/test/ProcessAccelerator_t.cpp +++ b/FWCore/Integration/test/ProcessAccelerator_t.cpp @@ -117,4 +117,39 @@ TEST_CASE("Configuration", s_tag) { edm::test::TestProcessor tester(config_auto); REQUIRE_NOTHROW(tester.testLuminosityBlockWithNoEvents()); } + + SECTION("Test2 enabled, acclerators=auto") { + edm::test::TestProcessor tester(config_auto); + auto event = tester.test(); + REQUIRE(event.get()->value == 2); + } + + SECTION("Test2 enabled, acclerators=test1") { + edm::test::TestProcessor tester(config_test1); + auto event = tester.test(); + REQUIRE(event.get()->value == 1); + } + + SECTION("Test2 enabled, acclerators=test2") { + edm::test::TestProcessor tester(config_test2); + auto event = tester.test(); + REQUIRE(event.get()->value == 2); + } + + SECTION("Test2 disabled, accelerators=auto") { + edm::test::TestProcessor tester(configTest2Disabled_auto); + auto event = tester.test(); + REQUIRE(event.get()->value == 1); + } + + SECTION("Test2 disabled, accelerators=test1") { + edm::test::TestProcessor tester(configTest2Disabled_test1); + auto event = tester.test(); + REQUIRE(event.get()->value == 1); + } + + SECTION("Test2 disabled, accelerators=test2") { + REQUIRE_THROWS_WITH(edm::test::TestProcessor(configTest2Disabled_test2), + Catch::Contains("Compute accelerators test2 were requested but are not available")); + } } diff --git a/FWCore/Integration/test/run_TestProcessAccelerator.sh b/FWCore/Integration/test/run_TestProcessAccelerator.sh new file mode 100755 index 0000000000000..8826eb6cadaba --- /dev/null +++ b/FWCore/Integration/test/run_TestProcessAccelerator.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +test=testProcessAccelerator +LOCAL_TEST_DIR=${CMSSW_BASE}/src/FWCore/Integration/test +LOCAL_TMP_DIR=${CMSSW_BASE}/tmp/${SCRAM_ARCH} + +function die { echo Failure $1: status $2 ; exit $2 ; } + +pushd ${LOCAL_TMP_DIR} + +echo "*************************************************" +echo "accelerators=auto" +cmsRun ${LOCAL_TEST_DIR}/${test}_cfg.py || die "cmsRun ${test}_cfg.py" $? + +echo "*************************************************" +echo "accelerators=auto, enableTest2" +cmsRun ${LOCAL_TEST_DIR}/${test}_cfg.py -- --enableTest2 || die "cmsRun ${test}_cfg.py -- --enableTest2" $? + +echo "*************************************************" +echo "accelerators=test1" +cmsRun ${LOCAL_TEST_DIR}/${test}_cfg.py -- --accelerators=test1 || die "cmsRun ${test}_cfg.py -- --accelerators=test1" $? + +echo "*************************************************" +echo "accelerators=test2" +cmsRun -j testProcessAccelerators_jobreport.xml ${LOCAL_TEST_DIR}/${test}_cfg.py -- --accelerators=test2 && die "cmsRun ${test}_cfg.py -- --accelerators=test2 did not fail" 1 +EXIT_CODE=$(edmFjrDump --exitCode testProcessAccelerators_jobreport.xml) +if [ "x${EXIT_CODE}" != "x8035" ]; then + echo "ProcessAccelerator test for unavailable accelerator reported exit code ${EXIT_CODE} which is different from the expected 8035" + exit 1 +fi + +echo "*************************************************" +echo "accelerators=test1, enableTest2" +cmsRun ${LOCAL_TEST_DIR}/${test}_cfg.py -- --accelerators=test1 --enableTest2 || die "cmsRun ${test}_cfg.py -- --accelerators=test1 --enableTest2" $? + +echo "*************************************************" +echo "accelerators=test2, enableTest2" +cmsRun ${LOCAL_TEST_DIR}/${test}_cfg.py -- --accelerators=test2 --enableTest2 || die "cmsRun ${test}_cfg.py -- --accelerators=test2 --enableTest2" $? diff --git a/FWCore/Integration/test/testProcessAccelerator_cfg.py b/FWCore/Integration/test/testProcessAccelerator_cfg.py new file mode 100644 index 0000000000000..d9d4dedba9d5e --- /dev/null +++ b/FWCore/Integration/test/testProcessAccelerator_cfg.py @@ -0,0 +1,68 @@ +import FWCore.ParameterSet.Config as cms + +import argparse +import sys + +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test ProcessAccelerator.') + +parser.add_argument("--enableTest2", help="Enable test2 accelerator", action="store_true") +parser.add_argument("--accelerators", type=str, help="Comma-separated string for accelerators to enable") + +argv = sys.argv[:] +if '--' in argv: + argv.remove("--") +args, unknown = parser.parse_known_args(argv) + +class ProcessAcceleratorTest(cms.ProcessAccelerator): + def __init__(self): + super(ProcessAcceleratorTest,self).__init__() + self._labels = ["test1", "test2"] + self._enabled = ["test1"] + if args.enableTest2: + self._enabled.append("test2") + def labels(self): + return self._labels + def enabledLabels(self): + return self._enabled + +class SwitchProducerTest(cms.SwitchProducer): + def __init__(self, **kargs): + super(SwitchProducerTest,self).__init__( + dict( + test1 = lambda accelerators: ("test1" in accelerators, -10), + test2 = lambda accelerators: ("test2" in accelerators, -9), + ), **kargs) + +process = cms.Process("PROD1") + +process.add_(ProcessAcceleratorTest()) + +process.source = cms.Source("EmptySource") +process.maxEvents.input = 3 +if args.accelerators is not None: + process.options.accelerators = args.accelerators.split(",") + +process.intProducer1 = cms.EDProducer("ManyIntProducer", ivalue = cms.int32(1)) +process.intProducer2 = cms.EDProducer("ManyIntProducer", ivalue = cms.int32(2)) + +if args.enableTest2 and ("test2" in process.options.accelerators or "auto" in process.options.accelerators): + process.intProducer1.throw = cms.untracked.bool(True) +else: + process.intProducer2.throw = cms.untracked.bool(True) + +process.intProducer = SwitchProducerTest( + test1 = cms.EDProducer("AddIntsProducer", labels = cms.VInputTag("intProducer1")), + test2 = cms.EDProducer("AddIntsProducer", labels = cms.VInputTag("intProducer2")) +) + +process.intConsumer = cms.EDProducer("AddIntsProducer", labels = cms.VInputTag("intProducer")) + +process.t = cms.Task( + process.intProducer1, + process.intProducer2, + process.intProducer +) +process.p = cms.Path( + process.intConsumer, + process.t +) diff --git a/FWCore/ParameterSet/python/Config.py b/FWCore/ParameterSet/python/Config.py index c71188595fa1e..0afe053598d8f 100644 --- a/FWCore/ParameterSet/python/Config.py +++ b/FWCore/ParameterSet/python/Config.py @@ -1388,8 +1388,8 @@ def __extractPSet(self,pset): return pset self.validate() - self.handleProcessAccelerators() processPSet.addString(True, "@process_name", self.name_()) + self.handleProcessAccelerators(processPSet) all_modules = self.producers_().copy() all_modules.update(self.filters_()) all_modules.update(self.analyzers_()) @@ -1442,7 +1442,7 @@ def validate(self): # raise RuntimeError("No input source was found for this process") pass - def handleProcessAccelerators(self): + def handleProcessAccelerators(self, parameterSet): # Sanity check useSet = set(self.options.accelerators.value()) accSet = set([label for acc in self.__dict__['_Process__accelerators'].values() for label in acc.labels()]) @@ -1455,15 +1455,19 @@ def handleProcessAccelerators(self): invalid, valid)) + availableAccelerators = set() + for acc in self.__dict__['_Process__accelerators'].values(): + for l in acc.enabledLabels(): + availableAccelerators.add(l) + availableAccelerators = sorted(list(availableAccelerators)) + parameterSet.addVString(False, "@available_accelerators", availableAccelerators) + # Resolve 'auto' if "auto" in self.options.accelerators: if len(self.options.accelerators) >= 2: raise ValueError("process.options.accelerators may contain 'auto' only as the only element, now it has {} elements".format(len(self.options.accelerators))) newValue = set() - for acc in self.__dict__['_Process__accelerators'].values(): - for l in acc.enabledLabels(): - newValue.add(l) - self.options.accelerators = list(newValue) + self.options.accelerators = list(availableAccelerators) # Customize for acc in self.__dict__['_Process__accelerators'].values(): @@ -3970,12 +3974,17 @@ def testProcessAccelerator(self): self.assertTrue("test1" in accelerators) self.assertTrue("test2" in accelerators) self.assertEqual((True, "AcceleratorTestProducer"), p.values["acceleratorTestProducer"][1].values["@module_type"]) + self.assertFalse(p.values["@available_accelerators"][0]) + availableAccelerators = p.values["@available_accelerators"][1] + self.assertTrue("test1" in availableAccelerators) + self.assertTrue("test2" in availableAccelerators) proc = Process("TEST") proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"]) p = TestMakePSet() proc.fillProcessDesc(p) self.assertEqual(["test1"], p.values["options"][1].values["accelerators"][1]) + self.assertEqual(["test1"], p.values["@available_accelerators"][1]) proc = Process("TEST") proc.ProcessAcceleratorTest = ProcessAcceleratorTest() @@ -3983,6 +3992,17 @@ def testProcessAccelerator(self): p = TestMakePSet() proc.fillProcessDesc(p) self.assertEqual(["test2"], p.values["options"][1].values["accelerators"][1]) + availableAccelerators = p.values["@available_accelerators"][1] + self.assertTrue("test1" in availableAccelerators) + self.assertTrue("test2" in availableAccelerators) + + proc = Process("TEST") + proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"]) + proc.options.accelerators = ["test2"] + p = TestMakePSet() + proc.fillProcessDesc(p) + self.assertEqual(["test2"], p.values["options"][1].values["accelerators"][1]) + self.assertEqual(["test1"], p.values["@available_accelerators"][1]) proc = Process("TEST") proc.ProcessAcceleratorTest = ProcessAcceleratorTest() @@ -4024,10 +4044,14 @@ def testProcessAccelerator(self): p = TestMakePSet() unpkl.fillProcessDesc(p) self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"]) + availableAccelerators = p.values["@available_accelerators"][1] + self.assertTrue("test1" in availableAccelerators) + self.assertTrue("test2" in availableAccelerators) unpkl = pickle.loads(pkl) unpkl.ProcessAcceleratorTest.setEnabled(["test1"]) p = TestMakePSet() unpkl.fillProcessDesc(p) self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"]) + self.assertEqual(["test1"], p.values["@available_accelerators"][1]) unittest.main() diff --git a/FWCore/TestProcessor/src/TestProcessor.cc b/FWCore/TestProcessor/src/TestProcessor.cc index bba07fd68b07e..9145acca9e49e 100644 --- a/FWCore/TestProcessor/src/TestProcessor.cc +++ b/FWCore/TestProcessor/src/TestProcessor.cc @@ -32,6 +32,7 @@ #include "FWCore/Framework/interface/TransitionInfoTypes.h" #include "FWCore/Framework/interface/ProductPutterBase.h" #include "FWCore/Framework/interface/DelayedReader.h" +#include "FWCore/Framework/interface/ensureAvailableAccelerators.h" #include "FWCore/ServiceRegistry/interface/ServiceRegistry.h" #include "FWCore/ServiceRegistry/interface/SystemBounds.h" @@ -94,6 +95,8 @@ namespace edm { validateTopLevelParameterSets(psetPtr.get()); + ensureAvailableAccelerators(*psetPtr); + labelOfTestModule_ = psetPtr->getParameter("@moduleToTest"); auto procDesc = desc.processDesc(); diff --git a/FWCore/Utilities/interface/EDMException.h b/FWCore/Utilities/interface/EDMException.h index 1b4b08e16d4c7..8ff864a0301f5 100644 --- a/FWCore/Utilities/interface/EDMException.h +++ b/FWCore/Utilities/interface/EDMException.h @@ -67,6 +67,8 @@ namespace edm { FileNameInconsistentWithGUID = 8034, + UnavailableAccelerator = 8035, + EventGenerationFailure = 8501, CaughtSignal = 9000 diff --git a/FWCore/Utilities/src/EDMException.cc b/FWCore/Utilities/src/EDMException.cc index 93d667507ed7d..c3d5530b47a1f 100644 --- a/FWCore/Utilities/src/EDMException.cc +++ b/FWCore/Utilities/src/EDMException.cc @@ -43,6 +43,7 @@ namespace edm { FILLENTRY(ExceededResourceTime), FILLENTRY(FileWriteError), FILLENTRY(FileNameInconsistentWithGUID), + FILLENTRY(UnavailableAccelerator), FILLENTRY(EventGenerationFailure), FILLENTRY(CaughtSignal)}; static const std::string kUnknownCode("unknownCode"); From 417bdcae9be4a2fa8d6140ab36f654dd224f0145 Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Thu, 6 Jan 2022 11:10:57 -0600 Subject: [PATCH 4/9] Add ProcessAcceleratorCUDA, and replace the direct use of CUDAService with it --- .../StandardSequences/python/Services_cff.py | 4 +- .../CUDACore/python/ProcessAcceleratorCUDA.py | 27 ++++++++++ .../python/ProcessAcceleratorCUDA_cfi.py | 3 ++ .../CUDACore/python/SwitchProducerCUDA.py | 11 ++-- HeterogeneousCore/CUDACore/test/BuildFile.xml | 1 + .../test/test_ProcessAcceleratorCUDA.cc | 54 +++++++++++++++++++ HeterogeneousCore/CUDATest/test/BuildFile.xml | 7 ++- HeterogeneousCore/CUDATest/test/runtests.sh | 40 +++++++++++++- .../CUDATest/test/testCUDASwitch_cfg.py | 32 +++++++---- .../test/test_TestCUDAProducerGPUFirst.cc | 4 +- 10 files changed, 156 insertions(+), 27 deletions(-) create mode 100644 HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA.py create mode 100644 HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA_cfi.py create mode 100644 HeterogeneousCore/CUDACore/test/test_ProcessAcceleratorCUDA.cc diff --git a/Configuration/StandardSequences/python/Services_cff.py b/Configuration/StandardSequences/python/Services_cff.py index f68e2e399e9e9..3925df5f9ed41 100644 --- a/Configuration/StandardSequences/python/Services_cff.py +++ b/Configuration/StandardSequences/python/Services_cff.py @@ -10,9 +10,7 @@ # load CUDA services when the "gpu" or "pixelNtupletFit" modifiers are enabled def _addCUDAServices(process): - process.load("HeterogeneousCore.CUDAServices.CUDAService_cfi") - process.load("FWCore.MessageService.MessageLogger_cfi") - process.MessageLogger.CUDAService = cms.untracked.PSet() + process.load("HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi") from Configuration.ProcessModifiers.gpu_cff import gpu from Configuration.ProcessModifiers.pixelNtupletFit_cff import pixelNtupletFit diff --git a/HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA.py b/HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA.py new file mode 100644 index 0000000000000..f9ce7f1590746 --- /dev/null +++ b/HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA.py @@ -0,0 +1,27 @@ +import FWCore.ParameterSet.Config as cms + +import os + +class ProcessAcceleratorCUDA(cms.ProcessAccelerator): + def __init__(self): + super(ProcessAcceleratorCUDA,self).__init__() + self._label = "gpu-nvidia" + def labels(self): + return [self._label] + def enabledLabels(self): + enabled = (os.system("cudaIsEnabled") == 0) + if enabled: + return self.labels() + else: + return [] + def apply(self, process): + if not hasattr(process, "CUDAService"): + process.load("HeterogeneousCore.CUDAServices.CUDAService_cfi") + + if self._label in process.options.accelerators: + process.CUDAService.enabled = True + process.MessageLogger.CUDAService = cms.untracked.PSet() + else: + process.CUDAService.enabled = False + +cms.specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorCUDA, "from HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA import ProcessAcceleratorCUDA") diff --git a/HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA_cfi.py b/HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA_cfi.py new file mode 100644 index 0000000000000..1627387b855d4 --- /dev/null +++ b/HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA_cfi.py @@ -0,0 +1,3 @@ +from HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA import ProcessAcceleratorCUDA as _ProcessAcceleratorCUDA + +ProcessAcceleratorCUDA = _ProcessAcceleratorCUDA() diff --git a/HeterogeneousCore/CUDACore/python/SwitchProducerCUDA.py b/HeterogeneousCore/CUDACore/python/SwitchProducerCUDA.py index ded114e2fddfe..aa9c6434900cc 100644 --- a/HeterogeneousCore/CUDACore/python/SwitchProducerCUDA.py +++ b/HeterogeneousCore/CUDACore/python/SwitchProducerCUDA.py @@ -1,13 +1,8 @@ import FWCore.ParameterSet.Config as cms -_cuda_enabled_cached = None - -def _switch_cuda(): - global _cuda_enabled_cached - if _cuda_enabled_cached is None: - import os - _cuda_enabled_cached = (os.system("cudaIsEnabled") == 0) - return (_cuda_enabled_cached, 2) +def _switch_cuda(useAccelerators): + have_gpu = ("gpu-nvidia" in useAccelerators) + return (have_gpu, 2) class SwitchProducerCUDA(cms.SwitchProducer): def __init__(self, **kargs): diff --git a/HeterogeneousCore/CUDACore/test/BuildFile.xml b/HeterogeneousCore/CUDACore/test/BuildFile.xml index 0cd9316b34b69..34c12e1273eae 100644 --- a/HeterogeneousCore/CUDACore/test/BuildFile.xml +++ b/HeterogeneousCore/CUDACore/test/BuildFile.xml @@ -8,6 +8,7 @@ + diff --git a/HeterogeneousCore/CUDACore/test/test_ProcessAcceleratorCUDA.cc b/HeterogeneousCore/CUDACore/test/test_ProcessAcceleratorCUDA.cc new file mode 100644 index 0000000000000..0e674e85bbbbd --- /dev/null +++ b/HeterogeneousCore/CUDACore/test/test_ProcessAcceleratorCUDA.cc @@ -0,0 +1,54 @@ +#include "catch.hpp" + +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSetReader/interface/ParameterSetReader.h" + +#include + +#include +#include + +static constexpr auto s_tag = "[ProcessAcceleratorCUDA]"; + +namespace { + std::string makeConfig(std::string_view cpu, std::string_view cuda, std::string_view accelerator) { + return fmt::format( + R"_(import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") +process.options.accelerators = [{}] + +process.load("HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi") +from HeterogeneousCore.CUDACore.SwitchProducerCUDA import * + +process.s = SwitchProducerCUDA( + cpu = {}, + cuda = {} +) +process.p = cms.Path(process.s) +)_", + accelerator, + cpu, + cuda); + } +} // namespace + +TEST_CASE("Configuration", s_tag) { + const std::string test1{"cms.EDProducer('IntProducer', ivalue = cms.int32(1))"}; + const std::string test2{"cms.EDProducer('ManyIntProducer', ivalue = cms.int32(2), values = cms.VPSet())"}; + + const std::string baseConfig_auto = makeConfig(test1, test2, "'auto'"); + const std::string baseConfig_cpu = makeConfig(test1, test2, ""); + const std::string baseConfig_cuda = makeConfig(test1, test2, "'gpu-nvidia'"); + + SECTION("Configuration hash is not changed") { + auto pset_auto = edm::readConfig(baseConfig_auto); + auto pset_cpu = edm::readConfig(baseConfig_cpu); + auto pset_cuda = edm::readConfig(baseConfig_cuda); + pset_auto->registerIt(); + pset_cpu->registerIt(); + pset_cuda->registerIt(); + REQUIRE(pset_auto->id() == pset_cpu->id()); + REQUIRE(pset_auto->id() == pset_cuda->id()); + } +} diff --git a/HeterogeneousCore/CUDATest/test/BuildFile.xml b/HeterogeneousCore/CUDATest/test/BuildFile.xml index f133347ea1271..60351fbfe9b88 100644 --- a/HeterogeneousCore/CUDATest/test/BuildFile.xml +++ b/HeterogeneousCore/CUDATest/test/BuildFile.xml @@ -5,5 +5,10 @@ - + + + + + + diff --git a/HeterogeneousCore/CUDATest/test/runtests.sh b/HeterogeneousCore/CUDATest/test/runtests.sh index 6a9050388ea2e..ea2d9417559a3 100755 --- a/HeterogeneousCore/CUDATest/test/runtests.sh +++ b/HeterogeneousCore/CUDATest/test/runtests.sh @@ -4,6 +4,42 @@ function die { echo Failure $1: status $2 ; exit $2 ; } TEST_DIR=src/HeterogeneousCore/CUDATest/test +if [ "x$#" != "x1" ]; then + die "Need exactly 1 argument ('cpu', 'gpu'), got $#" 1 +fi +if [ "x$1" = "xgpu" ]; then + TARGET=gpu +elif [ "x$1" = "xcpu" ]; then + # In non-_GPU_ IBs, if CUDA is enabled, run the GPU-targeted tests + cudaIsEnabled + CUDA_ENABLED=$? + if [ "x${CUDA_ENABLED}" == "x0" ]; then + TARGET=gpu + else + TARGET=cpu + fi +else + die "Argument needs to be 'cpu' or 'gpu', got $1" 1 +fi + +echo "*************************************************" +echo "CUDA producer configuration with SwitchProducer, automatic" +cmsRun ${TEST_DIR}/testCUDASwitch_cfg.py -- --silent || die "cmsRun testCUDASwitch_cfg.py --silent" $? + echo "*************************************************" -echo "CUDA producer configuration with SwitchProducer" -cmsRun ${TEST_DIR}/testCUDASwitch_cfg.py || die "cmsRun testCUDASwitch_cfg.py 1" $? +echo "CUDA producer configuration with SwitchProducer, force CPU" +cmsRun ${TEST_DIR}/testCUDASwitch_cfg.py -- --silent --accelerator="" || die "cmsRun testCUDASwitch_cfg.py --silent --accelerator=\"\"" $? + +if [ "x${TARGET}" == "xgpu" ]; then + echo "*************************************************" + echo "CUDA producer configuration with SwitchProducer, force GPU" + cmsRun ${TEST_DIR}/testCUDASwitch_cfg.py -- --silent --accelerator="gpu-nvidia" || die "cmsRun testCUDASwitch_cfg.py --silent --accelerator=gpu-nvidia" $? +elif [ "x${TARGET}" == "xcpu" ]; then + echo "*************************************************" + echo "CUDA producer configuration with SwitchProducer, force GPU, should fail" + cmsRun -j testCUDATest_jobreport.xml ${TEST_DIR}/testCUDASwitch_cfg.py -- --silent --accelerator="gpu-nvidia" && die "cmsRun testCUDASwitch_cfg.py --silent --accelerator=gpu-nvidia did not fail" 1 + EXIT_CODE=$(edmFjrDump --exitCode testCUDATest_jobreport.xml) + if [ "x${EXIT_CODE}" != "x8035" ]; then + echo "Test (that was expected to fail) reported exit code ${EXIT_CODE} instead of expected 8035" + fi +fi diff --git a/HeterogeneousCore/CUDATest/test/testCUDASwitch_cfg.py b/HeterogeneousCore/CUDATest/test/testCUDASwitch_cfg.py index 805617091686b..c3592dae9a61e 100644 --- a/HeterogeneousCore/CUDATest/test/testCUDASwitch_cfg.py +++ b/HeterogeneousCore/CUDATest/test/testCUDASwitch_cfg.py @@ -1,27 +1,37 @@ import FWCore.ParameterSet.Config as cms -silent = True -#silent = False +import argparse +import sys -#includeAnalyzer = True -includeAnalyzer = False +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test CUDA EDProducers') + +parser.add_argument("--silent", help="Silence printouts", action="store_true") +parser.add_argument("--includeAnalyzer", help="Include an EDAnalyzer", action="store_true") +parser.add_argument("--accelerator", type=str, help="String for accelerator to enable") + +argv = sys.argv[:] +if '--' in argv: + argv.remove("--") +args, unknown = parser.parse_known_args(argv) process = cms.Process("Test") process.load("FWCore.MessageService.MessageLogger_cfi") -process.load("HeterogeneousCore.CUDAServices.CUDAService_cfi") +process.load("HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi") process.source = cms.Source("EmptySource") process.maxEvents.input = 3 -if not silent: +if not args.silent: process.maxEvents.input = 10 process.MessageLogger.cerr.threshold = cms.untracked.string("INFO") process.MessageLogger.cerr.INFO.limit = process.MessageLogger.cerr.default.limit -#process.options.numberOfThreads = 4 -process.options.numberOfStreams = 0 -#process.Tracer = cms.Service("Tracer") +if args.accelerator is not None: + if len(args.accelerator) == 0: + process.options.accelerators = [] + else: + process.options.accelerators = [args.accelerator] # Flow diagram of the modules # @@ -69,7 +79,7 @@ # GPU analyzer (optionally) from HeterogeneousCore.CUDATest.testCUDAAnalyzerGPU_cfi import testCUDAAnalyzerGPU process.anaCUDA = testCUDAAnalyzerGPU.clone(src="prod6CUDA") -if silent: +if args.silent: process.anaCUDA.minValue = 2.3e7 process.anaCUDA.maxValue = 2.5e7 @@ -95,7 +105,7 @@ process.prod6Task ) process.p = cms.Path() -if includeAnalyzer: +if args.includeAnalyzer: process.p += process.anaCUDA process.p.associate(process.t) process.ep = cms.EndPath(process.out) diff --git a/HeterogeneousCore/CUDATest/test/test_TestCUDAProducerGPUFirst.cc b/HeterogeneousCore/CUDATest/test/test_TestCUDAProducerGPUFirst.cc index a7e4f16043a1f..47ebbd97c5286 100644 --- a/HeterogeneousCore/CUDATest/test/test_TestCUDAProducerGPUFirst.cc +++ b/HeterogeneousCore/CUDATest/test/test_TestCUDAProducerGPUFirst.cc @@ -16,7 +16,7 @@ TEST_CASE("Standard checks of TestCUDAProducerGPUFirst", s_tag) { const std::string baseConfig{ R"_(from FWCore.TestProcessor.TestProcess import * process = TestProcess() -process.load("HeterogeneousCore.CUDAServices.CUDAService_cfi") +process.load("HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi") process.toTest = cms.EDProducer("TestCUDAProducerGPUFirst") process.moduleToTest(process.toTest) )_"}; @@ -57,7 +57,7 @@ TEST_CASE("TestCUDAProducerGPUFirst operation", s_tag) { const std::string baseConfig{ R"_(from FWCore.TestProcessor.TestProcess import * process = TestProcess() -process.load("HeterogeneousCore.CUDAServices.CUDAService_cfi") +process.load("HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi") process.toTest = cms.EDProducer("TestCUDAProducerGPUFirst") process.moduleToTest(process.toTest) )_"}; From 6ccf36189fe484ded325c91a76ad77b48b42289e Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Sat, 8 Jan 2022 00:46:31 +0100 Subject: [PATCH 5/9] Migrate direct users of CUDAService to ProcessAcceleratorCUDA --- HLTrigger/Configuration/python/customizeHLTforPatatrack.py | 5 +---- .../EcalRecProducers/test/ecalRawDecodingAndMultifit.py | 5 +---- .../EcalRecProducers/test/testEcalRechitProducer_cfg.py | 2 +- .../test/testEcalUncalibRechitProducer_cfg.py | 7 +------ .../HGCalRecProducers/test/HeterogeneousHGCalRecHit_cfg.py | 2 +- .../test/HeterogeneousRecHitsTiming_cfg.py | 2 +- .../HcalRecProducers/test/make_GPUvsCPU_HCAL_rechits.py | 4 +--- .../test/HeterogeneousHGCalRecHitsValidator_cfg.py | 2 +- 8 files changed, 8 insertions(+), 21 deletions(-) diff --git a/HLTrigger/Configuration/python/customizeHLTforPatatrack.py b/HLTrigger/Configuration/python/customizeHLTforPatatrack.py index 158808c4203d8..14974d681219b 100644 --- a/HLTrigger/Configuration/python/customizeHLTforPatatrack.py +++ b/HLTrigger/Configuration/python/customizeHLTforPatatrack.py @@ -54,10 +54,7 @@ def customiseCommon(process): # Services - _load_if_missing(process, 'CUDAService', 'HeterogeneousCore.CUDAServices.CUDAService_cfi') - - if 'MessageLogger' in process.__dict__: - process.MessageLogger.CUDAService = cms.untracked.PSet() + _load_if_missing(process, 'ProcessAcceleratorCUDA', 'HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi') # # NVProfilerService is broken in CMSSW 12.0.x and later # _load_if_missing(process, 'NVProfilerService', 'HeterogeneousCore.CUDAServices.NVProfilerService_cfi') diff --git a/RecoLocalCalo/EcalRecProducers/test/ecalRawDecodingAndMultifit.py b/RecoLocalCalo/EcalRecProducers/test/ecalRawDecodingAndMultifit.py index a3d04e836f020..1bfa8c8c6dc73 100644 --- a/RecoLocalCalo/EcalRecProducers/test/ecalRawDecodingAndMultifit.py +++ b/RecoLocalCalo/EcalRecProducers/test/ecalRawDecodingAndMultifit.py @@ -6,7 +6,7 @@ # import of standard configurations process.load('Configuration.StandardSequences.Services_cff') process.load('FWCore.MessageService.MessageLogger_cfi') -process.load('HeterogeneousCore.CUDAServices.CUDAService_cfi') +process.load('HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi') process.load('Configuration.StandardSequences.GeometryRecoDB_cff') process.load('Configuration.StandardSequences.MagneticField_AutoFromDBCurrent_cff') process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff') @@ -167,6 +167,3 @@ SkipEvent = cms.untracked.vstring('ProductNotFound'), wantSummary = cms.untracked.bool(True) ) - -# report CUDAService messages -process.MessageLogger.categories.append("CUDAService") diff --git a/RecoLocalCalo/EcalRecProducers/test/testEcalRechitProducer_cfg.py b/RecoLocalCalo/EcalRecProducers/test/testEcalRechitProducer_cfg.py index 192eb1fd2cb90..768b7fc5096bc 100644 --- a/RecoLocalCalo/EcalRecProducers/test/testEcalRechitProducer_cfg.py +++ b/RecoLocalCalo/EcalRecProducers/test/testEcalRechitProducer_cfg.py @@ -7,7 +7,7 @@ # import of standard configurations process.load('Configuration.StandardSequences.Services_cff') process.load('FWCore.MessageService.MessageLogger_cfi') -process.load('HeterogeneousCore.CUDAServices.CUDAService_cfi') +process.load('HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi') process.load('Configuration.StandardSequences.GeometryRecoDB_cff') process.load('Configuration.StandardSequences.MagneticField_AutoFromDBCurrent_cff') process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff') diff --git a/RecoLocalCalo/EcalRecProducers/test/testEcalUncalibRechitProducer_cfg.py b/RecoLocalCalo/EcalRecProducers/test/testEcalUncalibRechitProducer_cfg.py index 5684e30e330b8..bb01e1f9fcc01 100644 --- a/RecoLocalCalo/EcalRecProducers/test/testEcalUncalibRechitProducer_cfg.py +++ b/RecoLocalCalo/EcalRecProducers/test/testEcalUncalibRechitProducer_cfg.py @@ -8,7 +8,7 @@ process.load('Configuration.StandardSequences.Services_cff') #process.load('SimGeneral.HepPDTESSource.pythiapdt_cfi') process.load('FWCore.MessageService.MessageLogger_cfi') -process.load('HeterogeneousCore.CUDAServices.CUDAService_cfi') +process.load('HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi') #process.load('Configuration.EventContent.EventContent_cff') process.load('Configuration.StandardSequences.GeometryRecoDB_cff') process.load('Configuration.StandardSequences.MagneticField_AutoFromDBCurrent_cff') @@ -225,8 +225,3 @@ SkipEvent = cms.untracked.vstring('ProductNotFound'), wantSummary = cms.untracked.bool(True) ) - -# report CUDAService messages -process.MessageLogger.categories.append("CUDAService") - - diff --git a/RecoLocalCalo/HGCalRecProducers/test/HeterogeneousHGCalRecHit_cfg.py b/RecoLocalCalo/HGCalRecProducers/test/HeterogeneousHGCalRecHit_cfg.py index 249ee0d700909..4ff82b8f5bd80 100644 --- a/RecoLocalCalo/HGCalRecProducers/test/HeterogeneousHGCalRecHit_cfg.py +++ b/RecoLocalCalo/HGCalRecProducers/test/HeterogeneousHGCalRecHit_cfg.py @@ -44,7 +44,7 @@ def getHeterogeneousRecHitsSource(pu): #process.load('Configuration.EventContent.EventContent_cff') process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff') process.load('Configuration.Geometry.GeometryExtended2026D49Reco_cff') -process.load('HeterogeneousCore.CUDAServices.CUDAService_cfi') +process.load('HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi') process.load('RecoLocalCalo.HGCalRecProducers.HGCalRecHit_cfi') process.load('SimCalorimetry.HGCalSimProducers.hgcalDigitizer_cfi') diff --git a/RecoLocalCalo/HGCalRecProducers/test/HeterogeneousRecHitsTiming_cfg.py b/RecoLocalCalo/HGCalRecProducers/test/HeterogeneousRecHitsTiming_cfg.py index 476006f1a20bd..b79e90f5421e7 100644 --- a/RecoLocalCalo/HGCalRecProducers/test/HeterogeneousRecHitsTiming_cfg.py +++ b/RecoLocalCalo/HGCalRecProducers/test/HeterogeneousRecHitsTiming_cfg.py @@ -15,7 +15,7 @@ process.load('Configuration.StandardSequences.MagneticField_cff') process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff') process.load('Configuration.Geometry.GeometryExtended2026D49Reco_cff') -process.load('HeterogeneousCore.CUDAServices.CUDAService_cfi') +process.load('HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi') process.load('RecoLocalCalo.HGCalRecProducers.HGCalRecHit_cfi') process.load('SimCalorimetry.HGCalSimProducers.hgcalDigitizer_cfi') process.load( "HLTrigger.Timer.FastTimerService_cfi" ) diff --git a/RecoLocalCalo/HcalRecProducers/test/make_GPUvsCPU_HCAL_rechits.py b/RecoLocalCalo/HcalRecProducers/test/make_GPUvsCPU_HCAL_rechits.py index 0ce9caf13fa88..84fb7a98132e2 100644 --- a/RecoLocalCalo/HcalRecProducers/test/make_GPUvsCPU_HCAL_rechits.py +++ b/RecoLocalCalo/HcalRecProducers/test/make_GPUvsCPU_HCAL_rechits.py @@ -7,7 +7,7 @@ # import of standard configurations process.load('Configuration.StandardSequences.Services_cff') process.load('FWCore.MessageService.MessageLogger_cfi') -process.load('HeterogeneousCore.CUDAServices.CUDAService_cfi') +process.load('HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi') process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff') from Configuration.AlCa.GlobalTag import GlobalTag @@ -146,6 +146,4 @@ wantSummary = cms.untracked.bool(True) ) -# report CUDAService messages process.MessageLogger.cerr.FwkReport.reportEvery = 100 -process.MessageLogger.CUDAService = cms.untracked.PSet() diff --git a/Validation/HGCalValidation/test/HeterogeneousHGCalRecHitsValidator_cfg.py b/Validation/HGCalValidation/test/HeterogeneousHGCalRecHitsValidator_cfg.py index 7b4a2eacebcfa..0fb974b588c58 100644 --- a/Validation/HGCalValidation/test/HeterogeneousHGCalRecHitsValidator_cfg.py +++ b/Validation/HGCalValidation/test/HeterogeneousHGCalRecHitsValidator_cfg.py @@ -42,7 +42,7 @@ def getHeterogeneousRecHitsSource(pu): #process.load('Configuration.EventContent.EventContent_cff') process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff') process.load('Configuration.Geometry.GeometryExtended2026D49Reco_cff') -process.load('HeterogeneousCore.CUDAServices.CUDAService_cfi') +process.load('HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi') process.load('RecoLocalCalo.HGCalRecProducers.HGCalRecHit_cfi') process.load('SimCalorimetry.HGCalSimProducers.hgcalDigitizer_cfi') From 023bf0e83ed815c3046fe8ff3498d7b6978adc3a Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Thu, 3 Feb 2022 23:23:48 +0100 Subject: [PATCH 6/9] Change semantics to set of allowed accelerators, add implicit 'cpu' accelerator, and support for wildcards --- .../src/ensureAvailableAccelerators.cc | 29 +--- .../Integration/test/ProcessAccelerator_t.cpp | 19 ++- .../test/run_TestProcessAccelerator.sh | 4 +- .../test/testProcessAccelerator_cfg.py | 12 +- FWCore/ParameterSet/python/Config.py | 154 +++++++++++++----- .../src/validateTopLevelParameterSets.cc | 13 +- .../test/test_ProcessAcceleratorCUDA.cc | 4 +- HeterogeneousCore/CUDATest/test/runtests.sh | 2 +- .../CUDATest/test/testCUDASwitch_cfg.py | 5 +- 9 files changed, 153 insertions(+), 89 deletions(-) diff --git a/FWCore/Framework/src/ensureAvailableAccelerators.cc b/FWCore/Framework/src/ensureAvailableAccelerators.cc index 8b4dc90492775..f706c215b99eb 100644 --- a/FWCore/Framework/src/ensureAvailableAccelerators.cc +++ b/FWCore/Framework/src/ensureAvailableAccelerators.cc @@ -8,31 +8,16 @@ namespace edm { void ensureAvailableAccelerators(edm::ParameterSet const& parameterSet) { ParameterSet const& optionsPset(parameterSet.getUntrackedParameterSet("options")); auto accelerators = optionsPset.getUntrackedParameter>("accelerators"); - if (not accelerators.empty()) { + if (accelerators.empty()) { + Exception ex(errors::UnavailableAccelerator); + ex << "The system has no compute accelerators that match the patterns specified in " + "process.options.accelerators.\nThe following compute accelerators are available:\n"; auto const& availableAccelerators = parameterSet.getUntrackedParameter>("@available_accelerators"); - std::sort(accelerators.begin(), accelerators.end()); - std::vector unavailableAccelerators; - std::set_difference(accelerators.begin(), - accelerators.end(), - availableAccelerators.begin(), - availableAccelerators.end(), - std::back_inserter(unavailableAccelerators)); - if (not unavailableAccelerators.empty()) { - Exception ex(errors::UnavailableAccelerator); - ex << "Compute accelerators "; - bool first = true; - for (auto const& acc : unavailableAccelerators) { - if (not first) { - ex << ", "; - } else { - first = true; - } - ex << acc; - } - ex << " were requested but are not available in this system."; - throw ex; + for (auto const& acc : availableAccelerators) { + ex << " " << acc << "\n"; } + throw ex; } } } // namespace edm diff --git a/FWCore/Integration/test/ProcessAccelerator_t.cpp b/FWCore/Integration/test/ProcessAccelerator_t.cpp index b26ed1c7debb3..4a80fafeaec5f 100644 --- a/FWCore/Integration/test/ProcessAccelerator_t.cpp +++ b/FWCore/Integration/test/ProcessAccelerator_t.cpp @@ -38,14 +38,16 @@ class SwitchProducerTest(cms.SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest,self).__init__( dict( - test1 = lambda accelerators: ("test1" in accelerators, -10), - test2 = lambda accelerators: ("test2" in accelerators, -9), + cpu = cms.SwitchProducer.getCpu(), + test1 = lambda accelerators: ("test1" in accelerators, 2), + test2 = lambda accelerators: ("test2" in accelerators, 3), ), **kargs) process = TestProcess() process.options.accelerators = ["{}"] process.ProcessAcceleratorTest = ProcessAcceleratorTest() process.s = SwitchProducerTest( + cpu = cms.EDProducer('IntProducer', ivalue = cms.int32(0)), test1 = {}, test2 = {} ) @@ -62,10 +64,10 @@ TEST_CASE("Configuration", s_tag) { const std::string test1{"cms.EDProducer('IntProducer', ivalue = cms.int32(1))"}; const std::string test2{"cms.EDProducer('ManyIntProducer', ivalue = cms.int32(2), values = cms.VPSet())"}; - const std::string baseConfig_auto = makeConfig(true, test1, test2, "auto"); + const std::string baseConfig_auto = makeConfig(true, test1, test2, "*"); const std::string baseConfig_test1 = makeConfig(true, test1, test2, "test1"); const std::string baseConfig_test2 = makeConfig(true, test1, test2, "test2"); - const std::string baseConfigTest2Disabled_auto = makeConfig(false, test1, test2, "auto"); + const std::string baseConfigTest2Disabled_auto = makeConfig(false, test1, test2, "*"); const std::string baseConfigTest2Disabled_test1 = makeConfig(false, test1, test2, "test1"); const std::string baseConfigTest2Disabled_test2 = makeConfig(false, test1, test2, "test2"); @@ -118,7 +120,7 @@ TEST_CASE("Configuration", s_tag) { REQUIRE_NOTHROW(tester.testLuminosityBlockWithNoEvents()); } - SECTION("Test2 enabled, acclerators=auto") { + SECTION("Test2 enabled, acclerators=*") { edm::test::TestProcessor tester(config_auto); auto event = tester.test(); REQUIRE(event.get()->value == 2); @@ -136,7 +138,7 @@ TEST_CASE("Configuration", s_tag) { REQUIRE(event.get()->value == 2); } - SECTION("Test2 disabled, accelerators=auto") { + SECTION("Test2 disabled, accelerators=*") { edm::test::TestProcessor tester(configTest2Disabled_auto); auto event = tester.test(); REQUIRE(event.get()->value == 1); @@ -149,7 +151,8 @@ TEST_CASE("Configuration", s_tag) { } SECTION("Test2 disabled, accelerators=test2") { - REQUIRE_THROWS_WITH(edm::test::TestProcessor(configTest2Disabled_test2), - Catch::Contains("Compute accelerators test2 were requested but are not available")); + REQUIRE_THROWS_WITH( + edm::test::TestProcessor(configTest2Disabled_test2), + Catch::Contains("The system has no compute accelerators that match the patterns") && Catch::Contains("test1")); } } diff --git a/FWCore/Integration/test/run_TestProcessAccelerator.sh b/FWCore/Integration/test/run_TestProcessAccelerator.sh index 8826eb6cadaba..3a6eef0a705f2 100755 --- a/FWCore/Integration/test/run_TestProcessAccelerator.sh +++ b/FWCore/Integration/test/run_TestProcessAccelerator.sh @@ -9,11 +9,11 @@ function die { echo Failure $1: status $2 ; exit $2 ; } pushd ${LOCAL_TMP_DIR} echo "*************************************************" -echo "accelerators=auto" +echo "accelerators=*" cmsRun ${LOCAL_TEST_DIR}/${test}_cfg.py || die "cmsRun ${test}_cfg.py" $? echo "*************************************************" -echo "accelerators=auto, enableTest2" +echo "accelerators=*, enableTest2" cmsRun ${LOCAL_TEST_DIR}/${test}_cfg.py -- --enableTest2 || die "cmsRun ${test}_cfg.py -- --enableTest2" $? echo "*************************************************" diff --git a/FWCore/Integration/test/testProcessAccelerator_cfg.py b/FWCore/Integration/test/testProcessAccelerator_cfg.py index d9d4dedba9d5e..b5f8b95694b1a 100644 --- a/FWCore/Integration/test/testProcessAccelerator_cfg.py +++ b/FWCore/Integration/test/testProcessAccelerator_cfg.py @@ -29,8 +29,9 @@ class SwitchProducerTest(cms.SwitchProducer): def __init__(self, **kargs): super(SwitchProducerTest,self).__init__( dict( - test1 = lambda accelerators: ("test1" in accelerators, -10), - test2 = lambda accelerators: ("test2" in accelerators, -9), + cpu = cms.SwitchProducer.getCpu(), + test1 = lambda accelerators: ("test1" in accelerators, 2), + test2 = lambda accelerators: ("test2" in accelerators, 3), ), **kargs) process = cms.Process("PROD1") @@ -44,13 +45,15 @@ def __init__(self, **kargs): process.intProducer1 = cms.EDProducer("ManyIntProducer", ivalue = cms.int32(1)) process.intProducer2 = cms.EDProducer("ManyIntProducer", ivalue = cms.int32(2)) +process.failIntProducer = cms.EDProducer("ManyIntProducer", ivalue = cms.int32(-1), throw = cms.untracked.bool(True)) -if args.enableTest2 and ("test2" in process.options.accelerators or "auto" in process.options.accelerators): +if args.enableTest2 and ("test2" in process.options.accelerators or "*" in process.options.accelerators): process.intProducer1.throw = cms.untracked.bool(True) else: process.intProducer2.throw = cms.untracked.bool(True) process.intProducer = SwitchProducerTest( + cpu = cms.EDProducer("AddIntsProducer", labels = cms.VInputTag("failIntProducer")), test1 = cms.EDProducer("AddIntsProducer", labels = cms.VInputTag("intProducer1")), test2 = cms.EDProducer("AddIntsProducer", labels = cms.VInputTag("intProducer2")) ) @@ -58,9 +61,10 @@ def __init__(self, **kargs): process.intConsumer = cms.EDProducer("AddIntsProducer", labels = cms.VInputTag("intProducer")) process.t = cms.Task( + process.failIntProducer, process.intProducer1, process.intProducer2, - process.intProducer + process.intProducer, ) process.p = cms.Path( process.intConsumer, diff --git a/FWCore/ParameterSet/python/Config.py b/FWCore/ParameterSet/python/Config.py index 0afe053598d8f..95b68368c7ebd 100644 --- a/FWCore/ParameterSet/python/Config.py +++ b/FWCore/ParameterSet/python/Config.py @@ -229,7 +229,7 @@ def defaultOptions_(): allowAnyLabel_ = required.untracked.uint32 ) ), - accelerators = untracked.vstring('auto'), + accelerators = untracked.vstring('*'), wantSummary = untracked.bool(False), fileMode = untracked.string('FULLMERGE'), forceEventSetupCacheClearOnNewRun = untracked.bool(False), @@ -1443,31 +1443,38 @@ def validate(self): pass def handleProcessAccelerators(self, parameterSet): - # Sanity check - useSet = set(self.options.accelerators.value()) - accSet = set([label for acc in self.__dict__['_Process__accelerators'].values() for label in acc.labels()]) - accSet.add("auto") - diff = useSet.difference(accSet) - if len(diff) > 0: - invalid = ",".join(diff) - valid = ",".join(sorted(list(accSet))) - raise ValueError("Invalid value{} of {} in process.options.accelerators, valid values are {}".format("s" if len(diff) > 2 else "", - invalid, - valid)) - - availableAccelerators = set() + allAccelerators = set(label for acc in self.__dict__['_Process__accelerators'].values() for label in acc.labels()) + # 'cpu' accelerator is always implicitly there + availableAccelerators = set(["cpu"]) for acc in self.__dict__['_Process__accelerators'].values(): for l in acc.enabledLabels(): availableAccelerators.add(l) availableAccelerators = sorted(list(availableAccelerators)) parameterSet.addVString(False, "@available_accelerators", availableAccelerators) - # Resolve 'auto' - if "auto" in self.options.accelerators: + # Resolve wildcards + if "*" in self.options.accelerators: if len(self.options.accelerators) >= 2: - raise ValueError("process.options.accelerators may contain 'auto' only as the only element, now it has {} elements".format(len(self.options.accelerators))) - newValue = set() - self.options.accelerators = list(availableAccelerators) + raise ValueError("process.options.accelerators may contain '*' only as the only element, now it has {} elements".format(len(self.options.accelerators))) + self.options.accelerators = availableAccelerators + else: + import fnmatch + resolved = set() + invalid = [] + for pattern in self.options.accelerators: + acc = [a for a in availableAccelerators if fnmatch.fnmatchcase(a, pattern)] + if len(acc) == 0: + if not any(fnmatch.fnmatchcase(a, pattern) for a in allAccelerators): + invalid.append(pattern) + else: + resolved.update(acc) + # Sanity check + if len(invalid) != 0: + raise ValueError("Invalid pattern{} of {} in process.options.accelerators, valid values are {} or a pattern matching to some of them.".format( + "s" if len(invalid) > 2 else "", + ",".join(invalid), + ",".join(sorted(list(allAccelerators))))) + self.options.accelerators = sorted(list(resolved)) # Customize for acc in self.__dict__['_Process__accelerators'].values(): @@ -1908,7 +1915,7 @@ def enabledLabels(self): def apply(self, process): """Override if need to customize the Process at worker node. The available accelerator labels are available via - 'process.options.accelerators' (the 'auto' has been expanded + 'process.options.accelerators' (the patterns, e.g. '*' have been expanded to concrete labels at this point). This function may touch only untracked parameters.""" @@ -2032,9 +2039,9 @@ def __init__(self, **kargs): specialImportRegistry.registerSpecialImportForType(SwitchProducerTest2, "from test import SwitchProducerTest2") class ProcessAcceleratorTest(ProcessAccelerator): - def __init__(self, enabled=["test1", "test2"]): + def __init__(self, enabled=["test1", "test2", "anothertest3"]): super(ProcessAcceleratorTest,self).__init__() - self._labels = ["test1", "test2"] + self._labels = ["test1", "test2", "anothertest3"] self.setEnabled(enabled) def setEnabled(self, enabled): invalid = set(enabled).difference(set(self._labels)) @@ -2230,7 +2237,7 @@ def testProcessDumpPython(self): IgnoreCompletely = cms.untracked.vstring(), Rethrow = cms.untracked.vstring(), SkipEvent = cms.untracked.vstring(), - accelerators = cms.untracked.vstring('auto'), + accelerators = cms.untracked.vstring('*'), allowUnscheduled = cms.obsolete.untracked.bool, canDeleteEarly = cms.untracked.vstring(), deleteNonConsumedUnscheduledModules = cms.untracked.bool(True), @@ -3912,6 +3919,14 @@ def testProcessFragment(self): p.extend(f) self.assertTrue(hasattr(p,'fltr')) def testProcessAccelerator(self): + proc = Process("TEST") + p = TestMakePSet() + proc.fillProcessDesc(p) + self.assertFalse(p.values["options"][1].values["accelerators"][0]) + self.assertTrue(["cpu"], p.values["options"][1].values["accelerators"][1]) + self.assertFalse(p.values["@available_accelerators"][0]) + self.assertTrue(["cpu"], p.values["@available_accelerators"][1]) + proc = Process("TEST") self.assertRaises(TypeError, setattr, proc, "processAcceleratorTest", ProcessAcceleratorTest()) proc.ProcessAcceleratorTest = ProcessAcceleratorTest() @@ -3936,7 +3951,7 @@ def testProcessAccelerator(self): IgnoreCompletely = cms.untracked.vstring(), Rethrow = cms.untracked.vstring(), SkipEvent = cms.untracked.vstring(), - accelerators = cms.untracked.vstring('auto'), + accelerators = cms.untracked.vstring('*'), allowUnscheduled = cms.obsolete.untracked.bool, canDeleteEarly = cms.untracked.vstring(), deleteNonConsumedUnscheduledModules = cms.untracked.bool(True), @@ -3962,7 +3977,7 @@ def testProcessAccelerator(self): ) process.ProcessAcceleratorTest = ProcessAcceleratorTest( - enabled = ['test1', 'test2'] + enabled = ['test1', 'test2', 'anothertest3'] ) @@ -3970,21 +3985,17 @@ def testProcessAccelerator(self): p = TestMakePSet() proc.fillProcessDesc(p) self.assertFalse(p.values["options"][1].values["accelerators"][0]) - accelerators = p.values["options"][1].values["accelerators"][1] - self.assertTrue("test1" in accelerators) - self.assertTrue("test2" in accelerators) + self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["options"][1].values["accelerators"][1]) self.assertEqual((True, "AcceleratorTestProducer"), p.values["acceleratorTestProducer"][1].values["@module_type"]) self.assertFalse(p.values["@available_accelerators"][0]) - availableAccelerators = p.values["@available_accelerators"][1] - self.assertTrue("test1" in availableAccelerators) - self.assertTrue("test2" in availableAccelerators) + self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1]) proc = Process("TEST") proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"]) p = TestMakePSet() proc.fillProcessDesc(p) - self.assertEqual(["test1"], p.values["options"][1].values["accelerators"][1]) - self.assertEqual(["test1"], p.values["@available_accelerators"][1]) + self.assertEqual(["cpu", "test1"], p.values["options"][1].values["accelerators"][1]) + self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1]) proc = Process("TEST") proc.ProcessAcceleratorTest = ProcessAcceleratorTest() @@ -3993,16 +4004,23 @@ def testProcessAccelerator(self): proc.fillProcessDesc(p) self.assertEqual(["test2"], p.values["options"][1].values["accelerators"][1]) availableAccelerators = p.values["@available_accelerators"][1] - self.assertTrue("test1" in availableAccelerators) - self.assertTrue("test2" in availableAccelerators) + self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1]) + + proc = Process("TEST") + proc.ProcessAcceleratorTest = ProcessAcceleratorTest() + proc.options.accelerators = ["test*"] + proc.fillProcessDesc(p) + accelerators = p.values["options"][1].values["accelerators"][1] + self.assertEqual(["test1", "test2"], p.values["options"][1].values["accelerators"][1]) + self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1]) proc = Process("TEST") proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"]) proc.options.accelerators = ["test2"] p = TestMakePSet() proc.fillProcessDesc(p) - self.assertEqual(["test2"], p.values["options"][1].values["accelerators"][1]) - self.assertEqual(["test1"], p.values["@available_accelerators"][1]) + self.assertEqual([], p.values["options"][1].values["accelerators"][1]) + self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1]) proc = Process("TEST") proc.ProcessAcceleratorTest = ProcessAcceleratorTest() @@ -4012,10 +4030,23 @@ def testProcessAccelerator(self): proc = Process("TEST") proc.ProcessAcceleratorTest = ProcessAcceleratorTest() - proc.options.accelerators = ["auto", "test1"] + proc.options.accelerators = ["*", "test1"] p = TestMakePSet() self.assertRaises(ValueError, proc.fillProcessDesc, p) + proc = Process("TEST") + proc.ProcessAcceleratorTest = ProcessAcceleratorTest() + proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo", + a = int32(1), + b = PSet(c = int32(2))), + test1 = EDProducer("Bar", + aa = int32(11), + bb = PSet(cc = int32(12)))) + proc.p = Path(proc.sp) + p = TestMakePSet() + proc.fillProcessDesc(p) + self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"]) + proc = Process("TEST") proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"]) proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo", @@ -4029,6 +4060,47 @@ def testProcessAccelerator(self): proc.fillProcessDesc(p) self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"]) + proc = Process("TEST") + proc.ProcessAcceleratorTest = ProcessAcceleratorTest() + proc.options.accelerators = ["test1"] + proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo", + a = int32(1), + b = PSet(c = int32(2))), + test1 = EDProducer("Bar", + aa = int32(11), + bb = PSet(cc = int32(12)))) + proc.p = Path(proc.sp) + p = TestMakePSet() + proc.fillProcessDesc(p) + self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"]) + + proc = Process("TEST") + proc.ProcessAcceleratorTest = ProcessAcceleratorTest() + proc.options.accelerators = ["test*"] + proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo", + a = int32(1), + b = PSet(c = int32(2))), + test1 = EDProducer("Bar", + aa = int32(11), + bb = PSet(cc = int32(12)))) + proc.p = Path(proc.sp) + p = TestMakePSet() + proc.fillProcessDesc(p) + self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"]) + + proc = Process("TEST") + proc.ProcessAcceleratorTest = ProcessAcceleratorTest() + proc.options.accelerators = ["anothertest3"] + proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo", + a = int32(1), + b = PSet(c = int32(2))), + test1 = EDProducer("Bar", + aa = int32(11), + bb = PSet(cc = int32(12)))) + proc.p = Path(proc.sp) + p = TestMakePSet() + self.assertRaises(RuntimeError, proc.fillProcessDesc, p) + import pickle proc = Process("TEST") proc.ProcessAcceleratorTest = ProcessAcceleratorTest() @@ -4044,14 +4116,12 @@ def testProcessAccelerator(self): p = TestMakePSet() unpkl.fillProcessDesc(p) self.assertEqual((False, "sp@test2"), p.values["sp"][1].values["@chosen_case"]) - availableAccelerators = p.values["@available_accelerators"][1] - self.assertTrue("test1" in availableAccelerators) - self.assertTrue("test2" in availableAccelerators) + self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1]) unpkl = pickle.loads(pkl) unpkl.ProcessAcceleratorTest.setEnabled(["test1"]) p = TestMakePSet() unpkl.fillProcessDesc(p) self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"]) - self.assertEqual(["test1"], p.values["@available_accelerators"][1]) + self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1]) unittest.main() diff --git a/FWCore/ParameterSet/src/validateTopLevelParameterSets.cc b/FWCore/ParameterSet/src/validateTopLevelParameterSets.cc index 1d04d84f1c681..042f4e5240f64 100644 --- a/FWCore/ParameterSet/src/validateTopLevelParameterSets.cc +++ b/FWCore/ParameterSet/src/validateTopLevelParameterSets.cc @@ -50,11 +50,16 @@ namespace edm { eventSetupDescription.addUntracked("forceNumberOfConcurrentIOVs", nestedDescription); description.addUntracked("eventSetup", eventSetupDescription); - description.addUntracked>("accelerators", {"auto"}) + description.addUntracked>("accelerators", {"*"}) ->setComment( - "Specify the compute accelerator(s) the job should use. Empty value means that no accelerators will be " - "used. Special value 'auto', can be used for automatically deduce what accelerators to use based on the " - "available hardware."); + "Specify the set of compute accelerator(s) the job is allowed to use. The values can contain the direct " + "names of accelerators supported by the ProcessAccelerators defined in the configuration, or patterns " + "matching to them (patterns use '*' and '?' wildcards similar to shell). The actual set of accelerators to " + "be used is determined on the worker node based on the available hardware. A CPU fallback with the name " + "'cpu' is always included in the set of available accelerators. If no accelerator matching to the patterns " + "are available on the worker node, the job is terminated with a specific error code. Same happens if an " + "empty value is given in the configuration. Default value is pattern '*', which implies use of any " + "supported and available hardware (including the CPU fallback)."); description.addUntracked("wantSummary", false) ->setComment("Set true to print a report on the trigger decisions and timing of modules"); description.addUntracked("fileMode", "FULLMERGE") diff --git a/HeterogeneousCore/CUDACore/test/test_ProcessAcceleratorCUDA.cc b/HeterogeneousCore/CUDACore/test/test_ProcessAcceleratorCUDA.cc index 0e674e85bbbbd..44336a993efbd 100644 --- a/HeterogeneousCore/CUDACore/test/test_ProcessAcceleratorCUDA.cc +++ b/HeterogeneousCore/CUDACore/test/test_ProcessAcceleratorCUDA.cc @@ -37,8 +37,8 @@ TEST_CASE("Configuration", s_tag) { const std::string test1{"cms.EDProducer('IntProducer', ivalue = cms.int32(1))"}; const std::string test2{"cms.EDProducer('ManyIntProducer', ivalue = cms.int32(2), values = cms.VPSet())"}; - const std::string baseConfig_auto = makeConfig(test1, test2, "'auto'"); - const std::string baseConfig_cpu = makeConfig(test1, test2, ""); + const std::string baseConfig_auto = makeConfig(test1, test2, "'*'"); + const std::string baseConfig_cpu = makeConfig(test1, test2, "'cpu'"); const std::string baseConfig_cuda = makeConfig(test1, test2, "'gpu-nvidia'"); SECTION("Configuration hash is not changed") { diff --git a/HeterogeneousCore/CUDATest/test/runtests.sh b/HeterogeneousCore/CUDATest/test/runtests.sh index ea2d9417559a3..c1d17e2c8a23e 100755 --- a/HeterogeneousCore/CUDATest/test/runtests.sh +++ b/HeterogeneousCore/CUDATest/test/runtests.sh @@ -28,7 +28,7 @@ cmsRun ${TEST_DIR}/testCUDASwitch_cfg.py -- --silent || die "cmsRun testCUDASwit echo "*************************************************" echo "CUDA producer configuration with SwitchProducer, force CPU" -cmsRun ${TEST_DIR}/testCUDASwitch_cfg.py -- --silent --accelerator="" || die "cmsRun testCUDASwitch_cfg.py --silent --accelerator=\"\"" $? +cmsRun ${TEST_DIR}/testCUDASwitch_cfg.py -- --silent --accelerator="cpu" || die "cmsRun testCUDASwitch_cfg.py --silent --accelerator=\"\"" $? if [ "x${TARGET}" == "xgpu" ]; then echo "*************************************************" diff --git a/HeterogeneousCore/CUDATest/test/testCUDASwitch_cfg.py b/HeterogeneousCore/CUDATest/test/testCUDASwitch_cfg.py index c3592dae9a61e..944c1c765ad7d 100644 --- a/HeterogeneousCore/CUDATest/test/testCUDASwitch_cfg.py +++ b/HeterogeneousCore/CUDATest/test/testCUDASwitch_cfg.py @@ -28,10 +28,7 @@ if args.accelerator is not None: - if len(args.accelerator) == 0: - process.options.accelerators = [] - else: - process.options.accelerators = [args.accelerator] + process.options.accelerators = [args.accelerator] # Flow diagram of the modules # From fc821ee64cdecaca6c5f86f31fdd19b9cb575942 Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Mon, 14 Feb 2022 16:01:06 +0100 Subject: [PATCH 7/9] Add a new Accelerators_cff to collect all 'standard' ProcessAccelerators Thanks to Andrea Bocci for the suggestion. --- .../StandardSequences/python/Accelerators_cff.py | 6 ++++++ Configuration/StandardSequences/python/Services_cff.py | 9 +++++---- 2 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 Configuration/StandardSequences/python/Accelerators_cff.py diff --git a/Configuration/StandardSequences/python/Accelerators_cff.py b/Configuration/StandardSequences/python/Accelerators_cff.py new file mode 100644 index 0000000000000..aeef6ea5f367b --- /dev/null +++ b/Configuration/StandardSequences/python/Accelerators_cff.py @@ -0,0 +1,6 @@ +import FWCore.ParameterSet.Config as cms + +# This fragment is intended to collect all ProcessAccelerator objects +# used in production + +from HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi import ProcessAcceleratorCUDA diff --git a/Configuration/StandardSequences/python/Services_cff.py b/Configuration/StandardSequences/python/Services_cff.py index 3925df5f9ed41..c8c0b1e63b226 100644 --- a/Configuration/StandardSequences/python/Services_cff.py +++ b/Configuration/StandardSequences/python/Services_cff.py @@ -8,13 +8,14 @@ # DQM store service from DQMServices.Core.DQMStore_cfi import * -# load CUDA services when the "gpu" or "pixelNtupletFit" modifiers are enabled -def _addCUDAServices(process): - process.load("HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi") +# load ProcessAccelerators (that set the e.g. the necessary CUDA +# stuff) when the "gpu" or "pixelNtupletFit" modifiers are enabled +def _addProcessAccelerators(process): + process.load("Configuration.StandardSequences.Accelerators_cff") from Configuration.ProcessModifiers.gpu_cff import gpu from Configuration.ProcessModifiers.pixelNtupletFit_cff import pixelNtupletFit -modifyConfigurationStandardSequencesServicesAddCUDAServices_ = (gpu | pixelNtupletFit).makeProcessModifier(_addCUDAServices) +modifyConfigurationStandardSequencesServicesAddProcessAccelerators_ = (gpu | pixelNtupletFit).makeProcessModifier(_addProcessAccelerators) # load TritonService when SONIC workflow is enabled def _addTritonService(process): From 5caf7b8ebd70362cfe3a3134e7b5ef33e827b86d Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Wed, 16 Feb 2022 21:27:45 +0100 Subject: [PATCH 8/9] Use specific vstring to pass selected accelerators instead of overwriting options.accelerators Also add unit tests for two ProcessAccelerator objects in the Process --- .../src/ensureAvailableAccelerators.cc | 13 ++- FWCore/ParameterSet/python/Config.py | 95 ++++++++++++++----- .../CUDACore/python/ProcessAcceleratorCUDA.py | 4 +- 3 files changed, 84 insertions(+), 28 deletions(-) diff --git a/FWCore/Framework/src/ensureAvailableAccelerators.cc b/FWCore/Framework/src/ensureAvailableAccelerators.cc index f706c215b99eb..2b6134ee29905 100644 --- a/FWCore/Framework/src/ensureAvailableAccelerators.cc +++ b/FWCore/Framework/src/ensureAvailableAccelerators.cc @@ -6,17 +6,24 @@ namespace edm { void ensureAvailableAccelerators(edm::ParameterSet const& parameterSet) { + auto const& selectedAccelerators = + parameterSet.getUntrackedParameter>("@selected_accelerators"); ParameterSet const& optionsPset(parameterSet.getUntrackedParameterSet("options")); - auto accelerators = optionsPset.getUntrackedParameter>("accelerators"); - if (accelerators.empty()) { + if (selectedAccelerators.empty()) { Exception ex(errors::UnavailableAccelerator); ex << "The system has no compute accelerators that match the patterns specified in " - "process.options.accelerators.\nThe following compute accelerators are available:\n"; + "process.options.accelerators:\n"; + auto const& patterns = optionsPset.getUntrackedParameter>("accelerators"); + for (auto const& pat : patterns) { + ex << " " << pat << "\n"; + } + ex << "\nThe following compute accelerators are available:\n"; auto const& availableAccelerators = parameterSet.getUntrackedParameter>("@available_accelerators"); for (auto const& acc : availableAccelerators) { ex << " " << acc << "\n"; } + throw ex; } } diff --git a/FWCore/ParameterSet/python/Config.py b/FWCore/ParameterSet/python/Config.py index 95b68368c7ebd..12138cda850ae 100644 --- a/FWCore/ParameterSet/python/Config.py +++ b/FWCore/ParameterSet/python/Config.py @@ -1159,9 +1159,10 @@ def _insertManyInto(self, parameterSet, label, itemDict, tracked): def _insertSwitchProducersInto(self, parameterSet, labelModules, labelAliases, itemDict, tracked): modules = parameterSet.getVString(tracked, labelModules) aliases = parameterSet.getVString(tracked, labelAliases) + accelerators = parameterSet.getVString(False, "@selected_accelerators") for name,value in itemDict.items(): value.appendToProcessDescLists_(modules, aliases, name) - value.insertInto(parameterSet, name, self.options.accelerators) + value.insertInto(parameterSet, name, accelerators) modules.sort() aliases.sort() parameterSet.addVString(tracked, labelModules, modules) @@ -1443,20 +1444,21 @@ def validate(self): pass def handleProcessAccelerators(self, parameterSet): - allAccelerators = set(label for acc in self.__dict__['_Process__accelerators'].values() for label in acc.labels()) # 'cpu' accelerator is always implicitly there + allAccelerators = set(["cpu"]) availableAccelerators = set(["cpu"]) for acc in self.__dict__['_Process__accelerators'].values(): - for l in acc.enabledLabels(): - availableAccelerators.add(l) + allAccelerators.update(acc.labels()) + availableAccelerators.update(acc.enabledLabels()) availableAccelerators = sorted(list(availableAccelerators)) parameterSet.addVString(False, "@available_accelerators", availableAccelerators) # Resolve wildcards + selectedAccelerators = [] if "*" in self.options.accelerators: if len(self.options.accelerators) >= 2: raise ValueError("process.options.accelerators may contain '*' only as the only element, now it has {} elements".format(len(self.options.accelerators))) - self.options.accelerators = availableAccelerators + selectedAccelerators = availableAccelerators else: import fnmatch resolved = set() @@ -1474,11 +1476,12 @@ def handleProcessAccelerators(self, parameterSet): "s" if len(invalid) > 2 else "", ",".join(invalid), ",".join(sorted(list(allAccelerators))))) - self.options.accelerators = sorted(list(resolved)) + selectedAccelerators = sorted(list(resolved)) + parameterSet.addVString(False, "@selected_accelerators", selectedAccelerators) # Customize for acc in self.__dict__['_Process__accelerators'].values(): - acc.apply(self) + acc.apply(self, selectedAccelerators) def prefer(self, esmodule,*args,**kargs): """Prefer this ES source or producer. The argument can @@ -1912,13 +1915,14 @@ def enabledLabels(self): """Override to return a list of strings for the accelerator labels that are enabled in the system the job is being run on.""" return [] - def apply(self, process): + def apply(self, process, accelerators): """Override if need to customize the Process at worker node. The - available accelerator labels are available via - 'process.options.accelerators' (the patterns, e.g. '*' have been expanded - to concrete labels at this point). + selected available accelerator labels are given in the + 'accelerators' argument (the patterns, e.g. '*' have been + expanded to concrete labels). - This function may touch only untracked parameters.""" + This function may touch only untracked parameters. + """ pass # Need to be a module-level function for the configuration with a @@ -2056,11 +2060,33 @@ def labels(self): return self._labels def enabledLabels(self): return self._enabled - def apply(self, process): + def apply(self, process, accelerators): process.acceleratorTestProducer = EDProducer("AcceleratorTestProducer") process.acceleratorTestPath = Path(process.acceleratorTestProducer) specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorTest, "from test import ProcessAcceleratorTest") + class ProcessAcceleratorTest2(ProcessAccelerator): + def __init__(self, enabled=["anothertest3", "anothertest4"]): + super(ProcessAcceleratorTest2,self).__init__() + self._labels = ["anothertest3", "anothertest4"] + self.setEnabled(enabled) + def setEnabled(self, enabled): + invalid = set(enabled).difference(set(self._labels)) + if len(invalid) > 0: + raise Exception("Tried to enabled nonexistent test accelerators {}".format(",".join(invalid))) + self._enabled = enabled[:] + def dumpPythonImpl(self,options): + result = "{}enabled = [{}]".format(options.indentation(), + ", ".join(["'{}'".format(e) for e in self._enabled])) + return result + def labels(self): + return self._labels + def enabledLabels(self): + return self._enabled + def apply(self, process, accelerators): + pass + specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorTest2, "from test import ProcessAcceleratorTest2") + class TestModuleCommand(unittest.TestCase): def setUp(self): """Nothing to do """ @@ -3922,10 +3948,9 @@ def testProcessAccelerator(self): proc = Process("TEST") p = TestMakePSet() proc.fillProcessDesc(p) - self.assertFalse(p.values["options"][1].values["accelerators"][0]) - self.assertTrue(["cpu"], p.values["options"][1].values["accelerators"][1]) - self.assertFalse(p.values["@available_accelerators"][0]) self.assertTrue(["cpu"], p.values["@available_accelerators"][1]) + self.assertFalse(p.values["@selected_accelerators"][0]) + self.assertTrue(["cpu"], p.values["@selected_accelerators"][1]) proc = Process("TEST") self.assertRaises(TypeError, setattr, proc, "processAcceleratorTest", ProcessAcceleratorTest()) @@ -3984,8 +4009,9 @@ def testProcessAccelerator(self): """) p = TestMakePSet() proc.fillProcessDesc(p) + self.assertEqual(["*"], p.values["options"][1].values["accelerators"][1]) self.assertFalse(p.values["options"][1].values["accelerators"][0]) - self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["options"][1].values["accelerators"][1]) + self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["@selected_accelerators"][1]) self.assertEqual((True, "AcceleratorTestProducer"), p.values["acceleratorTestProducer"][1].values["@module_type"]) self.assertFalse(p.values["@available_accelerators"][0]) self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1]) @@ -3994,7 +4020,7 @@ def testProcessAccelerator(self): proc.ProcessAcceleratorTest = ProcessAcceleratorTest(enabled=["test1"]) p = TestMakePSet() proc.fillProcessDesc(p) - self.assertEqual(["cpu", "test1"], p.values["options"][1].values["accelerators"][1]) + self.assertEqual(["cpu", "test1"], p.values["@selected_accelerators"][1]) self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1]) proc = Process("TEST") @@ -4002,16 +4028,14 @@ def testProcessAccelerator(self): proc.options.accelerators = ["test2"] p = TestMakePSet() proc.fillProcessDesc(p) - self.assertEqual(["test2"], p.values["options"][1].values["accelerators"][1]) - availableAccelerators = p.values["@available_accelerators"][1] + self.assertEqual(["test2"], p.values["@selected_accelerators"][1]) self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1]) proc = Process("TEST") proc.ProcessAcceleratorTest = ProcessAcceleratorTest() proc.options.accelerators = ["test*"] proc.fillProcessDesc(p) - accelerators = p.values["options"][1].values["accelerators"][1] - self.assertEqual(["test1", "test2"], p.values["options"][1].values["accelerators"][1]) + self.assertEqual(["test1", "test2"], p.values["@selected_accelerators"][1]) self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1]) proc = Process("TEST") @@ -4019,9 +4043,17 @@ def testProcessAccelerator(self): proc.options.accelerators = ["test2"] p = TestMakePSet() proc.fillProcessDesc(p) - self.assertEqual([], p.values["options"][1].values["accelerators"][1]) + self.assertEqual([], p.values["@selected_accelerators"][1]) self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1]) + proc = Process("TEST") + proc.ProcessAcceleratorTest = ProcessAcceleratorTest() + proc.options.accelerators = ["cpu*"] + p = TestMakePSet() + proc.fillProcessDesc(p) + self.assertEqual(["cpu"], p.values["@selected_accelerators"][1]) + self.assertEqual(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1]) + proc = Process("TEST") proc.ProcessAcceleratorTest = ProcessAcceleratorTest() proc.options.accelerators = ["test3"] @@ -4034,6 +4066,23 @@ def testProcessAccelerator(self): p = TestMakePSet() self.assertRaises(ValueError, proc.fillProcessDesc, p) + proc = Process("TEST") + proc.ProcessAcceleratorTest = ProcessAcceleratorTest() + proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2() + p = TestMakePSet() + proc.fillProcessDesc(p) + self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@selected_accelerators"][1]) + self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@available_accelerators"][1]) + + proc = Process("TEST") + proc.ProcessAcceleratorTest = ProcessAcceleratorTest() + proc.ProcessAcceleratorTest2 = ProcessAcceleratorTest2() + proc.options.accelerators = ["*test3", "c*"] + p = TestMakePSet() + proc.fillProcessDesc(p) + self.assertEqual(["anothertest3", "cpu"], p.values["@selected_accelerators"][1]) + self.assertEqual(["anothertest3", "anothertest4", "cpu", "test1", "test2"], p.values["@available_accelerators"][1]) + proc = Process("TEST") proc.ProcessAcceleratorTest = ProcessAcceleratorTest() proc.sp = SwitchProducerTest2(test2 = EDProducer("Foo", diff --git a/HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA.py b/HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA.py index f9ce7f1590746..3a1990bcc2824 100644 --- a/HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA.py +++ b/HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA.py @@ -14,11 +14,11 @@ def enabledLabels(self): return self.labels() else: return [] - def apply(self, process): + def apply(self, process, accelerators): if not hasattr(process, "CUDAService"): process.load("HeterogeneousCore.CUDAServices.CUDAService_cfi") - if self._label in process.options.accelerators: + if self._label in accelerators: process.CUDAService.enabled = True process.MessageLogger.CUDAService = cms.untracked.PSet() else: From e0d39f3428b026249f8097a60a6cb3fd82b47261 Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Fri, 18 Feb 2022 21:04:38 +0100 Subject: [PATCH 9/9] Wrap Process into more constraining class for ProcessAccelerator.apply() --- FWCore/ParameterSet/python/Config.py | 56 ++++++++++++++++--- .../CUDACore/python/ProcessAcceleratorCUDA.py | 3 +- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/FWCore/ParameterSet/python/Config.py b/FWCore/ParameterSet/python/Config.py index 12138cda850ae..0533e1bc96af9 100644 --- a/FWCore/ParameterSet/python/Config.py +++ b/FWCore/ParameterSet/python/Config.py @@ -1480,8 +1480,9 @@ def handleProcessAccelerators(self, parameterSet): parameterSet.addVString(False, "@selected_accelerators", selectedAccelerators) # Customize + wrapped = ProcessForProcessAccelerator(self) for acc in self.__dict__['_Process__accelerators'].values(): - acc.apply(self, selectedAccelerators) + acc.apply(wrapped, selectedAccelerators) def prefer(self, esmodule,*args,**kargs): """Prefer this ES source or producer. The argument can @@ -1883,10 +1884,14 @@ class ProcessAccelerator(_ConfigureComponent,_Unlabelable): specific customization can be applied to the Process on a worker node at the point where the python configuration is serialized for C++. - The customizations must touch only untracked parameters. Each - deriving class should have a specific unit test enabling all - combinations of accelerators and assert that the configuration - hash does not change.""" + The customization must not change the configuration hash. To + enforce this reuirement, the customization gets a + ProcessForProcessAccelerator wrapper that gives access to only + those parts of the configuration that can be changed. Nevertheless + it would be good to have specific unit test for each deriving + class to ensure that all combinations of the enabled accelerators + give the same configuration hash. + """ def __init__(self): pass def _place(self, name, proc): @@ -1925,6 +1930,28 @@ def apply(self, process, accelerators): """ pass +class ProcessForProcessAccelerator(object): + """This class is inteded to wrap the Process object to constrain the + available functionality for ProcessAccelerator.apply()""" + def __init__(self, process): + self.__process = process + def __getattr__(self, label): + value = getattr(self.__process, label) + if not isinstance(value, Service): + raise TypeError("ProcessAccelerator.apply() can get only Services. Tried to get {} with label {}".format(str(type(value)), label)) + return value + def __setattr__(self, label, value): + if label == "_ProcessForProcessAccelerator__process": + super().__setattr__(label, value) + else: + if not isinstance(value, Service): + raise TypeError("ProcessAccelerator.apply() can only set Services. Tried to set {} with label {}".format(str(type(value)), label)) + setattr(self.__process, label, value) + def add_(self, value): + if not isinstance(value, Service): + raise TypeError("ProcessAccelerator.apply() can only add Services. Tried to set {} with label {}".format(str(type(value)), label)) + self.__process.add_(value) + # Need to be a module-level function for the configuration with a # SwitchProducer to be pickleable. def _switchproducer_test2_case1(accelerators): @@ -2061,8 +2088,7 @@ def labels(self): def enabledLabels(self): return self._enabled def apply(self, process, accelerators): - process.acceleratorTestProducer = EDProducer("AcceleratorTestProducer") - process.acceleratorTestPath = Path(process.acceleratorTestProducer) + process.AcceleratorTestService = Service("AcceleratorTestService") specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorTest, "from test import ProcessAcceleratorTest") class ProcessAcceleratorTest2(ProcessAccelerator): @@ -3944,6 +3970,20 @@ def testProcessFragment(self): p = Process('PROCESS') p.extend(f) self.assertTrue(hasattr(p,'fltr')) + def testProcessForProcessAccelerator(self): + proc = Process("TEST") + p = ProcessForProcessAccelerator(proc) + p.TestService = Service("TestService") + self.assertTrue(hasattr(proc, "TestService")) + self.assertEqual(proc.TestService.type_(), "TestService") + self.assertRaises(TypeError, setattr, p, "a", EDProducer("Foo")) + p.add_(Service("TestServiceTwo")) + self.assertTrue(hasattr(proc, "TestServiceTwo")) + self.assertEqual(proc.TestServiceTwo.type_(), "TestServiceTwo") + p.TestService.foo = untracked.uint32(42) + self.assertEqual(proc.TestService.foo.value(), 42) + proc.mod = EDProducer("Producer") + self.assertRaises(TypeError, getattr, p, "mod") def testProcessAccelerator(self): proc = Process("TEST") p = TestMakePSet() @@ -4012,7 +4052,7 @@ def testProcessAccelerator(self): self.assertEqual(["*"], p.values["options"][1].values["accelerators"][1]) self.assertFalse(p.values["options"][1].values["accelerators"][0]) self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["@selected_accelerators"][1]) - self.assertEqual((True, "AcceleratorTestProducer"), p.values["acceleratorTestProducer"][1].values["@module_type"]) + self.assertEqual("AcceleratorTestService", p.values["services"][1][0].values["@service_type"][1]) self.assertFalse(p.values["@available_accelerators"][0]) self.assertTrue(["anothertest3", "cpu", "test1", "test2"], p.values["@available_accelerators"][1]) diff --git a/HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA.py b/HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA.py index 3a1990bcc2824..19aca3fc6e4ce 100644 --- a/HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA.py +++ b/HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA.py @@ -16,7 +16,8 @@ def enabledLabels(self): return [] def apply(self, process, accelerators): if not hasattr(process, "CUDAService"): - process.load("HeterogeneousCore.CUDAServices.CUDAService_cfi") + from HeterogeneousCore.CUDAServices.CUDAService_cfi import CUDAService + process.add_(CUDAService) if self._label in accelerators: process.CUDAService.enabled = True