Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add ProcessAcceleratorCUDA, and replace the direct use of CUDAService…
Browse files Browse the repository at this point in the history
… with it
makortel committed Jan 12, 2022
1 parent b705376 commit 29129e5
Showing 10 changed files with 157 additions and 27 deletions.
4 changes: 1 addition & 3 deletions Configuration/StandardSequences/python/Services_cff.py
Original file line number Diff line number Diff line change
@@ -10,9 +10,7 @@

# load CUDA services when the "gpu" or "pixelNtupletFit" modifiers are enabled
def _addCUDAServices(process):
process.load("HeterogeneousCore.CUDAServices.CUDAService_cfi")
process.load("FWCore.MessageService.MessageLogger_cfi")
process.MessageLogger.CUDAService = cms.untracked.PSet()
process.load("HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi")

from Configuration.ProcessModifiers.gpu_cff import gpu
from Configuration.ProcessModifiers.pixelNtupletFit_cff import pixelNtupletFit
27 changes: 27 additions & 0 deletions HeterogeneousCore/CUDACore/python/ProcessAcceleratorCUDA.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import FWCore.ParameterSet.Config as cms

import os

class ProcessAcceleratorCUDA(cms.ProcessAccelerator):
def __init__(self):
super(ProcessAcceleratorCUDA,self).__init__()
self._label = "gpu-nvidia"
def labels(self):
return [self._label]
def enabledLabels(self):
enabled = (os.system("cudaIsEnabled") == 0)
if enabled:
return self.labels()
else:
return []
def apply(self, process):
if not hasattr(process, "CUDAService"):
process.load("HeterogeneousCore.CUDAServices.CUDAService_cfi")

if self._label in process.options.accelerators:
process.CUDAService.enabled = True
process.MessageLogger.CUDAService = cms.untracked.PSet()
else:
process.CUDAService.enabled = False

cms.specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorCUDA, "from HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA import ProcessAcceleratorCUDA")
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
from HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA import ProcessAcceleratorCUDA as _ProcessAcceleratorCUDA

ProcessAcceleratorCUDA = _ProcessAcceleratorCUDA()
11 changes: 3 additions & 8 deletions HeterogeneousCore/CUDACore/python/SwitchProducerCUDA.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,8 @@
import FWCore.ParameterSet.Config as cms

_cuda_enabled_cached = None

def _switch_cuda():
global _cuda_enabled_cached
if _cuda_enabled_cached is None:
import os
_cuda_enabled_cached = (os.system("cudaIsEnabled") == 0)
return (_cuda_enabled_cached, 2)
def _switch_cuda(useAccelerators):
have_gpu = ("gpu-nvidia" in useAccelerators)
return (have_gpu, 2)

class SwitchProducerCUDA(cms.SwitchProducer):
def __init__(self, **kargs):
1 change: 1 addition & 0 deletions HeterogeneousCore/CUDACore/test/BuildFile.xml
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@
<use name="HeterogeneousCore/CUDACore"/>
<use name="catch2"/>
<use name="cuda"/>
<use name="fmt"/>
</bin>

<bin file="testStreamEvent.cu" name="testHeterogeneousCoreCUDACoreStreamEvent">
55 changes: 55 additions & 0 deletions HeterogeneousCore/CUDACore/test/test_ProcessAcceleratorCUDA.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
#include "catch.hpp"

#include "FWCore/ParameterSet/interface/ParameterSet.h"
#include "FWCore/ParameterSetReader/interface/ParameterSetReader.h"
#include "FWCore/TestProcessor/interface/TestProcessor.h"

#include <fmt/format.h>

#include <iostream>
#include <string_view>

static constexpr auto s_tag = "[ProcessAcceleratorCUDA]";

namespace {
std::string makeConfig(std::string_view cpu,
std::string_view cuda,
std::string_view accelerator) {
return fmt::format(
R"_(from FWCore.TestProcessor.TestProcess import *
import FWCore.ParameterSet.Config as cms
process = TestProcess()
process.options.accelerators = [{}]
process.load("HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi")
from HeterogeneousCore.CUDACore.SwitchProducerCUDA import *
process.s = SwitchProducerCUDA(
cpu = {},
cuda = {}
)
process.moduleToTest(process.s)
)_", accelerator, cpu, cuda);
}
} // namespace

TEST_CASE("Configuration", s_tag) {
const std::string test1{"cms.EDProducer('IntProducer', ivalue = cms.int32(1))"};
const std::string test2{"cms.EDProducer('ManyIntProducer', ivalue = cms.int32(2), values = cms.VPSet())"};

const std::string baseConfig_auto = makeConfig(test1, test2, "'auto'");
const std::string baseConfig_cpu = makeConfig(test1, test2, "");
const std::string baseConfig_cuda = makeConfig(test1, test2, "'gpu-nvidia'");

SECTION("Configuration hash is not changed") {
auto pset_auto = edm::readConfig(baseConfig_auto);
auto pset_cpu = edm::readConfig(baseConfig_cpu);
auto pset_cuda = edm::readConfig(baseConfig_cuda);
pset_auto->registerIt();
pset_cpu->registerIt();
pset_cuda->registerIt();
REQUIRE(pset_auto->id() == pset_cpu->id());
REQUIRE(pset_auto->id() == pset_cuda->id());
}
}
7 changes: 6 additions & 1 deletion HeterogeneousCore/CUDATest/test/BuildFile.xml
Original file line number Diff line number Diff line change
@@ -5,5 +5,10 @@
<use name="catch2"/>
</bin>

<test name="TestCUDATest" command="runtests.sh"/>
<ifrelease name="_GPU_">
<test name="TestCUDATest" command="runtests.sh gpu"/>
<else/>
<test name="TestCUDATest" command="runtests.sh cpu"/>
</ifrelease>

</iftool>
40 changes: 38 additions & 2 deletions HeterogeneousCore/CUDATest/test/runtests.sh
Original file line number Diff line number Diff line change
@@ -4,6 +4,42 @@ function die { echo Failure $1: status $2 ; exit $2 ; }

TEST_DIR=src/HeterogeneousCore/CUDATest/test

if [ "x$#" != "x1" ]; then
die "Need exactly 1 argument ('cpu', 'gpu'), got $#" 1
fi
if [ "x$1" = "xgpu" ]; then
TARGET=gpu
elif [ "x$1" = "xcpu" ]; then
# In non-_GPU_ IBs, if CUDA is enabled, run the GPU-targeted tests
cudaIsEnabled
CUDA_ENABLED=$?
if [ "x${CUDA_ENABLED}" == "x0" ]; then
TARGET=gpu
else
TARGET=cpu
fi
else
die "Argument needs to be 'cpu' or 'gpu', got $1" 1
fi

echo "*************************************************"
echo "CUDA producer configuration with SwitchProducer, automatic"
cmsRun ${TEST_DIR}/testCUDASwitch_cfg.py -- --silent || die "cmsRun testCUDASwitch_cfg.py --silent" $?

echo "*************************************************"
echo "CUDA producer configuration with SwitchProducer"
cmsRun ${TEST_DIR}/testCUDASwitch_cfg.py || die "cmsRun testCUDASwitch_cfg.py 1" $?
echo "CUDA producer configuration with SwitchProducer, force CPU"
cmsRun ${TEST_DIR}/testCUDASwitch_cfg.py -- --silent --accelerator="" || die "cmsRun testCUDASwitch_cfg.py --silent --accelerator=\"\"" $?

if [ "x${TARGET}" == "xgpu" ]; then
echo "*************************************************"
echo "CUDA producer configuration with SwitchProducer, force GPU"
cmsRun ${TEST_DIR}/testCUDASwitch_cfg.py -- --silent --accelerator="gpu-nvidia" || die "cmsRun testCUDASwitch_cfg.py --silent --accelerator=gpu-nvidia" $?
elif [ "x${TARGET}" == "xcpu" ]; then
echo "*************************************************"
echo "CUDA producer configuration with SwitchProducer, force GPU, should fail"
cmsRun -j testCUDATest_jobreport.xml ${TEST_DIR}/testCUDASwitch_cfg.py -- --silent --accelerator="gpu-nvidia" && die "cmsRun testCUDASwitch_cfg.py --silent --accelerator=gpu-nvidia did not fail" 1
EXIT_CODE=$(edmFjrDump --exitCode testCUDATest_jobreport.xml)
if [ "x${EXIT_CODE}" != "x8035" ]; then
echo "Test (that was expected to fail) reported exit code ${EXIT_CODE} instead of expected 8035"
fi
fi
32 changes: 21 additions & 11 deletions HeterogeneousCore/CUDATest/test/testCUDASwitch_cfg.py
Original file line number Diff line number Diff line change
@@ -1,27 +1,37 @@
import FWCore.ParameterSet.Config as cms

silent = True
#silent = False
import argparse
import sys

#includeAnalyzer = True
includeAnalyzer = False
parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test CUDA EDProducers')

parser.add_argument("--silent", help="Silence printouts", action="store_true")
parser.add_argument("--includeAnalyzer", help="Include an EDAnalyzer", action="store_true")
parser.add_argument("--accelerator", type=str, help="String for accelerator to enable")

argv = sys.argv[:]
if '--' in argv:
argv.remove("--")
args, unknown = parser.parse_known_args(argv)

process = cms.Process("Test")
process.load("FWCore.MessageService.MessageLogger_cfi")
process.load("HeterogeneousCore.CUDAServices.CUDAService_cfi")
process.load("HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi")

process.source = cms.Source("EmptySource")

process.maxEvents.input = 3
if not silent:
if not args.silent:
process.maxEvents.input = 10
process.MessageLogger.cerr.threshold = cms.untracked.string("INFO")
process.MessageLogger.cerr.INFO.limit = process.MessageLogger.cerr.default.limit


#process.options.numberOfThreads = 4
process.options.numberOfStreams = 0
#process.Tracer = cms.Service("Tracer")
if args.accelerator is not None:
if len(args.accelerator) == 0:
process.options.accelerators = []
else:
process.options.accelerators = [args.accelerator]

# Flow diagram of the modules
#
@@ -69,7 +79,7 @@
# GPU analyzer (optionally)
from HeterogeneousCore.CUDATest.testCUDAAnalyzerGPU_cfi import testCUDAAnalyzerGPU
process.anaCUDA = testCUDAAnalyzerGPU.clone(src="prod6CUDA")
if silent:
if args.silent:
process.anaCUDA.minValue = 2.3e7
process.anaCUDA.maxValue = 2.5e7

@@ -95,7 +105,7 @@
process.prod6Task
)
process.p = cms.Path()
if includeAnalyzer:
if args.includeAnalyzer:
process.p += process.anaCUDA
process.p.associate(process.t)
process.ep = cms.EndPath(process.out)
Original file line number Diff line number Diff line change
@@ -16,7 +16,7 @@ TEST_CASE("Standard checks of TestCUDAProducerGPUFirst", s_tag) {
const std::string baseConfig{
R"_(from FWCore.TestProcessor.TestProcess import *
process = TestProcess()
process.load("HeterogeneousCore.CUDAServices.CUDAService_cfi")
process.load("HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi")
process.toTest = cms.EDProducer("TestCUDAProducerGPUFirst")
process.moduleToTest(process.toTest)
)_"};
@@ -57,7 +57,7 @@ TEST_CASE("TestCUDAProducerGPUFirst operation", s_tag) {
const std::string baseConfig{
R"_(from FWCore.TestProcessor.TestProcess import *
process = TestProcess()
process.load("HeterogeneousCore.CUDAServices.CUDAService_cfi")
process.load("HeterogeneousCore.CUDACore.ProcessAcceleratorCUDA_cfi")
process.toTest = cms.EDProducer("TestCUDAProducerGPUFirst")
process.moduleToTest(process.toTest)
)_"};

0 comments on commit 29129e5

Please sign in to comment.