Skip to content

Commit

Permalink
Merge pull request #31269 from makortel/edaliasAllProducts
Browse files Browse the repository at this point in the history 
Add support for wildcard in EDAlias type field
  • Loading branch information
@cmsbuild
cmsbuild authored Sep 3, 2020
2 parents 194c5f2 + 5340da8 commit f2d3972
Show file tree
Hide file tree
Showing 4 changed files with 306 additions and 1 deletion.
42 changes: 41 additions & 1 deletion FWCore/Framework/src/Schedule.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,14 @@ namespace edm {

std::map<BranchKey, BranchKey> aliasKeys; // Used to search for duplicates or clashes.

// Auxiliary search structure to support wildcard for friendlyClassName
std::multimap<std::string, BranchKey> moduleLabelToBranches;
for (auto const& prod : preg.productList()) {
if (processName == prod.second.processName()) {
moduleLabelToBranches.emplace(prod.first.moduleLabel(), prod.first);
}
}

// Now, loop over the alias information and store it in aliasMap.
for (std::string const& alias : aliases) {
ParameterSet const& aliasPSet = proc_pset.getParameterSet(alias);
Expand All @@ -243,7 +251,39 @@ namespace edm {
std::string friendlyClassName = pset.getParameter<std::string>("type");
std::string productInstanceName = pset.getParameter<std::string>("fromProductInstance");
std::string instanceAlias = pset.getParameter<std::string>("toProductInstance");
if (productInstanceName == star) {

if (friendlyClassName == star) {
bool processHasLabel = false;
bool match = false;
for (auto it = moduleLabelToBranches.lower_bound(moduleLabel);
it != moduleLabelToBranches.end() && it->first == moduleLabel;
++it) {
processHasLabel = true;
if (productInstanceName != star and productInstanceName != it->second.productInstanceName()) {
continue;
}
match = true;

checkAndInsertAlias(it->second.friendlyClassName(),
moduleLabel,
it->second.productInstanceName(),
processName,
alias,
instanceAlias,
preg,
aliasMap,
aliasKeys);
}
if (not match and processHasLabel) {
// No product was found matching the alias.
// We throw an exception only if a module with the specified module label was created in this process.
// Note that if that condition is ever relatex, it might be best to throw an exception with different
// message (omitting productInstanceName) in case 'productInstanceName == start'
throw Exception(errors::Configuration, "EDAlias parameter set mismatch\n")
<< "There are no products with module label '" << moduleLabel << "' and product instance name '"
<< productInstanceName << "'.\n";
}
} else if (productInstanceName == star) {
bool match = false;
BranchKey lowerBound(friendlyClassName, moduleLabel, empty, empty);
for (ProductRegistry::ProductList::const_iterator it = preg.productList().lower_bound(lowerBound);
Expand Down
1 change: 1 addition & 0 deletions FWCore/Integration/test/BuildFile.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@
<use name="FWCore/TestProcessor"/>
<use name="DataFormats/Provenance"/>
<use name="catch2"/>
<use name="fmt"/>
</bin>

<library file="TestPSetAnalyzer.cc" name="TestPSetAnalyzer">
Expand Down
239 changes: 239 additions & 0 deletions FWCore/Integration/test/EDAlias_t.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include "FWCore/TestProcessor/interface/TestProcessor.h"

#include <iostream>
#include <fmt/format.h>

static constexpr auto s_tag = "[EDAlias]";

Expand Down Expand Up @@ -116,3 +117,241 @@ process.moduleToTest(process.test, cms.Task(process.intprod))
Catch::Contains("One has module label") && Catch::Contains("the other has module label"));
}
}

////////////////////////////////////////
TEST_CASE("Configuration with all products of a module", s_tag) {
const std::string baseConfig{
R"_(from FWCore.TestProcessor.TestProcess import *
import FWCore.ParameterSet.Config as cms
process = TestProcess()
process.intprod = cms.EDProducer('ManyIntProducer', ivalue = cms.int32(2),
values = cms.VPSet(
cms.PSet(instance=cms.string('foo'),value=cms.int32(3)),
cms.PSet(instance=cms.string('another'),value=cms.int32(4)),
)
)
process.intalias = cms.EDAlias(intprod = cms.EDAlias.allProducts())
# Module to be tested can not be an EDAlias
process.test = cms.EDProducer('AddIntsProducer',
labels = cms.VInputTag('intalias', ('intalias', 'foo'), ('intalias', 'another'))
)
process.moduleToTest(process.test, cms.Task(process.intprod))
)_"};

edm::test::TestProcessor::Config config{baseConfig};

SECTION("Base configuration is OK") { REQUIRE_NOTHROW(edm::test::TestProcessor(config)); }

SECTION("No event data") {
edm::test::TestProcessor tester(config);
REQUIRE_NOTHROW(tester.test());
}

SECTION("beginJob and endJob only") {
edm::test::TestProcessor tester(config);
REQUIRE_NOTHROW(tester.testBeginAndEndJobOnly());
}

SECTION("Run with no LuminosityBlocks") {
edm::test::TestProcessor tester(config);
REQUIRE_NOTHROW(tester.testRunWithNoLuminosityBlocks());
}

SECTION("LuminosityBlock with no Events") {
edm::test::TestProcessor tester(config);
REQUIRE_NOTHROW(tester.testLuminosityBlockWithNoEvents());
}

SECTION("Getting value") {
edm::test::TestProcessor tester(config);
auto event = tester.test();
REQUIRE(event.get<edmtest::IntProduct>()->value == 9);
}
}

TEST_CASE("Configuration with all products of a module with a given product instance name", s_tag) {
const std::string baseConfig{
R"_(from FWCore.TestProcessor.TestProcess import *
import FWCore.ParameterSet.Config as cms
process = TestProcess()
process.intprod = cms.EDProducer('ManyIntProducer', ivalue = cms.int32(2),
values = cms.VPSet(
cms.PSet(instance=cms.string('foo'),value=cms.int32(3)),
cms.PSet(instance=cms.string('another'),value=cms.int32(4)),
)
)
process.intalias = cms.EDAlias(intprod = cms.VPSet(cms.PSet(type = cms.string('*'), fromProductInstance = cms.string('another'))))
# Module to be tested can not be an EDAlias
process.test = cms.EDProducer('AddIntsProducer',
labels = cms.VInputTag({})
)
process.moduleToTest(process.test, cms.Task(process.intprod))
)_"};

edm::test::TestProcessor::Config config{fmt::format(baseConfig, "'intalias:another'")};

SECTION("Base configuration is OK") { REQUIRE_NOTHROW(edm::test::TestProcessor(config)); }

SECTION("No event data") {
edm::test::TestProcessor tester(config);
REQUIRE_NOTHROW(tester.test());
}

SECTION("beginJob and endJob only") {
edm::test::TestProcessor tester(config);
REQUIRE_NOTHROW(tester.testBeginAndEndJobOnly());
}

SECTION("Run with no LuminosityBlocks") {
edm::test::TestProcessor tester(config);
REQUIRE_NOTHROW(tester.testRunWithNoLuminosityBlocks());
}

SECTION("LuminosityBlock with no Events") {
edm::test::TestProcessor tester(config);
REQUIRE_NOTHROW(tester.testLuminosityBlockWithNoEvents());
}

SECTION("Getting value") {
edm::test::TestProcessor tester(config);
auto event = tester.test();
REQUIRE(event.get<edmtest::IntProduct>()->value == 4);
}

SECTION("Other product instances are not aliased") {
{
edm::test::TestProcessor::Config config{fmt::format(baseConfig, "'intalias:foo'")};
edm::test::TestProcessor tester(config);
REQUIRE_THROWS_WITH(tester.test(), Catch::Contains("ProductNotFound"));
}
{
edm::test::TestProcessor::Config config{fmt::format(baseConfig, "'intalias'")};
edm::test::TestProcessor tester(config);
REQUIRE_THROWS_WITH(tester.test(), Catch::Contains("ProductNotFound"));
}
}
}

////////////////////////////////////////
TEST_CASE("Configuration with all products of two modules", s_tag) {
const std::string baseConfig{
R"_(from FWCore.TestProcessor.TestProcess import *
import FWCore.ParameterSet.Config as cms
process = TestProcess()
process.intprod = cms.EDProducer('ManyIntProducer', ivalue = cms.int32(2),
values = cms.VPSet(
cms.PSet(instance=cms.string('foo'),value=cms.int32(3)),
cms.PSet(instance=cms.string('another'),value=cms.int32(4)),
)
)
process.intprod2 = cms.EDProducer('ManyIntProducer', ivalue = cms.int32(20),
values = cms.VPSet(
cms.PSet(instance=cms.string('foo2'),value=cms.int32(30)),
cms.PSet(instance=cms.string('another2'),value=cms.int32(40)),
)
)
process.intalias = cms.EDAlias(
intprod = cms.EDAlias.allProducts(),
# can't use allProducts() because the product instance '' would lead to duplicate brances to be aliased
intprod2 = cms.VPSet(
cms.PSet(type = cms.string('*'), fromProductInstance = cms.string('foo2')),
cms.PSet(type = cms.string('*'), fromProductInstance = cms.string('another2')),
)
)
# Module to be tested can not be an EDAlias
process.test = cms.EDProducer('AddIntsProducer',
labels = cms.VInputTag('intalias', ('intalias', 'foo'), ('intalias', 'another'), ('intalias', 'foo2'), ('intalias', 'another2'))
)
process.moduleToTest(process.test, cms.Task(process.intprod, process.intprod2))
)_"};

edm::test::TestProcessor::Config config{baseConfig};

SECTION("Base configuration is OK") { REQUIRE_NOTHROW(edm::test::TestProcessor(config)); }

SECTION("No event data") {
edm::test::TestProcessor tester(config);
REQUIRE_NOTHROW(tester.test());
}

SECTION("beginJob and endJob only") {
edm::test::TestProcessor tester(config);
REQUIRE_NOTHROW(tester.testBeginAndEndJobOnly());
}

SECTION("Run with no LuminosityBlocks") {
edm::test::TestProcessor tester(config);
REQUIRE_NOTHROW(tester.testRunWithNoLuminosityBlocks());
}

SECTION("LuminosityBlock with no Events") {
edm::test::TestProcessor tester(config);
REQUIRE_NOTHROW(tester.testLuminosityBlockWithNoEvents());
}

SECTION("Getting value") {
edm::test::TestProcessor tester(config);
auto event = tester.test();
REQUIRE(event.get<edmtest::IntProduct>()->value == 79);
}
}

////////////////////////////////////////
TEST_CASE("No products found with wildcards", s_tag) {
const std::string baseConfig{
R"_(from FWCore.TestProcessor.TestProcess import *
import FWCore.ParameterSet.Config as cms
process = TestProcess()
process.intprod = cms.EDProducer('ManyIntProducer', ivalue = cms.int32(2),
values = cms.VPSet(
cms.PSet(instance=cms.string('foo'),value=cms.int32(3)),
cms.PSet(instance=cms.string('another'),value=cms.int32(4)),
)
)
process.intalias = cms.EDAlias({})
# Module to be tested can not be an EDAlias
process.test = cms.EDProducer('AddIntsProducer',
labels = cms.VInputTag('intalias')
)
process.moduleToTest(process.test, cms.Task(process.intprod))
)_"};

SECTION("Type wildcard") {
edm::test::TestProcessor::Config config{fmt::format(
baseConfig,
"intprod = cms.VPSet(cms.PSet(type = cms.string('*'), fromProductInstance = cms.string('nonexistent')))")};

REQUIRE_THROWS_WITH(
edm::test::TestProcessor(config),
Catch::Contains("There are no products with module label 'intprod' and product instance name 'nonexistent'"));
}

SECTION("Instance wildcard") {
edm::test::TestProcessor::Config config{
fmt::format(baseConfig, "intprod = cms.VPSet(cms.PSet(type = cms.string('nonexistentType')))")};

REQUIRE_THROWS_WITH(edm::test::TestProcessor(config),
Catch::Contains("There are no products of type 'nonexistentType'") &&
Catch::Contains("with module label 'intprod'"));
}
}
25 changes: 25 additions & 0 deletions FWCore/ParameterSet/python/Types.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1362,6 +1362,15 @@ class EDAlias(_ConfigureComponent,_Labelable,_Parameterizable):
def __init__(self,*arg,**kargs):
super(EDAlias,self).__init__(**kargs)

@staticmethod
def allProducts():
"""A helper to specify that all products of a module are to be aliased for. Example usage:
process.someAlias = cms.EDAlias(
aliasForModuleLabel = cms.EDAlias.allProducts()
)
"""
return VPSet(PSet(type = string('*')))

def clone(self, *args, **params):
returnValue = EDAlias.__new__(type(self))
myparams = self.parameters_()
Expand Down Expand Up @@ -1974,6 +1983,22 @@ def testEDAlias(self):
self.assertTrue(hasattr(aliasfoo4, "foo3"))
self.assertEqual(aliasfoo4.foo3[0].type, "Foo3")

aliasfoo5 = EDAlias(foo5 = EDAlias.allProducts())
self.assertEqual(len(aliasfoo5.foo5), 1)
self.assertEqual(aliasfoo5.foo5[0].type.value(), "*")
self.assertFalse(hasattr(aliasfoo5.foo5[0], "fromProductInstance"))
self.assertFalse(hasattr(aliasfoo5.foo5[0], "toProductInstance"))

aliasfoo6 = aliasfoo5.clone(foo5 = None, foo6 = EDAlias.allProducts())
self.assertFalse(hasattr(aliasfoo6, "foo5"))
self.assertTrue(hasattr(aliasfoo6, "foo6"))
self.assertEqual(len(aliasfoo6.foo6), 1)
self.assertEqual(aliasfoo6.foo6[0].type.value(), "*")

aliasfoo7 = EDAlias(foo5 = EDAlias.allProducts(), foo6 = EDAlias.allProducts())
self.assertEqual(len(aliasfoo7.foo5), 1)
self.assertEqual(len(aliasfoo7.foo6), 1)

def testFileInPath(self):
f = FileInPath("FWCore/ParameterSet/python/Types.py")
self.assertEqual(f.configValue(), "'FWCore/ParameterSet/python/Types.py'")
Expand Down

0 comments on commit f2d3972

Please sign in to comment.