msgtime:2024-04-29 13:43:42
doc_type:cmsswlog
date:2024-04-29T11:43:42.403Z
run:380115
host:fu-c2b03-28-01
pid:1451372
doctype:stacktrace
severity:FATAL
severityVal:4
instance:global
lexicalId:549852445
message:A fatal system signal has occurred: segmentation violation
The following is the call stack containing the origin of the signal.
Mon Apr 29 13:43:22 CEST 2024
Thread 44 (Thread 0x7fe9f0b7f700 (LWP 1454027) "cmsRun"):
#0  0x00007feba41a6da6 in do_futex_wait.constprop () from /lib64/libpthread.so.0
#1  0x00007feba41a6e98 in __new_sem_wait_slow.constprop.0 () from /lib64/libpthread.so.0
#2  0x00007feb855a12ea in ?? () from /lib64/libcuda.so.1
#3  0x00007feb855bf47f in ?? () from /lib64/libcuda.so.1
#4  0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#5  0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 43 (Thread 0x7fea3b7ff700 (LWP 1453491) "cmsRun"):
#0  0x00007feba41a47aa in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007feb9d7827cf in FedRawDataInputSource::readSupervisor() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libEventFilterUtilities.so
#2  0x00007feba4830a73 in std::execute_native_thread_routine (__p=0x7fea56804260) at ../../../../../libstdc++-v3/src/c++11/thread.cc:82
#3  0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#4  0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 42 (Thread 0x7fea3f9ff700 (LWP 1453475) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feba41a782d in __lll_lock_wait () from /lib64/libpthread.so.0
#5  0x00007feba41a0ad9 in pthread_mutex_lock () from /lib64/libpthread.so.0
#6  0x00007feb3e7c9646 in cms::alpakatools::CachingAllocator<alpaka::DevUniformCudaHipRt<alpaka::ApiCudaRt>, alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >::tryReuseCachedBlock(cms::alpakatools::CachingAllocator<alpaka::DevUniformCudaHipRt<alpaka::ApiCudaRt>, alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >::BlockDescriptor&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginRecoLocalCaloEcalRecProducersPluginsPortableCudaAsync.so
#7  0x00007feb3e7caffe in std::enable_if<((isQueue<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >)&&(is_unbounded_array_v<std::byte []>))&&(!(is_array_v<std::remove_extent<std::byte []>::type>)), cms::alpakatools::detail::buffer_type<alpaka::trait::DevType<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false>, void>::type, std::byte [], std::enable_if<isDevice<alpaka::trait::DevType<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false>, void>::type>, void>::type>::type>::type cms::alpakatools::make_device_buffer<std::byte [], alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >(alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> const&, unsigned int) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginRecoLocalCaloEcalRecProducersPluginsPortableCudaAsync.so
#8  0x00007feb3e7e876a in PortableDeviceCollection<EcalUncalibratedRecHitSoALayout<128ul, false>, alpaka::DevUniformCudaHipRt<alpaka::ApiCudaRt>, void>::PortableDeviceCollection<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false>, void>(int, alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginRecoLocalCaloEcalRecProducersPluginsPortableCudaAsync.so
#9  0x00007feb3e7e8c1d in alpaka_cuda_async::EcalUncalibRecHitProducerPortable::produce(alpaka_cuda_async::device::Event&, alpaka_cuda_async::device::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginRecoLocalCaloEcalRecProducersPluginsPortableCudaAsync.so
#10 0x00007feb3e7ef48c in alpaka_cuda_async::stream::SynchronizingEDProducer<>::produce(edm::Event&, edm::EventSetup const&) [clone .lto_priv.0] () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginRecoLocalCaloEcalRecProducersPluginsPortableCudaAsync.so
#11 0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#12 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#13 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#14 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#15 0x00007feba682692e in tbb::detail::d1::function_task<edm::WaitingTaskHolder::doneWaiting(std::__exception_ptr::exception_ptr)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#16 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea47fc1100, waiter=..., this=0x7feb9f6c5400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#17 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c5400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#18 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#19 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#20 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8300) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#21 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8300) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#22 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#23 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 41 (Thread 0x7fea40dff700 (LWP 1453474) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feb280fbe41 in PixelCPEGenericBase::initializeLocalErrorVariables(float&, float&, bool&, bool&, bool&, bool&, int&, int&, int&, int&, unsigned int&, unsigned int&, PixelCPEBase::DetParam const&, PixelCPEGenericBase::ClusterParamGeneric const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/libRecoLocalTrackerSiPixelRecHits.so
#5  0x00007feb280fc2cb in PixelCPEGeneric::localError(PixelCPEBase::DetParam const&, PixelCPEBase::ClusterParam&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/libRecoLocalTrackerSiPixelRecHits.so
#6  0x00007feb280d46e6 in PixelCPEBase::getParameters(SiPixelCluster const&, GeomDet const&, LocalTrajectoryParameters const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/libRecoLocalTrackerSiPixelRecHits.so
#7  0x00007feb5019fe9a in TkPixelMeasurementDet::compHits(TrajectoryStateOnSurface const&, MeasurementTrackerEvent const&, float, float) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerMeasurementDetPlugins.so
#8  0x00007feb501a04e9 in TkPixelMeasurementDet::measurements(TrajectoryStateOnSurface const&, MeasurementEstimator const&, MeasurementTrackerEvent const&, tracking::TempMeasurements&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerMeasurementDetPlugins.so
#9  0x00007feb50107472 in LayerMeasurements::groupedMeasurements(DetLayer const&, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsMeasurementDet.so
#10 0x00007fea9e0534b3 in GroupedCkfTrajectoryBuilder::advanceOneLayer(TrajectorySeed const&, TempTrajectory&, TrajectoryFilter const*, Propagator const*, bool, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&) const [clone .constprop.0] () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#11 0x00007fea9e047a63 in GroupedCkfTrajectoryBuilder::groupedLimitedCandidates(TrajectorySeed const&, TempTrajectory const&, TrajectoryFilter const*, Propagator const*, bool, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#12 0x00007fea9e04ae8f in GroupedCkfTrajectoryBuilder::rebuildTrajectories(TempTrajectory const&, TrajectorySeed const&, std::vector<Trajectory, std::allocator<Trajectory> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#13 0x00007fea9cfd203c in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&)::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#14 0x00007fea9cfd31df in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#15 0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#16 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#17 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#18 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#19 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#20 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea41106900, waiter=..., this=0x7feb9f6c5380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#21 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c5380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#22 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#23 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#24 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8280) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#25 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8280) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#26 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#27 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 40 (Thread 0x7fea41bff700 (LWP 1453473) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feba41a782d in __lll_lock_wait () from /lib64/libpthread.so.0
#5  0x00007feba41a0ad9 in pthread_mutex_lock () from /lib64/libpthread.so.0
#6  0x00007feb3e7c9646 in cms::alpakatools::CachingAllocator<alpaka::DevUniformCudaHipRt<alpaka::ApiCudaRt>, alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >::tryReuseCachedBlock(cms::alpakatools::CachingAllocator<alpaka::DevUniformCudaHipRt<alpaka::ApiCudaRt>, alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >::BlockDescriptor&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginRecoLocalCaloEcalRecProducersPluginsPortableCudaAsync.so
#7  0x00007feb3e7caffe in std::enable_if<((isQueue<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >)&&(is_unbounded_array_v<std::byte []>))&&(!(is_array_v<std::remove_extent<std::byte []>::type>)), cms::alpakatools::detail::buffer_type<alpaka::trait::DevType<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false>, void>::type, std::byte [], std::enable_if<isDevice<alpaka::trait::DevType<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false>, void>::type>, void>::type>::type>::type cms::alpakatools::make_device_buffer<std::byte [], alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >(alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> const&, unsigned int) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginRecoLocalCaloEcalRecProducersPluginsPortableCudaAsync.so
#8  0x00007feb3e7e876a in PortableDeviceCollection<EcalUncalibratedRecHitSoALayout<128ul, false>, alpaka::DevUniformCudaHipRt<alpaka::ApiCudaRt>, void>::PortableDeviceCollection<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false>, void>(int, alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginRecoLocalCaloEcalRecProducersPluginsPortableCudaAsync.so
#9  0x00007feb3e7e8c1d in alpaka_cuda_async::EcalUncalibRecHitProducerPortable::produce(alpaka_cuda_async::device::Event&, alpaka_cuda_async::device::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginRecoLocalCaloEcalRecProducersPluginsPortableCudaAsync.so
#10 0x00007feb3e7ef48c in alpaka_cuda_async::stream::SynchronizingEDProducer<>::produce(edm::Event&, edm::EventSetup const&) [clone .lto_priv.0] () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginRecoLocalCaloEcalRecProducersPluginsPortableCudaAsync.so
#11 0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#12 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#13 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#14 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#15 0x00007feba682692e in tbb::detail::d1::function_task<edm::WaitingTaskHolder::doneWaiting(std::__exception_ptr::exception_ptr)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#16 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea3edc8b00, waiter=..., this=0x7feb9f6c5300) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#17 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c5300) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#18 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#19 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#20 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8200) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#21 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8200) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#22 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#23 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 39 (Thread 0x7fea42bff700 (LWP 1453472) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feb3f5b803c in MultipleScatteringUpdator::compute(TrajectoryStateOnSurface const&, PropagationDirection, materialEffect::Effect&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsMaterialEffects.so
#5  0x00007feb3f5b837f in CombinedMaterialEffectsUpdator::compute(TrajectoryStateOnSurface const&, PropagationDirection, materialEffect::Effect&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsMaterialEffects.so
#6  0x00007feb3f5b79bc in MaterialEffectsUpdator::updateStateInPlace(TrajectoryStateOnSurface&, PropagationDirection) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsMaterialEffects.so
#7  0x00007feb3f5b9dcf in PropagatorWithMaterial::propagateWithPath(TrajectoryStateOnSurface const&, Plane const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsMaterialEffects.so
#8  0x00007feb9092c90c in GeomDetCompatibilityChecker::isCompatible(GeomDet const*, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsDetLayers.so
#9  0x00007feb52f8ceda in CompatibleDetToGroupAdder::add(GeomDet const&, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&, std::vector<DetGroup, std::allocator<DetGroup> >&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerTkDetLayers.so
#10 0x00007feb52f8f910 in Phase1PixelBlade::groupedCompatibleDetsV(TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&, std::vector<DetGroup, std::allocator<DetGroup> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerTkDetLayers.so
#11 0x00007feb52f8dcd7 in CompatibleDetToGroupAdder::add(GeometricSearchDet const&, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&, std::vector<DetGroup, std::allocator<DetGroup> >&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerTkDetLayers.so
#12 0x00007feb52f94db0 in PixelForwardLayerPhase1::groupedCompatibleDetsV(TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&, std::vector<DetGroup, std::allocator<DetGroup> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerTkDetLayers.so
#13 0x00007feb9092a39a in GeometricSearchDet::compatibleDetsV(TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&, std::vector<std::pair<GeomDet const*, TrajectoryStateOnSurface>, std::allocator<std::pair<GeomDet const*, TrajectoryStateOnSurface> > >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsDetLayers.so
#14 0x00007feb909297a1 in GeometricSearchDet::compatibleDets(TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsDetLayers.so
#15 0x00007feb501065c4 in LayerMeasurements::measurements(DetLayer const&, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsMeasurementDet.so
#16 0x00007fea9cfe434e in CkfTrajectoryBuilder::findCompatibleMeasurements(TrajectorySeed const&, TempTrajectory const&, std::vector<TrajectoryMeasurement, std::allocator<TrajectoryMeasurement> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#17 0x00007fea9cfe2d53 in CkfTrajectoryBuilder::limitedCandidates(std::shared_ptr<TrajectorySeed const> const&, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&, std::vector<Trajectory, std::allocator<Trajectory> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#18 0x00007fea9cfdc71d in CkfTrajectoryBuilder::limitedCandidates(TrajectorySeed const&, TempTrajectory&, std::vector<Trajectory, std::allocator<Trajectory> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#19 0x00007fea9cfdc8ff in CkfTrajectoryBuilder::buildTrajectories(TrajectorySeed const&, std::vector<Trajectory, std::allocator<Trajectory> >&, unsigned int&, TrajectoryFilter const*) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#20 0x00007fea9cfd1c56 in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&)::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#21 0x00007fea9cfd31df in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#22 0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#23 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#24 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#25 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#26 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#27 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fe9ec013d00, waiter=..., this=0x7feb9f6c5200) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#28 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c5200) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#29 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#30 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#31 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8900) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#32 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8900) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#33 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#34 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 38 (Thread 0x7fea439fe700 (LWP 1453471) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feb8cfc3c41 in AnalyticalCurvilinearJacobian::computeFullJacobian(GlobalTrajectoryParameters const&, Point3DBase<float, GlobalTag> const&, Vector3DBase<float, GlobalTag> const&, Vector3DBase<float, GlobalTag> const&, double const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsAnalyticalJacobians.so
#5  0x00007feb8cfc49aa in AnalyticalCurvilinearJacobian::AnalyticalCurvilinearJacobian(GlobalTrajectoryParameters const&, Point3DBase<float, GlobalTag> const&, Vector3DBase<float, GlobalTag> const&, double const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsAnalyticalJacobians.so
#6  0x00007feb9090c463 in AnalyticalPropagator::propagatedStateWithPath(FreeTrajectoryState const&, Surface const&, GlobalTrajectoryParameters const&, double const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsGeomPropagators.so
#7  0x00007feb90917053 in AnalyticalPropagator::propagateWithPath(FreeTrajectoryState const&, Plane const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsGeomPropagators.so
#8  0x00007feb9090c255 in Propagator::propagateWithPath(TrajectoryStateOnSurface const&, Plane const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsGeomPropagators.so
#9  0x00007feb3f5b9d1e in PropagatorWithMaterial::propagateWithPath(TrajectoryStateOnSurface const&, Plane const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsMaterialEffects.so
#10 0x00007feb9092c90c in GeomDetCompatibilityChecker::isCompatible(GeomDet const*, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsDetLayers.so
#11 0x00007feb52f8ceda in CompatibleDetToGroupAdder::add(GeomDet const&, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&, std::vector<DetGroup, std::allocator<DetGroup> >&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerTkDetLayers.so
#12 0x00007feb52f8f910 in Phase1PixelBlade::groupedCompatibleDetsV(TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&, std::vector<DetGroup, std::allocator<DetGroup> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerTkDetLayers.so
#13 0x00007feb52f8dcd7 in CompatibleDetToGroupAdder::add(GeometricSearchDet const&, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&, std::vector<DetGroup, std::allocator<DetGroup> >&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerTkDetLayers.so
#14 0x00007feb52f94db0 in PixelForwardLayerPhase1::groupedCompatibleDetsV(TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&, std::vector<DetGroup, std::allocator<DetGroup> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerTkDetLayers.so
#15 0x00007feb909297ed in GeometricSearchDet::groupedCompatibleDets(TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsDetLayers.so
#16 0x00007feb501072e8 in LayerMeasurements::groupedMeasurements(DetLayer const&, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsMeasurementDet.so
#17 0x00007fea9e0534b3 in GroupedCkfTrajectoryBuilder::advanceOneLayer(TrajectorySeed const&, TempTrajectory&, TrajectoryFilter const*, Propagator const*, bool, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&) const [clone .constprop.0] () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#18 0x00007fea9e047a63 in GroupedCkfTrajectoryBuilder::groupedLimitedCandidates(TrajectorySeed const&, TempTrajectory const&, TrajectoryFilter const*, Propagator const*, bool, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#19 0x00007fea9e049db6 in GroupedCkfTrajectoryBuilder::buildTrajectories(TrajectorySeed const&, std::vector<Trajectory, std::allocator<Trajectory> >&, unsigned int&, TrajectoryFilter const*) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#20 0x00007fea9cfd1c56 in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&)::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#21 0x00007fea9cfd31df in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#22 0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#23 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#24 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#25 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#26 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#27 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7feb55c61b00, waiter=..., this=0x7feb9f6c5180) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#28 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c5180) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#29 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#30 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#31 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8880) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#32 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8880) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#33 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#34 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 37 (Thread 0x7fea447ff700 (LWP 1453470) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feb5011c44a in TrajectoryStateOnSurface (anonymous namespace)::lupdate<2u>(TrajectoryStateOnSurface const&, TrackingRecHit const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsKalmanUpdators.so
#5  0x00007feb50126306 in KFUpdator::update(TrajectoryStateOnSurface const&, TrackingRecHit const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsKalmanUpdators.so
#6  0x00007fea978b783e in SeedFromConsecutiveHitsCreator::makeSeed(std::vector<TrajectorySeed, std::allocator<TrajectorySeed> >&, SeedingHitSet const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerTkSeedGeneratorPlugins.so
#7  0x00007fea978b8a68 in SeedCreatorFromRegionHitsEDProducerT<SeedFromConsecutiveHitsCreator>::produce(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerTkSeedGeneratorPlugins.so
#8  0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#9  0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#10 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#12 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#13 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea3ed0d500, waiter=..., this=0x7feb9f6c5100) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#14 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c5100) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#15 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#16 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#17 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8a00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#18 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8a00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#19 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#20 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 36 (Thread 0x7fea457ff700 (LWP 1453469) "cmsRun"):
#0  0x00007feba3ef30e1 in poll () from /lib64/libc.so.6
#1  0x00007feb9b69d6af in full_read.constprop () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2  0x00007feb9b651dbc in edm::service::InitRootHandlers::stacktraceFromThread() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  0x00007feb9b652720 in sig_dostack_then_abort () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x00007feaa5495ef2 in HLTMuonL1TFilter::hltFilter(edm::Event&, edm::EventSetup const&, trigger::TriggerFilterObjectWithRefs&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginHLTriggerMuonAuto.so
#6  0x00007feb1f56fc2b in HLTFilter::filter(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libHLTriggerHLTcore.so
#7  0x00007feba6929ba2 in edm::global::EDFilterBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#8  0x00007feba692246e in edm::WorkerT<edm::global::EDFilterBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#9  0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#10 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00007feba682692e in tbb::detail::d1::function_task<edm::WaitingTaskHolder::doneWaiting(std::__exception_ptr::exception_ptr)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#12 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fe9ec14f100, waiter=..., this=0x7feb9f6c5080) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#13 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c5080) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#14 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#15 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#16 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8980) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#17 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8980) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#18 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#19 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 35 (Thread 0x7fea469ff700 (LWP 1453468) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feba41a782d in __lll_lock_wait () from /lib64/libpthread.so.0
#5  0x00007feba41a0ad9 in pthread_mutex_lock () from /lib64/libpthread.so.0
#6  0x00007feb3ef543d6 in cms::alpakatools::CachingAllocator<alpaka::DevUniformCudaHipRt<alpaka::ApiCudaRt>, alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >::tryReuseCachedBlock(cms::alpakatools::CachingAllocator<alpaka::DevUniformCudaHipRt<alpaka::ApiCudaRt>, alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >::BlockDescriptor&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginEventFilterEcalRawToDigiPluginsPortableCudaAsync.so
#7  0x00007feb3ef5778e in std::enable_if<((isQueue<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >)&&(is_unbounded_array_v<std::byte []>))&&(!(is_array_v<std::remove_extent<std::byte []>::type>)), cms::alpakatools::detail::buffer_type<alpaka::trait::DevType<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false>, void>::type, std::byte [], std::enable_if<isDevice<alpaka::trait::DevType<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false>, void>::type>, void>::type>::type>::type cms::alpakatools::make_device_buffer<std::byte [], alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >(alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> const&, unsigned int) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginEventFilterEcalRawToDigiPluginsPortableCudaAsync.so
#8  0x00007feb3ef62ada in PortableDeviceCollection<EcalDigiSoALayout<128ul, false>, alpaka::DevUniformCudaHipRt<alpaka::ApiCudaRt>, void>::PortableDeviceCollection<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false>, void>(int, alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginEventFilterEcalRawToDigiPluginsPortableCudaAsync.so
#9  0x00007feb3ef67299 in alpaka_cuda_async::EcalRawToDigiPortable::produce(alpaka_cuda_async::device::Event&, alpaka_cuda_async::device::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginEventFilterEcalRawToDigiPluginsPortableCudaAsync.so
#10 0x00007feb3ef62923 in alpaka_cuda_async::stream::EDProducer<>::produce(edm::Event&, edm::EventSetup const&) [clone .lto_priv.0] () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginEventFilterEcalRawToDigiPluginsPortableCudaAsync.so
#11 0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#12 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#13 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#14 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#15 0x00007feba682692e in tbb::detail::d1::function_task<edm::WaitingTaskHolder::doneWaiting(std::__exception_ptr::exception_ptr)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#16 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fe80c396800, waiter=..., this=0x7feb9f6c5000) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#17 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c5000) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#18 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#19 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#20 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#21 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#22 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#23 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 34 (Thread 0x7fea47bff700 (LWP 1453467) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feba41a782d in __lll_lock_wait () from /lib64/libpthread.so.0
#5  0x00007feba41a0ba4 in pthread_mutex_lock () from /lib64/libpthread.so.0
#6  0x00007feb8581018f in ?? () from /lib64/libcuda.so.1
#7  0x00007feb8552d4d7 in ?? () from /lib64/libcuda.so.1
#8  0x00007feb855fe009 in ?? () from /lib64/libcuda.so.1
#9  0x00007feb8716c1b7 in ?? () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/external/el8_amd64_gcc12/lib/libcudart.so.12
#10 0x00007feb871a8490 in cudaEventQuery () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/external/el8_amd64_gcc12/lib/libcudart.so.12
#11 0x00007feb3ef54514 in cms::alpakatools::CachingAllocator<alpaka::DevUniformCudaHipRt<alpaka::ApiCudaRt>, alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >::tryReuseCachedBlock(cms::alpakatools::CachingAllocator<alpaka::DevUniformCudaHipRt<alpaka::ApiCudaRt>, alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >::BlockDescriptor&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginEventFilterEcalRawToDigiPluginsPortableCudaAsync.so
#12 0x00007feb3ef5778e in std::enable_if<((isQueue<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >)&&(is_unbounded_array_v<std::byte []>))&&(!(is_array_v<std::remove_extent<std::byte []>::type>)), cms::alpakatools::detail::buffer_type<alpaka::trait::DevType<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false>, void>::type, std::byte [], std::enable_if<isDevice<alpaka::trait::DevType<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false>, void>::type>, void>::type>::type>::type cms::alpakatools::make_device_buffer<std::byte [], alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> >(alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> const&, unsigned int) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginEventFilterEcalRawToDigiPluginsPortableCudaAsync.so
#13 0x00007feb3ef62ada in PortableDeviceCollection<EcalDigiSoALayout<128ul, false>, alpaka::DevUniformCudaHipRt<alpaka::ApiCudaRt>, void>::PortableDeviceCollection<alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false>, void>(int, alpaka::uniform_cuda_hip::detail::QueueUniformCudaHipRt<alpaka::ApiCudaRt, false> const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginEventFilterEcalRawToDigiPluginsPortableCudaAsync.so
#14 0x00007feb3ef67299 in alpaka_cuda_async::EcalRawToDigiPortable::produce(alpaka_cuda_async::device::Event&, alpaka_cuda_async::device::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginEventFilterEcalRawToDigiPluginsPortableCudaAsync.so
#15 0x00007feb3ef62923 in alpaka_cuda_async::stream::EDProducer<>::produce(edm::Event&, edm::EventSetup const&) [clone .lto_priv.0] () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginEventFilterEcalRawToDigiPluginsPortableCudaAsync.so
#16 0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#17 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#18 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#19 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#20 0x00007feba682692e in tbb::detail::d1::function_task<edm::WaitingTaskHolder::doneWaiting(std::__exception_ptr::exception_ptr)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#21 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea49949500, waiter=..., this=0x7feb9f6c4f00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#22 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4f00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#23 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#24 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#25 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#26 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#27 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#28 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 33 (Thread 0x7fea489fe700 (LWP 1453466) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feba489c72c in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_assign (this=0x7fea489f68a0, __str=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_2_0_pre2-el8_amd64_gcc12/build/CMSSW_13_2_0_pre2-build/BUILD/el8_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/gcc-12.3.1/obj/x86_64-redhat-linux-gnu/libstdc++-v3/include/bits/basic_string.h:1078
#5  0x00007feb273ce117 in l1t::GlobalBoard::runGTL(edm::Event const&, edm::EventSetup const&, TriggerMenu const*, bool, int, std::unique_ptr<GlobalObjectMapRecord, std::default_delete<GlobalObjectMapRecord> >&, unsigned int, int, int, int, int, int) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libL1TriggerL1TGlobal.so
#6  0x00007feb274d12a0 in L1TGlobalProducer::produce(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginL1TriggerL1TGlobalPlugins.so
#7  0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#8  0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#9  0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#10 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#12 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fe9fb938c00, waiter=..., this=0x7feb9f6c4f80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#13 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4f80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#14 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#15 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#16 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#17 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#18 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#19 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 32 (Thread 0x7fea495ff700 (LWP 1453465) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feaa3dc4f0d in VirtualJetProducer::output(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoJetsJetProducers_plugins.so
#5  0x00007feaa3dbc6e3 in VirtualJetProducer::produce(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoJetsJetProducers_plugins.so
#6  0x00007feaa3d75c2d in FastjetJetProducer::produce(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoJetsJetProducers_plugins.so
#7  0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#8  0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#9  0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#10 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#12 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea3afd6200, waiter=..., this=0x7feb9f6c4e80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#13 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4e80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#14 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#15 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#16 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8800) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#17 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8800) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#18 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#19 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 31 (Thread 0x7fea4a396700 (LWP 1453464) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feb52f981c6 in TIDRing::groupedCompatibleDetsV(TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&, std::vector<DetGroup, std::allocator<DetGroup> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerTkDetLayers.so
#5  0x00007feb52f98821 in TIDLayer::groupedCompatibleDetsV(TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&, std::vector<DetGroup, std::allocator<DetGroup> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerTkDetLayers.so
#6  0x00007feb909297ed in GeometricSearchDet::groupedCompatibleDets(TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsDetLayers.so
#7  0x00007feb501072e8 in LayerMeasurements::groupedMeasurements(DetLayer const&, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsMeasurementDet.so
#8  0x00007fea9e0534b3 in GroupedCkfTrajectoryBuilder::advanceOneLayer(TrajectorySeed const&, TempTrajectory&, TrajectoryFilter const*, Propagator const*, bool, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&) const [clone .constprop.0] () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#9  0x00007fea9e047a63 in GroupedCkfTrajectoryBuilder::groupedLimitedCandidates(TrajectorySeed const&, TempTrajectory const&, TrajectoryFilter const*, Propagator const*, bool, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#10 0x00007fea9e049db6 in GroupedCkfTrajectoryBuilder::buildTrajectories(TrajectorySeed const&, std::vector<Trajectory, std::allocator<Trajectory> >&, unsigned int&, TrajectoryFilter const*) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#11 0x00007fea9cfd1c56 in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&)::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#12 0x00007fea9cfd31df in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#13 0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#14 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#15 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#16 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#17 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#18 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea44da8100, waiter=..., this=0x7feb9f6c4e00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#19 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4e00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#20 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#21 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#22 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8700) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#23 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8700) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#24 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#25 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 30 (Thread 0x7fea4bb97700 (LWP 1453463) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feb50198988 in BaseTrackerRecHit::surface() const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerMeasurementDetPlugins.so
#5  0x00007feb5328cae0 in KFTrajectoryFitter::fitOne(TrajectorySeed const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, TrajectoryStateOnSurface const&, TrajectoryFitter::fitType) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsTrackFitters.so
#6  0x00007fea9cfe1c7c in TransientInitialStateEstimator::innerState(Trajectory const&, bool) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#7  0x00007fea9cfd4276 in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#8  0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#9  0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#10 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#12 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#13 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea40390400, waiter=..., this=0x7feb9f6c4d80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#14 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4d80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#15 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#16 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#17 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#18 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#19 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#20 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 29 (Thread 0x7fea4ad97700 (LWP 1453462) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feba68b66e2 in edm::Path::runNextWorkerAsync(unsigned int, edm::EventTransitionInfo const&, edm::ServiceToken const&, edm::StreamID const&, edm::StreamContext const*, tbb::detail::d1::task_group&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#5  0x00007feba68b6c6c in edm::Path::workerFinished(std::__exception_ptr::exception_ptr const*, unsigned int, edm::EventTransitionInfo const&, edm::ServiceToken const&, edm::StreamID const&, edm::StreamContext const*, tbb::detail::d1::task_group&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#6  0x00007feba68b6d32 in edm::FunctorWaitingTask<edm::Path::runNextWorkerAsync(unsigned int, edm::EventTransitionInfo const&, edm::ServiceToken const&, edm::StreamID const&, edm::StreamContext const*, tbb::detail::d1::task_group&)::{lambda(std::__exception_ptr::exception_ptr const*)#1}>::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#7  0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#8  0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fe9fba2f900, waiter=..., this=0x7feb9f6c4d00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#9  tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4d00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#10 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#11 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#12 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8600) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#13 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8600) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#14 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#15 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 28 (Thread 0x7fea4c798700 (LWP 1453461) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feba444db38 in __ieee754_log_fma () from /lib64/libm.so.6
#5  0x00007feab171b182 in void brokenline::prepareBrokenLineData<Eigen::Map<Eigen::Matrix<double, 3, 3, 0, 3, 3>, 0, Eigen::Stride<98304, 32768> >, Eigen::Map<Eigen::Matrix<double, 4, 1, 0, 4, 1>, 0, Eigen::InnerStride<32768> >, 3>(Eigen::Map<Eigen::Matrix<double, 3, 3, 0, 3, 3>, 0, Eigen::Stride<98304, 32768> > const&, Eigen::Map<Eigen::Matrix<double, 4, 1, 0, 4, 1>, 0, Eigen::InnerStride<32768> > const&, double, brokenline::PreparedBrokenLineData<3>&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginRecoTrackerPixelSeedingPlugins.so
#6  0x00007feab171dc53 in HelixFitOnGPU<pixelTopology::Phase1>::launchBrokenLineKernelsOnCPU(TrackingRecHitSoA<pixelTopology::Phase1>::TrackingRecHitSoALayout<128ul, false>::ConstViewTemplateFreeParams<128ul, false, true, false> const&, unsigned int, unsigned int) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginRecoTrackerPixelSeedingPlugins.so
#7  0x00007feab175d5f5 in CAHitNtupletGeneratorOnGPU<pixelTopology::Phase1>::makeTuples(TrackingRecHitSoAHost<pixelTopology::Phase1> const&, float) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginRecoTrackerPixelSeedingPlugins.so
#8  0x00007feab1740061 in CAHitNtupletCUDAT<pixelTopology::Phase1>::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginRecoTrackerPixelSeedingPlugins.so
#9  0x00007feba6931406 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#10 0x00007feba692208e in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#12 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#13 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#14 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea3af6ef00, waiter=..., this=0x7feb9f6c4c80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#15 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4c80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#16 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#17 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#18 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8780) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#19 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8780) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#20 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#21 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 27 (Thread 0x7fea4db9a700 (LWP 1453460) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feba68314c1 in edm::ServiceRegistry::Operate::~Operate() [clone .lto_priv.0] () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#5  0x00007feba68b4b54 in edm::Worker::AcquireTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>, void>::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#6  0x00007feba682692e in tbb::detail::d1::function_task<edm::WaitingTaskHolder::doneWaiting(std::__exception_ptr::exception_ptr)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#7  0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea421bc100, waiter=..., this=0x7feb9f6c4c00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#8  tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4c00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#9  tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#10 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#11 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8680) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#12 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8680) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#13 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#14 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 26 (Thread 0x7fea4d199700 (LWP 1453459) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feba6b6de0c in edm::DataFrameContainer::sort() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libDataFormatsCommon.so
#5  0x00007feb2064116b in HcalRawToDigi::produce(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginEventFilterHcalRawToDigiPlugins.so
#6  0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#7  0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#8  0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#9  0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#10 0x00007feba682692e in tbb::detail::d1::function_task<edm::WaitingTaskHolder::doneWaiting(std::__exception_ptr::exception_ptr)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fe9f1803400, waiter=..., this=0x7feb9f6c5280) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#12 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c5280) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#13 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#14 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#15 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8580) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#16 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8580) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#17 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#18 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 25 (Thread 0x7fea807e2700 (LWP 1453432) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9d778ec7 in evf::FastMonitoringService::snapshotRunner() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libEventFilterUtilities.so
#3  0x00007feba4830a73 in std::execute_native_thread_routine (__p=0x7feb971e54a0) at ../../../../../libstdc++-v3/src/c++11/thread.cc:82
#4  0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#5  0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 24 (Thread 0x7fea9d7ff700 (LWP 1452643) "cmsRun"):
#0  0x00007feba41a445c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007feb3748084e in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#2  0x00007feb37480e03 in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#3  0x00007feb3747e578 in std::_Function_handler<void (), tsl::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#4  0x00007feb2a1f0a02 in tsl::(anonymous namespace)::PThread::ThreadFn(void*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#5  0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#6  0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 23 (Thread 0x7fea9e860700 (LWP 1452640) "cmsRun"):
#0  0x00007feba41a445c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007feb3748084e in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#2  0x00007feb37480e03 in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#3  0x00007feb3747e578 in std::_Function_handler<void (), tsl::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#4  0x00007feb2a1f0a02 in tsl::(anonymous namespace)::PThread::ThreadFn(void*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#5  0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#6  0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 22 (Thread 0x7fea9f061700 (LWP 1452639) "cmsRun"):
#0  0x00007feba41a445c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007feb3748084e in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#2  0x00007feb37480e03 in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#3  0x00007feb3747e578 in std::_Function_handler<void (), tsl::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#4  0x00007feb2a1f0a02 in tsl::(anonymous namespace)::PThread::ThreadFn(void*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#5  0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#6  0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 21 (Thread 0x7feac13fa700 (LWP 1452586) "cmsRun"):
#0  0x00007feba41a445c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007feb9d77ebf6 in FedRawDataInputSource::readWorker(unsigned int) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libEventFilterUtilities.so
#2  0x00007feba4830a73 in std::execute_native_thread_routine (__p=0x7feb947dca80) at ../../../../../libstdc++-v3/src/c++11/thread.cc:82
#3  0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#4  0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 20 (Thread 0x7feac1dfb700 (LWP 1452585) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feac46dc7c2 in std::__cxx11::list<int, std::allocator<int> >::unique() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginHLTriggerHLTfiltersAuto.so
#5  0x00007feac46df08d in HLTL1TSeed::seedsL1TriggerObjectMaps(edm::Event&, trigger::TriggerFilterObjectWithRefs&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginHLTriggerHLTfiltersAuto.so
#6  0x00007feac46d81a8 in HLTL1TSeed::hltFilter(edm::Event&, edm::EventSetup const&, trigger::TriggerFilterObjectWithRefs&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginHLTriggerHLTfiltersAuto.so
#7  0x00007feb1f57097e in HLTStreamFilter::filter(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libHLTriggerHLTcore.so
#8  0x00007feba6943aa1 in edm::stream::EDFilterAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#9  0x00007feba69289de in edm::WorkerT<edm::stream::EDFilterAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#10 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#12 0x00007feba682692e in tbb::detail::d1::function_task<edm::WaitingTaskHolder::doneWaiting(std::__exception_ptr::exception_ptr)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#13 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea3af56800, waiter=..., this=0x7feb9f6c4b80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#14 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4b80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#15 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#16 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#17 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8c00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#18 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8c00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#19 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#20 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 19 (Thread 0x7feac27fc700 (LWP 1452584) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feba682328d in std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#5  0x00007feba68b6d42 in edm::FunctorWaitingTask<edm::Path::runNextWorkerAsync(unsigned int, edm::EventTransitionInfo const&, edm::ServiceToken const&, edm::StreamID const&, edm::StreamContext const*, tbb::detail::d1::task_group&)::{lambda(std::__exception_ptr::exception_ptr const*)#1}>::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#6  0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#7  0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea403f7600, waiter=..., this=0x7feb9f6c4a80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#8  tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4a80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#9  tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#10 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#11 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8d00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#12 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8d00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#13 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#14 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 18 (Thread 0x7feac2ffd700 (LWP 1452583) "cmsRun"):
#0  0x00007feba41a7ab4 in read () from /lib64/libpthread.so.0
#1  0x00007feb9d77ed02 in FedRawDataInputSource::readWorker(unsigned int) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libEventFilterUtilities.so
#2  0x00007feba4830a73 in std::execute_native_thread_routine (__p=0x7feb947dcab0) at ../../../../../libstdc++-v3/src/c++11/thread.cc:82
#3  0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#4  0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 17 (Thread 0x7feac51fc700 (LWP 1452582) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feb8cfc5086 in JacobianCurvilinearToLocal::JacobianCurvilinearToLocal(Surface const&, LocalTrajectoryParameters const&, GlobalTrajectoryParameters const&, MagneticField const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsAnalyticalJacobians.so
#5  0x00007feb8fa74ece in BasicTrajectoryState::createLocalErrorFromCurvilinearError() const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsTrajectoryState.so
#6  0x00007feb50119ef5 in Chi2MeasurementEstimator::estimate(TrajectoryStateOnSurface const&, TrackingRecHit const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsKalmanUpdators.so
#7  0x00007feb5019c600 in bool TkStripMeasurementDet::filteredRecHits<edm::Ref<edmNew::DetSetVector<SiStripCluster>, SiStripCluster, edmNew::DetSetVector<SiStripCluster>::FindForDetSetVector> >(edm::Ref<edmNew::DetSetVector<SiStripCluster>, SiStripCluster, edmNew::DetSetVector<SiStripCluster>::FindForDetSetVector> const&, StripCPE::AlgoParam const&, TrajectoryStateOnSurface const&, MeasurementEstimator const&, std::vector<bool, std::allocator<bool> > const&, std::vector<SiStripRecHit2D, std::allocator<SiStripRecHit2D> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerMeasurementDetPlugins.so
#8  0x00007feb501a9ec3 in TkStripMeasurementDet::simpleRecHits(TrajectoryStateOnSurface const&, MeasurementEstimator const&, MeasurementTrackerEvent const&, std::vector<SiStripRecHit2D, std::allocator<SiStripRecHit2D> >&) const [clone .isra.0] () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerMeasurementDetPlugins.so
#9  0x00007feb50196b94 in TkGluedMeasurementDet::measurements(TrajectoryStateOnSurface const&, MeasurementEstimator const&, MeasurementTrackerEvent const&, tracking::TempMeasurements&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerMeasurementDetPlugins.so
#10 0x00007feb50107472 in LayerMeasurements::groupedMeasurements(DetLayer const&, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsMeasurementDet.so
#11 0x00007fea9e0534b3 in GroupedCkfTrajectoryBuilder::advanceOneLayer(TrajectorySeed const&, TempTrajectory&, TrajectoryFilter const*, Propagator const*, bool, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&) const [clone .constprop.0] () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#12 0x00007fea9e047a63 in GroupedCkfTrajectoryBuilder::groupedLimitedCandidates(TrajectorySeed const&, TempTrajectory const&, TrajectoryFilter const*, Propagator const*, bool, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#13 0x00007fea9e049db6 in GroupedCkfTrajectoryBuilder::buildTrajectories(TrajectorySeed const&, std::vector<Trajectory, std::allocator<Trajectory> >&, unsigned int&, TrajectoryFilter const*) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#14 0x00007fea9cfd1c56 in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&)::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#15 0x00007fea9cfd31df in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#16 0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#17 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#18 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#19 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#20 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#21 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fe9f17e4400, waiter=..., this=0x7feb9f6c4a00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#22 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4a00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#23 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#24 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#25 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8b80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#26 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8b80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#27 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#28 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 16 (Thread 0x7feac69fe700 (LWP 1452581) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feb8fa737bf in BasicTrajectoryState::BasicTrajectoryState(LocalTrajectoryParameters const&, LocalTrajectoryError const&, Surface const&, MagneticField const*, SurfaceSideDefinition::SurfaceSide) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsTrajectoryState.so
#5  0x00007feb5011d88b in TrajectoryStateOnSurface (anonymous namespace)::lupdate<2u>(TrajectoryStateOnSurface const&, TrackingRecHit const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsKalmanUpdators.so
#6  0x00007feb50126306 in KFUpdator::update(TrajectoryStateOnSurface const&, TrackingRecHit const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsKalmanUpdators.so
#7  0x00007fea978b783e in SeedFromConsecutiveHitsCreator::makeSeed(std::vector<TrajectorySeed, std::allocator<TrajectorySeed> >&, SeedingHitSet const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerTkSeedGeneratorPlugins.so
#8  0x00007fea978b8a68 in SeedCreatorFromRegionHitsEDProducerT<SeedFromConsecutiveHitsCreator>::produce(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerTkSeedGeneratorPlugins.so
#9  0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#10 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#12 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#13 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#14 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea411c8e00, waiter=..., this=0x7feb9f6c4900) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#15 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4900) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#16 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#17 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#18 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8c80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#19 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8c80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#20 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#21 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 15 (Thread 0x7feac39fe700 (LWP 1452580) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feb5011c74a in TrajectoryStateOnSurface (anonymous namespace)::lupdate<2u>(TrajectoryStateOnSurface const&, TrackingRecHit const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsKalmanUpdators.so
#5  0x00007feb50126306 in KFUpdator::update(TrajectoryStateOnSurface const&, TrackingRecHit const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsKalmanUpdators.so
#6  0x00007fea9cfdd85f in CkfTrajectoryBuilder::updateTrajectory(TempTrajectory&, TrajectoryMeasurement&&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#7  0x00007fea9cfe304e in CkfTrajectoryBuilder::limitedCandidates(std::shared_ptr<TrajectorySeed const> const&, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&, std::vector<Trajectory, std::allocator<Trajectory> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#8  0x00007fea9cfdc71d in CkfTrajectoryBuilder::limitedCandidates(TrajectorySeed const&, TempTrajectory&, std::vector<Trajectory, std::allocator<Trajectory> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#9  0x00007fea9cfdc8ff in CkfTrajectoryBuilder::buildTrajectories(TrajectorySeed const&, std::vector<Trajectory, std::allocator<Trajectory> >&, unsigned int&, TrajectoryFilter const*) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#10 0x00007fea9cfd1c56 in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&)::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#11 0x00007fea9cfd31df in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#12 0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#13 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#14 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#15 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#16 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#17 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea44d89600, waiter=..., this=0x7feb9f6c4880) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#18 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4880) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#19 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#20 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#21 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8b00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#22 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8b00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#23 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#24 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 14 (Thread 0x7feac43ff700 (LWP 1452579) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feba68b3823 in std::__exception_ptr::exception_ptr edm::Worker::runModuleDirectly<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#5  0x00007feba68b3987 in edm::Path::finished(std::__exception_ptr::exception_ptr, edm::StreamContext const*, edm::EventTransitionInfo const&, edm::StreamID const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#6  0x00007feba68b6ae0 in edm::Path::workerFinished(std::__exception_ptr::exception_ptr const*, unsigned int, edm::EventTransitionInfo const&, edm::ServiceToken const&, edm::StreamID const&, edm::StreamContext const*, tbb::detail::d1::task_group&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#7  0x00007feba68b6d32 in edm::FunctorWaitingTask<edm::Path::runNextWorkerAsync(unsigned int, edm::EventTransitionInfo const&, edm::ServiceToken const&, edm::StreamID const&, edm::StreamContext const*, tbb::detail::d1::task_group&)::{lambda(std::__exception_ptr::exception_ptr const*)#1}>::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#8  0x00007feba682692e in tbb::detail::d1::function_task<edm::WaitingTaskHolder::doneWaiting(std::__exception_ptr::exception_ptr)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#9  0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea42f2f200, waiter=..., this=0x7feb9f6c4780) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#10 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4780) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#11 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#12 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#13 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8a80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#14 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8a80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#15 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#16 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 13 (Thread 0x7feac5ffd700 (LWP 1452576) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feba3e9e046 in __memmove_avx_unaligned_erms () from /lib64/libc.so.6
#5  0x00007feba69513c7 in void std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct<char const*>(char const*, char const*, std::forward_iterator_tag) [clone .constprop.0] () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#6  0x00007feba68330df in edm::Event::commit_aux(std::vector<edm::propagate_const<std::unique_ptr<edm::WrapperBase, std::default_delete<edm::WrapperBase> > >, std::allocator<edm::propagate_const<std::unique_ptr<edm::WrapperBase, std::default_delete<edm::WrapperBase> > > > >&, edm::Hash<5>*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#7  0x00007feba68333fa in edm::Event::commit_(std::vector<unsigned int, std::allocator<unsigned int> > const&, edm::Hash<5>*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#8  0x00007feba6943abe in edm::stream::EDFilterAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#9  0x00007feba69289de in edm::WorkerT<edm::stream::EDFilterAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#10 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#12 0x00007feba682692e in tbb::detail::d1::function_task<edm::WaitingTaskHolder::doneWaiting(std::__exception_ptr::exception_ptr)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#13 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea42fa8800, waiter=..., this=0x7feb9f6c4700) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#14 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4700) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#15 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#16 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#17 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8e00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#18 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8e00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#19 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#20 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 12 (Thread 0x7feac71ff700 (LWP 1452574) "cmsRun"):
#0  0x00007feba41a445c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007feb9d77ebf6 in FedRawDataInputSource::readWorker(unsigned int) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libEventFilterUtilities.so
#2  0x00007feba4830a73 in std::execute_native_thread_routine (__p=0x7feb947dcae0) at ../../../../../libstdc++-v3/src/c++11/thread.cc:82
#3  0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#4  0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 11 (Thread 0x7feb201ff700 (LWP 1452572) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feaaa9ad4d8 in edm::OrphanHandle<edm::SortedCollection<EcalTriggerPrimitiveDigi, edm::StrictWeakOrdering<EcalTriggerPrimitiveDigi> > > edm::Event::putImpl<edm::SortedCollection<EcalTriggerPrimitiveDigi, edm::StrictWeakOrdering<EcalTriggerPrimitiveDigi> > >(unsigned int, std::unique_ptr<edm::SortedCollection<EcalTriggerPrimitiveDigi, edm::StrictWeakOrdering<EcalTriggerPrimitiveDigi> >, std::default_delete<edm::SortedCollection<EcalTriggerPrimitiveDigi, edm::StrictWeakOrdering<EcalTriggerPrimitiveDigi> > > >) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginEventFilterEcalRawToDigiPlugins.so
#5  0x00007feaaa9b0c31 in EcalRawToDigi::produce(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_0_5_patch1/lib/el8_amd64_gcc12/pluginEventFilterEcalRawToDigiPlugins.so
#6  0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#7  0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#8  0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#9  0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#10 0x00007feba682692e in tbb::detail::d1::function_task<edm::WaitingTaskHolder::doneWaiting(std::__exception_ptr::exception_ptr)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea421dfe00, waiter=..., this=0x7feb9f6c4600) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#12 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4600) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#13 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#14 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#15 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8d80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#16 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8d80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#17 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#18 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x7feb211fd700 (LWP 1452568) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feb50196fbb in TkGluedMeasurementDet::measurements(TrajectoryStateOnSurface const&, MeasurementEstimator const&, MeasurementTrackerEvent const&, tracking::TempMeasurements&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerMeasurementDetPlugins.so
#5  0x00007feb50107472 in LayerMeasurements::groupedMeasurements(DetLayer const&, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsMeasurementDet.so
#6  0x00007fea9e0534b3 in GroupedCkfTrajectoryBuilder::advanceOneLayer(TrajectorySeed const&, TempTrajectory&, TrajectoryFilter const*, Propagator const*, bool, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&) const [clone .constprop.0] () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#7  0x00007fea9e047a63 in GroupedCkfTrajectoryBuilder::groupedLimitedCandidates(TrajectorySeed const&, TempTrajectory const&, TrajectoryFilter const*, Propagator const*, bool, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#8  0x00007fea9e049db6 in GroupedCkfTrajectoryBuilder::buildTrajectories(TrajectorySeed const&, std::vector<Trajectory, std::allocator<Trajectory> >&, unsigned int&, TrajectoryFilter const*) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#9  0x00007fea9cfd1c56 in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&)::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#10 0x00007fea9cfd31df in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#11 0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#12 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#13 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#14 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#15 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#16 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7feb540c7d00, waiter=..., this=0x7feb9f6c4800) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#17 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4800) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#18 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#19 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#20 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8f00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#21 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8f00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#22 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#23 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x7feb21dfe700 (LWP 1452565) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feba3e9e4f9 in __memset_avx2_unaligned_erms () from /lib64/libc.so.6
#5  0x00007feb9b5d0aba in FEDRawData::resize(unsigned long, unsigned long) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libDataFormatsFEDRawData.so
#6  0x00007feb9d780c00 in FedRawDataInputSource::fillFEDRawDataCollection(FEDRawDataCollection&, bool&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libEventFilterUtilities.so
#7  0x00007feb9d785cf1 in FedRawDataInputSource::read(edm::EventPrincipal&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libEventFilterUtilities.so
#8  0x00007feba689d4c0 in void edm::callWithTryCatchAndPrint<void>(std::function<void ()>, char const*, bool) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#9  0x00007feba689d897 in edm::InputSource::readEvent(edm::EventPrincipal&, edm::StreamContext&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#10 0x00007feba684ab06 in edm::EventProcessor::readEvent(unsigned int) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00007feba6844c43 in edm::EventProcessor::readNextEventForStream(edm::WaitingTaskHolder const&, unsigned int, edm::LuminosityBlockProcessingStatus&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#12 0x00007feba684c5e3 in edm::EventProcessor::handleNextEventForStreamAsync(edm::WaitingTaskHolder, unsigned int)::{lambda()#1}::operator()() [clone .lto_priv.0] () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#13 0x00007feba6868258 in edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::EventProcessor::handleNextEventForStreamAsync(edm::WaitingTaskHolder, unsigned int)::{lambda()#1}>(tbb::detail::d1::task_group&, edm::EventProcessor::handleNextEventForStreamAsync(edm::WaitingTaskHolder, unsigned int)::{lambda()#1}&&)::{lambda()#1}>::execute() [clone .lto_priv.0] () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#14 0x00007feba6a6a1d0 in tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#15 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea403b0800, waiter=..., this=0x7feb9f6c4b00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#16 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4b00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#17 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#18 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#19 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b9000) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#20 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b9000) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#21 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#22 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x7feb227ff700 (LWP 1452564) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feba6b721e7 in edm::reftobase::RefHolderBase::RefHolderBase() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libDataFormatsCommon.so
#5  0x00007fea9cfd1ed8 in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&)::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#6  0x00007fea9cfd31df in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#7  0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#8  0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#9  0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#10 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#12 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fe9fb94db00, waiter=..., this=0x7feb9f6c4980) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#13 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4980) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#14 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#15 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#16 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8e80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#17 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8e80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#18 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#19 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x7feb237ff700 (LWP 1452563) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feb9090f482 in HelixArbitraryPlaneCrossing2Order::HelixArbitraryPlaneCrossing2Order(Basic3DVector<float> const&, Basic3DVector<float> const&, float, PropagationDirection) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsGeomPropagators.so
#5  0x00007feb9090f4f4 in HelixArbitraryPlaneCrossing::HelixArbitraryPlaneCrossing(Basic3DVector<float> const&, Basic3DVector<float> const&, float, PropagationDirection) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsGeomPropagators.so
#6  0x00007feb90917198 in AnalyticalPropagator::propagateWithPath(FreeTrajectoryState const&, Plane const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsGeomPropagators.so
#7  0x00007feb9090c255 in Propagator::propagateWithPath(TrajectoryStateOnSurface const&, Plane const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsGeomPropagators.so
#8  0x00007feb3f5b9d1e in PropagatorWithMaterial::propagateWithPath(TrajectoryStateOnSurface const&, Plane const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsMaterialEffects.so
#9  0x00007fea978b79f7 in SeedFromConsecutiveHitsCreator::makeSeed(std::vector<TrajectorySeed, std::allocator<TrajectorySeed> >&, SeedingHitSet const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerTkSeedGeneratorPlugins.so
#10 0x00007fea978b8a68 in SeedCreatorFromRegionHitsEDProducerT<SeedFromConsecutiveHitsCreator>::produce(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerTkSeedGeneratorPlugins.so
#11 0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#12 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#13 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#14 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#15 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#16 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fe80c1b1300, waiter=..., this=0x7feb9f6c4680) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#17 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4680) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#18 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#19 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#20 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b8f80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#21 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b8f80) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#22 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#23 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7feb24e83700 (LWP 1452561) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feb3f48d04f in SteppingHelixPropagator::getNextState(SteppingHelixStateInfo const&, SteppingHelixStateInfo&, double, CLHEP::Hep3Vector const&, CLHEP::Hep3Vector const&, double, double, ROOT::Math::SMatrix<double, 5u, 5u, ROOT::Math::MatRepStd<double, 5u, 5u> > const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackPropagationSteppingHelixPropagator.so
#5  0x00007feb3f48d329 in SteppingHelixPropagator::makeAtomStep(SteppingHelixStateInfo&, SteppingHelixStateInfo&, double, PropagationDirection, SteppingHelixPropagator::Fancy) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackPropagationSteppingHelixPropagator.so
#6  0x00007feb3f48f85f in SteppingHelixPropagator::propagate(SteppingHelixStateInfo (&) [8], int&, SteppingHelixPropagator::DestType, double const*, double) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackPropagationSteppingHelixPropagator.so
#7  0x00007feb3f49122f in SteppingHelixPropagator::propagate(SteppingHelixStateInfo const&, Plane const&, SteppingHelixStateInfo&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackPropagationSteppingHelixPropagator.so
#8  0x00007feb3f493e7f in SteppingHelixPropagator::propagateWithPath(FreeTrajectoryState const&, Plane const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackPropagationSteppingHelixPropagator.so
#9  0x00007feb3f486cf5 in Propagator::propagateWithPath(TrajectoryStateOnSurface const&, Plane const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackPropagationSteppingHelixPropagator.so
#10 0x00007feb9092c90c in GeomDetCompatibilityChecker::isCompatible(GeomDet const*, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsDetLayers.so
#11 0x00007feb9092cb9c in ForwardDetRingOneZ::add(int, std::vector<std::pair<GeomDet const*, TrajectoryStateOnSurface>, std::allocator<std::pair<GeomDet const*, TrajectoryStateOnSurface> > >&, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsDetLayers.so
#12 0x00007feb52e7dc7a in MuDetRing::compatibleDets(TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoMuonDetLayers.so
#13 0x00007feb52e7f3e2 in MuRingForwardLayer::compatibleDets(TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoMuonDetLayers.so
#14 0x00007feb52e7fcc3 in MuRingForwardDoubleLayer::groupedCompatibleDets(TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoMuonDetLayers.so
#15 0x00007feaa3347c78 in MuonDetLayerMeasurements::groupedMeasurements(DetLayer const*, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&, edm::Event const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoMuonMeasurementDet.so
#16 0x00007feaa3348989 in MuonDetLayerMeasurements::groupedMeasurements(DetLayer const*, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoMuonMeasurementDet.so
#17 0x00007feaa32d5d74 in StandAloneMuonFilter::findBestMeasurements(DetLayer const*, TrajectoryStateOnSurface const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoMuonStandAloneTrackFinder.so
#18 0x00007feaa32d9106 in StandAloneMuonFilter::refit(TrajectoryStateOnSurface const&, DetLayer const*, Trajectory&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoMuonStandAloneTrackFinder.so
#19 0x00007feaa32db6a4 in StandAloneMuonTrajectoryBuilder::trajectories(TrajectorySeed const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoMuonStandAloneTrackFinder.so
#20 0x00007feaa32d45c2 in ExhaustiveMuonTrajectoryBuilder::trajectories(TrajectorySeed const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoMuonStandAloneTrackFinder.so
#21 0x00007feb22cfe029 in MuonTrackFinder::reconstruct(edm::Handle<edm::View<TrajectorySeed> > const&, edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoMuonTrackingTools.so
#22 0x00007feaa32fda16 in L2MuonProducer::produce(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoMuonL2MuonProducer.so
#23 0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#24 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#25 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#26 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#27 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#28 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fea421dbb00, waiter=..., this=0x7feb9f6c4580) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#29 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4580) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#30 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#31 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#32 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b9100) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#33 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b9100) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#34 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#35 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7feb25884700 (LWP 1452559) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feb52fe50cd in StripCPEfromTrackAngle::localParameters(SiStripCluster const&, GeomDet const&, LocalTrajectoryParameters const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoLocalTrackerSiStripRecHitConverter.so
#5  0x00007feb52fe2464 in StripClusterParameterEstimator::localParameters(SiStripCluster const&, GeomDet const&, TrajectoryStateOnSurface const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoLocalTrackerSiStripRecHitConverter.so
#6  0x00007feb530a1d85 in TkClonerImpl::operator()(SiStripMatchedRecHit2D const&, TrajectoryStateOnSurface const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerTransientTrackingRecHit.so
#7  0x00007feb530a2117 in TkClonerImpl::makeShared(SiStripMatchedRecHit2D const&, TrajectoryStateOnSurface const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerTransientTrackingRecHit.so
#8  0x00007feb5e0ea4d4 in SiStripMatchedRecHit2D::cloneSH_(TkCloner const&, TrajectoryStateOnSurface const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libDataFormatsTrackerRecHit2D.so
#9  0x00007feb5328cb7e in KFTrajectoryFitter::fitOne(TrajectorySeed const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, TrajectoryStateOnSurface const&, TrajectoryFitter::fitType) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsTrackFitters.so
#10 0x00007feb3e6014e4 in (anonymous namespace)::KFFittingSmoother::fitOne(TrajectorySeed const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, TrajectoryStateOnSurface const&, TrajectoryFitter::fitType) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginTrackingToolsTrackFittersPlugins.so
#11 0x00007fea9ce5acf6 in TrackProducerAlgorithm<reco::Track>::buildTrack(TrajectoryFitter const*, Propagator const*, std::vector<AlgoProductTraits<reco::Track>::AlgoProduct, std::allocator<AlgoProductTraits<reco::Track>::AlgoProduct> >&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >&, TrajectoryStateOnSurface&, TrajectorySeed const&, float, reco::BeamSpot const&, edm::RefToBase<TrajectorySeed>, int, signed char) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerTrackProducer.so
#12 0x00007fea9cf3a67c in TrackProducerAlgorithm<reco::Track>::runWithCandidate(TrackingGeometry const*, MagneticField const*, std::vector<TrackCandidate, std::allocator<TrackCandidate> > const&, TrajectoryFitter const*, Propagator const*, TransientTrackingRecHitBuilder const*, reco::BeamSpot const&, std::vector<AlgoProductTraits<reco::Track>::AlgoProduct, std::allocator<AlgoProductTraits<reco::Track>::AlgoProduct> >&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerTrackProducerPlugins.so
#13 0x00007fea9cf391a4 in TrackProducer::produce(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginRecoTrackerTrackProducerPlugins.so
#14 0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#15 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#16 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#17 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#18 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#19 0x00007feba506795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7fe9f17d8f00, waiter=..., this=0x7feb9f6c4500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#20 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7feb9f6c4500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#21 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:137
#22 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/market.cpp:599
#23 0x00007feba5069b0e in tbb::detail::r1::rml::private_worker::run (this=0x7feb9f6b9080) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#24 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7feb9f6b9080) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#25 0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#26 0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7feb5e9ff700 (LWP 1452465) "cuda-EvtHandlr"):
#0  0x00007feba3ef30e1 in poll () from /lib64/libc.so.6
#1  0x00007feb855c638f in ?? () from /lib64/libcuda.so.1
#2  0x00007feb8568903f in ?? () from /lib64/libcuda.so.1
#3  0x00007feb855bf47f in ?? () from /lib64/libcuda.so.1
#4  0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#5  0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7feb5f663700 (LWP 1452462) "cuda-EvtHandlr"):
#0  0x00007feba3ef30e1 in poll () from /lib64/libc.so.6
#1  0x00007feb855c638f in ?? () from /lib64/libcuda.so.1
#2  0x00007feb8568903f in ?? () from /lib64/libcuda.so.1
#3  0x00007feb855bf47f in ?? () from /lib64/libcuda.so.1
#4  0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#5  0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7feb661aa700 (LWP 1452439) "cmsRun"):
#0  0x00007feba41a8672 in waitpid () from /lib64/libpthread.so.0
#1  0x00007feb9b64ee17 in edm::service::cmssw_stacktrace_fork() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2  0x00007feb9b651cea in edm::service::InitRootHandlers::stacktraceHelperThread() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  0x00007feba4830a73 in std::execute_native_thread_routine (__p=0x7feb9d291610) at ../../../../../libstdc++-v3/src/c++11/thread.cc:82
#4  0x00007feba419e1ca in start_thread () from /lib64/libpthread.so.0
#5  0x00007feba3e07e73 in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7feba3329640 (LWP 1451372) "cmsRun"):
#0  0x00007feba3ec89b8 in nanosleep () from /lib64/libc.so.6
#1  0x00007feba3ec88be in sleep () from /lib64/libc.so.6
#2  0x00007feb9b64ecc0 in sig_pause_for_stacktrace () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00007feb50112230 in __tls_get_addr@plt () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsKalmanUpdators.so
#5  0x00007feb5011d9dd in TrajectoryStateOnSurface (anonymous namespace)::lupdate<2u>(TrajectoryStateOnSurface const&, TrackingRecHit const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsKalmanUpdators.so
#6  0x00007feb50126306 in KFUpdator::update(TrajectoryStateOnSurface const&, TrackingRecHit const&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libTrackingToolsKalmanUpdators.so
#7  0x00007fea9cfdd85f in CkfTrajectoryBuilder::updateTrajectory(TempTrajectory&, TrajectoryMeasurement&&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#8  0x00007fea9cfe304e in CkfTrajectoryBuilder::limitedCandidates(std::shared_ptr<TrajectorySeed const> const&, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&, std::vector<Trajectory, std::allocator<Trajectory> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#9  0x00007fea9cfdc71d in CkfTrajectoryBuilder::limitedCandidates(TrajectorySeed const&, TempTrajectory&, std::vector<Trajectory, std::allocator<Trajectory> >&) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#10 0x00007fea9cfdc8ff in CkfTrajectoryBuilder::buildTrajectories(TrajectorySeed const&, std::vector<Trajectory, std::allocator<Trajectory> >&, unsigned int&, TrajectoryFilter const*) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#11 0x00007fea9cfd1c56 in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&)::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#12 0x00007fea9cfd31df in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libRecoTrackerCkfPattern.so
#13 0x00007feba6943e91 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#14 0x00007feba69287ae in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#15 0x00007feba68b3669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#16 0x00007feba68b3bd4 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#17 0x00007feba6a68f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#18 0x00007feba5070281 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x7feb9f6c4480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#19 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x7feb9f6c4480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#20 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#21 0x00007feba6836ceb in edm::FinalWaitingTask::wait() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#22 0x00007feba684065a in edm::EventProcessor::processRuns() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#23 0x00007feba6840bb1 in edm::EventProcessor::runToCompletion() () from /opt/offline/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_5/lib/el8_amd64_gcc12/libFWCoreFramework.so
#24 0x00000000004074ef in tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const ()
#25 0x00007feba505c9ad in tbb::detail::r1::task_arena_impl::execute (ta=..., d=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre1-el8_amd64_gcc12/build/CMSSW_14_1_0_pre1-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-c3903c50b52342174dbd3a52854a6e6d/tbb-v2021.9.0/src/tbb/arena.cpp:688
#26 0x0000000000408ed2 in main::{lambda()#1}::operator()() const ()
#27 0x000000000040517c in main ()
Current Modules:
Module: HLTMuonL1TFilter:hltL1fL1sCDCL1Filtered0 (crashed)
Module: TrackProducer:hltIter0PFlowCtfWithMaterialTracks
Module: none
Module: CAHitNtupletCUDAPhase1:hltPixelTracksCPUOnly
Module: CkfTrackCandidateMaker:hltIter0PFlowCkfTrackCandidates
Module: none
Module: L1TGlobalProducer:hltGtStage2ObjectMap
Module: CkfTrackCandidateMaker:hltDoubletRecoveryPFlowCkfTrackCandidates
Module: CkfTrackCandidateMaker:hltDoubletRecoveryPFlowCkfTrackCandidates
Module: SeedCreatorFromRegionConsecutiveHitsEDProducer:hltElePixelSeedsDoubletsUnseeded
Module: CkfTrackCandidateMaker:hltIterL3OITrackCandidatesNoVtx
Module: none
Module: HcalRawToDigi:hltHcalDigis
Module: CkfTrackCandidateMaker:hltIter0PFlowCkfTrackCandidates
Module: HLTL1TSeed:hltL1DoubleJet8535MassMin600Mu3
Module: L2MuonProducer:hltL2Muons
Module: HLTL1TSeed:hltL1sL1ZeroBiasFirstBunchAfterTrain
Module: none
Module: CkfTrackCandidateMaker:hltIter0PFlowCkfTrackCandidates
Module: EcalRawToDigiPortable@alpaka:hltEcalDigisPortableSoA
Module: EcalRawToDigi:hltEcalDigisLegacy
Module: EcalUncalibRecHitProducerPortable@alpaka:hltEcalUncalibRecHitPortableSoA
Module: CkfTrackCandidateMaker:hltIter0IterL3FromL1MuonCkfTrackCandidates
Module: none
Module: FastjetJetProducer:hltAK4PFJets
Module: CkfTrackCandidateMaker:hltMuCkfTrackCandidates
Module: EcalUncalibRecHitProducerPortable@alpaka:hltEcalUncalibRecHitPortableSoA
Module: EcalRawToDigiPortable@alpaka:hltEcalDigisPortableSoA
Module: SeedCreatorFromRegionConsecutiveHitsEDProducer:hltElePixelSeedsDoublets
Module: SeedCreatorFromRegionConsecutiveHitsEDProducer:hltElePixelSeedsDoubletsUnseeded
Module: CkfTrackCandidateMaker:hltIterL3OITrackCandidatesNoVtx
Module: CkfTrackCandidateMaker:hltIter0PFlowCkfTrackCandidates
A fatal system signal has occurred: segmentation violation