`line 447: cudaLaunchKernel returned status 98: invalid device function [acn01:103533:0:103533] Caught signal 11 (Segmentation fault: address not mapped to object at address 0x10) ==== backtrace (tid: 103533) ==== 0 0x0000000000039a70 __cudaPopCallConfiguration() ???:0 1 0x000000000003d1c4 __cudaPopCallConfiguration() ???:0 2 0x00000000000237bc __cudaPopCallConfiguration() ???:0 3 0x00000000000253ab __cudaPopCallConfiguration() ???:0 4 0x00000000000260ca __pgi_uacc_cuda_unregister_fat_binary() /proj/build/22A/Linux_x86_64/rte/accel-uni/build/Linux_x86_64/../../src/cuda_init.c:661 5 0x0000000000026092 __pgi_uacc_cuda_unregister_fat_binaries() /proj/build/22A/Linux_x86_64/rte/accel-uni/build/Linux_x86_64/../../src/cuda_init.c:651 6 0x0000000000039ce9 __run_exit_handlers() :0 7 0x0000000000039d37 __GI_exit() :0 8 0x0000000000045750 __pgi_uacc_exit() /proj/build/22A/Linux_x86_64/rte/accel-uni/build/Linux_x86_64/../../src/init.c:777 9 0x000000000001f451 __pgi_cuda_error_handler() /proj/build/22A/Linux_x86_64/rte/accel-uni/build/Linux_x86_64/../../src/cuda_launch.c:71 10 0x000000000002140a __pgi_uacc_cuda_launchk3() /proj/build/22A/Linux_x86_64/rte/accel-uni/build/Linux_x86_64/../../src/cuda_launch.c:559 11 0x000000000002262e __pgi_uacc_cuda_launch2() /proj/build/22A/Linux_x86_64/rte/accel-uni/build/Linux_x86_64/../../src/cuda_launche.c:157 12 0x00000000000227af __pgi_uacc_cuda_launch() /proj/build/22A/Linux_x86_64/rte/accel-uni/build/Linux_x86_64/../../src/cuda_launche.c:190 13 0x0000000000046ac3 __pgi_uacc_launch() /proj/build/22A/Linux_x86_64/rte/accel-uni/build/Linux_x86_64/../../src/launch.c:55 14 0x000000000051a47f DiagnosticScalar::compute() /scratch/project/open-28-44/smilei_GPU/Smilei/src/Diagnostic/DiagnosticScalar.cpp:447 15 0x0000000000ad7fbb VectorPatch::runAllDiags() /scratch/project/open-28-44/smilei_GPU/Smilei/src/Patch/VectorPatch.cpp:65 16 0x0000000000ceb221 main() /scratch/project/open-28-44/smilei_GPU/Smilei/src/Smilei.cpp:429 17 0x0000000000022555 __libc_start_main() ???:0 18 0x000000000049c82f _start() ???:0 ================================= Stack trace (most recent call last): #19 Object "[0xffffffffffffffff]", at 0xffffffffffffffff, in #18 Object "/scratch/project/open-28-44/smilei_GPU/Smilei/build/smilei", at 0x49c82e, in #17 Object "/lib64/libc.so.6", at 0x2b249ef7f554, in __libc_start_main #16 Object "/scratch/project/open-28-44/smilei_GPU/Smilei/build/smilei", at 0xceb220, in main #15 Object "/scratch/project/open-28-44/smilei_GPU/Smilei/build/smilei", at 0xad7fba, in VectorPatch::runAllDiags(Params&, SmileiMPI*, unsigned int, Timers&, SimWindow*) #14 Object "/scratch/project/open-28-44/smilei_GPU/Smilei/build/smilei", at 0x51a47e, in DiagnosticScalar::compute(Patch*, int) #13 Object "/apps/all/NVHPC/22.7-CUDA-11.7.0/Linux_x86_64/22.7/compilers/lib/libacchost.so", at 0x2b249ce30ac2, in __pgi_uacc_launch #12 Object "/apps/all/NVHPC/22.7-CUDA-11.7.0/Linux_x86_64/22.7/compilers/lib/libacccuda.so", at 0x2b249b6b97ae, in __pgi_uacc_cuda_launch #11 Object "/apps/all/NVHPC/22.7-CUDA-11.7.0/Linux_x86_64/22.7/compilers/lib/libacccuda.so", at 0x2b249b6b962d, in #10 Object "/apps/all/NVHPC/22.7-CUDA-11.7.0/Linux_x86_64/22.7/compilers/lib/libacccuda.so", at 0x2b249b6b8409, in __pgi_uacc_cuda_launchk3 #9 Object "/apps/all/NVHPC/22.7-CUDA-11.7.0/Linux_x86_64/22.7/compilers/lib/libacccuda.so", at 0x2b249b6b6450, in #8 Object "/apps/all/NVHPC/22.7-CUDA-11.7.0/Linux_x86_64/22.7/compilers/lib/libacchost.so", at 0x2b249ce2f74f, in __pgi_uacc_exit #7 Object "/lib64/libc.so.6", at 0x2b249ef96d36, in exit #6 Object "/lib64/libc.so.6", at 0x2b249ef96ce8, in #5 Object "/apps/all/NVHPC/22.7-CUDA-11.7.0/Linux_x86_64/22.7/compilers/lib/libacccuda.so", at 0x2b249b6bd091, in #4 Object "/apps/all/NVHPC/22.7-CUDA-11.7.0/Linux_x86_64/22.7/compilers/lib/libacccuda.so", at 0x2b249b6bd0c9, in #3 Object "/apps/all/CUDA/11.7.0/lib/libcudart.so.11.0", at 0x2b249581d3aa, in #2 Object "/apps/all/CUDA/11.7.0/lib/libcudart.so.11.0", at 0x2b249581b7bb, in #1 Object "/apps/all/CUDA/11.7.0/lib/libcudart.so.11.0", at 0x2b24958351c3, in #0 Object "/apps/all/CUDA/11.7.0/lib/libcudart.so.11.0", at 0x2b2495831a70, in Segmentation fault (Signal sent by tkill() [0x1e730001946d]) watf? exit -------------------------------------------------------------------------- Primary job terminated normally, but 1 process returned a non-zero exit code. Per user-direction, the job has been aborted. -------------------------------------------------------------------------- -------------------------------------------------------------------------- mpirun detected that one or more processes exited with non-zero status, thus causing the job to be terminated. The first process to do so was: Process name: [[42069,1],0] Exit code: 1 --------------------------------------------------------------------------`