From 748bd6b7f04e38ddd4d2e015947941823bb21d41 Mon Sep 17 00:00:00 2001 From: Stephen Copplestone Date: Mon, 21 Aug 2023 14:33:43 +0200 Subject: [PATCH 01/16] Abort if not all processes have surfaces for analysis because the MPI communicator requires this --- src/particles/surfacemodel/surfacemodel_analyze.f90 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/particles/surfacemodel/surfacemodel_analyze.f90 b/src/particles/surfacemodel/surfacemodel_analyze.f90 index 24ab955c0..bb78bb4b6 100644 --- a/src/particles/surfacemodel/surfacemodel_analyze.f90 +++ b/src/particles/surfacemodel/surfacemodel_analyze.f90 @@ -182,8 +182,12 @@ SUBROUTINE AnalyzeSurface(Time) INTEGER :: iEDCBC,i,iBoundary,iPartBound2 #endif /*USE_HDG*/ !=================================================================================================================================== -IF((nComputeNodeSurfSides.EQ.0).AND.(.NOT.CalcBoundaryParticleOutput).AND.(.NOT.UseNeutralization).AND.(.NOT.CalcElectronSEE)) RETURN +IF((.NOT.CalcBoundaryParticleOutput).AND.(.NOT.UseNeutralization).AND.(.NOT.CalcElectronSEE)) RETURN IF(.NOT.DoSurfModelAnalyze) RETURN + +! Currently the MPI routines below use Part%COMM (which contains all procs) and if one process returns here a hang-up occurs +IF(nComputeNodeSurfSides.EQ.0) CALL abort(__STAMP__,'Not all processes have surfaces for analysis but the communicator requires this.') + SurfModelAnalyzeSampleTime = Time - SurfModelAnalyzeSampleTime ! Set SurfModelAnalyzeSampleTime=Time at the end of this routine OutputCounter = 2 unit_index = 636 From 27625c54c1a684f0d3ea31b21f72dc99c05e1e7d Mon Sep 17 00:00:00 2001 From: Paul Nizenkov Date: Tue, 22 Aug 2023 18:24:49 +0200 Subject: [PATCH 02/16] Moved the initialization of nComputeNodeSurfSides,nComputeNodeSurfTotalSides,nComputeNodeSurfOutputSides and GlobalSide2SurfSide,SurfSide2GlobalSide to a new routine InitParticleBoundarySurfSides to avoid unnecessarily allocating the sampling variables --- .../boundary/particle_boundary_init.f90 | 307 +++++++++++++++++- .../boundary/particle_boundary_sampling.f90 | 284 +--------------- .../emission/particle_emission_init.f90 | 2 + src/particles/particle_init.f90 | 16 +- 4 files changed, 326 insertions(+), 283 deletions(-) diff --git a/src/particles/boundary/particle_boundary_init.f90 b/src/particles/boundary/particle_boundary_init.f90 index a0862f783..ef0fab172 100644 --- a/src/particles/boundary/particle_boundary_init.f90 +++ b/src/particles/boundary/particle_boundary_init.f90 @@ -26,7 +26,7 @@ MODULE MOD_Particle_Boundary_Init ! Private Part --------------------------------------------------------------------------------------------------------------------- ! Public Part ---------------------------------------------------------------------------------------------------------------------- -PUBLIC :: DefineParametersParticleBoundary, InitializeVariablesPartBoundary, FinalizeParticleBoundary +PUBLIC :: DefineParametersParticleBoundary, InitializeVariablesPartBoundary, InitParticleBoundarySurfSides, FinalizeParticleBoundary PUBLIC :: InitAdaptiveWallTemp, InitRotPeriodicMapping, InitRotPeriodicInterPlaneMapping !=================================================================================================================================== @@ -564,6 +564,299 @@ SUBROUTINE InitializeVariablesPartBoundary() END SUBROUTINE InitializeVariablesPartBoundary +SUBROUTINE InitParticleBoundarySurfSides() +!=================================================================================================================================== +! Initialize the counters (nComputeNodeSurfSides,nComputeNodeSurfTotalSides,nComputeNodeSurfOutputSides) and +! mappings (GlobalSide2SurfSide,SurfSide2GlobalSide) of the particle boundary surface sides +! 1) all procs identify surfaces on the node (plus halo region) for sampling and/or boundary conditions +! 2) the compute-node leaders communicate the number of surfaces +!=================================================================================================================================== +! MODULES ! +!----------------------------------------------------------------------------------------------------------------------------------! +USE MOD_Globals +USE MOD_Particle_Mesh_Vars ,ONLY: SideInfo_Shared +USE MOD_Particle_Boundary_Vars ,ONLY: PartBound +USE MOD_Particle_Boundary_Vars ,ONLY: nComputeNodeSurfSides,nComputeNodeSurfTotalSides,nComputeNodeSurfOutputSides +USE MOD_Particle_Boundary_Vars ,ONLY: GlobalSide2SurfSide,SurfSide2GlobalSide +#if USE_MPI +USE MOD_Particle_Mesh_Vars ,ONLY: ElemInfo_Shared +USE MOD_MPI_Shared +USE MOD_MPI_Shared_Vars ,ONLY: MPI_COMM_SHARED +USE MOD_MPI_Shared_Vars ,ONLY: myComputeNodeRank,nComputeNodeProcessors +USE MOD_Particle_Mesh_Vars ,ONLY: nNonUniqueGlobalSides +!USE MOD_Particle_Mesh_Vars ,ONLY: offsetComputeNodeElem,nComputeNodeElems +USE MOD_MPI_Shared_Vars ,ONLY: myLeaderGroupRank,nLeaderGroupProcs +USE MOD_Particle_Boundary_Vars ,ONLY: GlobalSide2SurfSide_Shared,GlobalSide2SurfSide_Shared_Win +USE MOD_Particle_Boundary_Vars ,ONLY: SurfSide2GlobalSide_Shared,SurfSide2GlobalSide_Shared_Win +USE MOD_Particle_Boundary_Vars ,ONLY: nComputeNodeInnerBCs +#else +USE MOD_Particle_Mesh_Vars ,ONLY: nComputeNodeSides +#endif /*USE_MPI*/ +#if USE_LOADBALANCE +USE MOD_LoadBalance_Vars ,ONLY: PerformLoadBalance +#endif /*USE_LOADBALANCE*/ +! IMPLICIT VARIABLE HANDLING +IMPLICIT NONE +!----------------------------------------------------------------------------------------------------------------------------------! +! INPUT VARIABLES +!----------------------------------------------------------------------------------------------------------------------------------! +! OUTPUT VARIABLES +!----------------------------------------------------------------------------------------------------------------------------------- +! LOCAL VARIABLES +INTEGER :: iSide,firstSide,lastSide,iSurfSide,GlobalSideID +INTEGER :: nSurfSidesProc +INTEGER :: offsetSurfTotalSidesProc +INTEGER,ALLOCATABLE :: GlobalSide2SurfSideProc(:,:) +#if USE_MPI +INTEGER :: offsetSurfSidesProc +INTEGER :: GlobalElemID,GlobalElemRank +INTEGER :: sendbuf,recvbuf +INTEGER :: NbGlobalElemID, NbElemRank, NbLeaderID, nSurfSidesTmp +#endif /*USE_MPI*/ +INTEGER :: NbGlobalSideID +!=================================================================================================================================== + +LBWRITE(UNIT_stdOut,'(A)') ' INIT SURFACE SIDES ...' + +! Allocate shared array for surf sides +#if USE_MPI +CALL Allocate_Shared((/3,nNonUniqueGlobalSides/),GlobalSide2SurfSide_Shared_Win,GlobalSide2SurfSide_Shared) +CALL MPI_WIN_LOCK_ALL(0,GlobalSide2SurfSide_Shared_Win,IERROR) +GlobalSide2SurfSide => GlobalSide2SurfSide_Shared +#else +ALLOCATE(GlobalSide2SurfSide(1:3,1:nComputeNodeSides)) +#endif /*USE_MPI*/ + +! only CN root nullifies +#if USE_MPI +IF (myComputeNodeRank.EQ.0) THEN +#endif /* USE_MPI*/ + GlobalSide2SurfSide = -1. +#if USE_MPI +END IF + +CALL BARRIER_AND_SYNC(GlobalSide2SurfSide_Shared_Win,MPI_COMM_SHARED) +#endif /* USE_MPI*/ + +! get number of BC-Sides +#if USE_MPI +! NO HALO REGION REDUCTION +firstSide = INT(REAL( myComputeNodeRank )*REAL(nNonUniqueGlobalSides)/REAL(nComputeNodeProcessors))+1 +lastSide = INT(REAL((myComputeNodeRank+1))*REAL(nNonUniqueGlobalSides)/REAL(nComputeNodeProcessors)) +ALLOCATE(GlobalSide2SurfSideProc(1:3,firstSide:lastSide)) +#else +firstSide = 1 +lastSide = nComputeNodeSides +ALLOCATE(GlobalSide2SurfSideProc(1:3,1:nComputeNodeSides)) +#endif /*USE_MPI*/ + +GlobalSide2SurfSideProc = -1 +nComputeNodeSurfSides = 0 +nSurfSidesProc = 0 + +! check every BC side +DO iSide = firstSide,lastSide + ! ignore non-BC sides + IF (SideInfo_Shared(SIDE_BCID,iSide).LE.0) CYCLE + +#if USE_MPI + ! ignore sides outside of halo region + IF (ElemInfo_Shared(ELEM_HALOFLAG,SideInfo_Shared(SIDE_ELEMID,iSide)).EQ.0) CYCLE +#endif /*USE_MPI*/ + + ! count number of reflective and rotationally periodic BC sides + IF ((PartBound%TargetBoundCond(PartBound%MapToPartBC(SideInfo_Shared(SIDE_BCID,iSide))).EQ.PartBound%ReflectiveBC) .OR. & + (PartBound%TargetBoundCond(PartBound%MapToPartBC(SideInfo_Shared(SIDE_BCID,iSide))).EQ.PartBound%RotPeriodicBC).OR. & + (PartBound%TargetBoundCond(PartBound%MapToPartBC(SideInfo_Shared(SIDE_BCID,iSide))).EQ.PartBound%RotPeriodicInterPlaneBC))THEN + nSurfSidesProc = nSurfSidesProc + 1 + ! check if element for this side is on the current compute-node + ! IF ((SideInfo_Shared(SIDE_ID,iSide).GT.ElemInfo_Shared(ELEM_FIRSTSIDEIND,offsetComputeNodeElem+1)) .AND. & + ! (SideInfo_Shared(SIDE_ID,iSide).LE.ElemInfo_Shared(ELEM_LASTSIDEIND ,offsetComputeNodeElem+nComputeNodeElems))) THEN +! IF ((iSide.GE.(ElemInfo_Shared(ELEM_FIRSTSIDEIND,offsetComputeNodeElem+1)+1)) .AND. & +! (iSide.LE.ElemInfo_Shared(ELEM_LASTSIDEIND ,offsetComputeNodeElem+nComputeNodeElems))) THEN +! nComputeNodeSurfSides = nComputeNodeSurfSides + 1 +! END IF + + ! TODO: Add another check to determine the surface side in halo_eps from current proc. Node-wide halo can become quite large with + ! with 128 procs! + + ! Write local mapping from Side to Surf side. The rank is already correct, the offset must be corrected by the proc offset later + GlobalSide2SurfSideProc(SURF_SIDEID,iSide) = nSurfSidesProc +#if USE_MPI + GlobalSide2SurfSideProc(SURF_RANK ,iSide) = ElemInfo_Shared(ELEM_RANK,SideInfo_Shared(SIDE_ELEMID,iSide)) + ! get global Elem ID + GlobalElemID = SideInfo_Shared(SIDE_ELEMID,iSide) + GlobalElemRank = ElemInfo_Shared(ELEM_RANK,GlobalElemID) + ! running on one node, everything belongs to us + IF (nLeaderGroupProcs.EQ.1) THEN + GlobalSide2SurfSideProc(SURF_LEADER,iSide) = myLeaderGroupRank + ELSE + ! find the compute node + GlobalSide2SurfSideProc(SURF_LEADER,iSide) = INT(GlobalElemRank/nComputeNodeProcessors) + END IF +#else + GlobalSide2SurfSideProc(SURF_RANK ,iSide) = 0 + GlobalSide2SurfSideProc(SURF_LEADER,iSide) = GlobalSide2SurfSideProc(SURF_RANK,iSide) +#endif /*USE_MPI*/ + +#if USE_MPI + ! check if element for this side is on the current compute-node. Alternative version to the check above + IF (GlobalSide2SurfSideProc(SURF_LEADER,iSide).EQ.myLeaderGroupRank) THEN +#endif /*USE_MPI*/ + nComputeNodeSurfSides = nComputeNodeSurfSides + 1 +#if USE_MPI + END IF +#endif /*USE_MPI*/ + END IF ! reflective side +END DO + +! Find CN global number of total surf sides and write Side to Surf Side mapping into shared array +#if USE_MPI +sendbuf = nSurfSidesProc - nComputeNodeSurfSides +recvbuf = 0 +CALL MPI_EXSCAN(sendbuf,recvbuf,1,MPI_INTEGER,MPI_SUM,MPI_COMM_SHARED,iError) +offsetSurfTotalSidesProc = recvbuf +! last proc knows CN total number of BC elems +sendbuf = offsetSurfTotalSidesProc + nSurfSidesProc - nComputeNodeSurfSides +CALL MPI_BCAST(sendbuf,1,MPI_INTEGER,nComputeNodeProcessors-1,MPI_COMM_SHARED,iError) +nComputeNodeSurfTotalSides = sendbuf + +! Find CN global number of local surf sides and write Side to Surf Side mapping into shared array +sendbuf = nComputeNodeSurfSides +recvbuf = 0 +CALL MPI_EXSCAN(sendbuf,recvbuf,1,MPI_INTEGER,MPI_SUM,MPI_COMM_SHARED,iError) +offsetSurfSidesProc = recvbuf +! last proc knows CN total number of BC elems +sendbuf = offsetSurfSidesProc + nComputeNodeSurfSides +CALL MPI_BCAST(sendbuf,1,MPI_INTEGER,nComputeNodeProcessors-1,MPI_COMM_SHARED,iError) +nComputeNodeSurfSides = sendbuf +nComputeNodeSurfTotalSides = nComputeNodeSurfTotalSides + nComputeNodeSurfSides + +! increment SURF_SIDEID by offset +nSurfSidesTmp = 0 +DO iSide = firstSide,lastSide + IF (GlobalSide2SurfSideProc(SURF_SIDEID,iSide).EQ.-1) CYCLE + + ! sort compute-node local sides first + IF (GlobalSide2SurfSideProc(SURF_LEADER,iSide).EQ.myLeaderGroupRank) THEN + nSurfSidesTmp = nSurfSidesTmp + 1 + + GlobalSide2SurfSide(: ,iSide) = GlobalSide2SurfSideProc(:,iSide) + GlobalSide2SurfSide(SURF_SIDEID,iSide) = nSurfSidesTmp + offsetSurfSidesProc + END IF +END DO + +nSurfSidesTmp = 0 +DO iSide = firstSide,lastSide + IF (GlobalSide2SurfSideProc(SURF_SIDEID,iSide).EQ.-1) CYCLE + + ! sampling sides in halo region follow at the end + IF (GlobalSide2SurfSideProc(SURF_LEADER,iSide).NE.myLeaderGroupRank) THEN + nSurfSidesTmp = nSurfSidesTmp + 1 + + GlobalSide2SurfSide(: ,iSide) = GlobalSide2SurfSideProc(:,iSide) + GlobalSide2SurfSide(SURF_SIDEID,iSide) = nSurfSidesTmp + nComputeNodeSurfSides + offsetSurfTotalSidesProc + END IF +END DO +#else +offsetSurfTotalSidesProc = 0 +nComputeNodeSurfTotalSides = nSurfSidesProc +GlobalSide2SurfSide(:,firstSide:lastSide) = GlobalSide2SurfSideProc(:,firstSide:lastSide) +#endif /*USE_MPI*/ + +! Build inverse mapping +#if USE_MPI +CALL Allocate_Shared((/3,nComputeNodeSurfTotalSides/),SurfSide2GlobalSide_Shared_Win,SurfSide2GlobalSide_Shared) +CALL MPI_WIN_LOCK_ALL(0,SurfSide2GlobalSide_Shared_Win,IERROR) +SurfSide2GlobalSide => SurfSide2GlobalSide_Shared + +DO iSide = firstSide,lastSide + IF (GlobalSide2SurfSideProc(SURF_SIDEID,iSide).EQ.-1) CYCLE + + SurfSide2GlobalSide(: ,GlobalSide2SurfSide(SURF_SIDEID,iSide)) = GlobalSide2SurfSide(:,iSide) + SurfSide2GlobalSide(SURF_SIDEID,GlobalSide2SurfSide(SURF_SIDEID,iSide)) = iSide +END DO + +CALL BARRIER_AND_SYNC(GlobalSide2SurfSide_Shared_Win,MPI_COMM_SHARED) +CALL BARRIER_AND_SYNC(SurfSide2GlobalSide_Shared_Win,MPI_COMM_SHARED) +#else +ALLOCATE(SurfSide2GlobalSide(1:1,1:nComputeNodeSurfTotalSides)) +DO iSide = firstSide,lastSide + IF (GlobalSide2SurfSide(SURF_SIDEID,iSide).EQ.-1) CYCLE + SurfSide2GlobalSide(SURF_SIDEID,GlobalSide2SurfSide(SURF_SIDEID,iSide)) =iSide +END DO +#endif /*USE_MPI*/ + +! Determine the number of surface output sides (inner BCs are not counted twice and rotationally periodic BCs excluded) +#if USE_MPI +IF (myComputeNodeRank.EQ.0) THEN + nComputeNodeInnerBCs = 0 +#endif /*USE_MPI*/ + nComputeNodeSurfOutputSides = 0 + DO iSurfSide = 1,nComputeNodeSurfSides + GlobalSideID = SurfSide2GlobalSide(SURF_SIDEID,iSurfSide) + ! Check if the surface side has a neighbor (and is therefore an inner BCs) + IF(SideInfo_Shared(SIDE_NBSIDEID,GlobalSideID).GT.0) THEN + ! Abort inner BC + Mortar! (too complex and confusing to implement) + ! This test catches large Mortar sides, i.e., SideInfo_Shared(SIDE_NBELEMID,NonUniqueGlobalSideID) gives the 2 or 4 + ! connecting small Mortar sides. It is assumed that inner BC result in being flagged as a "SurfSide" and therefore are checked + ! here. + IF(SideInfo_Shared(SIDE_LOCALID,GlobalSideID).EQ.-1)THEN + IPWRITE(UNIT_StdOut,'(I12,A,I0)') " NonUniqueGlobalSideID = ",GlobalSideID + IPWRITE(UNIT_StdOut,'(I12,A,I0)') " SideInfo_Shared(SIDE_LOCALID,NonUniqueGlobalSideID) = ",& + SideInfo_Shared(SIDE_LOCALID,GlobalSideID) + IPWRITE(UNIT_StdOut,'(I12,A,I0,A)') " SideInfo_Shared(SIDE_ELEMID,NonUniqueGlobalSideID) = ",& + SideInfo_Shared(SIDE_ELEMID,GlobalSideID)," (GlobalElemID)" + CALL abort(__STAMP__,'Inner BC + Mortar is not implemented!') + END IF + ! Only add the side with the smaller index + NbGlobalSideID = SideInfo_Shared(SIDE_NBSIDEID,GlobalSideID) + IF(GlobalSideID.GT.NbGlobalSideID)THEN +#if USE_MPI + !--- switcheroo check 1 of 2: Non-HALO sides + ! Only required for sampling on the larger NonUniqueGlobalSideID of the two sides of the inner BC + ! Count larger inner BCs as these may have to be sent to a different leader processor + NbGlobalElemID = SideInfo_Shared(SIDE_ELEMID,NbGlobalSideID) + NbElemRank = ElemInfo_Shared(ELEM_RANK,NbGlobalElemID) + NbLeaderID = INT(NbElemRank/nComputeNodeProcessors) + IF(NbLeaderID.NE.INT(myRank/nComputeNodeProcessors))THEN + nComputeNodeInnerBCs(1) = nComputeNodeInnerBCs(1) + 1 + END IF +#endif + CYCLE! Skip sides with the larger index + END IF + END IF + ! Skip rotationally periodic boundary sides for the output + IF(PartBound%TargetBoundCond(PartBound%MapToPartBC(SideInfo_Shared(SIDE_BCID,GlobalSideID))).EQ.PartBound%RotPeriodicBC) CYCLE + ! Skip intermediate planes BCs for the output + IF(PartBound%TargetBoundCond(PartBound%MapToPartBC(SideInfo_Shared(SIDE_BCID,GlobalSideID))).EQ.PartBound%RotPeriodicInterPlaneBC) CYCLE + ! Count the number of output sides + nComputeNodeSurfOutputSides = nComputeNodeSurfOutputSides + 1 + END DO +#if USE_MPI + !--- switcheroo check 2 of 2: HALO sides + ! Count number of inner BC in halo region + ! Only required for sampling on the larger NonUniqueGlobalSideID of the two sides of the inner BC + DO iSurfSide = nComputeNodeSurfSides+1, nComputeNodeSurfTotalSides + GlobalSideID = SurfSide2GlobalSide(SURF_SIDEID,iSurfSide) + ! Check if the surface side has a neighbor (and is therefore an inner BCs) + IF(SideInfo_Shared(SIDE_NBSIDEID,GlobalSideID).GT.0) THEN + ! Only add the side with the smaller index + IF(GlobalSideID.GT.SideInfo_Shared(SIDE_NBSIDEID,GlobalSideID))THEN + ! Count larger inner BCs as these may have to be sent to a different leader processor + nComputeNodeInnerBCs(2) = nComputeNodeInnerBCs(2) + 1 + END IF + END IF + END DO ! iSurfSide = nComputeNodeSurfSides+1, nComputeNodeSurfTotalSides +END IF +#endif + +! free temporary arrays +DEALLOCATE(GlobalSide2SurfSideProc) + +END SUBROUTINE InitParticleBoundarySurfSides + + SUBROUTINE InitParticleBoundaryRotPeriodic(nRotPeriodicBCs) !=================================================================================================================================== !> @@ -1757,6 +2050,18 @@ SUBROUTINE FinalizeParticleBoundary() SDEALLOCATE(PartBound%BoundaryParticleOutputHDF5) SDEALLOCATE(PartBound%RadiativeEmissivity) +! Mapping arrays are allocated even if the node does not have sampling surfaces +#if USE_MPI +CALL MPI_BARRIER(MPI_COMM_SHARED,iERROR) +CALL UNLOCK_AND_FREE(GlobalSide2SurfSide_Shared_Win) +CALL UNLOCK_AND_FREE(SurfSide2GlobalSide_Shared_Win) +CALL MPI_BARRIER(MPI_COMM_SHARED,iERROR) +ADEALLOCATE(GlobalSide2SurfSide_Shared) +ADEALLOCATE(SurfSide2GlobalSide_Shared) +#endif /*USE_MPI*/ +ADEALLOCATE(GlobalSide2SurfSide) +ADEALLOCATE(SurfSide2GlobalSide) + ! Rotational periodic boundary condition IF(PartBound%UseRotPeriodicBC)THEN #if USE_MPI diff --git a/src/particles/boundary/particle_boundary_sampling.f90 b/src/particles/boundary/particle_boundary_sampling.f90 index 6a3142239..a46d1c7ac 100644 --- a/src/particles/boundary/particle_boundary_sampling.f90 +++ b/src/particles/boundary/particle_boundary_sampling.f90 @@ -20,7 +20,6 @@ MODULE MOD_Particle_Boundary_Sampling ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE PRIVATE - !----------------------------------------------------------------------------------------------------------------------------------- INTERFACE DefineParametersParticleBoundarySampling MODULE PROCEDURE DefineParametersParticleBoundarySampling @@ -81,7 +80,7 @@ SUBROUTINE InitParticleBoundarySampling() USE MOD_Particle_Boundary_Vars ,ONLY: nComputeNodeSurfSides,nComputeNodeSurfTotalSides,nComputeNodeSurfOutputSides USE MOD_Particle_Boundary_Vars ,ONLY: nSurfBC,SurfBCName USE MOD_Particle_Boundary_Vars ,ONLY: nSurfTotalSides -USE MOD_Particle_Boundary_Vars ,ONLY: GlobalSide2SurfSide,SurfSide2GlobalSide +USE MOD_Particle_Boundary_Vars ,ONLY: SurfSide2GlobalSide USE MOD_SurfaceModel_Vars ,ONLY: nPorousBC USE MOD_Particle_Boundary_Vars ,ONLY: CalcSurfaceImpact USE MOD_Particle_Boundary_Vars ,ONLY: SurfSideArea,SurfSampSize,SurfOutputSize,SurfSpecOutputSize @@ -102,16 +101,11 @@ SUBROUTINE InitParticleBoundarySampling() USE MOD_Particle_Vars ,ONLY: Symmetry USE MOD_ReadInTools ,ONLY: GETINT,GETLOGICAL,GETINTARRAY #if USE_MPI -USE MOD_Particle_Mesh_Vars ,ONLY: ElemInfo_Shared USE MOD_MPI_Shared USE MOD_MPI_Shared_Vars ,ONLY: MPI_COMM_SHARED USE MOD_MPI_Shared_Vars ,ONLY: MPI_COMM_LEADERS_SURF,mySurfRank USE MOD_MPI_Shared_Vars ,ONLY: myComputeNodeRank,nComputeNodeProcessors -USE MOD_Particle_Mesh_Vars ,ONLY: nNonUniqueGlobalSides -!USE MOD_Particle_Mesh_Vars ,ONLY: offsetComputeNodeElem,nComputeNodeElems -USE MOD_MPI_Shared_Vars ,ONLY: myLeaderGroupRank,nLeaderGroupProcs -USE MOD_Particle_Boundary_Vars ,ONLY: GlobalSide2SurfSide_Shared,GlobalSide2SurfSide_Shared_Win -USE MOD_Particle_Boundary_Vars ,ONLY: SurfSide2GlobalSide_Shared,SurfSide2GlobalSide_Shared_Win +USE MOD_Particle_Boundary_Vars ,ONLY: SurfSide2GlobalSide_Shared USE MOD_Particle_Boundary_Vars ,ONLY: SurfSideArea_Shared,SurfSideArea_Shared_Win USE MOD_Particle_Boundary_Vars ,ONLY: SampWallState_Shared,SampWallState_Shared_Win USE MOD_Particle_Boundary_Vars ,ONLY: SampWallPumpCapacity_Shared,SampWallPumpCapacity_Shared_Win @@ -120,7 +114,6 @@ SUBROUTINE InitParticleBoundarySampling() USE MOD_Particle_Boundary_Vars ,ONLY: SampWallImpactAngle_Shared,SampWallImpactAngle_Shared_Win USE MOD_Particle_Boundary_Vars ,ONLY: SampWallImpactNumber_Shared,SampWallImpactNumber_Shared_Win USE MOD_Particle_MPI_Boundary_Sampling,ONLY: InitSurfCommunication -USE MOD_Particle_Boundary_Vars ,ONLY: nComputeNodeInnerBCs #else USE MOD_MPI_Shared_Vars ,ONLY: mySurfRank USE MOD_Particle_Mesh_Vars ,ONLY: nComputeNodeSides @@ -138,11 +131,7 @@ SUBROUTINE InitParticleBoundarySampling() !----------------------------------------------------------------------------------------------------------------------------------- ! LOCAL VARIABLES INTEGER :: iBC -INTEGER :: iSide,firstSide,lastSide,iSurfSide,GlobalSideID -INTEGER :: nSurfSidesProc -INTEGER :: offsetSurfTotalSidesProc -INTEGER,ALLOCATABLE :: GlobalSide2SurfSideProc(:,:) -!INTEGER,ALLOCATABLE :: SurfSide2GlobalSideProc(:,:) +INTEGER :: iSide,firstSide,lastSide CHARACTER(20) :: hilf CHARACTER(LEN=255),ALLOCATABLE :: BCName(:) ! surface area @@ -154,13 +143,6 @@ SUBROUTINE InitParticleBoundarySampling() REAL,DIMENSION(:),ALLOCATABLE :: Xi_NGeo,wGP_NGeo REAL :: XiOut(1:2),E,F,G,D,tmp1,tmpI2,tmpJ2 REAL :: xNod, zNod, yNod, Vector1(3), Vector2(3), nx, ny, nz -#if USE_MPI -INTEGER :: offsetSurfSidesProc -INTEGER :: GlobalElemID,GlobalElemRank -INTEGER :: sendbuf,recvbuf -INTEGER :: NbGlobalElemID, NbElemRank, NbLeaderID, nSurfSidesTmp -#endif /*USE_MPI*/ -INTEGER :: NbGlobalSideID !=================================================================================================================================== ! Get input parameters @@ -175,247 +157,6 @@ SUBROUTINE InitParticleBoundarySampling() ! Sampling of impact energy for each species (trans, rot, vib), impact vector (x,y,z) and angle CalcSurfaceImpact = GETLOGICAL('CalcSurfaceImpact') -! Allocate shared array for surf sides -#if USE_MPI -CALL Allocate_Shared((/3,nNonUniqueGlobalSides/),GlobalSide2SurfSide_Shared_Win,GlobalSide2SurfSide_Shared) -CALL MPI_WIN_LOCK_ALL(0,GlobalSide2SurfSide_Shared_Win,IERROR) -GlobalSide2SurfSide => GlobalSide2SurfSide_Shared -#else -ALLOCATE(GlobalSide2SurfSide(1:3,1:nComputeNodeSides)) -#endif /*USE_MPI*/ - -! only CN root nullifies -#if USE_MPI -IF (myComputeNodeRank.EQ.0) THEN -#endif /* USE_MPI*/ - GlobalSide2SurfSide = -1. -#if USE_MPI -END IF - -CALL BARRIER_AND_SYNC(GlobalSide2SurfSide_Shared_Win,MPI_COMM_SHARED) -#endif /* USE_MPI*/ - -! get number of BC-Sides -#if USE_MPI -! NO HALO REGION REDUCTION -firstSide = INT(REAL( myComputeNodeRank )*REAL(nNonUniqueGlobalSides)/REAL(nComputeNodeProcessors))+1 -lastSide = INT(REAL((myComputeNodeRank+1))*REAL(nNonUniqueGlobalSides)/REAL(nComputeNodeProcessors)) -ALLOCATE(GlobalSide2SurfSideProc(1:3,firstSide:lastSide)) - !,SurfSide2GlobalSideProc(1:3,1 :INT(nNonUniqueGlobalSides/REAL(nComputeNodeProcessors)))) -#else -firstSide = 1 -lastSide = nComputeNodeSides -ALLOCATE(GlobalSide2SurfSideProc(1:3,1:nComputeNodeSides)) - !,SurfSide2GlobalSideProc(1:3,1:nComputeNodeSides)) -#endif /*USE_MPI*/ - -GlobalSide2SurfSideProc = -1 -!SurfSide2GlobalSideProc = -1 -nComputeNodeSurfSides = 0 -nSurfSidesProc = 0 - -! check every BC side -DO iSide = firstSide,lastSide - ! ignore non-BC sides - IF (SideInfo_Shared(SIDE_BCID,iSide).LE.0) CYCLE - -#if USE_MPI - ! ignore sides outside of halo region - IF (ElemInfo_Shared(ELEM_HALOFLAG,SideInfo_Shared(SIDE_ELEMID,iSide)).EQ.0) CYCLE -#endif /*USE_MPI*/ - - ! count number of reflective and rotationally periodic BC sides - IF ((PartBound%TargetBoundCond(PartBound%MapToPartBC(SideInfo_Shared(SIDE_BCID,iSide))).EQ.PartBound%ReflectiveBC) .OR. & - (PartBound%TargetBoundCond(PartBound%MapToPartBC(SideInfo_Shared(SIDE_BCID,iSide))).EQ.PartBound%RotPeriodicBC).OR. & - (PartBound%TargetBoundCond(PartBound%MapToPartBC(SideInfo_Shared(SIDE_BCID,iSide))).EQ.PartBound%RotPeriodicInterPlaneBC))THEN - nSurfSidesProc = nSurfSidesProc + 1 - ! check if element for this side is on the current compute-node - ! IF ((SideInfo_Shared(SIDE_ID,iSide).GT.ElemInfo_Shared(ELEM_FIRSTSIDEIND,offsetComputeNodeElem+1)) .AND. & - ! (SideInfo_Shared(SIDE_ID,iSide).LE.ElemInfo_Shared(ELEM_LASTSIDEIND ,offsetComputeNodeElem+nComputeNodeElems))) THEN -! IF ((iSide.GE.(ElemInfo_Shared(ELEM_FIRSTSIDEIND,offsetComputeNodeElem+1)+1)) .AND. & -! (iSide.LE.ElemInfo_Shared(ELEM_LASTSIDEIND ,offsetComputeNodeElem+nComputeNodeElems))) THEN -! nComputeNodeSurfSides = nComputeNodeSurfSides + 1 -! END IF - - ! TODO: Add another check to determine the surface side in halo_eps from current proc. Node-wide halo can become quite large with - ! with 128 procs! - - ! Write local mapping from Side to Surf side. The rank is already correct, the offset must be corrected by the proc offset later - GlobalSide2SurfSideProc(SURF_SIDEID,iSide) = nSurfSidesProc -#if USE_MPI - GlobalSide2SurfSideProc(SURF_RANK ,iSide) = ElemInfo_Shared(ELEM_RANK,SideInfo_Shared(SIDE_ELEMID,iSide)) - ! get global Elem ID - GlobalElemID = SideInfo_Shared(SIDE_ELEMID,iSide) - GlobalElemRank = ElemInfo_Shared(ELEM_RANK,GlobalElemID) - ! running on one node, everything belongs to us - IF (nLeaderGroupProcs.EQ.1) THEN - GlobalSide2SurfSideProc(SURF_LEADER,iSide) = myLeaderGroupRank - ELSE - ! find the compute node - GlobalSide2SurfSideProc(SURF_LEADER,iSide) = INT(GlobalElemRank/nComputeNodeProcessors) - END IF -#else - GlobalSide2SurfSideProc(SURF_RANK ,iSide) = 0 - GlobalSide2SurfSideProc(SURF_LEADER,iSide) = GlobalSide2SurfSideProc(SURF_RANK,iSide) -#endif /*USE_MPI*/ - -#if USE_MPI - ! check if element for this side is on the current compute-node. Alternative version to the check above - IF (GlobalSide2SurfSideProc(SURF_LEADER,iSide).EQ.myLeaderGroupRank) THEN -#endif /*USE_MPI*/ - nComputeNodeSurfSides = nComputeNodeSurfSides + 1 -#if USE_MPI - END IF -#endif /*USE_MPI*/ - END IF ! reflective side -END DO - -! Find CN global number of total surf sides and write Side to Surf Side mapping into shared array -#if USE_MPI -sendbuf = nSurfSidesProc - nComputeNodeSurfSides -recvbuf = 0 -CALL MPI_EXSCAN(sendbuf,recvbuf,1,MPI_INTEGER,MPI_SUM,MPI_COMM_SHARED,iError) -offsetSurfTotalSidesProc = recvbuf -! last proc knows CN total number of BC elems -sendbuf = offsetSurfTotalSidesProc + nSurfSidesProc - nComputeNodeSurfSides -CALL MPI_BCAST(sendbuf,1,MPI_INTEGER,nComputeNodeProcessors-1,MPI_COMM_SHARED,iError) -nComputeNodeSurfTotalSides = sendbuf - -! Find CN global number of local surf sides and write Side to Surf Side mapping into shared array -sendbuf = nComputeNodeSurfSides -recvbuf = 0 -CALL MPI_EXSCAN(sendbuf,recvbuf,1,MPI_INTEGER,MPI_SUM,MPI_COMM_SHARED,iError) -offsetSurfSidesProc = recvbuf -! last proc knows CN total number of BC elems -sendbuf = offsetSurfSidesProc + nComputeNodeSurfSides -CALL MPI_BCAST(sendbuf,1,MPI_INTEGER,nComputeNodeProcessors-1,MPI_COMM_SHARED,iError) -nComputeNodeSurfSides = sendbuf -nComputeNodeSurfTotalSides = nComputeNodeSurfTotalSides + nComputeNodeSurfSides - -! increment SURF_SIDEID by offset -nSurfSidesTmp = 0 -DO iSide = firstSide,lastSide - IF (GlobalSide2SurfSideProc(SURF_SIDEID,iSide).EQ.-1) CYCLE - - ! sort compute-node local sides first - IF (GlobalSide2SurfSideProc(SURF_LEADER,iSide).EQ.myLeaderGroupRank) THEN - nSurfSidesTmp = nSurfSidesTmp + 1 - - GlobalSide2SurfSide(: ,iSide) = GlobalSide2SurfSideProc(:,iSide) - GlobalSide2SurfSide(SURF_SIDEID,iSide) = nSurfSidesTmp + offsetSurfSidesProc - END IF -END DO - -nSurfSidesTmp = 0 -DO iSide = firstSide,lastSide - IF (GlobalSide2SurfSideProc(SURF_SIDEID,iSide).EQ.-1) CYCLE - - ! sampling sides in halo region follow at the end - IF (GlobalSide2SurfSideProc(SURF_LEADER,iSide).NE.myLeaderGroupRank) THEN - nSurfSidesTmp = nSurfSidesTmp + 1 - - GlobalSide2SurfSide(: ,iSide) = GlobalSide2SurfSideProc(:,iSide) - GlobalSide2SurfSide(SURF_SIDEID,iSide) = nSurfSidesTmp + nComputeNodeSurfSides + offsetSurfTotalSidesProc - END IF -END DO -#else -offsetSurfTotalSidesProc = 0 -nComputeNodeSurfTotalSides = nSurfSidesProc -GlobalSide2SurfSide(:,firstSide:lastSide) = GlobalSide2SurfSideProc(:,firstSide:lastSide) -#endif /*USE_MPI*/ - -! Build inverse mapping -#if USE_MPI -CALL Allocate_Shared((/3,nComputeNodeSurfTotalSides/),SurfSide2GlobalSide_Shared_Win,SurfSide2GlobalSide_Shared) -CALL MPI_WIN_LOCK_ALL(0,SurfSide2GlobalSide_Shared_Win,IERROR) -SurfSide2GlobalSide => SurfSide2GlobalSide_Shared - -DO iSide = firstSide,lastSide - IF (GlobalSide2SurfSideProc(SURF_SIDEID,iSide).EQ.-1) CYCLE - - SurfSide2GlobalSide(: ,GlobalSide2SurfSide(SURF_SIDEID,iSide)) = GlobalSide2SurfSide(:,iSide) - SurfSide2GlobalSide(SURF_SIDEID,GlobalSide2SurfSide(SURF_SIDEID,iSide)) = iSide -END DO - -CALL BARRIER_AND_SYNC(GlobalSide2SurfSide_Shared_Win,MPI_COMM_SHARED) -CALL BARRIER_AND_SYNC(SurfSide2GlobalSide_Shared_Win,MPI_COMM_SHARED) -#else -ALLOCATE(SurfSide2GlobalSide(1:1,1:nComputeNodeSurfTotalSides)) -!SurfSide2GlobalSide = SurfSide2GlobalSideProc(:,1:nComputeNodeSurfTotalSides) -DO iSide = firstSide,lastSide - IF (GlobalSide2SurfSide(SURF_SIDEID,iSide).EQ.-1) CYCLE - SurfSide2GlobalSide(SURF_SIDEID,GlobalSide2SurfSide(SURF_SIDEID,iSide)) =iSide -END DO -#endif /*USE_MPI*/ - -! Determine the number of surface output sides (inner BCs are not counted twice and rotationally periodic BCs excluded) -#if USE_MPI -IF (myComputeNodeRank.EQ.0) THEN - nComputeNodeInnerBCs = 0 -#endif /*USE_MPI*/ - nComputeNodeSurfOutputSides = 0 - DO iSurfSide = 1,nComputeNodeSurfSides - GlobalSideID = SurfSide2GlobalSide(SURF_SIDEID,iSurfSide) - ! Check if the surface side has a neighbor (and is therefore an inner BCs) - IF(SideInfo_Shared(SIDE_NBSIDEID,GlobalSideID).GT.0) THEN - ! Abort inner BC + Mortar! (too complex and confusing to implement) - ! This test catches large Mortar sides, i.e., SideInfo_Shared(SIDE_NBELEMID,NonUniqueGlobalSideID) gives the 2 or 4 - ! connecting small Mortar sides. It is assumed that inner BC result in being flagged as a "SurfSide" and therefore are checked - ! here. - IF(SideInfo_Shared(SIDE_LOCALID,GlobalSideID).EQ.-1)THEN - IPWRITE(UNIT_StdOut,'(I12,A,I0)') " NonUniqueGlobalSideID = ",GlobalSideID - IPWRITE(UNIT_StdOut,'(I12,A,I0)') " SideInfo_Shared(SIDE_LOCALID,NonUniqueGlobalSideID) = ",& - SideInfo_Shared(SIDE_LOCALID,GlobalSideID) - IPWRITE(UNIT_StdOut,'(I12,A,I0,A)') " SideInfo_Shared(SIDE_ELEMID,NonUniqueGlobalSideID) = ",& - SideInfo_Shared(SIDE_ELEMID,GlobalSideID)," (GlobalElemID)" - CALL abort(__STAMP__,'Inner BC + Mortar is not implemented!') - END IF - ! Only add the side with the smaller index - NbGlobalSideID = SideInfo_Shared(SIDE_NBSIDEID,GlobalSideID) - IF(GlobalSideID.GT.NbGlobalSideID)THEN -#if USE_MPI - !--- switcheroo check 1 of 2: Non-HALO sides - ! Only required for sampling on the larger NonUniqueGlobalSideID of the two sides of the inner BC - ! Count larger inner BCs as these may have to be sent to a different leader processor - NbGlobalElemID = SideInfo_Shared(SIDE_ELEMID,NbGlobalSideID) - NbElemRank = ElemInfo_Shared(ELEM_RANK,NbGlobalElemID) - NbLeaderID = INT(NbElemRank/nComputeNodeProcessors) - IF(NbLeaderID.NE.INT(myRank/nComputeNodeProcessors))THEN - nComputeNodeInnerBCs(1) = nComputeNodeInnerBCs(1) + 1 - END IF -#endif - CYCLE! Skip sides with the larger index - END IF - END IF - ! Skip rotationally periodic boundary sides for the output - IF(PartBound%TargetBoundCond(PartBound%MapToPartBC(SideInfo_Shared(SIDE_BCID,GlobalSideID))).EQ.PartBound%RotPeriodicBC) CYCLE - ! Skip intermediate planes BCs for the output - IF(PartBound%TargetBoundCond(PartBound%MapToPartBC(SideInfo_Shared(SIDE_BCID,GlobalSideID))).EQ.PartBound%RotPeriodicInterPlaneBC) CYCLE - ! Count the number of output sides - nComputeNodeSurfOutputSides = nComputeNodeSurfOutputSides + 1 - END DO -#if USE_MPI - !--- switcheroo check 2 of 2: HALO sides - ! Count number of inner BC in halo region - ! Only required for sampling on the larger NonUniqueGlobalSideID of the two sides of the inner BC - DO iSurfSide = nComputeNodeSurfSides+1, nComputeNodeSurfTotalSides - GlobalSideID = SurfSide2GlobalSide(SURF_SIDEID,iSurfSide) - ! Check if the surface side has a neighbor (and is therefore an inner BCs) - IF(SideInfo_Shared(SIDE_NBSIDEID,GlobalSideID).GT.0) THEN - ! Only add the side with the smaller index - IF(GlobalSideID.GT.SideInfo_Shared(SIDE_NBSIDEID,GlobalSideID))THEN - ! Count larger inner BCs as these may have to be sent to a different leader processor - nComputeNodeInnerBCs(2) = nComputeNodeInnerBCs(2) + 1 - END IF - END IF - END DO ! iSurfSide = nComputeNodeSurfSides+1, nComputeNodeSurfTotalSides -END IF -#endif - -! free temporary arrays -DEALLOCATE(GlobalSide2SurfSideProc) -!DEALLOCATE(SurfSide2GlobalSideProc) - ! flag if there is at least one surf side on the node (sides in halo region do also count) SurfOnNode = MERGE(.TRUE.,.FALSE.,nComputeNodeSurfTotalSides.GT.0) @@ -608,6 +349,10 @@ SUBROUTINE InitParticleBoundarySampling() ! compute area of sub-faces tmp1=dXiEQ_SurfSample/2.0 !(b-a)/2 +#if USE_MPI +SurfSide2GlobalSide => SurfSide2GlobalSide_Shared +#endif + DO iSide = firstSide,LastSide ! get global SideID. This contains only nonUniqueSide, no special mortar treatment required SideID = SurfSide2GlobalSide(SURF_SIDEID,iSide) @@ -1195,18 +940,7 @@ SUBROUTINE FinalizeParticleBoundarySampling() !=================================================================================================================================== ! Return if nothing was allocated -IF (.NOT.WriteMacroSurfaceValues.AND..NOT.DSMC%CalcSurfaceVal.AND..NOT.(ANY(PartBound%Reactive)).AND..NOT.(nPorousBC.GT.0).AND..NOT.PartBound%UseRotPeriodicBC) RETURN - -! First, free every shared memory window. This requires MPI_BARRIER as per MPI3.1 specification -#if USE_MPI -! Mapping arrays are allocated even if the node does not have sampling surfaces -CALL MPI_BARRIER(MPI_COMM_SHARED,iERROR) -CALL UNLOCK_AND_FREE(GlobalSide2SurfSide_Shared_Win) -CALL UNLOCK_AND_FREE(SurfSide2GlobalSide_Shared_Win) -CALL MPI_BARRIER(MPI_COMM_SHARED,iERROR) -ADEALLOCATE(GlobalSide2SurfSide_Shared) -ADEALLOCATE(SurfSide2GlobalSide_Shared) -#endif /*USE_MPI*/ +IF (.NOT.WriteMacroSurfaceValues.AND..NOT.DSMC%CalcSurfaceVal.AND..NOT.(ANY(PartBound%Reactive))) RETURN ! Return if no sampling surfaces on node IF (.NOT.SurfOnNode) RETURN @@ -1253,8 +987,6 @@ SUBROUTINE FinalizeParticleBoundarySampling() SDEALLOCATE(SampWallImpactAngle) SDEALLOCATE(SampWallImpactNumber) ADEALLOCATE(SurfSideArea) -ADEALLOCATE(GlobalSide2SurfSide) -ADEALLOCATE(SurfSide2GlobalSide) SDEALLOCATE(MacroSurfaceVal) SDEALLOCATE(MacroSurfaceSpecVal) diff --git a/src/particles/emission/particle_emission_init.f90 b/src/particles/emission/particle_emission_init.f90 index d313c071a..c45b5ca21 100644 --- a/src/particles/emission/particle_emission_init.f90 +++ b/src/particles/emission/particle_emission_init.f90 @@ -148,6 +148,7 @@ SUBROUTINE InitializeVariablesSpeciesInits() USE MOD_LoadBalance_Vars ,ONLY: PerformLoadBalance #endif /*USE_MPI*/ USE MOD_Restart_Vars ,ONLY: DoRestart +USE MOD_Analyze_Vars ,ONLY: DoSurfModelAnalyze ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !----------------------------------------------------------------------------------------------------------------------------------- @@ -328,6 +329,7 @@ SUBROUTINE InitializeVariablesSpeciesInits() NeutralizationSource = TRIM(GETSTR('Part-Species'//TRIM(hilf2)//'-NeutralizationSource')) NeutralizationBalance = 0 UseNeutralization = .TRUE. + DoSurfModelAnalyze = .TRUE. IF((TRIM(Species(iSpec)%Init(iInit)%SpaceIC).EQ.'3D_Liu2010_neutralization').OR.& (TRIM(Species(iSpec)%Init(iInit)%SpaceIC).EQ.'3D_Liu2010_neutralization_Szabo'))THEN Species(iSpec)%Init(iInit)%FirstQuadrantOnly = GETLOGICAL('Part-Species'//TRIM(hilf2)//'-FirstQuadrantOnly') diff --git a/src/particles/particle_init.f90 b/src/particles/particle_init.f90 index 0f93b4c62..9d0ce0df7 100644 --- a/src/particles/particle_init.f90 +++ b/src/particles/particle_init.f90 @@ -273,6 +273,7 @@ SUBROUTINE InitParticles() USE MOD_SurfaceModel_Init ,ONLY: InitSurfaceModel USE MOD_Particle_Surfaces ,ONLY: InitParticleSurfaces USE MOD_Particle_Sampling_Adapt ,ONLY: InitAdaptiveBCSampling +USE MOD_Particle_Boundary_Init ,ONLY: InitParticleBoundarySurfSides USE MOD_Particle_Boundary_Init ,ONLY: InitRotPeriodicMapping, InitAdaptiveWallTemp, InitRotPeriodicInterPlaneMapping USE MOD_DSMC_BGGas ,ONLY: BGGas_InitRegions #if USE_MPI @@ -336,16 +337,19 @@ SUBROUTINE InitParticles() DSMC_Solution = 0.0 END IF -! Initialize surface sampling / rotational periodic mapping -! (the following IF arguments have to be considered in FinalizeParticleBoundarySampling as well) -IF (WriteMacroSurfaceValues.OR.DSMC%CalcSurfaceVal.OR.(ANY(PartBound%Reactive)).OR.(nPorousBC.GT.0).OR.PartBound%UseRotPeriodicBC) THEN +! Initialize the counters (nComputeNodeSurfSides,nComputeNodeSurfTotalSides,nComputeNodeSurfOutputSides) and +! mappings (GlobalSide2SurfSide,SurfSide2GlobalSide) of the particle boundary surface sides +CALL InitParticleBoundarySurfSides() +! Initialize rotational periodic mappings (requires InitParticleBoundarySurfSides) +IF(PartBound%UseRotPeriodicBC) CALL InitRotPeriodicMapping() +IF(PartBound%UseInterPlaneBC) CALL InitRotPeriodicInterPlaneMapping() +! Initialize surface sampling (the following IF arguments have to be considered in FinalizeParticleBoundarySampling as well) +IF (WriteMacroSurfaceValues.OR.DSMC%CalcSurfaceVal.OR.(ANY(PartBound%Reactive))) THEN CALL InitParticleBoundarySampling() - IF(PartBound%UseRotPeriodicBC) CALL InitRotPeriodicMapping() - IF(PartBound%UseInterPlaneBC) CALL InitRotPeriodicInterPlaneMapping() CALL InitAdaptiveWallTemp() END IF -! Initialize porous boundary condition (requires BCdata_auxSF and SurfMesh from InitParticleBoundarySampling) +! Initialize porous boundary condition (requires BCdata_auxSF and InitParticleBoundarySurfSides) IF(nPorousBC.GT.0) CALL InitPorousBoundaryCondition() ! Allocate sampling of near adaptive boundary element values From b92531d199bd7f0d2e863dc2ad023d9a5ed7c127 Mon Sep 17 00:00:00 2001 From: Paul Nizenkov Date: Fri, 25 Aug 2023 21:03:34 +0200 Subject: [PATCH 03/16] New SurfCOMM%UNICATOR between procs with a surface side --- docs/documentation/developerguide/mpi.md | 40 +++++++++-- .../command_line.ini | 2 +- .../parameter.ini | 8 ++- .../boundary/particle_boundary_init.f90 | 42 +++++++++--- .../boundary/particle_boundary_vars.f90 | 20 ++---- .../particle_mpi_boundary_sampling.f90 | 2 - src/particles/particle_operations.f90 | 4 +- .../surfacemodel/surfacemodel_analyze.f90 | 66 ++++++++----------- 8 files changed, 111 insertions(+), 73 deletions(-) diff --git a/docs/documentation/developerguide/mpi.md b/docs/documentation/developerguide/mpi.md index bd8143130..7ce6bf36e 100644 --- a/docs/documentation/developerguide/mpi.md +++ b/docs/documentation/developerguide/mpi.md @@ -2,12 +2,7 @@ This chapter describes how PICLas subroutines and functions are parallelized. - ## General Remarks: Things to consider -In case any new communicator (e.g. SurfCOMM%COMM) is built during init or anywhere else with -`CALL MPI_COMM_SPLIT(NEWCOMMUNICATOR,iERROR)` or such, it is necessary to finalize it with `CALL MPI_COMM_FREE(NEWCOMMUNICATOR,iERROR)`. - -Else, load balancing will produce undefined errors that are almost impossible to find. Debug MPI @@ -121,3 +116,38 @@ Additionally to conventional sides, mappings for the sides that belong to a boun PEM%GlobalElemID(iPart) ! Global element ID PEM%CNElemID(iPart) ! Compute-node local element ID (GlobalElem2CNTotalElem(PEM%GlobalElemID(iPart))) PEM%LocalElemID(iPart) ! Core local element ID (PEM%GlobalElemID(iPart) - offsetElem) + +## Custom communicators + +To limit the number of communicating processors, feature specific communicators can be built. In the following, an example is given +for a communicator, which only contains processors with a local surface side (part of the `InitParticleBoundarySurfSides` routine). First, a global variable `SurfCOMM`, which is based on the `tMPIGROUP` type, is required: +``` +TYPE tMPIGROUP + INTEGER :: UNICATOR !< MPI communicator for surface sides + INTEGER :: nProcs !< number of MPI processes + INTEGER :: MyRank !< MyRank, ordered from 0 to nProcs - 1 +END TYPE +TYPE (tMPIGROUP) :: SurfCOMM +``` +To create a subset of processors, a condition is required, which is defined by the `color` variable: +``` +color = MERGE(1337, MPI_UNDEFINED, nSurfSidesProc.GT.0) +``` +Here, every processor with the same `color` will be part of the same communicator. The condition `nSurfSidesProc.GT.0` in this case includes every processor with a surface side. Every other processor will be set to `MPI_UNDEFINED` and consequently be part of `MPI_COMM_NULL`. Now, the communicator itself can be created: +``` +CALL MPI_COMM_SPLIT(MPI_COMM_WORLD, color, MPI_INFO_NULL, SurfCOMM%UNICATOR, iError) +``` +`MPI_COMM_WORLD` denotes the original communicator containing every processor (but can also be a previously created subset) and the `MPI_INFO_NULL` entry denotes the rank assignment within the new communicator (default: numbering from 0 to nProcs - 1). Additional information can be stored within the created variable: +``` +IF(SurfCOMM%UNICATOR.NE.MPI_COMM_NULL) THEN + ! Stores the rank within the given communicator as MyRank + CALL MPI_COMM_RANK(SurfCOMM%UNICATOR, SurfCOMM%MyRank, iError) + ! Stores the total number of processors of the given communicator as nProcs + CALL MPI_COMM_SIZE(SurfCOMM%UNICATOR, SurfCOMM%nProcs, iError) +END IF +``` +Through the IF clause, only processors that are part of the communicator can be addressed. And finally, it is important to free the communicator during the finalization routine: +``` +IF(SurfCOMM%UNICATOR.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(SurfCOMM%UNICATOR,iERROR) +``` +Otherwise load balancing might produce undefined errors that are very difficult to find. \ No newline at end of file diff --git a/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/command_line.ini b/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/command_line.ini index fb34fa7f6..82716c168 100644 --- a/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/command_line.ini +++ b/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/command_line.ini @@ -1 +1 @@ -MPI=1,2,4 +MPI=1,4,20 diff --git a/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/parameter.ini b/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/parameter.ini index d04cc4703..049bc641f 100644 --- a/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/parameter.ini +++ b/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/parameter.ini @@ -14,7 +14,9 @@ NAnalyze = 1 ! Number of analyze points MeshFile = channel_mesh.h5 useCurveds = F ! if boundaries have to be changed (else they are used from Mesh directly): -TrackingMethod = triatracking +TrackingMethod = triatracking +CalcMeshInfo = T +CalcHaloInfo = T ! =============================================================================== ! ! OUTPUT / VISUALIZATION ! =============================================================================== ! @@ -57,6 +59,7 @@ BoundaryType = (/10,0/) ! =============================================================================== ! Part-maxParticleNumber=500000 Part-nSpecies=1 + Part-nBounds=6 Part-Boundary1-SourceName = BC_Xplus Part-Boundary1-Condition = reflective @@ -65,8 +68,8 @@ Part-Boundary1-SpeciesSwaps1 = (/1,0/) Part-Boundary2-SourceName = BC_Xminus Part-Boundary2-Condition = reflective - Part-Boundary2-WallTemp = 2700, 2635.24 + Part-Boundary3-SourceName = BC_Yplus Part-Boundary3-Condition = symmetric Part-Boundary4-SourceName = BC_Yminus @@ -109,3 +112,4 @@ Particles-HaloEpsVelo=2.0E+07 Part-NumberOfRandomSeeds=2 Particles-RandomSeed1=1 Particles-RandomSeed2=2 +NVisu=1 diff --git a/src/particles/boundary/particle_boundary_init.f90 b/src/particles/boundary/particle_boundary_init.f90 index ef0fab172..8b4f37cce 100644 --- a/src/particles/boundary/particle_boundary_init.f90 +++ b/src/particles/boundary/particle_boundary_init.f90 @@ -589,6 +589,7 @@ SUBROUTINE InitParticleBoundarySurfSides() USE MOD_Particle_Boundary_Vars ,ONLY: GlobalSide2SurfSide_Shared,GlobalSide2SurfSide_Shared_Win USE MOD_Particle_Boundary_Vars ,ONLY: SurfSide2GlobalSide_Shared,SurfSide2GlobalSide_Shared_Win USE MOD_Particle_Boundary_Vars ,ONLY: nComputeNodeInnerBCs +USE MOD_Particle_Boundary_Vars ,ONLY: SurfCOMM #else USE MOD_Particle_Mesh_Vars ,ONLY: nComputeNodeSides #endif /*USE_MPI*/ @@ -612,6 +613,7 @@ SUBROUTINE InitParticleBoundarySurfSides() INTEGER :: GlobalElemID,GlobalElemRank INTEGER :: sendbuf,recvbuf INTEGER :: NbGlobalElemID, NbElemRank, NbLeaderID, nSurfSidesTmp +INTEGER :: color #endif /*USE_MPI*/ INTEGER :: NbGlobalSideID !=================================================================================================================================== @@ -672,10 +674,10 @@ SUBROUTINE InitParticleBoundarySurfSides() ! check if element for this side is on the current compute-node ! IF ((SideInfo_Shared(SIDE_ID,iSide).GT.ElemInfo_Shared(ELEM_FIRSTSIDEIND,offsetComputeNodeElem+1)) .AND. & ! (SideInfo_Shared(SIDE_ID,iSide).LE.ElemInfo_Shared(ELEM_LASTSIDEIND ,offsetComputeNodeElem+nComputeNodeElems))) THEN -! IF ((iSide.GE.(ElemInfo_Shared(ELEM_FIRSTSIDEIND,offsetComputeNodeElem+1)+1)) .AND. & -! (iSide.LE.ElemInfo_Shared(ELEM_LASTSIDEIND ,offsetComputeNodeElem+nComputeNodeElems))) THEN -! nComputeNodeSurfSides = nComputeNodeSurfSides + 1 -! END IF + ! IF ((iSide.GE.(ElemInfo_Shared(ELEM_FIRSTSIDEIND,offsetComputeNodeElem+1)+1)) .AND. & + ! (iSide.LE.ElemInfo_Shared(ELEM_LASTSIDEIND ,offsetComputeNodeElem+nComputeNodeElems))) THEN + ! nComputeNodeSurfSides = nComputeNodeSurfSides + 1 + ! END IF ! TODO: Add another check to determine the surface side in halo_eps from current proc. Node-wide halo can become quite large with ! with 128 procs! @@ -693,7 +695,7 @@ SUBROUTINE InitParticleBoundarySurfSides() ELSE ! find the compute node GlobalSide2SurfSideProc(SURF_LEADER,iSide) = INT(GlobalElemRank/nComputeNodeProcessors) - END IF + END IF #else GlobalSide2SurfSideProc(SURF_RANK ,iSide) = 0 GlobalSide2SurfSideProc(SURF_LEADER,iSide) = GlobalSide2SurfSideProc(SURF_RANK,iSide) @@ -701,11 +703,7 @@ SUBROUTINE InitParticleBoundarySurfSides() #if USE_MPI ! check if element for this side is on the current compute-node. Alternative version to the check above - IF (GlobalSide2SurfSideProc(SURF_LEADER,iSide).EQ.myLeaderGroupRank) THEN -#endif /*USE_MPI*/ - nComputeNodeSurfSides = nComputeNodeSurfSides + 1 -#if USE_MPI - END IF + IF (GlobalSide2SurfSideProc(SURF_LEADER,iSide).EQ.myLeaderGroupRank) nComputeNodeSurfSides = nComputeNodeSurfSides + 1 #endif /*USE_MPI*/ END IF ! reflective side END DO @@ -760,6 +758,7 @@ SUBROUTINE InitParticleBoundarySurfSides() END DO #else offsetSurfTotalSidesProc = 0 +nComputeNodeSurfSides = nSurfSidesProc nComputeNodeSurfTotalSides = nSurfSidesProc GlobalSide2SurfSide(:,firstSide:lastSide) = GlobalSide2SurfSideProc(:,firstSide:lastSide) #endif /*USE_MPI*/ @@ -854,6 +853,28 @@ SUBROUTINE InitParticleBoundarySurfSides() ! free temporary arrays DEALLOCATE(GlobalSide2SurfSideProc) +! create a communicator between processors with a surface side (including sides in the halo region) +#if USE_MPI +! Set the control of subset assignment (nonnegative integer). Processes with the same color are in the same new communicator. +! Make sure to include the root +IF(MPIRoot) THEN + color = 1337 +ELSE + color = MERGE(1337, MPI_UNDEFINED, nSurfSidesProc.GT.0) +END IF +! Create new surface communicator. Pass MPI_INFO_NULL as rank to follow the original ordering +CALL MPI_COMM_SPLIT(MPI_COMM_WORLD, color, MPI_INFO_NULL, SurfCOMM%UNICATOR, iError) +! Find my rank on the shared communicator, comm size and proc name +IF(SurfCOMM%UNICATOR.NE.MPI_COMM_NULL)THEN + CALL MPI_COMM_RANK(SurfCOMM%UNICATOR, SurfCOMM%MyRank, iError) + CALL MPI_COMM_SIZE(SurfCOMM%UNICATOR, SurfCOMM%nProcs, iError) + ! inform about size of emission communicator + LBWRITE(UNIT_StdOut,'(A,I0,A)') ' Surface sides: Communicator on ', SurfCOMM%nProcs,' procs' +END IF ! nSurfSidesProc.GT.0 +#endif /*USE_MPI*/ + +LBWRITE(UNIT_stdOut,'(A)') ' INIT SURFACE SIDES DONE!' + END SUBROUTINE InitParticleBoundarySurfSides @@ -2052,6 +2073,7 @@ SUBROUTINE FinalizeParticleBoundary() ! Mapping arrays are allocated even if the node does not have sampling surfaces #if USE_MPI +IF(SurfCOMM%UNICATOR.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(SurfCOMM%UNICATOR,iERROR) CALL MPI_BARRIER(MPI_COMM_SHARED,iERROR) CALL UNLOCK_AND_FREE(GlobalSide2SurfSide_Shared_Win) CALL UNLOCK_AND_FREE(SurfSide2GlobalSide_Shared_Win) diff --git a/src/particles/boundary/particle_boundary_vars.f90 b/src/particles/boundary/particle_boundary_vars.f90 index d799fb292..5ad22e8c3 100644 --- a/src/particles/boundary/particle_boundary_vars.f90 +++ b/src/particles/boundary/particle_boundary_vars.f90 @@ -180,22 +180,14 @@ MODULE MOD_Particle_Boundary_Vars END TYPE #endif /*USE_MPI*/ -TYPE tSurfaceCOMM - LOGICAL :: MPIRoot ! if root of mpi communicator - INTEGER :: MyRank ! local rank in new group - INTEGER :: nProcs ! number of processes - LOGICAL :: MPIOutputRoot ! if root of mpi communicator - INTEGER :: MyOutputRank ! local rank in new group - INTEGER :: nOutputProcs ! number of output processes #if USE_MPI - LOGICAL :: InnerBCs ! are there InnerSides with reflective properties - INTEGER :: COMM=MPI_COMM_NULL ! communicator - INTEGER :: nMPINeighbors ! number of processes to communicate with - TYPE(tSurfaceSendList),ALLOCATABLE :: MPINeighbor(:) ! list containing all mpi neighbors - INTEGER :: OutputCOMM=MPI_COMM_NULL ! communicator for output -#endif /*USE_MPI*/ +TYPE tMPIGROUP + INTEGER :: UNICATOR !< MPI communicator for surface sides (including sides inside the halo region) + INTEGER :: nProcs !< number of MPI processes for particles + INTEGER :: MyRank !< MyRank of PartMPIVAR%COMM END TYPE -TYPE (tSurfaceCOMM) :: SurfCOMM +TYPE (tMPIGROUP) :: SurfCOMM +#endif /*USE_MPI*/ TYPE tSurfaceMesh INTEGER :: SampSize ! integer of sampsize diff --git a/src/particles/particle_mpi/particle_mpi_boundary_sampling.f90 b/src/particles/particle_mpi/particle_mpi_boundary_sampling.f90 index 151588f88..4d38fb90b 100644 --- a/src/particles/particle_mpi/particle_mpi_boundary_sampling.f90 +++ b/src/particles/particle_mpi/particle_mpi_boundary_sampling.f90 @@ -418,8 +418,6 @@ SUBROUTINE ExchangeSurfData() INTEGER :: MessageSize,iSurfSide,SurfSideID INTEGER :: nValues INTEGER :: RecvRequest(0:nSurfLeaders-1),SendRequest(0:nSurfLeaders-1) -!INTEGER :: iPos,p,q,iProc,iReact -!INTEGER :: recv_status_list(1:MPI_STATUS_SIZE,1:SurfCOMM%nMPINeighbors) !=================================================================================================================================== ! nodes without sampling surfaces do not take part in this routine IF (.NOT.SurfOnNode) RETURN diff --git a/src/particles/particle_operations.f90 b/src/particles/particle_operations.f90 index e30c4cfb0..a077f8465 100644 --- a/src/particles/particle_operations.f90 +++ b/src/particles/particle_operations.f90 @@ -194,8 +194,8 @@ SUBROUTINE RemoveParticle(PartID,BCID,alpha,crossedBC) PartEkinOut(iSpec)=PartEkinOut(iSpec)+CalcEkinPart(PartID) END IF ELSE - nPartOut(iSpec)=nPartOut(iSpec) + 1 - PartEkinOut(iSpec)=PartEkinOut(iSpec)+CalcEkinPart(PartID) + nPartOut(iSpec)=nPartOut(iSpec) + 1 + PartEkinOut(iSpec)=PartEkinOut(iSpec)+CalcEkinPart(PartID) END IF END IF ! CalcPartBalance diff --git a/src/particles/surfacemodel/surfacemodel_analyze.f90 b/src/particles/surfacemodel/surfacemodel_analyze.f90 index bb78bb4b6..df1615da8 100644 --- a/src/particles/surfacemodel/surfacemodel_analyze.f90 +++ b/src/particles/surfacemodel/surfacemodel_analyze.f90 @@ -150,9 +150,7 @@ SUBROUTINE AnalyzeSurface(Time) USE MOD_SurfaceModel_Analyze_Vars USE MOD_Restart_Vars ,ONLY: DoRestart USE MOD_Particle_Boundary_Vars ,ONLY: nComputeNodeSurfSides,PartBound -#if USE_MPI USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ USE MOD_SurfaceModel_Vars ,ONLY: nPorousBC, PorousBC USE MOD_Particle_Vars ,ONLY: nSpecies,UseNeutralization,NeutralizationBalanceGlobal,Species #if USE_HDG @@ -160,6 +158,7 @@ SUBROUTINE AnalyzeSurface(Time) USE MOD_Analyze_Vars ,ONLY: CalcElectricTimeDerivative USE MOD_HDG_Vars ,ONLY: UseBiasVoltage,BiasVoltage,BVDataLength #if USE_MPI +USE MOD_Particle_Boundary_Vars ,ONLY: SurfCOMM USE MOD_HDG ,ONLY: SynchronizeBV #endif /*USE_MPI*/ #endif /*USE_HDG*/ @@ -182,18 +181,17 @@ SUBROUTINE AnalyzeSurface(Time) INTEGER :: iEDCBC,i,iBoundary,iPartBound2 #endif /*USE_HDG*/ !=================================================================================================================================== -IF((.NOT.CalcBoundaryParticleOutput).AND.(.NOT.UseNeutralization).AND.(.NOT.CalcElectronSEE)) RETURN IF(.NOT.DoSurfModelAnalyze) RETURN -! Currently the MPI routines below use Part%COMM (which contains all procs) and if one process returns here a hang-up occurs -IF(nComputeNodeSurfSides.EQ.0) CALL abort(__STAMP__,'Not all processes have surfaces for analysis but the communicator requires this.') +! Only proceed with processors, which have a surface side (as determined in InitParticleBoundarySurfSides) +#if USE_MPI +IF(SurfCOMM%UNICATOR.EQ.MPI_COMM_NULL) RETURN +#endif /*USE_MPI*/ SurfModelAnalyzeSampleTime = Time - SurfModelAnalyzeSampleTime ! Set SurfModelAnalyzeSampleTime=Time at the end of this routine OutputCounter = 2 unit_index = 636 -#if USE_MPI IF(PartMPI%MPIRoot)THEN -#endif /*USE_MPI*/ INQUIRE(UNIT = unit_index , OPENED = isOpen) IF(.NOT.isOpen)THEN outfile = 'SurfaceAnalyze.csv' @@ -273,17 +271,17 @@ SUBROUTINE AnalyzeSurface(Time) WRITE(unit_index,'(A)') '' END IF END IF -#if USE_MPI END IF -#endif /*USE_MPI*/ !=================================================================================================================================== ! Analyze Routines !=================================================================================================================================== IF (CalcSurfCollCounter) CALL GetCollCounter(SurfCollNum,AdsorptionNum,DesorptionNum) IF (CalcPorousBCInfo) CALL GetPorousBCInfo() +#if USE_MPI IF (CalcBoundaryParticleOutput) CALL SyncBoundaryParticleOutput() IF (CalcElectronSEE) CALL SyncElectronSEE() +#endif /*USE_MPI*/ !=================================================================================================================================== ! Output Analyzed variables !=================================================================================================================================== @@ -595,6 +593,7 @@ SUBROUTINE GetCollCounter(SurfCollNum,AdsorbNum, DesorbNum) USE MOD_SurfaceModel_Analyze_Vars ,ONLY: SurfAnalyzeCount, SurfAnalyzeNumOfAds, SurfAnalyzeNumOfDes #if USE_MPI USE MOD_Particle_MPI_Vars ,ONLY: PartMPI +USE MOD_Particle_Boundary_Vars ,ONLY: SurfCOMM #endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE @@ -616,13 +615,13 @@ SUBROUTINE GetCollCounter(SurfCollNum,AdsorbNum, DesorbNum) #if USE_MPI IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,SurfCollNum ,nSpecies,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM,IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE,AdsorbNum ,nSpecies,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM,IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE,DesorbNum ,nSpecies,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM,IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,SurfCollNum ,nSpecies,MPI_INTEGER,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,AdsorbNum ,nSpecies,MPI_INTEGER,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,DesorbNum ,nSpecies,MPI_INTEGER,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) ELSE - CALL MPI_REDUCE(SurfCollNum ,SurfCollNum ,nSpecies,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM,IERROR) - CALL MPI_REDUCE(AdsorbNum ,AdsorbNum ,nSpecies,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM,IERROR) - CALL MPI_REDUCE(DesorbNum ,DesorbNum ,nSpecies,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM,IERROR) + CALL MPI_REDUCE(SurfCollNum ,SurfCollNum ,nSpecies,MPI_INTEGER,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) + CALL MPI_REDUCE(AdsorbNum ,AdsorbNum ,nSpecies,MPI_INTEGER,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) + CALL MPI_REDUCE(DesorbNum ,DesorbNum ,nSpecies,MPI_INTEGER,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) END IF #endif /*USE_MPI*/ @@ -642,8 +641,9 @@ SUBROUTINE GetPorousBCInfo() USE MOD_Globals USE MOD_SurfaceModel_Vars ,ONLY: nPorousBC USE MOD_SurfaceModel_Analyze_Vars ,ONLY: PorousBCOutput -#if USE_MPI USE MOD_Particle_MPI_Vars ,ONLY: PartMPI +#if USE_MPI +USE MOD_Particle_Boundary_Vars ,ONLY: SurfCOMM #endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE @@ -657,15 +657,13 @@ SUBROUTINE GetPorousBCInfo() !=================================================================================================================================== #if USE_MPI IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE , PorousBCOutput, 5*nPorousBC, MPI_DOUBLE_PRECISION, MPI_SUM, 0, PartMPI%COMM, iError) + CALL MPI_REDUCE(MPI_IN_PLACE , PorousBCOutput, 5*nPorousBC, MPI_DOUBLE_PRECISION, MPI_SUM, 0, SurfCOMM%UNICATOR, iError) ELSE - CALL MPI_REDUCE(PorousBCOutput, PorousBCOutput, 5*nPorousBC, MPI_DOUBLE_PRECISION, MPI_SUM, 0, PartMPI%COMM, iError) + CALL MPI_REDUCE(PorousBCOutput, PorousBCOutput, 5*nPorousBC, MPI_DOUBLE_PRECISION, MPI_SUM, 0, SurfCOMM%UNICATOR, iError) END IF #endif /*USE_MPI*/ -#if USE_MPI IF(PartMPI%MPIRoot)THEN -#endif /*USE_MPI*/ DO iPBC = 1, nPorousBC IF(PorousBCOutput(1,iPBC).GT.0.0)THEN ! Pumping Speed (Output(2)) is the sum of all elements (counter over particles exiting through pump) @@ -673,9 +671,7 @@ SUBROUTINE GetPorousBCInfo() PorousBCOutput(3:5,iPBC) = PorousBCOutput(3:5,iPBC) / PorousBCOutput(1,iPBC) END IF END DO -#if USE_MPI END IF -#endif /*USE_MPI*/ END SUBROUTINE GetPorousBCInfo @@ -683,13 +679,13 @@ END SUBROUTINE GetPorousBCInfo !=================================================================================================================================== !> Synchronize BoundaryParticleOutput analyze arrays !=================================================================================================================================== +#if USE_MPI SUBROUTINE SyncBoundaryParticleOutput() ! MODULES -#if USE_MPI USE MOD_Globals USE MOD_SurfaceModel_Analyze_Vars ,ONLY: BPO USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ +USE MOD_Particle_Boundary_Vars ,ONLY: SurfCOMM ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !----------------------------------------------------------------------------------------------------------------------------------- @@ -698,41 +694,38 @@ SUBROUTINE SyncBoundaryParticleOutput() ! OUTPUT VARIABLES !----------------------------------------------------------------------------------------------------------------------------------- ! LOCAL VARIABLES -#if USE_MPI REAL :: SendBuf(1:BPO%NPartBoundaries*BPO%NSpecies) INTEGER :: SendBufSize -#endif /*USE_MPI*/ !=================================================================================================================================== -#if USE_MPI SendBufSize = BPO%NPartBoundaries*BPO%NSpecies IF(PartMPI%MPIRoot)THEN ! Map 2D array to vector for sending via MPI SendBuf = RESHAPE(BPO%RealPartOut,(/SendBufSize/)) - CALL MPI_REDUCE(MPI_IN_PLACE,SendBuf(1:SendBufSize),SendBufSize,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,SendBuf(1:SendBufSize),SendBufSize,MPI_DOUBLE_PRECISION,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) ! MAP vector back to 2D array BPO%RealPartOut = RESHAPE(SendBuf,(/BPO%NPartBoundaries,BPO%NSpecies/)) ELSE ! Map 2D array to vector for sending via MPI SendBuf = RESHAPE(BPO%RealPartOut,(/SendBufSize/)) - CALL MPI_REDUCE(SendBuf(1:SendBufSize),0,SendBufSize,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) - ! Reset non PartMPI%MPIRoot counters, PartMPI%MPIRoot counters are reset after writing the data to the file + CALL MPI_REDUCE(SendBuf(1:SendBufSize),0,SendBufSize,MPI_DOUBLE_PRECISION,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) + ! Reset non SurfCOMM%UNICATOR counters, SurfCOMM%UNICATOR counters are reset after writing the data to the file BPO%RealPartOut = 0. END IF -#endif /*USE_MPI*/ END SUBROUTINE SyncBoundaryParticleOutput +#endif /*USE_MPI*/ !=================================================================================================================================== !> Synchronize CalcElectronSEE analyze arrays !=================================================================================================================================== +#if USE_MPI SUBROUTINE SyncElectronSEE() ! MODULES -#if USE_MPI USE MOD_Globals USE MOD_SurfaceModel_Analyze_Vars ,ONLY: SEE USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ +USE MOD_Particle_Boundary_Vars ,ONLY: SurfCOMM ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !----------------------------------------------------------------------------------------------------------------------------------- @@ -742,17 +735,16 @@ SUBROUTINE SyncElectronSEE() !----------------------------------------------------------------------------------------------------------------------------------- ! LOCAL VARIABLES !=================================================================================================================================== -#if USE_MPI IF (PartMPI%MPIRoot) THEN - CALL MPI_REDUCE(MPI_IN_PLACE , SEE%RealElectronOut, SEE%NPartBoundaries,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE , SEE%RealElectronOut, SEE%NPartBoundaries,MPI_DOUBLE_PRECISION,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) ELSE - CALL MPI_REDUCE(SEE%RealElectronOut , 0 , SEE%NPartBoundaries,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) + CALL MPI_REDUCE(SEE%RealElectronOut , 0 , SEE%NPartBoundaries,MPI_DOUBLE_PRECISION,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) ! Reset non PartMPI%MPIRoot counters, PartMPI%MPIRoot counters are reset after writing the data to the file SEE%RealElectronOut = 0. END IF -#endif /*USE_MPI*/ END SUBROUTINE SyncElectronSEE +#endif /*USE_MPI*/ !=================================================================================================================================== From 0f9f46c2918559b762891a0f0b8993f7432f43b0 Mon Sep 17 00:00:00 2001 From: Paul Nizenkov Date: Fri, 25 Aug 2023 21:17:53 +0200 Subject: [PATCH 04/16] Fix of wrong precompiler flags --- src/particles/surfacemodel/surfacemodel_analyze.f90 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/particles/surfacemodel/surfacemodel_analyze.f90 b/src/particles/surfacemodel/surfacemodel_analyze.f90 index df1615da8..f6be49250 100644 --- a/src/particles/surfacemodel/surfacemodel_analyze.f90 +++ b/src/particles/surfacemodel/surfacemodel_analyze.f90 @@ -153,12 +153,14 @@ SUBROUTINE AnalyzeSurface(Time) USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_SurfaceModel_Vars ,ONLY: nPorousBC, PorousBC USE MOD_Particle_Vars ,ONLY: nSpecies,UseNeutralization,NeutralizationBalanceGlobal,Species +#if USE_MPI +USE MOD_Particle_Boundary_Vars ,ONLY: SurfCOMM +#endif /*USE_MPI*/ #if USE_HDG USE MOD_Analyze_Vars ,ONLY: EDC USE MOD_Analyze_Vars ,ONLY: CalcElectricTimeDerivative USE MOD_HDG_Vars ,ONLY: UseBiasVoltage,BiasVoltage,BVDataLength #if USE_MPI -USE MOD_Particle_Boundary_Vars ,ONLY: SurfCOMM USE MOD_HDG ,ONLY: SynchronizeBV #endif /*USE_MPI*/ #endif /*USE_HDG*/ From adf579e8b38b09df1711971f5d069796cd62dd41 Mon Sep 17 00:00:00 2001 From: Paul Nizenkov Date: Sat, 26 Aug 2023 18:37:15 +0200 Subject: [PATCH 05/16] Initialize communicators with MPI_COMM_NULL and free EDC, FPC, EPC, BiasVoltage and CPP communicators during finalize --- docs/documentation/developerguide/mpi.md | 34 +++++++++-- src/analyze/analyze.f90 | 5 ++ src/analyze/analyze_vars.f90 | 15 +++-- src/equations/poisson/equation.f90 | 2 +- src/hdg/hdg.f90 | 14 ++++- src/hdg/hdg_vars.f90 | 17 +++--- .../boundary/particle_boundary_condition.f90 | 8 --- .../boundary/particle_boundary_init.f90 | 4 -- .../boundary/particle_boundary_vars.f90 | 61 ++++--------------- .../particle_mpi/particle_mpi_vars.f90 | 6 +- .../surfacemodel/surfacemodel_analyze.f90 | 4 +- 11 files changed, 79 insertions(+), 91 deletions(-) diff --git a/docs/documentation/developerguide/mpi.md b/docs/documentation/developerguide/mpi.md index 7ce6bf36e..a901be3b2 100644 --- a/docs/documentation/developerguide/mpi.md +++ b/docs/documentation/developerguide/mpi.md @@ -123,9 +123,9 @@ To limit the number of communicating processors, feature specific communicators for a communicator, which only contains processors with a local surface side (part of the `InitParticleBoundarySurfSides` routine). First, a global variable `SurfCOMM`, which is based on the `tMPIGROUP` type, is required: ``` TYPE tMPIGROUP - INTEGER :: UNICATOR !< MPI communicator for surface sides - INTEGER :: nProcs !< number of MPI processes - INTEGER :: MyRank !< MyRank, ordered from 0 to nProcs - 1 + INTEGER :: UNICATOR=MPI_COMM_NULL !< MPI communicator for surface sides + INTEGER :: nProcs !< number of MPI processes + INTEGER :: MyRank !< MyRank, ordered from 0 to nProcs - 1 END TYPE TYPE (tMPIGROUP) :: SurfCOMM ``` @@ -150,4 +150,30 @@ Through the IF clause, only processors that are part of the communicator can be ``` IF(SurfCOMM%UNICATOR.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(SurfCOMM%UNICATOR,iERROR) ``` -Otherwise load balancing might produce undefined errors that are very difficult to find. \ No newline at end of file +This works for communicators, which have been initialized with MPI_COMM_NULL, either initially during the variable definition or during the split call. +If not initialized initially, you have to make sure that the freeing call is only performed, if the respective split routine has been called to guarantee +that either a communicator exists and/or every (other) processor has been set to MPI_COMM_NULL. + +### Available communicators + +| Handle | Description | Derived from | +| ----------------------- | --------------------------- | ----------------------- | +| MPI_COMM_WORLD | Default global communicator | - | +| MPI_COMM_SHARED | | MPI_COMM_WORLD | +| MPI_COMM_NODE | | MPI_COMM_WORLD | +| MPI_COMM_LEADERS | | MPI_COMM_WORLD | +| MPI_COMM_WORKERS | | MPI_COMM_WORLD | +| MPI_COMM_LEADERS_SHARED | | MPI_COMM_WORLD | +| MPI_COMM_LEADERS_SURF | | MPI_COMM_LEADERS_SHARED | + +#### Feature-specific + +| Handle | Description | Derived from | +| ------------------------------------ | ---------------------------------------------------------------------- | -------------- | +| PartMPI%InitGroup(nInitRegions)%COMM | Emission groups | PartMPI%COMM | +| SurfCOMM%UNICATOR | Processors with a surface side (e.g. reflective), including halo sides | MPI_COMM_WORLD | +| CPPCOMM%UNICATOR | Coupled power potential | MPI_COMM_WORLD | +| EDC%COMM(iEDCBC)%UNICATOR | Electric displacement current (per BC) | MPI_COMM_WORLD | +| FPC%COMM(iUniqueFPCBC)%UNICATOR | Floating potential (per BC) | MPI_COMM_WORLD | +| EPC%COMM(iUniqueEPCBC)%UNICATOR | Electric potential (per BC) | MPI_COMM_WORLD | +| BiasVoltage%COMM%UNICATOR | Bias voltage | MPI_COMM_WORLD | \ No newline at end of file diff --git a/src/analyze/analyze.f90 b/src/analyze/analyze.f90 index c82d7d771..02718e01c 100644 --- a/src/analyze/analyze.f90 +++ b/src/analyze/analyze.f90 @@ -825,6 +825,7 @@ SUBROUTINE FinalizeAnalyze() ! Finalizes variables necessary for analyse subroutines !=================================================================================================================================== ! MODULES +USE MOD_Globals #if PP_nVar>=6 USE MOD_Analyze_Vars ,ONLY: CalcPoyntingInt USE MOD_AnalyzeField ,ONLY: FinalizePoyntingInt @@ -844,6 +845,7 @@ SUBROUTINE FinalizeAnalyze() ! OUTPUT VARIABLES !---------------------------------------------------------------------------------------------------------------------------------- ! LOCAL VARIABLES +INTEGER :: iEDCBC !=================================================================================================================================== #if PP_nVar>=6 IF(CalcPoyntingInt) CALL FinalizePoyntingInt() @@ -856,6 +858,9 @@ SUBROUTINE FinalizeAnalyze() SDEALLOCATE(EDC%FieldBoundaries) SDEALLOCATE(EDC%BCIDToEDCBCID) #if USE_MPI + DO iEDCBC = 1, EDC%NBoundaries + IF(EDC%COMM(iEDCBC)%UNICATOR.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(EDC%COMM(iEDCBC)%UNICATOR,iERROR) + END DO SDEALLOCATE(EDC%COMM) #endif /*USE_MPI*/ END IF ! CalcElectricTimeDerivative diff --git a/src/analyze/analyze_vars.f90 b/src/analyze/analyze_vars.f90 index 26b8f403e..95a63f36b 100644 --- a/src/analyze/analyze_vars.f90 +++ b/src/analyze/analyze_vars.f90 @@ -15,6 +15,9 @@ !=================================================================================================================================== MODULE MOD_Analyze_Vars ! MODULES +#if USE_MPI +USE MOD_Globals +#endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE PUBLIC @@ -94,14 +97,10 @@ MODULE MOD_Analyze_Vars #if USE_MPI TYPE tMPIGROUP - INTEGER :: ID !< MPI communicator ID - INTEGER :: UNICATOR !< MPI communicator for electric displacement current - INTEGER :: Request !< MPI request for asynchronous communication - INTEGER :: nProcs !< number of MPI processes for particles - INTEGER :: MyRank !< MyRank of PartMPIVAR%COMM - LOGICAL :: MPIRoot !< Root, MPIRank=0 - INTEGER,ALLOCATABLE :: GroupToComm(:) !< list containing the rank in PartMPI%COMM - INTEGER,ALLOCATABLE :: CommToGroup(:) !< list containing the rank in PartMPI%COMM + INTEGER :: ID !< MPI communicator ID + INTEGER :: UNICATOR=MPI_COMM_NULL !< MPI communicator for electric displacement current + INTEGER :: nProcs !< number of MPI processes for particles + INTEGER :: MyRank !< MyRank within communicator END TYPE #endif /*USE_MPI*/ diff --git a/src/equations/poisson/equation.f90 b/src/equations/poisson/equation.f90 index d8c12a2aa..65339207a 100644 --- a/src/equations/poisson/equation.f90 +++ b/src/equations/poisson/equation.f90 @@ -377,7 +377,7 @@ SUBROUTINE InitCoupledPowerPotential() ! set communicator id CPPCOMM%ID = CPPBoundaries -! create new emission communicator for electric potential boundary condition communication. Pass MPI_INFO_NULL as rank to follow the original ordering +! create new emission communicator for coupled power potential communication. Pass MPI_INFO_NULL as rank to follow the original ordering CALL MPI_COMM_SPLIT(MPI_COMM_WORLD, color, MPI_INFO_NULL, CPPCOMM%UNICATOR, iError) ! Find my rank on the shared communicator, comm size and proc name diff --git a/src/hdg/hdg.f90 b/src/hdg/hdg.f90 index 92454b3f2..998f98825 100644 --- a/src/hdg/hdg.f90 +++ b/src/hdg/hdg.f90 @@ -3392,14 +3392,13 @@ END SUBROUTINE RestartHDG !=================================================================================================================================== SUBROUTINE FinalizeHDG() ! MODULES -USE MOD_globals +USE MOD_Globals USE MOD_HDG_Vars #if USE_PETSC USE petsc #endif #if USE_LOADBALANCE USE MOD_LoadBalance_Vars ,ONLY: PerformLoadBalance,UseH5IOLoadBalance -USE MOD_HDG_Vars ,ONLY: lambda, nGP_face USE MOD_Particle_Mesh_Vars ,ONLY: ElemInfo_Shared USE MOD_Mesh_Vars ,ONLY: nElems,offsetElem,nSides,SideToNonUniqueGlobalSide USE MOD_Mesh_Tools ,ONLY: LambdaSideToMaster,GetMasteriLocSides @@ -3416,6 +3415,9 @@ SUBROUTINE FinalizeHDG() INTEGER :: NonUniqueGlobalSideID INTEGER :: iSide #endif /*USE_LOADBALANCE*/ +#if USE_MPI +INTEGER :: iBC +#endif /*USE_MPI*/ !=================================================================================================================================== HDGInitIsDone = .FALSE. #if USE_PETSC @@ -3475,6 +3477,9 @@ SUBROUTINE FinalizeHDG() SDEALLOCATE(FPC%VoltageProc) SDEALLOCATE(FPC%ChargeProc) #if USE_MPI +DO iBC = 1, FPC%nUniqueFPCBounds + IF(FPC%COMM(iBC)%UNICATOR.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(FPC%COMM(iBC)%UNICATOR,iERROR) +END DO SDEALLOCATE(FPC%COMM) #endif /*USE_MPI*/ @@ -3498,7 +3503,12 @@ SUBROUTINE FinalizeHDG() SDEALLOCATE(EPC%VoltageProc) SDEALLOCATE(EPC%ChargeProc) #if USE_MPI +DO iBC = 1, EPC%nUniqueEPCBounds + IF(EPC%COMM(iBC)%UNICATOR.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(EPC%COMM(iBC)%UNICATOR,iERROR) +END DO SDEALLOCATE(EPC%COMM) +IF(BiasVoltage%COMM%UNICATOR.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(BiasVoltage%COMM%UNICATOR,iERROR) +IF(CPPCOMM%UNICATOR.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(CPPCOMM%UNICATOR,iERROR) #endif /*USE_MPI*/ #if USE_LOADBALANCE diff --git a/src/hdg/hdg_vars.f90 b/src/hdg/hdg_vars.f90 index 1c1d83384..b0385b5ea 100644 --- a/src/hdg/hdg_vars.f90 +++ b/src/hdg/hdg_vars.f90 @@ -19,6 +19,9 @@ !=================================================================================================================================== MODULE MOD_HDG_Vars ! MODULES +#if USE_MPI +USE MOD_Globals +#endif /*USE_MPI*/ #if USE_PETSC USE PETSc #endif @@ -161,15 +164,11 @@ MODULE MOD_HDG_Vars #if USE_MPI TYPE tMPIGROUP - INTEGER :: ID !< MPI communicator ID - INTEGER :: UNICATOR !< MPI communicator for floating boundary condition - INTEGER :: Request !< MPI request for asynchronous communication - INTEGER :: nProcs !< number of MPI processes part of the FPC group - INTEGER :: nProcsWithSides !< number of MPI processes part of the FPC group and actual FPC sides - INTEGER :: MyRank !< MyRank of PartMPIVAR%COMM - LOGICAL :: MPIRoot !< Root, MPIRank=0 - INTEGER,ALLOCATABLE :: GroupToComm(:) !< list containing the rank in PartMPI%COMM - INTEGER,ALLOCATABLE :: CommToGroup(:) !< list containing the rank in PartMPI%COMM + INTEGER :: ID !< MPI communicator ID + INTEGER :: UNICATOR=MPI_COMM_NULL !< MPI communicator for floating boundary condition + INTEGER :: nProcs !< number of MPI processes part of the FPC group + INTEGER :: nProcsWithSides !< number of MPI processes part of the FPC group and actual FPC sides + INTEGER :: MyRank !< MyRank within communicator END TYPE #endif /*USE_MPI*/ diff --git a/src/particles/boundary/particle_boundary_condition.f90 b/src/particles/boundary/particle_boundary_condition.f90 index 6cc95575b..4ecf777a3 100644 --- a/src/particles/boundary/particle_boundary_condition.f90 +++ b/src/particles/boundary/particle_boundary_condition.f90 @@ -153,14 +153,6 @@ SUBROUTINE GetBoundaryInteraction(iPart,SideID,flip,ElemID,crossedBC,TriNum,IsIn !----------------------------------------------------------------------------------------------------------------------------------- CALL PeriodicBoundary(iPart,SideID,ElemID) !----------------------------------------------------------------------------------------------------------------------------------- - CASE(4) ! PartBound%SimpleAnodeBC - !----------------------------------------------------------------------------------------------------------------------------------- - CALL abort(__STAMP__,' ERROR: PartBound not associated!. (PartBound%SimpleAnodeBC)') - !----------------------------------------------------------------------------------------------------------------------------------- - CASE(5) ! PartBound%SimpleCathodeBC - !----------------------------------------------------------------------------------------------------------------------------------- - CALL abort(__STAMP__,' ERROR: PartBound not associated!. (PartBound%SimpleCathodeBC)') - !----------------------------------------------------------------------------------------------------------------------------------- CASE(6) ! PartBound%RotPeriodicBC !----------------------------------------------------------------------------------------------------------------------------------- CALL RotPeriodicBoundary(iPart,SideID,ElemID) diff --git a/src/particles/boundary/particle_boundary_init.f90 b/src/particles/boundary/particle_boundary_init.f90 index 8b4f37cce..89a39c4dc 100644 --- a/src/particles/boundary/particle_boundary_init.f90 +++ b/src/particles/boundary/particle_boundary_init.f90 @@ -435,10 +435,6 @@ SUBROUTINE InitializeVariablesPartBoundary() CASE('periodic') PartBound%TargetBoundCond(iPartBound) = PartBound%PeriodicBC PartMeshHasPeriodicBCs = .TRUE. - CASE('simple_anode') - PartBound%TargetBoundCond(iPartBound) = PartBound%SimpleAnodeBC - CASE('simple_cathode') - PartBound%TargetBoundCond(iPartBound) = PartBound%SimpleCathodeBC CASE('symmetric') #if defined(IMPA) || defined(ROS) PartMeshHasReflectiveBCs=.TRUE. diff --git a/src/particles/boundary/particle_boundary_vars.f90 b/src/particles/boundary/particle_boundary_vars.f90 index 5ad22e8c3..84f2f04ca 100644 --- a/src/particles/boundary/particle_boundary_vars.f90 +++ b/src/particles/boundary/particle_boundary_vars.f90 @@ -17,6 +17,7 @@ MODULE MOD_Particle_Boundary_Vars !=================================================================================================================================== ! MODULES #if USE_MPI +USE MOD_Globals USE mpi #endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING @@ -155,62 +156,23 @@ MODULE MOD_Particle_Boundary_Vars REAL,ALLOCATABLE :: XiEQ_SurfSample(:) ! position of XiEQ_SurfSample REAL :: dXiEQ_SurfSample ! deltaXi in [-1,1] INTEGER :: OffSetSurfSide ! offset of local surf side -INTEGER :: OffSetInnerSurfSide ! offset of local inner surf side INTEGER :: nSurfBC ! number of surface side BCs CHARACTER(LEN=255),ALLOCATABLE :: SurfBCName(:) ! names of belonging surface BC #if USE_MPI -INTEGER,ALLOCATABLE :: OffSetSurfSideMPI(:) ! integer offset for particle boundary sampling -INTEGER,ALLOCATABLE :: OffSetInnerSurfSideMPI(:) ! integer offset for particle boundary sampling (innerBC) INTEGER :: nComputeNodeInnerBCs(2) ! Number of inner BCs with a larger global side ID on node #endif /*USE_MPI*/ -#if USE_MPI -TYPE tSurfaceSendList - INTEGER :: NativeProcID - INTEGER,ALLOCATABLE :: SendList(:) ! list containing surfsideid of sides to send to proc - INTEGER,ALLOCATABLE :: RecvList(:) ! list containing surfsideid of sides to recv from proc - - INTEGER,ALLOCATABLE :: SurfDistSendList(:) ! list containing surfsideid of sides to send to proc - INTEGER,ALLOCATABLE :: SurfDistRecvList(:) ! list containing surfsideid of sides to recv from proc - INTEGER,ALLOCATABLE :: H2OSendList(:) ! list containing surfsideid of sides to send to proc - INTEGER,ALLOCATABLE :: H2ORecvList(:) ! list containing surfsideid of sides to recv from proc - INTEGER,ALLOCATABLE :: O2HSendList(:) ! list containing surfsideid of sides to send to proc - INTEGER,ALLOCATABLE :: O2HRecvList(:) ! list containing surfsideid of sides to recv from proc - -END TYPE -#endif /*USE_MPI*/ - #if USE_MPI TYPE tMPIGROUP - INTEGER :: UNICATOR !< MPI communicator for surface sides (including sides inside the halo region) - INTEGER :: nProcs !< number of MPI processes for particles - INTEGER :: MyRank !< MyRank of PartMPIVAR%COMM + INTEGER :: UNICATOR=MPI_COMM_NULL !< MPI communicator for surface sides (including sides inside the halo region) + INTEGER :: nProcs !< number of MPI processes for particles + INTEGER :: MyRank !< MyRank within communicator END TYPE TYPE (tMPIGROUP) :: SurfCOMM #endif /*USE_MPI*/ - -TYPE tSurfaceMesh - INTEGER :: SampSize ! integer of sampsize - INTEGER :: ReactiveSampSize ! additional sample size on the surface due to use of - ! reactive surface modelling (reactions, liquid, etc.) - LOGICAL :: SurfOnProc ! flag if reflective boundary condition is on proc - INTEGER :: nSides ! Number of Sides on Surface (reflective) - INTEGER :: nBCSides ! Number of OuterSides with Surface (reflective) properties - INTEGER :: nInnerSides ! Number of InnerSides with Surface (reflective) properties - INTEGER :: nOutputSides ! Number of surfaces that are assigned to an MPI rank for - ! surface sampling (MacroSurfaceVal and MacroSurfaceSpecVal) - ! and output to .h5 (SurfData) purposes: - ! nOutputSides = bcsides + maser_innersides - !INTEGER :: nTotalSides ! Number of Sides on Surface incl. HALO sides - INTEGER :: nGlobalSides ! Global number of Sides on Surfaces (reflective) - INTEGER,ALLOCATABLE :: SideIDToSurfID(:) ! Mapping of side ID to surface side ID (reflective) - REAL, ALLOCATABLE :: SurfaceArea(:,:,:) ! Area of Surface - INTEGER,ALLOCATABLE :: SurfIDToSideID(:) ! Mapping of surface side ID (reflective) to side ID - INTEGER,ALLOCATABLE :: innerBCSideToHaloMap(:) ! map of inner BC ID on slave side to corresp. HaloSide -END TYPE - -TYPE (tSurfaceMesh) :: SurfMesh - +!----------------------------------------------------------------------------------------------------------------------------------- +! Porous BC +!----------------------------------------------------------------------------------------------------------------------------------- INTEGER :: nPorousSides ! Number of porous sides per compute node INTEGER,ALLOCPOINT :: MapSurfSideToPorousSide_Shared(:) ! Mapping of surface side to porous side INTEGER,ALLOCPOINT :: PorousBCInfo_Shared(:,:) ! Info and mappings for porous BCs [1:3,1:nPorousSides] @@ -236,13 +198,13 @@ MODULE MOD_Particle_Boundary_Vars ! REAL variable since the particle weight is used ! 1: Impinging particles ! 2: Deleted particles - +!----------------------------------------------------------------------------------------------------------------------------------- +! Particle Boundary +!----------------------------------------------------------------------------------------------------------------------------------- TYPE tPartBoundary INTEGER :: OpenBC = 1 ! = 1 (s.u.) Boundary Condition Integer Definition INTEGER :: ReflectiveBC = 2 ! = 2 (s.u.) Boundary Condition Integer Definition INTEGER :: PeriodicBC = 3 ! = 3 (s.u.) Boundary Condition Integer Definition - INTEGER :: SimpleAnodeBC = 4 ! = 4 (s.u.) Boundary Condition Integer Definition - INTEGER :: SimpleCathodeBC = 5 ! = 5 (s.u.) Boundary Condition Integer Definition INTEGER :: RotPeriodicBC = 6 ! = 6 (s.u.) Boundary Condition Integer Definition INTEGER :: RotPeriodicInterPlaneBC = 7 ! = 7 (s.u.) Boundary Condition Integer Definition INTEGER :: SymmetryBC = 10 ! = 10 (s.u.) Boundary Condition Integer Definition @@ -316,8 +278,9 @@ MODULE MOD_Particle_Boundary_Vars INTEGER :: nPartBound ! number of particle boundaries TYPE(tPartBoundary) :: PartBound ! Boundary Data for Particles - +!----------------------------------------------------------------------------------------------------------------------------------- ! Boundary particle output +!----------------------------------------------------------------------------------------------------------------------------------- LOGICAL :: DoBoundaryParticleOutputHDF5 ! Flag set automatically if particles crossing specific ! ! boundaries are to be saved to .h5 (position of intersection, ! ! velocity, species, internal energies) diff --git a/src/particles/particle_mpi/particle_mpi_vars.f90 b/src/particles/particle_mpi/particle_mpi_vars.f90 index 0d0603107..0804ad34d 100644 --- a/src/particles/particle_mpi/particle_mpi_vars.f90 +++ b/src/particles/particle_mpi/particle_mpi_vars.f90 @@ -17,7 +17,7 @@ MODULE MOD_Particle_MPI_Vars ! Contains global variables provided by the particle surfaces routines !=================================================================================================================================== ! MODULES -!USE mpi +USE MOD_Globals ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE PUBLIC @@ -35,7 +35,7 @@ MODULE MOD_Particle_MPI_Vars LOGICAL :: AbortExchangeProcs ! Terminate run if proc communication is non-symmetric TYPE tPartMPIGROUP - INTEGER :: COMM ! MPI communicator for PIC GTS region + INTEGER :: COMM=MPI_COMM_NULL ! MPI communicator for PIC GTS region INTEGER :: Request ! MPI request for asynchronous communication INTEGER :: nProcs ! number of MPI processes for particles INTEGER :: MyRank ! MyRank of PartMPIVAR%COMM @@ -63,7 +63,7 @@ MODULE MOD_Particle_MPI_Vars TYPE(tPartMPIConnect) , ALLOCATABLE :: DepoBGMConnect(:) ! MPI connect for each process #endif /*USE_MPI*/ TYPE(tPartMPIGROUP),ALLOCATABLE :: InitGroup(:) ! small communicator for initialization - INTEGER :: COMM ! MPI communicator for PIC GTS region + INTEGER :: COMM=MPI_COMM_NULL ! MPI communicator for PIC GTS region INTEGER :: nProcs ! number of MPI processes for particles INTEGER :: MyRank ! MyRank of PartMPIVAR%COMM LOGICAL :: MPIRoot ! Root, MPIRank=0 diff --git a/src/particles/surfacemodel/surfacemodel_analyze.f90 b/src/particles/surfacemodel/surfacemodel_analyze.f90 index f6be49250..8f3c4fea8 100644 --- a/src/particles/surfacemodel/surfacemodel_analyze.f90 +++ b/src/particles/surfacemodel/surfacemodel_analyze.f90 @@ -149,7 +149,7 @@ SUBROUTINE AnalyzeSurface(Time) USE MOD_Analyze_Vars ,ONLY: DoSurfModelAnalyze USE MOD_SurfaceModel_Analyze_Vars USE MOD_Restart_Vars ,ONLY: DoRestart -USE MOD_Particle_Boundary_Vars ,ONLY: nComputeNodeSurfSides,PartBound +USE MOD_Particle_Boundary_Vars ,ONLY: PartBound USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_SurfaceModel_Vars ,ONLY: nPorousBC, PorousBC USE MOD_Particle_Vars ,ONLY: nSpecies,UseNeutralization,NeutralizationBalanceGlobal,Species @@ -849,8 +849,6 @@ SUBROUTINE InitBoundaryParticleOutput() ' OpenBC = 1 \n'//& ' ReflectiveBC = 2 \n'//& ' PeriodicBC = 3 \n'//& - ' SimpleAnodeBC = 4 \n'//& - ' SimpleCathodeBC = 5 \n'//& ' RotPeriodicBC = 6 \n'//& ' SymmetryBC = 10 \n'//& ' SymmetryAxis = 11 ' From d886330a28839a7cdea114c96a35cd6459c9d73c Mon Sep 17 00:00:00 2001 From: Paul Nizenkov Date: Sun, 27 Aug 2023 15:04:44 +0200 Subject: [PATCH 06/16] Included open BCs in new communicator as well, bugfixes --- docs/documentation/developerguide/mpi.md | 18 ++++++------ src/hdg/hdg.f90 | 2 ++ src/io_hdf5/hdf5_output_state.f90 | 3 +- .../boundary/particle_boundary_init.f90 | 11 ++++--- src/particles/dsmc/dsmc_vars.f90 | 3 -- .../particle_mpi_boundary_sampling.f90 | 2 +- .../particle_mpi/particle_mpi_vars.f90 | 19 ++---------- .../surfacemodel/surfacemodel_analyze.f90 | 29 ++++++++----------- .../surfacemodel/surfacemodel_main.f90 | 2 +- 9 files changed, 37 insertions(+), 52 deletions(-) diff --git a/docs/documentation/developerguide/mpi.md b/docs/documentation/developerguide/mpi.md index a901be3b2..d8fed88c4 100644 --- a/docs/documentation/developerguide/mpi.md +++ b/docs/documentation/developerguide/mpi.md @@ -156,15 +156,15 @@ that either a communicator exists and/or every (other) processor has been set to ### Available communicators -| Handle | Description | Derived from | -| ----------------------- | --------------------------- | ----------------------- | -| MPI_COMM_WORLD | Default global communicator | - | -| MPI_COMM_SHARED | | MPI_COMM_WORLD | -| MPI_COMM_NODE | | MPI_COMM_WORLD | -| MPI_COMM_LEADERS | | MPI_COMM_WORLD | -| MPI_COMM_WORKERS | | MPI_COMM_WORLD | -| MPI_COMM_LEADERS_SHARED | | MPI_COMM_WORLD | -| MPI_COMM_LEADERS_SURF | | MPI_COMM_LEADERS_SHARED | +| Handle | Description | Derived from | +| ----------------------- | --------------------------------------------- | ----------------------- | +| MPI_COMM_WORLD | Default global communicator | - | +| MPI_COMM_NODE | Processors on a node | MPI_COMM_WORLD | +| MPI_COMM_LEADERS | Group of node leaders | MPI_COMM_WORLD | +| MPI_COMM_WORKERS | All remaining processors, who are not leaders | MPI_COMM_WORLD | +| MPI_COMM_SHARED | Processors on a node | MPI_COMM_WORLD | +| MPI_COMM_LEADERS_SHARED | Group of node leaders (myComputeNodeRank = 0) | MPI_COMM_WORLD | +| MPI_COMM_LEADERS_SURF | Node leaders with surface sides | MPI_COMM_LEADERS_SHARED | #### Feature-specific diff --git a/src/hdg/hdg.f90 b/src/hdg/hdg.f90 index 998f98825..bd1533612 100644 --- a/src/hdg/hdg.f90 +++ b/src/hdg/hdg.f90 @@ -3507,8 +3507,10 @@ SUBROUTINE FinalizeHDG() IF(EPC%COMM(iBC)%UNICATOR.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(EPC%COMM(iBC)%UNICATOR,iERROR) END DO SDEALLOCATE(EPC%COMM) +#if defined(PARTICLES) IF(BiasVoltage%COMM%UNICATOR.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(BiasVoltage%COMM%UNICATOR,iERROR) IF(CPPCOMM%UNICATOR.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(CPPCOMM%UNICATOR,iERROR) +#endif /*defined(PARTICLES)*/ #endif /*USE_MPI*/ #if USE_LOADBALANCE diff --git a/src/io_hdf5/hdf5_output_state.f90 b/src/io_hdf5/hdf5_output_state.f90 index 76a77c3d9..544b15a6a 100644 --- a/src/io_hdf5/hdf5_output_state.f90 +++ b/src/io_hdf5/hdf5_output_state.f90 @@ -170,8 +170,9 @@ SUBROUTINE WriteStateToHDF5(MeshFileName,OutputTime,PreviousTime) INTEGER :: SortedOffset,SortedStart,SortedEnd #ifdef PARTICLES INTEGER :: i,j,k,iElem +REAL,ALLOCATABLE :: BVDataHDF5(:,:) #endif /*PARTICLES*/ -REAL,ALLOCATABLE :: FPCDataHDF5(:,:),EPCDataHDF5(:,:),BVDataHDF5(:,:) +REAL,ALLOCATABLE :: FPCDataHDF5(:,:),EPCDataHDF5(:,:) INTEGER :: nVarFPC,nVarEPC #endif /*USE_HDG*/ !=================================================================================================================================== diff --git a/src/particles/boundary/particle_boundary_init.f90 b/src/particles/boundary/particle_boundary_init.f90 index 89a39c4dc..4abe6f977 100644 --- a/src/particles/boundary/particle_boundary_init.f90 +++ b/src/particles/boundary/particle_boundary_init.f90 @@ -601,7 +601,7 @@ SUBROUTINE InitParticleBoundarySurfSides() !----------------------------------------------------------------------------------------------------------------------------------- ! LOCAL VARIABLES INTEGER :: iSide,firstSide,lastSide,iSurfSide,GlobalSideID -INTEGER :: nSurfSidesProc +INTEGER :: nSurfSidesProc, nBCSidesProc INTEGER :: offsetSurfTotalSidesProc INTEGER,ALLOCATABLE :: GlobalSide2SurfSideProc(:,:) #if USE_MPI @@ -650,6 +650,7 @@ SUBROUTINE InitParticleBoundarySurfSides() GlobalSide2SurfSideProc = -1 nComputeNodeSurfSides = 0 +nBCSidesProc = 0 nSurfSidesProc = 0 ! check every BC side @@ -662,6 +663,8 @@ SUBROUTINE InitParticleBoundarySurfSides() IF (ElemInfo_Shared(ELEM_HALOFLAG,SideInfo_Shared(SIDE_ELEMID,iSide)).EQ.0) CYCLE #endif /*USE_MPI*/ + nBCSidesProc = nBCSidesProc + 1 + ! count number of reflective and rotationally periodic BC sides IF ((PartBound%TargetBoundCond(PartBound%MapToPartBC(SideInfo_Shared(SIDE_BCID,iSide))).EQ.PartBound%ReflectiveBC) .OR. & (PartBound%TargetBoundCond(PartBound%MapToPartBC(SideInfo_Shared(SIDE_BCID,iSide))).EQ.PartBound%RotPeriodicBC).OR. & @@ -849,14 +852,14 @@ SUBROUTINE InitParticleBoundarySurfSides() ! free temporary arrays DEALLOCATE(GlobalSide2SurfSideProc) -! create a communicator between processors with a surface side (including sides in the halo region) +! create a communicator between processors with a BC side (including open BCs and sides in the halo region) #if USE_MPI ! Set the control of subset assignment (nonnegative integer). Processes with the same color are in the same new communicator. ! Make sure to include the root IF(MPIRoot) THEN color = 1337 ELSE - color = MERGE(1337, MPI_UNDEFINED, nSurfSidesProc.GT.0) + color = MERGE(1337, MPI_UNDEFINED, nBCSidesProc.GT.0) END IF ! Create new surface communicator. Pass MPI_INFO_NULL as rank to follow the original ordering CALL MPI_COMM_SPLIT(MPI_COMM_WORLD, color, MPI_INFO_NULL, SurfCOMM%UNICATOR, iError) @@ -866,7 +869,7 @@ SUBROUTINE InitParticleBoundarySurfSides() CALL MPI_COMM_SIZE(SurfCOMM%UNICATOR, SurfCOMM%nProcs, iError) ! inform about size of emission communicator LBWRITE(UNIT_StdOut,'(A,I0,A)') ' Surface sides: Communicator on ', SurfCOMM%nProcs,' procs' -END IF ! nSurfSidesProc.GT.0 +END IF ! nBCSidesProc.GT.0 #endif /*USE_MPI*/ LBWRITE(UNIT_stdOut,'(A)') ' INIT SURFACE SIDES DONE!' diff --git a/src/particles/dsmc/dsmc_vars.f90 b/src/particles/dsmc/dsmc_vars.f90 index 3b1ae16ef..990d8bc30 100644 --- a/src/particles/dsmc/dsmc_vars.f90 +++ b/src/particles/dsmc/dsmc_vars.f90 @@ -15,9 +15,6 @@ MODULE MOD_DSMC_Vars ! Contains the DSMC variables !=================================================================================================================================== ! MODULES -#if USE_MPI -USE MOD_Particle_MPI_Vars, ONLY: tPartMPIConnect -#endif ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE PUBLIC diff --git a/src/particles/particle_mpi/particle_mpi_boundary_sampling.f90 b/src/particles/particle_mpi/particle_mpi_boundary_sampling.f90 index 4d38fb90b..9f6fe8ddd 100644 --- a/src/particles/particle_mpi/particle_mpi_boundary_sampling.f90 +++ b/src/particles/particle_mpi/particle_mpi_boundary_sampling.f90 @@ -208,7 +208,7 @@ SUBROUTINE InitSurfCommunication() !--- Split communicator from MPI_COMM_LEADER_SHARED color = MERGE(1201,MPI_UNDEFINED,SurfOnNode) -! create new SurfMesh communicator for SurfMesh communication. Pass MPI_INFO_NULL as rank to follow the original ordering +! create new communicator between node leaders with surfaces. Pass MPI_INFO_NULL as rank to follow the original ordering CALL MPI_COMM_SPLIT(MPI_COMM_LEADERS_SHARED, color, MPI_INFO_NULL, MPI_COMM_LEADERS_SURF, IERROR) ! Do not participate in remainder of communication if no surf sides on node diff --git a/src/particles/particle_mpi/particle_mpi_vars.f90 b/src/particles/particle_mpi/particle_mpi_vars.f90 index 0804ad34d..87ab59a48 100644 --- a/src/particles/particle_mpi/particle_mpi_vars.f90 +++ b/src/particles/particle_mpi/particle_mpi_vars.f90 @@ -35,7 +35,9 @@ MODULE MOD_Particle_MPI_Vars LOGICAL :: AbortExchangeProcs ! Terminate run if proc communication is non-symmetric TYPE tPartMPIGROUP +#if USE_MPI INTEGER :: COMM=MPI_COMM_NULL ! MPI communicator for PIC GTS region +#endif /*USE_MPI*/ INTEGER :: Request ! MPI request for asynchronous communication INTEGER :: nProcs ! number of MPI processes for particles INTEGER :: MyRank ! MyRank of PartMPIVAR%COMM @@ -44,26 +46,11 @@ MODULE MOD_Particle_MPI_Vars INTEGER,ALLOCATABLE :: CommToGroup(:) ! list containing the rank in PartMPI%COMM END TYPE -TYPE tPeriodicPtr - INTEGER , ALLOCATABLE :: BGMPeriodicBorder(:,:) ! indices of periodic border nodes -END TYPE - -#if USE_MPI -TYPE tPartMPIConnect - TYPE(tPeriodicPtr) , ALLOCATABLE :: Periodic(:) ! data for different periodic borders for process - LOGICAL :: isBGMNeighbor ! Flag: which process is neighber wrt. bckgrnd mesh - LOGICAL :: isBGMPeriodicNeighbor ! Flag: which process is neighber wrt. bckgrnd mesh - INTEGER , ALLOCATABLE :: BGMBorder(:,:) ! indices of border nodes (1=min 2=max,xyz) - INTEGER :: BGMPeriodicBorderCount ! Number(#) of overlapping areas due to periodic bc -END TYPE -#endif /*USE_MPI*/ - TYPE tPartMPIVAR #if USE_MPI - TYPE(tPartMPIConnect) , ALLOCATABLE :: DepoBGMConnect(:) ! MPI connect for each process + INTEGER :: COMM=MPI_COMM_NULL ! MPI communicator for PIC GTS region #endif /*USE_MPI*/ TYPE(tPartMPIGROUP),ALLOCATABLE :: InitGroup(:) ! small communicator for initialization - INTEGER :: COMM=MPI_COMM_NULL ! MPI communicator for PIC GTS region INTEGER :: nProcs ! number of MPI processes for particles INTEGER :: MyRank ! MyRank of PartMPIVAR%COMM LOGICAL :: MPIRoot ! Root, MPIRank=0 diff --git a/src/particles/surfacemodel/surfacemodel_analyze.f90 b/src/particles/surfacemodel/surfacemodel_analyze.f90 index 8f3c4fea8..3e6e20134 100644 --- a/src/particles/surfacemodel/surfacemodel_analyze.f90 +++ b/src/particles/surfacemodel/surfacemodel_analyze.f90 @@ -150,7 +150,6 @@ SUBROUTINE AnalyzeSurface(Time) USE MOD_SurfaceModel_Analyze_Vars USE MOD_Restart_Vars ,ONLY: DoRestart USE MOD_Particle_Boundary_Vars ,ONLY: PartBound -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_SurfaceModel_Vars ,ONLY: nPorousBC, PorousBC USE MOD_Particle_Vars ,ONLY: nSpecies,UseNeutralization,NeutralizationBalanceGlobal,Species #if USE_MPI @@ -193,7 +192,7 @@ SUBROUTINE AnalyzeSurface(Time) SurfModelAnalyzeSampleTime = Time - SurfModelAnalyzeSampleTime ! Set SurfModelAnalyzeSampleTime=Time at the end of this routine OutputCounter = 2 unit_index = 636 -IF(PartMPI%MPIRoot)THEN +IF(MPIRoot)THEN INQUIRE(UNIT = unit_index , OPENED = isOpen) IF(.NOT.isOpen)THEN outfile = 'SurfaceAnalyze.csv' @@ -288,7 +287,7 @@ SUBROUTINE AnalyzeSurface(Time) ! Output Analyzed variables !=================================================================================================================================== #if USE_MPI -IF(PartMPI%MPIRoot)THEN +IF(MPIRoot)THEN #endif /*USE_MPI*/ WRITE(unit_index,'(E23.16E3)',ADVANCE='NO') Time IF(CalcSurfCollCounter)THEN @@ -416,8 +415,8 @@ SUBROUTINE AnalyzeSurface(Time) ! Reset BPO containers DO iPartBound = 1, BPO%NPartBoundaries DO iSpec = 1, BPO%NSpecies - ! Reset PartMPI%MPIRoot counters after writing the data to the file, - ! non-PartMPI%MPIRoot are reset in SyncBoundaryParticleOutput() + ! Reset MPIRoot counters after writing the data to the file, + ! non-MPIRoot are reset in SyncBoundaryParticleOutput() BPO%RealPartOut(iPartBound,iSpec) = 0. END DO ! iSpec = 1, BPO%NSpecies END DO ! iPartBound = 1, BPO%NPartBoundaries @@ -432,8 +431,8 @@ SUBROUTINE AnalyzeSurface(Time) ELSE CALL WriteDataInfo(unit_index,RealScalar=SEE%RealElectronOut(iPartBound)/SurfModelAnalyzeSampleTime) END IF ! ABS(SurfModelAnalyzeSampleTime).LE.0.0 - ! Reset PartMPI%MPIRoot counters after writing the data to the file, - ! non-PartMPI%MPIRoot are reset in SyncBoundaryParticleOutput() + ! Reset MPIRoot counters after writing the data to the file, + ! non-MPIRoot are reset in SyncBoundaryParticleOutput() SEE%RealElectronOut(iPartBound) = 0. END DO ! iPartBound = 1, SEE%NPartBoundaries END IF ! CalcElectronSEE @@ -594,7 +593,6 @@ SUBROUTINE GetCollCounter(SurfCollNum,AdsorbNum, DesorbNum) USE MOD_Particle_Vars ,ONLY: nSpecies USE MOD_SurfaceModel_Analyze_Vars ,ONLY: SurfAnalyzeCount, SurfAnalyzeNumOfAds, SurfAnalyzeNumOfDes #if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_Particle_Boundary_Vars ,ONLY: SurfCOMM #endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING @@ -616,7 +614,7 @@ SUBROUTINE GetCollCounter(SurfCollNum,AdsorbNum, DesorbNum) END DO #if USE_MPI -IF(PartMPI%MPIRoot)THEN +IF(MPIRoot)THEN CALL MPI_REDUCE(MPI_IN_PLACE,SurfCollNum ,nSpecies,MPI_INTEGER,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) CALL MPI_REDUCE(MPI_IN_PLACE,AdsorbNum ,nSpecies,MPI_INTEGER,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) CALL MPI_REDUCE(MPI_IN_PLACE,DesorbNum ,nSpecies,MPI_INTEGER,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) @@ -643,7 +641,6 @@ SUBROUTINE GetPorousBCInfo() USE MOD_Globals USE MOD_SurfaceModel_Vars ,ONLY: nPorousBC USE MOD_SurfaceModel_Analyze_Vars ,ONLY: PorousBCOutput -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI #if USE_MPI USE MOD_Particle_Boundary_Vars ,ONLY: SurfCOMM #endif /*USE_MPI*/ @@ -658,14 +655,14 @@ SUBROUTINE GetPorousBCInfo() INTEGER :: iPBC !=================================================================================================================================== #if USE_MPI -IF(PartMPI%MPIRoot)THEN +IF(MPIRoot)THEN CALL MPI_REDUCE(MPI_IN_PLACE , PorousBCOutput, 5*nPorousBC, MPI_DOUBLE_PRECISION, MPI_SUM, 0, SurfCOMM%UNICATOR, iError) ELSE CALL MPI_REDUCE(PorousBCOutput, PorousBCOutput, 5*nPorousBC, MPI_DOUBLE_PRECISION, MPI_SUM, 0, SurfCOMM%UNICATOR, iError) END IF #endif /*USE_MPI*/ -IF(PartMPI%MPIRoot)THEN +IF(MPIRoot)THEN DO iPBC = 1, nPorousBC IF(PorousBCOutput(1,iPBC).GT.0.0)THEN ! Pumping Speed (Output(2)) is the sum of all elements (counter over particles exiting through pump) @@ -686,7 +683,6 @@ SUBROUTINE SyncBoundaryParticleOutput() ! MODULES USE MOD_Globals USE MOD_SurfaceModel_Analyze_Vars ,ONLY: BPO -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_Particle_Boundary_Vars ,ONLY: SurfCOMM ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE @@ -700,7 +696,7 @@ SUBROUTINE SyncBoundaryParticleOutput() INTEGER :: SendBufSize !=================================================================================================================================== SendBufSize = BPO%NPartBoundaries*BPO%NSpecies -IF(PartMPI%MPIRoot)THEN +IF(MPIRoot)THEN ! Map 2D array to vector for sending via MPI SendBuf = RESHAPE(BPO%RealPartOut,(/SendBufSize/)) CALL MPI_REDUCE(MPI_IN_PLACE,SendBuf(1:SendBufSize),SendBufSize,MPI_DOUBLE_PRECISION,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) @@ -726,7 +722,6 @@ SUBROUTINE SyncElectronSEE() ! MODULES USE MOD_Globals USE MOD_SurfaceModel_Analyze_Vars ,ONLY: SEE -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_Particle_Boundary_Vars ,ONLY: SurfCOMM ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE @@ -737,11 +732,11 @@ SUBROUTINE SyncElectronSEE() !----------------------------------------------------------------------------------------------------------------------------------- ! LOCAL VARIABLES !=================================================================================================================================== -IF (PartMPI%MPIRoot) THEN +IF (MPIRoot) THEN CALL MPI_REDUCE(MPI_IN_PLACE , SEE%RealElectronOut, SEE%NPartBoundaries,MPI_DOUBLE_PRECISION,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) ELSE CALL MPI_REDUCE(SEE%RealElectronOut , 0 , SEE%NPartBoundaries,MPI_DOUBLE_PRECISION,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) - ! Reset non PartMPI%MPIRoot counters, PartMPI%MPIRoot counters are reset after writing the data to the file + ! Reset non MPIRoot counters, MPIRoot counters are reset after writing the data to the file SEE%RealElectronOut = 0. END IF diff --git a/src/particles/surfacemodel/surfacemodel_main.f90 b/src/particles/surfacemodel/surfacemodel_main.f90 index 9f06c8796..d84406800 100644 --- a/src/particles/surfacemodel/surfacemodel_main.f90 +++ b/src/particles/surfacemodel/surfacemodel_main.f90 @@ -919,7 +919,7 @@ SUBROUTINE StickingCoefficientModel(PartID,SideID,n_Loc) IF(ParticleSticks) THEN ! Remove the particle from the simulation (total energy was added to the sampled heat flux before the interaction) - CALL RemoveParticle(PartID) + CALL RemoveParticle(PartID,BCID=locBCID) ELSE ! Perform regular Maxwell scattering CALL MaxwellScattering(PartID,SideID,n_Loc) From 291b9d088298411131d6ddc0d2df1a91ce469c8c Mon Sep 17 00:00:00 2001 From: Paul Nizenkov Date: Tue, 29 Aug 2023 01:38:23 +0200 Subject: [PATCH 07/16] Looping over nBCSides to build communicator; Added comparison of electric current at the exit of the thermionic emission test case --- .../PartAnalyze_ref.csv | 25 +++++++++------- .../SurfaceAnalyze_ref.csv | 17 +++++++++++ .../analyze.ini | 15 +++++++--- .../channel_mesh.h5 | Bin 23858 -> 15858 bytes .../command_line.ini | 2 +- .../hopr.ini | 4 +-- .../parameter.ini | 28 +++++++++++------- .../readme.md | 1 + .../2D_Landmark/command_line.ini | 1 - .../2D_Landmark/parameter.ini | 10 ------- .../PartAnalyze_ref.csv | 24 --------------- .../turner_bias-voltage_AC-DC/analyze.ini | 7 ----- .../boundary/particle_boundary_condition.f90 | 3 ++ .../boundary/particle_boundary_init.f90 | 15 +++++++--- .../surfacemodel/surfacemodel_analyze.f90 | 7 ++--- 15 files changed, 81 insertions(+), 78 deletions(-) create mode 100644 regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/SurfaceAnalyze_ref.csv delete mode 100644 regressioncheck/NIG_PIC_poisson_RK3/turner_bias-voltage_AC-DC/PartAnalyze_ref.csv diff --git a/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/PartAnalyze_ref.csv b/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/PartAnalyze_ref.csv index 90436bcef..faab753f3 100644 --- a/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/PartAnalyze_ref.csv +++ b/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/PartAnalyze_ref.csv @@ -1,12 +1,17 @@ 001-TIME,002-Current-Spec-001-SF-001 0.0000000000000000E+000,0.0000000000000000E+000 -0.1000000000000000E-011,0.1467593701480000E+001 -0.2000000000000000E-011,0.1467433483827000E+001 -0.3000000000000000E-011,0.1467433483827000E+001 -0.4000000000000000E-011,0.1467433483827000E+001 -0.5000000000000000E-011,0.1467593701480000E+001 -0.5999999999999999E-011,0.1467593701480000E+001 -0.6999999999999999E-011,0.1467433483827000E+001 -0.8000000000000000E-011,0.1467593701480000E+001 -0.9000000000000000E-011,0.1467433483827000E+001 -0.9999999999999999E-011,0.1467593701480000E+001 +0.2000000000000000E-010,0.1467513592653500E+001 +0.4000000000000000E-010,0.1467513592653500E+001 +0.6000000000000000E-010,0.1467513592653500E+001 +0.8000000000000000E-010,0.1467433483827000E+001 +0.1000000000000000E-009,0.1467513592653499E+001 +0.1200000000000000E-009,0.1467593701480000E+001 +0.1400000000000000E-009,0.1467433483827000E+001 +0.1600000000000000E-009,0.1467513592653500E+001 +0.1800000000000000E-009,0.1467593701480000E+001 +0.2000000000000000E-009,0.1467513592653501E+001 +0.2200000000000000E-009,0.1467433483827000E+001 +0.2400000000000000E-009,0.1467513592653500E+001 +0.2600000000000000E-009,0.1467513592653500E+001 +0.2800000000000000E-009,0.1467513592653500E+001 +0.3000000000000000E-009,0.1467513592653498E+001 diff --git a/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/SurfaceAnalyze_ref.csv b/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/SurfaceAnalyze_ref.csv new file mode 100644 index 000000000..8f0ace133 --- /dev/null +++ b/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/SurfaceAnalyze_ref.csv @@ -0,0 +1,17 @@ +001-TIME,002-Flux-Spec-001-BC_Xplus,003-TotalElectricCurrent-BC_Xplus +0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000 +0.2000000000000000E-010,0.0000000000000000E+000,0.0000000000000000E+000 +0.4000000000000000E-010,0.0000000000000000E+000,0.0000000000000000E+000 +0.6000000000000000E-010,0.0000000000000000E+000,0.0000000000000000E+000 +0.8000000000000000E-010,0.0000000000000000E+000,0.0000000000000000E+000 +0.1000000000000000E-009,0.0000000000000000E+000,0.0000000000000000E+000 +0.1200000000000000E-009,0.0000000000000000E+000,0.0000000000000000E+000 +0.1400000000000000E-009,0.0000000000000000E+000,0.0000000000000000E+000 +0.1600000000000000E-009,0.0000000000000000E+000,0.0000000000000000E+000 +0.1800000000000000E-009,0.0000000000000000E+000,0.0000000000000000E+000 +0.2000000000000000E-009,0.0000000000000000E+000,0.0000000000000000E+000 +0.2200000000000000E-009,0.0000000000000000E+000,0.0000000000000000E+000 +0.2400000000000000E-009,0.2500000000000002E+017,-.4005441325000004E-002 +0.2600000000000000E-009,0.6887500000000006E+019,-.1103499085037501E+001 +0.2800000000000000E-009,0.9169500000000008E+019,-.1469115769183501E+001 +0.3000000000000000E-009,0.9144999999999985E+019,-.1465190436684997E+001 diff --git a/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/analyze.ini b/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/analyze.ini index acbbd87ef..e72bb0a44 100644 --- a/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/analyze.ini +++ b/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/analyze.ini @@ -1,5 +1,12 @@ -! compare the last line of PartAnalyze.csv with a reference file -compare_data_file_name = PartAnalyze.csv -compare_data_file_reference = PartAnalyze_ref.csv -compare_data_file_tolerance = 5e-3 +! compare the current entering through the surface flux +compare_column_file = PartAnalyze.csv +compare_column_reference_file = PartAnalyze_ref.csv +compare_column_index = 1 +compare_column_tolerance_value = 0.01 +compare_column_tolerance_type = relative + +! compare the current leaving the domain +compare_data_file_name = SurfaceAnalyze.csv +compare_data_file_reference = SurfaceAnalyze_ref.csv +compare_data_file_tolerance = 0.01 compare_data_file_tolerance_type = relative diff --git a/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/channel_mesh.h5 b/regressioncheck/CHE_poisson/SurfFlux_ThermionicEmission_Schottky/channel_mesh.h5 index dc042beac4f368c5e3517b1ed87c99819b8fe5e3..0edf3c72693af2da41c430f25582736a1fcd990c 100644 GIT binary patch delta 2595 zcmZuzZD>r4}qCb898x44!G+UeF*tFRA-1IO@>cJ9MY zQR|OE8QW+)Fs8z|?8kl#Ce^x8!lq_027;sRgN=bK!Uj{=Kz|JxbLY7q_8Rtvllz|M zJs;0G@0<7LLQTATbhg)bw_MF`><$T{AP5EvhQc3=WQ+9>CaWjY%gr`hk?e;MA3R@c z^NBN5_r1K{Xt&Y5&SFRywq~Lb5+shh?cn60Jue-4b^pNw7OwgP!B3yIB-WJr_`_I8 z*yeaFuJkEpqZ8#QKLxfR&k2I4SJ=uAewZiwTmvdmc$93jfop* z#Gj66wuj(w7c_d)KPon;VOs431U_@?2Y;|vt!V`v@X3Fv~9ZRRNf=Q+Ds63o-;AKl5 zmEdg9wKnObe@GtA+E&v(SOq{|5T>j3w`ftW`L5*Xo%W=aZGNu(f+*M4+2(tNrJxPJ zO~Y-AOG5*VV7%Y)9L z`Ps>WQ{wQouv_=<-^UqoQdpQl=Z&-&qpGfw)EP=44*b!L0E^fEH};olw6lkH7#dAR zHNA^1+~In5(}=F>4Ym#Pk~i1ybd$6E5Y;lj@+#OlC)UcCVjUU1Rjfj+*bTN0HO0Ek zn4j{o9&xqBJrvDFu;<=zZZ``Cs{6@_XyvDBR1Lc9dnsb5`j5Q6ob9~P za&$*iDHqqo@99NDE$4Y%X7TdvV;xMrUAxY)RK~oqidL+PF-p4KExVgcq`w8eBz^YBbFv39tz^h zL7Xxe>6>{5u3nGF;jwa-T=z~a4AX01#n%$5lSU`^ikKGpKksqt&_IEeqiD4)Ms=up_bE(qN`pG7ugy;@t(Y= zSa*jTdpliEsQMQUCvWVhtxm0F2kgE#_IXOJP)lQsqH^!Kz`cg7-gEI4#||^rPw%dX z(m$G{7gz%?Z_J(OzP^J|oNjN-U8p|)4M*wV6)(d%#;}XtPGUVo;>zy%={0RFbi7xg z16hSGWn?P$eL&N_5>@gGr_vJ5=J=t|Tt=cspGnghg%;O-PJMZql(h>~%quim4AZ58 zLVx9#lgabrMTOTD+FUf~+w3M9E`;e^K_)ZvG5yPS?+wb-7%*veAWY$b8WoG2t$%^c z0;gLyM-8@FEX`SQ!=)N6W;*GG;wPuFY~Igd3QS|m^a=cmQ5SlIZlZfSi+gHtlSWyv z=Xi}a+w=QQh_SeK*z35*u35(};vRJJoUoi%Fn}a0S>TK-5TvkaE{^0$78eKE)c@!p zzlsGLg&%TrJS1PuV%wb%ghyCe!_v!w)16_#naZ)?M7lWcA>2d zBAV^jGq!=nwRaI)>yM&?9Eonr)_n57j)~nlj(@$0F$M7G5u1@Uqoheh-Zu@P|e_@W(mEE%M z-T?ltY_wekNrr~LVEipX!`Z<2BQQU{XUtz+>>%^GGxW>zTQm@O!&(%CMd$e+J=?ZZ literal 23858 zcmeI4e~_J3dB@+~1Xx3`fl!N9anXSBcSOMWbC*p5i9iStAViQ2WFgUz#AHFh8lznm zsJ}%8Ly@IvFi9j?)f)zxO?#d!BRd-Xu!_ znI>oE?DIa~^F7b^InO!wz3=Y5vo|hZdfE|lo;9a&8yae6H}g7|{CG6A6EA5Ie8$Jq z#Xzz*B>Tm|_F|wpyz$sE37?nZ56^0Wg^9QN+-1v}X0RbXHce~R9NJzDCE&sH+Y92t zJ81uZR$%$EQ&)Lq_od}=&PL}xvFqi$KGeQKn(-tzw^`n~8fb=^qjkD3nAo;`jc}OHhV_qitjei9Ma5Hyp>l@Y_UI2!DjoS8Seb- z7l(K`&S9N6o0n~xxN7_U$644>X*74V)oW6m^ER%Zm_eL9{o<_LGNU-7X<@nEN3`oY zfAhw-T|JSq+W+gL7%74cGb`ycP}6mOt8v~!_XQW^IGe1|m z=eHHueN?Q(Q{`Oks~)56%dO!>QTwy>`Z1;7eY*eaZRZK89>?yzE*s+T6Mo%WkM@a( z_X*aIo%wm#xil?;Bi3)_13!839(Go`f6pImnnx99FXwB8b(-e4`=~Q3YuQhgpMK0(vH6P5GmCTb8*}vi z#=2z7mTl`hRd1TpH*L9W-6o?emKqvcvU1&3dcYW4viiEM6HViIW9znEclpHTNj+M1 z{0k>GUb$hiU9rcPY`NNl9f18>*bevo*}H%34Bn^NxAvJU?>qj?Uq9EM-6k_$&>~ z`5gzoG5hYr>frYM#%auF2;A6$ge?lVu@|WuTO9C+jRxGjQ`C(uS0}!)*ClLKz>STm z8(R}_W1eqpT%9<^Rwisiz>WD4)7aL48(W*OYXfdfff`U>@(>Nlu^8}qfX zj|AM9?|jB?3Ai!GHg<D%@6ZV6E8*?tk{#~8;#_mnn&jW7kYwE`K1l-tz3HwdJjqO%9=7UdsW8Y5L zP{56SSKZjWfE)VWP<-Pq!Q8}mAijjGSq zwM@4isw-oIU8MDm{jdJ0_B7d7rZHpxBh1`q1S~#)c+9-tif7BOalSrGorbP)=kua8 zVE;FoJ4ZI^IuDxPTg)?uhVCPp>+z`TYmdJ^kv(^=x^2|`wdR@|b)PPuhVCB#$f7$h zV6KN_dyQUO)ICx@%^dgtlCJZKy8oTl5$imkiL&S(9WZqNP}q=q)FqDT=>9?)N63%5 zKNse_;hXzRb?5h7bvI+r>Rh!y(EVe%-s7lyjC_yBI{!o(u4~kNj(i$*K30JDBkFQ( zF1o{ZQsZ15C*L)Qx<8Vx_b%!_Kh3j2cY&^bw$So=f12tN&(-n5Y3QB+AnSSV|E#$_ zA5r(;gt=C+&I{%HoJZY<#q&C&?hED9&}E&*(fxtux&~4ApM-gBQP=DCIgPsiD4zE` z>b_Jy4c!+DaL(vk`(HKJYm2%slTV}0&dujI>i)gvS~KdtT)uOT=W3CB=NENb^og4PBpkUq{^&A<1&CUZQKSE$R~6bn5&w>3R-h#ve-Sh;=?H&561v38SHVvH+i# zsCx<|S#*Uv{Z z>ild0-jAqDY;)0lgPqhkS0nOWgQ&Y(x~^f=JvGg@Y-UXzplBCA9c@UqAYbD)wOF7b-${)^Mys-v*bHx_TKAT zsqXxu?pHL|?_E)Mm3$hyuNB}k9(B)wB+I#4p=+-#>Yf{T)cMQO_4`!R{mZnDSm#%X zEGwQC9Ph-7j3vf-N?nRJf(Onxb_V-P~e1@X# zF6sLHJL+B%=AwIsFz;K`{R_?Y9>x8AbC`?nr2(VPcWADkC8F*Zgt=C+&Zo=wzD3>7 ziRb!7-F5Q4wpi!OGdyF5l~jx}Qop$MwEN92&YC1h}Sge?OtQ#|w+Po0ure{$8wW=Mi;3uDL#w zQTHnO&NV8yn7Yd8ITjkTxy-w!1MEy^SL%)toJ%$3)G|T`;_0Xuvq7J z$fu$ECjhdn_niSl_a?=Aj_|nN9rB;A9(CU>p8cr%F8N+ttn*I!G<4q$AdBvfw2r9z zryBQraMYa&bE)(7!klx|eNUPf>-^p@7v1*-j5@zx*l}sThn@EgzDNz-8|5!hkGeO> zcg|7wy0o@`$0&u=zd7o-jAsJ;lQKL?@W6Xb*~ZTwZ%GrM83~` z)ct6PhwjG$#<}`rD5Sm*a^ zuFpZ#{S*^rsq-zmK3+ZQzFl)&m#BNId>Xp932@F)cSBlR)crIQWzl`Bu6_TDy6ZL9 zdmeQ^BcF!uX8~lX^IJ97HHf-b2=m%vov)Pd`(M;uC!Y5_>V8hX*A~y!9r9`DZq;1x zQPjOunAZ_?x6AjvJ?eg5K8IarQtn_x_8N^QRgiJ{N59FcR`Z1 zZol@9qwhWaKdzq|+57lpcN~6eGhWYU96saVwGR%z^s^3lIe%)T#_RFgM+Y2BoYLm_ z;3dBv$H(^OPn_0f9{A%EpSak`RHwY(r9Qk~OFmfgWPXj;e$6laT%U4Y&ZEXlKd*~? z%J{^&xjM&_BM%=j$h(Aeu?M!C7$`EZswOb{5miA zH4cBWdfmgXarjuq;n#TW*L-{|d@T6Q-TQdI`m{Ln;4>e*pM~E$Ue=d*kKg{5@zJm2 zgJa19pZqh<>!OU$_^zJ&Km4-3nP21Z%lPC|$FJ{`#LfINem_3*c2(y!-#5y0Q1j8N z`S2L8;b)wBl7CrO`1$o)^2r|yAHQBNc`^<^f|`!@X-Tr{T!ch_$4pr0}l%yzqB*Y zr=6eANf|%mEicDMx8##I7XDQC^;hyU{`-kPHB#n7-sDy1U-RK%pZNVP&q>aw#;31m z=CgmT?T1f}+~0Zrzqh-8%q#20`fGm1OMb5N^zrL_h%;5?lkqw}^{C^6m;8LrOMb>n zKKa+{&3MTt&zhfc_~eHrAABr)_5&P#jl+k>IQ$xi4-X3;9}B;Z1HZ=MGam~d9}B;X zlg~qq!)JbLGru2B9PEFdp1Xh96s~0@bR(m%Q#cr`qwyo z=C?NUYkYcsygNnGffr&OhVVcmB@*weC4br;L;HFZslo?4vt5Qu2vY z=b7=6PyCvnarosv=6q@#KI_EpYrSQhoKKBUA3yWaYwc9GKE%P24?ebz4-TLB;PCMo zhhO9H;bGz9V{1O~YCd?$?{}Zfc**CSp1yng9M!^84kP@sgkO zFZ0ZJ$FoWv;OxvFu`!ivG z);^?um^%0zUHiMc;B$5D&&+Gz-{)PKD^XKLM4wJvP z5b+6suBLMf>Ul?$&-QNiq zR)4X&*Zvaqn(wv0RDD?8yjss=FH2)Dmvt_S)Sb&K)IGj zRNWegK#z-JDC+tpRUb z*VcfyPS?)yGIi%z^T99InAb=g^jvduTw8Nob8}o2`0Pp^6-Th;G zy#4ZteSP{13zknrcJ##`=zIdjI8XTNN5I>^^xxW@U*qjxLU;7=!Q1@Sr%wWN9Pl>Y zhr01=yp97OoP5B``8(2YLU(rK!*74%xifv@mGiH0;(_C1+rL2kc{hHIlV?AieCjyh z3origUq7<&OFenkIPt;pu{BQoUHZH9_wTx|Cx7rd{~GTX2Ygi$k_~?RDPw?LV%yLZUcENur@^3J1*vI~# zSZ?zJuiEH9Yulf_|DopEIQ*nv_tv9bIsdum@RZSDvb8_+|J?IK{{o=q-$!!mt;^Ot z_#|Gn|6y--=Rc2F_BQ{Tw3oIxjqQHgUPv0^Jl!|6cV2rMo92$g_Rc#vdawcqD{!y^ z2P^PcDzL|Yuv0r|XIh61?S#&MZ}R2F I = 1.4675 A (A = 1 cm^2) + * In the case with the Schottky effect, the current reduces slightly over time as the electrons in the domain reduce the potential difference diff --git a/regressioncheck/NIG_PIC_poisson_Boris-Leapfrog/2D_Landmark/command_line.ini b/regressioncheck/NIG_PIC_poisson_Boris-Leapfrog/2D_Landmark/command_line.ini index 65c96f0f6..906585720 100644 --- a/regressioncheck/NIG_PIC_poisson_Boris-Leapfrog/2D_Landmark/command_line.ini +++ b/regressioncheck/NIG_PIC_poisson_Boris-Leapfrog/2D_Landmark/command_line.ini @@ -1,2 +1 @@ MPI = 4,10 -!restart_file = 2Dplasma_test_State_000.00000000000000000.h5, 2Dplasma_test_State_000.00000005000000000.h5 diff --git a/regressioncheck/NIG_PIC_poisson_Boris-Leapfrog/2D_Landmark/parameter.ini b/regressioncheck/NIG_PIC_poisson_Boris-Leapfrog/2D_Landmark/parameter.ini index 10b209494..50f855e0e 100644 --- a/regressioncheck/NIG_PIC_poisson_Boris-Leapfrog/2D_Landmark/parameter.ini +++ b/regressioncheck/NIG_PIC_poisson_Boris-Leapfrog/2D_Landmark/parameter.ini @@ -71,13 +71,11 @@ PIC-AlgebraicExternalField = 1 ! 1: Charoy 2019 magnetic + electric field Part-maxParticleNumber = 500000 Part-nSpecies = 2 Part-FIBGMdeltas = (/2.5e-2 , 1.28e-2 , 0.01e-2/) -!Part-FactorFIBGM = (/ 500 , 256 , 1/) Part-FactorFIBGM = (/ 10 , 10 , 1/) PIC-DoDeposition = T PIC-DoInterpolation = T PIC-Deposition-Type = cell_volweight_mean -!PIC-AlgebraicExternalField = 1 DisplayLostParticles=T Part-Species1-MacroParticleFactor = 1.67e4 ! 1.67e2 originally used for z=1e-4 m (case2: 75 parts per cell with dx=dy=5e-5 m) @@ -91,17 +89,9 @@ Part-nBounds = 6 Part-Boundary1-SourceName = BC_ANODE Part-Boundary1-Condition = open -!Part-Boundary1-NbrOfSpeciesSwaps = 3 -!Part-Boundary1-SpeciesSwaps1 = (/1,0/) -!Part-Boundary1-SpeciesSwaps2 = (/2,0/) -!Part-Boundary1-SpeciesSwaps3 = (/3,0/) Part-Boundary2-SourceName = BC_CATHODE Part-Boundary2-Condition = open -!Part-Boundary2-NbrOfSpeciesSwaps = 3 -!Part-Boundary2-SpeciesSwaps1 = (/1,0/) -!Part-Boundary2-SpeciesSwaps2 = (/2,0/) -!Part-Boundary2-SpeciesSwaps3 = (/3,0/) Part-Boundary3-SourceName = BC_periodicy+ Part-Boundary3-Condition = periodic diff --git a/regressioncheck/NIG_PIC_poisson_RK3/turner_bias-voltage_AC-DC/PartAnalyze_ref.csv b/regressioncheck/NIG_PIC_poisson_RK3/turner_bias-voltage_AC-DC/PartAnalyze_ref.csv deleted file mode 100644 index fbf3ffa2d..000000000 --- a/regressioncheck/NIG_PIC_poisson_RK3/turner_bias-voltage_AC-DC/PartAnalyze_ref.csv +++ /dev/null @@ -1,24 +0,0 @@ -001-TIME,002-nPart-Spec-001,003-nPart-Spec-002,004-nPart-Spec-003,005-nPart-Spec-004,006-nPart-Spec-005,007-E-Vib001,008-E-Vib002,009-E-Vib003,010-E-Vib004,011-E-Vib005,012-E-Rot001,013-E-Rot002,014-E-Rot003,015-E-Rot004,016-E-Rot005,017-E-Elec001,018-E-Elec002,019-E-Elec003,020-E-Elec004,021-E-Elec005,022-E-TotalPart -0.0000000000000000E+000,0.0000000000000000E+000,0.4012000000000000E+004,0.4012000000000000E+004,0.0000000000000000E+000,0.8024000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2517594416481664E-013 -0.4580000000000000E-010,0.0000000000000000E+000,0.4011000000000000E+004,0.4012000000000000E+004,0.0000000000000000E+000,0.8023000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2518698200783414E-013 -0.9160000000000000E-010,0.0000000000000000E+000,0.4010000000000000E+004,0.4012000000000000E+004,0.0000000000000000E+000,0.8022000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2522462715369796E-013 -0.1374000000000000E-009,0.0000000000000000E+000,0.4008000000000000E+004,0.4012000000000000E+004,0.0000000000000000E+000,0.8020000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2528153121183654E-013 -0.1832000000000000E-009,0.0000000000000000E+000,0.4008000000000000E+004,0.4012000000000000E+004,0.0000000000000000E+000,0.8020000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2537316774570108E-013 -0.2290000000000000E-009,0.0000000000000000E+000,0.4007000000000000E+004,0.4012000000000000E+004,0.0000000000000000E+000,0.8019000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2547895782038987E-013 -0.2748000000000000E-009,0.0000000000000000E+000,0.4007000000000000E+004,0.4012000000000000E+004,0.0000000000000000E+000,0.8019000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2561151429942829E-013 -0.3206000000000000E-009,0.0000000000000000E+000,0.4005000000000000E+004,0.4012000000000000E+004,0.0000000000000000E+000,0.8017000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2575341266174985E-013 -0.3663999999999999E-009,0.0000000000000000E+000,0.4004000000000000E+004,0.4012000000000000E+004,0.0000000000000000E+000,0.8016000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2592967043329345E-013 -0.4121999999999999E-009,0.0000000000000000E+000,0.4002000000000000E+004,0.4013000000000000E+004,0.0000000000000000E+000,0.8015000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2606812409604058E-013 -0.4579999999999999E-009,0.0000000000000000E+000,0.4000000000000000E+004,0.4013000000000000E+004,0.0000000000000000E+000,0.8013000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2624571920908253E-013 -0.5000000000000000E-009,0.0000000000000000E+000,0.3999000000000000E+004,0.4013000000000000E+004,0.0000000000000000E+000,0.8012000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2641262427096647E-013 -0.5458000000000000E-009,0.0000000000000000E+000,0.3997000000000000E+004,0.4013000000000000E+004,0.0000000000000000E+000,0.8010000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2657709585005035E-013 -0.5916000000000000E-009,0.0000000000000000E+000,0.3997000000000000E+004,0.4013000000000000E+004,0.0000000000000000E+000,0.8010000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2680186010280453E-013 -0.6374000000000000E-009,0.0000000000000000E+000,0.3996000000000000E+004,0.4013000000000000E+004,0.0000000000000000E+000,0.8009000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2703333139144175E-013 -0.6832000000000000E-009,0.0000000000000000E+000,0.3993000000000000E+004,0.4013000000000000E+004,0.0000000000000000E+000,0.8006000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2722071667924760E-013 -0.7290000000000000E-009,0.0000000000000000E+000,0.3991000000000000E+004,0.4013000000000000E+004,0.0000000000000000E+000,0.8004000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2736908745529201E-013 -0.7748000000000000E-009,0.0000000000000000E+000,0.3989000000000000E+004,0.4013000000000000E+004,0.0000000000000000E+000,0.8002000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2756365047732255E-013 -0.8205999999999999E-009,0.0000000000000000E+000,0.3985000000000000E+004,0.4013000000000000E+004,0.0000000000000000E+000,0.7998000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2774018444632331E-013 -0.8663999999999999E-009,0.0000000000000000E+000,0.3982000000000000E+004,0.4013000000000000E+004,0.0000000000000000E+000,0.7995000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2784862968426264E-013 -0.9121999999999999E-009,0.0000000000000000E+000,0.3980000000000000E+004,0.4013000000000000E+004,0.0000000000000000E+000,0.7993000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2799497523940786E-013 -0.9580000000000000E-009,0.0000000000000000E+000,0.3978000000000000E+004,0.4013000000000000E+004,0.0000000000000000E+000,0.7991000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2813652599466704E-013 -0.1000000000000000E-008,0.0000000000000000E+000,0.3978000000000000E+004,0.4013000000000000E+004,0.0000000000000000E+000,0.7991000000000000E+004,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.0000000000000000E+000,0.2828197009080410E-013 diff --git a/regressioncheck/NIG_PIC_poisson_RK3/turner_bias-voltage_AC-DC/analyze.ini b/regressioncheck/NIG_PIC_poisson_RK3/turner_bias-voltage_AC-DC/analyze.ini index 91b4f21f3..dccb8ea36 100644 --- a/regressioncheck/NIG_PIC_poisson_RK3/turner_bias-voltage_AC-DC/analyze.ini +++ b/regressioncheck/NIG_PIC_poisson_RK3/turner_bias-voltage_AC-DC/analyze.ini @@ -1,10 +1,3 @@ -! compare the last row in PartAnalyze.csv with a reference file -!compare_data_file_name = PartAnalyze.csv -!compare_data_file_reference = PartAnalyze_ref.csv -!compare_data_file_tolerance = 20.0e-2 -!compare_data_file_tolerance_type = relative -!compare_data_file_max_differences = 2 - ! integrate columns x:y in a data file as integral(y(x), x, x(1), x(end)) integrate_line_file = SurfaceAnalyze.csv ! data file name integrate_line_columns = 0:9 ! columns x:y diff --git a/src/particles/boundary/particle_boundary_condition.f90 b/src/particles/boundary/particle_boundary_condition.f90 index 4ecf777a3..56119d81c 100644 --- a/src/particles/boundary/particle_boundary_condition.f90 +++ b/src/particles/boundary/particle_boundary_condition.f90 @@ -168,6 +168,7 @@ SUBROUTINE GetBoundaryInteraction(iPart,SideID,flip,ElemID,crossedBC,TriNum,IsIn END IF ! DoDdielectric !----------------------------------------------------------------------------------------------------------------------------------- CASE(7) ! PartBound%RotPeriodicInterPlaneBC + !----------------------------------------------------------------------------------------------------------------------------------- IF(PRESENT(IsInterPlanePart)) THEN CALL RotPeriodicInterPlaneBoundary(iPart,SideID,ElemID,IsInterplanePart) ELSE @@ -177,7 +178,9 @@ SUBROUTINE GetBoundaryInteraction(iPart,SideID,flip,ElemID,crossedBC,TriNum,IsIn CASE(10,11) ! PartBound%SymmetryBC !----------------------------------------------------------------------------------------------------------------------------------- CALL PerfectReflection(iPart,SideID,n_loc,opt_Symmetry=.TRUE.) + !----------------------------------------------------------------------------------------------------------------------------------- CASE DEFAULT + !----------------------------------------------------------------------------------------------------------------------------------- CALL abort(__STAMP__,' ERROR: PartBound not associated!. (unknown case)') END SELECT !PartBound%MapToPartBC(SideInfo_Shared(SIDE_BCID,SideID) END ASSOCIATE diff --git a/src/particles/boundary/particle_boundary_init.f90 b/src/particles/boundary/particle_boundary_init.f90 index 4abe6f977..b6355c5aa 100644 --- a/src/particles/boundary/particle_boundary_init.f90 +++ b/src/particles/boundary/particle_boundary_init.f90 @@ -575,6 +575,7 @@ SUBROUTINE InitParticleBoundarySurfSides() USE MOD_Particle_Boundary_Vars ,ONLY: nComputeNodeSurfSides,nComputeNodeSurfTotalSides,nComputeNodeSurfOutputSides USE MOD_Particle_Boundary_Vars ,ONLY: GlobalSide2SurfSide,SurfSide2GlobalSide #if USE_MPI +USE MOD_Mesh_Vars ,ONLY: nBCSides, offsetElem, SideToElem USE MOD_Particle_Mesh_Vars ,ONLY: ElemInfo_Shared USE MOD_MPI_Shared USE MOD_MPI_Shared_Vars ,ONLY: MPI_COMM_SHARED @@ -609,7 +610,7 @@ SUBROUTINE InitParticleBoundarySurfSides() INTEGER :: GlobalElemID,GlobalElemRank INTEGER :: sendbuf,recvbuf INTEGER :: NbGlobalElemID, NbElemRank, NbLeaderID, nSurfSidesTmp -INTEGER :: color +INTEGER :: color #endif /*USE_MPI*/ INTEGER :: NbGlobalSideID !=================================================================================================================================== @@ -650,7 +651,6 @@ SUBROUTINE InitParticleBoundarySurfSides() GlobalSide2SurfSideProc = -1 nComputeNodeSurfSides = 0 -nBCSidesProc = 0 nSurfSidesProc = 0 ! check every BC side @@ -663,8 +663,6 @@ SUBROUTINE InitParticleBoundarySurfSides() IF (ElemInfo_Shared(ELEM_HALOFLAG,SideInfo_Shared(SIDE_ELEMID,iSide)).EQ.0) CYCLE #endif /*USE_MPI*/ - nBCSidesProc = nBCSidesProc + 1 - ! count number of reflective and rotationally periodic BC sides IF ((PartBound%TargetBoundCond(PartBound%MapToPartBC(SideInfo_Shared(SIDE_BCID,iSide))).EQ.PartBound%ReflectiveBC) .OR. & (PartBound%TargetBoundCond(PartBound%MapToPartBC(SideInfo_Shared(SIDE_BCID,iSide))).EQ.PartBound%RotPeriodicBC).OR. & @@ -854,6 +852,15 @@ SUBROUTINE InitParticleBoundarySurfSides() ! create a communicator between processors with a BC side (including open BCs and sides in the halo region) #if USE_MPI +! Count the number of BC sides per processors. Note: cannot be done in the loop above, since it might happen that a processor +! might not get his own elements and thus will not be added to the communicator. +nBCSidesProc = 0 +DO iSide=1,nBCSides + GlobalElemID = SideToElem(S2E_ELEM_ID,iSide) + offsetElem + IF (ElemInfo_Shared(ELEM_HALOFLAG,GlobalElemID).EQ.0) CYCLE + nBCSidesProc = nBCSidesProc + 1 +END DO + ! Set the control of subset assignment (nonnegative integer). Processes with the same color are in the same new communicator. ! Make sure to include the root IF(MPIRoot) THEN diff --git a/src/particles/surfacemodel/surfacemodel_analyze.f90 b/src/particles/surfacemodel/surfacemodel_analyze.f90 index 3e6e20134..5210b01c6 100644 --- a/src/particles/surfacemodel/surfacemodel_analyze.f90 +++ b/src/particles/surfacemodel/surfacemodel_analyze.f90 @@ -696,15 +696,14 @@ SUBROUTINE SyncBoundaryParticleOutput() INTEGER :: SendBufSize !=================================================================================================================================== SendBufSize = BPO%NPartBoundaries*BPO%NSpecies + +! Map 2D array to vector for sending via MPI +SendBuf = RESHAPE(BPO%RealPartOut,(/SendBufSize/)) IF(MPIRoot)THEN - ! Map 2D array to vector for sending via MPI - SendBuf = RESHAPE(BPO%RealPartOut,(/SendBufSize/)) CALL MPI_REDUCE(MPI_IN_PLACE,SendBuf(1:SendBufSize),SendBufSize,MPI_DOUBLE_PRECISION,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) ! MAP vector back to 2D array BPO%RealPartOut = RESHAPE(SendBuf,(/BPO%NPartBoundaries,BPO%NSpecies/)) ELSE - ! Map 2D array to vector for sending via MPI - SendBuf = RESHAPE(BPO%RealPartOut,(/SendBufSize/)) CALL MPI_REDUCE(SendBuf(1:SendBufSize),0,SendBufSize,MPI_DOUBLE_PRECISION,MPI_SUM,0,SurfCOMM%UNICATOR,IERROR) ! Reset non SurfCOMM%UNICATOR counters, SurfCOMM%UNICATOR counters are reset after writing the data to the file BPO%RealPartOut = 0. From 8e9f4198ec91dc8b837709290a4aab6e4fb185b8 Mon Sep 17 00:00:00 2001 From: Felix Garmirian Date: Tue, 5 Sep 2023 11:25:18 +0200 Subject: [PATCH 08/16] PartMPI/MPI_COMM_WORLD replaced by MPI_COMM_PICLAS --- docs/documentation/developerguide/mpi.md | 2 +- src/analyze/analyze.f90 | 78 +++---- src/analyze/analyzefield.f90 | 36 ++-- src/equations/magnetostatic/calctimestep.f90 | 2 +- .../magnetostatic_poisson/calctimestep.f90 | 2 +- src/equations/maxwell/calctimestep.f90 | 2 +- src/equations/maxwell/equation.f90 | 10 +- src/equations/maxwell/getboundaryflux.f90 | 4 +- src/equations/maxwell_pois/calctimestep.f90 | 2 +- src/equations/maxwell_pois/equation.f90 | 4 +- .../maxwell_pois/getboundaryflux.f90 | 4 +- src/equations/poisson/calctimestep.f90 | 2 +- src/equations/poisson/equation.f90 | 6 +- src/equations/poisson/getboundaryflux.f90 | 2 +- src/globals/globals.f90 | 1 + src/hdg/fillmortar_hdg.f90 | 1 - src/hdg/hdg.f90 | 35 ++- src/interfaces/interfaces.f90 | 6 +- src/io_hdf5/hdf5_input.f90 | 6 +- src/io_hdf5/hdf5_input_particle.f90 | 5 +- src/io_hdf5/hdf5_output.f90 | 10 +- src/io_hdf5/hdf5_output_elemdata.f90 | 4 +- src/io_hdf5/hdf5_output_field.f90 | 24 +-- src/io_hdf5/hdf5_output_particle.f90 | 72 +++---- src/io_hdf5/hdf5_output_state.f90 | 6 +- .../linearoperator/linearoperator.f90 | 2 +- src/linearsolver/linearsolver.f90 | 2 +- src/linearsolver/newton.f90 | 2 +- src/linearsolver/particlesolver.f90 | 29 ++- src/loadbalance/loadbalance.f90 | 12 +- src/mesh/mesh.f90 | 2 +- src/mesh/mesh_readin.f90 | 6 +- src/mesh/mesh_tools.f90 | 2 +- src/mpi/mpi.f90 | 48 +++-- src/mpi/mpi_shared.f90 | 8 +- src/output/output_vtk.f90 | 12 +- src/particles/analyze/particle_analyze.f90 | 61 +++--- .../analyze/particle_analyze_code.f90 | 3 +- .../analyze/particle_analyze_output.f90 | 6 +- .../analyze/particle_analyze_tools.f90 | 200 ++++++++---------- .../boundary/particle_boundary_init.f90 | 10 +- src/particles/dsmc/dsmc_analyze.f90 | 14 +- src/particles/dsmc/dsmc_bg_gas.f90 | 4 +- src/particles/dsmc/dsmc_init.f90 | 6 +- .../emission/particle_br_electron_fluid.f90 | 12 +- src/particles/emission/particle_emission.f90 | 18 +- .../emission/particle_emission_init.f90 | 5 +- .../particle_position_and_velocity.f90 | 40 ++-- .../emission/particle_surface_flux.f90 | 15 +- .../emission/particle_surface_flux_init.f90 | 21 +- src/particles/particle_init.f90 | 13 +- src/particles/particle_mesh/particle_bgm.f90 | 12 +- src/particles/particle_mpi/particle_mpi.f90 | 38 +--- .../particle_mpi/particle_mpi_emission.f90 | 136 ++++++------ .../particle_mpi/particle_mpi_halo.f90 | 18 +- .../particle_mpi/particle_mpi_vars.f90 | 12 +- src/particles/particle_operations.f90 | 2 +- src/particles/particle_timestep.f90 | 8 +- src/particles/pic/analyze/pic_analyze.f90 | 22 +- src/particles/pic/deposition/pic_depo.f90 | 46 ++-- .../pic/deposition/pic_depo_method.f90 | 12 +- .../pic/deposition/pic_depo_tools.f90 | 2 +- .../pic/interpolation/init_BGField.f90 | 2 +- src/particles/restart/particle_readin.f90 | 18 +- src/particles/restart/particle_restart.f90 | 57 +++-- .../sampling/particle_sampling_adaptive.f90 | 27 ++- src/particles/tracking/particle_tracing.f90 | 2 +- src/particles/ttm/ttm_init.f90 | 5 +- src/posti/dmd/dmd_main.f90 | 28 +-- src/posti/piclas2vtk/piclas2vtk.f90 | 16 +- src/posti/superB/superB_tools.f90 | 8 +- src/precond/precond.f90 | 2 +- src/readIMD/readIMD.f90 | 38 ++-- src/readintools/readintools.f90 | 6 +- src/recordpoints/recordpoints.f90 | 4 +- src/restart/restart.f90 | 2 +- src/restart/restart_field.f90 | 12 +- src/timedisc/timedisc.f90 | 4 +- .../timedisc_TimeStepByImplicitRK.f90 | 7 +- .../timedisc_TimeStepByRosenbrock.f90 | 7 +- 80 files changed, 665 insertions(+), 767 deletions(-) diff --git a/docs/documentation/developerguide/mpi.md b/docs/documentation/developerguide/mpi.md index d8fed88c4..ba3ba0a48 100644 --- a/docs/documentation/developerguide/mpi.md +++ b/docs/documentation/developerguide/mpi.md @@ -170,7 +170,7 @@ that either a communicator exists and/or every (other) processor has been set to | Handle | Description | Derived from | | ------------------------------------ | ---------------------------------------------------------------------- | -------------- | -| PartMPI%InitGroup(nInitRegions)%COMM | Emission groups | PartMPI%COMM | +| PartMPIInitGroup(nInitRegions)%COMM | Emission groups | MPI_COMM_PICLAS| | SurfCOMM%UNICATOR | Processors with a surface side (e.g. reflective), including halo sides | MPI_COMM_WORLD | | CPPCOMM%UNICATOR | Coupled power potential | MPI_COMM_WORLD | | EDC%COMM(iEDCBC)%UNICATOR | Electric displacement current (per BC) | MPI_COMM_WORLD | diff --git a/src/analyze/analyze.f90 b/src/analyze/analyze.f90 index 02718e01c..d3aca80ed 100644 --- a/src/analyze/analyze.f90 +++ b/src/analyze/analyze.f90 @@ -333,11 +333,11 @@ SUBROUTINE InitAnalyze() END DO ! iElem = 1, nElems #if USE_MPI IF(MPIroot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE , PPWCellMin , 1 , MPI_DOUBLE_PRECISION , MPI_MIN , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(MPI_IN_PLACE , PPWCellMax , 1 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(MPI_IN_PLACE , PPWCellMin , 1 , MPI_DOUBLE_PRECISION , MPI_MIN , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(MPI_IN_PLACE , PPWCellMax , 1 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) ELSE - CALL MPI_REDUCE(PPWCellMin , 0 , 1 , MPI_DOUBLE_PRECISION , MPI_MIN , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(PPWCellMax , 0 , 1 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(PPWCellMin , 0 , 1 , MPI_DOUBLE_PRECISION , MPI_MIN , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(PPWCellMax , 0 , 1 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) ! in this case the receive value is not relevant. END IF #endif /*USE_MPI*/ @@ -421,11 +421,11 @@ SUBROUTINE CalcError(time,L_2_Error,L_Inf_Error) END DO ! iElem=1,PP_nElems #if USE_MPI IF(MPIroot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE , L_2_Error , PP_nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(MPI_IN_PLACE , L_Inf_Error , PP_nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(MPI_IN_PLACE , L_2_Error , PP_nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(MPI_IN_PLACE , L_Inf_Error , PP_nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) ELSE - CALL MPI_REDUCE(L_2_Error , 0 , PP_nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(L_Inf_Error , 0 , PP_nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(L_2_Error , 0 , PP_nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(L_Inf_Error , 0 , PP_nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) ! in this case the receive value is not relevant. END IF #endif /*USE_MPI*/ @@ -497,11 +497,11 @@ SUBROUTINE CalcErrorPartSource(PartSource_nVar,L_2_PartSource,L_Inf_PartSource) END DO ! iElem=1,PP_nElems #if USE_MPI IF(MPIroot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE , L_2_PartSource , PartSource_nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(MPI_IN_PLACE , L_Inf_PartSource , PartSource_nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(MPI_IN_PLACE , L_2_PartSource , PartSource_nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(MPI_IN_PLACE , L_Inf_PartSource , PartSource_nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) ELSE - CALL MPI_REDUCE(L_2_PartSource , 0 , PartSource_nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(L_Inf_PartSource , 0 , PartSource_nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(L_2_PartSource , 0 , PartSource_nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(L_Inf_PartSource , 0 , PartSource_nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) ! in this case the receive value is not relevant. END IF #endif /*USE_MPI*/ @@ -602,13 +602,13 @@ SUBROUTINE CalcErrorStateFiles(nVar,N1,N2,U1,U2) END DO ! iElem=1,nElems #if USE_MPI IF(MPIroot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE , L_2_Error , nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(MPI_IN_PLACE , volume , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(MPI_IN_PLACE , L_Inf_Error , nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(MPI_IN_PLACE , L_2_Error , nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(MPI_IN_PLACE , volume , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(MPI_IN_PLACE , L_Inf_Error , nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) ELSE - CALL MPI_REDUCE(L_2_Error , L_2_Error2 , nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(volume , volume2 , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(L_Inf_Error , L_Inf_Error2 , nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(L_2_Error , L_2_Error2 , nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(volume , volume2 , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(L_Inf_Error , L_Inf_Error2 , nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) ! in this case the receive value is not relevant. END IF #endif /*USE_MPI*/ @@ -707,13 +707,13 @@ SUBROUTINE CalcErrorStateFileSigma(nVar,N1,U1) END DO ! iElem=1,nElems #if USE_MPI IF(MPIroot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE , L_2_Error , nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(MPI_IN_PLACE , volume , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(MPI_IN_PLACE , L_Inf_Error , nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(MPI_IN_PLACE , L_2_Error , nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(MPI_IN_PLACE , volume , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(MPI_IN_PLACE , L_Inf_Error , nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) ELSE - CALL MPI_REDUCE(L_2_Error , L_2_Error2 , nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(volume , volume2 , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(L_Inf_Error , L_Inf_Error2 , nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(L_2_Error , L_2_Error2 , nVar , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(volume , volume2 , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(L_Inf_Error , L_Inf_Error2 , nVar , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) ! in this case the receive value is not relevant. END IF #endif /*USE_MPI*/ @@ -1243,13 +1243,13 @@ SUBROUTINE PerformAnalyze(OutputTime,FirstOrLastIter,OutPutHDF5) IF(OutPutHDF5 .AND. MeasureTrackTime)THEN #if USE_MPI IF(MPIRoot) THEN - CALL MPI_REDUCE(MPI_IN_PLACE , nTracks , 1 , MPI_INTEGER , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE , tTracking , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE , tLocalization , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE , nTracks , 1 , MPI_INTEGER , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE , tTracking , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE , tLocalization , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) ELSE ! no Root - CALL MPI_REDUCE(nTracks , RECI , 1 , MPI_INTEGER , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) - CALL MPI_REDUCE(tTracking , RECR , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) - CALL MPI_REDUCE(tLocalization , RECR , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) + CALL MPI_REDUCE(nTracks , RECI , 1 , MPI_INTEGER , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) + CALL MPI_REDUCE(tTracking , RECR , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) + CALL MPI_REDUCE(tLocalization , RECR , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) END IF #endif /*USE_MPI*/ SWRITE(UNIT_StdOut,'(132("-"))') @@ -1281,9 +1281,9 @@ SUBROUTINE PerformAnalyze(OutputTime,FirstOrLastIter,OutPutHDF5) ! TotalSideBoundingBoxVolume=SUM(SideBoundingBoxVolume) !#if USE_MPI ! IF(MPIRoot) THEN -! CALL MPI_REDUCE(MPI_IN_PLACE, TotalSideBoundingBoxVolume , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) +! CALL MPI_REDUCE(MPI_IN_PLACE, TotalSideBoundingBoxVolume , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) ! ELSE ! no Root -! CALL MPI_REDUCE(TotalSideBoundingBoxVolume , 0 , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) +! CALL MPI_REDUCE(TotalSideBoundingBoxVolume , 0 , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) ! END IF !#endif /*USE_MPI*/ ! SWRITE(UNIT_stdOut,'(A35,E15.7)') ' Total Volume of SideBoundingBox: ' , TotalSideBoundingBoxVolume @@ -1413,13 +1413,13 @@ SUBROUTINE CodeAnalyzeOutput(TIME) ! MPI Communication #if USE_MPI IF(MPIRoot) THEN - CALL MPI_REDUCE(MPI_IN_PLACE , rBoundingBoxChecks , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE , rPerformBezierClip , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE , rPerformBezierNewton , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE , rBoundingBoxChecks , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE , rPerformBezierClip , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE , rPerformBezierNewton , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) ELSE ! no Root - CALL MPI_REDUCE(rBoundingBoxChecks , rDummy , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) - CALL MPI_REDUCE(rPerformBezierClip , rDummy , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) - CALL MPI_REDUCE(rPerformBezierNewton , rDummy , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) + CALL MPI_REDUCE(rBoundingBoxChecks , rDummy , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) + CALL MPI_REDUCE(rPerformBezierClip , rDummy , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) + CALL MPI_REDUCE(rPerformBezierNewton , rDummy , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) END IF #endif /*USE_MPI*/ @@ -1559,7 +1559,7 @@ SUBROUTINE InitCalcElectricTimeDerivativeSurface() EDC%COMM(iEDCBC)%ID=iEDCBC ! create new emission communicator for electric displacement current communication. Pass MPI_INFO_NULL as rank to follow the original ordering - CALL MPI_COMM_SPLIT(MPI_COMM_WORLD, color, MPI_INFO_NULL, EDC%COMM(iEDCBC)%UNICATOR, iError) + CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS, color, MPI_INFO_NULL, EDC%COMM(iEDCBC)%UNICATOR, iError) ! Find my rank on the shared communicator, comm size and proc name IF(BConProc(iEDCBC))THEN diff --git a/src/analyze/analyzefield.f90 b/src/analyze/analyzefield.f90 index 3e9080bc2..0804bb206 100644 --- a/src/analyze/analyzefield.f90 +++ b/src/analyze/analyzefield.f90 @@ -558,7 +558,7 @@ SUBROUTINE CalcPoyntingIntegral(PoyntingIntegral,doProlong) END DO ! iElems #if USE_MPI - CALL MPI_REDUCE (PoyntingIntegral(:) , SumSabs(:) , nPoyntingIntPlanes , MPI_DOUBLE_PRECISION ,MPI_SUM, 0, MPI_COMM_WORLD,IERROR) + CALL MPI_REDUCE(PoyntingIntegral(:) , SumSabs(:) , nPoyntingIntPlanes , MPI_DOUBLE_PRECISION ,MPI_SUM, 0, MPI_COMM_PICLAS,IERROR) PoyntingIntegral(:) = SumSabs(:) #endif /*USE_MPI*/ @@ -803,7 +803,7 @@ SUBROUTINE GetPoyntingIntPlane() ! prolonged values of mu_r and no MPI information has to be sent. The master side cannot currently be outside of the dielectric ! region (e.g. in vacuum) because that is not allowed. If this would be allowed that MPI rank would need the information of the ! prolonged dielectric material properties from the slave side - CALL MPI_ALLREDUCE(MPI_IN_PLACE,PoyntingUseMuR_Inv,1,MPI_LOGICAL,MPI_LOR,MPI_COMM_WORLD,iError) + CALL MPI_ALLREDUCE(MPI_IN_PLACE,PoyntingUseMuR_Inv,1,MPI_LOGICAL,MPI_LOR,MPI_COMM_PICLAS,iError) #endif ! Determine mu_r on faces within a dielectric region for calculating the Poynting vector and communicate the ! prolonged values via MPI @@ -815,9 +815,9 @@ SUBROUTINE GetPoyntingIntPlane() #if USE_MPI sumFaces=0 sumAllFaces=0 - CALL MPI_REDUCE(nFaces , sumFaces , nPoyntingIntPlanes , MPI_INTEGER, MPI_SUM,0, MPI_COMM_WORLD, IERROR) + CALL MPI_REDUCE(nFaces , sumFaces , nPoyntingIntPlanes , MPI_INTEGER, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) !nFaces(:) = sumFaces(:) - CALL MPI_REDUCE(nPoyntingIntSides , sumAllFaces , 1 , MPI_INTEGER, MPI_SUM,0, MPI_COMM_WORLD, IERROR) + CALL MPI_REDUCE(nPoyntingIntSides , sumAllFaces , 1 , MPI_INTEGER, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) !nPoyntingIntSides = sumAllFaces #else sumFaces=nFaces @@ -996,18 +996,18 @@ SUBROUTINE CalcPotentialEnergy(WEl) #if USE_MPI ! todo: only one reduce with array IF(MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,WEl , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_WORLD, IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,WEl , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) #if (PP_nVar==8) - CALL MPI_REDUCE(MPI_IN_PLACE,WMag , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_WORLD, IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE,Wphi , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_WORLD, IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE,Wpsi , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_WORLD, IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,WMag , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,Wphi , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,Wpsi , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) #endif /*PP_nVar=8*/ ELSE - CALL MPI_REDUCE(WEl ,RD , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_WORLD, IERROR) + CALL MPI_REDUCE(WEl ,RD , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) #if (PP_nVar==8) - CALL MPI_REDUCE(WMag ,RD , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_WORLD, IERROR) - CALL MPI_REDUCE(Wphi ,RD , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_WORLD, IERROR) - CALL MPI_REDUCE(Wpsi ,RD , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_WORLD, IERROR) + CALL MPI_REDUCE(WMag ,RD , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(Wphi ,RD , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(Wpsi ,RD , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) #endif /*PP_nVar=8*/ END IF #endif /*USE_MPI*/ @@ -1197,14 +1197,14 @@ SUBROUTINE CalcPotentialEnergy_Dielectric(WEl) #if USE_MPI IF(MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,WEl , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_WORLD, IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,WEl , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) #if (PP_nVar==8) - CALL MPI_REDUCE(MPI_IN_PLACE,WMag , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_WORLD, IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,WMag , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) #endif /*PP_nVar=8*/ ELSE - CALL MPI_REDUCE(WEl ,RD , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_WORLD, IERROR) + CALL MPI_REDUCE(WEl ,RD , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) #if (PP_nVar==8) - CALL MPI_REDUCE(WMag ,RD , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_WORLD, IERROR) + CALL MPI_REDUCE(WMag ,RD , 1 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) #endif /*PP_nVar=8*/ END IF #endif /*USE_MPI*/ @@ -1502,7 +1502,7 @@ SUBROUTINE CalculateAverageElectricPotential() !AverageElectricPotentialProc = AverageElectricPotentialProc / (1e-4 * 1.28e-2) #if USE_MPI - CALL MPI_ALLREDUCE(AverageElectricPotentialProc , AverageElectricPotential , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , MPI_COMM_WORLD , IERROR) + CALL MPI_ALLREDUCE(AverageElectricPotentialProc , AverageElectricPotential , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , MPI_COMM_PICLAS , IERROR) #else AverageElectricPotential = AverageElectricPotentialProc #endif /*USE_MPI*/ @@ -1585,7 +1585,7 @@ SUBROUTINE GetAverageElectricPotentialPlane() END DO !iElem=1,nElems #if USE_MPI -CALL MPI_ALLREDUCE(nAverageElecPotSides , AverageElectricPotentialFaces , 1 , MPI_INTEGER , MPI_SUM , MPI_COMM_WORLD , IERROR) +CALL MPI_ALLREDUCE(nAverageElecPotSides , AverageElectricPotentialFaces , 1 , MPI_INTEGER , MPI_SUM , MPI_COMM_PICLAS , IERROR) #else AverageElectricPotentialFaces=nAverageElecPotSides #endif /*USE_MPI*/ diff --git a/src/equations/magnetostatic/calctimestep.f90 b/src/equations/magnetostatic/calctimestep.f90 index d9563065c..4f671a249 100644 --- a/src/equations/magnetostatic/calctimestep.f90 +++ b/src/equations/magnetostatic/calctimestep.f90 @@ -74,7 +74,7 @@ FUNCTION CALCTIMESTEP() END DO ! iElem=1,PP_nElems TimeStep(2)=MIN(TimeStep(2),4./maxLambda_v) #if USE_MPI -CALL MPI_ALLREDUCE(MPI_IN_PLACE,TimeStep,2,MPI_DOUBLE_PRECISION,MPI_MIN,MPI_COMM_WORLD,iError) +CALL MPI_ALLREDUCE(MPI_IN_PLACE,TimeStep,2,MPI_DOUBLE_PRECISION,MPI_MIN,MPI_COMM_PICLAS,iError) #endif CalcTimeStep=MINVAL(TimeStep) IF(CalcTimeStep.NE.CalcTimeStep)THEN diff --git a/src/equations/magnetostatic_poisson/calctimestep.f90 b/src/equations/magnetostatic_poisson/calctimestep.f90 index 4ab2db30a..89b9f318f 100644 --- a/src/equations/magnetostatic_poisson/calctimestep.f90 +++ b/src/equations/magnetostatic_poisson/calctimestep.f90 @@ -74,7 +74,7 @@ FUNCTION CALCTIMESTEP() END DO ! iElem=1,PP_nElems TimeStep(2)=MIN(TimeStep(2),4./maxLambda_v) #if USE_MPI -CALL MPI_ALLREDUCE(MPI_IN_PLACE,TimeStep,2,MPI_DOUBLE_PRECISION,MPI_MIN,MPI_COMM_WORLD,iError) +CALL MPI_ALLREDUCE(MPI_IN_PLACE,TimeStep,2,MPI_DOUBLE_PRECISION,MPI_MIN,MPI_COMM_PICLAS,iError) #endif CalcTimeStep=MINVAL(TimeStep) IF(CalcTimeStep.NE.CalcTimeStep)THEN diff --git a/src/equations/maxwell/calctimestep.f90 b/src/equations/maxwell/calctimestep.f90 index 44cb6b4e0..c4bbf5d7a 100644 --- a/src/equations/maxwell/calctimestep.f90 +++ b/src/equations/maxwell/calctimestep.f90 @@ -103,7 +103,7 @@ FUNCTION CALCTIMESTEP() END IF END DO ! iElem #if USE_MPI -CALL MPI_ALLREDUCE(locTimeStepConv,TimeStepConv,1,MPI_DOUBLE_PRECISION,MPI_MIN,MPI_COMM_WORLD,iError) +CALL MPI_ALLREDUCE(locTimeStepConv,TimeStepConv,1,MPI_DOUBLE_PRECISION,MPI_MIN,MPI_COMM_PICLAS,iError) #else TimeStepConv=locTimeStepConv #endif /*USE_MPI*/ diff --git a/src/equations/maxwell/equation.f90 b/src/equations/maxwell/equation.f90 index 34e6231f4..265e2016e 100644 --- a/src/equations/maxwell/equation.f90 +++ b/src/equations/maxwell/equation.f90 @@ -1326,7 +1326,7 @@ SUBROUTINE GetWaveGuideRadius(DoSide) END DO #if USE_MPI -CALL MPI_ALLREDUCE(MPI_IN_PLACE,TERadius,1,MPI_DOUBLE_PRECISION,MPI_MAX,MPI_COMM_WORLD,iError) +CALL MPI_ALLREDUCE(MPI_IN_PLACE,TERadius,1,MPI_DOUBLE_PRECISION,MPI_MAX,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ LBWRITE(UNIT_StdOut,*) ' Found waveguide radius of ', TERadius @@ -1342,7 +1342,7 @@ SUBROUTINE InitExactFlux() USE MOD_PreProc USE MOD_Globals ,ONLY: abort,UNIT_stdOut,mpiroot,CollectiveStop #if USE_MPI -USE MOD_Globals ,ONLY: MPI_COMM_WORLD,MPI_SUM,MPI_INTEGER,IERROR +USE MOD_Globals ,ONLY: MPI_COMM_PICLAS,MPI_SUM,MPI_INTEGER,IERROR #endif USE MOD_Mesh_Vars ,ONLY: nElems,ElemToSide,SideToElem,lastMPISide_MINE USE MOD_Interfaces ,ONLY: FindElementInRegion,FindInterfacesInRegion,CountAndCreateMappings @@ -1406,7 +1406,7 @@ SUBROUTINE InitExactFlux() #if USE_MPI sumExactFluxMasterInterFaces=0 - CALL MPI_REDUCE(nExactFluxMasterInterFaces , sumExactFluxMasterInterFaces , 1 , MPI_INTEGER, MPI_SUM,0, MPI_COMM_WORLD, IERROR) + CALL MPI_REDUCE(nExactFluxMasterInterFaces , sumExactFluxMasterInterFaces , 1 , MPI_INTEGER, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) #else sumExactFluxMasterInterFaces=nExactFluxMasterInterFaces #endif /*USE_MPI*/ @@ -1430,7 +1430,7 @@ SUBROUTINE InitExactFlux() #if USE_MPI sumExactFluxMasterInterFaces=0 - CALL MPI_REDUCE(nExactFluxMasterInterFaces , sumExactFluxMasterInterFaces , 1 , MPI_INTEGER, MPI_SUM,0, MPI_COMM_WORLD, IERROR) + CALL MPI_REDUCE(nExactFluxMasterInterFaces , sumExactFluxMasterInterFaces , 1 , MPI_INTEGER, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) #else sumExactFluxMasterInterFaces=nExactFluxMasterInterFaces #endif /*USE_MPI*/ @@ -1462,7 +1462,7 @@ SUBROUTINE InitExactFlux() SDEALLOCATE(ElemToExactFlux) SDEALLOCATE(FaceToExactFlux) SDEALLOCATE(FaceToExactFluxInter) -!CALL MPI_BARRIER(MPI_COMM_WORLD, iError) +!CALL MPI_BARRIER(MPI_COMM_PICLAS, iError) !stop END SUBROUTINE InitExactFlux diff --git a/src/equations/maxwell/getboundaryflux.f90 b/src/equations/maxwell/getboundaryflux.f90 index 2a42d0ac2..4bc0c47f1 100644 --- a/src/equations/maxwell/getboundaryflux.f90 +++ b/src/equations/maxwell/getboundaryflux.f90 @@ -97,7 +97,7 @@ SUBROUTINE InitBC() END DO MaxBCStateGLobal=MaxBCState #if USE_MPI -CALL MPI_ALLREDUCE(MPI_IN_PLACE,MaxBCStateGlobal,1,MPI_INTEGER,MPI_MAX,MPI_COMM_WORLD,iError) +CALL MPI_ALLREDUCE(MPI_IN_PLACE,MaxBCStateGlobal,1,MPI_INTEGER,MPI_MAX,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ ! Sanity check for BCs @@ -451,7 +451,7 @@ SUBROUTINE ReadBCFlow(FileName) REAL,ALLOCATABLE :: xGP_tmp(:),wBary_tmp(:),wGP_tmp(:) !=================================================================================================================================== SWRITE(UNIT_StdOut,'(A,A)')' Read BC state from file "',FileName -CALL OpenDataFile(FileName,create=.FALSE.,readOnly=.TRUE.,single=.FALSE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(FileName,create=.FALSE.,readOnly=.TRUE.,single=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) CALL GetDataProps('DG_Solution',nVar_HDF5,N_HDF5,nElems_HDF5,NodeType_HDF5) IF(((N_HDF5.NE.PP_N) .OR. (TRIM(NodeType_HDF5).NE.TRIM(NodeType))))THEN InterpolateSolution=.TRUE. diff --git a/src/equations/maxwell_pois/calctimestep.f90 b/src/equations/maxwell_pois/calctimestep.f90 index 60da337f2..828da56b4 100644 --- a/src/equations/maxwell_pois/calctimestep.f90 +++ b/src/equations/maxwell_pois/calctimestep.f90 @@ -102,7 +102,7 @@ FUNCTION CALCTIMESTEP() END IF END DO ! iElem #if USE_MPI -CALL MPI_ALLREDUCE(MPI_IN_PLACE,TimeStepConv,1,MPI_DOUBLE_PRECISION,MPI_MIN,MPI_COMM_WORLD,iError) +CALL MPI_ALLREDUCE(MPI_IN_PLACE,TimeStepConv,1,MPI_DOUBLE_PRECISION,MPI_MIN,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ CalcTimeStep=TimeStepConv END FUNCTION CALCTIMESTEP diff --git a/src/equations/maxwell_pois/equation.f90 b/src/equations/maxwell_pois/equation.f90 index 824bbe885..231e0aa0e 100644 --- a/src/equations/maxwell_pois/equation.f90 +++ b/src/equations/maxwell_pois/equation.f90 @@ -1121,7 +1121,7 @@ SUBROUTINE StartExchangeMPIData_Pois(FaceData,LowerBound,UpperBound,SendRequest, SideID_start=OffsetMPISides_send(iNbProc-1,SendID)+1 SideID_end =OffsetMPISides_send(iNbProc,SendID) CALL MPI_ISEND(FaceData(:,:,:,SideID_start:SideID_end),nSendVal,MPI_DOUBLE_PRECISION, & - nbProc(iNbProc),0,MPI_COMM_WORLD,SendRequest(iNbProc),iError) + nbProc(iNbProc),0,MPI_COMM_PICLAS,SendRequest(iNbProc),iError) END IF ! Start receive face data IF(nMPISides_rec(iNbProc,SendID).GT.0)THEN @@ -1129,7 +1129,7 @@ SUBROUTINE StartExchangeMPIData_Pois(FaceData,LowerBound,UpperBound,SendRequest, SideID_start=OffsetMPISides_rec(iNbProc-1,SendID)+1 SideID_end =OffsetMPISides_rec(iNbProc,SendID) CALL MPI_IRECV(FaceData(:,:,:,SideID_start:SideID_end),nRecVal,MPI_DOUBLE_PRECISION, & - nbProc(iNbProc),0,MPI_COMM_WORLD,RecRequest(iNbProc),iError) + nbProc(iNbProc),0,MPI_COMM_PICLAS,RecRequest(iNbProc),iError) END IF END DO !iProc=1,nNBProcs END SUBROUTINE StartExchangeMPIData_Pois diff --git a/src/equations/maxwell_pois/getboundaryflux.f90 b/src/equations/maxwell_pois/getboundaryflux.f90 index a17c2959e..ceb927151 100644 --- a/src/equations/maxwell_pois/getboundaryflux.f90 +++ b/src/equations/maxwell_pois/getboundaryflux.f90 @@ -102,7 +102,7 @@ SUBROUTINE InitBC() END DO MaxBCStateGLobal=MaxBCState #if USE_MPI -CALL MPI_ALLREDUCE(MPI_IN_PLACE,MaxBCStateGlobal,1,MPI_INTEGER,MPI_MAX,MPI_COMM_WORLD,iError) +CALL MPI_ALLREDUCE(MPI_IN_PLACE,MaxBCStateGlobal,1,MPI_INTEGER,MPI_MAX,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ ! Sanity check for BCs @@ -422,7 +422,7 @@ SUBROUTINE ReadBCFlow(FileName) REAL,ALLOCATABLE :: xGP_tmp(:),wBary_tmp(:),wGP_tmp(:) !=================================================================================================================================== SWRITE(UNIT_StdOut,'(A,A)')' Read BC state from file "',FileName -CALL OpenDataFile(FileName,create=.FALSE.,readOnly=.TRUE.,single=.FALSE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(FileName,create=.FALSE.,readOnly=.TRUE.,single=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) CALL GetDataProps('DG_Solution',nVar_HDF5,N_HDF5,nELems_HDF5,NodeType_HDF5) IF(((N_HDF5.NE.PP_N) .OR. (TRIM(NodeType_HDF5).NE.TRIM(NodeType))))THEN diff --git a/src/equations/poisson/calctimestep.f90 b/src/equations/poisson/calctimestep.f90 index 4ab2db30a..89b9f318f 100644 --- a/src/equations/poisson/calctimestep.f90 +++ b/src/equations/poisson/calctimestep.f90 @@ -74,7 +74,7 @@ FUNCTION CALCTIMESTEP() END DO ! iElem=1,PP_nElems TimeStep(2)=MIN(TimeStep(2),4./maxLambda_v) #if USE_MPI -CALL MPI_ALLREDUCE(MPI_IN_PLACE,TimeStep,2,MPI_DOUBLE_PRECISION,MPI_MIN,MPI_COMM_WORLD,iError) +CALL MPI_ALLREDUCE(MPI_IN_PLACE,TimeStep,2,MPI_DOUBLE_PRECISION,MPI_MIN,MPI_COMM_PICLAS,iError) #endif CalcTimeStep=MINVAL(TimeStep) IF(CalcTimeStep.NE.CalcTimeStep)THEN diff --git a/src/equations/poisson/equation.f90 b/src/equations/poisson/equation.f90 index 65339207a..4bd4702fe 100644 --- a/src/equations/poisson/equation.f90 +++ b/src/equations/poisson/equation.f90 @@ -287,7 +287,7 @@ SUBROUTINE InitCoupledPowerPotential() USE MOD_ReadInTools ,ONLY: GETREALARRAY,GETREAL,GETINTFROMSTR,CountOption USE MOD_Mesh_Vars ,ONLY: BoundaryType,nBCs #if USE_MPI -USE MOD_Globals ,ONLY: IERROR,MPI_COMM_NULL,MPI_DOUBLE_PRECISION,MPI_COMM_WORLD,MPI_INFO_NULL,MPI_UNDEFINED,MPIRoot +USE MOD_Globals ,ONLY: IERROR,MPI_COMM_NULL,MPI_DOUBLE_PRECISION,MPI_COMM_PICLAS,MPI_INFO_NULL,MPI_UNDEFINED,MPIRoot USE MOD_Globals ,ONLY: UNIT_StdOut USE MOD_HDG_Vars ,ONLY: CPPCOMM USE MOD_Mesh_Vars ,ONLY: nBCSides,BC @@ -378,7 +378,7 @@ SUBROUTINE InitCoupledPowerPotential() CPPCOMM%ID = CPPBoundaries ! create new emission communicator for coupled power potential communication. Pass MPI_INFO_NULL as rank to follow the original ordering -CALL MPI_COMM_SPLIT(MPI_COMM_WORLD, color, MPI_INFO_NULL, CPPCOMM%UNICATOR, iError) +CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS, color, MPI_INFO_NULL, CPPCOMM%UNICATOR, iError) ! Find my rank on the shared communicator, comm size and proc name IF(BConProc)THEN @@ -739,7 +739,7 @@ SUBROUTINE ExactFunc(ExactFunction,x,resu,t,ElemID,iRefState,iLinState,BCState) SWRITE(*,*) "r1=",r1 CALL abort(__STAMP__,'Point source in dielectric region: Cannot evaluate the exact function at the singularity!') END IF - resu(1:PP_nVar) = (2.0*Q/eps12) * 1./r1 + resu(1:PP_nVar) = (2.0*Q/eps12) * 1./r1 END IF END ASSOCIATE CASE(500) ! Coaxial capacitor with Floating Boundary Condition (FPC) with from diff --git a/src/equations/poisson/getboundaryflux.f90 b/src/equations/poisson/getboundaryflux.f90 index 2d8ad66fb..d767fafb3 100644 --- a/src/equations/poisson/getboundaryflux.f90 +++ b/src/equations/poisson/getboundaryflux.f90 @@ -95,7 +95,7 @@ SUBROUTINE InitBC() END DO MaxBCStateGLobal=MaxBCState #if USE_MPI -CALL MPI_ALLREDUCE(MPI_IN_PLACE,MaxBCStateGlobal,1,MPI_INTEGER,MPI_MAX,MPI_COMM_WORLD,iError) +CALL MPI_ALLREDUCE(MPI_IN_PLACE,MaxBCStateGlobal,1,MPI_INTEGER,MPI_MAX,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ ! Sanity check for BCs diff --git a/src/globals/globals.f90 b/src/globals/globals.f90 index ce1737da8..4a62619c8 100644 --- a/src/globals/globals.f90 +++ b/src/globals/globals.f90 @@ -39,6 +39,7 @@ MODULE MOD_Globals INTEGER :: MPI_COMM_NODE ! local node subgroup INTEGER :: MPI_COMM_LEADERS ! all node masters INTEGER :: MPI_COMM_WORKERS ! all non-master nodes +INTEGER :: MPI_COMM_PICLAS ! all nodes LOGICAL :: MPIRoot,MPILocalRoot #if USE_MPI !#include "mpif.h" diff --git a/src/hdg/fillmortar_hdg.f90 b/src/hdg/fillmortar_hdg.f90 index 3a1ca87ea..b25250440 100644 --- a/src/hdg/fillmortar_hdg.f90 +++ b/src/hdg/fillmortar_hdg.f90 @@ -67,7 +67,6 @@ SUBROUTINE InitMortar_HDG() #if USE_PETSC USE MOD_HDG_Vars ,ONLY: SmallMortarType #if USE_MPI -USE MOD_MPI_Shared_Vars ,ONLY: MPI_COMM_WORLD USE MOD_MPI ,ONLY: StartReceiveMPIDataInt,StartSendMPIDataInt,FinishExchangeMPIData USE MOD_MPI_Vars #endif /*USE_MPI*/ diff --git a/src/hdg/hdg.f90 b/src/hdg/hdg.f90 index bd1533612..f4cd42c66 100644 --- a/src/hdg/hdg.f90 +++ b/src/hdg/hdg.f90 @@ -128,7 +128,6 @@ SUBROUTINE InitHDG() USE PETSc USE MOD_Mesh_Vars ,ONLY: nMPISides_YOUR #if USE_MPI -USE MOD_MPI_Shared_Vars ,ONLY: MPI_COMM_WORLD USE MOD_MPI ,ONLY: StartReceiveMPIDataInt,StartSendMPIDataInt,FinishExchangeMPIData USE MOD_MPI_Vars #endif /*USE_MPI*/ @@ -322,7 +321,7 @@ SUBROUTINE InitHDG() END DO nPETScUniqueSides = nSides-nDirichletBCSides-nMPISides_YOUR-nMortarMasterSides-nConductorBCsides IF(ZeroPotentialSideID.GT.0) nPETScUniqueSides = nPETScUniqueSides - 1 -CALL MPI_ALLGATHER(nPETScUniqueSides,1,MPI_INTEGER,OffsetPETScSideMPI,1,MPI_INTEGER,MPI_COMM_WORLD,IERROR) +CALL MPI_ALLGATHER(nPETScUniqueSides,1,MPI_INTEGER,OffsetPETScSideMPI,1,MPI_INTEGER,MPI_COMM_PICLAS,IERROR) DO iProc=1, myrank OffsetPETScSide = OffsetPETScSide + OffsetPETScSideMPI(iProc) END DO @@ -446,7 +445,7 @@ SUBROUTINE InitHDG() ! ALLOCATE(FPC%GroupGlobal(1:FPC%nFPCBounds)) ! FPC%GroupGlobal(1:FPC%nFPCBounds) = FPC%Group(1:FPC%nFPCBounds,3) ! ! TODO is this allreduce required? -! !CALL MPI_ALLREDUCE(FPC%Group(1:FPC%nFPCBounds,3),FPC%GroupGlobal(1:FPC%nFPCBounds), FPC%nFPCBounds, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_WORLD, IERROR) +! !CALL MPI_ALLREDUCE(FPC%Group(1:FPC%nFPCBounds,3),FPC%GroupGlobal(1:FPC%nFPCBounds), FPC%nFPCBounds, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_PICLAS, IERROR) ! nAffectedBlockSides = MAXVAL(FPC%GroupGlobal(:)) ! DEALLOCATE(FPC%GroupGlobal) ! nAffectedBlockSides = MAX(22,nAffectedBlockSides*6) @@ -620,11 +619,11 @@ SUBROUTINE InitZeroPotential() #if USE_MPI ! Combine number of found zero potential sides to make sure that at least one is found IF(MPIroot)THEN - CALL MPI_REDUCE(nZeroPotentialSides , nZeroPotentialSidesGlobal , 1 , MPI_INTEGER , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) - CALL MPI_REDUCE(nZeroPotentialSides , nZeroPotentialSidesMax , 1 , MPI_INTEGER , MPI_MAX , 0 , MPI_COMM_WORLD , IERROR) + CALL MPI_REDUCE(nZeroPotentialSides , nZeroPotentialSidesGlobal , 1 , MPI_INTEGER , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) + CALL MPI_REDUCE(nZeroPotentialSides , nZeroPotentialSidesMax , 1 , MPI_INTEGER , MPI_MAX , 0 , MPI_COMM_PICLAS , IERROR) ELSE - CALL MPI_REDUCE(nZeroPotentialSides , 0 , 1 , MPI_INTEGER , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) - CALL MPI_REDUCE(nZeroPotentialSides , 0 , 1 , MPI_INTEGER , MPI_MAX , 0 , MPI_COMM_WORLD , IERROR) + CALL MPI_REDUCE(nZeroPotentialSides , 0 , 1 , MPI_INTEGER , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) + CALL MPI_REDUCE(nZeroPotentialSides , 0 , 1 , MPI_INTEGER , MPI_MAX , 0 , MPI_COMM_PICLAS , IERROR) END IF #else nZeroPotentialSidesGlobal = nZeroPotentialSides @@ -664,7 +663,7 @@ END SUBROUTINE InitZeroPotential !=================================================================================================================================== SUBROUTINE InitFPC() ! MODULES -USE MOD_Globals ! ,ONLY: MPIRoot,iError,myrank,UNIT_stdOut,MPI_COMM_WORLD +USE MOD_Globals ! ,ONLY: MPIRoot,iError,myrank,UNIT_stdOut,MPI_COMM_PICLAS USE MOD_Preproc USE MOD_Mesh_Vars ,ONLY: nBCs,BoundaryType USE MOD_Analyze_Vars ,ONLY: DoFieldAnalyze @@ -881,7 +880,7 @@ SUBROUTINE InitFPC() FPC%COMM(iUniqueFPCBC)%ID=iUniqueFPCBC ! create new emission communicator for floating boundary condition communication. Pass MPI_INFO_NULL as rank to follow the original ordering - CALL MPI_COMM_SPLIT(MPI_COMM_WORLD, color, MPI_INFO_NULL, FPC%COMM(iUniqueFPCBC)%UNICATOR, iError) + CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS, color, MPI_INFO_NULL, FPC%COMM(iUniqueFPCBC)%UNICATOR, iError) ! Find my rank on the shared communicator, comm size and proc name IF(BConProc(iUniqueFPCBC))THEN @@ -946,7 +945,7 @@ END SUBROUTINE InitFPC !=================================================================================================================================== SUBROUTINE InitEPC() ! MODULES -USE MOD_Globals ! ,ONLY: MPIRoot,iError,myrank,UNIT_stdOut,MPI_COMM_WORLD +USE MOD_Globals ! ,ONLY: MPIRoot,iError,myrank,UNIT_stdOut,MPI_COMM_PICLAS USE MOD_Preproc USE MOD_Mesh_Vars ,ONLY: nBCs,BoundaryType USE MOD_Analyze_Vars ,ONLY: DoFieldAnalyze @@ -1160,7 +1159,7 @@ SUBROUTINE InitEPC() ! Create new emission communicator for Electric potential boundary condition communication. ! Pass MPI_INFO_NULL as rank to follow the original ordering - CALL MPI_COMM_SPLIT(MPI_COMM_WORLD, color, MPI_INFO_NULL, EPC%COMM(iUniqueEPCBC)%UNICATOR, iError) + CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS, color, MPI_INFO_NULL, EPC%COMM(iUniqueEPCBC)%UNICATOR, iError) ! Find my rank on the shared communicator, comm size and proc name IF(BConProc(iUniqueEPCBC))THEN @@ -1233,7 +1232,7 @@ SUBROUTINE InitBV() USE MOD_LoadBalance_Vars ,ONLY: PerformLoadBalance #endif /*USE_LOADBALANCE*/ #if USE_MPI -USE MOD_Globals ,ONLY: IERROR,MPI_COMM_NULL,MPI_DOUBLE_PRECISION,MPI_COMM_WORLD,MPI_INFO_NULL,MPI_UNDEFINED,MPIRoot +USE MOD_Globals ,ONLY: IERROR,MPI_COMM_NULL,MPI_DOUBLE_PRECISION,MPI_COMM_PICLAS,MPI_INFO_NULL,MPI_UNDEFINED,MPIRoot USE MOD_Mesh_Vars ,ONLY: nBCSides,BC #endif /*USE_MPI*/ IMPLICIT NONE @@ -1334,7 +1333,7 @@ SUBROUTINE InitBV() BiasVoltage%COMM%ID = BVBoundaries ! Create new emission communicator for electric potential boundary condition communication. Pass MPI_INFO_NULL as rank to follow the original ordering -CALL MPI_COMM_SPLIT(MPI_COMM_WORLD, color, MPI_INFO_NULL, BiasVoltage%COMM%UNICATOR, iError) +CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS, color, MPI_INFO_NULL, BiasVoltage%COMM%UNICATOR, iError) ! Find my rank on the shared communicator, comm size and process name IF(BConProc)THEN @@ -2720,7 +2719,7 @@ SUBROUTINE CheckNonLinRes(RHS,lambda,converged,Norm_R2) #if USE_MPI IF(MPIroot) converged=(Norm_R2.LT.EpsNonLinear**2) - CALL MPI_BCAST(converged,1,MPI_LOGICAL,0,MPI_COMM_WORLD,iError) + CALL MPI_BCAST(converged,1,MPI_LOGICAL,0,MPI_COMM_PICLAS,iError) #else converged=(Norm_R2.LT.EpsNonLinear**2) #endif /*USE_MPI*/ @@ -2804,14 +2803,14 @@ SUBROUTINE CG_solver(RHS,lambda,iVar) IF(useRelativeAbortCrit)THEN #if USE_MPI IF(MPIroot) converged=(Norm_R2.LT.1e-16) - CALL MPI_BCAST(converged,1,MPI_LOGICAL,0,MPI_COMM_WORLD,iError) + CALL MPI_BCAST(converged,1,MPI_LOGICAL,0,MPI_COMM_PICLAS,iError) #else converged=(Norm_R2.LT.1e-16) #endif /*USE_MPI*/ ELSE #if USE_MPI IF(MPIroot) converged=(Norm_R2.LT.EpsCG**2) - CALL MPI_BCAST(converged,1,MPI_LOGICAL,0,MPI_COMM_WORLD,iError) + CALL MPI_BCAST(converged,1,MPI_LOGICAL,0,MPI_COMM_PICLAS,iError) #else converged=(Norm_R2.LT.EpsCG**2) #endif /*USE_MPI*/ @@ -2873,7 +2872,7 @@ SUBROUTINE CG_solver(RHS,lambda,iVar) CALL SYSTEM_CLOCK(count=CounterStart) #endif /*defined(MEASURE_MPI_WAIT)*/ - CALL MPI_BCAST(converged,1,MPI_LOGICAL,0,MPI_COMM_WORLD,iError) + CALL MPI_BCAST(converged,1,MPI_LOGICAL,0,MPI_COMM_PICLAS,iError) #if defined(MEASURE_MPI_WAIT) CALL SYSTEM_CLOCK(count=CounterEnd, count_rate=Rate) @@ -3238,7 +3237,7 @@ SUBROUTINE VectorDotProduct(dim1,A,B,Resu) #if USE_MPI ResuSend=Resu - CALL MPI_ALLREDUCE(ResuSend,Resu,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,iError) + CALL MPI_ALLREDUCE(ResuSend,Resu,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_PICLAS,iError) #endif #if defined(MEASURE_MPI_WAIT) diff --git a/src/interfaces/interfaces.f90 b/src/interfaces/interfaces.f90 index 868e33f6d..c09ca59c2 100644 --- a/src/interfaces/interfaces.f90 +++ b/src/interfaces/interfaces.f90 @@ -822,9 +822,9 @@ SUBROUTINE CountAndCreateMappings(TypeName,& END DO sumGlobalFaces = 0 sumGlobalInterFaces = 0 - CALL MPI_REDUCE(nElems ,nGlobalSpecialElems,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,iError) - CALL MPI_REDUCE(nMasterfaces ,nGlobalFaces ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,IERROR) - CALL MPI_REDUCE(nMasterInterFaces,nGlobalInterfaces ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,IERROR) + CALL MPI_REDUCE(nElems ,nGlobalSpecialElems,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,iError) + CALL MPI_REDUCE(nMasterfaces ,nGlobalFaces ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) + CALL MPI_REDUCE(nMasterInterFaces,nGlobalInterfaces ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) #else nGlobalSpecialElems = nElems sumGlobalFaces = nFaces diff --git a/src/io_hdf5/hdf5_input.f90 b/src/io_hdf5/hdf5_input.f90 index ce039326a..061c57c3c 100644 --- a/src/io_hdf5/hdf5_input.f90 +++ b/src/io_hdf5/hdf5_input.f90 @@ -182,7 +182,7 @@ FUNCTION ISVALIDMESHFILE(MeshFileName) CALL H5PCREATE_F(H5P_FILE_ACCESS_F, Plist_ID, iError) #if USE_MPI ! Setup file access property list with parallel I/O access (MPI) -CALL H5PSET_FAPL_MPIO_F(Plist_ID,MPI_COMM_WORLD, MPIInfo, iError) +CALL H5PSET_FAPL_MPIO_F(Plist_ID,MPI_COMM_PICLAS, MPIInfo, iError) #endif /*USE_MPI*/ ! Check if file exists @@ -699,7 +699,7 @@ SUBROUTINE GetHDF5NextFileName(FileName,NextFileName_HDF5) ! INPUT/OUTPUT VARIABLES CHARACTER(LEN=*),INTENT(IN) :: FileName !< filename to check #if USE_MPI -LOGICAL,INTENT(IN) :: single !< switch whether file is being accessed in parallel my MPI_COMM_WORLD +LOGICAL,INTENT(IN) :: single !< switch whether file is being accessed in parallel my MPI_COMM_PICLAS #endif CHARACTER(LEN=255),INTENT(OUT) :: NextFileName_HDF5 !< output: follow up file according to checked file opened !----------------------------------------------------------------------------------------------------------------------------------- @@ -720,7 +720,7 @@ SUBROUTINE GetHDF5NextFileName(FileName,NextFileName_HDF5) #if USE_MPI IF(.NOT.single)THEN ! Set property list to MPI IO - CALL H5PSET_FAPL_MPIO_F(Plist_ID, MPI_COMM_WORLD, MPI_INFO_NULL, iError) + CALL H5PSET_FAPL_MPIO_F(Plist_ID, MPI_COMM_PICLAS, MPI_INFO_NULL, iError) END IF #endif /*USE_MPI*/ ! Open file diff --git a/src/io_hdf5/hdf5_input_particle.f90 b/src/io_hdf5/hdf5_input_particle.f90 index 9eade00d6..35b8134d9 100644 --- a/src/io_hdf5/hdf5_input_particle.f90 +++ b/src/io_hdf5/hdf5_input_particle.f90 @@ -187,7 +187,6 @@ SUBROUTINE ReadEmissionVariablesFromHDF5() USE MOD_Globals !USE MOD_PreProc USE MOD_Particle_Vars ,ONLY: Species,nSpecies -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_Particle_Vars ,ONLY: NeutralizationBalanceGlobal !USE MOD_Particle_Vars ,ONLY: NeutralizationBalance USE MOD_HDF5_Input ,ONLY: ReadArray,DatasetExists @@ -215,7 +214,7 @@ SUBROUTINE ReadEmissionVariablesFromHDF5() ! Only the root reads the data and replaces his value, which will be communicated via the all-reduce (he also does the ! initial output) - IF(PartMPI%MPIRoot)THEN + IF(MPIRoot)THEN IF(.NOT.FILEEXISTS(RestartFile)) & CALL abort(__STAMP__,'Error in ReadEmissionVariablesFromHDF5() becuase RestartFile does not exist: '//TRIM(RestartFile)) @@ -242,7 +241,7 @@ SUBROUTINE ReadEmissionVariablesFromHDF5() ! ! Communicate number of particles with all procs in the same init group to the global root for output ! InitGroup=Species(iSpec)%Init(iInit)%InitCOMM ! ! Only processors which are part of group take part in the communication -! CALL MPI_BCAST(Box_X, 3, MPI_DOUBLE_PRECISION, 0, MPI_COMM_WORLD, iError) +! CALL MPI_BCAST(Box_X, 3, MPI_DOUBLE_PRECISION, 0, MPI_COMM_PICLAS, iError) !#endif /*USE_MPI*/ END SELECT diff --git a/src/io_hdf5/hdf5_output.f90 b/src/io_hdf5/hdf5_output.f90 index 0658c5ae7..9c2e72e58 100644 --- a/src/io_hdf5/hdf5_output.f90 +++ b/src/io_hdf5/hdf5_output.f90 @@ -117,7 +117,7 @@ SUBROUTINE WriteTimeAverage(MeshFileName,OutputTime,PreviousTime,VarNamesAvg,Var CALL CloseDataFile() END IF #if USE_MPI - CALL MPI_BARRIER(MPI_COMM_WORLD,iError) + CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ ! Reopen file and write DG solution @@ -147,7 +147,7 @@ SUBROUTINE WriteTimeAverage(MeshFileName,OutputTime,PreviousTime,VarNamesAvg,Var CALL CloseDataFile() END IF #if USE_MPI - CALL MPI_BARRIER(MPI_COMM_WORLD,iError) + CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ ! Reopen file and write DG solution @@ -825,7 +825,7 @@ SUBROUTINE GatheredWriteArray(FileName,create,DataSetName,rank,nValGlobal,nVal,o SDEALLOCATE(UStr) ELSE #endif - CALL OpenDataFile(FileName,create=create,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(FileName,create=create,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) IF(PRESENT(RealArray)) CALL WriteArrayToHDF5(DataSetName , rank , nValGlobal , nVal , & offset , collective , RealArray=RealArray) IF(PRESENT(IntegerArray)) CALL WriteArrayToHDF5(DataSetName , rank , nValGlobal , nVal , & @@ -921,8 +921,8 @@ SUBROUTINE DistributedWriteArray(FileName,DataSetName,rank,nValGlobal,nVal,offse ELSE ! 3: else write with all procs of the given communicator ! communicator_opt has to be the given communicator or else procs that are not in the given communicator might block the write out - ! e.g. surface communicator contains only procs with physical surface and MPI_COMM_WORLD contains every proc - ! Consequently, MPI_COMM_WORLD would block communication + ! e.g. surface communicator contains only procs with physical surface and MPI_COMM_PICLAS contains every proc + ! Consequently, MPI_COMM_PICLAS would block communication CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=communicator) IF(PRESENT(RealArray)) CALL WriteArrayToHDF5(DataSetName , rank , nValGlobal , nVal , & offset , collective , RealArray=RealArray) diff --git a/src/io_hdf5/hdf5_output_elemdata.f90 b/src/io_hdf5/hdf5_output_elemdata.f90 index 312bd2e2d..3524500ca 100644 --- a/src/io_hdf5/hdf5_output_elemdata.f90 +++ b/src/io_hdf5/hdf5_output_elemdata.f90 @@ -169,7 +169,7 @@ SUBROUTINE WriteMyInvisibleRankToHDF5() FileName=TRIM(ProjectName)//'_MyInvisibleRank.h5' IF(MPIRoot) CALL GenerateFileSkeleton('MyInvisibleRank',N_variables,StrVarNames,TRIM(MeshFile),OutputTime,FileNameIn=FileName) #if USE_MPI - CALL MPI_BARRIER(MPI_COMM_WORLD,iError) + CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif ! Write all 'ElemData' arrays to a single container in the state.h5 file @@ -212,7 +212,7 @@ SUBROUTINE WriteLostRotPeriodicSidesToHDF5() FileName=TRIM(ProjectName)//'_LostRotPeriodicSides.h5' IF(MPIRoot) CALL GenerateFileSkeleton('LostRotPeriodicSides',N_variables,StrVarNames,TRIM(MeshFile),OutputTime,FileNameIn=FileName) #if USE_MPI - CALL MPI_BARRIER(MPI_COMM_WORLD,iError) + CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif ! Write all 'ElemData' arrays to a single container in the state.h5 file diff --git a/src/io_hdf5/hdf5_output_field.f90 b/src/io_hdf5/hdf5_output_field.f90 index 53b54b70e..856793131 100644 --- a/src/io_hdf5/hdf5_output_field.f90 +++ b/src/io_hdf5/hdf5_output_field.f90 @@ -96,10 +96,10 @@ SUBROUTINE WriteDielectricGlobalToHDF5() FileName=TRIM(TIMESTAMP(TRIM(ProjectName)//'_DielectricGlobal',OutputTime))//'.h5' IF(MPIRoot) CALL GenerateFileSkeleton('DielectricGlobal',N_variables,StrVarNames,TRIM(MeshFile),OutputTime) #if USE_MPI - CALL MPI_BARRIER(MPI_COMM_WORLD,iError) + CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif IF(MPIRoot)THEN - CALL OpenDataFile(FileName,create=.FALSE.,single=.TRUE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(FileName,create=.FALSE.,single=.TRUE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) CALL WriteAttributeToHDF5(File_ID,'VarNamesDielectricGlobal',N_variables,StrArray=StrVarNames) CALL CloseDataFile() END IF ! MPIRoot @@ -175,9 +175,9 @@ SUBROUTINE WriteBRAverageElemToHDF5(isBRAverageElem) FileName=TRIM(TIMESTAMP(TRIM(ProjectName)//'_BRAverageElem',OutputTime))//'.h5' IF(MPIRoot) CALL GenerateFileSkeleton('BRAverageElem',N_variables,StrVarNames,TRIM(MeshFile),OutputTime) #if USE_MPI - CALL MPI_BARRIER(MPI_COMM_WORLD,iError) + CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif - CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) CALL WriteAttributeToHDF5(File_ID,'VarNamesBRAverageElem',N_variables,StrArray=StrVarNames) CALL CloseDataFile() @@ -258,9 +258,9 @@ SUBROUTINE WritePMLzetaGlobalToHDF5() FileName=TRIM(TIMESTAMP(TRIM(ProjectName)//'_PMLZetaGlobal',OutputTime))//'.h5' IF(MPIRoot) CALL GenerateFileSkeleton('PMLZetaGlobal',N_variables,StrVarNames,TRIM(MeshFile),OutputTime) #if USE_MPI - CALL MPI_BARRIER(MPI_COMM_WORLD,iError) + CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif - CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) CALL WriteAttributeToHDF5(File_ID,'VarNamesPMLzetaGlobal',N_variables,StrArray=StrVarNames) CALL CloseDataFile() @@ -372,9 +372,9 @@ SUBROUTINE WriteBGFieldToHDF5(OutputTime) CALL copy_userblock(TRIM(FileName)//C_NULL_CHAR,TRIM(UserblockTmpFile)//C_NULL_CHAR) END IF #if USE_MPI -CALL MPI_BARRIER(MPI_COMM_WORLD,iError) +CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ -CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) nVal=nGlobalElems ! For the MPI case this must be replaced by the global number of elements (sum over all procs) @@ -477,9 +477,9 @@ SUBROUTINE WriteBGFieldAnalyticToHDF5() CALL copy_userblock(TRIM(FileName)//C_NULL_CHAR,TRIM(UserblockTmpFile)//C_NULL_CHAR) END IF #if USE_MPI -CALL MPI_BARRIER(MPI_COMM_WORLD,iError) +CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ -CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) nVal=nGlobalElems ! For the MPI case this must be replaced by the global number of elements (sum over all procs) @@ -663,9 +663,9 @@ SUBROUTINE WriteErrorNormsToHDF5(OutputTime) CALL copy_userblock(TRIM(FileName)//C_NULL_CHAR,TRIM(UserblockTmpFile)//C_NULL_CHAR) END IF #if USE_MPI -CALL MPI_BARRIER(MPI_COMM_WORLD,iError) +CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ -CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) nVal=nGlobalElems ! For the MPI case this must be replaced by the global number of elements (sum over all procs) diff --git a/src/io_hdf5/hdf5_output_particle.f90 b/src/io_hdf5/hdf5_output_particle.f90 index 106243fbc..71621cf8e 100644 --- a/src/io_hdf5/hdf5_output_particle.f90 +++ b/src/io_hdf5/hdf5_output_particle.f90 @@ -182,10 +182,10 @@ SUBROUTINE WriteNodeSourceExtToHDF5(OutputTime) FileName=TRIM(TIMESTAMP(TRIM(ProjectName)//'_NodeSourceExtGlobal',OutputTime))//'.h5' IF(MPIRoot) CALL GenerateFileSkeleton('NodeSourceExtGlobal',N_variables,StrVarNames,TRIM(MeshFile),OutputTime) #if USE_MPI - CALL MPI_BARRIER(MPI_COMM_WORLD,iError) + CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif IF(MPIRoot)THEN - CALL OpenDataFile(FileName,create=.FALSE.,single=.TRUE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(FileName,create=.FALSE.,single=.TRUE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) CALL WriteAttributeToHDF5(File_ID,'VarNamesNodeSourceExtGlobal',N_variables,StrArray=StrVarNames) CALL CloseDataFile() END IF ! MPIRoot @@ -277,9 +277,6 @@ SUBROUTINE WriteParticleToHDF5(FileName) USE MOD_Particle_Vars ,ONLY: PartInt,PartData,PartDataSize,locnPart,offsetnPart,PartIntSize,PartDataVarNames USE MOD_part_tools ,ONLY: UpdateNextFreePosition USE MOD_DSMC_Vars ,ONLY: UseDSMC, DSMC -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ #if USE_LOADBALANCE USE MOD_LoadBalance_Vars ,ONLY: PerformLoadBalance,UseH5IOLoadBalance #endif /*USE_LOADBALANCE*/ @@ -360,7 +357,7 @@ SUBROUTINE WriteParticleToHDF5(FileName) nVal = (/PartDataSize , locnPart /) , & offset = (/0_IK , offsetnPart/) , & collective = UseCollectiveIO , offSetDim= 2 , & - communicator = PartMPI%COMM , RealArray= PartData) + communicator = MPI_COMM_PICLAS , RealArray= PartData) ! Output of the element-wise time step as a separate container in state file IF(VarTimeStep%UseDistribution) THEN CALL DistributedWriteArray(FileName , & @@ -369,7 +366,7 @@ SUBROUTINE WriteParticleToHDF5(FileName) nVal = (/PP_nElems , 1_IK/) , & offset = (/offsetElem , 0_IK/) , & collective = UseCollectiveIO , offSetDim = 1 , & - communicator = PartMPI%COMM , RealArray = VarTimeStep%ElemFac) + communicator = MPI_COMM_PICLAS , RealArray = VarTimeStep%ElemFac) END IF #else CALL OpenDataFile(FileName,create=.FALSE.,single=.TRUE.,readOnly=.FALSE.) @@ -414,7 +411,7 @@ SUBROUTINE WriteParticleToHDF5(FileName) nVal = (/MaxQuantNum , locnPart /) , & offset = (/0_IK , offsetnPart /) , & collective = UseCollectiveIO , offSetDim = 2 , & - communicator = PartMPI%COMM , IntegerArray_i4 = VibQuantData) + communicator = MPI_COMM_PICLAS , IntegerArray_i4 = VibQuantData) #else CALL OpenDataFile(FileName,create=.FALSE.,single=.TRUE.,readOnly=.FALSE.) CALL WriteArrayToHDF5(DataSetName = 'VibQuantData' , rank = 2 , & @@ -459,7 +456,7 @@ SUBROUTINE WriteParticleToHDF5(FileName) nVal = (/MaxElecQuant , locnPart /) , & offset = (/0_IK , offsetnPart /) , & collective = UseCollectiveIO , offSetDim = 2 , & - communicator = PartMPI%COMM , RealArray = ElecDistriData) + communicator = MPI_COMM_PICLAS , RealArray = ElecDistriData) #else CALL OpenDataFile(FileName,create=.FALSE.,single=.TRUE.,readOnly=.FALSE.) CALL WriteArrayToHDF5(DataSetName = 'ElecDistriData' , rank = 2 , & @@ -502,7 +499,7 @@ SUBROUTINE WriteParticleToHDF5(FileName) nVal = (/3_IK , locnPart /) , & offset = (/0_IK , offsetnPart /) , & collective = UseCollectiveIO , offSetDim = 2 , & - communicator = PartMPI%COMM , RealArray = AD_Data) + communicator = MPI_COMM_PICLAS , RealArray = AD_Data) #else CALL OpenDataFile(FileName,create=.FALSE.,single=.TRUE.,readOnly=.FALSE.) CALL WriteArrayToHDF5(DataSetName = 'ADVeloData' , rank = 2 , & @@ -552,9 +549,6 @@ SUBROUTINE WriteBoundaryParticleToHDF5(MeshFileName,OutputTime,PreviousTime) USE MOD_Particle_Boundary_Vars ,ONLY: PartStateBoundary,PartStateBoundaryVecLength,nVarPartStateBoundary USE MOD_Particle_Analyze_Tools ,ONLY: CalcEkinPart2 USE MOD_TimeDisc_Vars ,ONLY: iter -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !----------------------------------------------------------------------------------------------------------------------------------- @@ -606,7 +600,7 @@ SUBROUTINE WriteBoundaryParticleToHDF5(MeshFileName,OutputTime,PreviousTime) ! Reopen file and write DG solution #if USE_MPI -CALL MPI_BARRIER(MPI_COMM_WORLD,iError) +CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif ! 3xPos [m], 3xvelo [m/s], species [-] @@ -720,7 +714,7 @@ SUBROUTINE WriteBoundaryParticleToHDF5(MeshFileName,OutputTime,PreviousTime) nVal = (/ PartDataSizeLoc, locnPart /) , & offset = (/ 0_IK , offsetnPart /) , & collective = UseCollectiveIO , offSetDim = 2 , & - communicator = PartMPI%COMM , RealArray = PartData) + communicator = MPI_COMM_PICLAS , RealArray = PartData) #else CALL OpenDataFile(FileName,create=.FALSE.,single=.TRUE.,readOnly=.FALSE.) CALL WriteArrayToHDF5(DataSetName = 'PartData' , rank = 2 , & @@ -763,9 +757,6 @@ SUBROUTINE WriteLostParticlesToHDF5(MeshFileName,OutputTime) USE MOD_Particle_Tracking_Vars ,ONLY: TotalNbrOfMissingParticlesSum USE MOD_Equation_Vars ,ONLY: StrVarNames USE MOD_Particle_Analyze_Tools ,ONLY: CalcEkinPart2 -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !----------------------------------------------------------------------------------------------------------------------------------- @@ -803,7 +794,7 @@ SUBROUTINE WriteLostParticlesToHDF5(MeshFileName,OutputTime) ! Reopen file and write DG solution #if USE_MPI -CALL MPI_BARRIER(MPI_COMM_WORLD,iError) +CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif ! Set number of local particles @@ -914,7 +905,7 @@ SUBROUTINE WriteLostParticlesToHDF5(MeshFileName,OutputTime) nVal = (/ PartLostDataSize , locnPart /) , & offset = (/ 0_IK , offsetnPart /) , & collective = UseCollectiveIO , offSetDim = 2 , & - communicator = PartMPI%COMM , RealArray = PartData) + communicator = MPI_COMM_PICLAS , RealArray = PartData) #else CALL OpenDataFile(FileName,create=.FALSE.,single=.TRUE.,readOnly=.FALSE.) CALL WriteArrayToHDF5(DataSetName = 'PartData' , rank = 2 , & @@ -1011,7 +1002,7 @@ SUBROUTINE WriteAdaptiveInfoToHDF5(FileName) END DO WRITE(H5_Name,'(A)') 'AdaptiveInfo' -CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) ! Associate construct for integer KIND=8 possibility ASSOCIATE (& @@ -1084,7 +1075,7 @@ SUBROUTINE WriteAdaptiveRunningAverageToHDF5(FileName) CALL CloseDataFile() END IF -CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) ! Associate construct for integer KIND=8 possibility ASSOCIATE (& AdaptBCSampleElemNumGlobal => INT(AdaptBCSampleElemNumGlobal,IK) ,& @@ -1122,9 +1113,6 @@ SUBROUTINE WriteAdaptBCPartNumOutToHDF5(FileName) USE MOD_Restart_Vars ,ONLY: DoRestart USE MOD_Particle_Vars ,ONLY: nSpecies, Species USE MOD_Particle_Sampling_Vars ,ONLY: AdaptBCPartNumOut -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !----------------------------------------------------------------------------------------------------------------------------------- @@ -1146,9 +1134,9 @@ SUBROUTINE WriteAdaptBCPartNumOutToHDF5(FileName) #if USE_MPI IF(MPIRoot)THEN ALLOCATE(AdaptBCPartNumOutTemp(1:nSpecies,1:nSurfacefluxBCs)) - CALL MPI_REDUCE(AdaptBCPartNumOut,AdaptBCPartNumOutTemp,nSpecies*nSurfacefluxBCs,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM,IERROR) + CALL MPI_REDUCE(AdaptBCPartNumOut,AdaptBCPartNumOutTemp,nSpecies*nSurfacefluxBCs,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) ELSE - CALL MPI_REDUCE(AdaptBCPartNumOut,MPI_IN_PLACE ,nSpecies*nSurfacefluxBCs,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM,IERROR) + CALL MPI_REDUCE(AdaptBCPartNumOut,MPI_IN_PLACE ,nSpecies*nSurfacefluxBCs,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) END IF #endif @@ -1275,7 +1263,7 @@ SUBROUTINE WriteVibProbInfoToHDF5(FileName) END IF WRITE(H5_Name,'(A)') 'VibProbInfo' - CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) ! Associate construct for integer KIND=8 possibility ASSOCIATE (& @@ -1293,7 +1281,7 @@ SUBROUTINE WriteVibProbInfoToHDF5(FileName) SDEALLOCATE(StrVarNames) ELSE ! DSMC%VibRelaxProb < 2.0 #if USE_MPI - CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) #else CALL OpenDataFile(FileName,create=.FALSE.,single=.TRUE.,readOnly=.FALSE.) #endif @@ -1483,7 +1471,7 @@ SUBROUTINE WriteClonesToHDF5(FileName) END IF #if USE_MPI -CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) #else CALL OpenDataFile(FileName,create=.FALSE.,single=.TRUE.,readOnly=.FALSE.) #endif @@ -1612,9 +1600,9 @@ SUBROUTINE WriteElectroMagneticPICFieldToHDF5() CALL copy_userblock(TRIM(FileName)//C_NULL_CHAR,TRIM(UserblockTmpFile)//C_NULL_CHAR) END IF #if USE_MPI -CALL MPI_BARRIER(MPI_COMM_WORLD,iError) +CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ -CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) nVal=nGlobalElems ! For the MPI case this must be replaced by the global number of elements (sum over all procs) @@ -1659,7 +1647,6 @@ SUBROUTINE WriteEmissionVariablesToHDF5(FileName) USE MOD_Globals !USE MOD_PreProc USE MOD_Particle_Vars ,ONLY: Species,nSpecies -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_Particle_Vars ,ONLY: NeutralizationBalanceGlobal ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE @@ -1675,7 +1662,7 @@ SUBROUTINE WriteEmissionVariablesToHDF5(FileName) INTEGER(KIND=IK) :: NeutralizationBalanceTmp(1:1) ! This is a dummy array of size 1 ! !=================================================================================================================================== ! Only root writes the data -IF(.NOT.PartMPI%MPIRoot) RETURN +IF(.NOT.MPIRoot) RETURN ! Loop over all species and inits DO iSpec=1,nSpecies @@ -1718,9 +1705,6 @@ SUBROUTINE GetOffsetAndGlobalNumberOfParts(CallingRoutine,offsetnPart,globnPart, ! MODULES USE MOD_PreProc USE MOD_Globals -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !----------------------------------------------------------------------------------------------------------------------------------- @@ -1742,11 +1726,11 @@ SUBROUTINE GetOffsetAndGlobalNumberOfParts(CallingRoutine,offsetnPart,globnPart, #if USE_MPI locnPart8 = INT(locnPart,8) locnPart8Recv = 0_IK -CALL MPI_EXSCAN(locnPart8,locnPart8Recv,1,MPI_INTEGER8,MPI_SUM,MPI_COMM_WORLD,iError) +CALL MPI_EXSCAN(locnPart8,locnPart8Recv,1,MPI_INTEGER8,MPI_SUM,MPI_COMM_PICLAS,iError) offsetnPart = INT(locnPart8Recv,KIND=IK) ! Last proc calculates the global number and broadcasts it IF(myrank.EQ.nProcessors-1) locnPart8=locnPart8Recv+locnPart8 -CALL MPI_BCAST(locnPart8,1,MPI_INTEGER8,nProcessors-1,MPI_COMM_WORLD,iError) +CALL MPI_BCAST(locnPart8,1,MPI_INTEGER8,nProcessors-1,MPI_COMM_PICLAS,iError) !global numbers globnPart8=locnPart8 GlobalNbrOfParticlesUpdated = .TRUE. @@ -1766,12 +1750,12 @@ SUBROUTINE GetOffsetAndGlobalNumberOfParts(CallingRoutine,offsetnPart,globnPart, SimNumSpecMin = 0 SimNumSpecMax = 0 IF(GetMinMaxNbrOfParticles)THEN - IF (PartMPI%MPIRoot) THEN - CALL MPI_REDUCE(locnPart , SimNumSpecMin , 1 , MPI_INTEGER_INT_KIND , MPI_MIN , 0 , PartMPI%COMM , IERROR) - CALL MPI_REDUCE(locnPart , SimNumSpecMax , 1 , MPI_INTEGER_INT_KIND , MPI_MAX , 0 , PartMPI%COMM , IERROR) + IF (MPIRoot) THEN + CALL MPI_REDUCE(locnPart , SimNumSpecMin , 1 , MPI_INTEGER_INT_KIND , MPI_MIN , 0 , MPI_COMM_PICLAS , IERROR) + CALL MPI_REDUCE(locnPart , SimNumSpecMax , 1 , MPI_INTEGER_INT_KIND , MPI_MAX , 0 , MPI_COMM_PICLAS , IERROR) ELSE - CALL MPI_REDUCE(locnPart , 0 , 1 , MPI_INTEGER_INT_KIND , MPI_MIN , 0 , PartMPI%COMM , IERROR) - CALL MPI_REDUCE(locnPart , 0 , 1 , MPI_INTEGER_INT_KIND , MPI_MAX , 0 , PartMPI%COMM , IERROR) + CALL MPI_REDUCE(locnPart , 0 , 1 , MPI_INTEGER_INT_KIND , MPI_MIN , 0 , MPI_COMM_PICLAS , IERROR) + CALL MPI_REDUCE(locnPart , 0 , 1 , MPI_INTEGER_INT_KIND , MPI_MAX , 0 , MPI_COMM_PICLAS , IERROR) END IF END IF ! GetMinMaxNbrOfParticles diff --git a/src/io_hdf5/hdf5_output_state.f90 b/src/io_hdf5/hdf5_output_state.f90 index 544b15a6a..af92f7d8c 100644 --- a/src/io_hdf5/hdf5_output_state.f90 +++ b/src/io_hdf5/hdf5_output_state.f90 @@ -238,7 +238,7 @@ SUBROUTINE WriteStateToHDF5(MeshFileName,OutputTime,PreviousTime) ! Reopen file and write DG solution #if USE_MPI -CALL MPI_BARRIER(MPI_COMM_WORLD,iError) +CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif ! Associate construct for integer KIND=8 possibility @@ -492,7 +492,7 @@ SUBROUTINE WriteStateToHDF5(MeshFileName,OutputTime,PreviousTime) #ifdef PARTICLES ! output of last source term #if USE_MPI - CALL MPI_BARRIER(MPI_COMM_WORLD,iError) + CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ IF(OutputSource) THEN #if USE_HDG @@ -564,7 +564,7 @@ SUBROUTINE WriteStateToHDF5(MeshFileName,OutputTime,PreviousTime) IF(RadialWeighting%PerformCloning) CALL WriteClonesToHDF5(FileName) IF (PartBound%OutputWallTemp) CALL WriteAdaptiveWallTempToHDF5(FileName) #if USE_MPI -CALL MPI_BARRIER(MPI_COMM_WORLD,iError) +CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ ! For restart purposes, store the electron bulk temperature in .h5 state ! Only root writes the container diff --git a/src/linearsolver/linearoperator/linearoperator.f90 b/src/linearsolver/linearoperator/linearoperator.f90 index 65c1eee4a..1ee47f069 100644 --- a/src/linearsolver/linearoperator/linearoperator.f90 +++ b/src/linearsolver/linearoperator/linearoperator.f90 @@ -311,7 +311,7 @@ SUBROUTINE VectorDotProduct(a,b,resu) #if USE_MPI ResuSend=Resu - CALL MPI_ALLREDUCE(ResuSend,resu,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,iError) + CALL MPI_ALLREDUCE(ResuSend,resu,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_PICLAS,iError) #endif END SUBROUTINE VectorDotProduct diff --git a/src/linearsolver/linearsolver.f90 b/src/linearsolver/linearsolver.f90 index bd8c36bf3..66cf8b44a 100644 --- a/src/linearsolver/linearsolver.f90 +++ b/src/linearsolver/linearsolver.f90 @@ -170,7 +170,7 @@ SUBROUTINE InitLinearSolver() #if !(USE_HDG) nDofGlobalMPI=nDofGlobal #if USE_MPI - CALL MPI_ALLREDUCE(MPI_IN_PLACE,nDofGlobalMPI,1,MPI_INTEGER,MPI_SUM,MPI_COMM_WORLD,iError) + CALL MPI_ALLREDUCE(MPI_IN_PLACE,nDofGlobalMPI,1,MPI_INTEGER,MPI_SUM,MPI_COMM_PICLAS,iError) #endif eps_LinearSolver = GETREAL('eps_LinearSolver','1e-3') diff --git a/src/linearsolver/newton.f90 b/src/linearsolver/newton.f90 index 9c8832d96..db31953c4 100644 --- a/src/linearsolver/newton.f90 +++ b/src/linearsolver/newton.f90 @@ -172,7 +172,7 @@ SUBROUTINE ImplicitNorm(t,coeff,R,Norm_R,Delta_Norm_R,Delta_Norm_Rel,First) NormArray(1)=Norm_R NormArray(2)=Delta_Norm_R NormArray(3)=Delta_Norm_Rel -CALL MPI_ALLREDUCE(NormArray,GlobalNormArray,3,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,iError) +CALL MPI_ALLREDUCE(NormArray,GlobalNormArray,3,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_PICLAS,iError) Norm_R = SQRT(GlobalNormArray(1)) Delta_Norm_R = SQRT(GlobalNormArray(2)) Delta_Norm_Rel = SQRT(GlobalNormArray(3)) diff --git a/src/linearsolver/particlesolver.f90 b/src/linearsolver/particlesolver.f90 index 02ef9d0a6..8b6a8de2e 100644 --- a/src/linearsolver/particlesolver.f90 +++ b/src/linearsolver/particlesolver.f90 @@ -159,9 +159,6 @@ SUBROUTINE SelectImplicitParticles() USE MOD_TimeDisc_Vars ,ONLY: dt,nRKStages,iter USE MOD_Globals_Vars ,ONLY: c2_inv USE MOD_LinearSolver_Vars ,ONLY: DoPrintConvInfo -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ !----------------------------------------------------------------------------------------------------------------------------------! ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE @@ -232,12 +229,12 @@ SUBROUTINE SelectImplicitParticles() IF(.NOT.PartIsImplicit(iPart)) nExp=nExp+1 END DO #if USE_MPI - IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,nExp,1,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM, IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE,nImp,1,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM, IERROR) + IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE,nExp,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,nImp,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) ELSE - CALL MPI_REDUCE(nExp ,iPart,1,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM, IERROR) - CALL MPI_REDUCE(nImp ,iPart,1,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM, IERROR) + CALL MPI_REDUCE(nExp ,iPart,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(nImp ,iPart,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) END IF #endif /*USE_MPI*/ SWRITE(UNIT_StdOut,'(A,I0,x,I0)') ' Particles explicit/implicit ', nExp, nImp @@ -264,7 +261,6 @@ SUBROUTINE ParticleNewton(t,coeff,Mode,doParticle_In,opt_In,AbortTol_In) USE MOD_Part_RHS ,ONLY: CalcPartRHS #if USE_MPI USE MOD_Particle_MPI ,ONLY: IRecvNbOfParticles, MPIParticleSend,MPIParticleRecv,SendNbOfparticles -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI #if USE_LOADBALANCE USE MOD_LoadBalance_Timers ,ONLY: LBStartTime,LBPauseTime,LBSplitTime #endif /*USE_LOADBALANCE*/ @@ -422,7 +418,7 @@ SUBROUTINE ParticleNewton(t,coeff,Mode,doParticle_In,opt_In,AbortTol_In) IF(ANY(DoPartInNewton)) DoNewton=.TRUE. #if USE_MPI !set T if at least 1 proc has to do newton -CALL MPI_ALLREDUCE(MPI_IN_PLACE,DoNewton,1,MPI_LOGICAL,MPI_LOR,PartMPI%COMM,iError) +CALL MPI_ALLREDUCE(MPI_IN_PLACE,DoNewton,1,MPI_LOGICAL,MPI_LOR,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ IF(DoPrintConvInfo)THEN @@ -435,7 +431,7 @@ SUBROUTINE ParticleNewton(t,coeff,Mode,doParticle_In,opt_In,AbortTol_In) END DO ! iPart #if USE_MPI !set T if at least 1 proc has to do newton - CALL MPI_ALLREDUCE(MPI_IN_PLACE,Counter,1,MPI_INTEGER,MPI_SUM,PartMPI%COMM,iError) + CALL MPI_ALLREDUCE(MPI_IN_PLACE,Counter,1,MPI_INTEGER,MPI_SUM,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ SWRITE(UNIT_StdOut,'(A,I0)') ' Initial particle number in newton: ',Counter END IF @@ -487,7 +483,7 @@ SUBROUTINE ParticleNewton(t,coeff,Mode,doParticle_In,opt_In,AbortTol_In) IF(ANY(DoPartInNewton)) DoNewton=.TRUE. #if USE_MPI !set T if at least 1 proc has to do newton - CALL MPI_ALLREDUCE(MPI_IN_PLACE,DoNewton,1,MPI_LOGICAL,MPI_LOR,PartMPI%COMM,iError) + CALL MPI_ALLREDUCE(MPI_IN_PLACE,DoNewton,1,MPI_LOGICAL,MPI_LOR,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ IF(DoPrintConvInfo)THEN Counter=0 @@ -498,7 +494,7 @@ SUBROUTINE ParticleNewton(t,coeff,Mode,doParticle_In,opt_In,AbortTol_In) END DO ! iPart #if USE_MPI !set T if at least 1 proc has to do newton - CALL MPI_ALLREDUCE(MPI_IN_PLACE,Counter,1,MPI_INTEGER,MPI_SUM,PartMPI%COMM,iError) + CALL MPI_ALLREDUCE(MPI_IN_PLACE,Counter,1,MPI_INTEGER,MPI_SUM,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ END IF END DO @@ -706,7 +702,6 @@ SUBROUTINE Particle_Armijo(t,coeff,AbortTol,nInnerPartNewton) USE MOD_LinearSolver_Vars ,ONLY: DoFullNewton!,PartNewtonRelaxation #if USE_MPI USE MOD_Particle_MPI ,ONLY: IRecvNbOfParticles, MPIParticleSend,MPIParticleRecv,SendNbOfparticles -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI #if USE_LOADBALANCE USE MOD_LoadBalance_Timers ,ONLY: LBStartTime,LBPauseTime,LBSplitTime #endif /*USE_LOADBALANCE*/ @@ -950,7 +945,7 @@ SUBROUTINE Particle_Armijo(t,coeff,AbortTol,nInnerPartNewton) CALL LBStartTime(tLBStart) #endif /*USE_LOADBALANCE*/ !set T if at least 1 proc has to do newton -CALL MPI_ALLREDUCE(MPI_IN_PLACE,DoSetLambda,1,MPI_LOGICAL,MPI_LOR,PartMPI%COMM,iError) +CALL MPI_ALLREDUCE(MPI_IN_PLACE,DoSetLambda,1,MPI_LOGICAL,MPI_LOR,MPI_COMM_PICLAS,iError) #if USE_LOADBALANCE CALL LBSplitTime(LB_PARTCOMM,tLBStart) #endif /*USE_LOADBALANCE*/ @@ -1129,7 +1124,7 @@ SUBROUTINE Particle_Armijo(t,coeff,AbortTol,nInnerPartNewton) IF(ANY(.NOT.PartLambdaAccept)) DoSetLambda=.TRUE. #if USE_MPI !set T if at least 1 proc has to do newton - CALL MPI_ALLREDUCE(MPI_IN_PLACE,DoSetLambda,1,MPI_LOGICAL,MPI_LOR,PartMPI%COMM,iError) + CALL MPI_ALLREDUCE(MPI_IN_PLACE,DoSetLambda,1,MPI_LOGICAL,MPI_LOR,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ iCounter=0 DO iPart=1,PDM%ParticleVecLength @@ -1141,7 +1136,7 @@ SUBROUTINE Particle_Armijo(t,coeff,AbortTol,nInnerPartNewton) IF(DoPrintConvInfo)THEN #if USE_MPI !set T if at least 1 proc has to do newton - CALL MPI_ALLREDUCE(MPI_IN_PLACE,iCounter,1,MPI_INTEGER,MPI_SUM,PartMPI%COMM,iError) + CALL MPI_ALLREDUCE(MPI_IN_PLACE,iCounter,1,MPI_INTEGER,MPI_SUM,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ SWRITE(UNIT_stdOut,'(A20,2x,L,2x,I10)') ' Accept?: ', DoSetLambda,iCounter END IF diff --git a/src/loadbalance/loadbalance.f90 b/src/loadbalance/loadbalance.f90 index 25d3ca63a..9b7c30f3f 100644 --- a/src/loadbalance/loadbalance.f90 +++ b/src/loadbalance/loadbalance.f90 @@ -517,15 +517,15 @@ SUBROUTINE ComputeImbalance() #else ! Collect ElemTime for particles and field separately (only on root process) ! Skip the reduce for DSMC timedisc - CALL MPI_REDUCE(ElemTimeField , ElemTimeFieldTot , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) + CALL MPI_REDUCE(ElemTimeField , ElemTimeFieldTot , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) WeightSum = ElemTimeFieldTot ! only correct on MPI root #endif /*(PP_TimeDiscMethod==4) || (PP_TimeDiscMethod==42) || (PP_TimeDiscMethod==300) || (PP_TimeDiscMethod==400)*/ #ifdef PARTICLES - CALL MPI_REDUCE(ElemTimePart , ElemTimePartTot , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IERROR) + CALL MPI_REDUCE(ElemTimePart , ElemTimePartTot , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) WeightSum = WeightSum + ElemTimePartTot ! only correct on MPI root #endif /*PARTICLES*/ ! send WeightSum from MPI root to all other procs - CALL MPI_BCAST(WeightSum,1,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,iError) + CALL MPI_BCAST(WeightSum,1,MPI_DOUBLE_PRECISION,0,MPI_COMM_PICLAS,iError) ! Sanity check IF(.NOT.ISFINITE(WeightSum)) CALL abort(__STAMP__,'Loadbalance: WeightSum is infinite!') @@ -536,9 +536,9 @@ SUBROUTINE ComputeImbalance() IPWRITE(*,*) 'Info: The measured time of all elems is zero. ALMOSTZERO(WeightSum)=.TRUE., SUM(ElemTime)=',WeightSum_loc END IF - !CALL MPI_ALLREDUCE(WeightSum_loc,TargetWeight,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,iError) - CALL MPI_ALLREDUCE(WeightSum_loc,MaxWeight ,1,MPI_DOUBLE_PRECISION,MPI_MAX,MPI_COMM_WORLD,iError) - CALL MPI_ALLREDUCE(WeightSum_loc,MinWeight ,1,MPI_DOUBLE_PRECISION,MPI_MIN,MPI_COMM_WORLD,iError) + !CALL MPI_ALLREDUCE(WeightSum_loc,TargetWeight,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_PICLAS,iError) + CALL MPI_ALLREDUCE(WeightSum_loc,MaxWeight ,1,MPI_DOUBLE_PRECISION,MPI_MAX,MPI_COMM_PICLAS,iError) + CALL MPI_ALLREDUCE(WeightSum_loc,MinWeight ,1,MPI_DOUBLE_PRECISION,MPI_MIN,MPI_COMM_PICLAS,iError) !WeightSum = TargetWeight ! Set total weight for writing to file !IF(MPIRoot)THEN diff --git a/src/mesh/mesh.f90 b/src/mesh/mesh.f90 index 8e6761d02..a6ea69782 100644 --- a/src/mesh/mesh.f90 +++ b/src/mesh/mesh.f90 @@ -930,7 +930,7 @@ SUBROUTINE setSideRanges() USE MOD_Mesh_Vars ,ONLY: nGlobalUniqueSidesFromMesh,nGlobalUniqueSides,nMortarMPISides,nUniqueSides #if USE_MPI USE MOD_Globals ,ONLY: myrank -USE MOD_Globals ,ONLY: iError,MPI_COMM_WORLD +USE MOD_Globals ,ONLY: iError USE mpi #endif /*USE_MPI*/ #endif /*USE_HDG*/ diff --git a/src/mesh/mesh_readin.f90 b/src/mesh/mesh_readin.f90 index 52a7e9f74..4ff69cd50 100644 --- a/src/mesh/mesh_readin.f90 +++ b/src/mesh/mesh_readin.f90 @@ -283,7 +283,7 @@ SUBROUTINE ReadMesh(FileString,ReadNodes) GETTIME(StartT) ! Get ElemInfo from Mesh file - CALL OpenDataFile(FileString,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(FileString,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL GetDataSize(File_ID,'ElemInfo',nDims,HSize) CALL ReadAttribute(File_ID,'nUniqueSides',1,IntScalar=nGlobalUniqueSidesFromMesh) CALL ReadAttribute(File_ID,'nSides',1,IntScalar=nNonUniqueGlobalSides) @@ -1007,7 +1007,7 @@ SUBROUTINE ReadMeshNodes() offsetNodeID => INT(offsetNodeID,IK) ) ALLOCATE(NodeCoords_indx(3,nNodeIDs)) ! read all nodes - CALL OpenDataFile(MeshFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(MeshFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL ReadArray('NodeCoords',2,(/3_IK,nNodeIDs/),offsetNodeID,2,RealArray=NodeCoords_indx) CALL CloseDataFile() END ASSOCIATE @@ -1020,7 +1020,7 @@ SUBROUTINE ReadMeshNodes() nNodeIDs => INT(nNodeIDs,IK) ,& offsetNodeID => INT(offsetNodeID,IK) ) ALLOCATE(NodeInfo(FirstNodeInd:LastNodeInd)) - CALL OpenDataFile(MeshFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(MeshFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL ReadArray('GlobalNodeIDs',1,(/nNodeIDs/),offsetNodeID,1,IntegerArray_i4=NodeInfo) CALL CloseDataFile() END ASSOCIATE diff --git a/src/mesh/mesh_tools.f90 b/src/mesh/mesh_tools.f90 index aee6bafca..5cfda3979 100644 --- a/src/mesh/mesh_tools.f90 +++ b/src/mesh/mesh_tools.f90 @@ -408,7 +408,7 @@ END FUNCTION GetGlobalSide2CNTotalSide SUBROUTINE GetMasteriLocSides() ! MODULES USE MOD_PreProc -USE MOD_globals ,ONLY: abort,MPI_COMM_WORLD +USE MOD_globals ,ONLY: abort USE MOD_Mesh_Vars ,ONLY: MortarType,SideToElem,MortarInfo USE MOD_Mesh_Vars ,ONLY: firstMortarInnerSide,lastMortarInnerSide USE MOD_HDG_Vars ,ONLY: nGP_face, iLocSides diff --git a/src/mpi/mpi.f90 b/src/mpi/mpi.f90 index 5e4c10f40..1266098d3 100644 --- a/src/mpi/mpi.f90 +++ b/src/mpi/mpi.f90 @@ -194,16 +194,19 @@ SUBROUTINE InitMPIvars() RecRequest_Geo = MPI_REQUEST_NULL DataSizeSide =(PP_N+1)*(PP_N+1) +! General communicator +CALL MPI_COMM_DUP (MPI_COMM_WORLD,MPI_COMM_PICLAS,iError) + ! split communicator into smaller groups (e.g. for local nodes) GroupSize=GETINT('GroupSize','0') IF(GroupSize.LT.1)THEN ! group procs by node ! Split the node communicator (shared memory) from the global communicator on physical processor or node level #if (CORE_SPLIT==1) - CALL MPI_COMM_SPLIT(MPI_COMM_WORLD,myRank,0,MPI_COMM_NODE,iError) + CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS,myRank,0,MPI_COMM_NODE,iError) #elif (CORE_SPLIT==0) ! Note that using SharedMemoryMethod=OMPI_COMM_TYPE_CORE somehow does not work in every case (intel/amd processors) ! Also note that OMPI_COMM_TYPE_CORE is undefined when not using OpenMPI - CALL MPI_COMM_SPLIT_TYPE(MPI_COMM_WORLD,SharedMemoryMethod,0,MPI_INFO_NULL,MPI_COMM_NODE,IERROR) + CALL MPI_COMM_SPLIT_TYPE(MPI_COMM_PICLAS,SharedMemoryMethod,0,MPI_INFO_NULL,MPI_COMM_NODE,IERROR) #else ! Check if more nodes than procs are required or ! if the resulting split would create unequal procs per node @@ -216,11 +219,11 @@ SUBROUTINE InitMPIvars() ! Group procs so that every CORE_SPLIT procs are in the same group color = INT(REAL(myrank*CORE_SPLIT)/REAL(nProcessors_Global))+1 END IF ! (CORE_SPLIT.GE.nProcessors_Global).OR.(MOD().GT.0) - CALL MPI_COMM_SPLIT(MPI_COMM_WORLD,color,0,MPI_COMM_NODE,iError) + CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS,color,0,MPI_COMM_NODE,iError) #endif ELSE ! use groupsize color=myRank/GroupSize - CALL MPI_COMM_SPLIT(MPI_COMM_WORLD,color,0,MPI_COMM_NODE,iError) + CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS,color,0,MPI_COMM_NODE,iError) END IF CALL MPI_COMM_RANK(MPI_COMM_NODE,myLocalRank,iError) CALL MPI_COMM_SIZE(MPI_COMM_NODE,nLocalProcs,iError) @@ -240,12 +243,12 @@ SUBROUTINE InitMPIvars() myLeaderRank=-1 myWorkerRank=-1 IF(myLocalRank.EQ.0)THEN - CALL MPI_COMM_SPLIT(MPI_COMM_WORLD,0,0,MPI_COMM_LEADERS,iError) + CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS,0,0,MPI_COMM_LEADERS,iError) CALL MPI_COMM_RANK( MPI_COMM_LEADERS,myLeaderRank,iError) CALL MPI_COMM_SIZE( MPI_COMM_LEADERS,nLeaderProcs,iError) nWorkerProcs=nProcessors-nLeaderProcs ELSE - CALL MPI_COMM_SPLIT(MPI_COMM_WORLD,1,0,MPI_COMM_WORKERS,iError) + CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS,1,0,MPI_COMM_WORKERS,iError) CALL MPI_COMM_RANK( MPI_COMM_WORKERS,myWorkerRank,iError) CALL MPI_COMM_SIZE( MPI_COMM_WORKERS,nWorkerProcs,iError) nLeaderProcs=nProcessors-nWorkerProcs @@ -284,7 +287,7 @@ SUBROUTINE StartReceiveMPIData(firstDim,FaceData,LowerBound,UpperBound,MPIReques SideID_start=OffsetMPISides_rec(iNbProc-1,SendID)+1 SideID_end =OffsetMPISides_rec(iNbProc,SendID) CALL MPI_IRECV(FaceData(:,:,:,SideID_start:SideID_end),nRecVal,MPI_DOUBLE_PRECISION, & - nbProc(iNbProc),0,MPI_COMM_WORLD,MPIRequest(iNbProc),iError) + nbProc(iNbProc),0,MPI_COMM_PICLAS,MPIRequest(iNbProc),iError) ELSE MPIRequest(iNbProc)=MPI_REQUEST_NULL END IF @@ -319,7 +322,7 @@ SUBROUTINE StartSendMPIData(firstDim,FaceData,LowerBound,UpperBound,MPIRequest,S SideID_start=OffsetMPISides_send(iNbProc-1,SendID)+1 SideID_end =OffsetMPISides_send(iNbProc,SendID) CALL MPI_ISEND(FaceData(:,:,:,SideID_start:SideID_end),nSendVal,MPI_DOUBLE_PRECISION, & - nbProc(iNbProc),0,MPI_COMM_WORLD,MPIRequest(iNbProc),iError) + nbProc(iNbProc),0,MPI_COMM_PICLAS,MPIRequest(iNbProc),iError) ELSE MPIRequest(iNbProc)=MPI_REQUEST_NULL END IF @@ -413,14 +416,14 @@ SUBROUTINE StartReceiveMPIDataInt(firstDim,FaceData,LowerBound,UpperBound,MPIReq SideID_start=OffsetMPISides_rec(iNbProc-1,SendID)+1 SideID_end =OffsetMPISides_rec(iNbProc,SendID) CALL MPI_IRECV(FaceData(:,SideID_start:SideID_end),nRecVal,MPI_INTEGER, & - nbProc(iNbProc),0,MPI_COMM_WORLD,MPIRequest(iNbProc),iError) + nbProc(iNbProc),0,MPI_COMM_PICLAS,MPIRequest(iNbProc),iError) ELSE MPIRequest(iNbProc)=MPI_REQUEST_NULL END IF END DO !iProc=1,nNBProcs END SUBROUTINE StartReceiveMPIDataInt - - + + !=================================================================================================================================== !> See above, but for for send direction !=================================================================================================================================== @@ -448,7 +451,7 @@ SUBROUTINE StartSendMPIDataInt(firstDim,FaceData,LowerBound,UpperBound,MPIReques SideID_start=OffsetMPISides_send(iNbProc-1,SendID)+1 SideID_end =OffsetMPISides_send(iNbProc,SendID) CALL MPI_ISEND(FaceData(:,SideID_start:SideID_end),nSendVal,MPI_INTEGER, & - nbProc(iNbProc),0,MPI_COMM_WORLD,MPIRequest(iNbProc),iError) + nbProc(iNbProc),0,MPI_COMM_PICLAS,MPIRequest(iNbProc),iError) ELSE MPIRequest(iNbProc)=MPI_REQUEST_NULL END IF @@ -506,6 +509,7 @@ SUBROUTINE FinalizeMPI() ! Free the communicators IF(MPI_COMM_NODE .NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(MPI_COMM_NODE ,IERROR) IF(MPI_COMM_LEADERS.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(MPI_COMM_LEADERS,IERROR) +IF(MPI_COMM_PICLAS .NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(MPI_COMM_PICLAS ,IERROR) #if USE_LOADBALANCE IF (.NOT.(PerformLoadBalance.AND.(.NOT.UseH5IOLoadBalance))) THEN @@ -614,19 +618,19 @@ SUBROUTINE OutputMPIW8Time() IF(MPIroot)THEN ALLOCATE(MPIW8TimeProc(MPIW8SIZE*nProcessors)) ALLOCATE(MPIW8CountProc(MPIW8SIZE*nProcessors)) - CALL MPI_REDUCE(MPIW8TimeSim , MPIW8TimeSimeGlobal , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(MPIW8Time , MPIW8TimeGlobal , MPIW8SIZE , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(MPIW8Count , MPIW8CountGlobal , MPIW8SIZE , MPI_INTEGER8 , MPI_SUM , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(MPIW8TimeSim , MPIW8TimeSimeGlobal , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(MPIW8Time , MPIW8TimeGlobal , MPIW8SIZE , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(MPIW8Count , MPIW8CountGlobal , MPIW8SIZE , MPI_INTEGER8 , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) - CALL MPI_GATHER(MPIW8Time , MPIW8SIZE , MPI_DOUBLE_PRECISION , MPIW8TimeProc , MPIW8SIZE , MPI_DOUBLE_PRECISION , 0 , MPI_COMM_WORLD , iError) - CALL MPI_GATHER(MPIW8Count , MPIW8SIZE , MPI_INTEGER8 , MPIW8CountProc , MPIW8SIZE , MPI_INTEGER8 , 0 , MPI_COMM_WORLD , iError) + CALL MPI_GATHER(MPIW8Time , MPIW8SIZE , MPI_DOUBLE_PRECISION , MPIW8TimeProc , MPIW8SIZE , MPI_DOUBLE_PRECISION , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_GATHER(MPIW8Count , MPIW8SIZE , MPI_INTEGER8 , MPIW8CountProc , MPIW8SIZE , MPI_INTEGER8 , 0 , MPI_COMM_PICLAS , iError) ELSE - CALL MPI_REDUCE(MPIW8TimeSim , 0 , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IError) - CALL MPI_REDUCE(MPIW8Time , 0 , MPIW8SIZE , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , IError) - CALL MPI_REDUCE(MPIW8Count , 0 , MPIW8SIZE , MPI_INTEGER8 , MPI_SUM , 0 , MPI_COMM_WORLD , IError) + CALL MPI_REDUCE(MPIW8TimeSim , 0 , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IError) + CALL MPI_REDUCE(MPIW8Time , 0 , MPIW8SIZE , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IError) + CALL MPI_REDUCE(MPIW8Count , 0 , MPIW8SIZE , MPI_INTEGER8 , MPI_SUM , 0 , MPI_COMM_PICLAS , IError) - CALL MPI_GATHER(MPIW8Time , MPIW8SIZE , MPI_DOUBLE_PRECISION , 0 , 0 , 0 , 0 , MPI_COMM_WORLD , iError) - CALL MPI_GATHER(MPIW8Count , MPIW8SIZE , MPI_INTEGER8 , 0 , 0 , 0 , 0 , MPI_COMM_WORLD , iError) + CALL MPI_GATHER(MPIW8Time , MPIW8SIZE , MPI_DOUBLE_PRECISION , 0 , 0 , 0 , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_GATHER(MPIW8Count , MPIW8SIZE , MPI_INTEGER8 , 0 , 0 , 0 , 0 , MPI_COMM_PICLAS , iError) END IF ! -------------------------------------------------- diff --git a/src/mpi/mpi_shared.f90 b/src/mpi/mpi_shared.f90 index 57af7f263..34b142f32 100644 --- a/src/mpi/mpi_shared.f90 +++ b/src/mpi/mpi_shared.f90 @@ -143,11 +143,11 @@ SUBROUTINE InitMPIShared() ! Split the node communicator (shared memory) from the global communicator on physical processor or node level #if (CORE_SPLIT==1) - CALL MPI_COMM_SPLIT(MPI_COMM_WORLD,myRank,0,MPI_COMM_SHARED,iError) + CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS,myRank,0,MPI_COMM_SHARED,iError) #elif (CORE_SPLIT==0) ! Note that using SharedMemoryMethod=OMPI_COMM_TYPE_CORE somehow does not work in every case (intel/amd processors) ! Also note that OMPI_COMM_TYPE_CORE is undefined when not using OpenMPI - CALL MPI_COMM_SPLIT_TYPE(MPI_COMM_WORLD,SharedMemoryMethod,0,MPI_INFO_NULL,MPI_COMM_SHARED,IERROR) + CALL MPI_COMM_SPLIT_TYPE(MPI_COMM_PICLAS,SharedMemoryMethod,0,MPI_INFO_NULL,MPI_COMM_SHARED,IERROR) #else ! Check if more nodes than procs are required or ! if the resulting split would create unequal procs per node @@ -160,7 +160,7 @@ SUBROUTINE InitMPIShared() ! Group procs so that every CORE_SPLIT procs are in the same group color = INT(REAL(myRank)/REAL(CORE_SPLIT)) END IF ! (CORE_SPLIT.GE.nProcessors_Global).OR.(MOD().GT.0) - CALL MPI_COMM_SPLIT(MPI_COMM_WORLD,color,0,MPI_COMM_SHARED,iError) + CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS,color,0,MPI_COMM_SHARED,iError) #endif ! Find my rank on the shared communicator, comm size and proc name @@ -187,7 +187,7 @@ SUBROUTINE InitMPIShared() MPI_COMM_LEADERS_SHARED=MPI_COMM_NULL myLeaderGroupRank=-1 color = MERGE(101,MPI_UNDEFINED,myComputeNodeRank.EQ.0) -CALL MPI_COMM_SPLIT(MPI_COMM_WORLD,color,0,MPI_COMM_LEADERS_SHARED,IERROR) +CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS,color,0,MPI_COMM_LEADERS_SHARED,IERROR) IF(myComputeNodeRank.EQ.0)THEN CALL MPI_COMM_RANK(MPI_COMM_LEADERS_SHARED,myLeaderGroupRank,IERROR) CALL MPI_COMM_SIZE(MPI_COMM_LEADERS_SHARED,nLeaderGroupProcs,IERROR) diff --git a/src/output/output_vtk.f90 b/src/output/output_vtk.f90 index bacb6a3d9..1a7264bb5 100644 --- a/src/output/output_vtk.f90 +++ b/src/output/output_vtk.f90 @@ -207,7 +207,7 @@ SUBROUTINE WriteDataToVTK(nVal,NVisu,nElems,VarNames,Coord,Value,FileString,dim, ! get total number of elements on all processors #if USE_MPI -CALL MPI_GATHER(nElems,1,MPI_INTEGER,nElems_glob,1,MPI_INTEGER,0,MPI_COMM_WORLD,iError) +CALL MPI_GATHER(nElems,1,MPI_INTEGER,nElems_glob,1,MPI_INTEGER,0,MPI_COMM_PICLAS,iError) #else nElems_glob(0) = nElems #endif @@ -309,16 +309,16 @@ SUBROUTINE WriteDataToVTK(nVal,NVisu,nElems,VarNames,Coord,Value,FileString,dim, DO iProc=1,nProcessors-1 nElems_proc=nElems_glob(iProc) IF (nElems_proc.GT.0) THEN - CALL MPI_RECV(buf(:,:,:,1:nElems_proc),nElems_proc*NVisu_elem,MPI_DOUBLE_PRECISION,iProc,0,MPI_COMM_WORLD,MPIstatus,iError) + CALL MPI_RECV(buf(:,:,:,1:nElems_proc),nElems_proc*NVisu_elem,MPI_DOUBLE_PRECISION,iProc,0,MPI_COMM_PICLAS,MPIstatus,iError) WRITE(ivtk) REAL(buf(:,:,:,1:nElems_proc),4) END IF END DO !iProc ELSE IF (nElems.GT.0) THEN IF (nValAtLastDimension_loc) THEN - CALL MPI_SEND(Value(:,:,:,:,iVal),nElems*NVisu_elem,MPI_DOUBLE_PRECISION, 0,0,MPI_COMM_WORLD,iError) + CALL MPI_SEND(Value(:,:,:,:,iVal),nElems*NVisu_elem,MPI_DOUBLE_PRECISION, 0,0,MPI_COMM_PICLAS,iError) ELSE - CALL MPI_SEND(Value(iVal,:,:,:,:),nElems*NVisu_elem,MPI_DOUBLE_PRECISION, 0,0,MPI_COMM_WORLD,iError) + CALL MPI_SEND(Value(iVal,:,:,:,:),nElems*NVisu_elem,MPI_DOUBLE_PRECISION, 0,0,MPI_COMM_PICLAS,iError) END IF END IF #endif /*USE_MPI*/ @@ -341,13 +341,13 @@ SUBROUTINE WriteDataToVTK(nVal,NVisu,nElems,VarNames,Coord,Value,FileString,dim, DO iProc=1,nProcessors-1 nElems_proc=nElems_glob(iProc) IF (nElems_proc.GT.0) THEN - CALL MPI_RECV(buf2(:,:,:,:,1:nElems_proc),nElems_proc*NVisu_elem*3,MPI_DOUBLE_PRECISION,iProc,0,MPI_COMM_WORLD,MPIstatus,iError) + CALL MPI_RECV(buf2(:,:,:,:,1:nElems_proc),nElems_proc*NVisu_elem*3,MPI_DOUBLE_PRECISION,iProc,0,MPI_COMM_PICLAS,MPIstatus,iError) WRITE(ivtk) REAL(buf2(:,:,:,:,1:nElems_proc),4) END IF END DO !iProc ELSE IF (nElems.GT.0) THEN - CALL MPI_SEND(Coord(:,:,:,:,:),nElems*NVisu_elem*3,MPI_DOUBLE_PRECISION, 0,0,MPI_COMM_WORLD,iError) + CALL MPI_SEND(Coord(:,:,:,:,:),nElems*NVisu_elem*3,MPI_DOUBLE_PRECISION, 0,0,MPI_COMM_PICLAS,iError) END IF #endif /*USE_MPI*/ END IF !MPIroot diff --git a/src/particles/analyze/particle_analyze.f90 b/src/particles/analyze/particle_analyze.f90 index 2c19eb39f..95971ddcb 100644 --- a/src/particles/analyze/particle_analyze.f90 +++ b/src/particles/analyze/particle_analyze.f90 @@ -578,7 +578,7 @@ SUBROUTINE InitParticleAnalyze() !-- Coupled Power CalcCoupledPower = GETLOGICAL('CalcCoupledPower') #if USE_HDG -IF(UseCoupledPowerPotential.AND.(.NOT.CalcCoupledPower)) CALL abort(__STAMP__,'Coupled power potential requires CalcCoupledPower=T') +IF(UseCoupledPowerPotential.AND.(.NOT.CalcCoupledPower)) CALL abort(__STAMP__,'Coupled power potential requires CalcCoupledPower=T') #endif /*USE_HDG*/ IF(CalcCoupledPower) THEN @@ -814,7 +814,7 @@ SUBROUTINE InitBulkElectronTemp() END IF ! MPIRoot #if USE_MPI ! Broadcast from root to other processors. Only root knows if BulkElectronTempExists=T/F so always broadcast message - CALL MPI_BCAST(BulkElectronTemp,1, MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,iERROR) + CALL MPI_BCAST(BulkElectronTemp,1, MPI_DOUBLE_PRECISION,0,MPI_COMM_PICLAS,iERROR) IF(SurfModSEEelectronTempAutoamtic) BulkElectronTempSEE = BulkElectronTemp #endif /*USE_MPI*/ END IF ! CalcBulkElectronTemp @@ -843,7 +843,6 @@ SUBROUTINE AnalyzeParticles(Time) USE MOD_FPFlow_Vars ,ONLY: FP_MaxRelaxFactor,FP_MaxRotRelaxFactor,FP_MeanRelaxFactor,FP_MeanRelaxFactorCounter USE MOD_FPFlow_Vars ,ONLY: FP_PrandtlNumber,FPInitDone USE MOD_Particle_Analyze_Vars -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_Particle_Vars ,ONLY: Species,nSpecies USE MOD_PIC_Analyze ,ONLY: CalcDepositedCharge USE MOD_Restart_Vars ,ONLY: RestartTime,DoRestart @@ -941,7 +940,7 @@ SUBROUTINE AnalyzeParticles(Time) #endif OutputCounter = 2 unit_index = 535 - IF (PartMPI%MPIRoot) THEN + IF (MPIRoot) THEN INQUIRE(UNIT = unit_index , OPENED = isOpen) IF (.NOT.isOpen) THEN outfile = 'PartAnalyze.csv' @@ -1323,20 +1322,20 @@ SUBROUTINE AnalyzeParticles(Time) IF(CalcLaserInteraction)THEN CALL CalcKineticEnergyAndMaximum(Ekin,EkinMax) #if USE_MPI - IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE , EkinMax , nSpecies , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , PartMPI%COMM , iError) + IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE , EkinMax , nSpecies , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) ELSE - CALL MPI_REDUCE(EkinMax , 0. , nSpecies , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , PartMPI%COMM , iError) + CALL MPI_REDUCE(EkinMax , 0. , nSpecies , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) END IF #endif /*USE_MPI*/ ELSE CALL CalcKineticEnergy(Ekin) END IF #if USE_MPI - IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE , Ekin , nSpecAnalyze , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , PartMPI%COMM , IERROR) + IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE , Ekin , nSpecAnalyze , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) ELSE - CALL MPI_REDUCE(Ekin , 0. , nSpecAnalyze , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , PartMPI%COMM , IERROR) + CALL MPI_REDUCE(Ekin , 0. , nSpecAnalyze , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) END IF #endif /*USE_MPI*/ END IF @@ -1346,7 +1345,7 @@ SUBROUTINE AnalyzeParticles(Time) ! CalcEint(1) is required for Ekin IF(CalcTemp(1).AND.CalcEint(1)) THEN CALL CalcMixtureTemp(NumSpec,Temp,IntTemp,IntEn,TempTotal,Xi_Vib,Xi_Elec) ! contains MPI Communication - IF(PartMPI%MPIRoot) ETotal = Ekin(nSpecAnalyze) + IntEn(nSpecAnalyze,1) + IntEn(nSpecAnalyze,2) + IntEn(nSpecAnalyze,3) + IF(MPIRoot) ETotal = Ekin(nSpecAnalyze) + IntEn(nSpecAnalyze,1) + IntEn(nSpecAnalyze,2) + IntEn(nSpecAnalyze,3) END IF !----------------------------------------------------------------------------------------------------------------------------------- ! Determine the maximal collision probability for whole reservoir and mean collision probability (only for one cell reservoirs, @@ -1378,7 +1377,7 @@ SUBROUTINE AnalyzeParticles(Time) IF(iter.GT.0) THEN MaxCollProb = DSMC%CollProbMax IF(DSMC%CollProbMeanCount.GT.0) MeanCollProb = DSMC%CollProbMean / DSMC%CollProbMeanCount - IF (PartMPI%MPIRoot) THEN + IF (MPIRoot) THEN IF(TempTotal(nSpecAnalyze).GT.0.0) MeanFreePath = CalcMeanFreePath(NumSpecTmp(1:nSpecies), NumSpecTmp(nSpecAnalyze), & MeshVolume, TempTotal(nSpecAnalyze)) END IF @@ -1400,8 +1399,8 @@ SUBROUTINE AnalyzeParticles(Time) #if USE_MPI ! Collect sum on MPIRoot tmpArray = (/PCoupl, PCouplAverage/) - IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE, tmpArray, 2, MPI_DOUBLE_PRECISION, MPI_SUM, 0, PartMPI%COMM, IERROR) + IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE, tmpArray, 2, MPI_DOUBLE_PRECISION, MPI_SUM, 0, MPI_COMM_PICLAS, IERROR) PCoupl = tmpArray(1) PCouplAverage = tmpArray(2) #endif /*USE_MPI*/ @@ -1426,33 +1425,33 @@ SUBROUTINE AnalyzeParticles(Time) #endif /*USE_HDG*/ #if USE_MPI ELSE - CALL MPI_REDUCE(tmpArray, 0, 2, MPI_DOUBLE_PRECISION, MPI_SUM, 0, PartMPI%COMM, IERROR) + CALL MPI_REDUCE(tmpArray, 0, 2, MPI_DOUBLE_PRECISION, MPI_SUM, 0, MPI_COMM_PICLAS, IERROR) ! Reset for all processes execpt the MPIRoot (this process keeps the old value, which is therefore considered only once in the ! next MPI reduce call) PCouplAverage = 0. - END IF ! PartMPI%MPIRoot + END IF ! MPIRoot END IF ! Switch between root and non-root processes - IF (PartMPI%MPIRoot) THEN + IF (MPIRoot) THEN IF (CalcPartBalance)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,nPartIn(1:nSpecAnalyze) ,nSpecAnalyze,MPI_INTEGER ,MPI_SUM,0,PartMPI%COMM,IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE,nPartOut(1:nSpecAnalyze) ,nSpecAnalyze,MPI_INTEGER ,MPI_SUM,0,PartMPI%COMM,IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE,PartEkinIn(1:nSpecAnalyze) ,nSpecAnalyze,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE,PartEkinOut(1:nSpecAnalyze),nSpecAnalyze,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,nPartIn(1:nSpecAnalyze) ,nSpecAnalyze,MPI_INTEGER ,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,nPartOut(1:nSpecAnalyze) ,nSpecAnalyze,MPI_INTEGER ,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,PartEkinIn(1:nSpecAnalyze) ,nSpecAnalyze,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,PartEkinOut(1:nSpecAnalyze),nSpecAnalyze,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) END IF ELSE ! no Root IF (CalcPartBalance)THEN - CALL MPI_REDUCE(nPartIn ,0,nSpecAnalyze,MPI_INTEGER ,MPI_SUM,0,PartMPI%COMM,IERROR) - CALL MPI_REDUCE(nPartOut ,0,nSpecAnalyze,MPI_INTEGER ,MPI_SUM,0,PartMPI%COMM,IERROR) - CALL MPI_REDUCE(PartEkinIn ,0,nSpecAnalyze,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) - CALL MPI_REDUCE(PartEkinOut,0,nSpecAnalyze,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) + CALL MPI_REDUCE(nPartIn ,0,nSpecAnalyze,MPI_INTEGER ,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) + CALL MPI_REDUCE(nPartOut ,0,nSpecAnalyze,MPI_INTEGER ,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) + CALL MPI_REDUCE(PartEkinIn ,0,nSpecAnalyze,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) + CALL MPI_REDUCE(PartEkinOut,0,nSpecAnalyze,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) END IF #endif /*USE_MPI*/ END IF !----------------------------------------------------------------------------------------------------------------------------------- ! Analyze Routines that require MPI_REDUCE of other variables ! Moving Average of PCoupl: -IF(CalcCoupledPower.AND.PartMPI%MPIRoot) THEN +IF(CalcCoupledPower.AND.MPIRoot) THEN ! Moving Average of PCoupl: TimeDelta = Time-RestartTime IF(ABS(TimeDelta).GT.0.0) PCouplAverage = PCouplAverage / TimeDelta @@ -1465,7 +1464,7 @@ SUBROUTINE AnalyzeParticles(Time) #endif /*USE_HDG*/ !----------------------------------------------------------------------------------------------------------------------------------- ! Perform averaging/summation of the MPI communicated variables on the root only (and for the non-MPI case, MPIRoot is set to true) -IF(PartMPI%MPIRoot) THEN +IF(MPIRoot) THEN IF (CalcPartBalance)THEN IF(nSpecies.GT.1) THEN nPartIn(nSpecies+1) = SUM(nPartIn(1:nSpecies)) @@ -1488,7 +1487,7 @@ SUBROUTINE AnalyzeParticles(Time) IF(BGK_MeanRelaxFactorCounter.GT.0) BGK_MeanRelaxFactor = BGK_MeanRelaxFactor / REAL(BGK_MeanRelaxFactorCounter) END IF END IF -END IF ! PartMPI%MPIRoot +END IF ! MPIRoot IF(CalcCoupledPower) THEN ! Moving Average of PCoupl for each species @@ -1517,7 +1516,7 @@ SUBROUTINE AnalyzeParticles(Time) ! Output Routines !=================================================================================================================================== #if USE_MPI -IF (PartMPI%MPIROOT) THEN +IF (MPIRoot) THEN #endif /*USE_MPI*/ WRITE(unit_index,'(E23.16E3)',ADVANCE='NO') Time IF (CalcSimNumSpec) THEN @@ -1745,11 +1744,11 @@ SUBROUTINE AnalyzeParticles(Time) ! Finish the line with new line character WRITE(unit_index,'(A)') '' #if USE_MPI -END IF ! PartMPI%MPIROOT +END IF ! MPIRoot #endif /*USE_MPI*/ !----------------------------------------------------------------------------------------------------------------------------------- ! Reset coupled power to particles if output of coupled power is active -IF (CalcCoupledPower.AND.PartMPI%MPIRoot) THEN +IF (CalcCoupledPower.AND.MPIRoot) THEN IF(ABS(TimeDelta).GT.0.0) PCouplAverage = PCouplAverage * TimeDelta ! PCouplAverage is reset END IF !----------------------------------------------------------------------------------------------------------------------------------- diff --git a/src/particles/analyze/particle_analyze_code.f90 b/src/particles/analyze/particle_analyze_code.f90 index 6fda4b41e..0a68ed0fd 100644 --- a/src/particles/analyze/particle_analyze_code.f90 +++ b/src/particles/analyze/particle_analyze_code.f90 @@ -572,7 +572,6 @@ SUBROUTINE AnalyticParticleMovement(time,iter) USE MOD_Analyze_Vars ,ONLY: OutputErrorNorms USE MOD_Particle_Analyze_Vars ,ONLY: TrackParticlePosition USE MOD_PICInterpolation_Vars ,ONLY: L_2_Error_Part,AnalyticPartDim -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !---------------------------------------------------------------------------------------------------------------------------------- @@ -588,7 +587,7 @@ SUBROUTINE AnalyticParticleMovement(time,iter) !=================================================================================================================================== CALL CalcErrorParticle(time,iter,PartStateAnalytic) -IF(PartMPI%MPIRoot.AND.OutputErrorNorms) THEN +IF(MPIRoot.AND.OutputErrorNorms) THEN WRITE(UNIT_StdOut,'(A13,ES16.7)')' Sim time : ',time WRITE(formatStr,'(A5,I1,A7)')'(A13,',AnalyticPartDim,'ES16.7)' WRITE(UNIT_StdOut,formatStr)' L2_Part : ',L_2_Error_Part diff --git a/src/particles/analyze/particle_analyze_output.f90 b/src/particles/analyze/particle_analyze_output.f90 index d7b172548..66e3b8f84 100644 --- a/src/particles/analyze/particle_analyze_output.f90 +++ b/src/particles/analyze/particle_analyze_output.f90 @@ -37,7 +37,7 @@ SUBROUTINE WriteParticleTrackingData(time,iter) !----------------------------------------------------------------------------------------------------------------------------------! ! MODULES ! !----------------------------------------------------------------------------------------------------------------------------------! -USE MOD_Globals ,ONLY: FILEEXISTS,unit_stdout,DOTPRODUCT,abort,MPI_COMM_WORLD,iError,MPIRoot +USE MOD_Globals ,ONLY: FILEEXISTS,unit_stdout,DOTPRODUCT,abort,MPI_COMM_PICLAS,iError,MPIRoot USE MOD_Restart_Vars ,ONLY: DoRestart USE MOD_Particle_Vars ,ONLY: PartState, PDM, PEM USE MOD_Particle_Analyze_Vars ,ONLY: printDiff,printDiffVec,printDiffTime @@ -112,7 +112,7 @@ SUBROUTINE WriteParticleTrackingData(time,iter) #if USE_MPI ! Barrier is required is root creates file and other processor prints to this file -IF(CreateFile) CALL MPI_BARRIER(MPI_COMM_WORLD,iError) +IF(CreateFile) CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ CALL UpdateNextFreePosition() @@ -214,7 +214,7 @@ SUBROUTINE DisplayCoupledPowerPart() ! Sum the power #if USE_MPI -CALL MPI_REDUCE(PTotal(1:nSpecies),SumPTotal(1:nSpecies),nSpecies,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_WORLD,iError) +CALL MPI_REDUCE(PTotal(1:nSpecies),SumPTotal(1:nSpecies),nSpecies,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,iError) #else SumPTotal(1:nSpecies)=PTotal(1:nSpecies) #endif diff --git a/src/particles/analyze/particle_analyze_tools.f90 b/src/particles/analyze/particle_analyze_tools.f90 index d8a7878db..b1ee3777e 100644 --- a/src/particles/analyze/particle_analyze_tools.f90 +++ b/src/particles/analyze/particle_analyze_tools.f90 @@ -184,9 +184,6 @@ SUBROUTINE CalcNumPartsOfSpec(NumSpec,SimNumSpec,CalcNumSpec_IN,CalcSimNumSpec_I USE MOD_DSMC_Vars ,ONLY: BGGas, DSMC USE MOD_Particle_Vars ,ONLY: nSpecies, Species USE MOD_part_tools ,ONLY: GetParticleWeight -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif !----------------------------------------------------------------------------------------------------------------------------------! ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE @@ -238,16 +235,16 @@ SUBROUTINE CalcNumPartsOfSpec(NumSpec,SimNumSpec,CalcNumSpec_IN,CalcSimNumSpec_I END IF #if USE_MPI -IF (PartMPI%MPIRoot) THEN - IF(CalcNumSpec_IN) CALL MPI_REDUCE(MPI_IN_PLACE , NumSpec , nSpecAnalyze , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , PartMPI%COMM , IERROR) - IF(CalcSimNumSpec_IN) CALL MPI_REDUCE(SUM(SimNumSpec) , SimNumSpecMin , 1 , MPI_INTEGER_INT_KIND , MPI_MIN , 0 , PartMPI%COMM , IERROR) - IF(CalcSimNumSpec_IN) CALL MPI_REDUCE(SUM(SimNumSpec) , SimNumSpecMax , 1 , MPI_INTEGER_INT_KIND , MPI_MAX , 0 , PartMPI%COMM , IERROR) - IF(CalcSimNumSpec_IN) CALL MPI_REDUCE(MPI_IN_PLACE , SimNumSpec , nSpecAnalyze , MPI_INTEGER_INT_KIND , MPI_SUM , 0 , PartMPI%COMM , IERROR) +IF (MPIRoot) THEN + IF(CalcNumSpec_IN) CALL MPI_REDUCE(MPI_IN_PLACE , NumSpec , nSpecAnalyze , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) + IF(CalcSimNumSpec_IN) CALL MPI_REDUCE(SUM(SimNumSpec) , SimNumSpecMin , 1 , MPI_INTEGER_INT_KIND , MPI_MIN , 0 , MPI_COMM_PICLAS , IERROR) + IF(CalcSimNumSpec_IN) CALL MPI_REDUCE(SUM(SimNumSpec) , SimNumSpecMax , 1 , MPI_INTEGER_INT_KIND , MPI_MAX , 0 , MPI_COMM_PICLAS , IERROR) + IF(CalcSimNumSpec_IN) CALL MPI_REDUCE(MPI_IN_PLACE , SimNumSpec , nSpecAnalyze , MPI_INTEGER_INT_KIND , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) ELSE - IF(CalcNumSpec_IN) CALL MPI_REDUCE(NumSpec , 0 , nSpecAnalyze , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , PartMPI%COMM , IERROR) - IF(CalcSimNumSpec_IN) CALL MPI_REDUCE(SUM(SimNumSpec) , 0 , 1 , MPI_INTEGER_INT_KIND , MPI_MIN , 0 , PartMPI%COMM , IERROR) - IF(CalcSimNumSpec_IN) CALL MPI_REDUCE(SUM(SimNumSpec) , 0 , 1 , MPI_INTEGER_INT_KIND , MPI_MAX , 0 , PartMPI%COMM , IERROR) - IF(CalcSimNumSpec_IN) CALL MPI_REDUCE(SimNumSpec , 0 , nSpecAnalyze , MPI_INTEGER_INT_KIND , MPI_SUM , 0 , PartMPI%COMM , IERROR) + IF(CalcNumSpec_IN) CALL MPI_REDUCE(NumSpec , 0 , nSpecAnalyze , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) + IF(CalcSimNumSpec_IN) CALL MPI_REDUCE(SUM(SimNumSpec) , 0 , 1 , MPI_INTEGER_INT_KIND , MPI_MIN , 0 , MPI_COMM_PICLAS , IERROR) + IF(CalcSimNumSpec_IN) CALL MPI_REDUCE(SUM(SimNumSpec) , 0 , 1 , MPI_INTEGER_INT_KIND , MPI_MAX , 0 , MPI_COMM_PICLAS , IERROR) + IF(CalcSimNumSpec_IN) CALL MPI_REDUCE(SimNumSpec , 0 , nSpecAnalyze , MPI_INTEGER_INT_KIND , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) END IF #endif /*USE_MPI*/ @@ -255,7 +252,7 @@ SUBROUTINE CalcNumPartsOfSpec(NumSpec,SimNumSpec,CalcNumSpec_IN,CalcSimNumSpec_I IF(CalcSimNumSpec_IN)THEN GlobalNbrOfParticlesUpdated = .TRUE. #if USE_MPI - IF(PartMPI%MPIRoot)THEN + IF(MPIRoot)THEN #endif /*USE_MPI*/ nGlobalNbrOfParticles(1) = INT(SimNumSpecMin,KIND=IK) nGlobalNbrOfParticles(2) = INT(SimNumSpecMax,KIND=IK) @@ -264,7 +261,7 @@ SUBROUTINE CalcNumPartsOfSpec(NumSpec,SimNumSpec,CalcNumSpec_IN,CalcSimNumSpec_I nGlobalNbrOfParticles(5) = MAX(nGlobalNbrOfParticles(2),nGlobalNbrOfParticles(5)) nGlobalNbrOfParticles(6) = MAX(nGlobalNbrOfParticles(3),nGlobalNbrOfParticles(6)) #if USE_MPI - END IF ! PartMPI%MPIRoot + END IF ! MPIRoot #endif /*USE_MPI*/ END IF ! CalcSimNumSpec_IN @@ -386,7 +383,6 @@ SUBROUTINE CalculatePartElemData() USE MOD_Particle_Analyze_Vars ,ONLY: CalcCyclotronFrequency #if USE_MPI USE MOD_Globals -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_Particle_Analyze_Vars ,ONLY: PPDCellResolved,PICTimeCellResolved,PICValidPlasmaCellSum #endif /*USE_MPI*/ !----------------------------------------------------------------------------------------------------------------------------------! @@ -457,8 +453,8 @@ SUBROUTINE CalculatePartElemData() tmpArray(6) = PICValidPlasmaCellSum ! Collect sum on MPIRoot - IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE , tmpArray , 6 , MPI_INTEGER , MPI_SUM , 0 , PartMPI%COMM , IERROR) + IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE , tmpArray , 6 , MPI_INTEGER , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) IF(CalcPointsPerDebyeLength)THEN PPDCellResolved(1) = tmpArray(1) PPDCellResolved(2) = tmpArray(2) @@ -470,8 +466,8 @@ SUBROUTINE CalculatePartElemData() END IF ! CalcPICTimeStep PICValidPlasmaCellSum = tmpArray(6) ELSE - CALL MPI_REDUCE(tmpArray , 0 , 6 , MPI_INTEGER , MPI_SUM , 0 , PartMPI%COMM , IERROR) - END IF ! PartMPI%MPIRoot + CALL MPI_REDUCE(tmpArray , 0 , 6 , MPI_INTEGER , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) + END IF ! MPIRoot END IF ! CalcPointsPerDebyeLength.OR.CalcPICTimeStep #endif /*USE_MPI*/ @@ -781,7 +777,7 @@ SUBROUTINE CalcShapeEfficiencyR() USE MOD_Particle_Vars USE MOD_Preproc #if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI +USE MOD_Globals #endif ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE @@ -856,8 +852,8 @@ SUBROUTINE CalcShapeEfficiencyR() END DO ! i IF(NbrOfComps.GT.0.0)THEN #if USE_MPI - WRITE(*,*) 'ShapeEfficiency (Proc,%,%Elems)',PartMPI%MyRank,100*NbrWithinRadius/NbrOfComps,100*NbrOfElemsWithinRadius/NbrOfElems - WRITE(*,*) 'ShapeEfficiency (Elems) for Proc',PartMPI%MyRank,'is',100*NbrOfElemsWithinRadius/NbrOfElems,'%' + WRITE(*,*) 'ShapeEfficiency (Proc,%,%Elems)',myRank,100*NbrWithinRadius/NbrOfComps,100*NbrOfElemsWithinRadius/NbrOfElems + WRITE(*,*) 'ShapeEfficiency (Elems) for Proc',myRank,'is',100*NbrOfElemsWithinRadius/NbrOfElems,'%' #else WRITE(*,*) 'ShapeEfficiency (%,%Elems)',100*NbrWithinRadius/NbrOfComps, 100*NbrOfElemsWithinRadius/NbrOfElems WRITE(*,*) 'ShapeEfficiency (Elems) is',100*NbrOfElemsWithinRadius/NbrOfElems,'%' @@ -917,8 +913,8 @@ SUBROUTINE CalcShapeEfficiencyR() END DO ! i IF(NbrOfComps.GT.0)THEN #if USE_MPI - WRITE(*,*) 'ShapeEfficiency (Proc,%,%Elems)',PartMPI%MyRank,100*NbrWithinRadius/NbrOfComps,100*NbrOfElemsWithinRadius/NbrOfElems - WRITE(*,*) 'ShapeEfficiency (Elems) for Proc',PartMPI%MyRank,'is',100*NbrOfElemsWithinRadius/NbrOfElems,'%' + WRITE(*,*) 'ShapeEfficiency (Proc,%,%Elems)',myRank,100*NbrWithinRadius/NbrOfComps,100*NbrOfElemsWithinRadius/NbrOfElems + WRITE(*,*) 'ShapeEfficiency (Elems) for Proc',myRank,'is',100*NbrOfElemsWithinRadius/NbrOfElems,'%' #else WRITE(*,*) 'ShapeEfficiency (%,%Elems)',100*NbrWithinRadius/NbrOfComps,100*NbrOfElemsWithinRadius/NbrOfElems WRITE(*,*) 'ShapeEfficiency (Elems) is',100*NbrOfElemsWithinRadius/NbrOfElems,'%' @@ -1178,7 +1174,6 @@ PPURE SUBROUTINE CalcNumberDensity(NumSpec,NumDens) USE MOD_Particle_Analyze_Vars ,ONLY: nSpecAnalyze USE MOD_Particle_Vars ,ONLY: Species,nSpecies,usevMPF USE MOD_Particle_Mesh_Vars ,ONLY: MeshVolume -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI !----------------------------------------------------------------------------------------------------------------------------------! ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE @@ -1193,7 +1188,7 @@ PPURE SUBROUTINE CalcNumberDensity(NumSpec,NumDens) !=================================================================================================================================== ! Only root does calculation -IF(.NOT.PartMPI%MPIRoot) RETURN +IF(.NOT.MPIRoot) RETURN IF(usevMPF.OR.RadialWeighting%DoRadialWeighting) THEN NumDens(1:nSpecies) = NumSpec(1:nSpecies) / MeshVolume @@ -1230,9 +1225,6 @@ SUBROUTINE CalcNumberDensityBGGasDistri() USE MOD_Particle_Vars ,ONLY: nSpecies USE MOD_Particle_Mesh_Vars ,ONLY: MeshVolume,ElemVolume_Shared USE MOD_Mesh_Tools ,ONLY: GetCNElemID -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ USE MOD_Mesh_Vars ,ONLY: nElems,offSetElem !----------------------------------------------------------------------------------------------------------------------------------! ! IMPLICIT VARIABLE HANDLING @@ -1265,10 +1257,10 @@ SUBROUTINE CalcNumberDensityBGGasDistri() ! Communicate #if USE_MPI -IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE , DistriNumDens, BGGas%NumberOfSpecies, MPI_DOUBLE_PRECISION, MPI_SUM, 0,PartMPI%COMM ,IERROR) +IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE , DistriNumDens, BGGas%NumberOfSpecies, MPI_DOUBLE_PRECISION, MPI_SUM, 0,MPI_COMM_PICLAS ,IERROR) ELSE - CALL MPI_REDUCE(DistriNumDens, 0. , BGGas%NumberOfSpecies, MPI_DOUBLE_PRECISION, MPI_SUM, 0,PartMPI%COMM ,IERROR) + CALL MPI_REDUCE(DistriNumDens, 0. , BGGas%NumberOfSpecies, MPI_DOUBLE_PRECISION, MPI_SUM, 0,MPI_COMM_PICLAS ,IERROR) END IF #endif /*USE_MPI*/ BGGas%DistributionNumDens = DistriNumDens @@ -1296,7 +1288,6 @@ SUBROUTINE CalcSurfaceFluxInfo() USE MOD_Particle_Sampling_Vars ,ONLY: UseAdaptive, AdaptBCMacroVal, AdaptBCMapElemToSample, AdaptBCAreaSurfaceFlux USE MOD_Particle_Sampling_Vars ,ONLY: AdaptBCAverageValBC, AdaptBCAverageMacroVal USE MOD_Mesh_Vars ,ONLY: SideToElem -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI #if USE_MPI USE MOD_Particle_Analyze_Vars ,ONLY: nSpecAnalyze #endif /*USE_MPI*/ @@ -1371,21 +1362,21 @@ SUBROUTINE CalcSurfaceFluxInfo() ! distribution at the BC, ie. NOT if AdaptBCAverageValBC = T) #if USE_MPI MaxSurfaceFluxBCs = MAXVAL(Species(:)%nSurfacefluxBCs) -IF (PartMPI%MPIRoot) THEN +IF (MPIRoot) THEN CALL MPI_REDUCE(MPI_IN_PLACE,FlowRateSurfFlux(1:nSpecAnalyze,1:MaxSurfaceFluxBCs),nSpecAnalyze*MaxSurfaceFluxBCs,& - MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) + MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) IF(UseAdaptive.AND.(.NOT.AdaptBCAverageValBC)) THEN CALL MPI_REDUCE(MPI_IN_PLACE,PressureAdaptiveBC(1:nSpecAnalyze,1:MaxSurfaceFluxBCs),nSpecAnalyze*MaxSurfaceFluxBCs,& - MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) + MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) END IF ELSE ! no Root - CALL MPI_REDUCE(FlowRateSurfFlux,FlowRateSurfFlux,nSpecAnalyze*MaxSurfaceFluxBCs,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) - IF(UseAdaptive.AND.(.NOT.AdaptBCAverageValBC)) CALL MPI_REDUCE(PressureAdaptiveBC,PressureAdaptiveBC,nSpecAnalyze*MaxSurfaceFluxBCs,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) + CALL MPI_REDUCE(FlowRateSurfFlux,FlowRateSurfFlux,nSpecAnalyze*MaxSurfaceFluxBCs,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) + IF(UseAdaptive.AND.(.NOT.AdaptBCAverageValBC)) CALL MPI_REDUCE(PressureAdaptiveBC,PressureAdaptiveBC,nSpecAnalyze*MaxSurfaceFluxBCs,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) END IF #endif /*USE_MPI*/ ! 3) Consider Part-AnalyzeStep for FlowRateSurfFlux and determine the average pressure (value does not depend on the Part-AnalyzeStep) -IF (PartMPI%MPIRoot) THEN +IF (MPIRoot) THEN IF(PartAnalyzeStep.GT.1)THEN IF(PartAnalyzeStep.EQ.HUGE(PartAnalyzeStep))THEN FlowRateSurfFlux = FlowRateSurfFlux / iter @@ -1423,7 +1414,6 @@ SUBROUTINE CalcMixtureTemp(NumSpec,Temp,IntTemp,IntEn,TempTotal,Xi_Vib,Xi_Elec) USE MOD_Globals USE MOD_PARTICLE_Vars ,ONLY: nSpecies USE MOD_Particle_Analyze_Vars ,ONLY: nSpecAnalyze -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_DSMC_Vars ,ONLY: SpecDSMC, CollisMode, DSMC USE MOD_part_tools ,ONLY: CalcXiElec USE MOD_DSMC_Relaxation ,ONLY: CalcXiVib @@ -1447,7 +1437,7 @@ SUBROUTINE CalcMixtureTemp(NumSpec,Temp,IntTemp,IntEn,TempTotal,Xi_Vib,Xi_Elec) IF (CollisMode.GT.1) THEN CALL CalcIntTempsAndEn(NumSpec,IntTemp,IntEn) - IF(PartMPI%MPIRoot)THEN + IF(MPIRoot)THEN TempTotal = 0.0 Xi_Vib = 0.0 Xi_Elec = 0.0 @@ -1486,7 +1476,7 @@ SUBROUTINE CalcMixtureTemp(NumSpec,Temp,IntTemp,IntEn,TempTotal,Xi_Vib,Xi_Elec) END IF END IF ELSE - IF(PartMPI%MPIRoot)THEN + IF(MPIRoot)THEN TempTotal = Temp IntTemp = 0.0 IntEn = 0.0 @@ -1507,7 +1497,6 @@ SUBROUTINE CalcIntTempsAndEn(NumSpec,IntTemp,IntEn) USE MOD_Globals_Vars ,ONLY: BoltzmannConst USE MOD_Particle_Vars ,ONLY: PartSpecies, Species, PDM, nSpecies, usevMPF USE MOD_DSMC_Vars ,ONLY: PartStateIntEn, SpecDSMC, DSMC -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_Particle_Analyze_Vars ,ONLY: nSpecAnalyze USE MOD_part_tools ,ONLY: GetParticleWeight USE MOD_DSMC_Vars ,ONLY: RadialWeighting @@ -1549,19 +1538,19 @@ SUBROUTINE CalcIntTempsAndEn(NumSpec,IntTemp,IntEn) END DO #if USE_MPI -IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,EVib ,nSpecies, MPI_DOUBLE_PRECISION, MPI_SUM,0, PartMPI%COMM, IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE,ERot ,nSpecies, MPI_DOUBLE_PRECISION, MPI_SUM,0, PartMPI%COMM, IERROR) - IF(DSMC%ElectronicModel.GT.0) CALL MPI_REDUCE(MPI_IN_PLACE,Eelec,nSpecies, MPI_DOUBLE_PRECISION, MPI_SUM,0, PartMPI%COMM, IERROR) +IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE,EVib ,nSpecies, MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,ERot ,nSpecies, MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) + IF(DSMC%ElectronicModel.GT.0) CALL MPI_REDUCE(MPI_IN_PLACE,Eelec,nSpecies, MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) ELSE - CALL MPI_REDUCE(EVib ,RD ,nSpecies, MPI_DOUBLE_PRECISION, MPI_SUM,0, PartMPI%COMM, IERROR) - CALL MPI_REDUCE(ERot ,RD ,nSpecies, MPI_DOUBLE_PRECISION, MPI_SUM,0, PartMPI%COMM, IERROR) - IF(DSMC%ElectronicModel.GT.0) CALL MPI_REDUCE(Eelec ,RD ,nSpecies, MPI_DOUBLE_PRECISION, MPI_SUM,0, PartMPI%COMM, IERROR) + CALL MPI_REDUCE(EVib ,RD ,nSpecies, MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(ERot ,RD ,nSpecies, MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) + IF(DSMC%ElectronicModel.GT.0) CALL MPI_REDUCE(Eelec ,RD ,nSpecies, MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) END IF #endif /*USE_MPI*/ ! final computation is only done for the root -IF(PartMPI%MPIRoot)THEN +IF(MPIRoot)THEN ! Calc TVib, TRot DO iSpec = 1, nSpecies NumSpecTemp = NumSpec(iSpec) @@ -1632,7 +1621,6 @@ SUBROUTINE CalcTransTemp(NumSpec, Temp) USE MOD_part_tools ,ONLY: GetParticleWeight USE MOD_Particle_Vars ,ONLY: PartSpecies, PartState, Species, PDM USE MOD_Particle_Analyze_Vars ,ONLY: nSpecAnalyze -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_DSMC_Vars ,ONLY: DSMC, AmbipolElecVelo USE MOD_Particle_Vars ,ONLY: CalcBulkElectronTemp,BulkElectronTemp,BulkElectronTempSpecID #if USE_MPI @@ -1672,14 +1660,14 @@ SUBROUTINE CalcTransTemp(NumSpec, Temp) END DO #if USE_MPI -IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,PartVandV2,nSpecies*6,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM, IERROR) +IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE,PartVandV2,nSpecies*6,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) ELSE - CALL MPI_REDUCE(PartVandV2 ,PartVandV2,nSpecies*6,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM, IERROR) + CALL MPI_REDUCE(PartVandV2 ,PartVandV2,nSpecies*6,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) END IF #endif /*USE_MPI*/ -IF(PartMPI%MPIRoot)THEN +IF(MPIRoot)THEN DO iSpec=1, nSpecies IF(NumSpec(iSpec).NE.0) THEN ! Compute velocity averages @@ -1707,7 +1695,7 @@ SUBROUTINE CalcTransTemp(NumSpec, Temp) END IF ! For SEE model that used the bulk electron temperature, use the global electron temperature -IF(PartMPI%MPIRoot.AND.CalcBulkElectronTemp)THEN +IF(MPIRoot.AND.CalcBulkElectronTemp)THEN ASSOCIATE( Te => BulkElectronTemp ,& T => Temp(BulkElectronTempSpecID)*Kelvin2eV) ! Smooth the bulk electron temperature by adjusting the difference by 50% @@ -1715,7 +1703,7 @@ SUBROUTINE CalcTransTemp(NumSpec, Temp) END ASSOCIATE END IF #if USE_MPI -IF(CalcBulkElectronTemp) CALL MPI_BCAST(BulkElectronTemp,1,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,iError) +IF(CalcBulkElectronTemp) CALL MPI_BCAST(BulkElectronTemp,1,MPI_DOUBLE_PRECISION,0,MPI_COMM_PICLAS,iError) IF(SurfModSEEelectronTempAutoamtic) BulkElectronTempSEE = BulkElectronTemp #endif /*USE_MPI*/ @@ -1899,9 +1887,6 @@ SUBROUTINE CalcRelaxProbRotVib(RotRelaxProb,VibRelaxProb) USE MOD_Particle_Vars ,ONLY: nSpecies USE MOD_DSMC_Vars ,ONLY: DSMC, VarVibRelaxProb, CollisMode USE MOD_Mesh_Vars ,ONLY: nElems, nGlobalElems -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !----------------------------------------------------------------------------------------------------------------------------------- @@ -1929,13 +1914,13 @@ SUBROUTINE CalcRelaxProbRotVib(RotRelaxProb,VibRelaxProb) RotRelaxProb(2) = 0 END IF #if USE_MPI - IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,RotRelaxProb(1),1,MPI_DOUBLE_PRECISION,MPI_MAX,0,PartMPI%COMM, IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE,RotRelaxProb(2),1,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM, IERROR) - RotRelaxProb(2) = RotRelaxProb(2) / REAL(PartMPI%nProcs) + IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE,RotRelaxProb(1),1,MPI_DOUBLE_PRECISION,MPI_MAX,0,MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,RotRelaxProb(2),1,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) + RotRelaxProb(2) = RotRelaxProb(2) / REAL(nProcessors) ELSE - CALL MPI_REDUCE(RotRelaxProb(1),RotRelaxProb(1),1,MPI_DOUBLE_PRECISION,MPI_MAX,0,PartMPI%COMM, IERROR) - CALL MPI_REDUCE(RotRelaxProb(2),RotRelaxProb(2),1,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM, IERROR) + CALL MPI_REDUCE(RotRelaxProb(1),RotRelaxProb(1),1,MPI_DOUBLE_PRECISION,MPI_MAX,0,MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(RotRelaxProb(2),RotRelaxProb(2),1,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) END IF #endif /*USE_MPI*/ ELSE @@ -1953,25 +1938,25 @@ SUBROUTINE CalcRelaxProbRotVib(RotRelaxProb,VibRelaxProb) END DO VibRelaxProb(2)=VibRelaxProb(2)/(REAL(nSpecies)*REAL(nGlobalElems)) #if USE_MPI - IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,VibRelaxProb(1),1,MPI_DOUBLE_PRECISION,MPI_MAX,0,PartMPI%COMM, IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE,VibRelaxProb(2),1,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM, IERROR) + IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE,VibRelaxProb(1),1,MPI_DOUBLE_PRECISION,MPI_MAX,0,MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,VibRelaxProb(2),1,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) ELSE - CALL MPI_REDUCE(VibRelaxProb(1),VibRelaxProb(1),1,MPI_DOUBLE_PRECISION,MPI_MAX,0,PartMPI%COMM, IERROR) - CALL MPI_REDUCE(VibRelaxProb(2),VibRelaxProb(2),1,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM, IERROR) + CALL MPI_REDUCE(VibRelaxProb(1),VibRelaxProb(1),1,MPI_DOUBLE_PRECISION,MPI_MAX,0,MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(VibRelaxProb(2),VibRelaxProb(2),1,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) END IF #endif /*USE_MPI*/ ELSE VibRelaxProb = 0. #if USE_MPI - IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,DSMC%CalcVibProb(1:nSpecies,1),nSpecies,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM, IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE,DSMC%CalcVibProb(1:nSpecies,2),nSpecies,MPI_DOUBLE_PRECISION,MPI_MAX,0,PartMPI%COMM, IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE,DSMC%CalcVibProb(1:nSpecies,3),nSpecies,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM, IERROR) + IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE,DSMC%CalcVibProb(1:nSpecies,1),nSpecies,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,DSMC%CalcVibProb(1:nSpecies,2),nSpecies,MPI_DOUBLE_PRECISION,MPI_MAX,0,MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE,DSMC%CalcVibProb(1:nSpecies,3),nSpecies,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) ELSE - CALL MPI_REDUCE(DSMC%CalcVibProb(1:nSpecies,1),DSMC%CalcVibProb(1:nSpecies,1),nSpecies,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM, IERROR) - CALL MPI_REDUCE(DSMC%CalcVibProb(1:nSpecies,2),DSMC%CalcVibProb(1:nSpecies,2),nSpecies,MPI_DOUBLE_PRECISION,MPI_MAX,0,PartMPI%COMM, IERROR) - CALL MPI_REDUCE(DSMC%CalcVibProb(1:nSpecies,3),DSMC%CalcVibProb(1:nSpecies,3),nSpecies,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM, IERROR) + CALL MPI_REDUCE(DSMC%CalcVibProb(1:nSpecies,1),DSMC%CalcVibProb(1:nSpecies,1),nSpecies,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(DSMC%CalcVibProb(1:nSpecies,2),DSMC%CalcVibProb(1:nSpecies,2),nSpecies,MPI_DOUBLE_PRECISION,MPI_MAX,0,MPI_COMM_PICLAS, IERROR) + CALL MPI_REDUCE(DSMC%CalcVibProb(1:nSpecies,3),DSMC%CalcVibProb(1:nSpecies,3),nSpecies,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) END IF #endif /*USE_MPI*/ IF(MPIRoot)THEN @@ -2004,7 +1989,6 @@ SUBROUTINE CalcVelocities(PartVtrans, PartVtherm,NumSpec,SimNumSpec) USE MOD_Globals USE MOD_Particle_Analyze_Vars ,ONLY: VeloDirs USE MOD_Particle_Vars ,ONLY: PartState, PartSpecies, PDM, nSpecies, PartMPF, usevMPF -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_Particle_Analyze_Vars ,ONLY: nSpecAnalyze ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE @@ -2044,14 +2028,14 @@ SUBROUTINE CalcVelocities(PartVtrans, PartVtherm,NumSpec,SimNumSpec) END DO #if USE_MPI - IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,PartVtrans ,4*nSpecies,MPI_DOUBLE_PRECISION, MPI_SUM,0, PartMPI%COMM, IERROR) + IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE,PartVtrans ,4*nSpecies,MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) ELSE - CALL MPI_REDUCE(PartVtrans ,RD ,4*nSpecies,MPI_DOUBLE_PRECISION, MPI_SUM,0, PartMPI%COMM, IERROR) + CALL MPI_REDUCE(PartVtrans ,RD ,4*nSpecies,MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) END IF #endif /*USE_MPI*/ - IF(PartMPI%MPIRoot)THEN + IF(MPIRoot)THEN IF (usevMPF) THEN DO dir = 1,3 IF (VeloDirs(dir) .OR. VeloDirs(4)) THEN @@ -2080,7 +2064,7 @@ SUBROUTINE CalcVelocities(PartVtrans, PartVtherm,NumSpec,SimNumSpec) END IF #if USE_MPI - CALL MPI_BCAST(PartVtrans,4*nSpecies, MPI_DOUBLE_PRECISION,0,PartMPI%COMM,iERROR) + CALL MPI_BCAST(PartVtrans,4*nSpecies, MPI_DOUBLE_PRECISION,0,MPI_COMM_PICLAS,iERROR) #endif /*USE_MPI*/ ! calculate thermal velocity @@ -2101,14 +2085,14 @@ SUBROUTINE CalcVelocities(PartVtrans, PartVtherm,NumSpec,SimNumSpec) END DO #if USE_MPI - IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,PartVtherm,4*nSpecies,MPI_DOUBLE_PRECISION, MPI_SUM,0, PartMPI%COMM, IERROR) + IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE,PartVtherm,4*nSpecies,MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) ELSE - CALL MPI_REDUCE(PartVtherm ,RD ,4*nSpecies,MPI_DOUBLE_PRECISION, MPI_SUM,0, PartMPI%COMM, IERROR) + CALL MPI_REDUCE(PartVtherm ,RD ,4*nSpecies,MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) END IF #endif /*USE_MPI*/ - IF(PartMPI%MPIRoot)THEN + IF(MPIRoot)THEN DO dir = 1,3 IF (VeloDirs(dir) .OR. VeloDirs(4)) THEN DO iSpec = 1,nSpecies @@ -2149,7 +2133,6 @@ SUBROUTINE CollRates(CRate) USE MOD_TimeDisc_Vars ,ONLY: dt, iter USE MOD_Particle_Vars ,ONLY: VarTimeStep USE MOD_Particle_Analyze_Vars ,ONLY: PartAnalyzeStep -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_Particle_TimeStep ,ONLY: GetSpeciesTimeStep ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE @@ -2165,14 +2148,14 @@ SUBROUTINE CollRates(CRate) !=================================================================================================================================== #if USE_MPI -IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,DSMC%NumColl,CollInf%NumCase + 1,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) +IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE,DSMC%NumColl,CollInf%NumCase + 1,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) ELSE - CALL MPI_REDUCE(DSMC%NumColl,DSMC%NumColl,CollInf%NumCase + 1,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) + CALL MPI_REDUCE(DSMC%NumColl,DSMC%NumColl,CollInf%NumCase + 1,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) END IF #endif /*USE_MPI*/ -IF(PartMPI%MPIRoot)THEN +IF(MPIRoot)THEN DSMC%NumColl(CollInf%NumCase + 1) = SUM(DSMC%NumColl(1:CollInf%NumCase)) DO iCase=1, CollInf%NumCase ! Species-specific time step @@ -2218,9 +2201,6 @@ SUBROUTINE CalcRelaxRates(NumSpec,VibRelaxProbCase) USE MOD_TimeDisc_Vars ,ONLY: dt, iter USE MOD_Particle_Analyze_Vars ,ONLY: PartAnalyzeStep USE MOD_Particle_TimeStep ,ONLY: GetSpeciesTimeStep -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !----------------------------------------------------------------------------------------------------------------------------------- @@ -2236,15 +2216,15 @@ SUBROUTINE CalcRelaxRates(NumSpec,VibRelaxProbCase) #if USE_MPI IF(XSec_Relaxation) THEN - IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,SpecXSec(:)%VibCount,CollInf%NumCase,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM, IERROR) + IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE,SpecXSec(:)%VibCount,CollInf%NumCase,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) ELSE - CALL MPI_REDUCE(SpecXSec(:)%VibCount,SpecXSec(:)%VibCount,CollInf%NumCase,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM, IERROR) + CALL MPI_REDUCE(SpecXSec(:)%VibCount,SpecXSec(:)%VibCount,CollInf%NumCase,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) END IF END IF #endif /*USE_MPI*/ VibRelaxProbCase = 0. -IF(PartMPI%MPIRoot)THEN +IF(MPIRoot)THEN IF(XSec_Relaxation) THEN DO iSpec=1,nSpecies IF(NumSpec(iSpec).LE.0.0) CYCLE @@ -2293,7 +2273,6 @@ SUBROUTINE CalcRelaxRatesElec(ElecRelaxRate) USE MOD_MCC_Vars ,ONLY: SpecXSec USE MOD_TimeDisc_Vars ,ONLY: dt, iter USE MOD_Particle_Analyze_Vars ,ONLY: PartAnalyzeStep -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_Particle_TimeStep ,ONLY: GetSpeciesTimeStep ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE @@ -2321,14 +2300,14 @@ SUBROUTINE CalcRelaxRatesElec(ElecRelaxRate) END DO #if USE_MPI -IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,ElecRelaxRate,CollInf%NumCase*MaxLevel,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) +IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE,ElecRelaxRate,CollInf%NumCase*MaxLevel,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) ELSE - CALL MPI_REDUCE(ElecRelaxRate,ElecRelaxRate,CollInf%NumCase*MaxLevel,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) + CALL MPI_REDUCE(ElecRelaxRate,ElecRelaxRate,CollInf%NumCase*MaxLevel,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) END IF #endif /*USE_MPI*/ -IF(PartMPI%MPIRoot)THEN +IF(MPIRoot)THEN DO iCase=1, CollInf%NumCase IF(SpecXSec(iCase)%UseElecXSec) THEN ! Species-specific time step @@ -2371,7 +2350,6 @@ SUBROUTINE ReacRates(NumSpec, RRate) USE MOD_TimeDisc_Vars ,ONLY: dt, iter USE MOD_Particle_Vars ,ONLY: Species, nSpecies, VarTimeStep USE MOD_Particle_Mesh_Vars ,ONLY: MeshVolume -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_Particle_Analyze_Vars ,ONLY: PartAnalyzeStep USE MOD_Particle_TimeStep ,ONLY: GetSpeciesTimeStep ! IMPLICIT VARIABLE HANDLING @@ -2392,14 +2370,14 @@ SUBROUTINE ReacRates(NumSpec, RRate) !=================================================================================================================================== #if USE_MPI -IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE ,ChemReac%NumReac,ChemReac%NumOfReact,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) +IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE ,ChemReac%NumReac,ChemReac%NumOfReact,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) ELSE - CALL MPI_REDUCE(ChemReac%NumReac,RD ,ChemReac%NumOfReact,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,IERROR) + CALL MPI_REDUCE(ChemReac%NumReac,RD ,ChemReac%NumOfReact,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) END IF #endif /*USE_MPI*/ -IF(PartMPI%MPIRoot)THEN +IF(MPIRoot)THEN DO iReac=1, ChemReac%NumOfReact iCase = ChemReac%ReactCase(iReac) ! Species-specific time step diff --git a/src/particles/boundary/particle_boundary_init.f90 b/src/particles/boundary/particle_boundary_init.f90 index b6355c5aa..2d5d7dc5f 100644 --- a/src/particles/boundary/particle_boundary_init.f90 +++ b/src/particles/boundary/particle_boundary_init.f90 @@ -540,7 +540,7 @@ SUBROUTINE InitializeVariablesPartBoundary() IF(ANY(PartBound%SurfaceModel.EQ.1)) THEN ! Open the species database - CALL OpenDataFile(TRIM(SpeciesDatabase),create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(TRIM(SpeciesDatabase),create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) ! Check if the correct dataset exists StickingCoefficientExists = .FALSE. dsetname = TRIM('/Surface-Chemistry/StickingCoefficient') @@ -869,7 +869,7 @@ SUBROUTINE InitParticleBoundarySurfSides() color = MERGE(1337, MPI_UNDEFINED, nBCSidesProc.GT.0) END IF ! Create new surface communicator. Pass MPI_INFO_NULL as rank to follow the original ordering -CALL MPI_COMM_SPLIT(MPI_COMM_WORLD, color, MPI_INFO_NULL, SurfCOMM%UNICATOR, iError) +CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS, color, MPI_INFO_NULL, SurfCOMM%UNICATOR, iError) ! Find my rank on the shared communicator, comm size and proc name IF(SurfCOMM%UNICATOR.NE.MPI_COMM_NULL)THEN CALL MPI_COMM_RANK(SurfCOMM%UNICATOR, SurfCOMM%MyRank, iError) @@ -1284,7 +1284,7 @@ SUBROUTINE InitRotPeriodicMapping() DatasetName = 'RotPeriodicMap-v'//TRIM(hilf) WRITE(UNIT=hilf,FMT='(ES10.4)') ManualTimeStep DatasetName = TRIM(DatasetName)//'-dt'//TRIM(hilf) -CALL OpenDataFile(MeshFile,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(MeshFile,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) CALL DatasetExists(File_ID,TRIM(DatasetName),DatasetFound) CALL CloseDataFile() @@ -1716,7 +1716,7 @@ SUBROUTINE BuildParticleBoundaryRotPeriodic(notMappedTotal) CALL BARRIER_AND_SYNC(NumRotPeriodicNeigh_Shared_Win, MPI_COMM_SHARED) CALL BARRIER_AND_SYNC(RotPeriodicSideMapping_temp_Shared_Win, MPI_COMM_SHARED) ! The allreduce is only required when a global array for writing to .h5 is to be used -!CALL MPI_ALLREDUCE(MAXVAL(NumRotPeriodicNeigh) , MaxNumRotPeriodicNeigh , 1 , MPI_INTEGER , MPI_MAX , MPI_COMM_WORLD , iError) +!CALL MPI_ALLREDUCE(MAXVAL(NumRotPeriodicNeigh) , MaxNumRotPeriodicNeigh , 1 , MPI_INTEGER , MPI_MAX , MPI_COMM_PICLAS , iError) #endif /*USE_MPI*/ MaxNumRotPeriodicNeigh = MAXVAL(NumRotPeriodicNeigh) @@ -1748,7 +1748,7 @@ SUBROUTINE BuildParticleBoundaryRotPeriodic(notMappedTotal) END DO #if USE_MPI -CALL MPI_ALLREDUCE(notMapped , notMappedTotal , 1 , MPI_INTEGER , MPI_SUM , MPI_COMM_WORLD , IERROR) +CALL MPI_ALLREDUCE(notMapped , notMappedTotal , 1 , MPI_INTEGER , MPI_SUM , MPI_COMM_PICLAS , IERROR) #else notMappedTotal = notMapped #endif /*USE_MPI*/ diff --git a/src/particles/dsmc/dsmc_analyze.f90 b/src/particles/dsmc/dsmc_analyze.f90 index 9be9fc49e..6d28374e1 100644 --- a/src/particles/dsmc/dsmc_analyze.f90 +++ b/src/particles/dsmc/dsmc_analyze.f90 @@ -771,7 +771,7 @@ SUBROUTINE DSMC_output_calc(nVar,nVar_quality,nVarloc,DSMC_MacroVal) IF (DoVirtualCellMerge) THEN DO iElem = 1, nElems IF (VirtMergedCells(iElem)%isMerged) THEN - DSMC_MacroVal(:,iElem) = DSMC_MacroVal(:,VirtMergedCells(iElem)%MasterCell-offSetElem) + DSMC_MacroVal(:,iElem) = DSMC_MacroVal(:,VirtMergedCells(iElem)%MasterCell-offSetElem) END IF END DO END IF @@ -781,7 +781,7 @@ END SUBROUTINE DSMC_output_calc SUBROUTINE CalcMacroElecExcitation(MacroElecExcitation) !=================================================================================================================================== -!> +!> !=================================================================================================================================== ! MODULES USE MOD_PreProc @@ -802,7 +802,7 @@ SUBROUTINE CalcMacroElecExcitation(MacroElecExcitation) ! OUTPUT VARIABLES !----------------------------------------------------------------------------------------------------------------------------------- ! LOCAL VARIABLES -! INTEGER :: +! INTEGER :: REAL :: TimeSample !=================================================================================================================================== @@ -820,7 +820,7 @@ SUBROUTINE CalcMacroElecExcitation(MacroElecExcitation) TimeSample = (Time-(1-DSMC%TimeFracSamp)*TEnd) END IF -! Rate +! Rate MacroElecExcitation = ExcitationSampleData / TimeSample END SUBROUTINE CalcMacroElecExcitation @@ -1017,7 +1017,7 @@ SUBROUTINE WriteDSMCToHDF5(MeshFileName,OutputTime) CALL WriteAttributeToHDF5(File_ID,'Time',1,RealScalar=OutputTime) CALL WriteAttributeToHDF5(File_ID,'MeshFile',1,StrScalar=(/TRIM(MeshFileName)/)) CALL WriteAttributeToHDF5(File_ID,'NSpecies',1,IntegerScalar=nSpecies) - ! Standard variable names + ! Standard variable names CALL WriteAttributeToHDF5(File_ID,'VarNamesAdd',nVar+nVar_quality,StrArray=StrVarNames) ! Additional variable names: electronic excitation rate output IF(SampleElecExcitation) CALL WriteAttributeToHDF5(File_ID,'VarNamesExci',ExcitationLevelCounter,StrArray=StrVarNamesElecExci) @@ -1025,11 +1025,11 @@ SUBROUTINE WriteDSMCToHDF5(MeshFileName,OutputTime) END IF #if USE_MPI -CALL MPI_BARRIER(MPI_COMM_WORLD,iError) +CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif ! Open data file for parallel output -CALL OpenDataFile(FileName,create=.false.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(FileName,create=.false.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) ALLOCATE(DSMC_MacroVal(1:nVar+nVar_quality,nElems), STAT=ALLOCSTAT) IF (ALLOCSTAT.NE.0) THEN diff --git a/src/particles/dsmc/dsmc_bg_gas.f90 b/src/particles/dsmc/dsmc_bg_gas.f90 index dcbddbd00..2317de807 100644 --- a/src/particles/dsmc/dsmc_bg_gas.f90 +++ b/src/particles/dsmc/dsmc_bg_gas.f90 @@ -862,7 +862,7 @@ SUBROUTINE BGGas_ReadInDistribution() LBWRITE(UNIT_stdOut,*) 'BGGas distribution - Using macroscopic values from file: ',TRIM(MacroRestartFileName) -CALL OpenDataFile(MacroRestartFileName,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(MacroRestartFileName,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL GetDataSize(File_ID,'ElemData',nDims,HSize,attrib=.FALSE.) nVarHDF5 = INT(HSize(1),4) @@ -1145,7 +1145,7 @@ END SUBROUTINE BGGas_InitRegions !=================================================================================================================================== -!> Background gas regions: Set the internal temperatures in case of DSMC and CollisMode = 2/3 (not yet available during +!> Background gas regions: Set the internal temperatures in case of DSMC and CollisMode = 2/3 (not yet available during !> BGGas_InitRegions). Loop over all elements, species and inits per species to set values for molecules and/or atoms. !=================================================================================================================================== SUBROUTINE BGGas_RegionsSetInternalTemp() diff --git a/src/particles/dsmc/dsmc_init.f90 b/src/particles/dsmc/dsmc_init.f90 index 60e1be226..f2d7ff97d 100644 --- a/src/particles/dsmc/dsmc_init.f90 +++ b/src/particles/dsmc/dsmc_init.f90 @@ -1191,7 +1191,7 @@ SUBROUTINE SetVarVibProb2Elems() ! Set initial vibrational relaxation probability to all elements !=================================================================================================================================== ! MODULES ! -USE MOD_Globals ,ONLY: abort, IK, MPI_COMM_WORLD +USE MOD_Globals ,ONLY: abort, IK, MPI_COMM_PICLAS USE MOD_PARTICLE_Vars ,ONLY: nSpecies, Species USE MOD_Restart_Vars ,ONLY: DoRestart,RestartFile USE MOD_Particle_Vars ,ONLY: nSpecies, PartSpecies @@ -1228,7 +1228,7 @@ SUBROUTINE SetVarVibProb2Elems() VarVibRelaxProb%ProbVibAv = 0 VibProbInitDone = .FALSE. IF (DoRestart) THEN - CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) ! read local ParticleInfo from HDF5 CALL DatasetExists(File_ID,'VibProbInfo',VibProbDataExists) IF(VibProbDataExists)THEN @@ -1518,7 +1518,7 @@ RECURSIVE SUBROUTINE DeleteNodeVolume(Node) INTEGER :: iLoop, nLoop !=================================================================================================================================== nLoop = 2**Symmetry%Order -IF(ASSOCIATED(Node%SubNode)) THEN +IF(ASSOCIATED(Node%SubNode)) THEN DO iLoop = 1, nLoop CALL DeleteNodeVolume(Node%SubNode(iLoop)) END DO diff --git a/src/particles/emission/particle_br_electron_fluid.f90 b/src/particles/emission/particle_br_electron_fluid.f90 index df94d3872..1abef0f12 100644 --- a/src/particles/emission/particle_br_electron_fluid.f90 +++ b/src/particles/emission/particle_br_electron_fluid.f90 @@ -257,7 +257,7 @@ SUBROUTINE InitializeVariablesElectronFluidRegions() END IF ! MPIRoot #if USE_MPI ! Broadcast from root to other processors - CALL MPI_BCAST(RegionElectronRefHDF5,3, MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,iERROR) + CALL MPI_BCAST(RegionElectronRefHDF5,3, MPI_DOUBLE_PRECISION,0,MPI_COMM_PICLAS,iERROR) #endif /*USE_MPI*/ END IF ! DoRestart @@ -613,7 +613,7 @@ SUBROUTINE CalculateBRAutomaticElectronRef() ! Sum the number of elements (later required for averaging the cell-constant values globally on each processor) #if USE_MPI -CALL MPI_ALLREDUCE(RegionElectronRefNew , RegionElectronRefNewGlobal , 3 , MPI_DOUBLE_PRECISION , MPI_SUM , MPI_COMM_WORLD , IERROR) +CALL MPI_ALLREDUCE(RegionElectronRefNew , RegionElectronRefNewGlobal , 3 , MPI_DOUBLE_PRECISION , MPI_SUM , MPI_COMM_PICLAS , IERROR) #else RegionElectronRefNewGlobal = RegionElectronRefNew #endif /*USE_MPI*/ @@ -903,7 +903,7 @@ END FUNCTION LesserThanWithTolerance SUBROUTINE CreateElectronsFromBRFluid(CreateFromRestartFile) ! MODULES USE MOD_Globals -USE MOD_Globals ,ONLY: abort,MPIRoot,UNIT_stdOut,IK,MPI_COMM_WORLD +USE MOD_Globals ,ONLY: abort,MPIRoot,UNIT_stdOut,IK,MPI_COMM_PICLAS USE MOD_Globals_Vars ,ONLY: ElementaryCharge,BoltzmannConst USE MOD_PreProc USE MOD_Particle_Vars ,ONLY: PDM,PEM,PartState,nSpecies,Species,PartSpecies,usevMPF @@ -947,7 +947,7 @@ SUBROUTINE CreateElectronsFromBRFluid(CreateFromRestartFile) IF(CreateFromRestartFile)THEN ALLOCATE(ElectronDensityCell(1,1:PP_nElems)) ALLOCATE(ElectronTemperatureCell(1,1:PP_nElems)) - CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL DatasetExists(File_ID,'ElectronDensityCell',ElectronDensityCellExists) IF(ElectronDensityCellExists)THEN ! Associate construct for integer KIND=8 possibility @@ -1069,7 +1069,7 @@ SUBROUTINE CreateElectronsFromBRFluid(CreateFromRestartFile) END IF ! CreateFromRestartFile #if USE_MPI -CALL MPI_ALLREDUCE(MPI_IN_PLACE,BRNbrOfElectronsCreated,1,MPI_INTEGER,MPI_SUM,MPI_COMM_WORLD,iError) +CALL MPI_ALLREDUCE(MPI_IN_PLACE,BRNbrOfElectronsCreated,1,MPI_INTEGER,MPI_SUM,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ SWRITE(UNIT_StdOut,'(A,I0,A)') ' Created a total of ',BRNbrOfElectronsCreated,' electrons.' @@ -1205,7 +1205,7 @@ SUBROUTINE InitBRAutomaticElectronRefElements() ! Sum the number of elements (later required for averaging the cell-constant values globally on each processor) #if USE_MPI -CALL MPI_ALLREDUCE(nBRAverageElems , nBRAverageElemsGlobal , 1 , MPI_INTEGER , MPI_SUM , MPI_COMM_WORLD , IERROR) +CALL MPI_ALLREDUCE(nBRAverageElems , nBRAverageElemsGlobal , 1 , MPI_INTEGER , MPI_SUM , MPI_COMM_PICLAS , IERROR) #else nBRAverageElemsGlobal = nBRAverageElems #endif /*USE_MPI*/ diff --git a/src/particles/emission/particle_emission.f90 b/src/particles/emission/particle_emission.f90 index 9d19810ca..7f772b69c 100644 --- a/src/particles/emission/particle_emission.f90 +++ b/src/particles/emission/particle_emission.f90 @@ -34,7 +34,7 @@ SUBROUTINE ParticleInserting() !=================================================================================================================================== ! Modules #if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI +USE MOD_Particle_MPI_Vars ,ONLY: PartMPIInitGroup #endif /*USE_MPI*/ USE MOD_Globals USE MOD_Globals_Vars ,ONLY: c,PI @@ -178,10 +178,10 @@ SUBROUTINE ParticleInserting() #if USE_MPI ! communicate number of particles with all procs in the same init group InitGroup=Species(i)%Init(iInit)%InitCOMM - IF(PartMPI%InitGroup(InitGroup)%COMM.NE.MPI_COMM_NULL) THEN + IF(PartMPIInitGroup(InitGroup)%COMM.NE.MPI_COMM_NULL) THEN ! only procs which are part of group take part in the communication !NbrOfParticle based on RandVals! - CALL MPI_BCAST(NbrOfParticle, 1, MPI_INTEGER,0,PartMPI%InitGroup(InitGroup)%COMM,IERROR) + CALL MPI_BCAST(NbrOfParticle, 1, MPI_INTEGER,0,PartMPIInitGroup(InitGroup)%COMM,IERROR) ELSE NbrOfParticle=0 END IF @@ -280,11 +280,11 @@ SUBROUTINE ParticleInserting() ! Communicate number of particles with all procs in the same init group InitGroup=Species(i)%Init(iInit)%InitCOMM NeutralizationBalanceGlobal=0 ! always nullify - IF(PartMPI%InitGroup(InitGroup)%COMM.NE.MPI_COMM_NULL) THEN + IF(PartMPIInitGroup(InitGroup)%COMM.NE.MPI_COMM_NULL) THEN ! Loop over all elements and count the ion surplus per element if element-local emission is used IF(nNeutralizationElems.GT.0) CALL CountNeutralizationParticles() ! Only processors which are part of group take part in the communication - CALL MPI_ALLREDUCE(NeutralizationBalance,NeutralizationBalanceGlobal,1,MPI_INTEGER,MPI_SUM,PartMPI%InitGroup(InitGroup)%COMM,IERROR) + CALL MPI_ALLREDUCE(NeutralizationBalance,NeutralizationBalanceGlobal,1,MPI_INTEGER,MPI_SUM,PartMPIInitGroup(InitGroup)%COMM,IERROR) ELSE NeutralizationBalanceGlobal=0 END IF @@ -359,18 +359,18 @@ SUBROUTINE ParticleInserting() ! Complete check if all particles were emitted successfully #if USE_MPI InitGroup=Species(i)%Init(iInit)%InitCOMM - IF (PartMPI%InitGroup(InitGroup)%COMM.NE.MPI_COMM_NULL .AND. Species(i)%Init(iInit)%sumOfRequestedParticles.GT.0) THEN + IF (PartMPIInitGroup(InitGroup)%COMM.NE.MPI_COMM_NULL .AND. Species(i)%Init(iInit)%sumOfRequestedParticles.GT.0) THEN #if defined(MEASURE_MPI_WAIT) CALL SYSTEM_CLOCK(count=CounterStart) #endif /*defined(MEASURE_MPI_WAIT)*/ - CALL MPI_WAIT(PartMPI%InitGroup(InitGroup)%Request, MPI_STATUS_IGNORE, iError) + CALL MPI_WAIT(PartMPIInitGroup(InitGroup)%Request, MPI_STATUS_IGNORE, iError) #if defined(MEASURE_MPI_WAIT) CALL SYSTEM_CLOCK(count=CounterEnd, count_rate=Rate) MPIW8TimePart(5) = MPIW8TimePart(5) + REAL(CounterEnd-CounterStart,8)/Rate MPIW8CountPart(5) = MPIW8CountPart(5) + 1_8 #endif /*defined(MEASURE_MPI_WAIT)*/ - IF(PartMPI%InitGroup(InitGroup)%MPIRoot) THEN + IF(PartMPIInitGroup(InitGroup)%MPIRoot) THEN #endif ! add number of matching error to particle emission to fit ! number of added particles @@ -390,7 +390,7 @@ SUBROUTINE ParticleInserting() ! WRITE(UNIT_stdOut,'(A,I0,A)') 'ParticleEmission_parallel: matched all (',NbrOfParticle,') particles!' END IF #if USE_MPI - END IF ! PartMPI%iProc.EQ.0 + END IF ! PartMPIInitGroup(InitGroup)%MPIRoot END IF #endif END DO ! iInit diff --git a/src/particles/emission/particle_emission_init.f90 b/src/particles/emission/particle_emission_init.f90 index c45b5ca21..a8dd2dcc6 100644 --- a/src/particles/emission/particle_emission_init.f90 +++ b/src/particles/emission/particle_emission_init.f90 @@ -747,9 +747,6 @@ SUBROUTINE DetermineInitialParticleNumber() !> !=================================================================================================================================== ! MODULES -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ USE MOD_Globals USE MOD_Globals_Vars ,ONLY: PI USE MOD_DSMC_Vars ,ONLY: RadialWeighting, DSMC @@ -833,7 +830,7 @@ SUBROUTINE DetermineInitialParticleNumber() END IF ! Sum-up the number of particles to be inserted #if USE_MPI - insertParticles = insertParticles + INT(REAL(Species(iSpec)%Init(iInit)%ParticleNumber)/REAL(PartMPI%nProcs),8) + insertParticles = insertParticles + INT(REAL(Species(iSpec)%Init(iInit)%ParticleNumber)/REAL(nProcessors),8) #else insertParticles = insertParticles + INT(Species(iSpec)%Init(iInit)%ParticleNumber,8) #endif diff --git a/src/particles/emission/particle_position_and_velocity.f90 b/src/particles/emission/particle_position_and_velocity.f90 index b27368573..a67d4a9cc 100644 --- a/src/particles/emission/particle_position_and_velocity.f90 +++ b/src/particles/emission/particle_position_and_velocity.f90 @@ -52,7 +52,7 @@ SUBROUTINE SetParticlePositionCellLocal(FractNbr,iInit,NbrOfParticle) USE MOD_Particle_Vars ,ONLY: Species,Symmetry USE MOD_part_Emission_Tools ,ONLY: IntegerDivide,SetCellLocalParticlePosition #if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI +USE MOD_Particle_MPI_Vars ,ONLY: PartMPIInitGroup USE MOD_Particle_Mesh_Vars ,ONLY: LocalVolume #endif /*USE_MPI*/ !---------------------------------------------------------------------------------------------------------------------------------- @@ -88,15 +88,15 @@ SUBROUTINE SetParticlePositionCellLocal(FractNbr,iInit,NbrOfParticle) #if USE_MPI ! emission group communicator InitGroup=Species(FractNbr)%Init(iInit)%InitCOMM -IF(PartMPI%InitGroup(InitGroup)%COMM.EQ.MPI_COMM_NULL) THEN +IF(PartMPIInitGroup(InitGroup)%COMM.EQ.MPI_COMM_NULL) THEN NbrofParticle=0 RETURN END IF -IF (PartMPI%InitGroup(InitGroup)%nProcs.GT.1) THEN +IF (PartMPIInitGroup(InitGroup)%nProcs.GT.1) THEN IF (DoExactPartNumInsert) THEN !###$ ToDo - IF (PartMPI%InitGroup(InitGroup)%MPIROOT) THEN - ALLOCATE(ProcMeshVol(0:PartMPI%InitGroup(InitGroup)%nProcs-1)) - ALLOCATE(ProcNbrOfParticle(0:PartMPI%InitGroup(InitGroup)%nProcs-1)) + IF (PartMPIInitGroup(InitGroup)%MPIROOT) THEN + ALLOCATE(ProcMeshVol(0:PartMPIInitGroup(InitGroup)%nProcs-1)) + ALLOCATE(ProcNbrOfParticle(0:PartMPIInitGroup(InitGroup)%nProcs-1)) ProcMeshVol=0. ProcNbrOfParticle=0 ELSE ! to reduce global memory allocation if a lot of procs are used @@ -106,11 +106,11 @@ SUBROUTINE SetParticlePositionCellLocal(FractNbr,iInit,NbrOfParticle) ProcNbrOfParticle=0 END IF !InitGroup%MPIroot CALL MPI_GATHER(LocalVolume,1,MPI_DOUBLE_PRECISION & - ,ProcMeshVol,1,MPI_DOUBLE_PRECISION,0,PartMPI%InitGroup(InitGroup)%COMM,iError) - IF (PartMPI%InitGroup(InitGroup)%MPIROOT) THEN - CALL IntegerDivide(NbrOfParticle,PartMPI%InitGroup(InitGroup)%nProcs,ProcMeshVol,ProcNbrOfParticle) + ,ProcMeshVol,1,MPI_DOUBLE_PRECISION,0,PartMPIInitGroup(InitGroup)%COMM,iError) + IF (PartMPIInitGroup(InitGroup)%MPIROOT) THEN + CALL IntegerDivide(NbrOfParticle,PartMPIInitGroup(InitGroup)%nProcs,ProcMeshVol,ProcNbrOfParticle) END IF - CALL MPI_SCATTER(ProcNbrOfParticle, 1, MPI_INTEGER, chunksize, 1, MPI_INTEGER, 0, PartMPI%InitGroup(InitGroup)%COMM, IERROR) + CALL MPI_SCATTER(ProcNbrOfParticle, 1, MPI_INTEGER, chunksize, 1, MPI_INTEGER, 0, PartMPIInitGroup(InitGroup)%COMM, IERROR) SDEALLOCATE(ProcMeshVol) SDEALLOCATE(ProcNbrOfParticle) END IF @@ -149,7 +149,7 @@ SUBROUTINE SetParticlePosition(FractNbr,iInit,NbrOfParticle) USE MOD_DSMC_Vars ,ONLY: RadialWeighting #if USE_MPI USE MOD_Particle_MPI_Emission ,ONLY: SendEmissionParticlesToProcs -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI +USE MOD_Particle_MPI_Vars ,ONLY: PartMPIInitGroup #endif /*USE_MPI*/ !---------------------------------------------------------------------------------------------------------------------------------- ! IMPLICIT VARIABLE HANDLING @@ -185,7 +185,7 @@ SUBROUTINE SetParticlePosition(FractNbr,iInit,NbrOfParticle) ! emission group communicator #if USE_MPI InitGroup=Species(FractNbr)%Init(iInit)%InitCOMM -IF(PartMPI%InitGroup(InitGroup)%COMM.EQ.MPI_COMM_NULL) THEN +IF(PartMPIInitGroup(InitGroup)%COMM.EQ.MPI_COMM_NULL) THEN NbrofParticle=0 RETURN END IF @@ -194,15 +194,15 @@ SUBROUTINE SetParticlePosition(FractNbr,iInit,NbrOfParticle) (TRIM(Species(FractNbr)%Init(iInit)%SpaceIC).EQ.'circle' ) .OR. & (TRIM(Species(FractNbr)%Init(iInit)%SpaceIC).EQ.'line_with_equidistant_distribution' )) THEN nChunks = 1 -ELSE IF (nbrOfParticle.GT.(PartMPI%InitGroup(InitGroup)%nProcs*10)) THEN - nChunks = PartMPI%InitGroup(InitGroup)%nProcs +ELSE IF (nbrOfParticle.GT.(PartMPIInitGroup(InitGroup)%nProcs*10)) THEN + nChunks = PartMPIInitGroup(InitGroup)%nProcs ELSE nChunks = 1 END IF ! Set default chunkSize chunkSize = INT(nbrOfParticle/nChunks) -IF (PartMPI%InitGroup(InitGroup)%MPIROOT) THEN +IF (PartMPIInitGroup(InitGroup)%MPIROOT) THEN chunkSize = chunkSize*(1-nChunks) + nbrOfParticle END IF @@ -218,7 +218,7 @@ SUBROUTINE SetParticlePosition(FractNbr,iInit,NbrOfParticle) #if USE_MPI ! all proc taking part in particle inserting -IF (PartMPI%InitGroup(InitGroup)%MPIROOT.OR.nChunks.GT.1) THEN +IF (PartMPIInitGroup(InitGroup)%MPIROOT.OR.nChunks.GT.1) THEN #endif ! Allocate part pos buffer IF(RadialWeighting%DoRadialWeighting.AND.(chunkSize.GT.PDM%maxParticleNumber)) THEN @@ -296,7 +296,7 @@ SUBROUTINE SetParticlePosition(FractNbr,iInit,NbrOfParticle) chunkSize = 0 END IF ! Need to open MPI communication regardless of the chunk number. Make it only dependent on the number of procs -IF (PartMPI%InitGroup(InitGroup)%nProcs.GT.1) THEN +IF (PartMPIInitGroup(InitGroup)%nProcs.GT.1) THEN CALL SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,Species(FractNbr)%Init(iInit)%mySumOfMatchedParticles,particle_positions) ! Finish emission on local proc ELSE @@ -331,8 +331,8 @@ SUBROUTINE SetParticlePosition(FractNbr,iInit,NbrOfParticle) , MPI_INTEGER & , MPI_SUM & , 0 & - , PartMPI%InitGroup(InitGroup)%COMM & - , PartMPI%InitGroup(InitGroup)%Request & + , PartMPIInitGroup(InitGroup)%COMM & + , PartMPIInitGroup(InitGroup)%Request & , IERROR) #else ! in the serial case, particles are only emitted on the current processor @@ -546,7 +546,7 @@ SUBROUTINE SetPartPosAndVeloEmissionDistribution(iSpec,iInit,NbrOfParticle) ! modules !USE MOD_Globals USE MOD_PreProc -USE MOD_Globals ,ONLY: myrank,UNIT_StdOut,MPI_COMM_WORLD,abort +USE MOD_Globals ,ONLY: myrank,UNIT_StdOut,abort USE MOD_part_tools ,ONLY: InitializeParticleMaxwell,InterpolateEmissionDistribution2D USE MOD_Mesh_Vars ,ONLY: nElems,offsetElem USE MOD_Particle_Vars ,ONLY: Species, PDM, PartState, PEM, LastPartPos diff --git a/src/particles/emission/particle_surface_flux.f90 b/src/particles/emission/particle_surface_flux.f90 index 936941611..a1aa239e7 100644 --- a/src/particles/emission/particle_surface_flux.f90 +++ b/src/particles/emission/particle_surface_flux.f90 @@ -56,9 +56,6 @@ SUBROUTINE ParticleSurfaceflux() USE MOD_LoadBalance_Vars ,ONLY: nSurfacefluxPerElem USE MOD_LoadBalance_Timers ,ONLY: LBStartTime, LBElemSplitTime, LBPauseTime #endif /*USE_LOADBALANCE*/ -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !----------------------------------------------------------------------------------------------------------------------------------- @@ -96,7 +93,7 @@ SUBROUTINE ParticleSurfaceflux() ! Adaptive BC, Type = 4 (Const. massflow): Sum-up the global number of particles exiting through BC and calculate new weights IF(SF%AdaptiveType.EQ.4) THEN #if USE_MPI - CALL MPI_ALLREDUCE(MPI_IN_PLACE,AdaptBCPartNumOut(iSpec,iSF),1,MPI_INTEGER,MPI_SUM,PartMPI%COMM,IERROR) + CALL MPI_ALLREDUCE(MPI_IN_PLACE,AdaptBCPartNumOut(iSpec,iSF),1,MPI_INTEGER,MPI_SUM,MPI_COMM_PICLAS,IERROR) #endif IF(.NOT.ALMOSTEQUAL(SF%AdaptiveMassflow,0.)) CALL CalcConstMassflowWeight(iSpec,iSF) END IF @@ -329,9 +326,6 @@ SUBROUTINE CalcPartInsSubSidesStandardCase(iSpec, iSF, PartInsSubSides) USE MOD_TimeDisc_Vars ,ONLY: dt, RKdtFrac, RKdtFracTotal, Time USE MOD_Particle_Surfaces_Vars ,ONLY: SurfFluxSideSize, BCdata_auxSF USE MOD_Part_Emission_Tools ,ONLY: IntegerDivide, SamplePoissonDistri -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !----------------------------------------------------------------------------------------------------------------------------------- @@ -404,7 +398,7 @@ SUBROUTINE CalcPartInsSubSidesStandardCase(iSpec, iSF, PartInsSubSides) END IF !ReduceNoise, MPIroot #if USE_MPI IF (SF%ReduceNoise) THEN !scatter PartInsProc into PartInsSF of procs - CALL MPI_SCATTER(PartInsProc(0:nProcessors-1),1,MPI_INTEGER,PartInsSF,1,MPI_INTEGER,0,PartMPI%COMM,IERROR) + CALL MPI_SCATTER(PartInsProc(0:nProcessors-1),1,MPI_INTEGER,PartInsSF,1,MPI_INTEGER,0,MPI_COMM_PICLAS,IERROR) END IF !ReduceNoise #endif /*USE_MPI*/ !-- calc global to-be-inserted number of parts and distribute to SubSides (proc local) @@ -1059,9 +1053,6 @@ SUBROUTINE CalcConstMassflowWeight(iSpec,iSF) USE MOD_Particle_Surfaces_Vars ,ONLY: SurfMeshSubSideData, BCdata_auxSF, SurfFluxSideSize USE MOD_Mesh_Vars ,ONLY: SideToElem, offsetElem USE MOD_Particle_Mesh_Tools ,ONLY: GetGlobalNonUniqueSideID -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif !----------------------------------------------------------------------------------------------------------------------------------! ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE @@ -1152,7 +1143,7 @@ SUBROUTINE CalcConstMassflowWeight(iSpec,iSF) ! Calculate the total volume flow rate over the whole BC across all processors #if USE_MPI -CALL MPI_ALLREDUCE(MPI_IN_PLACE,nVFRTotal,1,MPI_DOUBLE,MPI_SUM,PartMPI%COMM,IERROR) +CALL MPI_ALLREDUCE(MPI_IN_PLACE,nVFRTotal,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_PICLAS,IERROR) #endif ! Determine the weight of each side compared to the total volume flow rate diff --git a/src/particles/emission/particle_surface_flux_init.f90 b/src/particles/emission/particle_surface_flux_init.f90 index 3dc9f6fd1..17d5a0d1f 100644 --- a/src/particles/emission/particle_surface_flux_init.f90 +++ b/src/particles/emission/particle_surface_flux_init.f90 @@ -139,7 +139,6 @@ SUBROUTINE InitializeParticleSurfaceflux() USE MOD_DSMC_Vars ,ONLY: AmbiPolarSFMapping, DSMC, useDSMC #if USE_MPI USE MOD_Particle_Vars ,ONLY: DoPoissonRounding, DoTimeDepInflow -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI #endif /*USE_MPI*/ #ifdef CODE_ANALYZE USE MOD_Particle_Vars ,ONLY: CountCircInflowType @@ -182,8 +181,8 @@ SUBROUTINE InitializeParticleSurfaceflux() CALL ReadInAndPrepareSurfaceFlux(MaxSurfacefluxBCs, nDataBC) #if USE_MPI -CALL MPI_ALLREDUCE(MPI_IN_PLACE,DoPoissonRounding,1,MPI_LOGICAL,MPI_LAND,PartMPI%COMM,iError) !set T if this is for all procs -CALL MPI_ALLREDUCE(MPI_IN_PLACE,DoTimeDepInflow,1,MPI_LOGICAL,MPI_LAND,PartMPI%COMM,iError) !set T if this is for all procs +CALL MPI_ALLREDUCE(MPI_IN_PLACE,DoPoissonRounding,1,MPI_LOGICAL,MPI_LAND,MPI_COMM_PICLAS,iError) !set T if this is for all procs +CALL MPI_ALLREDUCE(MPI_IN_PLACE,DoTimeDepInflow,1,MPI_LOGICAL,MPI_LAND,MPI_COMM_PICLAS,iError) !set T if this is for all procs #endif /*USE_MPI*/ CALL CreateSideListAndFinalizeAreasSurfFlux(nDataBC, BCdata_auxSFTemp) @@ -253,7 +252,7 @@ SUBROUTINE InitializeParticleSurfaceflux() IF(.NOT.Species(iSpec)%Surfaceflux(iSF)%CircularInflow) THEN totalAreaSF_global = 0.0 CALL MPI_ALLREDUCE(Species(iSpec)%Surfaceflux(iSF)%totalAreaSF,totalAreaSF_global,1, & - MPI_DOUBLE_PRECISION,MPI_SUM,PartMPI%COMM,IERROR) + MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_PICLAS,IERROR) Species(iSpec)%Surfaceflux(iSF)%totalAreaSF = totalAreaSF_global END IF #endif @@ -340,7 +339,7 @@ SUBROUTINE InitializeParticleSurfaceflux() END DO ! iSpec=1,nSpecies #if USE_MPI -CALL MPI_ALLREDUCE(MPI_IN_PLACE,DoSurfaceFlux,1,MPI_LOGICAL,MPI_LOR,PartMPI%COMM,iError) !set T if at least 1 proc have SFs +CALL MPI_ALLREDUCE(MPI_IN_PLACE,DoSurfaceFlux,1,MPI_LOGICAL,MPI_LOR,MPI_COMM_PICLAS,iError) !set T if at least 1 proc have SFs #endif /*USE_MPI*/ IF (.NOT.DoSurfaceFlux) THEN !-- no SFs defined LBWRITE(*,*) 'WARNING: No Sides for SurfacefluxBCs found! DoSurfaceFlux is now disabled!' @@ -695,9 +694,6 @@ SUBROUTINE CreateSideListAndFinalizeAreasSurfFlux(nDataBC, BCdata_auxSFTemp) USE MOD_DSMC_Symmetry ,ONLY: DSMC_1D_CalcSymmetryArea, DSMC_2D_CalcSymmetryArea, DSMC_2D_CalcSymmetryAreaSubSides USE MOD_DSMC_Vars ,ONLY: RadialWeighting USE MOD_Particle_Surfaces ,ONLY: CalcNormAndTangTriangle -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !----------------------------------------------------------------------------------------------------------------------------------- @@ -872,7 +868,7 @@ SUBROUTINE CreateSideListAndFinalizeAreasSurfFlux(nDataBC, BCdata_auxSFTemp) DO iPartBound=1,nPartBound areasLoc(iPartBound)=BCdata_auxSF(iPartBound)%LocalArea END DO - CALL MPI_ALLREDUCE(areasLoc,areasGlob,nPartBound,MPI_DOUBLE_PRECISION,MPI_SUM,PartMPI%COMM,IERROR) + CALL MPI_ALLREDUCE(areasLoc,areasGlob,nPartBound,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_PICLAS,IERROR) #endif DO iPartBound=1,nPartBound #if USE_MPI @@ -1049,9 +1045,6 @@ SUBROUTINE InitReduceNoiseSF(iSpec, iSF) ! MODULES USE MOD_Globals USE MOD_Particle_Vars ,ONLY: Species -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !----------------------------------------------------------------------------------------------------------------------------------- @@ -1073,9 +1066,9 @@ SUBROUTINE InitReduceNoiseSF(iSpec, iSF) END IF !MPIroot #if USE_MPI CALL MPI_GATHER(Species(iSpec)%Surfaceflux(iSF)%VFR_total,1,MPI_DOUBLE_PRECISION & - ,Species(iSpec)%Surfaceflux(iSF)%VFR_total_allProcs,1,MPI_DOUBLE_PRECISION,0,PartMPI%COMM,iError) + ,Species(iSpec)%Surfaceflux(iSF)%VFR_total_allProcs,1,MPI_DOUBLE_PRECISION,0,MPI_COMM_PICLAS,iError) IF(MPIroot)THEN - DO iProc=0,PartMPI%nProcs-1 + DO iProc=0,nProcessors-1 Species(iSpec)%Surfaceflux(iSF)%VFR_total_allProcsTotal = Species(iSpec)%Surfaceflux(iSF)%VFR_total_allProcsTotal & + Species(iSpec)%Surfaceflux(iSF)%VFR_total_allProcs(iProc) END DO diff --git a/src/particles/particle_init.f90 b/src/particles/particle_init.f90 index 9d0ce0df7..308dd4106 100644 --- a/src/particles/particle_init.f90 +++ b/src/particles/particle_init.f90 @@ -130,7 +130,7 @@ SUBROUTINE DefineParametersParticles() 'i.e. how deep the merge extends into the mesh starting from \n'//& 'each cell. 0 is the least aggressive merge, 2 the most \n'//& 'aggressive merge.','0') -CALL prms%CreateIntOption( 'Part-MaxNumbCellsMerge' ,'Maximum number of cells to be merged.','4') +CALL prms%CreateIntOption( 'Part-MaxNumbCellsMerge' ,'Maximum number of cells to be merged.','4') CALL prms%SetSection("IMD") ! IMD things @@ -408,7 +408,6 @@ SUBROUTINE InitializeVariables() #if USE_MPI USE MOD_Particle_MPI_Emission ,ONLY: InitEmissionComm USE MOD_Particle_MPI_Halo ,ONLY: IdentifyPartExchangeProcs -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI #endif /*USE_MPI*/ #ifdef CODE_ANALYZE USE MOD_PICInterpolation_Vars ,ONLY: DoInterpolationAnalytic @@ -504,7 +503,7 @@ SUBROUTINE InitializeVariables() #if USE_MPI CALL InitEmissionComm() -CALL MPI_BARRIER(PartMPI%COMM,IERROR) +CALL MPI_BARRIER(MPI_COMM_PICLAS,IERROR) #endif /*USE_MPI*/ #if defined(PARTICLES) && USE_HDG @@ -690,7 +689,7 @@ SUBROUTINE InitializeVariablesVirtualCellMerge() DoVirtualCellMerge = GETLOGICAL('Part-DoVirtualCellMerge') IF(DoVirtualCellMerge)THEN #if USE_MPI -DoParticleLatencyHiding = .FALSE. +DoParticleLatencyHiding = .FALSE. #endif VirtualCellMergeSpread = GETINT('Part-CellMergeSpread') MaxNumOfMergedCells = GETINT('Part-MaxNumbCellsMerge') @@ -1449,7 +1448,7 @@ SUBROUTINE InitRandomSeed(nRandomSeeds,SeedSize,Seeds) !=================================================================================================================================== ! MODULES #if USE_MPI -USE MOD_Particle_MPI_Vars, ONLY:PartMPI +USE MOD_Globals #endif ! IMPLICIT VARIABLE HANDLING !=================================================================================================================================== @@ -1501,9 +1500,9 @@ SUBROUTINE InitRandomSeed(nRandomSeeds,SeedSize,Seeds) DO iSeed = 1, SeedSize #if USE_MPI IF (nRandomSeeds.EQ.0) THEN - AuxilaryClock=AuxilaryClock+PartMPI%MyRank + AuxilaryClock=AuxilaryClock+myRank ELSE IF(nRandomSeeds.GT.0) THEN - AuxilaryClock=AuxilaryClock+(PartMPI%MyRank+1)*INT(Seeds(iSeed),8)*37 + AuxilaryClock=AuxilaryClock+(myRank+1)*INT(Seeds(iSeed),8)*37 END IF #else IF (nRandomSeeds.GT.0) THEN diff --git a/src/particles/particle_mesh/particle_bgm.f90 b/src/particles/particle_mesh/particle_bgm.f90 index f863fe648..e90e53ca2 100644 --- a/src/particles/particle_mesh/particle_bgm.f90 +++ b/src/particles/particle_mesh/particle_bgm.f90 @@ -383,7 +383,7 @@ SUBROUTINE BuildBGMAndIdentifyHaloRegion() END SELECT CharacteristicLengthMax = MAX(CharacteristicLengthMax,CharacteristicLength) END DO ! iElem = 1, nElems - CALL MPI_ALLREDUCE(MPI_IN_PLACE,CharacteristicLengthMax,1,MPI_DOUBLE_PRECISION,MPI_MAX,MPI_COMM_WORLD,iError) + CALL MPI_ALLREDUCE(MPI_IN_PLACE,CharacteristicLengthMax,1,MPI_DOUBLE_PRECISION,MPI_MAX,MPI_COMM_PICLAS,iError) r_sf = 1.1 * CharacteristicLengthMax ! Increase by 10% IF(CharacteristicLength.LE.0.) CALL abort(__STAMP__,'CharacteristicLength.LE.0. is not allowed.') CALL PrintOption('Global shape function radius from elements: PIC-shapefunction-radius' , 'INFO.' , RealOpt=r_sf) @@ -710,7 +710,7 @@ SUBROUTINE BuildBGMAndIdentifyHaloRegion() IF (MeshHasPeriodic) CALL CheckPeriodicSides (EnlargeBGM) CALL BARRIER_AND_SYNC(ElemInfo_Shared_Win,MPI_COMM_SHARED) IF (PartBound%UseRotPeriodicBC) CALL CheckRotPeriodicSides(EnlargeBGM) - CALL BARRIER_AND_SYNC(ElemInfo_Shared_Win,MPI_COMM_SHARED) + CALL BARRIER_AND_SYNC(ElemInfo_Shared_Win,MPI_COMM_SHARED) IF (PartBound%UseInterPlaneBC) CALL CheckInterPlaneSides(EnlargeBGM) CALL BARRIER_AND_SYNC(ElemInfo_Shared_Win,MPI_COMM_SHARED) @@ -1158,7 +1158,7 @@ SUBROUTINE BuildBGMAndIdentifyHaloRegion() END IF ! .NOT.PerformLoadBalance #endif /*USE_LOADBALANCE*/ END IF -CALL MPI_BARRIER(MPI_COMM_WORLD,iError) +CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #else hilf=' ' #endif /*CODE_ANALYZE*/ @@ -2001,10 +2001,10 @@ END SUBROUTINE CheckRotPeriodicSides SUBROUTINE CheckInterPlaneSides(EnlargeBGM) !=================================================================================================================================== !> checks the elements against inter plane -!> In addition to halo flat elements (normal halo region), find all elements on both side of a intermediate plane that +!> In addition to halo flat elements (normal halo region), find all elements on both side of a intermediate plane that !> are within range of the proc during a loop over all BCs: !> (1) Loop over all compute-node elements and check if they are within InterplaneRegion => Node is within InterplaneRegion -!> (2) Loop over all elements that are NOT on the compute node and add them as halo elements if they are within the corresponding +!> (2) Loop over all elements that are NOT on the compute node and add them as halo elements if they are within the corresponding !> InterplaneRegion !=================================================================================================================================== ! MODULES ! @@ -2058,7 +2058,7 @@ SUBROUTINE CheckInterPlaneSides(EnlargeBGM) END IF END DO IF(InInterPlaneRegion) THEN -! (2) Loop over all elements that are NOT on the compute node and add them as halo elements if they are within the corresponding +! (2) Loop over all elements that are NOT on the compute node and add them as halo elements if they are within the corresponding ! InterplaneRegion DO iElem = firstElem,lastElem ! only consider elements that are not already flagged diff --git a/src/particles/particle_mpi/particle_mpi.f90 b/src/particles/particle_mpi/particle_mpi.f90 index b82cb41d1..c12ae88a8 100644 --- a/src/particles/particle_mpi/particle_mpi.f90 +++ b/src/particles/particle_mpi/particle_mpi.f90 @@ -123,21 +123,6 @@ SUBROUTINE InitParticleMPI() IF(DoParticleLatencyHiding) CALL abort(__STAMP__,'DoParticleLatencyHiding=T not imeplemented for this time disc!') #endif /*!(PP_TimeDiscMethod==400)*/ -#if USE_MPI -CALL MPI_COMM_DUP (MPI_COMM_WORLD,PartMPI%COMM,iError) -CALL MPI_COMM_RANK(PartMPI%COMM,PartMPI%myRank,iError) -CALL MPI_COMM_SIZE(PartMPI%COMM,PartMPI%nProcs,iError) - -IF(PartMPI%nProcs.NE.nProcessors) CALL ABORT(__STAMP__,' MPI Communicater-size does not match!', IERROR) -PartCommSize = 0 -PartMPI%MPIRoot = .FALSE. -IF(PartMPI%MyRank.EQ.0) PartMPI%MPIRoot=.TRUE. -#else -PartMPI%myRank = 0 -PartMPI%nProcs = 1 -PartMPI%MPIRoot = .TRUE. -#endif /*USE_MPI*/ - ParticleMPIInitIsDone=.TRUE. LBWRITE(UNIT_stdOut,'(A)')' INIT PARTICLE MPI DONE!' LBWRITE(UNIT_StdOut,'(132("-"))') @@ -261,7 +246,7 @@ SUBROUTINE IRecvNbOfParticles() ! MODULES USE MOD_Globals USE MOD_Preproc -USE MOD_Particle_MPI_Vars, ONLY:PartMPI,PartMPIExchange +USE MOD_Particle_MPI_Vars, ONLY:PartMPIExchange USE MOD_Particle_MPI_Vars, ONLY:nExchangeProcessors,ExchangeProcToGlobalProc ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE @@ -279,7 +264,7 @@ SUBROUTINE IRecvNbOfParticles() , MPI_INTEGER & , ExchangeProcToGlobalProc(EXCHANGE_PROC_RANK,iProc) & , 1001 & - , PartMPI%COMM & + , MPI_COMM_PICLAS & , PartMPIExchange%RecvRequest(1,iProc) & , IERROR ) ! IF(IERROR.NE.MPI_SUCCESS) CALL ABORT(__STAMP__& @@ -306,7 +291,7 @@ SUBROUTINE SendNbOfParticles(doParticle_In) USE MOD_Part_Tools ,ONLY: isDepositParticle USE MOD_DSMC_Vars ,ONLY: DSMC,SpecDSMC, useDSMC, PolyatomMolDSMC USE MOD_Particle_Mesh_Vars ,ONLY: ElemInfo_Shared -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI,PartMPIExchange,PartTargetProc +USE MOD_Particle_MPI_Vars ,ONLY: PartMPIExchange,PartTargetProc USE MOD_Particle_MPI_Vars, ONLY: nExchangeProcessors,ExchangeProcToGlobalProc,GlobalProcToExchangeProc, halo_eps_velo USE MOD_Particle_Vars ,ONLY: PartState,PartSpecies,PEM,PDM,Species ! variables for parallel deposition @@ -397,7 +382,7 @@ SUBROUTINE SendNbOfParticles(doParticle_In) , MPI_INTEGER & , ExchangeProcToGlobalProc(EXCHANGE_PROC_RANK,iProc) & , 1001 & - , PartMPI%COMM & + , MPI_COMM_PICLAS & , PartMPIExchange%SendRequest(1,iProc) & , IERROR ) IF(IERROR.NE.MPI_SUCCESS) CALL ABORT(& @@ -428,7 +413,7 @@ SUBROUTINE MPIParticleSend(UseOldVecLength) USE MOD_Preproc USE MOD_DSMC_Vars, ONLY:useDSMC, CollisMode, DSMC, PartStateIntEn, SpecDSMC, PolyatomMolDSMC, VibQuantsPar USE MOD_DSMC_Vars, ONLY:ElectronicDistriPart, AmbipolElecVelo -USE MOD_Particle_MPI_Vars, ONLY:PartMPI,PartMPIExchange,PartCommSize,PartSendBuf,PartRecvBuf,PartTargetProc!,PartHaloElemToProc +USE MOD_Particle_MPI_Vars, ONLY:PartMPIExchange,PartCommSize,PartSendBuf,PartRecvBuf,PartTargetProc!,PartHaloElemToProc USE MOD_Particle_MPI_Vars, ONLY:nExchangeProcessors,ExchangeProcToGlobalProc USE MOD_Particle_Tracking_Vars, ONLY:TrackingMethod USE MOD_Particle_Vars, ONLY:PartState,PartSpecies,usevMPF,PartMPF,PEM,PDM,PartPosRef,Species @@ -827,7 +812,7 @@ SUBROUTINE MPIParticleSend(UseOldVecLength) , MPI_DOUBLE_PRECISION & , ExchangeProcToGlobalProc(EXCHANGE_PROC_RANK,iProc) & , 1002 & - , PartMPI%COMM & + , MPI_COMM_PICLAS & , PartMPIExchange%RecvRequest(2,iProc) & , IERROR ) IF(IERROR.NE.MPI_SUCCESS) CALL ABORT(__STAMP__,' MPI Communication error', IERROR) @@ -858,7 +843,7 @@ SUBROUTINE MPIParticleSend(UseOldVecLength) , MPI_DOUBLE_PRECISION & , ExchangeProcToGlobalProc(EXCHANGE_PROC_RANK,iProc) & , 1002 & - , PartMPI%COMM & + , MPI_COMM_PICLAS & , PartMPIExchange%SendRequest(2,iProc) & , IERROR ) IF(IERROR.NE.MPI_SUCCESS) CALL ABORT(__STAMP__,' MPI Communication error', IERROR) @@ -882,7 +867,7 @@ SUBROUTINE MPIParticleRecv(DoMPIUpdateNextFreePos) USE MOD_Preproc USE MOD_DSMC_Vars ,ONLY: useDSMC, CollisMode, DSMC, PartStateIntEn, SpecDSMC, PolyatomMolDSMC, VibQuantsPar USE MOD_DSMC_Vars ,ONLY: ElectronicDistriPart, AmbipolElecVelo -USE MOD_Particle_MPI_Vars ,ONLY: PartMPIExchange,PartCommSize,PartRecvBuf,PartSendBuf!,PartMPI +USE MOD_Particle_MPI_Vars ,ONLY: PartMPIExchange,PartCommSize,PartRecvBuf,PartSendBuf USE MOD_Particle_MPI_Vars ,ONLY: nExchangeProcessors USE MOD_Particle_Tracking_Vars ,ONLY: TrackingMethod USE MOD_Particle_Vars ,ONLY: PartState,PartSpecies,usevMPF,PartMPF,PEM,PDM, PartPosRef, Species, LastPartPos @@ -1356,18 +1341,17 @@ SUBROUTINE FinalizeParticleMPI() END DO ! iSpec IF(nInitRegions.GT.0) THEN DO iInitRegions=1,nInitRegions - IF(PartMPI%InitGroup(iInitRegions)%COMM.NE.MPI_COMM_NULL) THEN - CALL MPI_COMM_FREE(PartMPI%InitGroup(iInitRegions)%Comm,iERROR) + IF(PartMPIInitGroup(iInitRegions)%COMM.NE.MPI_COMM_NULL) THEN + CALL MPI_COMM_FREE(PartMPIInitGroup(iInitRegions)%Comm,iERROR) END IF END DO ! iInitRegions END IF -IF(PartMPI%COMM.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(PartMPI%COMM,iERROR) SDEALLOCATE( PartMPIExchange%nPartsSend) SDEALLOCATE( PartMPIExchange%nPartsRecv) SDEALLOCATE( PartMPIExchange%RecvRequest) SDEALLOCATE( PartMPIExchange%SendRequest) -SDEALLOCATE( PartMPI%InitGroup) +SDEALLOCATE( PartMPIInitGroup) SDEALLOCATE( PartSendBuf) SDEALLOCATE( PartRecvBuf) SDEALLOCATE( ExchangeProcToGlobalProc) diff --git a/src/particles/particle_mpi/particle_mpi_emission.f90 b/src/particles/particle_mpi/particle_mpi_emission.f90 index 805067cd8..8ca5d1df7 100644 --- a/src/particles/particle_mpi/particle_mpi_emission.f90 +++ b/src/particles/particle_mpi/particle_mpi_emission.f90 @@ -42,7 +42,7 @@ SUBROUTINE InitEmissionComm() ! MODULES USE MOD_Globals USE MOD_Preproc -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI,MPI_halo_eps +USE MOD_Particle_MPI_Vars ,ONLY: PartMPIInitGroup,MPI_halo_eps USE MOD_Particle_Vars ,ONLY: Species,nSpecies USE MOD_Particle_Mesh_Vars ,ONLY: GEO,SideInfo_Shared USE MOD_Mesh_Vars ,ONLY: nElems,BoundaryName @@ -79,7 +79,7 @@ SUBROUTINE InitEmissionComm() IF(nInitRegions.EQ.0) RETURN ! allocate communicators -ALLOCATE( PartMPI%InitGroup(1:nInitRegions)) +ALLOCATE( PartMPIInitGroup(1:nInitRegions)) ! Default value for neutralization regions (Landmark and Liu2010) nNeutralizationElems = -1 @@ -507,18 +507,18 @@ SUBROUTINE InitEmissionComm() END SELECT ! Sanity check if at least one proc will be on the new emission communicator - CALL MPI_ALLREDUCE(RegionOnProc,RegionExists,1,MPI_LOGICAL,MPI_LOR,MPI_COMM_WORLD,iError) + CALL MPI_ALLREDUCE(RegionOnProc,RegionExists,1,MPI_LOGICAL,MPI_LOR,MPI_COMM_PICLAS,iError) IF (.NOT. RegionExists) THEN WRITE(hilf,'(A,I0,A,I0)') 'Species',iSpec,'-Init',iInit CALL CollectiveStop(__STAMP__,'The emission region was not found on any processor. No processor in range for '//TRIM(hilf)) END IF - ! Add PartMPI%MPIRoot to specific inits automatically for output of analysis data to disk + ! Add MPIRoot to specific inits automatically for output of analysis data to disk ! The root sometimes also reads data during restart and broadcasts it to the other processors in the communicator SELECT CASE(TRIM(Species(iSpec)%Init(iInit)%SpaceIC)) CASE('2D_landmark_neutralization','2D_Liu2010_neutralization','3D_Liu2010_neutralization','2D_Liu2010_neutralization_Szabo',& '3D_Liu2010_neutralization_Szabo') - IF(PartMPI%MPIRoot) RegionOnProc=.TRUE. + IF(MPIRoot) RegionOnProc=.TRUE. END SELECT ! create new communicator @@ -527,37 +527,37 @@ SUBROUTINE InitEmissionComm() ! set communicator id Species(iSpec)%Init(iInit)%InitCOMM=nInitRegions ! create new emission communicator for emission communication. Pass MPI_INFO_NULL as rank to follow the original ordering - CALL MPI_COMM_SPLIT(PartMPI%COMM,color,MPI_INFO_NULL,PartMPI%InitGroup(nInitRegions)%COMM,iError) + CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS,color,MPI_INFO_NULL,PartMPIInitGroup(nInitRegions)%COMM,iError) ! Find my rank on the shared communicator, comm size and proc name IF (RegionOnProc) THEN - CALL MPI_COMM_RANK(PartMPI%InitGroup(nInitRegions)%COMM,PartMPI%InitGroup(nInitRegions)%MyRank,iError) - CALL MPI_COMM_SIZE(PartMPI%InitGroup(nInitRegions)%COMM,PartMPI%InitGroup(nInitRegions)%nProcs,iError) + CALL MPI_COMM_RANK(PartMPIInitGroup(nInitRegions)%COMM,PartMPIInitGroup(nInitRegions)%MyRank,iError) + CALL MPI_COMM_SIZE(PartMPIInitGroup(nInitRegions)%COMM,PartMPIInitGroup(nInitRegions)%nProcs,iError) ! inform about size of emission communicator - IF (PartMPI%InitGroup(nInitRegions)%MyRank.EQ.0) THEN + IF (PartMPIInitGroup(nInitRegions)%MyRank.EQ.0) THEN #if USE_LOADBALANCE IF(.NOT.PerformLoadBalance)& #endif /*USE_LOADBALANCE*/ WRITE(UNIT_StdOut,'(A,I0,A,I0,A,I0,A)') ' Emission-Region,Emission-Communicator: ',nInitRegions,' on ',& - PartMPI%InitGroup(nInitRegions)%nProcs,' procs ('//TRIM(Species(iSpec)%Init(iInit)%SpaceIC)//', iSpec=',iSpec,')' + PartMPIInitGroup(nInitRegions)%nProcs,' procs ('//TRIM(Species(iSpec)%Init(iInit)%SpaceIC)//', iSpec=',iSpec,')' END IF END IF ! build mapping for procs on emission communicator - IF(PartMPI%InitGroup(nInitRegions)%COMM.NE.MPI_COMM_NULL) THEN - PartMPI%InitGroup(nInitRegions)%MPIRoot=MERGE(.TRUE.,.FALSE.,PartMPI%InitGroup(nInitRegions)%MyRank.EQ.0) - - ALLOCATE(PartMPI%InitGroup(nInitRegions)%GroupToComm(0:PartMPI%InitGroup(nInitRegions)%nProcs-1)) - PartMPI%InitGroup(nInitRegions)%GroupToComm(PartMPI%InitGroup(nInitRegions)%MyRank) = PartMPI%MyRank - CALL MPI_ALLGATHER(PartMPI%MyRank,1,MPI_INTEGER& - ,PartMPI%InitGroup(nInitRegions)%GroupToComm(0:PartMPI%InitGroup(nInitRegions)%nProcs-1)& - ,1,MPI_INTEGER,PartMPI%InitGroup(nInitRegions)%COMM,iERROR) - - ALLOCATE(PartMPI%InitGroup(nInitRegions)%CommToGroup(0:PartMPI%nProcs-1)) - PartMPI%InitGroup(nInitRegions)%CommToGroup(0:PartMPI%nProcs-1) = -1 - DO iRank = 0,PartMPI%InitGroup(nInitRegions)%nProcs-1 - PartMPI%InitGroup(nInitRegions)%CommToGroup(PartMPI%InitGroup(nInitRegions)%GroupToComm(iRank))=iRank + IF(PartMPIInitGroup(nInitRegions)%COMM.NE.MPI_COMM_NULL) THEN + PartMPIInitGroup(nInitRegions)%MPIRoot=MERGE(.TRUE.,.FALSE.,PartMPIInitGroup(nInitRegions)%MyRank.EQ.0) + + ALLOCATE(PartMPIInitGroup(nInitRegions)%GroupToComm(0:PartMPIInitGroup(nInitRegions)%nProcs-1)) + PartMPIInitGroup(nInitRegions)%GroupToComm(PartMPIInitGroup(nInitRegions)%MyRank) = myRank + CALL MPI_ALLGATHER(myRank,1,MPI_INTEGER& + ,PartMPIInitGroup(nInitRegions)%GroupToComm(0:PartMPIInitGroup(nInitRegions)%nProcs-1)& + ,1,MPI_INTEGER,PartMPIInitGroup(nInitRegions)%COMM,iERROR) + + ALLOCATE(PartMPIInitGroup(nInitRegions)%CommToGroup(0:nProcessors-1)) + PartMPIInitGroup(nInitRegions)%CommToGroup(0:nProcessors-1) = -1 + DO iRank = 0,PartMPIInitGroup(nInitRegions)%nProcs-1 + PartMPIInitGroup(nInitRegions)%CommToGroup(PartMPIInitGroup(nInitRegions)%GroupToComm(iRank))=iRank END DO ! iRank END IF @@ -582,7 +582,7 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf !USE MOD_Mesh_Tools ,ONLY: GetCNElemID !USE MOD_Particle_Mesh_Vars ,ONLY: FIBGM_nElems, FIBGM_offsetElem, FIBGM_Element USE MOD_Particle_Mesh_Vars ,ONLY: FIBGM_nElems,FIBGM_nTotalElems -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI,PartMPIInsert,PartMPILocate +USE MOD_Particle_MPI_Vars ,ONLY: PartMPIInitGroup,PartMPIInsert,PartMPILocate USE MOD_Particle_MPI_Vars ,ONLY: EmissionSendBuf,EmissionRecvBuf USE MOD_Particle_Vars ,ONLY: PDM,PEM,PartState,PartPosRef,Species USE MOD_Particle_Tracking_Vars ,ONLY: TrackingMethod @@ -623,11 +623,11 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf InitGroup = Species(FractNbr)%Init(iInit)%InitCOMM ! Arrays for communication of particles not located in final element -ALLOCATE( PartMPIInsert%nPartsSend (2,0:PartMPI%InitGroup(InitGroup)%nProcs-1) & - , PartMPIInsert%nPartsRecv (1,0:PartMPI%InitGroup(InitGroup)%nProcs-1) & - , PartMPIInsert%SendRequest (2,0:PartMPI%InitGroup(InitGroup)%nProcs-1) & - , PartMPIInsert%RecvRequest (2,0:PartMPI%InitGroup(InitGroup)%nProcs-1) & - , PartMPIInsert%send_message( 0:PartMPI%InitGroup(InitGroup)%nProcs-1) & +ALLOCATE( PartMPIInsert%nPartsSend (2,0:PartMPIInitGroup(InitGroup)%nProcs-1) & + , PartMPIInsert%nPartsRecv (1,0:PartMPIInitGroup(InitGroup)%nProcs-1) & + , PartMPIInsert%SendRequest (2,0:PartMPIInitGroup(InitGroup)%nProcs-1) & + , PartMPIInsert%RecvRequest (2,0:PartMPIInitGroup(InitGroup)%nProcs-1) & + , PartMPIInsert%send_message( 0:PartMPIInitGroup(InitGroup)%nProcs-1) & , STAT=ALLOCSTAT) IF (ALLOCSTAT.NE.0) & CALL ABORT(__STAMP__,' Cannot allocate particle emission MPI arrays! ALLOCSTAT',ALLOCSTAT) @@ -636,12 +636,12 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf PartMPIInsert%nPartsRecv=0 ! Inter-CN communication -ALLOCATE( PartMPILocate%nPartsSend (2,0:PartMPI%InitGroup(InitGroup)%nProcs-1) & - , PartMPILocate%nPartsRecv (1,0:PartMPI%InitGroup(InitGroup)%nProcs-1) & - , PartMPILocate%SendRequest(2,0:PartMPI%InitGroup(InitGroup)%nProcs-1) & - , PartMPILocate%RecvRequest(2,0:PartMPI%InitGroup(InitGroup)%nProcs-1) & - , EmissionRecvBuf ( 0:PartMPI%InitGroup(InitGroup)%nProcs-1) & - , EmissionSendBuf ( 0:PartMPI%InitGroup(InitGroup)%nProcs-1) & +ALLOCATE( PartMPILocate%nPartsSend (2,0:PartMPIInitGroup(InitGroup)%nProcs-1) & + , PartMPILocate%nPartsRecv (1,0:PartMPIInitGroup(InitGroup)%nProcs-1) & + , PartMPILocate%SendRequest(2,0:PartMPIInitGroup(InitGroup)%nProcs-1) & + , PartMPILocate%RecvRequest(2,0:PartMPIInitGroup(InitGroup)%nProcs-1) & + , EmissionRecvBuf ( 0:PartMPIInitGroup(InitGroup)%nProcs-1) & + , EmissionSendBuf ( 0:PartMPIInitGroup(InitGroup)%nProcs-1) & , STAT=ALLOCSTAT) IF (ALLOCSTAT.NE.0) & CALL ABORT(__STAMP__,' Cannot allocate particle emission MPI arrays! ALLOCSTAT',ALLOCSTAT) @@ -665,8 +665,8 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf chunkState = -1 !--- 1/10 Open receive buffer (located and non-located particles) -DO iProc=0,PartMPI%InitGroup(InitGroup)%nProcs-1 - IF (iProc.EQ.PartMPI%InitGroup(InitGroup)%myRank) CYCLE +DO iProc=0,PartMPIInitGroup(InitGroup)%nProcs-1 + IF (iProc.EQ.PartMPIInitGroup(InitGroup)%myRank) CYCLE !--- MPI_IRECV lengths of lists of particles entering local mesh CALL MPI_IRECV( PartMPIInsert%nPartsRecv(:,iProc) & @@ -674,7 +674,7 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf , MPI_INTEGER & , iProc & , 1011 & - , PartMPI%InitGroup(InitGroup)%COMM & + , PartMPIInitGroup(InitGroup)%COMM & , PartMPIInsert%RecvRequest(1,iProc) & , IERROR) @@ -684,7 +684,7 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf , MPI_INTEGER & , iProc & , 1111 & - , PartMPI%InitGroup(InitGroup)%COMM & + , PartMPIInitGroup(InitGroup)%COMM & , PartMPILocate%RecvRequest(1,iProc) & , IERROR) END DO @@ -749,7 +749,7 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf ProcID = FIBGMProcs(iProc) IF (ProcID.EQ.myRank) CYCLE - tProc=PartMPI%InitGroup(InitGroup)%CommToGroup(ProcID) + tProc=PartMPIInitGroup(InitGroup)%CommToGroup(ProcID) ! Processor is not on emission communicator IF(tProc.EQ.-1) CYCLE @@ -761,8 +761,8 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf END DO ! i = 1, chunkSize !--- 2/10 Send number of non-located particles -DO iProc=0,PartMPI%InitGroup(InitGroup)%nProcs-1 - IF (iProc.EQ.PartMPI%InitGroup(InitGroup)%myRank) CYCLE +DO iProc=0,PartMPIInitGroup(InitGroup)%nProcs-1 + IF (iProc.EQ.PartMPIInitGroup(InitGroup)%myRank) CYCLE ! send particles !--- MPI_ISEND lengths of lists of particles leaving local mesh @@ -771,7 +771,7 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf , MPI_INTEGER & , iProc & , 1011 & - , PartMPI%InitGroup(InitGroup)%COMM & + , PartMPIInitGroup(InitGroup)%COMM & , PartMPIInsert%SendRequest(1,iProc) & , IERROR) IF (PartMPIInsert%nPartsSend(1,iProc).GT.0) THEN @@ -806,7 +806,7 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf ProcID = FIBGMProcs(iProc) IF (ProcID.EQ.myRank) CYCLE - tProc=PartMPI%InitGroup(InitGroup)%CommToGroup(ProcID) + tProc=PartMPIInitGroup(InitGroup)%CommToGroup(ProcID) ! Processor is not on emission communicator IF (tProc.EQ.-1) CYCLE @@ -827,8 +827,8 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf #if defined(MEASURE_MPI_WAIT) CALL SYSTEM_CLOCK(count=CounterStart) #endif /*defined(MEASURE_MPI_WAIT)*/ -DO iProc=0,PartMPI%InitGroup(InitGroup)%nProcs-1 - IF (iProc.EQ.PartMPI%InitGroup(InitGroup)%myRank) CYCLE +DO iProc=0,PartMPIInitGroup(InitGroup)%nProcs-1 + IF (iProc.EQ.PartMPIInitGroup(InitGroup)%myRank) CYCLE CALL MPI_WAIT(PartMPIInsert%SendRequest(1,iProc),msg_status(:),IERROR) IF (IERROR.NE.MPI_SUCCESS) CALL abort(__STAMP__,' MPI Communication error', IERROR) @@ -845,8 +845,8 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf ! Inter-CN communication ALLOCATE(recvPartPos(1:SUM(PartMPIInsert%nPartsRecv(1,:)*DimSend)), STAT=ALLOCSTAT) TotalNbrOfRecvParts = 0 -DO iProc=0,PartMPI%InitGroup(InitGroup)%nProcs-1 - IF (iProc.EQ.PartMPI%InitGroup(InitGroup)%myRank) CYCLE +DO iProc=0,PartMPIInitGroup(InitGroup)%nProcs-1 + IF (iProc.EQ.PartMPIInitGroup(InitGroup)%myRank) CYCLE IF (PartMPIInsert%nPartsRecv(1,iProc).GT.0) THEN !--- MPI_IRECV lengths of lists of particles entering local mesh @@ -855,7 +855,7 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf , MPI_DOUBLE_PRECISION & , iProc & , 1022 & - , PartMPI%InitGroup(InitGroup)%COMM & + , PartMPIInitGroup(InitGroup)%COMM & , PartMPIInsert%RecvRequest(2,iProc) & , IERROR) TotalNbrOfRecvParts = TotalNbrOfRecvParts + PartMPIInsert%nPartsRecv(1,iProc) @@ -867,7 +867,7 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf , MPI_DOUBLE_PRECISION & , iProc & , 1022 & - , PartMPI%InitGroup(InitGroup)%COMM & + , PartMPIInitGroup(InitGroup)%COMM & , PartMPIInsert%SendRequest(2,iProc) & , IERROR) END IF @@ -894,7 +894,7 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf IF (.NOT.InsideMyBGM(2,i)) CYCLE ! ProcID on emission communicator - tProc=PartMPI%InitGroup(InitGroup)%CommToGroup(ProcID) + tProc=PartMPIInitGroup(InitGroup)%CommToGroup(ProcID) ! Processor is not on emission communicator IF(tProc.EQ.-1) CALL ABORT(__STAMP__,'Error in particle_mpi_emission: proc not on emission communicator') @@ -933,8 +933,8 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf !--- / Send number of located particles ! Inter-CN communication -DO iProc=0,PartMPI%InitGroup(InitGroup)%nProcs-1 - IF (iProc.EQ.PartMPI%InitGroup(InitGroup)%myRank) CYCLE +DO iProc=0,PartMPIInitGroup(InitGroup)%nProcs-1 + IF (iProc.EQ.PartMPIInitGroup(InitGroup)%myRank) CYCLE ! send particles !--- MPI_ISEND lengths of lists of particles leaving local mesh @@ -943,7 +943,7 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf , MPI_INTEGER & , iProc & , 1111 & - , PartMPI%InitGroup(InitGroup)%COMM & + , PartMPIInitGroup(InitGroup)%COMM & , PartMPILocate%SendRequest(1,iProc) & , IERROR) IF (PartMPILocate%nPartsSend(1,iProc).GT.0) THEN @@ -963,7 +963,7 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf ProcID = ElemInfo_Shared(ELEM_RANK,ElemID) IF (ProcID.NE.myRank) THEN ! ProcID on emission communicator - tProc=PartMPI%InitGroup(InitGroup)%CommToGroup(ProcID) + tProc=PartMPIInitGroup(InitGroup)%CommToGroup(ProcID) ! Processor is not on emission communicator IF(tProc.EQ.-1) CYCLE @@ -980,8 +980,8 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf #if defined(MEASURE_MPI_WAIT) CALL SYSTEM_CLOCK(count=CounterStart) #endif /*defined(MEASURE_MPI_WAIT)*/ -DO iProc=0,PartMPI%InitGroup(InitGroup)%nProcs-1 - IF (iProc.EQ.PartMPI%InitGroup(InitGroup)%myRank) CYCLE +DO iProc=0,PartMPIInitGroup(InitGroup)%nProcs-1 + IF (iProc.EQ.PartMPIInitGroup(InitGroup)%myRank) CYCLE CALL MPI_WAIT(PartMPILocate%SendRequest(1,iProc),msg_status(:),IERROR) IF(IERROR.NE.MPI_SUCCESS) CALL abort(__STAMP__,' MPI Communication error', IERROR) @@ -994,8 +994,8 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf MPIW8CountPart(5) = MPIW8CountPart(5) + 1_8 #endif /*defined(MEASURE_MPI_WAIT)*/ -DO iProc=0,PartMPI%InitGroup(InitGroup)%nProcs-1 - IF (iProc.EQ.PartMPI%InitGroup(InitGroup)%myRank) CYCLE +DO iProc=0,PartMPIInitGroup(InitGroup)%nProcs-1 + IF (iProc.EQ.PartMPIInitGroup(InitGroup)%myRank) CYCLE ! Allocate receive array and open receive buffer if expecting particles from iProc IF (PartMPILocate%nPartsRecv(1,iProc).GT.0) THEN @@ -1011,7 +1011,7 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf , MPI_DOUBLE_PRECISION & , iProc & , 1122 & - , PartMPI%InitGroup(InitGroup)%COMM & + , PartMPIInitGroup(InitGroup)%COMM & , PartMPILocate%RecvRequest(2,iProc) & , IERROR ) IF(IERROR.NE.MPI_SUCCESS) & @@ -1026,7 +1026,7 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf , MPI_DOUBLE_PRECISION & , iProc & , 1122 & - , PartMPI%InitGroup(InitGroup)%COMM & + , PartMPIInitGroup(InitGroup)%COMM & , PartMPILocate%SendRequest(2,iProc) & , IERROR ) IF(IERROR.NE.MPI_SUCCESS) & @@ -1038,8 +1038,8 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf #if defined(MEASURE_MPI_WAIT) CALL SYSTEM_CLOCK(count=CounterStart) #endif /*defined(MEASURE_MPI_WAIT)*/ -DO iProc=0,PartMPI%InitGroup(InitGroup)%nProcs-1 - IF (iProc.EQ.PartMPI%InitGroup(InitGroup)%myRank) CYCLE +DO iProc=0,PartMPIInitGroup(InitGroup)%nProcs-1 + IF (iProc.EQ.PartMPIInitGroup(InitGroup)%myRank) CYCLE IF (PartMPIInsert%nPartsSend(1,iProc).GT.0) THEN CALL MPI_WAIT(PartMPIInsert%SendRequest(2,iProc),msg_status(:),IERROR) @@ -1091,8 +1091,8 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf #if defined(MEASURE_MPI_WAIT) CALL SYSTEM_CLOCK(count=CounterStart) #endif /*defined(MEASURE_MPI_WAIT)*/ -DO iProc=0,PartMPI%InitGroup(InitGroup)%nProcs-1 - IF (iProc.EQ.PartMPI%InitGroup(InitGroup)%myRank) CYCLE +DO iProc=0,PartMPIInitGroup(InitGroup)%nProcs-1 + IF (iProc.EQ.PartMPIInitGroup(InitGroup)%myRank) CYCLE IF (PartMPILocate%nPartsSend(1,iProc).GT.0) THEN CALL MPI_WAIT(PartMPILocate%SendRequest(2,iProc),msg_status(:),IERROR) @@ -1110,8 +1110,8 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf #endif /*defined(MEASURE_MPI_WAIT)*/ !--- 10/10 Write located particles -DO iProc=0,PartMPI%InitGroup(InitGroup)%nProcs-1 - IF (iProc.EQ.PartMPI%InitGroup(InitGroup)%myRank) CYCLE +DO iProc=0,PartMPIInitGroup(InitGroup)%nProcs-1 + IF (iProc.EQ.PartMPIInitGroup(InitGroup)%myRank) CYCLE IF (PartMPILocate%nPartsRecv(1,iProc).EQ.0) CYCLE DO i = 1,PartMPILocate%nPartsRecv(1,iProc) @@ -1138,7 +1138,7 @@ SUBROUTINE SendEmissionParticlesToProcs(chunkSize,DimSend,FractNbr,iInit,mySumOf !--- Clean up SDEALLOCATE(recvPartPos) SDEALLOCATE(chunkState) -DO iProc=0,PartMPI%InitGroup(InitGroup)%nProcs-1 +DO iProc=0,PartMPIInitGroup(InitGroup)%nProcs-1 SDEALLOCATE(EmissionRecvBuf(iProc)%content) SDEALLOCATE(EmissionSendBuf(iProc)%content) END DO diff --git a/src/particles/particle_mpi/particle_mpi_halo.f90 b/src/particles/particle_mpi/particle_mpi_halo.f90 index 5b4d3ec05..1656cffc1 100644 --- a/src/particles/particle_mpi/particle_mpi_halo.f90 +++ b/src/particles/particle_mpi/particle_mpi_halo.f90 @@ -167,7 +167,7 @@ SUBROUTINE IdentifyPartExchangeProcs , MPI_LOGICAL & , iProc & , 1999 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequest(iProc) & , IERROR) END DO @@ -970,7 +970,7 @@ SUBROUTINE IdentifyPartExchangeProcs END IF END DO IF(InInterPlaneRegion) THEN -! (2) Loop over all elements on the compute node and add the procs as halo_procs if they are within the corresponding +! (2) Loop over all elements on the compute node and add the procs as halo_procs if they are within the corresponding ! InterplaneRegion DO iElem = 1,nComputeNodeTotalElems ElemID = GetGlobalElemID(iElem) @@ -1010,7 +1010,7 @@ SUBROUTINE IdentifyPartExchangeProcs , MPI_LOGICAL & , iProc & , 1999 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequest(iProc) & , IERROR) END DO @@ -1050,7 +1050,7 @@ SUBROUTINE IdentifyPartExchangeProcs END DO ! On smooth grids, nNonSymmetricExchangeProcs should be zero. Only output if previously missing particle exchange procs are found - CALL MPI_ALLREDUCE(nNonSymmetricExchangeProcs, nNonSymmetricExchangeProcsGlob, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD, IERROR) + CALL MPI_ALLREDUCE(nNonSymmetricExchangeProcs, nNonSymmetricExchangeProcsGlob, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_PICLAS, IERROR) ! Check sum of nNonSymmetricExchangeProcs over all processors IF(nNonSymmetricExchangeProcsGlob.GT.0)THEN SWRITE(UNIT_StdOut,'(X,131("~"))') @@ -1097,7 +1097,7 @@ SUBROUTINE IdentifyPartExchangeProcs END DO ! -- Average number of exchange processors -CALL MPI_REDUCE(nExchangeProcessors,nExchangeProcessorsGlobal,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,iError) +CALL MPI_REDUCE(nExchangeProcessors,nExchangeProcessorsGlobal,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,iError) LBWRITE(UNIT_stdOut,'(A,I0,A)') ' | Started particle exchange communication with average ', & nExchangeProcessorsGlobal/nProcessors_Global , & ' partners per proc' @@ -1448,7 +1448,7 @@ SUBROUTINE IdentifyPartExchangeProcs , MPI_INTEGER & , ShapeMapping(iProc)%Rank & , 2003 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequest(iProc) & , IERROR) END DO @@ -1459,7 +1459,7 @@ SUBROUTINE IdentifyPartExchangeProcs , MPI_INTEGER & , ShapeMapping(iProc)%Rank & , 2003 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequest(iProc) & , IERROR) END DO @@ -1479,7 +1479,7 @@ SUBROUTINE IdentifyPartExchangeProcs , MPI_INTEGER & , ShapeMapping(iProc)%Rank & , 2003 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequest(iProc) & , IERROR) END DO @@ -1489,7 +1489,7 @@ SUBROUTINE IdentifyPartExchangeProcs , MPI_INTEGER & , ShapeMapping(iProc)%Rank & , 2003 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequest(iProc) & , IERROR) END DO diff --git a/src/particles/particle_mpi/particle_mpi_vars.f90 b/src/particles/particle_mpi/particle_mpi_vars.f90 index 87ab59a48..8c4ef23c2 100644 --- a/src/particles/particle_mpi/particle_mpi_vars.f90 +++ b/src/particles/particle_mpi/particle_mpi_vars.f90 @@ -46,17 +46,7 @@ MODULE MOD_Particle_MPI_Vars INTEGER,ALLOCATABLE :: CommToGroup(:) ! list containing the rank in PartMPI%COMM END TYPE -TYPE tPartMPIVAR -#if USE_MPI - INTEGER :: COMM=MPI_COMM_NULL ! MPI communicator for PIC GTS region -#endif /*USE_MPI*/ - TYPE(tPartMPIGROUP),ALLOCATABLE :: InitGroup(:) ! small communicator for initialization - INTEGER :: nProcs ! number of MPI processes for particles - INTEGER :: MyRank ! MyRank of PartMPIVAR%COMM - LOGICAL :: MPIRoot ! Root, MPIRank=0 -END TYPE - -TYPE (tPartMPIVAR) :: PartMPI +TYPE(tPartMPIGROUP),ALLOCATABLE :: PartMPIInitGroup(:) ! small communicator for initialization REAL :: SafetyFactor ! Factor to scale the halo region with MPI REAL :: halo_eps_velo ! halo_eps_velo diff --git a/src/particles/particle_operations.f90 b/src/particles/particle_operations.f90 index a077f8465..41ab0bda3 100644 --- a/src/particles/particle_operations.f90 +++ b/src/particles/particle_operations.f90 @@ -321,7 +321,7 @@ SUBROUTINE RemoveAllElectrons() END DO #if USE_MPI -CALL MPI_ALLREDUCE(MPI_IN_PLACE,NbrOfElectronsRemoved,1,MPI_INTEGER,MPI_SUM,MPI_COMM_WORLD,iError) +CALL MPI_ALLREDUCE(MPI_IN_PLACE,NbrOfElectronsRemoved,1,MPI_INTEGER,MPI_SUM,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ IF(NbrOfElectronsRemoved.GT.0.AND.MPIRoot) WRITE(UNIT_StdOut,'(A,I0,A)') ' Removed a total of ',NbrOfElectronsRemoved,' electrons.' diff --git a/src/particles/particle_timestep.f90 b/src/particles/particle_timestep.f90 index cb5a09c59..6c95a5849 100644 --- a/src/particles/particle_timestep.f90 +++ b/src/particles/particle_timestep.f90 @@ -210,7 +210,7 @@ SUBROUTINE VarTimeStep_InitDistribution() IF(DoRestart) THEN ! Try to get the time step factor distribution directly from state file - CALL OpenDataFile(TRIM(RestartFile),create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(TRIM(RestartFile),create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL DatasetExists(File_ID,'ElemTimeStep',TimeStepExists) IF(TimeStepExists) THEN ! Allocate the array for the element-wise time step factor @@ -247,7 +247,7 @@ SUBROUTINE VarTimeStep_InitDistribution() 'ERROR: It is required to use a restart and macroscopic restart when adapting the time step distribution!') END IF ! Open DSMC state file - CALL OpenDataFile(MacroRestartFileName,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(MacroRestartFileName,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL GetDataProps('ElemData',nVar_HDF5,N_HDF5,nGlobalElems) @@ -404,7 +404,7 @@ END SUBROUTINE VarTimeStep_InitDistribution REAL FUNCTION GetParticleTimeStep(xPos, yPos, iElem) !=================================================================================================================================== -!> Calculates/determines the time step +!> Calculates/determines the time step !> a) at a position x/y (only in 2D/Axi) [VarTimeStep%UseLinearScaling] !> b) of the given element number (3D and VTS distribution) [VarTimeStep%UseDistribution] !=================================================================================================================================== @@ -565,4 +565,4 @@ SUBROUTINE VarTimeStep_CalcElemFacs() END SUBROUTINE VarTimeStep_CalcElemFacs -END MODULE MOD_Particle_TimeStep +END MODULE MOD_Particle_TimeStep diff --git a/src/particles/pic/analyze/pic_analyze.f90 b/src/particles/pic/analyze/pic_analyze.f90 index bcb83f108..a2c703655 100644 --- a/src/particles/pic/analyze/pic_analyze.f90 +++ b/src/particles/pic/analyze/pic_analyze.f90 @@ -55,9 +55,6 @@ SUBROUTINE VerifyDepositedCharge() #else USE MOD_PICDepo_Vars ,ONLY: PartSource #endif -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ USE MOD_ChangeBasis ,ONLY: ChangeBasis3D ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE @@ -117,12 +114,12 @@ SUBROUTINE VerifyDepositedCharge() ! Collect info on MPI root process #if USE_MPI - IF(PartMPI%MPIRoot) THEN - CALL MPI_REDUCE(MPI_IN_PLACE , ChargeAnalytical , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , PartMPI%COMM , IERROR) - CALL MPI_REDUCE(MPI_IN_PLACE , ChargeNumerical , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , PartMPI%COMM , IERROR) + IF(MPIRoot) THEN + CALL MPI_REDUCE(MPI_IN_PLACE , ChargeAnalytical , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) + CALL MPI_REDUCE(MPI_IN_PLACE , ChargeNumerical , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) ELSE - CALL MPI_REDUCE(ChargeAnalytical , 0 , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , PartMPI%COMM , IERROR) - CALL MPI_REDUCE(ChargeNumerical , 0 , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , PartMPI%COMM , IERROR) + CALL MPI_REDUCE(ChargeAnalytical , 0 , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) + CALL MPI_REDUCE(ChargeNumerical , 0 , 1 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , IERROR) END IF #endif @@ -164,7 +161,6 @@ SUBROUTINE CalcDepositedCharge() #else USE MOD_PICDepo_Vars, ONLY:PartSource #endif -USE MOD_Particle_MPI_Vars, ONLY:PartMPI ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !----------------------------------------------------------------------------------------------------------------------------------- @@ -213,14 +209,14 @@ SUBROUTINE CalcDepositedCharge() ! MPI Communication #if USE_MPI -IF (PartMPI%MPIRoot) THEN - CALL MPI_REDUCE(MPI_IN_PLACE,Charge , 2 , MPI_DOUBLE_PRECISION, MPI_SUM,0, PartMPI%COMM, IERROR) +IF (MPIRoot) THEN + CALL MPI_REDUCE(MPI_IN_PLACE,Charge , 2 , MPI_DOUBLE_PRECISION, MPI_SUM,0, MPI_COMM_PICLAS, IERROR) ELSE ! no Root - CALL MPI_REDUCE(Charge,RECBR ,2,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM, IERROR) + CALL MPI_REDUCE(Charge,RECBR ,2,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) END IF #endif -IF (PartMPI%MPIRoot) THEN +IF (MPIRoot) THEN PartCharge(1)=Charge(1) ! absolute error PartCharge(2)=ABS(Charge(2)-Charge(1)) diff --git a/src/particles/pic/deposition/pic_depo.f90 b/src/particles/pic/deposition/pic_depo.f90 index b258b5d6a..5a212e060 100644 --- a/src/particles/pic/deposition/pic_depo.f90 +++ b/src/particles/pic/deposition/pic_depo.f90 @@ -382,7 +382,7 @@ SUBROUTINE InitializeDeposition() , MPI_INTEGER & , iProc & , 1999 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequestNonSymDepo(iProc) & , IERROR) END DO @@ -395,7 +395,7 @@ SUBROUTINE InitializeDeposition() , MPI_INTEGER & , iProc & , 1999 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequestNonSymDepo(iProc) & , IERROR) END DO @@ -437,7 +437,7 @@ SUBROUTINE InitializeDeposition() , MPI_INTEGER & , NodeRecvDepoRankToGlobalRank(iProc) & , 666 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequest(iProc) & , IERROR) END DO @@ -464,7 +464,7 @@ SUBROUTINE InitializeDeposition() , MPI_INTEGER & , NodeSendDepoRankToGlobalRank(iProc) & , 666 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequest(iProc) & , IERROR) END DO @@ -981,7 +981,7 @@ SUBROUTINE ExchangeNodeSourceExtTmp() , MPI_DOUBLE_PRECISION & , NodeRecvDepoRankToGlobalRank(iProc) & , 666 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequest(iProc) & , IERROR) END DO @@ -997,7 +997,7 @@ SUBROUTINE ExchangeNodeSourceExtTmp() , MPI_DOUBLE_PRECISION & , NodeSendDepoRankToGlobalRank(iProc) & , 666 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequest(iProc) & , IERROR) END DO @@ -1298,7 +1298,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 1667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequestNonSymDepo(iProc) & , IERROR) CALL MPI_ISEND( SendPeriodicNodes(iProc) & @@ -1306,7 +1306,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 1667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequestNonSymDepo(iProc) & , IERROR) END DO @@ -1329,7 +1329,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequestNonSymDepo(iProc) & , IERROR) END IF @@ -1339,7 +1339,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequestNonSymDepo(iProc) & , IERROR) END IF @@ -1437,7 +1437,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 1667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequestNonSymDepo(iProc) & , IERROR) CALL MPI_ISEND( SendPeriodicNodes(iProc) & @@ -1445,7 +1445,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 1667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequestNonSymDepo(iProc) & , IERROR) END DO @@ -1468,7 +1468,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequestNonSymDepo(iProc) & , IERROR) END IF @@ -1478,7 +1478,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequestNonSymDepo(iProc) & , IERROR) END IF @@ -1520,7 +1520,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequestNonSymDepo(iProc) & , IERROR) END IF @@ -1530,7 +1530,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequestNonSymDepo(iProc) & , IERROR) END IF @@ -1648,7 +1648,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 1667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequestNonSymDepo(iProc) & , IERROR) CALL MPI_ISEND( SendPeriodicNodes(iProc) & @@ -1656,7 +1656,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 1667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequestNonSymDepo(iProc) & , IERROR) END DO @@ -1679,7 +1679,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequestNonSymDepo(iProc) & , IERROR) END IF @@ -1689,7 +1689,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequestNonSymDepo(iProc) & , IERROR) END IF @@ -1733,7 +1733,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequestNonSymDepo(iProc) & , IERROR) END IF @@ -1743,7 +1743,7 @@ SUBROUTINE InitializePeriodicNodes(& , MPI_INTEGER & , iProc & , 667 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequestNonSymDepo(iProc) & , IERROR) END IF @@ -1786,7 +1786,7 @@ SUBROUTINE InitializePeriodicNodes(& IF (ANY(PeriodicNodeMap(iNode)%Mapping.EQ.0)) THEN DO jNode = 1, PeriodicNodeMap(iNode)%nPeriodicNodes IF (PeriodicNodeMap(iNode)%Mapping(jNode).EQ.0) THEN - DO kNode =1, jNode - 1 + DO kNode =1, jNode - 1 zGlobalNode = PeriodicNodeMap(iNode)%Mapping(kNode) DO zNode = 1, PeriodicNodeMap(zGlobalNode)%nPeriodicNodes IF ((PeriodicNodeMap(zGlobalNode)%Mapping(zNode).NE.0).AND.(PeriodicNodeMap(zGlobalNode)%Mapping(zNode).NE.iNode)) THEN diff --git a/src/particles/pic/deposition/pic_depo_method.f90 b/src/particles/pic/deposition/pic_depo_method.f90 index 895e9e54e..dd4951ca6 100644 --- a/src/particles/pic/deposition/pic_depo_method.f90 +++ b/src/particles/pic/deposition/pic_depo_method.f90 @@ -545,7 +545,7 @@ SUBROUTINE DepositionMethod_CVWM(doParticle_In, stage_opt) , MPI_DOUBLE_PRECISION & , NodeRecvDepoRankToGlobalRank(iProc) & , 666 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequest(iProc) & , IERROR) END DO @@ -561,7 +561,7 @@ SUBROUTINE DepositionMethod_CVWM(doParticle_In, stage_opt) , MPI_DOUBLE_PRECISION & , NodeSendDepoRankToGlobalRank(iProc) & , 666 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequest(iProc) & , IERROR) END DO @@ -593,7 +593,7 @@ SUBROUTINE DepositionMethod_CVWM(doParticle_In, stage_opt) , MPI_DOUBLE_PRECISION & , NodeRecvDepoRankToGlobalRank(iProc) & , 666 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequest(iProc) & , IERROR) END DO @@ -608,7 +608,7 @@ SUBROUTINE DepositionMethod_CVWM(doParticle_In, stage_opt) , MPI_DOUBLE_PRECISION & , NodeSendDepoRankToGlobalRank(iProc) & , 666 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequest(iProc) & , IERROR) END DO @@ -794,7 +794,7 @@ SUBROUTINE DepositionMethod_SF(doParticle_In, stage_opt) , MPI_DOUBLE_PRECISION & , ShapeMapping(iProc)%Rank & , 2001 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , RecvRequest(iProc) & , IERROR) ! IF (myComputeNodeRank.NE.0) THEN @@ -803,7 +803,7 @@ SUBROUTINE DepositionMethod_SF(doParticle_In, stage_opt) , MPI_DOUBLE_PRECISION & , ShapeMapping(iProc)%Rank & , 2001 & - , MPI_COMM_WORLD & + , MPI_COMM_PICLAS & , SendRequest(iProc) & , IERROR) ! END IF diff --git a/src/particles/pic/deposition/pic_depo_tools.f90 b/src/particles/pic/deposition/pic_depo_tools.f90 index 10440aa81..035664778 100644 --- a/src/particles/pic/deposition/pic_depo_tools.f90 +++ b/src/particles/pic/deposition/pic_depo_tools.f90 @@ -378,7 +378,7 @@ SUBROUTINE ReadTimeAverage(FileName) IF(.NOT.FILEEXISTS(FileName)) CALL abort(__STAMP__, & 'TimeAverage-File "'//TRIM(FileName)//'" does not exist',999,999.) END IF -CALL OpenDataFile(TRIM(FileName),create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(TRIM(FileName),create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) ! get attributes CALL DatasetExists(File_ID,'DG_Solution',SolutionExists) diff --git a/src/particles/pic/interpolation/init_BGField.f90 b/src/particles/pic/interpolation/init_BGField.f90 index c4cbc6338..df2504d1f 100644 --- a/src/particles/pic/interpolation/init_BGField.f90 +++ b/src/particles/pic/interpolation/init_BGField.f90 @@ -129,7 +129,7 @@ SUBROUTINE InitializeBackgroundField ELSE BGFieldScaling = GETREAL('PIC-BGFieldScaling','1.') ! 2b) Read-in the parameters from the BGField file - CALL OpenDataFile(BGFileName,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(BGFileName,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL DatasetExists(File_ID,'BGField',BGFieldExists) ! backward compatibility CALL DatasetExists(File_ID,'DG_Solution',DG_SolutionExists) IF(BGFieldExists) THEN diff --git a/src/particles/restart/particle_readin.f90 b/src/particles/restart/particle_readin.f90 index 529cc4cda..7349770ce 100644 --- a/src/particles/restart/particle_readin.f90 +++ b/src/particles/restart/particle_readin.f90 @@ -156,7 +156,7 @@ SUBROUTINE ParticleReadin() CALL MPI_TYPE_CREATE_STRUCT(1,MPI_LENGTH,MPI_DISPLACEMENT,MPI_TYPE,MPI_STRUCT,iError) CALL MPI_TYPE_COMMIT(MPI_STRUCT,iError) - CALL MPI_ALLTOALLV(PartSourceLB,counts_send,disp_send,MPI_STRUCT,PartSource_HDF5,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_WORLD,iError) + CALL MPI_ALLTOALLV(PartSourceLB,counts_send,disp_send,MPI_STRUCT,PartSource_HDF5,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_PICLAS,iError) END ASSOCIATE DEALLOCATE(PartSourceLB) @@ -212,7 +212,7 @@ SUBROUTINE ParticleReadin() CALL MPI_TYPE_CREATE_STRUCT(1,MPI_LENGTH,MPI_DISPLACEMENT,MPI_TYPE,MPI_STRUCT,iError) CALL MPI_TYPE_COMMIT(MPI_STRUCT,iError) - CALL MPI_ALLTOALLV(NodeSourceExtEquiLB,counts_send,disp_send,MPI_STRUCT,NodeSourceExtEquiLBTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_WORLD,iError) + CALL MPI_ALLTOALLV(NodeSourceExtEquiLB,counts_send,disp_send,MPI_STRUCT,NodeSourceExtEquiLBTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_PICLAS,iError) END ASSOCIATE DEALLOCATE(NodeSourceExtEquiLB) ! Loop over all elements and store absolute charge values in equidistantly distributed nodes of PP_N=1 @@ -285,7 +285,7 @@ SUBROUTINE ParticleReadin() CALL MPI_TYPE_COMMIT(MPI_STRUCT,iError) ! Communicate PartInt over MPI - CALL MPI_ALLTOALLV(PartInt,counts_send,disp_send,MPI_STRUCT,PartIntTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_WORLD,iError) + CALL MPI_ALLTOALLV(PartInt,counts_send,disp_send,MPI_STRUCT,PartIntTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_PICLAS,iError) END ASSOCIATE ! Calculate the PartInt deltas @@ -336,7 +336,7 @@ SUBROUTINE ParticleReadin() CALL MPI_TYPE_COMMIT(MPI_STRUCT,iError) ! Communicate PartData over MPI - CALL MPI_ALLTOALLV(PartData,counts_send,disp_send,MPI_STRUCT,PartDataTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_WORLD,iError) + CALL MPI_ALLTOALLV(PartData,counts_send,disp_send,MPI_STRUCT,PartDataTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_PICLAS,iError) END ASSOCIATE CALL MOVE_ALLOC(PartDataTmp,PartData) PartDataExists = .TRUE. @@ -361,7 +361,7 @@ SUBROUTINE ParticleReadin() CALL MPI_TYPE_COMMIT(MPI_STRUCT,iError) ! Communicate VibQuantData over MPI - CALL MPI_ALLTOALLV(VibQuantData,counts_send,disp_send,MPI_STRUCT,VibQuantDataTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_WORLD,iError) + CALL MPI_ALLTOALLV(VibQuantData,counts_send,disp_send,MPI_STRUCT,VibQuantDataTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_PICLAS,iError) END ASSOCIATE CALL MOVE_ALLOC(VibQuantDataTmp,VibQuantData) END IF @@ -383,7 +383,7 @@ SUBROUTINE ParticleReadin() CALL MPI_TYPE_COMMIT(MPI_STRUCT,iError) ! Communicate ElecDistriData over MPI - CALL MPI_ALLTOALLV(ElecDistriData,counts_send,disp_send,MPI_STRUCT,ElecDistriDataTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_WORLD,iError) + CALL MPI_ALLTOALLV(ElecDistriData,counts_send,disp_send,MPI_STRUCT,ElecDistriDataTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_PICLAS,iError) END ASSOCIATE CALL MOVE_ALLOC(ElecDistriDataTmp,ElecDistriData) END IF @@ -405,7 +405,7 @@ SUBROUTINE ParticleReadin() CALL MPI_TYPE_COMMIT(MPI_STRUCT,iError) ! Communicate AD_Data over MPI - CALL MPI_ALLTOALLV(AD_Data,counts_send,disp_send,MPI_STRUCT,AD_DataTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_WORLD,iError) + CALL MPI_ALLTOALLV(AD_Data,counts_send,disp_send,MPI_STRUCT,AD_DataTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_PICLAS,iError) END ASSOCIATE CALL MOVE_ALLOC(AD_DataTmp,AD_Data) END IF @@ -426,7 +426,7 @@ SUBROUTINE ParticleReadin() IF(.NOT.RestartNullifySolution)THEN ! Use the solution in the restart file !-- read PartSource if relaxation is performed (might be needed for RestartHDG) IF (DoDeposition .AND. RelaxDeposition) THEN - CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL DatasetExists(File_ID,'DG_Source',DGSourceExists) IF(DGSourceExists)THEN IF(.NOT.InterpolateSolution)THEN! No interpolation needed, read solution directly from file @@ -482,7 +482,7 @@ SUBROUTINE ParticleReadin() END DO END IF - CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) ! ------------------------------------------------ ! NodeSourceExt (external/additional charge source terms) ! ------------------------------------------------ diff --git a/src/particles/restart/particle_restart.f90 b/src/particles/restart/particle_restart.f90 index aec9ecfac..cb5922bd3 100644 --- a/src/particles/restart/particle_restart.f90 +++ b/src/particles/restart/particle_restart.f90 @@ -75,7 +75,6 @@ SUBROUTINE ParticleRestart() #endif /*USE_HDG*/ ! MPI #if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI USE MOD_Particle_Vars ,ONLY: PartDataSize #endif /*USE_MPI*/ USE MOD_Particle_Vars ,ONLY: VibQuantData,ElecDistriData,AD_Data @@ -112,14 +111,14 @@ SUBROUTINE ParticleRestart() INTEGER,ALLOCATABLE :: IndexOfFoundParticles(:),CompleteIndexOfFoundParticles(:) INTEGER :: CompleteNbrOfLost,CompleteNbrOfFound,CompleteNbrOfDuplicate REAL, ALLOCATABLE :: RecBuff(:,:) -INTEGER :: TotalNbrOfMissingParticles(0:PartMPI%nProcs-1), Displace(0:PartMPI%nProcs-1),CurrentPartNum -INTEGER :: OffsetTotalNbrOfMissingParticles(0:PartMPI%nProcs-1) -INTEGER :: NbrOfFoundParts, RecCount(0:PartMPI%nProcs-1) +INTEGER :: TotalNbrOfMissingParticles(0:nProcessors-1), Displace(0:nProcessors-1),CurrentPartNum +INTEGER :: OffsetTotalNbrOfMissingParticles(0:nProcessors-1) +INTEGER :: NbrOfFoundParts, RecCount(0:nProcessors-1) INTEGER, ALLOCATABLE :: SendBuffPoly(:), RecBuffPoly(:) REAL, ALLOCATABLE :: SendBuffAmbi(:), RecBuffAmbi(:), SendBuffElec(:), RecBuffElec(:) -INTEGER :: LostPartsPoly(0:PartMPI%nProcs-1), DisplacePoly(0:PartMPI%nProcs-1) -INTEGER :: LostPartsElec(0:PartMPI%nProcs-1), DisplaceElec(0:PartMPI%nProcs-1) -INTEGER :: LostPartsAmbi(0:PartMPI%nProcs-1), DisplaceAmbi(0:PartMPI%nProcs-1) +INTEGER :: LostPartsPoly(0:nProcessors-1), DisplacePoly(0:nProcessors-1) +INTEGER :: LostPartsElec(0:nProcessors-1), DisplaceElec(0:nProcessors-1) +INTEGER :: LostPartsAmbi(0:nProcessors-1), DisplaceAmbi(0:nProcessors-1) INTEGER :: iProc #endif /*USE_MPI*/ !=================================================================================================================================== @@ -444,12 +443,12 @@ SUBROUTINE ParticleRestart() ! Step 2: All particles that are not found within MyProc need to be communicated to the others and located there ! Combine number of lost particles of all processes and allocate variables ! Note: Particles that are lost on MyProc are also searched for here again - CALL MPI_ALLGATHER(NbrOfLostParticles, 1, MPI_INTEGER, TotalNbrOfMissingParticles, 1, MPI_INTEGER, PartMPI%COMM, IERROR) + CALL MPI_ALLGATHER(NbrOfLostParticles, 1, MPI_INTEGER, TotalNbrOfMissingParticles, 1, MPI_INTEGER, MPI_COMM_PICLAS, IERROR) NbrOfLostParticles=0 IF (useDSMC) THEN - IF (DSMC%NumPolyatomMolecs.GT.0) CALL MPI_ALLGATHER(CounterPoly, 1, MPI_INTEGER, LostPartsPoly, 1, MPI_INTEGER, PartMPI%COMM, IERROR) - IF (DSMC%ElectronicModel.EQ.2) CALL MPI_ALLGATHER(CounterElec, 1, MPI_INTEGER, LostPartsElec, 1, MPI_INTEGER, PartMPI%COMM, IERROR) - IF (DSMC%DoAmbipolarDiff) CALL MPI_ALLGATHER(CounterAmbi, 1, MPI_INTEGER, LostPartsAmbi, 1, MPI_INTEGER, PartMPI%COMM, IERROR) + IF (DSMC%NumPolyatomMolecs.GT.0) CALL MPI_ALLGATHER(CounterPoly, 1, MPI_INTEGER, LostPartsPoly, 1, MPI_INTEGER, MPI_COMM_PICLAS, IERROR) + IF (DSMC%ElectronicModel.EQ.2) CALL MPI_ALLGATHER(CounterElec, 1, MPI_INTEGER, LostPartsElec, 1, MPI_INTEGER, MPI_COMM_PICLAS, IERROR) + IF (DSMC%DoAmbipolarDiff) CALL MPI_ALLGATHER(CounterAmbi, 1, MPI_INTEGER, LostPartsAmbi, 1, MPI_INTEGER, MPI_COMM_PICLAS, IERROR) END IF ! useDSMC !TotalNbrOfMissingParticlesSum = SUM(INT(TotalNbrOfMissingParticles,8)) @@ -460,9 +459,9 @@ SUBROUTINE ParticleRestart() ! Set offsets OffsetTotalNbrOfMissingParticles(0) = 0 - DO iProc = 1, PartMPI%nProcs-1 + DO iProc = 1, nProcessors-1 OffsetTotalNbrOfMissingParticles(iProc) = OffsetTotalNbrOfMissingParticles(iProc-1) + TotalNbrOfMissingParticles(iProc-1) - END DO ! iProc = 0, PartMPI%nProcs-1 + END DO ! iProc = 0, nProcessors-1 ALLOCATE(RecBuff(PartDataSize,1:TotalNbrOfMissingParticlesSum)) IF (useDSMC) THEN @@ -484,7 +483,7 @@ SUBROUTINE ParticleRestart() END IF ! useDSMC ! Fill SendBuffer - NbrOfMissingParticles = OffsetTotalNbrOfMissingParticles(PartMPI%MyRank) + 1 + NbrOfMissingParticles = OffsetTotalNbrOfMissingParticles(myRank) + 1 CounterPoly = 0 CounterAmbi = 0 CounterElec = 0 @@ -550,7 +549,7 @@ SUBROUTINE ParticleRestart() CounterElec = 0 CounterAmbi = 0 - DO iProc = 0, PartMPI%nProcs-1 + DO iProc = 0, nProcessors-1 RecCount(iProc) = TotalNbrOfMissingParticles(iProc) Displace(iProc) = NbrOfMissingParticles NbrOfMissingParticles = NbrOfMissingParticles + TotalNbrOfMissingParticles(iProc) @@ -572,7 +571,7 @@ SUBROUTINE ParticleRestart() CounterAmbi = CounterAmbi + LostPartsAmbi(iProc) END IF END IF ! useDSMC - END DO ! iProc = 0, PartMPI%nProcs-1 + END DO ! iProc = 0, nProcessors-1 CALL MPI_ALLGATHERV( MPI_IN_PLACE & , 0 & @@ -581,19 +580,19 @@ SUBROUTINE ParticleRestart() , PartDataSize*TotalNbrOfMissingParticles(:) & , PartDataSize*OffsetTotalNbrOfMissingParticles(:) & , MPI_DOUBLE_PRECISION & - , PartMPI%COMM & + , MPI_COMM_PICLAS & , IERROR) IF (useDSMC) THEN ! Polyatomic - IF (DSMC%NumPolyatomMolecs.GT.0) CALL MPI_ALLGATHERV(SendBuffPoly, LostPartsPoly(PartMPI%MyRank), MPI_INTEGER, & - RecBuffPoly, LostPartsPoly, DisplacePoly, MPI_INTEGER, PartMPI%COMM, IERROR) + IF (DSMC%NumPolyatomMolecs.GT.0) CALL MPI_ALLGATHERV(SendBuffPoly, LostPartsPoly(myRank), MPI_INTEGER, & + RecBuffPoly, LostPartsPoly, DisplacePoly, MPI_INTEGER, MPI_COMM_PICLAS, IERROR) ! Electronic - IF (DSMC%ElectronicModel.EQ.2) CALL MPI_ALLGATHERV(SendBuffElec, LostPartsElec(PartMPI%MyRank), MPI_INTEGER, & - RecBuffElec, LostPartsElec, DisplaceElec, MPI_DOUBLE_PRECISION, PartMPI%COMM, IERROR) + IF (DSMC%ElectronicModel.EQ.2) CALL MPI_ALLGATHERV(SendBuffElec, LostPartsElec(myRank), MPI_INTEGER, & + RecBuffElec, LostPartsElec, DisplaceElec, MPI_DOUBLE_PRECISION, MPI_COMM_PICLAS, IERROR) ! Ambipolar Diffusion - IF (DSMC%DoAmbipolarDiff) CALL MPI_ALLGATHERV(SendBuffAmbi, LostPartsAmbi(PartMPI%MyRank), MPI_INTEGER, & - RecBuffAmbi, LostPartsAmbi, DisplaceAmbi, MPI_DOUBLE_PRECISION, PartMPI%COMM, IERROR) + IF (DSMC%DoAmbipolarDiff) CALL MPI_ALLGATHERV(SendBuffAmbi, LostPartsAmbi(myRank), MPI_INTEGER, & + RecBuffAmbi, LostPartsAmbi, DisplaceAmbi, MPI_DOUBLE_PRECISION, MPI_COMM_PICLAS, IERROR) END IF ! Keep track which particles are found on the current proc @@ -622,8 +621,8 @@ SUBROUTINE ParticleRestart() ! Do not search particles twice: Skip my own particles, because these have already been searched for before they are ! sent to all other procs - ASSOCIATE( myFirst => OffsetTotalNbrOfMissingParticles(PartMPI%MyRank) + 1 ,& - myLast => OffsetTotalNbrOfMissingParticles(PartMPI%MyRank) + TotalNbrOfMissingParticles(PartMPI%MyRank)) + ASSOCIATE( myFirst => OffsetTotalNbrOfMissingParticles(myRank) + 1 ,& + myLast => OffsetTotalNbrOfMissingParticles(myRank) + TotalNbrOfMissingParticles(myRank)) IF((iPart.GE.myFirst).AND.(iPart.LE.myLast))THEN IndexOfFoundParticles(iPart) = 0 CYCLE @@ -721,9 +720,9 @@ SUBROUTINE ParticleRestart() ! Combine number of found particles to make sure none are lost completely or found twice IF(MPIroot)THEN - CALL MPI_REDUCE(IndexOfFoundParticles,CompleteIndexOfFoundParticles,TotalNbrOfMissingParticlesSum,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM,IERROR) + CALL MPI_REDUCE(IndexOfFoundParticles,CompleteIndexOfFoundParticles,TotalNbrOfMissingParticlesSum,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) ELSE - CALL MPI_REDUCE(IndexOfFoundParticles,0 ,TotalNbrOfMissingParticlesSum,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM,IERROR) + CALL MPI_REDUCE(IndexOfFoundParticles,0 ,TotalNbrOfMissingParticlesSum,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) END IF CompleteNbrOfFound = 0 @@ -766,7 +765,7 @@ SUBROUTINE ParticleRestart() DEALLOCATE(CompleteIndexOfFoundParticles) END IF ! MPIRoot - CALL MPI_BCAST(NbrOfLostParticlesTotal,1,MPI_INTEGER,0,MPI_COMM_WORLD,iError) + CALL MPI_BCAST(NbrOfLostParticlesTotal,1,MPI_INTEGER,0,MPI_COMM_PICLAS,iError) NbrOfLostParticlesTotal_old = NbrOfLostParticlesTotal END IF ! TotalNbrOfMissingParticlesSum.GT.0 @@ -1112,7 +1111,7 @@ SUBROUTINE MacroscopicRestart() SWRITE(UNIT_stdOut,*) 'Using macroscopic values from file: ',TRIM(MacroRestartFileName) -CALL OpenDataFile(MacroRestartFileName,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(MacroRestartFileName,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) ! Check if the provided file is a DSMC state file. CALL ReadAttribute(File_ID,'File_Type',1,StrScalar=File_Type) diff --git a/src/particles/sampling/particle_sampling_adaptive.f90 b/src/particles/sampling/particle_sampling_adaptive.f90 index 157ebc19d..05f912a84 100644 --- a/src/particles/sampling/particle_sampling_adaptive.f90 +++ b/src/particles/sampling/particle_sampling_adaptive.f90 @@ -81,9 +81,6 @@ SUBROUTINE InitAdaptiveBCSampling() USE MOD_Particle_Mesh_Vars ,ONLY: SideInfo_Shared, ElemVolume_Shared USE MOD_LoadBalance_Vars ,ONLY: DoLoadBalance, PerformLoadBalance, UseH5IOLoadBalance USE MOD_Mesh_Tools ,ONLY: GetCNElemID -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ ! IMPLICIT VARIABLE HANDLING IMPLICIT NONE !----------------------------------------------------------------------------------------------------------------------------------- @@ -166,16 +163,16 @@ SUBROUTINE InitAdaptiveBCSampling() #if USE_MPI IF(UseCircularInflow) THEN - IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,AdaptBCAreaSurfaceFlux,nSpecies*nSurfacefluxBCs,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,iError) + IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE,AdaptBCAreaSurfaceFlux,nSpecies*nSurfacefluxBCs,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,iError) ELSE - CALL MPI_REDUCE(AdaptBCAreaSurfaceFlux,0,nSpecies*nSurfacefluxBCs,MPI_DOUBLE_PRECISION,MPI_SUM,0,PartMPI%COMM,iError) + CALL MPI_REDUCE(AdaptBCAreaSurfaceFlux,0,nSpecies*nSurfacefluxBCs,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,iError) END IF END IF #endif /*USE_MPI*/ #if USE_MPI -CALL MPI_ALLREDUCE(MPI_IN_PLACE,AdaptBCVolSurfaceFlux,nSpecies*nSurfacefluxBCs,MPI_DOUBLE_PRECISION,MPI_SUM,PartMPI%COMM,iError) +CALL MPI_ALLREDUCE(MPI_IN_PLACE,AdaptBCVolSurfaceFlux,nSpecies*nSurfacefluxBCs,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ ! 1b) Add elements for the porous BCs @@ -214,9 +211,9 @@ SUBROUTINE InitAdaptiveBCSampling() #if USE_MPI ! Gather the number of sampling elements per proc -CALL MPI_GATHER(AdaptBCSampleElemNum,1,MPI_INTEGER_INT_KIND,offSetElemAdaptBCSampleMPI,1,MPI_INTEGER_INT_KIND,0,MPI_COMM_WORLD,iError) +CALL MPI_GATHER(AdaptBCSampleElemNum,1,MPI_INTEGER_INT_KIND,offSetElemAdaptBCSampleMPI,1,MPI_INTEGER_INT_KIND,0,MPI_COMM_PICLAS,iError) ! Distribute the number of elements per proc to each each proc -CALL MPI_BCAST(offSetElemAdaptBCSampleMPI,nProcessors,MPI_INTEGER,0,MPI_COMM_WORLD,iERROR) +CALL MPI_BCAST(offSetElemAdaptBCSampleMPI,nProcessors,MPI_INTEGER,0,MPI_COMM_PICLAS,iERROR) ! Determine the offset for the sampling elements IF(myRank.EQ.0) THEN offSetElemAdaptBCSample = 0 @@ -292,7 +289,7 @@ SUBROUTINE InitAdaptiveBCSampling() ! 4) If restart is done, check if adaptiveinfo exists in state, read it in and write to AdaptBCMacroValues IF (DoRestart) THEN - CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) ! read local ParticleInfo from HDF5 CALL DatasetExists(File_ID,'AdaptiveInfo',AdaptiveDataExists) IF(AdaptiveDataExists)THEN @@ -588,7 +585,7 @@ SUBROUTINE AdaptiveBCSampling(initSampling_opt,initTruncAverage_opt) RelaxationFactor = AdaptBCRelaxFactor IF(AdaptBCSampIter.GT.0) THEN IF(AdaptBCTruncAverage.AND.(RestartSampIter.GT.0).AND.(RestartSampIter.LT.AdaptBCSampIter)) THEN - ! Truncated average: get the correct number of samples to calculate the average number density while the + ! Truncated average: get the correct number of samples to calculate the average number density while the ! sampling array is populated SamplingIteration = RestartSampIter ELSE @@ -665,9 +662,9 @@ SUBROUTINE AdaptiveBCSampling(initSampling_opt,initTruncAverage_opt) ! MPI Communication #if USE_MPI IF(MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,AdaptBCMeanValues,8*nSpecies*nSurfacefluxBCs,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_WORLD,iError) + CALL MPI_REDUCE(MPI_IN_PLACE,AdaptBCMeanValues,8*nSpecies*nSurfacefluxBCs,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,iError) ELSE - CALL MPI_REDUCE(AdaptBCMeanValues,0.,8*nSpecies*nSurfacefluxBCs,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_WORLD,iError) + CALL MPI_REDUCE(AdaptBCMeanValues,0.,8*nSpecies*nSurfacefluxBCs,MPI_DOUBLE_PRECISION,MPI_SUM,0,MPI_COMM_PICLAS,iError) END IF #endif /*USE_MPI*/ IF(MPIRoot) THEN @@ -726,8 +723,8 @@ SUBROUTINE AdaptiveBCSampling(initSampling_opt,initTruncAverage_opt) END DO END IF #if USE_MPI - CALL MPI_BCAST(AdaptBCMeanValues,8*nSpecies*nSurfacefluxBCs, MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,iERROR) - CALL MPI_BCAST(AdaptBCAverageMacroVal,3*nSpecies*nSurfacefluxBCs, MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,iERROR) + CALL MPI_BCAST(AdaptBCMeanValues,8*nSpecies*nSurfacefluxBCs, MPI_DOUBLE_PRECISION,0,MPI_COMM_PICLAS,iERROR) + CALL MPI_BCAST(AdaptBCAverageMacroVal,3*nSpecies*nSurfacefluxBCs, MPI_DOUBLE_PRECISION,0,MPI_COMM_PICLAS,iERROR) #endif /*USE_MPI*/ END IF END IF diff --git a/src/particles/tracking/particle_tracing.f90 b/src/particles/tracking/particle_tracing.f90 index 54f6d830c..c1414fcf5 100644 --- a/src/particles/tracking/particle_tracing.f90 +++ b/src/particles/tracking/particle_tracing.f90 @@ -594,7 +594,7 @@ SUBROUTINE ParticleTracing() !---------------------------------------------CODE_ANALYZE-------------------------------------------------------------------------- ! check if particle is still inside of bounding box of domain and in element #if USE_MPI -CALL MPI_BARRIER(MPI_COMM_WORLD,iError) +CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ DO iPart=1,PDM%ParticleVecLength #ifdef IMPA diff --git a/src/particles/ttm/ttm_init.f90 b/src/particles/ttm/ttm_init.f90 index 4bd85de33..9d7c0f9dd 100644 --- a/src/particles/ttm/ttm_init.f90 +++ b/src/particles/ttm/ttm_init.f90 @@ -114,9 +114,6 @@ SUBROUTINE InitTTM() USE MOD_ReadInTools USE MOD_TTM_Vars USE MOD_Restart_Vars ,ONLY: DoRestart -#if USE_MPI -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI -#endif /*USE_MPI*/ USE MOD_Mesh_Vars ,ONLY: ElemBaryNGeo USE MOD_Globals_Vars ,ONLY: BoltzmannConst,eps0 USE MOD_IO_HDF5 ,ONLY: AddToElemData,ElementOut @@ -196,7 +193,7 @@ SUBROUTINE InitTTM() ElemIsDone=.FALSE. LBWRITE(UNIT_stdOut,'(A,A)') " Reading TTM data from file (TTMFile): ",TRIM(TTMFile) #if USE_MPI - IF(.NOT.PartMPI%MPIROOT)THEN + IF(.NOT.MPIRoot)THEN CALL abort(& __STAMP__& ,'ERROR: Cannot SetParticlePosition in multi-core environment for SpaceIC=IMD!') diff --git a/src/posti/dmd/dmd_main.f90 b/src/posti/dmd/dmd_main.f90 index e52300131..08e17bc94 100644 --- a/src/posti/dmd/dmd_main.f90 +++ b/src/posti/dmd/dmd_main.f90 @@ -147,7 +147,7 @@ SUBROUTINE InitDMD() END IF ! nModes .GT. (nFiles-1) ! Open the first statefile to read necessary attributes -CALL OpenDataFile(Args(2),create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(Args(2),create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL ReadAttribute(File_ID,'MeshFile', 1,StrScalar=MeshFile_state) CALL ReadAttribute(File_ID,'Project_Name',1,StrScalar=ProjectName) CALL ReadAttribute(File_ID,'Time', 1,RealScalar=starttime) @@ -202,7 +202,7 @@ SUBROUTINE InitDMD() !WRITE(UNIT_stdOut,'(132("="))') ! Read Statefiles - CALL OpenDataFile(Args(iFile),create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(Args(iFile),create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) offset=0 CALL ReadAttribute(File_ID,'Time', 1,RealScalar=time) CALL ReadArray('DG_Solution',5,& @@ -225,7 +225,7 @@ SUBROUTINE InitDMD() TimeEnd_State = time ! IF (useBaseflow) THEN -! CALL OpenDataFile(Baseflow,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) +! CALL OpenDataFile(Baseflow,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) ! CALL ReadAttribute(File_ID,'File_Type',1,StrScalar =FileType) ! SELECT CASE(TRIM(FileType)) ! CASE('State') @@ -233,9 +233,9 @@ SUBROUTINE InitDMD() ! CALL ReadAttribute(File_ID,'Time', 1,RealScalar=time) ! CALL ReadArray('DG_Solution',5,& ! (/nVar_State,N_State+1,N_State+1,N_StateZ+1,nElems_State/),0,5,RealArray=Utmp) -! +! ! CALL CloseDataFile() -! +! ! CALL CalcEquationDMD(Utmp,Ktmp(:)) ! DO iFile = 2,nFiles+1 ! K(:,iFile-1) = K(:,iFile-1) - Ktmp @@ -256,24 +256,24 @@ SUBROUTINE InitDMD() ! END IF ! END DO ! END DO -! +! ! ALLOCATE(Ktmp(nDoFs*nVarDMD)) ! DEALLOCATE(Utmp) ! ALLOCATE(Utmp (HSize(1),0:N_State,0:N_State,0:N_StateZ,nElems_State)) -! +! ! CALL ReadArray('Mean',5,& ! (/INT(HSize(1)),N_State+1,N_State+1,N_StateZ+1,nElems_State/),0,5,RealArray=Utmp) ! CALL CloseDataFile() -! +! ! ktmp(:) = RESHAPE(Utmp(VarSortTimeAvg,:,:,:,:), (/nDoFs*nVarDMD/)) -! +! ! DO iFile = 2,nFiles+1 ! K(:,iFile-1) = K(:,iFile-1) - Ktmp ! END DO -! +! ! DEALLOCATE(ktmp) ! END SELECT -! +! ! END IF DEALLOCATE(Utmp) @@ -566,7 +566,7 @@ SUBROUTINE WriteDmdStateFile() IF(MPIRoot) CALL GenerateFileSkeleton('DMD',N_variables,StrVarNames,TRIM(MeshFile),Time_State,FileNameIn=TRIM(FileName),& WriteUserblockIn = .FALSE.) #if USE_MPI - CALL MPI_BARRIER(MPI_COMM_WORLD,iError) + CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif !write(*,*) "" @@ -574,7 +574,7 @@ SUBROUTINE WriteDmdStateFile() !WRITE (*,*) "nDofs,nVarDMD,nModes =", nDofs,nVarDMD,nModes !write(*,*) "OpenDataFile(TRIM(FileName),create=.FALSE.,single=.FALSE.,readOnly=.FALSE" -CALL OpenDataFile(TRIM(FileName),create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(TRIM(FileName),create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) CALL WriteAttributeToHDF5( File_ID , 'DMD Start Time' , 1 , RealScalar=Time_State ) CALL WriteAttributeToHDF5( File_ID , 'DMD END Time' , 1 , RealScalar=TimeEnd_State ) CALL WriteAttributeToHDF5( File_ID , 'DMD dt' , 1 , RealScalar=dt ) @@ -587,7 +587,7 @@ SUBROUTINE WriteDmdStateFile() !WRITE (*,*) "N_State,N_StateZ,nElems_State =", N_State,N_StateZ,nElems_State !================= Actual data output =======================! ! Write DMD -CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(FileName,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) IF (PlotSingleMode) THEN k=1 DO i = 1, nModes diff --git a/src/posti/piclas2vtk/piclas2vtk.f90 b/src/posti/piclas2vtk/piclas2vtk.f90 index 81667208f..5b7aa76d8 100644 --- a/src/posti/piclas2vtk/piclas2vtk.f90 +++ b/src/posti/piclas2vtk/piclas2vtk.f90 @@ -243,7 +243,7 @@ PROGRAM piclas2vtk SurfaceDataExists = .FALSE. PartDataExists = .FALSE. AdaptiveInfoExists = .FALSE. - CALL OpenDataFile(InputStateFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(InputStateFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) ! Get the type of the .h5 file CALL ReadAttribute(File_ID,'File_Type',1,StrScalar=File_Type) ! Check which containers are present @@ -284,7 +284,7 @@ PROGRAM piclas2vtk END IF ! Build connectivity for element/volume output IF(ElemDataExists.AND..NOT.ElemMeshInit) THEN - CALL OpenDataFile(MeshFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(MeshFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL ReadAttribute(File_ID,'nUniqueNodes',1,IntScalar=nUniqueNodes) CALL CloseDataFile() ALLOCATE(ElemUniqueNodeID(1:8,1:nGlobalElems)) @@ -608,7 +608,7 @@ SUBROUTINE ConvertDGSolution(InputStateFile,NVisu,NodeTypeVisuOut,OutputName,DGS CHARACTER(LEN=255) :: DMDFields(1:16), Dataset !=================================================================================================================================== ! 1.) Open given file to get the number of elements, the order and the name of the mesh file -CALL OpenDataFile(InputStateFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(InputStateFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) ! Check if data set exists, if not return CALL DatasetExists(File_ID,TRIM(DGSolutionDataset),DGSolutionDatasetExists) @@ -695,7 +695,7 @@ SUBROUTINE ConvertDGSolution(InputStateFile,NVisu,NodeTypeVisuOut,OutputName,DGS END DO ! Read in solution -CALL OpenDataFile(InputStateFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(InputStateFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) ! Associate construct for integer KIND=8 possibility ASSOCIATE (& @@ -866,7 +866,7 @@ SUBROUTINE ConvertPartData(InputStateFile) LOGICAL :: FileVersionExists !=================================================================================================================================== -CALL OpenDataFile(InputStateFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(InputStateFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL ReadAttribute(File_ID,'Project_Name',1,StrScalar=ProjectName) CALL ReadAttribute(File_ID,'Time',1,RealScalar=OutputTime) @@ -979,7 +979,7 @@ SUBROUTINE ConvertElemData(InputStateFile,ArrayName,VarName) !=================================================================================================================================== ! Read in solution -CALL OpenDataFile(InputStateFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(InputStateFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL ReadAttribute(File_ID,'Project_Name',1,StrScalar=ProjectName) CALL ReadAttribute(File_ID,'Time',1,RealScalar=OutputTime) CALL ReadAttribute(File_ID,'File_Type',1,StrScalar=File_Type) @@ -1048,7 +1048,7 @@ SUBROUTINE ConvertSurfaceData(InputStateFile) !=================================================================================================================================== ! Read in solution -CALL OpenDataFile(InputStateFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(InputStateFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL ReadAttribute(File_ID,'Project_Name',1,StrScalar=ProjectName) CALL ReadAttribute(File_ID,'Time',1,RealScalar=OutputTime) CALL ReadAttribute(File_ID,'DSMC_nSurfSample',1,IntScalar=nSurfSample) @@ -1134,7 +1134,7 @@ SUBROUTINE BuildSurfMeshConnectivity(InputStateFile) REAL, ALLOCATABLE :: TempNodeCoords(:,:) LOGICAL :: IsSortedSurfNode !=================================================================================================================================== -CALL OpenDataFile(InputStateFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(InputStateFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) SWRITE(UNIT_stdOut,'(A,A)')' GET NUMBER AND NAMES OF SURFACE-BCSIDES IN HDF5 FILE... ' CALL GetDataSize(File_ID,'BC_Surf',nDims,HSize,attrib=.true.) diff --git a/src/posti/superB/superB_tools.f90 b/src/posti/superB/superB_tools.f90 index a1a591fd6..2504d898c 100644 --- a/src/posti/superB/superB_tools.f90 +++ b/src/posti/superB/superB_tools.f90 @@ -190,11 +190,11 @@ SUBROUTINE CalcErrorSuperB(L_2_Error,L_Inf_Error,ExactFunctionNumber,iCoilOrMagn END DO ! iElem=1,PP_nElems #if USE_MPI IF(MPIroot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE , L_2_Error , 4 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(MPI_IN_PLACE , L_Inf_Error , 4 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(MPI_IN_PLACE , L_2_Error , 4 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(MPI_IN_PLACE , L_Inf_Error , 4 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) ELSE - CALL MPI_REDUCE(L_2_Error , 0 , 4 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(L_Inf_Error , 0 , 4 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(L_2_Error , 0 , 4 , MPI_DOUBLE_PRECISION , MPI_SUM , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(L_Inf_Error , 0 , 4 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) ! in this case the receive value is not relevant. END IF #endif /*USE_MPI*/ diff --git a/src/precond/precond.f90 b/src/precond/precond.f90 index 9ffdfacf0..513d4034f 100644 --- a/src/precond/precond.f90 +++ b/src/precond/precond.f90 @@ -378,7 +378,7 @@ SUBROUTINE BuildPrecond(t,tStage,tDeriv,alpha,dt) END DO ! ! iELEM #if USE_MPI -CALL MPI_REDUCE(TotalTime,TotalTimeMPI ,3,MPI_DOUBLE_PRECISION,MPI_MAX,0,MPI_COMM_WORLD,IERROR) +CALL MPI_REDUCE(TotalTime,TotalTimeMPI ,3,MPI_DOUBLE_PRECISION,MPI_MAX,0,MPI_COMM_PICLAS,IERROR) IF(MPIRoot) THEN TotalTime=TotalTimeMPI END IF diff --git a/src/readIMD/readIMD.f90 b/src/readIMD/readIMD.f90 index 8450ec681..5dc97a324 100644 --- a/src/readIMD/readIMD.f90 +++ b/src/readIMD/readIMD.f90 @@ -108,7 +108,7 @@ subroutine read_IMD_results() write(UNIT_errOut,'(A34,I8,6A)')'Error during MPI_File_open! Rank: ', myRank, '\n',& 'Error-Message: ', trim(errorString), ' "',trim(filenameIMDresults), '"',& 'The program will be terminated!!!' - call MPI_Abort(MPI_COMM_WORLD, mpiFileError, iError) + call MPI_Abort(MPI_COMM_PICLAS, mpiFileError, iError) end if call MPI_FILE_READ_AT(filehandle,3_8,disp_tmp,2,MPI_BYTE,ioStatus,iError) @@ -127,12 +127,12 @@ subroutine read_IMD_results() end if - call MPI_BCAST(Box_X, 3, MPI_DOUBLE_PRECISION, 0, MPI_COMM_WORLD, iError) - call MPI_BCAST(Box_Y, 3, MPI_DOUBLE_PRECISION, 0, MPI_COMM_WORLD, iError) - call MPI_BCAST(Box_Z, 3, MPI_DOUBLE_PRECISION, 0, MPI_COMM_WORLD, iError) - call MPI_BCAST(nGlobalAtoms, 8, MPI_BYTE, 0, MPI_COMM_WORLD, iError) - call MPI_BCAST(observables, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, iError) - call MPI_BCAST(disp, 8, MPI_BYTE, 0, MPI_COMM_WORLD, iError) + call MPI_BCAST(Box_X, 3, MPI_DOUBLE_PRECISION, 0, MPI_COMM_PICLAS, iError) + call MPI_BCAST(Box_Y, 3, MPI_DOUBLE_PRECISION, 0, MPI_COMM_PICLAS, iError) + call MPI_BCAST(Box_Z, 3, MPI_DOUBLE_PRECISION, 0, MPI_COMM_PICLAS, iError) + call MPI_BCAST(nGlobalAtoms, 8, MPI_BYTE, 0, MPI_COMM_PICLAS, iError) + call MPI_BCAST(observables, 1, MPI_INTEGER, 0, MPI_COMM_PICLAS, iError) + call MPI_BCAST(disp, 8, MPI_BYTE, 0, MPI_COMM_PICLAS, iError) nAtoms = nGlobalAtoms/nProcessors SWRITE(UNIT_stdOut,*)'Number of atoms per proc: ',nAtoms @@ -170,7 +170,7 @@ subroutine read_IMD_results() StartT=MPI_WTIME() end if - call MPI_FILE_OPEN(MPI_COMM_WORLD, trim(filenameIMDresults), MPI_MODE_RDONLY,& + call MPI_FILE_OPEN(MPI_COMM_PICLAS, trim(filenameIMDresults), MPI_MODE_RDONLY,& mpiInfo, filehandle, mpiFileError) if( mpiFileError /= 0)then @@ -178,7 +178,7 @@ subroutine read_IMD_results() write(UNIT_errOut,'(A34,I8,6A)')'Error during MPI_File_open! Rank: ', myRank, '\n',& 'Error-Message: ', trim(errorString), ' "',trim(filenameIMDresults), '"',& 'The program will be terminated!!!' - call MPI_Abort(MPI_COMM_WORLD, mpiFileError, iError) + call MPI_Abort(MPI_COMM_PICLAS, mpiFileError, iError) end if call MPI_FILE_SET_VIEW(filehandle, myFileOffset, MPI_PACKED, MPI_PACKED, 'native', mpiInfo, iError) @@ -217,12 +217,12 @@ subroutine read_IMD_results() do iPart=1,PDM%ParticleVecLength call MPI_UNPACK(AtomsBuffer, atomBufferSize, atomsBufferPos, PartState(1:6,iPart),& - 6_4, MPI_DOUBLE_PRECISION, MPI_COMM_WORLD, iError) + 6_4, MPI_DOUBLE_PRECISION, MPI_COMM_PICLAS, iError) if ( iError .NE. 0 ) & IPWRITE(UNIT_stdOut,'(I0,A,I0)')'Error unpacking particle position to PartState(1:6,iPart) with iPart=',iPart call MPI_UNPACK(AtomsBuffer, atomBufferSize, atomsBufferPos, PartStateIntEn(1:2,iPart),& - int( observables-6_8, 4 ), MPI_DOUBLE_PRECISION, MPI_COMM_WORLD, iError) + int( observables-6_8, 4 ), MPI_DOUBLE_PRECISION, MPI_COMM_PICLAS, iError) if ( iError .NE. 0 ) & IPWRITE(UNIT_stdOut,'(I0,A,I0)')'Error unpacking particle charge and electron temperature to PartState(1:2,iPart) with iPart=',iPart end do @@ -238,7 +238,7 @@ subroutine read_IMD_results() PartState = PartState * 1e-10_8 PartState(4:6,:) = PartState(4:6,:) * 10.18e15_8 - ! Free an info object + ! Free an info object call MPI_Info_free(mpiInfo, iError) ! Get minimum and maximum extend of the complete particle distribution in the domain @@ -250,14 +250,14 @@ subroutine read_IMD_results() MaxY = MAXVAL(PartState(2,:)) MaxZ = MAXVAL(PartState(3,:)) - CALL MPI_REDUCE(MaxX , MaxX_glob , 1 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(MinX , MinX_glob , 1 , MPI_DOUBLE_PRECISION , MPI_MIN , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(MaxX , MaxX_glob , 1 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(MinX , MinX_glob , 1 , MPI_DOUBLE_PRECISION , MPI_MIN , 0 , MPI_COMM_PICLAS , iError) - CALL MPI_REDUCE(MaxY , MaxY_glob , 1 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(MinY , MinY_glob , 1 , MPI_DOUBLE_PRECISION , MPI_MIN , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(MaxY , MaxY_glob , 1 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(MinY , MinY_glob , 1 , MPI_DOUBLE_PRECISION , MPI_MIN , 0 , MPI_COMM_PICLAS , iError) - CALL MPI_REDUCE(MaxZ , MaxZ_glob , 1 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) - CALL MPI_REDUCE(MinZ , MinZ_glob , 1 , MPI_DOUBLE_PRECISION , MPI_MIN , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(MaxZ , MaxZ_glob , 1 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) + CALL MPI_REDUCE(MinZ , MinZ_glob , 1 , MPI_DOUBLE_PRECISION , MPI_MIN , 0 , MPI_COMM_PICLAS , iError) if( mpiroot )then EndT=MPI_WTIME() @@ -281,7 +281,7 @@ subroutine read_IMD_results() NbrOfLostParticles=NbrOfLostParticles+1 end if end do - CALL MPI_REDUCE(NbrOfLostParticles , NbrOfLostParticlesGlobal , 1 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(NbrOfLostParticles , NbrOfLostParticlesGlobal , 1 , MPI_DOUBLE_PRECISION , MPI_MAX , 0 , MPI_COMM_PICLAS , iError) if( mpiroot )then EndT=MPI_WTIME() WRITE(UNIT_stdOut,'(A,F0.3,A)',ADVANCE='YES')'DONE [',EndT-StartT,'s]' diff --git a/src/readintools/readintools.f90 b/src/readintools/readintools.f90 index 500e204c5..6b19805fd 100644 --- a/src/readintools/readintools.f90 +++ b/src/readintools/readintools.f90 @@ -817,7 +817,7 @@ SUBROUTINE read_options(this, filename, furtherini) !broadcast number of lines, read and broadcast file content #if USE_MPI -CALL MPI_BCAST(nLines,1,MPI_INTEGER,0,MPI_COMM_WORLD,iError) +CALL MPI_BCAST(nLines,1,MPI_INTEGER,0,MPI_COMM_PICLAS,iError) #endif ALLOCATE(FileContent(nLines)) @@ -828,7 +828,7 @@ SUBROUTINE read_options(this, filename, furtherini) END IF IF (MPIROOT) CLOSE(iniUnit) #if USE_MPI -CALL MPI_BCAST(FileContent,LEN(FileContent)*nLines,MPI_CHARACTER,0,MPI_COMM_WORLD,iError) +CALL MPI_BCAST(FileContent,LEN(FileContent)*nLines,MPI_CHARACTER,0,MPI_COMM_PICLAS,iError) #endif ! infinte loop. Exit at EOF @@ -2140,7 +2140,7 @@ SUBROUTINE ExtractParameterFile(filename,prmfile,userblockFound) CLOSE(iniUnit) END IF #if USE_MPI -CALL MPI_BCAST(userblockFound,1,MPI_LOGICAL,0,MPI_COMM_WORLD,iError) +CALL MPI_BCAST(userblockFound,1,MPI_LOGICAL,0,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ END SUBROUTINE ExtractParameterFile diff --git a/src/recordpoints/recordpoints.f90 b/src/recordpoints/recordpoints.f90 index 32284f86c..990eaa898 100644 --- a/src/recordpoints/recordpoints.f90 +++ b/src/recordpoints/recordpoints.f90 @@ -151,7 +151,7 @@ SUBROUTINE InitRPCommunicator() ! create new RP communicator for RP output color = MERGE(2,MPI_UNDEFINED,RP_onProc) -CALL MPI_COMM_SPLIT(MPI_COMM_WORLD,color,myRPrank,RP_COMM,iError) +CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS,color,myRPrank,RP_COMM,iError) IF (RP_onProc) THEN CALL MPI_COMM_RANK (RP_COMM,myRPrank,iError) CALL MPI_COMM_SIZE(RP_COMM, nRP_Procs,iError) @@ -197,7 +197,7 @@ SUBROUTINE ReadRPList(FileString) SWRITE(UNIT_stdOut,'(A)',ADVANCE='NO')' Read recordpoint definitions from data file "'//TRIM(FileString)//'" ...' ! Open data file -CALL OpenDataFile(FileString,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_WORLD) +CALL OpenDataFile(FileString,create=.FALSE.,single=.FALSE.,readOnly=.FALSE.,communicatorOpt=MPI_COMM_PICLAS) ! compare mesh file names CALL ReadAttribute(File_ID,'MeshFile',1,StrScalar=MeshFile_RPList) diff --git a/src/restart/restart.f90 b/src/restart/restart.f90 index ee544f64b..b780721a6 100644 --- a/src/restart/restart.f90 +++ b/src/restart/restart.f90 @@ -149,7 +149,7 @@ SUBROUTINE InitRestart() IF (LEN_TRIM(RestartFile).GT.0) THEN ! Read in the state file we want to restart from DoRestart = .TRUE. - CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) #ifdef PP_POIS #if (PP_nVar==8) !The following arrays are read from the file diff --git a/src/restart/restart_field.f90 b/src/restart/restart_field.f90 index c46f53310..e5c724ca8 100644 --- a/src/restart/restart_field.f90 +++ b/src/restart/restart_field.f90 @@ -184,9 +184,9 @@ SUBROUTINE FieldRestart() !IPWRITE(UNIT_StdOut,*) "firstSide:lastSide,Nbr,nSides =", firstSide,lastSide,lastSide-firstSide+1,nSides ALLOCATE(lambdaLBTmp(PP_nVar,nGP_face,firstSide:lastSide)) END ASSOCIATE - !CALL MPI_BARRIER(MPI_COMM_WORLD,iError) + !CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) !IPWRITE(UNIT_StdOut,*) "MPInSideSend,MPIoffsetSideSend,MPInSideRecv,MPIoffsetSideRecv =", MPInSideSend,MPIoffsetSideSend,MPInSideRecv,MPIoffsetSideRecv - !CALL MPI_BARRIER(MPI_COMM_WORLD,iError) + !CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) ASSOCIATE (& counts_send => (INT(MPInSideSend )) ,& @@ -199,7 +199,7 @@ SUBROUTINE FieldRestart() MPI_TYPE = MPI_DOUBLE_PRECISION CALL MPI_TYPE_CREATE_STRUCT(1,MPI_LENGTH,MPI_DISPLACEMENT,MPI_TYPE,MPI_STRUCT,iError) CALL MPI_TYPE_COMMIT(MPI_STRUCT,iError) - CALL MPI_ALLTOALLV(lambdaLB,counts_send,disp_send,MPI_STRUCT,lambdaLBTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_WORLD,iError) + CALL MPI_ALLTOALLV(lambdaLB,counts_send,disp_send,MPI_STRUCT,lambdaLBTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_PICLAS,iError) END ASSOCIATE DEALLOCATE(lambdaLB) @@ -287,7 +287,7 @@ SUBROUTINE FieldRestart() MPI_TYPE = MPI_DOUBLE_PRECISION CALL MPI_TYPE_CREATE_STRUCT(1,MPI_LENGTH,MPI_DISPLACEMENT,MPI_TYPE,MPI_STRUCT,iError) CALL MPI_TYPE_COMMIT(MPI_STRUCT,iError) - CALL MPI_ALLTOALLV(U,counts_send,disp_send,MPI_STRUCT,UTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_WORLD,iError) + CALL MPI_ALLTOALLV(U,counts_send,disp_send,MPI_STRUCT,UTmp,counts_recv,disp_recv,MPI_STRUCT,MPI_COMM_PICLAS,iError) END ASSOCIATE CALL MOVE_ALLOC(UTmp,U) END IF ! ALLOCATED(U) @@ -310,10 +310,10 @@ SUBROUTINE FieldRestart() ! =========================================================================== IF(RestartNullifySolution)THEN ! Open the restart file and neglect the DG solution (only read particles if present) SWRITE(UNIT_stdOut,*)'Restarting from File: ',TRIM(RestartFile),' (but without reading the DG solution)' - CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) ELSE ! Use the solution in the restart file SWRITE(UNIT_stdOut,*)'Restarting from File: ',TRIM(RestartFile) - CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) ! Read in time from restart file !CALL ReadAttribute(File_ID,'Time',1,RealScalar=RestartTime) ! Read in state diff --git a/src/timedisc/timedisc.f90 b/src/timedisc/timedisc.f90 index 734a97a2e..de308d2d9 100644 --- a/src/timedisc/timedisc.f90 +++ b/src/timedisc/timedisc.f90 @@ -233,7 +233,7 @@ SUBROUTINE TimeDisc() tTracking=0 tLocalization=0 END IF -IF(CalcEMFieldOutput) CALL WriteElectroMagneticPICFieldToHDF5() ! Write magnetic field to file +IF(CalcEMFieldOutput) CALL WriteElectroMagneticPICFieldToHDF5() ! Write magnetic field to file #endif /*PARTICLES*/ ! No computation needed if tEnd=tStart! @@ -539,7 +539,7 @@ SUBROUTINE WriteInfoStdOut() #if USE_MPI NbrOfLostParticlesTotal_old_tmp = NbrOfLostParticlesTotal ! keep old value ! Allreduce is required because of the particle output to .h5 in which all processors must take place - CALL MPI_ALLREDUCE(NbrOfLostParticles , NbrOfLostParticlesTotal , 1 , MPI_INTEGER , MPI_SUM , MPI_COMM_WORLD , IERROR) + CALL MPI_ALLREDUCE(NbrOfLostParticles , NbrOfLostParticlesTotal , 1 , MPI_INTEGER , MPI_SUM , MPI_COMM_PICLAS , IERROR) NbrOfLostParticlesTotal = NbrOfLostParticlesTotal + NbrOfLostParticlesTotal_old_tmp ! add old value #else NbrOfLostParticlesTotal = NbrOfLostParticlesTotal + NbrOfLostParticles diff --git a/src/timedisc/timedisc_TimeStepByImplicitRK.f90 b/src/timedisc/timedisc_TimeStepByImplicitRK.f90 index 66e7db5a2..c94101fa2 100644 --- a/src/timedisc/timedisc_TimeStepByImplicitRK.f90 +++ b/src/timedisc/timedisc_TimeStepByImplicitRK.f90 @@ -93,7 +93,6 @@ SUBROUTINE TimeStepByImplicitRK() #if USE_MPI USE MOD_Particle_MPI ,ONLY: IRecvNbOfParticles, MPIParticleSend,MPIParticleRecv,SendNbOfparticles USE MOD_Particle_MPI_Vars ,ONLY: PartMPIExchange -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI #endif /*USE_MPI*/ USE MOD_PIC_Analyze ,ONLY: CalcDepositedCharge USE MOD_part_tools ,ONLY: UpdateNextFreePosition @@ -307,10 +306,10 @@ SUBROUTINE TimeStepByImplicitRK() #endif /*USE_LOADBALANCE*/ IF(DoPrintConvInfo)THEN #if USE_MPI - IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,nParts,1,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM, IERROR) + IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE,nParts,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) ELSE - CALL MPI_REDUCE(nParts ,iPart,1,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM, IERROR) + CALL MPI_REDUCE(nParts ,iPart,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) END IF #endif /*USE_MPI*/ SWRITE(UNIT_StdOut,'(A,I10)') ' SurfaceFlux-Particles: ',nParts diff --git a/src/timedisc/timedisc_TimeStepByRosenbrock.f90 b/src/timedisc/timedisc_TimeStepByRosenbrock.f90 index 0507f14c3..365202bf1 100644 --- a/src/timedisc/timedisc_TimeStepByRosenbrock.f90 +++ b/src/timedisc/timedisc_TimeStepByRosenbrock.f90 @@ -80,7 +80,6 @@ SUBROUTINE TimeStepByRosenbrock() #if USE_MPI USE MOD_Particle_MPI ,ONLY: IRecvNbOfParticles, MPIParticleSend,MPIParticleRecv,SendNbOfparticles USE MOD_Particle_MPI_Vars ,ONLY: PartMPIExchange -USE MOD_Particle_MPI_Vars ,ONLY: PartMPI #endif /*USE_MPI*/ USE MOD_PIC_Analyze ,ONLY: CalcDepositedCharge USE MOD_part_tools ,ONLY: UpdateNextFreePosition @@ -232,10 +231,10 @@ SUBROUTINE TimeStepByRosenbrock() #endif /*USE_LOADBALANCE*/ IF(DoPrintConvInfo)THEN #if USE_MPI - IF(PartMPI%MPIRoot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,nParts,1,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM, IERROR) + IF(MPIRoot)THEN + CALL MPI_REDUCE(MPI_IN_PLACE,nParts,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) ELSE - CALL MPI_REDUCE(nParts ,iPart,1,MPI_INTEGER,MPI_SUM,0,PartMPI%COMM, IERROR) + CALL MPI_REDUCE(nParts ,iPart,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS, IERROR) END IF #endif /*USE_MPI*/ SWRITE(UNIT_StdOut,'(A,I10)') ' SurfaceFlux-Particles: ',nParts From 4c5063b75bcb28947564d748fb667dd276cc3e65 Mon Sep 17 00:00:00 2001 From: Felix Garmirian Date: Tue, 5 Sep 2023 17:11:35 +0200 Subject: [PATCH 09/16] forgot one variable --- src/particles/analyze/particle_analyze_code.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/particles/analyze/particle_analyze_code.f90 b/src/particles/analyze/particle_analyze_code.f90 index 0a68ed0fd..761abe2e4 100644 --- a/src/particles/analyze/particle_analyze_code.f90 +++ b/src/particles/analyze/particle_analyze_code.f90 @@ -568,7 +568,7 @@ END SUBROUTINE CalcErrorParticle SUBROUTINE AnalyticParticleMovement(time,iter) ! MODULES USE MOD_Preproc -USE MOD_Globals ,ONLY: UNIT_StdOut +USE MOD_Globals ,ONLY: UNIT_StdOut,MPIRoot USE MOD_Analyze_Vars ,ONLY: OutputErrorNorms USE MOD_Particle_Analyze_Vars ,ONLY: TrackParticlePosition USE MOD_PICInterpolation_Vars ,ONLY: L_2_Error_Part,AnalyticPartDim From dd004978c9da51b8f7b7d974bc304a0672244446 Mon Sep 17 00:00:00 2001 From: Paul Nizenkov Date: Tue, 5 Sep 2023 23:41:55 +0200 Subject: [PATCH 10/16] Updated developer guide with respect to the new MPI_COMM_PICLAS communicator --- docs/documentation/developerguide/mpi.md | 33 ++++++++++++------------ 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/docs/documentation/developerguide/mpi.md b/docs/documentation/developerguide/mpi.md index ba3ba0a48..40044a8f9 100644 --- a/docs/documentation/developerguide/mpi.md +++ b/docs/documentation/developerguide/mpi.md @@ -135,9 +135,9 @@ color = MERGE(1337, MPI_UNDEFINED, nSurfSidesProc.GT.0) ``` Here, every processor with the same `color` will be part of the same communicator. The condition `nSurfSidesProc.GT.0` in this case includes every processor with a surface side. Every other processor will be set to `MPI_UNDEFINED` and consequently be part of `MPI_COMM_NULL`. Now, the communicator itself can be created: ``` -CALL MPI_COMM_SPLIT(MPI_COMM_WORLD, color, MPI_INFO_NULL, SurfCOMM%UNICATOR, iError) +CALL MPI_COMM_SPLIT(MPI_COMM_PICLAS, color, MPI_INFO_NULL, SurfCOMM%UNICATOR, iError) ``` -`MPI_COMM_WORLD` denotes the original communicator containing every processor (but can also be a previously created subset) and the `MPI_INFO_NULL` entry denotes the rank assignment within the new communicator (default: numbering from 0 to nProcs - 1). Additional information can be stored within the created variable: +`MPI_COMM_PICLAS` denotes the global PICLas communicator containing every processor (but can also be a previously created subset) and the `MPI_INFO_NULL` entry denotes the rank assignment within the new communicator (default: numbering from 0 to nProcs - 1). Additional information can be stored within the created variable: ``` IF(SurfCOMM%UNICATOR.NE.MPI_COMM_NULL) THEN ! Stores the rank within the given communicator as MyRank @@ -159,21 +159,22 @@ that either a communicator exists and/or every (other) processor has been set to | Handle | Description | Derived from | | ----------------------- | --------------------------------------------- | ----------------------- | | MPI_COMM_WORLD | Default global communicator | - | -| MPI_COMM_NODE | Processors on a node | MPI_COMM_WORLD | -| MPI_COMM_LEADERS | Group of node leaders | MPI_COMM_WORLD | -| MPI_COMM_WORKERS | All remaining processors, who are not leaders | MPI_COMM_WORLD | -| MPI_COMM_SHARED | Processors on a node | MPI_COMM_WORLD | -| MPI_COMM_LEADERS_SHARED | Group of node leaders (myComputeNodeRank = 0) | MPI_COMM_WORLD | +| MPI_COMM_PICLAS | Duplicate of MPI_COMM_WORLD | MPI_COMM_PICLAS | +| MPI_COMM_NODE | Processors on a node | MPI_COMM_PICLAS | +| MPI_COMM_LEADERS | Group of node leaders | MPI_COMM_PICLAS | +| MPI_COMM_WORKERS | All remaining processors, who are not leaders | MPI_COMM_PICLAS | +| MPI_COMM_SHARED | Processors on a node | MPI_COMM_PICLAS | +| MPI_COMM_LEADERS_SHARED | Group of node leaders (myComputeNodeRank = 0) | MPI_COMM_PICLAS | | MPI_COMM_LEADERS_SURF | Node leaders with surface sides | MPI_COMM_LEADERS_SHARED | #### Feature-specific -| Handle | Description | Derived from | -| ------------------------------------ | ---------------------------------------------------------------------- | -------------- | -| PartMPIInitGroup(nInitRegions)%COMM | Emission groups | MPI_COMM_PICLAS| -| SurfCOMM%UNICATOR | Processors with a surface side (e.g. reflective), including halo sides | MPI_COMM_WORLD | -| CPPCOMM%UNICATOR | Coupled power potential | MPI_COMM_WORLD | -| EDC%COMM(iEDCBC)%UNICATOR | Electric displacement current (per BC) | MPI_COMM_WORLD | -| FPC%COMM(iUniqueFPCBC)%UNICATOR | Floating potential (per BC) | MPI_COMM_WORLD | -| EPC%COMM(iUniqueEPCBC)%UNICATOR | Electric potential (per BC) | MPI_COMM_WORLD | -| BiasVoltage%COMM%UNICATOR | Bias voltage | MPI_COMM_WORLD | \ No newline at end of file +| Handle | Description | Derived from | +| ----------------------------------- | ---------------------------------------------------------------------- | --------------- | +| PartMPIInitGroup(nInitRegions)%COMM | Emission groups | MPI_COMM_PICLAS | +| SurfCOMM%UNICATOR | Processors with a surface side (e.g. reflective), including halo sides | MPI_COMM_PICLAS | +| CPPCOMM%UNICATOR | Coupled power potential | MPI_COMM_PICLAS | +| EDC%COMM(iEDCBC)%UNICATOR | Electric displacement current (per BC) | MPI_COMM_PICLAS | +| FPC%COMM(iUniqueFPCBC)%UNICATOR | Floating potential (per BC) | MPI_COMM_PICLAS | +| EPC%COMM(iUniqueEPCBC)%UNICATOR | Electric potential (per BC) | MPI_COMM_PICLAS | +| BiasVoltage%COMM%UNICATOR | Bias voltage | MPI_COMM_PICLAS | \ No newline at end of file From 1adf95794c527df44a579cca83fdc477c1890708 Mon Sep 17 00:00:00 2001 From: Felix Garmirian Date: Wed, 6 Sep 2023 11:53:44 +0200 Subject: [PATCH 11/16] (almost) no more MPI_COMM_WORLD --- src/globals/globals.f90 | 2 +- src/loadbalance/loadbalance_tools.f90 | 16 +++++----- src/loadbalance/loaddistribution.f90 | 14 ++++----- src/mesh/mesh.f90 | 12 ++++---- src/mesh/mesh_readin.f90 | 6 ++-- src/mesh/mesh_tools.f90 | 4 +-- src/mesh/metrics.f90 | 4 +-- src/mesh/prepare_mesh.f90 | 30 +++++++++---------- src/mpi/mpi.f90 | 4 --- src/mpi/mpi_shared.f90 | 4 +++ .../particle_mesh/particle_mesh_tools.f90 | 18 +++++------ 11 files changed, 57 insertions(+), 57 deletions(-) diff --git a/src/globals/globals.f90 b/src/globals/globals.f90 index 4a62619c8..474ae952f 100644 --- a/src/globals/globals.f90 +++ b/src/globals/globals.f90 @@ -805,7 +805,7 @@ FUNCTION PICLASTIME() IF(PRESENT(Comm))THEN CALL MPI_BARRIER(Comm,iError) ELSE - CALL MPI_BARRIER(MPI_COMM_WORLD,iError) + CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) END IF PiclasTime=MPI_WTIME() #else diff --git a/src/loadbalance/loadbalance_tools.f90 b/src/loadbalance/loadbalance_tools.f90 index 1efe20742..066ba9805 100644 --- a/src/loadbalance/loadbalance_tools.f90 +++ b/src/loadbalance/loadbalance_tools.f90 @@ -146,7 +146,7 @@ SUBROUTINE DomainDecomposition() END IF ! 2) Distribute logical information ElemTimeExists - CALL MPI_BCAST(ElemTimeExists,1,MPI_LOGICAL,0,MPI_COMM_WORLD,iError) + CALL MPI_BCAST(ElemTimeExists,1,MPI_LOGICAL,0,MPI_COMM_PICLAS,iError) ! Distribute the elements according to the selected distribution method CALL ApplyWeightDistributionMethod(ElemTimeExists) @@ -155,7 +155,7 @@ SUBROUTINE DomainDecomposition() CALL WeightDistribution_Equal(nProcessors,nGlobalElems,offsetElemMPI) ! Send the load distribution to all other procs - CALL MPI_BCAST(offsetElemMPI,nProcessors+1,MPI_INTEGER,0,MPI_COMM_WORLD,iERROR) + CALL MPI_BCAST(offsetElemMPI,nProcessors+1,MPI_INTEGER,0,MPI_COMM_PICLAS,iERROR) END IF ! IF(DoRestart.OR.PerformLoadBalance) @@ -325,7 +325,7 @@ SUBROUTINE ReadElemTime(single) DO iProc = 0,nProcessors-1 ElemPerProc(iProc) = offsetElemMPIOld(iProc+1) - offsetElemMPIOld(iProc) END DO - CALL MPI_GATHERV(ElemTime,nElems,MPI_DOUBLE_PRECISION,ElemGlobalTime,ElemPerProc,offsetElemMPIOld(0:nProcessors-1),MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,iError) + CALL MPI_GATHERV(ElemTime,nElems,MPI_DOUBLE_PRECISION,ElemGlobalTime,ElemPerProc,offsetElemMPIOld(0:nProcessors-1),MPI_DOUBLE_PRECISION,0,MPI_COMM_PICLAS,iError) ELSE ALLOCATE(ElemTimeTmp(1:nElems)) @@ -335,7 +335,7 @@ SUBROUTINE ReadElemTime(single) counts_recv => INT(MPInElemRecv ) ,& disp_recv => INT(MPIoffsetElemRecv)) ! Communicate PartInt over MPI - CALL MPI_ALLTOALLV(ElemTime,counts_send,disp_send,MPI_DOUBLE_PRECISION,ElemTimeTmp,counts_recv,disp_recv,MPI_DOUBLE_PRECISION,MPI_COMM_WORLD,iError) + CALL MPI_ALLTOALLV(ElemTime,counts_send,disp_send,MPI_DOUBLE_PRECISION,ElemTimeTmp,counts_recv,disp_recv,MPI_DOUBLE_PRECISION,MPI_COMM_PICLAS,iError) END ASSOCIATE DEALLOCATE(ElemTime) @@ -367,7 +367,7 @@ SUBROUTINE ReadElemTime(single) ! When this happens, the root process and its processors that are on the same node always return ElemTimeExists=T ! This points to a corrupt state file (accompanied by SpecID=0 particles within the file) ! If the load balance step is performed without h5 I/O in the future, this check can be removed - CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL DatasetExists(File_ID,'ElemTime',ElemTimeExists) IF(.NOT.ElemTimeExists) CALL abort(__STAMP__,'ElemTime does not exit for some processors in .h5 which indicates a corrupt state file') CALL CloseDataFile() @@ -392,8 +392,8 @@ SUBROUTINE ReadElemTime(single) END IF ! MPIRoot ! Send from root to all other processes - CALL MPI_SCATTERV(ElemGlobalTime, ElemProc, offsetElemMPI, MPI_DOUBLE_PRECISION, ElemTime_tmp, nElems, MPI_DOUBLE_PRECISION, 0, MPI_COMM_WORLD, IERROR) - + CALL MPI_SCATTERV(ElemGlobalTime, ElemProc, offsetElemMPI, MPI_DOUBLE_PRECISION, ElemTime_tmp, nElems, MPI_DOUBLE_PRECISION, 0, MPI_COMM_PICLAS, IERROR) + ! Deallocate temporary array IF(MPIRoot) DEALLOCATE(ElemProc) END IF ! FlushInitialState @@ -403,7 +403,7 @@ SUBROUTINE ReadElemTime(single) SDEALLOCATE(ElemTime_tmp) ALLOCATE(ElemTime_tmp(1:nElems)) ElemTime_tmp = 0. - CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(RestartFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL ReadArray('ElemTime',2,(/1_IK,INT(nElems,IK)/),INT(offsetElem,IK),2,RealArray=ElemTime_tmp) CALL CloseDataFile() diff --git a/src/loadbalance/loaddistribution.f90 b/src/loadbalance/loaddistribution.f90 index 1004d2831..c2acdd359 100644 --- a/src/loadbalance/loaddistribution.f90 +++ b/src/loadbalance/loaddistribution.f90 @@ -91,7 +91,7 @@ SUBROUTINE SingleStepOptimalPartition(nProcs,OldElems,NewElems,ElemTime) LoadSend = PreSum(OldElems) -CALL MPI_EXSCAN(LoadSend,LowerBoundary,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,iERROR) +CALL MPI_EXSCAN(LoadSend,LowerBoundary,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_PICLAS,iERROR) IF(MPIRoot) LowerBoundary = 0. UpperBoundary = LowerBoundary + PreSum(OldElems) @@ -143,7 +143,7 @@ SUBROUTINE SingleStepOptimalPartition(nProcs,OldElems,NewElems,ElemTime) END IF END DO ! iRank=minRank,maxRank -CALL MPI_ALLTOALL(send_count,1,MPI_INTEGER,recv_count,1, MPI_INTEGER,MPI_COMM_WORLD,iERROR) +CALL MPI_ALLTOALL(send_count,1,MPI_INTEGER,recv_count,1, MPI_INTEGER,MPI_COMM_PICLAS,iERROR) NewElems = SUM(recv_count) DEALLOCATE(PreSum,send_count,recv_count,split) @@ -213,7 +213,7 @@ SUBROUTINE ApplyWeightDistributionMethod(ElemTimeExists) END DO ! Sanity check IF(.NOT.ALLOCATED(PartInt)) CALL abort(__STAMP__,'PartInt is not allocated') ! Missing call to FillParticleData() - CALL MPI_GATHERV(PartInt,nElemsOld,MPI_DOUBLE_PRECISION,PartIntGlob,ElemPerProc,offsetElemMPIOld(0:nProcessors-1),MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,iError) + CALL MPI_GATHERV(PartInt,nElemsOld,MPI_DOUBLE_PRECISION,PartIntGlob,ElemPerProc,offsetElemMPIOld(0:nProcessors-1),MPI_DOUBLE_PRECISION,0,MPI_COMM_PICLAS,iError) PartIntExists = .TRUE. ELSE ! Readin of PartInt: Read in only by MPIRoot in single mode because the root performs the distribution of elements (domain decomposition) @@ -285,7 +285,7 @@ SUBROUTINE ApplyWeightDistributionMethod(ElemTimeExists) #endif /*PARTICLES*/ ! Distribute PartsInElem to all procs (Every proc needs to get the information to arrive at the same timedisc) -CALL MPI_BCAST(PartsInElem,nGlobalElems,MPI_INTEGER,0,MPI_COMM_WORLD,iError) +CALL MPI_BCAST(PartsInElem,nGlobalElems,MPI_INTEGER,0,MPI_COMM_PICLAS,iError) ! Every proc needs to get the information to arrive at the same timedisc ! No historical data and no particles in restart file @@ -332,7 +332,7 @@ SUBROUTINE ApplyWeightDistributionMethod(ElemTimeExists) END SELECT ! WeightDistributionMethod_loc ! Send the load distribution to all other procs -CALL MPI_BCAST(offsetElemMPI,nProcs+1,MPI_INTEGER,0,MPI_COMM_WORLD,iERROR) +CALL MPI_BCAST(offsetElemMPI,nProcs+1,MPI_INTEGER,0,MPI_COMM_PICLAS,iERROR) #ifdef PARTICLES ! Set PartDistri for every processor @@ -600,7 +600,7 @@ SUBROUTINE WeightDistribution_SingleStepOptimal(nProcs,nGlobalElems,ElemGlobalTi REAL :: LoadDiff( 0:nProcs-1) !=================================================================================================================================== ! Distribute ElemGlobalTime to all procs -CALL MPI_BCAST(ElemGlobalTime,nGlobalElems,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,iError) +CALL MPI_BCAST(ElemGlobalTime,nGlobalElems,MPI_DOUBLE_PRECISION,0,MPI_COMM_PICLAS,iError) ! Do Rebalance WeightSum = 0. @@ -698,7 +698,7 @@ SUBROUTINE WeightDistribution_SingleStepOptimal(nProcs,nGlobalElems,ElemGlobalTi ErrorCode = 0 IF(NewElems.LE.0) ErrorCode = ErrorCode+100 - CALL MPI_ALLGATHER(NewElems,1,MPI_INTEGER,ElemDistri(:),1,MPI_INTEGER,MPI_COMM_WORLD,iERROR) + CALL MPI_ALLGATHER(NewElems,1,MPI_INTEGER,ElemDistri(:),1,MPI_INTEGER,MPI_COMM_PICLAS,iERROR) ! calculate proc offset offsetElemMPI(0) = 0 diff --git a/src/mesh/mesh.f90 b/src/mesh/mesh.f90 index a6ea69782..6aeffd41d 100644 --- a/src/mesh/mesh.f90 +++ b/src/mesh/mesh.f90 @@ -199,7 +199,7 @@ SUBROUTINE InitMesh(meshMode,MeshFile_IN) END IF IF (.NOT.(PerformLoadBalance.AND.(.NOT.UseH5IOLoadBalance))) THEN #endif /*USE_LOADBALANCE*/ - CALL OpenDataFile(MeshFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(MeshFile,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL ReadAttribute(File_ID,'Ngeo',1,IntScalar=NGeo) CALL PrintOption('NGeo','INFO',IntOpt=NGeo) CALL CloseDataFile() @@ -719,9 +719,9 @@ SUBROUTINE GetMeshMinMaxBoundaries() xmax_loc(3) = xyzMinMax(6) ! Find global min - CALL MPI_ALLREDUCE(xmin_loc(1:3),xmin(1:3), 3, MPI_DOUBLE_PRECISION, MPI_MIN, MPI_COMM_WORLD, IERROR) + CALL MPI_ALLREDUCE(xmin_loc(1:3),xmin(1:3), 3, MPI_DOUBLE_PRECISION, MPI_MIN, MPI_COMM_PICLAS, IERROR) ! Find global max - CALL MPI_ALLREDUCE(xmax_loc(1:3),xmax(1:3), 3, MPI_DOUBLE_PRECISION, MPI_MAX, MPI_COMM_WORLD, IERROR) + CALL MPI_ALLREDUCE(xmax_loc(1:3),xmax(1:3), 3, MPI_DOUBLE_PRECISION, MPI_MAX, MPI_COMM_PICLAS, IERROR) ! Map global min/max values to xyzMinMax(1:6) xyzMinMax(1) = xmin(1) @@ -891,7 +891,7 @@ SUBROUTINE InitElemVolumes() CALL MPI_BCAST(MeshVolume,1, MPI_DOUBLE_PRECISION,0,MPI_COMM_SHARED,iERROR) #else ! In this case, no shared array is created and all arrays are processor-local -CALL MPI_ALLREDUCE(LocalVolume,MeshVolume,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_WORLD,IERROR) +CALL MPI_ALLREDUCE(LocalVolume,MeshVolume,1,MPI_DOUBLE_PRECISION,MPI_SUM,MPI_COMM_PICLAS,IERROR) #endif /*PARTICLES*/ #else MeshVolume = LocalVolume @@ -920,7 +920,7 @@ SUBROUTINE setSideRanges() ! !----------------------------------------------------------------------------------------------------------------------------------! ! MODULES ! -USE MOD_Globals ,ONLY: abort +USE MOD_Globals ,ONLY: abort,MPI_COMM_PICLAS USE MOD_Mesh_Vars ,ONLY: firstBCSide,firstMortarInnerSide,firstInnerSide,firstMPISide_MINE,firstMPISide_YOUR USE MOD_Mesh_Vars ,ONLY: nMPISides_MINE,nMPISides_YOUR,nInnerSides,nMortarInnerSides,nBCSides USE MOD_Mesh_Vars ,ONLY: lastBCSide,lastMortarInnerSide,lastInnerSide,lastMPISide_MINE,lastMPISide_YOUR,lastMortarMPISide @@ -973,7 +973,7 @@ SUBROUTINE setSideRanges() #if USE_HDG nUniqueSides = lastMPISide_MINE + nMortarMPISides !big mortars are at the end of the side list! #if USE_MPI -CALL MPI_ALLREDUCE(nUniqueSides,nGlobalUniqueSides,1,MPI_INTEGER,MPI_SUM,MPI_COMM_WORLD,iError) +CALL MPI_ALLREDUCE(nUniqueSides,nGlobalUniqueSides,1,MPI_INTEGER,MPI_SUM,MPI_COMM_PICLAS,iError) #else nGlobalUniqueSides=nSides #endif /*USE_MPI*/ diff --git a/src/mesh/mesh_readin.f90 b/src/mesh/mesh_readin.f90 index 4ff69cd50..70b7ce553 100644 --- a/src/mesh/mesh_readin.f90 +++ b/src/mesh/mesh_readin.f90 @@ -383,7 +383,7 @@ SUBROUTINE ReadMesh(FileString,ReadNodes) #if defined(PARTICLES) && USE_LOADBALANCE IF (.NOT.PerformLoadBalance) THEN #endif /*defined(PARTICLES) && USE_LOADBALANCE*/ - CALL OpenDataFile(FileString,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(FileString,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) CALL ReadBCs() #if defined(PARTICLES) && USE_LOADBALANCE END IF @@ -633,7 +633,7 @@ SUBROUTINE ReadMesh(FileString,ReadNodes) #if defined(PARTICLES) && USE_LOADBALANCE IF (.NOT.PerformLoadBalance) & #endif /*defined(PARTICLES) && USE_LOADBALANCE*/ - CALL OpenDataFile(FileString,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_WORLD) + CALL OpenDataFile(FileString,create=.FALSE.,single=.FALSE.,readOnly=.TRUE.,communicatorOpt=MPI_COMM_PICLAS) ! Backup required if useCurveds=F NGeoOld = NGeo @@ -799,7 +799,7 @@ SUBROUTINE ReadMesh(FileString,ReadNodes) #endif #if USE_MPI -CALL MPI_ALLREDUCE(MPI_IN_PLACE,ReduceData,11,MPI_INTEGER,MPI_SUM,MPI_COMM_WORLD,iError) +CALL MPI_ALLREDUCE(MPI_IN_PLACE,ReduceData,11,MPI_INTEGER,MPI_SUM,MPI_COMM_PICLAS,iError) #endif /*USE_MPI*/ nGlobalMortarSides=ReduceData(9) diff --git a/src/mesh/mesh_tools.f90 b/src/mesh/mesh_tools.f90 index 5cfda3979..612c1dc9d 100644 --- a/src/mesh/mesh_tools.f90 +++ b/src/mesh/mesh_tools.f90 @@ -547,7 +547,7 @@ END SUBROUTINE LambdaSideToMaster SUBROUTINE BuildSideToNonUniqueGlobalSide() ! MODULES #if USE_DEBUG -USE MOD_Globals ,ONLY: myrank,UNIT_StdOut,MPI_COMM_WORLD +USE MOD_Globals ,ONLY: myrank,UNIT_StdOut,MPI_COMM_PICLAS #endif /*USE_DEBUG*/ USE MOD_Globals ,ONLY: iError USE MOD_Mesh_Vars ,ONLY: MortarType,ElemInfo,SideToNonUniqueGlobalSide,nSides,nElems,ElemToSide,offsetElem,MortarInfo @@ -625,7 +625,7 @@ SUBROUTINE BuildSideToNonUniqueGlobalSide() END DO END DO ! iElem #if USE_DEBUG -IF(myrank.eq.0.AND.checkRank.GT.-1) read*; CALL MPI_BARRIER(MPI_COMM_WORLD,iError) +IF(myrank.eq.0.AND.checkRank.GT.-1) read*; CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) #endif /*USE_DEBUG*/ END SUBROUTINE BuildSideToNonUniqueGlobalSide #endif /*USE_LOADBALANCE*/ diff --git a/src/mesh/metrics.f90 b/src/mesh/metrics.f90 index e9372a682..39ae37390 100644 --- a/src/mesh/metrics.f90 +++ b/src/mesh/metrics.f90 @@ -409,9 +409,9 @@ SUBROUTINE CalcMetrics(XCL_NGeo_Out,dXCL_NGeo_out) ! Communicate smallest ref. Jacobian and display #if USE_MPI IF(MPIroot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE , SmallestscaledJacRef , 1 , MPI_DOUBLE_PRECISION , MPI_MIN , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(MPI_IN_PLACE , SmallestscaledJacRef , 1 , MPI_DOUBLE_PRECISION , MPI_MIN , 0 , MPI_COMM_PICLAS , iError) ELSE - CALL MPI_REDUCE(SmallestscaledJacRef , 0 , 1 , MPI_DOUBLE_PRECISION , MPI_MIN , 0 , MPI_COMM_WORLD , iError) + CALL MPI_REDUCE(SmallestscaledJacRef , 0 , 1 , MPI_DOUBLE_PRECISION , MPI_MIN , 0 , MPI_COMM_PICLAS , iError) END IF #endif /*USE_MPI*/ LBWRITE (*,'(A,ES18.10E3,A,I0,A,ES13.5E3)') " Smallest scaled Jacobian in reference system: ",SmallestscaledJacRef,& diff --git a/src/mesh/prepare_mesh.f90 b/src/mesh/prepare_mesh.f90 index a02668ab6..7209f6129 100644 --- a/src/mesh/prepare_mesh.f90 +++ b/src/mesh/prepare_mesh.f90 @@ -623,7 +623,7 @@ SUBROUTINE setLocalSideIDs() ! CAUTION: MY-MORTAR-MPI-Sides are missing IF(ALLOCATED(offsetSideMPI))DEALLOCATE(offsetSideMPI) ALLOCATE(offsetSideMPI(nProcessors)) -CALL MPI_ALLGATHER(nSides-nMPISides_YOUR,1,MPI_INTEGER,offsetSideMPI,1,MPI_INTEGER,MPI_COMM_WORLD,IERROR) +CALL MPI_ALLGATHER(nSides-nMPISides_YOUR,1,MPI_INTEGER,offsetSideMPI,1,MPI_INTEGER,MPI_COMM_PICLAS,IERROR) offsetSide=0 ! set default for restart!!! DO iProc=1, myrank offsetSide = offsetSide + offsetSideMPI(iProc) @@ -659,8 +659,8 @@ SUBROUTINE setLocalSideIDs() ALLOCATE(nNBProcs_glob(1)) ! dummy for debug ALLOCATE(ProcInfo_glob(1,1)) ! dummy for debug END IF !MPIroot -CALL MPI_GATHER(nNBProcs,1,MPI_INTEGER,nNBProcs_glob,1,MPI_INTEGER,0,MPI_COMM_WORLD,iError) -CALL MPI_GATHER(ProcInfo,9,MPI_INTEGER,ProcInfo_glob,9,MPI_INTEGER,0,MPI_COMM_WORLD,iError) +CALL MPI_GATHER(nNBProcs,1,MPI_INTEGER,nNBProcs_glob,1,MPI_INTEGER,0,MPI_COMM_PICLAS,iError) +CALL MPI_GATHER(ProcInfo,9,MPI_INTEGER,ProcInfo_glob,9,MPI_INTEGER,0,MPI_COMM_PICLAS,iError) IF(MPIroot)THEN nNBmax=MAXVAL(nNBProcs_glob) ! count, total number of columns in table ALLOCATE(NBinfo_glob(6,nNBmax,0:nProcessors)) @@ -668,7 +668,7 @@ SUBROUTINE setLocalSideIDs() ELSE ALLOCATE(NBinfo_glob(1,1,1)) ! dummy for debug END IF -CALL MPI_BCAST(nNBmax,1,MPI_INTEGER,0,MPI_COMM_WORLD,iError) +CALL MPI_BCAST(nNBmax,1,MPI_INTEGER,0,MPI_COMM_PICLAS,iError) ALLOCATE(NBinfo(6,nNbmax)) NBinfo=0 NBinfo(1,1:nNBProcs)=NBProc @@ -677,7 +677,7 @@ SUBROUTINE setLocalSideIDs() NBinfo(4,1:nNBProcs)=nMPISides_YOUR_Proc NBinfo(5,1:nNBProcs)=offsetMPISides_MINE(0:nNBProcs-1) NBinfo(6,1:nNBProcs)=offsetMPISides_YOUR(0:nNBProcs-1) -CALL MPI_GATHER(NBinfo,6*nNBmax,MPI_INTEGER,NBinfo_glob,6*nNBmax,MPI_INTEGER,0,MPI_COMM_WORLD,iError) +CALL MPI_GATHER(NBinfo,6*nNBmax,MPI_INTEGER,NBinfo_glob,6*nNBmax,MPI_INTEGER,0,MPI_COMM_PICLAS,iError) DEALLOCATE(NBinfo) IF(MPIroot)THEN OPEN(NEWUNIT=ioUnit,FILE=filename,STATUS='REPLACE') @@ -932,11 +932,11 @@ SUBROUTINE fillMeshInfo() #if USE_MPI IF(MPIroot)THEN - CALL MPI_REDUCE(MPI_IN_PLACE,nSides_flip,5,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,iError) - CALL MPI_REDUCE(MPI_IN_PLACE ,nSides_MortarType,3,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,iError) + CALL MPI_REDUCE(MPI_IN_PLACE,nSides_flip,5,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,iError) + CALL MPI_REDUCE(MPI_IN_PLACE ,nSides_MortarType,3,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,iError) ELSE - CALL MPI_REDUCE(nSides_flip,dummy,5,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,iError) - CALL MPI_REDUCE(nSides_MortarType,nSides_MortarType,3,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,iError) + CALL MPI_REDUCE(nSides_flip,dummy,5,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,iError) + CALL MPI_REDUCE(nSides_MortarType,nSides_MortarType,3,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,iError) END IF #endif /*USE_MPI*/ LBWRITE(UNIT_StdOut,'(132("."))') @@ -1192,7 +1192,7 @@ SUBROUTINE exchangeFlip() SideID_start=OffsetMPISides_MINE(iNbProc-1)+1 SideID_end =OffsetMPISides_MINE(iNbProc) CALL MPI_ISEND(Flip_MINE(SideID_start:SideID_end),nSendVal,MPI_INTEGER, & - nbProc(iNbProc),0,MPI_COMM_WORLD,SendRequest(iNbProc),iError) + nbProc(iNbProc),0,MPI_COMM_PICLAS,SendRequest(iNbProc),iError) END IF ! Start receive flip to YOUR IF(nMPISides_YOUR_Proc(iNbProc).GT.0)THEN @@ -1200,7 +1200,7 @@ SUBROUTINE exchangeFlip() SideID_start=OffsetMPISides_YOUR(iNbProc-1)+1 SideID_end =OffsetMPISides_YOUR(iNbProc) CALL MPI_IRECV(Flip_YOUR(SideID_start:SideID_end),nRecVal,MPI_INTEGER, & - nbProc(iNbProc),0,MPI_COMM_WORLD,RecRequest(iNbProc),iError) + nbProc(iNbProc),0,MPI_COMM_PICLAS,RecRequest(iNbProc),iError) END IF END DO !iProc=1,nNBProcs DO iNbProc=1,nNbProcs @@ -1287,7 +1287,7 @@ SUBROUTINE exchangeElemID() SideID_end =OffsetMPISides_send(iNbProc,2) IF(nSendVal.GT.0)THEN CALL MPI_ISEND(ElemID_MINE(SideID_start:SideID_end),nSendVal,MPI_INTEGER, & - nbProc(iNbProc),0,MPI_COMM_WORLD,SendRequest(iNbProc),iError) + nbProc(iNbProc),0,MPI_COMM_PICLAS,SendRequest(iNbProc),iError) END IF ! Start receive flip to YOUR nRecVal =nMPISides_rec(iNbProc,2) @@ -1295,7 +1295,7 @@ SUBROUTINE exchangeElemID() SideID_end =OffsetMPISides_rec(iNbProc,2) IF(nRecVal.GT.0)THEN CALL MPI_IRECV(ElemID_YOUR(SideID_start:SideID_end),nRecVal,MPI_INTEGER, & - nbProc(iNbProc),0,MPI_COMM_WORLD,RecRequest(iNbProc),iError) + nbProc(iNbProc),0,MPI_COMM_PICLAS,RecRequest(iNbProc),iError) END IF END DO !iProc=1,nNBProcs DO iNbProc=1,nNbProcs @@ -1315,7 +1315,7 @@ SUBROUTINE exchangeElemID() SideID_end =OffsetMPISides_send(iNbProc,1) IF(nSendVal.GT.0)THEN CALL MPI_ISEND(ElemID_MINE(SideID_start:SideID_end),nSendVal,MPI_INTEGER, & - nbProc(iNbProc),0,MPI_COMM_WORLD,SendRequest(iNbProc),iError) + nbProc(iNbProc),0,MPI_COMM_PICLAS,SendRequest(iNbProc),iError) END IF ! Start receive flip to YOUR nRecVal =nMPISides_rec(iNbProc,1) @@ -1323,7 +1323,7 @@ SUBROUTINE exchangeElemID() SideID_end =OffsetMPISides_rec(iNbProc,1) IF(nRecVal.GT.0)THEN CALL MPI_IRECV(ElemID_YOUR(SideID_start:SideID_end),nRecVal,MPI_INTEGER, & - nbProc(iNbProc),0,MPI_COMM_WORLD,RecRequest(iNbProc),iError) + nbProc(iNbProc),0,MPI_COMM_PICLAS,RecRequest(iNbProc),iError) END IF END DO !iProc=1,nNBProcs DO iNbProc=1,nNbProcs diff --git a/src/mpi/mpi.f90 b/src/mpi/mpi.f90 index 1266098d3..bb2eef7d3 100644 --- a/src/mpi/mpi.f90 +++ b/src/mpi/mpi.f90 @@ -194,9 +194,6 @@ SUBROUTINE InitMPIvars() RecRequest_Geo = MPI_REQUEST_NULL DataSizeSide =(PP_N+1)*(PP_N+1) -! General communicator -CALL MPI_COMM_DUP (MPI_COMM_WORLD,MPI_COMM_PICLAS,iError) - ! split communicator into smaller groups (e.g. for local nodes) GroupSize=GETINT('GroupSize','0') IF(GroupSize.LT.1)THEN ! group procs by node @@ -509,7 +506,6 @@ SUBROUTINE FinalizeMPI() ! Free the communicators IF(MPI_COMM_NODE .NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(MPI_COMM_NODE ,IERROR) IF(MPI_COMM_LEADERS.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(MPI_COMM_LEADERS,IERROR) -IF(MPI_COMM_PICLAS .NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(MPI_COMM_PICLAS ,IERROR) #if USE_LOADBALANCE IF (.NOT.(PerformLoadBalance.AND.(.NOT.UseH5IOLoadBalance))) THEN diff --git a/src/mpi/mpi_shared.f90 b/src/mpi/mpi_shared.f90 index 34b142f32..d818da02e 100644 --- a/src/mpi/mpi_shared.f90 +++ b/src/mpi/mpi_shared.f90 @@ -129,6 +129,9 @@ SUBROUTINE InitMPIShared() SWRITE(UNIT_StdOut,'(132("-"))') SWRITE(UNIT_stdOut,'(A)') ' INIT MPI SHARED COMMUNICATION ...' +! General communicator +CALL MPI_COMM_DUP (MPI_COMM_WORLD,MPI_COMM_PICLAS,iError) + ! Save the global number of procs nProcessors_Global = nProcessors @@ -1138,6 +1141,7 @@ SUBROUTINE FinalizeMPIShared() ! Free the shared communicator IF(MPI_COMM_SHARED .NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(MPI_COMM_SHARED ,IERROR) IF(MPI_COMM_LEADERS_SHARED.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(MPI_COMM_LEADERS_SHARED,IERROR) +IF(MPI_COMM_PICLAS .NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(MPI_COMM_PICLAS ,IERROR) MPISharedInitIsDone=.FALSE. END SUBROUTINE FinalizeMPIShared diff --git a/src/particles/particle_mesh/particle_mesh_tools.f90 b/src/particles/particle_mesh/particle_mesh_tools.f90 index 808e2db3e..c22a726c9 100644 --- a/src/particles/particle_mesh/particle_mesh_tools.f90 +++ b/src/particles/particle_mesh/particle_mesh_tools.f90 @@ -1050,13 +1050,13 @@ SUBROUTINE IdentifyElemAndSideType() END DO #if USE_MPI -CALL MPI_REDUCE(nPlanarRectangular ,nPlanarRectangularTot ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,IERROR) -CALL MPI_REDUCE(nPlanarNonRectangular,nPlanarNonRectangularTot,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,IERROR) -CALL MPI_REDUCE(nBilinear ,nBilinearTot ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,IERROR) -CALL MPI_REDUCE(nPlanarCurved ,nPlanarCurvedTot ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,IERROR) -CALL MPI_REDUCE(nCurved ,nCurvedTot ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,IERROR) -CALL MPI_REDUCE(nLinearElems ,nLinearElemsTot ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,IERROR) -CALL MPI_REDUCE(nCurvedElems ,nCurvedElemsTot ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,IERROR) +CALL MPI_REDUCE(nPlanarRectangular ,nPlanarRectangularTot ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) +CALL MPI_REDUCE(nPlanarNonRectangular,nPlanarNonRectangularTot,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) +CALL MPI_REDUCE(nBilinear ,nBilinearTot ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) +CALL MPI_REDUCE(nPlanarCurved ,nPlanarCurvedTot ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) +CALL MPI_REDUCE(nCurved ,nCurvedTot ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) +CALL MPI_REDUCE(nLinearElems ,nLinearElemsTot ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) +CALL MPI_REDUCE(nCurvedElems ,nCurvedElemsTot ,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_PICLAS,IERROR) #else nPlanarRectangularTot = nPlanarRectangular nPlanarNonRectangularTot = nPlanarNonRectangular @@ -1868,11 +1868,11 @@ SUBROUTINE ComputePeriodicVec() ! https://stackoverflow.com/questions/56307320/mpi-allreduce-not-synchronizing-properly !CALL MPI_ALLREDUCE(MPI_IN_PLACE,sendbuf,GEO%nPeriodicVectors,MPI_2DOUBLE_PRECISION,MPI_MINLOC,MPI_COMM_SHARED,iERROR) - CALL MPI_ALLGATHER(sendbuf,1,MPI_DOUBLE_PRECISION,recvbuf(:),1,MPI_DOUBLE_PRECISION,MPI_COMM_WORLD,iERROR) + CALL MPI_ALLGATHER(sendbuf,1,MPI_DOUBLE_PRECISION,recvbuf(:),1,MPI_DOUBLE_PRECISION,MPI_COMM_PICLAS,iERROR) IF (ALL(recvbuf(:).EQ.HUGE(1.))) CALL CollectiveStop(__STAMP__,'No periodic vector for BC_ALPHA found!',IntInfo=iVec) ! MINLOC does not follow array bounds, so root rank = 1 - CALL MPI_BCAST(GEO%PeriodicVectors(:,iVec),3,MPI_DOUBLE_PRECISION,MINLOC(recvbuf(:),1)-1,MPI_COMM_WORLD,iError) + CALL MPI_BCAST(GEO%PeriodicVectors(:,iVec),3,MPI_DOUBLE_PRECISION,MINLOC(recvbuf(:),1)-1,MPI_COMM_PICLAS,iError) END DO #endif /*USE_MPI*/ From 499f823c960d40684ba948031a3acb12a9ff7d4c Mon Sep 17 00:00:00 2001 From: Felix Garmirian Date: Wed, 6 Sep 2023 13:55:43 +0200 Subject: [PATCH 12/16] forgot one variable part 2 --- src/mesh/mesh.f90 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/mesh/mesh.f90 b/src/mesh/mesh.f90 index 6aeffd41d..ba982a68a 100644 --- a/src/mesh/mesh.f90 +++ b/src/mesh/mesh.f90 @@ -805,6 +805,8 @@ SUBROUTINE InitElemVolumes() USE MOD_Particle_Mesh_Vars ,ONLY: nComputeNodeElems,offsetComputeNodeElem USE MOD_MPI_Shared_Vars ,ONLY: MPI_COMM_SHARED,myComputeNodeRank,MPI_COMM_LEADERS_SHARED USE MOD_Particle_Mesh_Vars ,ONLY: ElemVolume_Shared_Win,ElemCharLength_Shared_Win +#else +USE MOD_Globals ,ONLY: MPI_COMM_PICLAS #endif /*PARTICLES*/ #endif /*USE_MPI*/ #if USE_LOADBALANCE @@ -920,7 +922,7 @@ SUBROUTINE setSideRanges() ! !----------------------------------------------------------------------------------------------------------------------------------! ! MODULES ! -USE MOD_Globals ,ONLY: abort,MPI_COMM_PICLAS +USE MOD_Globals ,ONLY: abort USE MOD_Mesh_Vars ,ONLY: firstBCSide,firstMortarInnerSide,firstInnerSide,firstMPISide_MINE,firstMPISide_YOUR USE MOD_Mesh_Vars ,ONLY: nMPISides_MINE,nMPISides_YOUR,nInnerSides,nMortarInnerSides,nBCSides USE MOD_Mesh_Vars ,ONLY: lastBCSide,lastMortarInnerSide,lastInnerSide,lastMPISide_MINE,lastMPISide_YOUR,lastMortarMPISide @@ -929,8 +931,7 @@ SUBROUTINE setSideRanges() USE MOD_Globals ,ONLY: UNIT_StdOut USE MOD_Mesh_Vars ,ONLY: nGlobalUniqueSidesFromMesh,nGlobalUniqueSides,nMortarMPISides,nUniqueSides #if USE_MPI -USE MOD_Globals ,ONLY: myrank -USE MOD_Globals ,ONLY: iError +USE MOD_Globals ,ONLY: myrank,MPI_COMM_PICLAS,iError USE mpi #endif /*USE_MPI*/ #endif /*USE_HDG*/ From eb3d47dc61e96507d50c0fbabf9259a111a35ec8 Mon Sep 17 00:00:00 2001 From: Paul Nizenkov Date: Tue, 17 Oct 2023 16:56:12 +0200 Subject: [PATCH 13/16] Moved the duplication of MPI_COMM_PICLAS to InitMPI and OutputMPIW8Time into FinalizePiclas, where the last remaining communicator is freed aftewards --- src/globals/globals.f90 | 14 +++++++------- src/init/piclas_init.f90 | 9 +++++++++ src/mpi/mpi.f90 | 13 +++++-------- src/mpi/mpi_shared.f90 | 4 ---- src/mpi/mpi_vars.f90 | 1 - src/piclas.f90 | 5 ----- 6 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/globals/globals.f90 b/src/globals/globals.f90 index 474ae952f..7a3e901a0 100644 --- a/src/globals/globals.f90 +++ b/src/globals/globals.f90 @@ -36,16 +36,16 @@ MODULE MOD_Globals INTEGER :: myRank,myLocalRank,myLeaderRank,myWorkerRank INTEGER :: nProcessors,nLocalProcs,nLeaderProcs,nWorkerProcs LOGICAL :: GlobalNbrOfParticlesUpdated ! When FALSE, then global number of particles needs to be determined -INTEGER :: MPI_COMM_NODE ! local node subgroup -INTEGER :: MPI_COMM_LEADERS ! all node masters -INTEGER :: MPI_COMM_WORKERS ! all non-master nodes -INTEGER :: MPI_COMM_PICLAS ! all nodes LOGICAL :: MPIRoot,MPILocalRoot #if USE_MPI !#include "mpif.h" INTEGER :: MPIStatus(MPI_STATUS_SIZE) +INTEGER :: MPI_COMM_NODE ! local node subgroup +INTEGER :: MPI_COMM_LEADERS ! all node masters +INTEGER :: MPI_COMM_WORKERS ! all non-master nodes +INTEGER :: MPI_COMM_PICLAS ! all nodes #else -INTEGER,PARAMETER :: MPI_COMM_WORLD=-1 ! DUMMY when compiling single (MPI=OFF) +INTEGER,PARAMETER :: MPI_COMM_PICLAS=-1 ! DUMMY when compiling single (MPI=OFF) #endif LOGICAL :: MemoryMonitor !> Flag for turning RAM monitoring ON/OFF. Used for the detection of RAM overflows (e.g. due to memory leaks) @@ -388,7 +388,7 @@ SUBROUTINE AbortProg(SourceFile,SourceLine,CompDate,CompTime,ErrorMessage,IntInf #if USE_MPI signalout=2 ! MPI_ABORT requires an output error-code /=0 errOut = 1 -CALL MPI_ABORT(MPI_COMM_WORLD,signalout,errOut) +CALL MPI_ABORT(MPI_COMM_PICLAS,signalout,errOut) #endif STOP 2 END SUBROUTINE AbortProg @@ -461,7 +461,7 @@ SUBROUTINE CollectiveStop(SourceFile,SourceLine,CompDate,CompTime,ErrorMessage,I CALL FLUSH(UNIT_stdOut) #if USE_MPI -CALL MPI_BARRIER(MPI_COMM_WORLD,iError) +CALL MPI_BARRIER(MPI_COMM_PICLAS,iError) CALL MPI_FINALIZE(iError) #endif ERROR STOP 1 diff --git a/src/init/piclas_init.f90 b/src/init/piclas_init.f90 index 1727796d6..621bd1447 100644 --- a/src/init/piclas_init.f90 +++ b/src/init/piclas_init.f90 @@ -287,6 +287,9 @@ SUBROUTINE FinalizePiclas(IsLoadBalance) #if USE_MPI USE MOD_Particle_MPI ,ONLY: FinalizeParticleMPI USE MOD_Particle_MPI_Vars ,ONLY: ParticleMPIInitisdone +#if defined(MEASURE_MPI_WAIT) +USE MOD_MPI ,ONLY: OutputMPIW8Time +#endif /*defined(MEASURE_MPI_WAIT)*/ #endif /*USE_MPI*/ #endif /*PARTICLES*/ USE MOD_IO_HDF5 ,ONLY: FinalizeElemData,ElementOut @@ -380,6 +383,12 @@ SUBROUTINE FinalizePiclas(IsLoadBalance) #if USE_MPI ! Free the communicators! CALL FinalizeMPIShared() +#if defined(MEASURE_MPI_WAIT) + ! Collect the MPI_WAIT() over all procs and output + IF(nProcessors.GT.1) CALL OutputMPIW8Time() +#endif /*defined(MEASURE_MPI_WAIT)*/ + ! Free the last communicator after OutputMPIW8Time + IF(MPI_COMM_PICLAS.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(MPI_COMM_PICLAS,IERROR) #endif /*USE_MPI*/ ELSE CALL DisplaySimulationTime(Time, StartTime, 'RUNNING') diff --git a/src/mpi/mpi.f90 b/src/mpi/mpi.f90 index bb2eef7d3..39f8447ed 100644 --- a/src/mpi/mpi.f90 +++ b/src/mpi/mpi.f90 @@ -110,17 +110,15 @@ SUBROUTINE InitMPI() CALL MPI_INIT(iError) CALL MPI_INITIALIZED(initDone,iError) IF(.NOT.initDone) CALL MPI_INIT(iError) - IF(iError .NE. 0) & - CALL Abort(__STAMP__,'Error in MPI_INIT',iError) - MPI_COMM_LOC = MPI_COMM_WORLD + IF(iError .NE. 0) CALL Abort(__STAMP__,'Error in MPI_INIT',iError) + ! General communicator + CALL MPI_COMM_DUP (MPI_COMM_WORLD,MPI_COMM_PICLAS,iError) + MPI_COMM_LOC = MPI_COMM_PICLAS END IF CALL MPI_COMM_RANK(MPI_COMM_LOC, myRank , iError) CALL MPI_COMM_SIZE(MPI_COMM_LOC, nProcessors, iError) -IF(iError .NE. 0) & - CALL Abort(& - __STAMP__& - ,'Could not get rank and number of processors',iError) +IF(iError .NE. 0) CALL Abort(__STAMP__,'Could not get rank and number of processors',iError) MPIRoot=(myRank .EQ. 0) #else /*USE_MPI*/ myRank = 0 @@ -130,7 +128,6 @@ SUBROUTINE InitMPI() MPILocalRoot=.TRUE. #endif /*USE_MPI*/ -! At this point the initialization is not completed. We first have to create a new MPI communicator. MPIInitIsDone will be set END SUBROUTINE InitMPI diff --git a/src/mpi/mpi_shared.f90 b/src/mpi/mpi_shared.f90 index d818da02e..34b142f32 100644 --- a/src/mpi/mpi_shared.f90 +++ b/src/mpi/mpi_shared.f90 @@ -129,9 +129,6 @@ SUBROUTINE InitMPIShared() SWRITE(UNIT_StdOut,'(132("-"))') SWRITE(UNIT_stdOut,'(A)') ' INIT MPI SHARED COMMUNICATION ...' -! General communicator -CALL MPI_COMM_DUP (MPI_COMM_WORLD,MPI_COMM_PICLAS,iError) - ! Save the global number of procs nProcessors_Global = nProcessors @@ -1141,7 +1138,6 @@ SUBROUTINE FinalizeMPIShared() ! Free the shared communicator IF(MPI_COMM_SHARED .NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(MPI_COMM_SHARED ,IERROR) IF(MPI_COMM_LEADERS_SHARED.NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(MPI_COMM_LEADERS_SHARED,IERROR) -IF(MPI_COMM_PICLAS .NE.MPI_COMM_NULL) CALL MPI_COMM_FREE(MPI_COMM_PICLAS ,IERROR) MPISharedInitIsDone=.FALSE. END SUBROUTINE FinalizeMPIShared diff --git a/src/mpi/mpi_vars.f90 b/src/mpi/mpi_vars.f90 index 038ec10be..211700866 100644 --- a/src/mpi/mpi_vars.f90 +++ b/src/mpi/mpi_vars.f90 @@ -32,7 +32,6 @@ MODULE MOD_MPI_Vars INTEGER :: iNbProc INTEGER :: nSendVal,nRecVal,DataSizeSide INTEGER :: SideID_start,SideID_end -LOGICAL :: MPIInitIsDone=.FALSE. #if USE_MPI INTEGER :: nNbProcs ! number of neighbor procs INTEGER,ALLOCATABLE :: NbProc(:) ! iProc list of neighbor procs; allocated from 1:nNbProcs diff --git a/src/piclas.f90 b/src/piclas.f90 index 4f21d4847..cd7043962 100644 --- a/src/piclas.f90 +++ b/src/piclas.f90 @@ -58,11 +58,6 @@ PROGRAM Piclas ! Finalize CALL FinalizePiclas(IsLoadBalance=.FALSE.) -#if defined(MEASURE_MPI_WAIT) -! Collect the MPI_WAIT() over all procs and output -IF(nProcessors.GT.1) CALL OutputMPIW8Time() -#endif /*defined(MEASURE_MPI_WAIT)*/ - ! MPI #if USE_MPI ! We also have to finalize MPI itself here From 5c754769993da6437c942b0c9ae19ac4cf35a6eb Mon Sep 17 00:00:00 2001 From: Paul Nizenkov Date: Thu, 19 Oct 2023 19:04:08 +0200 Subject: [PATCH 14/16] WEK_DSMC/ChannelFlow_AdaptiveBoundary_ConstPressure_TruncAverage: Increased number of particles to avoid strong fluctuations --- .../parameter.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regressioncheck/WEK_DSMC/ChannelFlow_AdaptiveBoundary_ConstPressure_TruncAverage/parameter.ini b/regressioncheck/WEK_DSMC/ChannelFlow_AdaptiveBoundary_ConstPressure_TruncAverage/parameter.ini index 6aabb6569..2bd70e77a 100644 --- a/regressioncheck/WEK_DSMC/ChannelFlow_AdaptiveBoundary_ConstPressure_TruncAverage/parameter.ini +++ b/regressioncheck/WEK_DSMC/ChannelFlow_AdaptiveBoundary_ConstPressure_TruncAverage/parameter.ini @@ -74,7 +74,7 @@ Part-FIBGMdeltas=(/2.5e-6,1e-5,1e-5/) ! Species1 - O2 ! =============================================================================== ! Part-Species1-MassIC=5.31352E-26 -Part-Species1-MacroParticleFactor=2E3 +Part-Species1-MacroParticleFactor=1E3 Part-Species1-nInits=1 Part-Species1-Init1-SpaceIC=cell_local From d38957edfaf6dc1331cc7b3c24298dda6caae88d Mon Sep 17 00:00:00 2001 From: Paul Nizenkov Date: Thu, 19 Oct 2023 19:47:38 +0200 Subject: [PATCH 15/16] Description of profile-guided optimization in the workflow documentation --- CMakeListsMachine.txt | 2 +- docs/documentation/userguide/workflow.md | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CMakeListsMachine.txt b/CMakeListsMachine.txt index 1fd41e266..4bac8bc4f 100644 --- a/CMakeListsMachine.txt +++ b/CMakeListsMachine.txt @@ -290,7 +290,7 @@ IF (USE_PGO) SET(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -fprofile-use") SET(CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -fprofile-generate") ELSE() - MESSAGE( SEND_ERROR "Profile-guided optimization (PGO) currently only supported for GNU compiler. Either set USE_GPO=OFF or use the GNU compiler." ) + MESSAGE( SEND_ERROR "Profile-guided optimization (PGO) currently only supported for GNU compiler. Either set USE_PGO=OFF or use the GNU compiler." ) ENDIF() ENDIF() diff --git a/docs/documentation/userguide/workflow.md b/docs/documentation/userguide/workflow.md index 9e4c29038..09b8117f3 100644 --- a/docs/documentation/userguide/workflow.md +++ b/docs/documentation/userguide/workflow.md @@ -193,6 +193,18 @@ the mesh is simply divided into parts along the space filling curve. Thus, domai not limited by e.g. an integer factor between the number of cores and elements. The only limitation is that the number of cores may not exceed the number of elements. +### Profile-guided optimization (PGO) + +To further increase performance for production runs, profile-guided optimization can be utilized with the GNU compiler. This requires the execution of a representative simulation run with PICLas compiled using profiling instrumentation. For this purpose, the code has to be configured and compiled using the following additional settings and the `Profile` build type: + + -DPICLAS_PERFORMANCE=ON -DUSE_PGO=ON -DCMAKE_BUILD_TYPE=Profile + +A short representative simulation has to be performed, where additional files with the profiling information will be stored. Note that the test run should be relatively short as the code will be substantially slower than the regular `Release` build type. Afterwards, the code can be configured and compiled again for the production runs, using the `Release` build type: + + -DPICLAS_PERFORMANCE=ON -DUSE_PGO=ON -DCMAKE_BUILD_TYPE=Release + +Warnings regarding missing profiling files (`-Wmissing-profile`) can be ignored, if they concern modules not relevant for the current simulation method (e.g. `bgk_colloperator.f90` will be missing profile information if only a DSMC simulation has been performed). + ## Post-processing **PICLas** comes with a tool for visualization. The piclas2vtk tool converts the HDF5 files generated by **PICLas** to the binary From 5b5660203fa45a17f450aa6a7ae242a647e392c4 Mon Sep 17 00:00:00 2001 From: Paul Nizenkov Date: Thu, 19 Oct 2023 23:50:43 +0200 Subject: [PATCH 16/16] WEK_DSMC/ChannelFlow_AdaptiveBoundary_ConstMassflow: increased sampling duration of adaptive BC --- .../ChannelFlow_AdaptiveBoundary_ConstMassflow/parameter.ini | 2 +- .../parameter.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/regressioncheck/WEK_DSMC/ChannelFlow_AdaptiveBoundary_ConstMassflow/parameter.ini b/regressioncheck/WEK_DSMC/ChannelFlow_AdaptiveBoundary_ConstMassflow/parameter.ini index e5f06d50d..75243fdf6 100644 --- a/regressioncheck/WEK_DSMC/ChannelFlow_AdaptiveBoundary_ConstMassflow/parameter.ini +++ b/regressioncheck/WEK_DSMC/ChannelFlow_AdaptiveBoundary_ConstMassflow/parameter.ini @@ -105,7 +105,7 @@ Part-Species1-Surfaceflux2-Adaptive-Pressure=2.5 ! Continuous sample, where the current value is added partially, multiplied the relaxation factor AdaptiveBC-RelaxationFactor = 0.1 ! Sample is reset every given number of iterations -AdaptiveBC-SamplingIteration = 0,10,10,10 +AdaptiveBC-SamplingIteration = 0,20,20,20 ! Sample is not reset, current value replaces the oldest (DEFAULT: TRUE) AdaptiveBC-TruncateRunningAverage = F,F,T,T ! Average the sample over the whole BC diff --git a/regressioncheck/WEK_DSMC/ChannelFlow_AdaptiveBoundary_ConstPressure_TruncAverage/parameter.ini b/regressioncheck/WEK_DSMC/ChannelFlow_AdaptiveBoundary_ConstPressure_TruncAverage/parameter.ini index 2bd70e77a..3de5cda38 100644 --- a/regressioncheck/WEK_DSMC/ChannelFlow_AdaptiveBoundary_ConstPressure_TruncAverage/parameter.ini +++ b/regressioncheck/WEK_DSMC/ChannelFlow_AdaptiveBoundary_ConstPressure_TruncAverage/parameter.ini @@ -90,7 +90,7 @@ Part-Species1-nSurfaceFluxBCs=2 ! Inlet: Constant pressure, Type 1 Part-Species1-Surfaceflux1-BC=1 Part-Species1-Surfaceflux1-velocityDistribution=maxwell_lpn -Part-Species1-Surfaceflux1-VeloIC=5 +Part-Species1-Surfaceflux1-VeloIC=20 Part-Species1-Surfaceflux1-VeloVecIC=(/-1.0,0.0,0.0/) Part-Species1-Surfaceflux1-MWTemperatureIC=300. Part-Species1-Surfaceflux1-TempVib=300. @@ -102,7 +102,7 @@ Part-Species1-Surfaceflux1-Adaptive-Pressure=5.0 ! Outlet: Constant pressure, Type 2 Part-Species1-Surfaceflux2-BC=2 Part-Species1-Surfaceflux2-velocityDistribution=maxwell_lpn -Part-Species1-Surfaceflux2-VeloIC=5 +Part-Species1-Surfaceflux2-VeloIC=20 Part-Species1-Surfaceflux2-VeloVecIC=(/1.0,0.0,0.0/) Part-Species1-Surfaceflux2-MWTemperatureIC=300. Part-Species1-Surfaceflux2-TempVib=300.