Merge branch 'fix.initial.LB.no.state.file.output' into 'master.dev'

[fix.initial.LB.no.state.file.output] Fixed h5 output during initial load balancing after the first time step. Also...

See merge request piclas/piclas!723

regressioncheck/NIG_PIC_poisson_Boris-Leapfrog/2D_HET_Liu2010/analyze.ini

@@ -1,6 +1,6 @@
 ! integrate the number of electrons impinging on the anode
 integrate_line_file            = SurfaceAnalyze.csv    ! Data file name
 integrate_line_columns         = 0:1                   ! Columns x:y for integration y over x
-integrate_line_integral_value  = 1.6e-4                ! Number of removed electrons through the anode BC: integrated value from one exemplary simulation and rounded
-integrate_line_tolerance_value = 10e-2                 ! Tolerance
+integrate_line_integral_value  = 0.000118725           ! Number of removed electrons through the anode BC: integrated value from the first single-core simulation
+integrate_line_tolerance_value = 12e-2                 ! Tolerance (largest difference was observed to be 10.739%)
 integrate_line_tolerance_type  = relative              ! Relative or absolute tolerance

regressioncheck/NIG_PIC_poisson_Boris-Leapfrog/2D_HET_Liu2010/command_line.ini

@@ -1,2 +1,2 @@
-MPI        = 3,6,12
+MPI        = 1,3,6,12
 cmd_suffix = DSMC.ini

src/io_hdf5/hdf5_output_particle.f90

@@ -1814,7 +1814,7 @@ SUBROUTINE FillParticleData()
 USE MOD_Particle_Vars          ,ONLY: locnPart,offsetnPart
 USE MOD_DSMC_Vars              ,ONLY: UseDSMC,CollisMode,DSMC,PolyatomMolDSMC,SpecDSMC
 USE MOD_Particle_Vars          ,ONLY: VibQuantData,ElecDistriData,AD_Data,MaxQuantNum,MaxElecQuant
-USE MOD_Particle_Vars          ,ONLY: PDM, PartState, PartSpecies, PartMPF, usevMPF, nSpecies, Species
+USE MOD_Particle_Vars          ,ONLY: PartState, PartSpecies, PartMPF, usevMPF, nSpecies, Species
 USE MOD_DSMC_Vars              ,ONLY: UseDSMC, CollisMode,PartStateIntEn, DSMC, PolyatomMolDSMC, SpecDSMC, VibQuantsPar
 USE MOD_DSMC_Vars              ,ONLY: ElectronicDistriPart, AmbipolElecVelo
 USE MOD_LoadBalance_Vars       ,ONLY: nPartsPerElem
@@ -2090,9 +2090,7 @@ SUBROUTINE FillParticleData()
       END DO
       iPart = PartInt(2,iElem_glob)
     ELSE
-      CALL abort(&
-      __STAMP__&
-      , " Particle HDF5-Output method not supported! PEM%pNumber not associated")
+      CALL abort(__STAMP__, " Particle HDF5-Output method not supported! PEM%pNumber not associated")
     END IF
     PartInt(2,iElem_glob)=iPart
   END DO

src/loadbalance/loadbalance.f90

@@ -209,7 +209,6 @@ SUBROUTINE ComputeElemLoad()
 #ifdef PARTICLES
   ElemTimePartTot = 0.
 #endif /*PARTICLES*/
-
 ! If elem times are calculated by time measurement (PerformLBSample) and no Partweight Loadbalance is enabled
 IF(PerformLBSample .AND. LoadBalanceSample.GT.0) THEN
 
@@ -319,8 +318,7 @@ SUBROUTINE ComputeElemLoad()
   END DO ! iElem=1,PP_nElems
   IF((MAXVAL(nPartsPerElem).GT.0).AND.(MAXVAL(ElemTime).LE.1.0))THEN
     IPWRITE (*,*) "parts, time =", MAXVAL(nPartsPerElem),MAXVAL(ElemTime)
-    CALL abort(&
-        __STAMP__&
+    CALL abort(__STAMP__&
         ,' ERROR: MAXVAL(nPartsPerElem).GT.0 but MAXVAL(ElemTime).LE.1.0 with ParticleMPIWeight=',RealInfoOpt=ParticleMPIWeight)
   END IF
 #endif /*PARTICLES*/

src/loadbalance/loaddistribution.f90

@@ -170,6 +170,7 @@ SUBROUTINE ApplyWeightDistributionMethod(ElemTimeExists)
 USE MOD_LoadBalance_Vars ,ONLY: PerformLoadBalance,UseH5IOLoadBalance
 USE MOD_LoadBalance_Vars ,ONLY: offsetElemMPIOld
 USE MOD_Particle_Vars    ,ONLY: PartInt
+USE MOD_HDF5_Output_Particles  ,ONLY: FillParticleData
 #endif /*PARTICLES*/
 ! IMPLICIT VARIABLE HANDLING
 IMPLICIT NONE
@@ -209,6 +210,8 @@ SUBROUTINE ApplyWeightDistributionMethod(ElemTimeExists)
   DO iProc = 0,nProcessors-1
     ElemPerProc(iProc) = offsetElemMPIOld(iProc+1) - offsetElemMPIOld(iProc)
   END DO
+  ! Sanity check
+  IF(.NOT.ALLOCATED(PartInt)) CALL abort(__STAMP__,'PartInt is not allocated') ! Missing call to FillParticleData()
   CALL MPI_GATHERV(PartInt,nElemsOld,MPI_DOUBLE_PRECISION,PartIntGlob,ElemPerProc,offsetElemMPIOld(0:nProcessors-1),MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,iError)
   PartIntExists = .TRUE.
 ELSE

src/mesh/mesh.f90

@@ -203,7 +203,7 @@ SUBROUTINE InitMesh(meshMode,MeshFile_IN)
 
 useCurveds=GETLOGICAL('useCurveds')
 #if USE_LOADBALANCE
-IF ( (DoLoadBalance.OR.DoInitialAutoRestart) .AND. .NOT.DoWriteStateToHDF5) THEN
+IF ( (DoLoadBalance.OR.DoInitialAutoRestart) .AND. (.NOT.DoWriteStateToHDF5) .AND. UseH5IOLoadBalance) THEN
   DoWriteStateToHDF5=.TRUE.
   CALL PrintOption('Loadbalancing or InitialAutoRestart enabled: DoWriteStateToHDF5','INFO',LogOpt=DoWriteStateToHDF5)
 END IF

src/timedisc/timedisc.f90

@@ -216,7 +216,6 @@ SUBROUTINE TimeDisc()
 #if defined(PARTICLES)
   CALL FillParticleData() ! Fill the SFC-ordered particle arrays
 #endif /*defined(PARTICLES)*/
-
   CALL WriteStateToHDF5(TRIM(MeshFile),time,tPreviousAnalyze) ! Write initial state to file
 END IF
 
@@ -350,8 +349,8 @@ SUBROUTINE TimeDisc()
   !IF ((dt.EQ.dt_Min(DT_ANALYZE)).OR.(dt.EQ.dt_Min(DT_END))) THEN   ! timestep is equal to time to analyze or end
 #if USE_LOADBALANCE
   ! For automatic initial restart, check if the number of sampling steps has been achieved and force a load balance step, but skip
-  ! this procedure in the final iteration after which the simulation if finished
-  !      DoInitialAutoRestart: user-activated load balance restart in first time step (could already be a restart)
+  ! this procedure in the final iteration after which the simulation is finished
+  !      DoInitialAutoRestart: user-activated load balance restart in first time step (could also be during a normal restart)
   ! iter.GE.LoadBalanceSample: as soon as the number of time steps for sampling is reached, perform the load balance restart
   !                 finalIter: prevent removal of last state file even though no load balance restart was performed
   IF(DoInitialAutoRestart.AND.(iter.GE.LoadBalanceSample).AND.(.NOT.finalIter)) ForceInitialLoadBalance=.TRUE.
@@ -406,7 +405,9 @@ SUBROUTINE TimeDisc()
     ! finalIter=T: last iteration of the simulation is reached, hence, always perform analysis and output to hdf5
 #if USE_LOADBALANCE
     ! PerformLoadBalance.AND.UseH5IOLoadBalance: Load balance step will be performed and load balance restart via hdf5 IO active
-    IF(MOD(iAnalyze,nSkipAnalyze).EQ.0 .OR. (PerformLoadBalance.AND.UseH5IOLoadBalance) .OR. finalIter)THEN
+    ! .NOT.(DoInitialAutoRestart.AND.(.NOT.UseH5IOLoadBalance)): Skip I/O for initial LB because MOD might give 0
+    IF( ((.NOT.(DoInitialAutoRestart.AND.(.NOT.UseH5IOLoadBalance))).AND.MOD(iAnalyze,nSkipAnalyze).EQ.0)&
+        .OR. (PerformLoadBalance.AND.UseH5IOLoadBalance) .OR. finalIter )THEN
 #else
     IF(MOD(iAnalyze,nSkipAnalyze).EQ.0 .OR. finalIter)THEN
 #endif /*USE_LOADBALANCE*/
@@ -415,7 +416,7 @@ SUBROUTINE TimeDisc()
       ! write information out to std-out of console
       CALL WriteInfoStdOut()
 #if defined(PARTICLES)
-      CALL FillParticleData() ! Fill the SFC-ordered particle arrays
+      CALL FillParticleData() ! Fill the SFC-ordered particle arrays for LB or I/O
 #endif /*defined(PARTICLES)*/
       ! Write state to file
       CALL WriteStateToHDF5(TRIM(MeshFile),time,tPreviousAnalyze)
@@ -425,6 +426,10 @@ SUBROUTINE TimeDisc()
       tPreviousAnalyze        = tAnalyze
       tPreviousAverageAnalyze = tAnalyze
       SWRITE(UNIT_StdOut,'(132("-"))')
+#if USE_LOADBALANCE && defined(PARTICLES)
+    ELSEIF(PerformLoadBalance) THEN
+      CALL FillParticleData() ! Fill the SFC-ordered particle arrays for LB
+#endif /*USE_LOADBALANCE && defined(PARTICLES)*/
     END IF ! actual analyze is done
 
     iter_PID=0

src/timedisc/timedisc_init.f90

@@ -75,7 +75,7 @@ SUBROUTINE InitTime()
 #endif /*PARTICLES*/
 iAnalyze=1
 ! Determine the first analyze time
-tAnalyze=MIN(RestartTime+REAL(iAnalyze)*Analyze_dt,tEnd)
+tAnalyze=MIN(RestartTime+Analyze_dt,tEnd)
 
 ! fill initial analyze stuff
 dt_Min(DT_ANALYZE) = tAnalyze-Time ! Time to next analysis, put in extra variable so number does not change due to numerical errors