NCEP GFSv16/GEFSv12 (#140)

Changes including new features and bugfixes for adding WW3 to the GEFSv12 and GFSv16 coupled systems at NCEP:

Added option to write single output (gridded and point) files at individual specified output time steps (#141 ), to allow inline post-processing;
Added option to write second stream of restart files (#1 ) with write stride different to first (default) restart file,
Updated ww3_grib for grib2 encoding following recent additions/changes in WMO tables,
Several bugfixes to new output and restart options, including alignment of MPI calls and resources, backwards compatibility with multiple output configurations specified in regtests,
Changes to compile options for WCOSS to allow b4b.
Please find results from the regtest matrix comparisons in the comment below.

model/bin/comp.wcoss_dell_p3

@@ -80,17 +80,9 @@
 
 # opt="-c $list -O3 -assume byterecl -ip -module $path_m"
 
-opt="-c $list -module $path_m -no-fma -ip -g -i4 -real-size 32 -fp-model precise -assume byterecl -convert big_endian -fno-alias -fno-fnalias -sox"
-opt="$opt  -xHOST -qno-opt-dynamic-align -auto -nowarn -align array64byte -Wp,-w"
+  opt="-c $list -module $path_m -no-fma -ip -g -i4 -real-size 32 -fp-model precise -assume byterecl -convert big_endian -fno-alias -sox -xhost -O2"
 # Add profiling
 # opt="$opt -p"
-###  opt="-c $list -O2 -Wp,-w -fno-alias -auto -ftz -assume byterecl -nowarn -sox -align array64byte"
-###  opt="$opt -xHOST -qno-opt-dynamic-align -fp-model source -ip -module $path_m"
-##  opt="$opt -nowarn -sox -align array64byte -real-size 32 -xHOST -qno-opt-dynamic-align"
-##  opt="$opt -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3"
-# opt="-c $list -O3 -assume byterecl -xSSE4.2 -ip  -module $path_m"
-# opt="-c $list -O3 -assume byterecl -prec-div -prec-sqrt -xHost -align array128byte -ip  -module $path_m"
-# opt="-c $list -O0 -assume byterecl -g -traceback -check all -fpe0 -ftrapuv -module $path_m"
 
   if [ "$name" != 'ww3_gspl' ]
   then

model/bin/link.wcoss_dell_p3

@@ -86,17 +86,12 @@
 
   # linking options
   libs=""
-#  opt="-O2 -ip -o $prog"
-opt="-o $prog -g"
-opt="$opt -xhost"
+
+  opt="-o $prog -g -xhost -O2"
+
 # Add profiling
 # opt="$opt -p"
 
-# opt="-O3 -ip -o $prog"
-# opt="-O3 -xSSE4.2 -ip -o $prog"
-# opt="-O3 -prec-div -prec-sqrt -xHost -align array128byte -ip -o $prog"
-# opt="-O0 -g -traceback -check all -fpe0 -ftrapuv -o $prog"
-
   # mpi implementation
   if [ "$mpi_mod" = 'yes' ]
   then

model/bin/switch_NCEP_gwm

@@ -1 +1 @@
-F90 NCO NOPA LRB4 NOGRB DIST MPI OMPH OMPG SCRIP SCRIPNC NC4 PR3 UQ ST4 STAB0 FLX0 LN1 NL1 BT1 DB1 IC4 IS0 REF0 TR0 BS0 XX0 WNX1 WNT1 CRX1 CRT1 RWND O0 O1 O2 O4 O5 O6 O7 O14 O15
+F90 NCO NOPA NOGRB SHRD SCRIP SCRIPNC NC4 PR3 UQ ST4 STAB0 FLX0 LN1 NL1 BT1 DB1 IC0 IS0 REF0 TR0 BS0 XX0 WNX1 WNT1 CRX1 CRT1 RWND O0 O1 O2 O4 O5 O6 O7 O14 O15

model/ftn/w3initmd.ftn

@@ -117,9 +117,11 @@
 !/
       CONTAINS
 !/ ------------------------------------------------------------------- /
-      SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT,FLGRD,         &
+      SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT      &
+                          , FLGRD,                               &
                            FLGR2, FLGD, FLG2, NPT, XPT, YPT, PNAMES,   &
-                          IPRT, PRTFRM, MPI_COMM, FLAGSTIDEIN )
+                          IPRT, PRTFRM, MPI_COMM, FLAGSTIDEIN)
+
 !/
 !/                  +-----------------------------------+
 !/                  | WAVEWATCH III           NOAA/NCEP |
@@ -213,6 +215,7 @@
 !                          16-20 Id. for OTYPE = 4; restart files.
 !                          21-25 Id. for OTYPE = 5; boundary data.
 !                          31-35 Id. for OTYPE = 7; coupling data.
+!                          36-40 Id. for OTYPE = 8; second restart file
 !       FLGRD   L.A.   I   Flags for gridded output.
 !       FLGR2   L.A.   I   Flags for coupling output.
 !       NPT     Int.   I   Number of output points
@@ -388,7 +391,8 @@
 !/ Parameter list
 !/
       INTEGER, INTENT(IN)           :: IMOD, MDS(13), MTRACE(2),      &
-                                       ODAT(35), NPT, IPRT(6), MPI_COMM
+                                       ODAT(40),NPT, IPRT(6),&
+                                       MPI_COMM
       LOGICAL, INTENT(IN)           :: IsMulti
       REAL, INTENT(INOUT)           :: XPT(NPT), YPT(NPT)
       LOGICAL, INTENT(INOUT)        :: FLGRD(NOGRP,NGRPP), FLGD(NOGRP),&
@@ -851,7 +855,7 @@
 !/DEBUGINIT     WRITE(740+IAPROC,*) ' After ALL_VA_INTEGRAL_PRINT'
 !/DEBUGINIT     FLUSH(740+IAPROC)
 !/TIMINGS       CALL PRINT_MY_TIME("Before W3IORS")
-      CALL W3IORS ( 'READ', NDS(6), SIG(NK), INTYPE, IMOD )
+      CALL W3IORS ( 'READ', NDS(6), SIG(NK), INTYPE, IMOD)
 !/TIMINGS       CALL PRINT_MY_TIME("After W3IORS")
 !/MEMCHECK      WRITE(740+IAPROC,*) 'memcheck_____:', 'WW3_INIT SECTION 3a'
 !/MEMCHECK      call getMallocInfo(mallinfos)
@@ -954,6 +958,20 @@
         TOLAST(2,J) =        ODAT(J0+5)
       END DO
 !
+! J=8, second stream of restart files
+        J=8
+        J0 = (J-1)*5
+      IF(ODAT(J0+1) .NE. 0) THEN
+        TONEXT(1,J) =        ODAT(J0+1) 
+        TONEXT(2,J) =        ODAT(J0+2)
+        DTOUT (  J) = REAL ( ODAT(J0+3) )
+        TOLAST(1,J) =        ODAT(J0+4) 
+        TOLAST(2,J) =        ODAT(J0+5)
+        FLOUT(8) = .TRUE.
+      ELSE
+        FLOUT(8) = .FALSE.
+      END IF
+!
 ! 4.b Check if output available
 !
       FLOUT(1) = .FALSE.
@@ -1060,6 +1078,57 @@
 !
         END DO
 !
+! J=8, second stream of restart files
+!
+      J=8
+!
+! ... check time step  
+!
+        DTOUT(J) = MAX ( 0. , DTOUT(J) )
+        FLOUT(J) = FLOUT(J) .AND. ( DTOUT(J) .GT. 0.5 )
+!
+! ... get first time 
+!
+        IF ( FLOUT(J) ) THEN
+            TOUT = TONEXT(:,J)
+            TLST = TOLAST(:,J)
+!
+            DO
+              DTTST   = DSEC21 ( TIME , TOUT )
+              IF ( ( J.NE.4 .AND. DTTST.LT.0. ) .OR.                  &
+                   ( J.EQ.4 .AND. DTTST.LE.0. ) ) THEN
+                  CALL TICK21 ( TOUT, DTOUT(J) )
+                ELSE
+                  EXIT
+                END IF
+              END DO
+!
+! ... reset first time
+!
+            TONEXT(:,J) = TOUT
+!
+! ... check last time
+!
+            DTTST  = DSEC21 ( TOUT , TLST )
+            IF ( DTTST.LT.0.) FLOUT(J) = .FALSE.
+!
+! ... check overall first time
+!
+            IF ( FLOUT(J) ) THEN
+                IF ( TOFRST(1).EQ.-1 ) THEN
+                    TOFRST = TOUT
+                  ELSE
+                    DTTST  = DSEC21 ( TOUT , TOFRST )
+                    IF ( DTTST.GT.0.) THEN
+                        TOFRST = TOUT
+                      END IF
+                  END IF
+              END IF
+!
+          END IF
+! END J=8
+!
+!
 !/MEMCHECK       WRITE(740+IAPROC,*) 'memcheck_____:', 'WW3_INIT SECTION 5'
 !/MEMCHECK       call getMallocInfo(mallinfos)
 !/MEMCHECK       call printMallInfo(IAPROC,mallInfos)
@@ -3591,7 +3660,7 @@
 !/MPI      IH     = 0
 !/MPI      IROOT  = NAPRST - 1
 !
-!/MPI      IF ( FLOUT(4) ) THEN
+!/MPI      IF ( FLOUT(4) .OR. FLOUT(8) ) THEN
 !/MPI          ALLOCATE ( OUTPTS(IMOD)%OUT4%IRQRS(3*NAPROC) )
 !/MPI          IRQRS  => OUTPTS(IMOD)%OUT4%IRQRS
 !

model/ftn/w3iogomd.ftn

@@ -53,6 +53,8 @@
 !/    22-Aug-2018 : Add WBT output parameter            ( version 6.06 )
 !/    25-Sep-2019 : Corrected th2m and sth2m            ( version 6.07 )
 !/                  calculations. (J Dykes, NRL)
+!/    04-Oct-2019 : Optional one file per output stride ( version 7.00 )
+!/                  (Roberto Padilla-Hernandez & J.H. Alves)
 !/
 !/    Copyright 2009-2014 National Weather Service (NWS),
 !/       National Oceanic and Atmospheric Administration.  All rights
@@ -2343,6 +2345,7 @@
 !/
       END SUBROUTINE W3OUTG
 !/ ------------------------------------------------------------------- /
+!/
       SUBROUTINE W3IOGO ( INXOUT, NDSOG, IOTST, IMOD )
 !/
 !/                  +-----------------------------------+
@@ -2483,6 +2486,7 @@
 !/
       USE W3SERVMD, ONLY: EXTCDE
       USE W3ODATMD, only : IAPROC
+      USE W3ODATMD, ONLY :  OFILES
 !/SETUP     USE W3WDATMD, ONLY: ZETA_SETUP
 !/S      USE W3SERVMD, ONLY: STRACE
 !
@@ -2495,12 +2499,14 @@
       INTEGER, INTENT(IN)           :: NDSOG
       INTEGER, INTENT(IN), OPTIONAL :: IMOD
       CHARACTER, INTENT(IN)         :: INXOUT*(*)
+      CHARACTER(LEN=15) :: TIMETAG
 !/
 !/ ------------------------------------------------------------------- /
 !/ Local parameters
 !/
       INTEGER                 :: IGRD, IERR, I, J, IX, IY, MOGRP,     &
-                                 MGRPP, ISEA, MOSWLL, IK, IFI, IFJ
+                                 MGRPP, ISEA, MOSWLL, IK, IFI, IFJ    &
+                                 ,IFILOUT
       INTEGER, ALLOCATABLE    :: MAPTMP(:,:)
 !/S      INTEGER, SAVE           :: IENT = 0
       REAL                    :: AUX1(NSEA), AUX2(NSEA),              &
@@ -2535,7 +2541,7 @@
           CALL EXTCDE ( 1 )
         END IF
 !
-      IF ( IPASS.EQ.1 ) THEN
+      IF ( IPASS.EQ.1 .AND. OFILES(1) .EQ. 0) THEN
           WRITE  = INXOUT.EQ.'WRITE'
         ELSE
           IF ( WRITE .AND. INXOUT.EQ.'READ' ) THEN
@@ -2546,11 +2552,11 @@
 !
 !/T      WRITE (NDST,9000) IPASS, INXOUT, WRITE, NDSOG, IGRD, FILEXT
 !
+!
 ! open file ---------------------------------------------------------- *
 ! ( IPASS = 1 )
 !
-      IF ( IPASS.EQ.1 ) THEN
-!
+      IF ( IPASS.EQ.1 .AND. OFILES(1) .EQ. 0) THEN
           I      = LEN_TRIM(FILEXT)
           J      = LEN_TRIM(FNMPRE)
 !
@@ -2604,6 +2610,76 @@
 !
         END IF
 !
+!  IN CASE OF GENERATION OF A NEW FILE OUTPUT EVERY DELTA OUTPUT
+! open file ---------------------------------------------------------- *
+! ( IPASS = 1 )
+!
+      IF ( IPASS.GE.1 .AND. OFILES(1) .EQ. 1) THEN
+          WRITE  = INXOUT.EQ.'WRITE'
+        ELSE
+          IF ( WRITE .AND. INXOUT.EQ.'READ' ) THEN
+              WRITE (NDSE,901) INXOUT
+              CALL EXTCDE ( 2 )
+            END IF
+        END IF
+
+!
+      IF ( IPASS.GE.1 .AND. OFILES(1) .EQ. 1) THEN
+          I      = LEN_TRIM(FILEXT)
+          J      = LEN_TRIM(FNMPRE)
+!
+! Create TIMETAG for file name using YYYYMMDD.HHMMS prefix
+          WRITE(TIMETAG,"(i8.8,'.'i6.6)")TIME(1),TIME(2)
+!/T          WRITE (NDST,9001) FNMPRE(:J)//TIMETAG//'.out_grd.'//FILEXT(:I)
+          IF ( WRITE ) THEN
+              OPEN (NDSOG,FILE=FNMPRE(:J)//TIMETAG//'.out_grd.'  &
+                    //FILEXT(:I),FORM='UNFORMATTED',ERR=800,IOSTAT=IERR)
+          ELSE
+              OPEN (NDSOG,FILE=FNMPRE(:J)//'out_grd.'//FILEXT(:I),    &
+                    FORM='UNFORMATTED',ERR=800,IOSTAT=IERR,STATUS='OLD')
+            END IF
+!
+          REWIND ( NDSOG )
+!
+! test info --------------------------------------------------------- *
+! ( IPASS >= 1 & OFILES(1) = 1)
+!
+          IF ( WRITE ) THEN
+              WRITE (NDSOG)                                           &
+                IDSTR, VEROGR, GNAME, NOGRP, NGRPP, NSEA, NX, NY,     &
+                UNDEF, NOSWLL
+            ELSE
+              READ (NDSOG,END=801,ERR=802,IOSTAT=IERR)                &
+                IDTST, VERTST, TNAME, MOGRP, MGRPP, NSEA, NX, NY,     &
+                UNDEF, MOSWLL
+!
+              IF ( IDTST .NE. IDSTR ) THEN
+                  WRITE (NDSE,902) IDTST, IDSTR
+                  CALL EXTCDE ( 20 )
+                END IF
+              IF ( VERTST .NE. VEROGR ) THEN
+                  WRITE (NDSE,903) VERTST, VEROGR
+                  CALL EXTCDE ( 21 )
+                END IF
+              IF ( NOGRP .NE. MOGRP .OR. NGRPP .NE. MGRPP ) THEN
+                  WRITE (NDSE,904) MOGRP, MGRPP, NOGRP, NGRPP
+                  CALL EXTCDE ( 22 )
+                END IF
+              IF ( TNAME .NE. GNAME ) THEN
+                  WRITE (NDSE,905) TNAME, GNAME
+                END IF
+              IF ( NOSWLL .NE. MOSWLL ) THEN
+                  WRITE (NDSE,906) MOSWLL, NOSWLL
+                  CALL EXTCDE ( 24 )
+                END IF
+!
+            END IF
+!
+!/T          WRITE (NDST,9002) IDSTR, VEROGR, GNAME, NSEA, NX, NY,    &
+!/T                            UNDEF
+!
+        END IF
+!
 ! TIME and flags ----------------------------------------------------- *
 !
       IF ( WRITE ) THEN
@@ -3417,6 +3493,8 @@
 ! Flush the buffers for write
 !
       IF ( WRITE ) CALL FLUSH ( NDSOG )
+!
+      IF(OFILES(1) .EQ. 1) CLOSE(NDSOG)
 !
 !/MPI      CALL W3SETA ( IGRD, NDSE, NDST )
 !