From d95a4f35ee7800af0bacbd8a61359f8ebe104519 Mon Sep 17 00:00:00 2001
From: TRasmussen <33480590+TillRasmussen@users.noreply.github.com>
Date: Tue, 3 Nov 2020 18:55:10 +0100
Subject: [PATCH] Add DMI Cray XC50 hpc (freya) to machines. Based on Banting
 (#528)

* Add DMI Cray XC50 hpc (freya) to machines. Based on Banting

* separation of Banting and Freya

* Removed depreciated cpp and added gnu to freya
---
 configuration/scripts/cice.batch.csh          |  8 +++
 configuration/scripts/cice.launch.csh         | 12 ++++
 .../scripts/machines/Macros.freya_gnu         | 40 +++++++++++
 .../scripts/machines/Macros.freya_intel       | 69 +++++++++++++++++++
 configuration/scripts/machines/env.freya_gnu  | 39 +++++++++++
 .../scripts/machines/env.freya_intel          | 38 ++++++++++
 6 files changed, 206 insertions(+)
 create mode 100644 configuration/scripts/machines/Macros.freya_gnu
 create mode 100644 configuration/scripts/machines/Macros.freya_intel
 create mode 100755 configuration/scripts/machines/env.freya_gnu
 create mode 100755 configuration/scripts/machines/env.freya_intel

diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh
index 54a2be711..d6f78313f 100755
--- a/configuration/scripts/cice.batch.csh
+++ b/configuration/scripts/cice.batch.csh
@@ -193,6 +193,14 @@ cat >> ${jobfile} << EOFB
 #PBS -l walltime=${batchtime}
 EOFB
 
+else if (${ICE_MACHINE} =~ freya* ) then
+cat >> ${jobfile} << EOFB
+#PBS -N ${ICE_CASENAME}
+#PBS -j oe
+#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}
+#PBS -l walltime=${batchtime}
+EOFB
+
 else if (${ICE_MACHINE} =~ hera*) then
 cat >> ${jobfile} << EOFB
 #SBATCH -J ${ICE_CASENAME}
diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh
index dfbffd6ab..a05b3a9d3 100755
--- a/configuration/scripts/cice.launch.csh
+++ b/configuration/scripts/cice.launch.csh
@@ -141,6 +141,18 @@ aprun -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_F
 EOFR
 endif
 
+#=======
+else if (${ICE_MACHINE} =~ freya*) then
+if (${ICE_COMMDIR} =~ serial*) then
+cat >> ${jobfile} << EOFR
+aprun -n 1 -N 1 -d 1 ./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+else
+cat >> ${jobfile} << EOFR
+aprun -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+endif
+
 #=======
 else if (${ICE_MACHINE} =~ hera*) then
 cat >> ${jobfile} << EOFR
diff --git a/configuration/scripts/machines/Macros.freya_gnu b/configuration/scripts/machines/Macros.freya_gnu
new file mode 100644
index 000000000..dafa4fd9b
--- /dev/null
+++ b/configuration/scripts/machines/Macros.freya_gnu
@@ -0,0 +1,40 @@
+#==============================================================================
+# Makefile macros for DMI freya
+#==============================================================================
+# For use with GNU compiler
+#==============================================================================
+
+#INCLDIR := -I. -I/usr/include
+#SLIBS   :=
+
+#--- Compiler/preprocessor ---
+FC         := ftn 
+CC         := cc
+CXX        := CC
+CPP        := /usr/bin/cpp
+CPPFLAGS   := -P -traditional  # ALLOW fortran double backslash "\\"
+SCC   := gcc
+SFC   := ftn
+
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 
+#-xHost
+
+FREEFLAGS  := -ffree-form
+FFLAGS     := -fconvert=big-endian -fbacktrace -ffree-line-length-none
+#-xHost
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow 
+else
+  FFLAGS     += -O2 #FROM BANTING
+  #FFLAGS     := -O2 -ffloat-store -march=native -ffree-line-length-non # DMI BUILD
+endif
+LD:= $(FC)
+LDFLAGS    := $(FFLAGS) -v
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -fopenmp 
+   CFLAGS += -fopenmp 
+   FFLAGS += -fopenmp 
+endif
diff --git a/configuration/scripts/machines/Macros.freya_intel b/configuration/scripts/machines/Macros.freya_intel
new file mode 100644
index 000000000..f40ca4e23
--- /dev/null
+++ b/configuration/scripts/machines/Macros.freya_intel
@@ -0,0 +1,69 @@
+#==============================================================================
+# Makefile macros for DMI Freya based on ECCC banting
+#==============================================================================
+# For use with intel compiler
+#==============================================================================
+
+#INCLDIR := -I. -I/usr/include
+#SLIBS   :=
+
+#--- Compiler/preprocessor ---
+FC         := ftn 
+CC         := cc
+CXX        := CC
+CPP        := /usr/bin/cpp
+CPPFLAGS   := -P -traditional  # ALLOW fortran double backslash "\\"
+SCC   := gcc
+SFC   := ftn
+
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -fp-model precise
+# Additional flags
+FIXEDFLAGS := -132
+FREEFLAGS  := -FR
+FFLAGS     := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -no-wrap-margin
+#-xHost
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created
+# -heap-arrays 1024 
+else
+  FFLAGS     += -O2
+endif
+LD         := $(FC)
+LDFLAGS    := $(FFLAGS) -v
+#ifeq ($(ICE_BLDDEBUG), true)
+#FFLAGS     := -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created
+#FFLAGS     := -g -O0 -traceback -fp-model precise -fp-stack-check -fpe0
+#else
+#FFLAGS     := -r8 -i4 -O2 -align all -w -ftz -assume byterecl
+# FFLAGS     := -O2 -fp-model precise -assume byterecl -ftz -traceback -xHost
+#endif
+# Preprocessor flags
+#CPPDEFS    := -DLINUX $(ICE_CPPDEFS)
+
+# Linker flags
+
+# Additional flags
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -qopenmp 
+   CFLAGS += -qopenmp 
+   FFLAGS += -qopenmp 
+endif
+
+#--- NetCDF ---
+#ifeq ($(IO_TYPE), netcdf)
+#   
+#endif
+#
+#ifeq ($(IO_TYPE), netcdf_bin)
+#   CPPDEFS :=  $(CPPDEFS) -Dncdf
+#endif
+
+### if using parallel I/O, load all 3 libraries.  PIO must be first!
+#ifeq ($(ICE_IOTYPE), pio)
+#   PIO_PATH:=/usr/projects/climate/SHARED_CLIMATE/software/conejo/pio/1.7.2/intel-13.0.1/openmpi-1.6.3/netcdf-3.6.3-parallel-netcdf-1.3.1/include
+#   INCLDIR += -I$(PIO_PATH)
+#   SLIBS   := $(SLIBS) -L$(PIO_PATH) -lpio
+#endif
diff --git a/configuration/scripts/machines/env.freya_gnu b/configuration/scripts/machines/env.freya_gnu
new file mode 100755
index 000000000..b655d6dd0
--- /dev/null
+++ b/configuration/scripts/machines/env.freya_gnu
@@ -0,0 +1,39 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source /opt/modules/default/init/csh # Initialize modules for csh
+ Clear environment
+module rm PrgEnv-intel
+module rm PrgEnv-cray
+module rm PrgEnv-gnu
+module add PrgEnv-gnu
+#module load PrgEnv-intel # Intel compiler
+#module load cray-mpich # MPI (Cray MPICH)
+module add cray-netcdf # NetCDF
+module add cray-hdf5 # HDF5
+#setenv HDF5_USE_FILE_LOCKING FALSE # necessary since data is on an NFS filesystem
+setenv HDF5_USE_FILE_LOCKING FALSE # necessary since data is on an NFS filesystem
+
+endif
+
+setenv ICE_MACHINE_MACHNAME freya
+setenv ICE_MACHINE_MACHINFO "Cray XC50, GNU Xeon Gold 6148 (Skylake) NOT SURE-TILL"
+setenv ICE_MACHINE_ENVNAME gnu
+setenv ICE_MACHINE_ENVINFO "gcc/7.2.0, cray-mpich/7.7.0, cray-netcdf/4.4.1.1.6"
+setenv ICE_MACHINE_MAKE make
+setenv ICE_MACHINE_WKDIR /data/${USER}/cice_original/run/ 
+setenv ICE_MACHINE_INPUTDATA /data/${USER}/cice_original/
+setenv ICE_MACHINE_BASELINE /data/${USER}/cice_original/dbaselines/
+setenv ICE_MACHINE_SUBMIT "qsub"
+setenv ICE_MACHINE_TPNODE 36    # tasks per node
+#setenv ICE_MACHINE_MAXRUNLENGTH 9
+setenv ICE_MACHINE_ACCT P0000000
+setenv ICE_MACHINE_QUEUE "development"
+setenv ICE_MACHINE_BLDTHRDS 18
+setenv ICE_MACHINE_QSTAT "qstat "
diff --git a/configuration/scripts/machines/env.freya_intel b/configuration/scripts/machines/env.freya_intel
new file mode 100755
index 000000000..dcbc1f8ba
--- /dev/null
+++ b/configuration/scripts/machines/env.freya_intel
@@ -0,0 +1,38 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source /opt/modules/default/init/csh # Initialize modules for csh
+# Clear environment
+module rm PrgEnv-intel
+module rm PrgEnv-cray
+module rm PrgEnv-gnu
+module add PrgEnv-intel
+#module load PrgEnv-intel # Intel compiler
+#module load cray-mpich # MPI (Cray MPICH)
+module add cray-netcdf # NetCDF
+module add cray-hdf5 # HDF5
+#setenv HDF5_USE_FILE_LOCKING FALSE # necessary since data is on an NFS filesystem
+
+endif
+
+setenv ICE_MACHINE_MACHNAME freya
+setenv ICE_MACHINE_MACHINFO "Cray XC50, Intel Xeon Gold 6148 (Skylake) NOT SURE-TILL"
+setenv ICE_MACHINE_ENVNAME intel
+setenv ICE_MACHINE_ENVINFO "Intel 18.0.0.128, cray-mpich/7.7.0, cray-netcdf/4.4.1.1.6"
+setenv ICE_MACHINE_MAKE make
+setenv ICE_MACHINE_WKDIR /data/${USER}/cice_original/run/ 
+setenv ICE_MACHINE_INPUTDATA /data/${USER}/cice_original/
+setenv ICE_MACHINE_BASELINE /data/${USER}/cice_original/dbaselines/
+setenv ICE_MACHINE_SUBMIT "qsub"
+setenv ICE_MACHINE_TPNODE 36     # tasks per node
+#setenv ICE_MACHINE_MAXRUNLENGTH 9
+setenv ICE_MACHINE_ACCT P0000000
+setenv ICE_MACHINE_QUEUE "development"
+setenv ICE_MACHINE_BLDTHRDS 18
+setenv ICE_MACHINE_QSTAT "qstat "