pipeline: compile preproc executables on login nodes

In order to adapt pipelines to work from a read-only NFS space we need the
pre-processing step for tc4 to be [optionally] broken into two steps: a
compile step and a data generation step.

Changes:
- Added the target "executables to tc4/Makefile, so only compilation occurs
  when invoked. The "all" target still compiles if needed.
- Added target "preproc-compile" to .testing/Makefile whcih invokes the
  "executables" target of tc4/Makefile. The target "preproc" still compiles
  if needed.
- Added `make-preproc-compile` to the gitlab pipeline prior to submitting
  the "make test" job to the compute nodes.
- I also disconnected the "clean.stats" target from "clean.preproc" since
  the latter removes the pre-processing executables too and given that we
  can't re-compile those and re-create the stats from the same location
  it makes sense to keep those clean steps separated.

.gitlab-ci.yml

@@ -182,6 +182,7 @@ actions:gnu:
     - cd .testing
     - module unload PrgEnv-pgi PrgEnv-intel PrgEnv-gnu darshan ; module load PrgEnv-gnu ; module unload netcdf gcc ; module load gcc/7.3.0 cray-hdf5 cray-netcdf
     - make -s -j
+    - make preproc-compile -s -j
     - echo -e "\e[0Ksection_end:`date +%s`:compile\r\e[0K"
     - (echo '#!/bin/bash';echo 'make MPIRUN="srun -mblock --exclusive" test -s -j') > job.sh
     - sbatch --clusters=c3,c4 --nodes=5 --time=0:05:00 --account=gfdl_o --qos=debug --job-name=MOM6.gnu.testing --output=log.$CI_JOB_ID --wait job.sh || ( cat log.$CI_JOB_ID ; exit 911 )
@@ -201,6 +202,7 @@ actions:intel:
     - cd .testing
     - module unload PrgEnv-pgi PrgEnv-intel PrgEnv-gnu darshan; module load PrgEnv-intel; module unload netcdf intel; module load intel/18.0.6.288 cray-hdf5 cray-netcdf
     - make -s -j
+    - make preproc-compile -s -j
     - echo -e "\e[0Ksection_end:`date +%s`:compile\r\e[0K"
     - (echo '#!/bin/bash';echo 'make MPIRUN="srun -mblock --exclusive" test -s -j') > job.sh
     - sbatch --clusters=c3,c4 --nodes=5 --time=0:05:00 --account=gfdl_o --qos=debug --job-name=MOM6.intel.testing --output=log.$CI_JOB_ID --wait job.sh || ( cat log.$CI_JOB_ID ; exit 911 )

.testing/Makefile

@@ -523,6 +523,8 @@ $(foreach c,$(CONFIGS),$(eval $(call CONFIG_DIM_RULE,$(c))))
 .PHONY: preproc
 preproc: tc4/Makefile
 	cd tc4 && $(MAKE) LAUNCHER="$(MPIRUN)"
+preproc-compile: tc4/Makefile
+	cd tc4 && $(MAKE) executables
 
 tc4/Makefile: tc4/configure tc4/Makefile.in
 	cd $(@D) && ./configure || (cat config.log && false)
@@ -795,7 +797,7 @@ clean.build:
 
 
 .PHONY: clean.stats
-clean.stats: clean.preproc
+clean.stats:
 	@[ $$(basename $$(pwd)) = .testing ]
 	rm -rf work results
 

.testing/tc4/Makefile.in

@@ -16,6 +16,7 @@ OUT = ocean_hgrid.nc topog.nc temp_salt_ic.nc sponge.nc
 
 # Program output
 all: ocean_hgrid.nc temp_salt_ic.nc
+executables: gen_data gen_grid
 
 ocean_hgrid.nc: gen_grid
 	$(LAUNCHER) ./gen_grid