Add oneAPI config for Narwhal, add GNU config for Atlantis, remove Sa…

…ndy platform; make `python` variant for `esmf` configurable (#1318) - Add a [email protected] config for Narwhal (using icx, icpx, ifort) - Add a [email protected] config for Atlantis - Remove Sandy as a supported platform - Update submodule pointer for spack to bring in [email protected] (official release) and replace [email protected] with [email protected] in all configs/** - Make python variant for esmf configurable
JCSDA · Oct 21, 2024 · 55967c1 · 55967c1
1 parent 7538aa0
commit 55967c1
Show file tree

Hide file tree

Showing 22 changed files with 127 additions and 167 deletions.
diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml
@@ -136,6 +136,8 @@ modules:
           ^[email protected]+debug snapshot=b04: 'esmf-8.7.0b04-debug'
           ^[email protected]~debug snapshot=b11: 'esmf-8.7.0b11'
           ^[email protected]+debug snapshot=b11: 'esmf-8.7.0b11-debug'
+          ^[email protected]~debug snapshot=none: 'esmf-8.7.0'
+          ^[email protected]+debug snapshot=none: 'esmf-8.7.0-debug'
       openmpi:
         environment:
           set:

diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml
@@ -138,6 +138,8 @@ modules:
           ^[email protected]+debug snapshot=b04: 'esmf-8.7.0b04-debug'
           ^[email protected]~debug snapshot=b11: 'esmf-8.7.0b11'
           ^[email protected]+debug snapshot=b11: 'esmf-8.7.0b11-debug'
+          ^[email protected]~debug snapshot=none: 'esmf-8.7.0'
+          ^[email protected]+debug snapshot=none: 'esmf-8.7.0-debug'
       openmpi:
         environment:
           set:

diff --git a/configs/common/packages b/configs/common/packages
diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml
@@ -54,17 +54,19 @@ packages:
   # Also, check the acorn and derecho site configs which have esmf modifications.
   esmf:
     require:
-      - '~xerces ~pnetcdf +shared +external-parallelio +python'
-      - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11']
-      - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"']
-        when: "%intel"
-        message: "Extra ESMF compile options for Intel"
-      #- any_of: ['']
-      #  when: "%gcc"
-      #  message: "Extra ESMF compile options for GCC"
-      #- any_of: ['']
-      #  when: "%apple-clang"
-      #  message: "Extra ESMF compile options for GCC"
+    - '~xerces ~pnetcdf +shared +external-parallelio'
+    - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0 snapshot=none']
+    - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"']
+      when: "%intel"
+      message: "Extra ESMF compile options for Intel"
+    #- any_of: ['']
+    #  when: "%gcc"
+    #  message: "Extra ESMF compile options for GCC"
+    #- any_of: ['']
+    #  when: "%apple-clang"
+    #  message: "Extra ESMF compile options for GCC"
+    prefer:
+    - '+python'
   fckit:
     require: '@0.11.0 +eckit'
   fftw:

diff --git a/configs/sites/tier1/sandy/mirrors.yaml → configs/sites/tier1/atlantis/mirrors.yaml b/configs/sites/tier1/sandy/mirrors.yaml → configs/sites/tier1/atlantis/mirrors.yaml
@@ -1,15 +1,15 @@
 mirrors:
   local-source:
     fetch:
-      url: file:///gpfs/fs1/neptune/spack-stack/source-cache
+      url: file:///neptune_diagnostics/spack-stack/source-cache/
       access_pair:
       - null
       - null
       access_token: null
       profile: null
       endpoint_url: null
     push:
-      url: file:///gpfs/fs1/neptune/spack-stack/source-cache
+      url: file:///neptune_diagnostics/spack-stack/source-cache/
       access_pair:
       - null
       - null

diff --git a/configs/sites/tier1/atlantis/packages_gcc.yaml b/configs/sites/tier1/atlantis/packages_gcc.yaml
@@ -0,0 +1,13 @@
+packages:
+  all:
+    compiler:: [[email protected]]
+    providers:
+      mpi:: [[email protected]]
+  mpi:
+    buildable: False
+  openmpi:
+    buildable: False
+    externals:
+    - spec: [email protected]%gcc@=11.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm
+      modules:
+      - openmpi/mlnx/gcc/64/4.1.5a1
diff --git a/configs/sites/tier1/atlantis/packages_intel.yaml b/configs/sites/tier1/atlantis/packages_intel.yaml
@@ -15,6 +15,6 @@ packages:
   intel-oneapi-mkl:
     externals:
     - spec: [email protected]
+      prefix: /cm/shared/apps/intel/oneapi
       modules:
       -  mkl/2022.1.0
-      prefix: /cm/shared/apps/intel/oneapi
diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml
@@ -21,6 +21,30 @@ compilers::
         set:
           CRAYPE_LINK_TYPE: 'dynamic'
       extra_rpaths: []
+  - compiler:
+      spec: [email protected]
+      paths:
+        cc: cc
+        cxx: CC
+        f77: ftn
+        fc: ftn
+      flags: {}
+      operating_system: sles15
+      modules:
+      - PrgEnv-intel/8.4.0
+      - intel/2024.2
+      - cray-libsci/23.05.1.4
+      - libfabric/1.12.1.2.2.1
+      environment:
+        prepend_path:
+          PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin'
+          CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include'
+          LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64'
+        append_path:
+          CPATH: '/opt/intel/oneapi_2024.2.0.634/compiler/2024.2/opt/compiler/include/intel64'
+        set:
+          CRAYPE_LINK_TYPE: 'dynamic'
+      extra_rpaths: []
   - compiler:
       spec: [email protected]
       paths:

diff --git a/configs/sites/tier1/narwhal/mirrors.yaml b/configs/sites/tier1/narwhal/mirrors.yaml
@@ -1,15 +1,15 @@
 mirrors:
   local-source:
     fetch:
-      url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache
+      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
       access_pair:
       - null
       - null
       access_token: null
       profile: null
       endpoint_url: null
     push:
-      url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache
+      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
       access_pair:
       - null
       - null

diff --git a/configs/sites/tier1/narwhal/packages_oneapi.yaml b/configs/sites/tier1/narwhal/packages_oneapi.yaml
@@ -0,0 +1,27 @@
+packages:
+  all:
+    compiler:: [[email protected],[email protected]]
+    providers:
+      mpi:: [[email protected]]
+  mpi:
+    buildable: False
+  cray-mpich:
+    externals:
+    - spec: [email protected]%[email protected] ~wrappers
+      modules:
+      - cray-mpich-ucx/8.1.21
+      - craype-network-ucx
+  intel-oneapi-mkl:
+    externals:
+    - spec: [email protected]%[email protected]
+      prefix: /opt/intel/oneapi_2024.2.0.634
+  intel-oneapi-tbb:
+    externals:
+    - spec: [email protected]%[email protected]
+      prefix: /opt/intel/oneapi_2024.2.0.634
+      modules:
+      - tbb/2021.13
+  intel-oneapi-runtime:
+    externals:
+    - spec: [email protected]%[email protected]
+      prefix: /opt/intel/oneapi_2024.2.0.634
diff --git a/configs/sites/tier1/nautilus/mirrors.yaml b/configs/sites/tier1/nautilus/mirrors.yaml
@@ -1,15 +1,15 @@
 mirrors:
   local-source:
     fetch:
-      url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache
+      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
       access_pair:
       - null
       - null
       access_token: null
       profile: null
       endpoint_url: null
     push:
-      url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache
+      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
       access_pair:
       - null
       - null

diff --git a/configs/sites/tier1/sandy/compilers.yaml b/configs/sites/tier1/sandy/compilers.yaml
diff --git a/configs/sites/tier1/sandy/config.yaml b/configs/sites/tier1/sandy/config.yaml
diff --git a/configs/sites/tier1/sandy/modules.yaml b/configs/sites/tier1/sandy/modules.yaml
diff --git a/configs/sites/tier1/sandy/packages.yaml b/configs/sites/tier1/sandy/packages.yaml
diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml
@@ -8,8 +8,8 @@ spack:
   definitions:
   - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi']
   - packages:
-      - neptune-env +espc        ^[email protected].0b11 snapshot=b11
-      - neptune-python-env +xnrl ^[email protected].0b11 snapshot=b11
+      - neptune-env +espc        ^[email protected].0
+      - neptune-python-env +xnrl ^[email protected].0
 
   specs:
     - matrix:

diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml
@@ -14,11 +14,11 @@ spack:
     - jedi-fv3-env
     - jedi-geos-env         ^esmf@=8.6.1
     - jedi-mpas-env
-    - jedi-neptune-env      ^esmf@=8.7.0b11 snapshot=b11
+    - jedi-neptune-env      ^esmf@=8.7.0
     - jedi-ufs-env          ^esmf@=8.6.1
     - jedi-um-env
-    - neptune-env           ^esmf@=8.7.0b11 snapshot=b11
-    - neptune-python-env    ^esmf@=8.7.0b11 snapshot=b11
+    - neptune-env           ^esmf@=8.7.0
+    - neptune-python-env    ^esmf@=8.7.0
     - soca-env
 
     # Various crtm tags (list all to avoid duplicate packages)
@@ -27,7 +27,7 @@ spack:
 
     # Various esmf tags (list all to avoid duplicate packages)
     - esmf@=8.6.1 snapshot=none
-    - esmf@=8.7.0b11 snapshot=b11
+    - esmf@=8.7.0 snapshot=none
 
   specs:
   - matrix:

diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml
@@ -17,12 +17,12 @@ spack:
     - jedi-fv3-env
     - jedi-geos-env         ^esmf@=8.6.1
     - jedi-mpas-env
-    - jedi-neptune-env      ^esmf@=8.7.0b11 snapshot=b11
+    - jedi-neptune-env      ^esmf@=8.7.0
     - jedi-tools-env
     - jedi-ufs-env          ^esmf@=8.6.1
     - jedi-um-env
-    - neptune-env           ^esmf@=8.7.0b11 snapshot=b11
-    - neptune-python-env    ^esmf@=8.7.0b11 snapshot=b11
+    - neptune-env           ^esmf@=8.7.0
+    - neptune-python-env    ^esmf@=8.7.0
     - soca-env
     - ufs-srw-app-env       ^esmf@=8.6.1
     - ufs-weather-model-env ^esmf@=8.6.1
@@ -33,7 +33,7 @@ spack:
 
     # Various esmf tags (list all to avoid duplicate packages)
     - esmf@=8.6.1 snapshot=none
-    - esmf@=8.7.0b11 snapshot=b11
+    - esmf@=8.7.0 snapshot=none
 
     # MADIS for WCOSS2 decoders.
     - [email protected]