Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update crtm(-fix), wgrib2 #510

Merged
merged 3 commits into from
Jan 30, 2025
Merged

Update crtm(-fix), wgrib2 #510

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c9bc27c
wgrib2: add cmake builder/v3.5.0
AlexanderRichert-NOAA Jan 30, 2025
0a87ef0
crtm: add v3.1.1-build1 (#48451)
AlexanderRichert-NOAA Jan 15, 2025
1c3d3d5
crtm: add 3.1.1 fix dep
AlexanderRichert-NOAA Jan 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions var/spack/repos/builtin/packages/crtm-fix/package.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

Expand All @@ -18,6 +17,7 @@ class CrtmFix(Package):
"BenjaminTJohnson", "edwardhartnett", "AlexanderRichert-NOAA", "Hang-Lei-NOAA", "climbfuji"
)

version("3.1.1.2", sha256="c2e289f690d82a3aa82d2239cbb567cd514fa0f476a8b498ceba11670685ca66")
version(
"2.4.0.1_emc", sha256="6e4005b780435c8e280d6bfa23808d8f12609dfd72f77717d046d4795cac0457"
)
Expand All @@ -27,6 +27,7 @@ class CrtmFix(Package):
variant("big_endian", default=True, description="Install big_endian fix files")
variant("little_endian", default=False, description="Install little endian fix files")
variant("netcdf", default=True, description="Install netcdf fix files")
variant("testfiles", default=False, description="Install test files", when="@3:")

conflicts("+big_endian", when="+little_endian", msg="big_endian and little_endian conflict")

Expand All @@ -52,7 +53,11 @@ def install(self, spec, prefix):

fix_files = []
for d in endian_dirs:
fix_files = fix_files + find(".", "*/{}/*".format(d))
fix_files = fix_files + find(".", "*/{}/*".format(d), recursive=False)
fix_files = fix_files + find(".", "*/*/{}/*".format(d), recursive=False)
if self.spec.satisfies("~testfiles"):
fix_files = [f for f in fix_files if "/fix/test_data/" not in f]
fix_files = [f for f in fix_files if os.path.isfile(f)]

# Big_Endian amsua_metop-c.SpcCoeff.bin is incorrect
# Little_Endian amsua_metop-c_v2.SpcCoeff.bin is what it's supposed to be.
Expand Down
9 changes: 7 additions & 2 deletions var/spack/repos/builtin/packages/crtm/package.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ class Crtm(CMakePackage):
depends_on("[email protected]_emc", when="@2.3.0 +fix")
depends_on("[email protected]_emc", when="@=2.4.0 +fix")
depends_on("[email protected]_emc", when="@2.4.0.1 +fix")
depends_on("[email protected]", when="@3.1.1 +fix")

depends_on("ecbuild", type=("build"), when="@v2.3-jedi.4")
depends_on("ecbuild", type=("build"), when="@v2.4-jedi.1")
Expand All @@ -51,6 +52,9 @@ class Crtm(CMakePackage):

license("CC0-1.0")

version(
"v3.1.1-build1", sha256="1ed49e594da5d3769cbaa52cc7fc19c1bb0325ee6324f6057227c31e2d95ca67"
)
version(
"v3.1.0-skylabv8",
sha256="a475c8a444072aef1e8c2babba3d12f13ab0fb6c7ecab88edad57130839e29ff",
Expand Down Expand Up @@ -81,8 +85,9 @@ class Crtm(CMakePackage):
depends_on("fortran", type="build") # generated

def url_for_version(self, version):
if self.spec.satisfies("@v3") or version >= Version("3.0.0"):
return f"https://github.com/JCSDA/crtmv3/archive/refs/tags/{version}.tar.gz"
if version > Version("v3") or version >= Version("3"):
fmtversion = str(version).replace("-build", "+build")
return f"https://github.com/JCSDA/CRTMv3/archive/refs/tags/{fmtversion}.tar.gz"
else:
return f"https://github.com/JCSDA/crtm/archive/refs/tags/{version}.tar.gz"

Expand Down
221 changes: 144 additions & 77 deletions var/spack/repos/builtin/packages/wgrib2/package.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,32 +1,68 @@
# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

import os
import re

import spack.build_systems.cmake
import spack.build_systems.makefile
from spack.package import *


class Wgrib2(MakefilePackage):
variant_map_common = {
"netcdf3": "USE_NETCDF3",
"netcdf4": "USE_NETCDF4",
"mysql": "USE_MYSQL",
"udf": "USE_UDF",
"regex": "USE_REGEX",
"tigge": "USE_TIGGE",
"proj4": "USE_PROJ4",
"aec": "USE_AEC",
"g2c": "USE_G2CLIB",
"png": "USE_PNG",
"jasper": "USE_JASPER",
"openmp": "USE_OPENMP",
"wmo_validation": "USE_WMO_VALIDATION",
"ipolates": "USE_IPOLATES",
"disable_alarm": "DISABLE_ALARM",
"fortran_api": "MAKE_FTN_API",
"disable_stat": "DISABLE_STAT",
"openjpeg": "USE_OPENJPEG",
}

variant_map_makefile = {"spectral": "USE_SPECTRAL"}

variant_map_cmake = {
"netcdf": "USE_NETCDF",
"disable_timezone": "DISABLE_TIMEZONE",
"enable_docs": "ENABLE_DOCS",
}

variant_map_makefile.update(variant_map_common)
variant_map_cmake.update(variant_map_common)


class Wgrib2(MakefilePackage, CMakePackage):
"""Utility for interacting with GRIB2 files"""

homepage = "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2"
url = "https://www.ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz.v2.0.8"
git = "https://github.com/NOAA-EMC/wgrib2"

maintainers("t-brown", "AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett")

version(
"3.1.1",
sha256="9236f6afddad76d868c2cfdf5c4227f5bdda5e85ae40c18bafb37218e49bc04a",
extension="tar.gz",
maintainers(
"AlysonStahl-NOAA", "t-brown", "AlexanderRichert-NOAA", "Hang-Lei-NOAA", "edwardhartnett"
)
version(
"3.1.0",
sha256="5757ef9016b19ae87491918e0853dce2d3616b14f8c42efe3b2f41219c16b78f",
extension="tar.gz",

build_system(
conditional("cmake", when="@3.2:"), conditional("makefile", when="@:3.1"), default="cmake"
)

version("develop", branch="develop")
version("3.5.0", sha256="b27b48228442a08bddc3d511d0c6335afca47252ae9f0e41ef6948f804afa3a1")
version("3.4.0", sha256="ecbce2209c09bd63f1bca824f58a60aa89db6762603bda7d7d3fa2148b4a0536")
version("3.3.0", sha256="010827fba9c31f05807e02375240950927e9e51379e1444388153284f08f58e2")
version("3.2.0", sha256="ac3ace77a32c2809cbc4538608ad64aabda2c9c1e44e7851da79764a6eb3c369")
version("3.1.1", sha256="9236f6afddad76d868c2cfdf5c4227f5bdda5e85ae40c18bafb37218e49bc04a")
version("3.1.0", sha256="5757ef9016b19ae87491918e0853dce2d3616b14f8c42efe3b2f41219c16b78f")
version(
"2.0.8",
sha256="5e6a0d6807591aa2a190d35401606f7e903d5485719655aea1c4866cc2828160",
Expand All @@ -38,27 +74,48 @@ class Wgrib2(MakefilePackage):
extension="tar.gz",
)

depends_on("c", type="build") # generated
depends_on("fortran", type="build") # generated
def url_for_version(self, version):
if version >= Version("3.2.0"):
url_fmt = "https://github.com/NOAA-EMC/wgrib2/archive/refs/tags/v{0}.tar.gz"
else:
url_fmt = "https://www.ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz.v{0}"
return url_fmt.format(version)

variant("netcdf3", default=True, description="Link in netcdf3 library to write netcdf3 files")
variant(
"netcdf4", default=False, description="Link in netcdf4 library to write netcdf3/4 files"
"netcdf3",
default=True,
description="Link in netcdf3 library to write netcdf3 files",
when="@:3.1",
)
variant(
"ipolates",
default="3",
description="Use to interpolate to new grids (0 = OFF, 1 = ip, 3 = ip2)",
values=("0", "1", "3"),
"netcdf4",
default=False,
description="Link in netcdf4 library to write netcdf3/4 files",
when="@:3.3",
)
variant("spectral", default=False, description="Spectral interpolation in -new_grid")
variant(
"netcdf",
default=False,
description="Link in netcdf4 library to write netcdf3/4 files",
when="@3.4:",
)
variant("ipolates", default=False, description="Use to interpolate to new grids")
variant(
"spectral", default=False, description="Spectral interpolation in -new_grid", when="@:3.1"
)
# Not currently working for @3.2:
variant(
"fortran_api",
default=True,
description="Make wgrib2api which allows fortran code to read/write grib2",
)
# Not currently working for @3.2:
# variant("lib", default=True, description="Build library", when="@3.2:")
variant(
"mysql", default=False, description="Link in interface to MySQL to write to mysql database"
"mysql",
default=False,
description="Link in interface to MySQL to write to mysql database",
when="@:3.1",
)
variant(
"udf",
Expand All @@ -67,11 +124,21 @@ class Wgrib2(MakefilePackage):
)
variant("regex", default=True, description="Use regular expression library (POSIX-2)")
variant("tigge", default=True, description="Ability for TIGGE-like variable names")
variant("proj4", default=False, description="The proj4 library is used to verify gctpc.")
variant(
"proj4",
default=False,
description="The proj4 library is used to verify gctpc.",
when="@:3.1",
)
variant(
"aec", default=True, description="Use the libaec library for packing with GRIB2 template"
)
variant("g2c", default=False, description="Include NCEP g2clib (mainly for testing purposes)")
variant(
"g2c",
default=False,
description="Include NCEP g2clib (mainly for testing purposes)",
when="@:3.1",
)
variant(
"disable_timezone", default=False, description="Some machines do not support timezones"
)
Expand All @@ -84,62 +151,73 @@ class Wgrib2(MakefilePackage):
variant("jasper", default=True, description="JPEG compression using Jasper")
variant("openmp", default=True, description="OpenMP parallelization")
variant("wmo_validation", default=False, description="WMO validation")
# variant("shared", default=False, description="Enable shared library", when="+lib")
variant("disable_stat", default=False, description="Disable POSIX feature", when="@:3.1")
variant("openjpeg", default=False, description="Enable OpenJPEG")
variant(
"enable_docs", default=False, description="Build doxygen documentation", when="@3.4.0:"
)

conflicts("+netcdf3", when="+netcdf4")
conflicts("+netcdf3", when="+netcdf")
conflicts("+openmp", when="%apple-clang")

depends_on("wget", type=("build"), when="+netcdf4")
# makefile behavior with shell commands/character escapes breaks with [email protected]:
depends_on("gmake@:4.2")

variant_map = {
"netcdf3": "USE_NETCDF3",
"netcdf4": "USE_NETCDF4",
"spectral": "USE_SPECTRAL",
"mysql": "USE_MYSQL",
"udf": "USE_UDF",
"regex": "USE_REGEX",
"tigge": "USE_TIGGE",
"proj4": "USE_PROJ4",
"aec": "USE_AEC",
"g2c": "USE_G2CLIB",
"png": "USE_PNG",
"jasper": "USE_JASPER",
"openmp": "USE_OPENMP",
"wmo_validation": "USE_WMO_VALIDATION",
"ipolates": "USE_IPOLATES",
"disable_timezone": "DISABLE_TIMEZONE",
"disable_alarm": "DISABLE_ALARM",
"fortran_api": "MAKE_FTN_API",
}

# Disable parallel build
parallel = False
depends_on("[email protected]:", when="@3.5: +ipolates")
depends_on("lapack", when="@3.5: +ipolates")
depends_on("[email protected]:", when="@3.2: +aec")
depends_on("netcdf-c", when="@3.2: +netcdf4")
depends_on("jasper@:2", when="@3.2:3.4 +jasper")
depends_on("g2c", when="@3.5: +jasper")
depends_on("zlib-api", when="@3.2: +png")
depends_on("libpng", when="@3.2: +png")
depends_on("openjpeg", when="@3.2:3.4 +openjpeg")
depends_on("g2c +openjpeg", when="@3.5: +openjpeg")
requires("^[email protected]:", when="@3.5: ^g2c")

@when("@:2 ^[email protected]:")
def patch(self):
filter_file(r"\\#define", "#define", "makefile")

# Use Spack compiler wrapper flags
def inject_flags(self, name, flags):
spec = self.spec
if name == "cflags":
if spec.satisfies("%apple-clang"):
if self.spec.compiler.name == "apple-clang":
flags.append("-Wno-error=implicit-function-declaration")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@AlexanderRichert-NOAA This seems to be a bug, or at least it is causing a build error? https://github.com/JCSDA/spack-stack/actions/runs/13077825823/job/36494196074?pr=1486

Maybe the compiler name is still clang, not apple-clang, even if the spec is apple-clang?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did you already take care of this? I can't test it, so feel free to do whatever makes work...

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My first attempt failed, I will need to look again. Unlikely I will be able to finish this today, other things are more urgent.


# When mixing Clang/gfortran need to link to -lgfortran
# Find this by searching for gfortran/../lib
if spec.satisfies("%apple-clang") or spec.satisfies("%clang"):
if self.spec.compiler.name in ["apple-clang", "clang"]:
if "gfortran" in self.compiler.fc:
output = Executable(self.compiler.fc)("-###", output=str, error=str)
libdir = re.search("--libdir=(.+?) ", output).group(1)
flags.append("-L{}".format(libdir))

return (flags, None, None)


class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder):
# Disable parallel build
parallel = False

def cmake_args(self):
args = [self.define_from_variant(variant_map_cmake[k], k) for k in variant_map_cmake]
# args.append(self.define_from_variant("BUILD_LIB", "lib"))
# args.append(self.define_from_variant("BUILD_SHARED_LIB", "shared"))

return args


class MakefileBuilder(spack.build_systems.makefile.MakefileBuilder):
# Disable parallel build
parallel = False

flag_handler = inject_flags

def url_for_version(self, version):
url = "https://www.ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz.v{}"
return url.format(version)

def edit(self, spec, prefix):
def edit(self, pkg, spec, prefix):
makefile = FileFilter("makefile")

# ifort no longer accepts -openmp
Expand All @@ -151,31 +229,20 @@ def edit(self, spec, prefix):
if spec.satisfies("%clang") or spec.satisfies("%apple-clang"):
makefile.filter(r"--fast-math", "-ffast-math")

for variant_name, makefile_option in self.variant_map.items():
for variant_name, makefile_option in variant_map_makefile.items():
value = int(spec.variants[variant_name].value)
makefile.filter(r"^%s=.*" % makefile_option, "{}={}".format(makefile_option, value))

def setup_build_environment(self, env):
spec = self.spec
if spec.satisfies("%oneapi") or spec.satisfies("%intel"):
if self.spec.compiler.name in ["oneapi", "intel"]:
comp_sys = "intel_linux"
elif spec.satisfies("%gcc") or spec.satisfies("%clang") or spec.satisfies("%apple-clang"):
elif self.spec.compiler.name in ["gcc", "clang", "apple-clang"]:
comp_sys = "gnu_linux"

env.set("COMP_SYS", comp_sys)

def build(self, spec, prefix):
# Get source files for netCDF4 builds
if self.spec.satisfies("+netcdf4"):
with working_dir(self.build_directory):
os.system(
"wget https://downloads.unidata.ucar.edu/netcdf-c/4.8.1/netcdf-c-4.8.1.tar.gz"
)
os.system(
"wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.1/src/hdf5-1.12.1.tar.gz"
)

make()
def build(self, pkg, spec, prefix):
make("-j1")

# Move wgrib2 executable to a tempoary directory
mkdir("install")
Expand All @@ -190,7 +257,7 @@ def build(self, spec, prefix):
makefile = FileFilter("makefile")

# Disable all options
for variant_name, makefile_option in self.variant_map.items():
for variant_name, makefile_option in variant_map_makefile.items():
value = 0
makefile.filter(
r"^%s=.*" % makefile_option, "{}={}".format(makefile_option, value)
Expand All @@ -199,13 +266,13 @@ def build(self, spec, prefix):
# Need USE_REGEX in addition to MAKE_FTN_API to build lib
makefile.filter(r"^MAKE_FTN_API=.*", "MAKE_FTN_API=1")
makefile.filter(r"^USE_REGEX=.*", "USE_REGEX=1")
make("lib")
make("lib", "-j1")
mkdir(join_path("install", "lib"))
mkdir(join_path("install", "include"))

move(join_path("lib", "libwgrib2.a"), join_path("install", "lib"))
move(join_path("lib", "wgrib2api.mod"), join_path("install", "include"))
move(join_path("lib", "wgrib2lowapi.mod"), join_path("install", "include"))

def install(self, spec, prefix):
def install(self, pkg, spec, prefix):
install_tree("install/", prefix)
Loading