From 728dfb67d9247d82b35e021a6d1e2d23120e9eae Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Wed, 10 Apr 2024 14:25:56 +0200 Subject: [PATCH 01/31] bump PartMC submodule version to v2.8.0 --- gitmodules/partmc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitmodules/partmc b/gitmodules/partmc index e73f6534..afe7bed5 160000 --- a/gitmodules/partmc +++ b/gitmodules/partmc @@ -1 +1 @@ -Subproject commit e73f6534160abeda86d6d58ab43ba4dc0934e602 +Subproject commit afe7bed5ac2f0f2044b3d91f74b8158830d0d470 From f42923206c6dccbc73a6d9bade969d58993d4796 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Fri, 12 Apr 2024 14:37:16 +0200 Subject: [PATCH 02/31] add new file to partmc source list --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f2c32f5a..d3a150d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -176,7 +176,7 @@ set(partmclib_SOURCES condense_solver.c aero_state.F90 integer_varray.F90 intege gas_state.F90 coagulation.F90 exact_soln.F90 coagulation_dist.F90 coag_kernel.F90 spec_line.F90 rand.F90 aero_particle.F90 aero_particle_array.F90 mpi.F90 netcdf.F90 aero_info.F90 aero_info_array.F90 nucleate.F90 condense.F90 fractal.F90 chamber.F90 camp_interface.F90 - photolysis.F90 + photolysis.F90 aero_component.F90 ) add_prefix(gitmodules/partmc/src/ partmclib_SOURCES) list(APPEND partmclib_SOURCES src/spec_file_pypartmc.F90 src/sys.F90) From 0213c2aa3118210f553b1720fbe97a11fefa0254 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Fri, 12 Apr 2024 14:51:42 +0200 Subject: [PATCH 03/31] fix preprocessor define logic in parameter files --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d3a150d7..3471225a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -435,7 +435,7 @@ add_dependencies(partmclib ${SUNDIALS_items}) string(CONCAT cmd "import sys;" - "print(''.join([f'#define {line.split()[3]} {line.split()[5]}\\n' for line in sys.stdin if 'parameter ::' in line]))" + "print(''.join([line if line.startswith('#') else f'#define {line.split()[3]} {line.split()[5]}\\n' for line in sys.stdin if 'parameter ::' in line or line.startswith('#')]))" ) include(CheckFunctionExists) foreach(file aero_data;gas_data;output) From 40c9fb5e3374587ca952ae7c9e4fbabb678e6cd2 Mon Sep 17 00:00:00 2001 From: Jeffrey Curtis Date: Fri, 12 Apr 2024 15:13:58 -0500 Subject: [PATCH 04/31] add false for read_aero_weight_classes --- src/aero_dist.F90 | 2 +- src/aero_mode.F90 | 2 +- src/scenario.F90 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/aero_dist.F90 b/src/aero_dist.F90 index 4d729cfb..aa8764a6 100644 --- a/src/aero_dist.F90 +++ b/src/aero_dist.F90 @@ -38,7 +38,7 @@ subroutine f_aero_dist_from_json(ptr_c, aero_data_ptr_c) bind(C) call c_f_pointer(ptr_c, aero_dist) call c_f_pointer(aero_data_ptr_c, aero_data_ptr_f) - call spec_file_read_aero_dist(file, aero_data_ptr_f, aero_dist) + call spec_file_read_aero_dist(file, aero_data_ptr_f, .false., aero_dist) end subroutine subroutine f_aero_dist_n_mode(ptr_c, n_mode) bind(C) diff --git a/src/aero_mode.F90 b/src/aero_mode.F90 index 416eb399..1b514205 100644 --- a/src/aero_mode.F90 +++ b/src/aero_mode.F90 @@ -179,7 +179,7 @@ subroutine f_aero_mode_from_json(ptr_c, aero_data_ptr_c) bind(C) call c_f_pointer(ptr_c, ptr_f) call c_f_pointer(aero_data_ptr_c, aero_data_ptr_f) - call spec_file_read_aero_mode(file, aero_data_ptr_f, ptr_f, eof) + call spec_file_read_aero_mode(file, aero_data_ptr_f, .false., ptr_f, eof) end subroutine diff --git a/src/scenario.F90 b/src/scenario.F90 index d7e18527..a832300e 100644 --- a/src/scenario.F90 +++ b/src/scenario.F90 @@ -36,7 +36,7 @@ subroutine f_scenario_from_json(gas_ptr_c, aer_ptr_c, ptr_c) bind(C) call c_f_pointer(gas_ptr_c, gas_ptr_f) call c_f_pointer(aer_ptr_c, aer_ptr_f) call c_f_pointer(ptr_c, ptr_f) - call spec_file_read_scenario(file, gas_ptr_f, aer_ptr_f, ptr_f) + call spec_file_read_scenario(file, gas_ptr_f, aer_ptr_f, .false., ptr_f) end subroutine subroutine f_scenario_loss_rate( & From 38d110ec513bf98d3c24bd536228cd29067212bf Mon Sep 17 00:00:00 2001 From: Jeffrey Curtis Date: Fri, 12 Apr 2024 15:14:30 -0500 Subject: [PATCH 05/31] deal with int64 for ids --- src/aero_state.F90 | 6 +++--- src/aero_state.hpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/aero_state.F90 b/src/aero_state.F90 index 24e220e9..72991192 100644 --- a/src/aero_state.F90 +++ b/src/aero_state.F90 @@ -495,8 +495,8 @@ subroutine f_aero_state_add_aero_dist_sample(ptr_c, ptr_aero_data_c, & call c_f_pointer(ptr_aero_dist_c,ptr_aero_dist_f) call aero_state_add_aero_dist_sample(ptr_f, ptr_aero_data_f, & - ptr_aero_dist_f, sample_prop, create_time, LOGICAL(allow_doubling), & - logical(allow_halving), n_part_add) + ptr_aero_dist_f, sample_prop, 1.0d0, create_time, LOGICAL(allow_doubling), & + logical(allow_halving), n_part_add) end subroutine @@ -555,7 +555,7 @@ subroutine f_aero_state_zero(ptr_c) bind(C) subroutine f_aero_state_ids(ptr_c, ids, n_parts) bind(C) type(c_ptr), intent(in) :: ptr_c integer(c_int), intent(in) :: n_parts - integer(c_int), intent(out) :: ids(n_parts) + integer(c_int64_t), intent(out) :: ids(n_parts) type(aero_state_t), pointer :: ptr_f => null() call c_f_pointer(ptr_c, ptr_f) diff --git a/src/aero_state.hpp b/src/aero_state.hpp index c99f90c9..a3c86399 100644 --- a/src/aero_state.hpp +++ b/src/aero_state.hpp @@ -178,7 +178,7 @@ extern "C" void f_aero_state_zero( extern "C" void f_aero_state_ids( const void *ptr_c, - int *ids, + int64_t *ids, const int *n_parts ) noexcept; @@ -480,7 +480,7 @@ struct AeroState { self.ptr.f_arg(), &len ); - std::valarray ids(len); + std::valarray ids(len); f_aero_state_ids( self.ptr.f_arg(), From 9bb1eb8670bbde848856bef1a09ee425f921d492 Mon Sep 17 00:00:00 2001 From: Jeffrey Curtis Date: Fri, 12 Apr 2024 15:15:07 -0500 Subject: [PATCH 06/31] first pass at dealing with scalars becoming arrays --- src/aero_particle.F90 | 34 ++++++++++++++++++++++------------ src/aero_particle.hpp | 30 ++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/src/aero_particle.F90 b/src/aero_particle.F90 index 31ed8e9d..9fe4657c 100644 --- a/src/aero_particle.F90 +++ b/src/aero_particle.F90 @@ -389,12 +389,14 @@ subroutine f_aero_particle_set_vols( & subroutine f_aero_particle_absorb_cross_sect( & aero_particle_ptr_c, & - absorb_cross_sect & + absorb_cross_sect, & + n_wavelengths & ) bind(C) type(aero_particle_t), pointer :: aero_particle_ptr_f => null() type(c_ptr), intent(in) :: aero_particle_ptr_c - real(c_double), intent(out) :: absorb_cross_sect + integer(c_int), intent(in) :: n_wavelengths + real(c_double), dimension(n_wavelengths), intent(out) :: absorb_cross_sect call c_f_pointer(aero_particle_ptr_c, aero_particle_ptr_f) @@ -404,12 +406,14 @@ subroutine f_aero_particle_absorb_cross_sect( & subroutine f_aero_particle_scatter_cross_sect( & aero_particle_ptr_c, & - scatter_cross_sect & + scatter_cross_sect, & + n_wavelengths & ) bind(C) type(aero_particle_t), pointer :: aero_particle_ptr_f => null() type(c_ptr), intent(in) :: aero_particle_ptr_c - real(c_double), intent(out) :: scatter_cross_sect + integer(c_int), intent(in) :: n_wavelengths + real(c_double), dimension(n_wavelengths), intent(out) :: scatter_cross_sect call c_f_pointer(aero_particle_ptr_c, aero_particle_ptr_f) @@ -419,12 +423,14 @@ subroutine f_aero_particle_scatter_cross_sect( & subroutine f_aero_particle_asymmetry( & aero_particle_ptr_c, & - asymmetry & + asymmetry, & + n_wavelengths & ) bind(C) type(aero_particle_t), pointer :: aero_particle_ptr_f => null() type(c_ptr), intent(in) :: aero_particle_ptr_c - real(c_double), intent(out) :: asymmetry + integer(c_int), intent(in) :: n_wavelengths + real(c_double), dimension(n_wavelengths), intent(out) :: asymmetry call c_f_pointer(aero_particle_ptr_c, aero_particle_ptr_f) @@ -475,7 +481,7 @@ subroutine f_aero_particle_n_orig_part( & call c_f_pointer(aero_particle_ptr_c, aero_particle_ptr_f) - n_orig_part = aero_particle_ptr_f%n_orig_part + n_orig_part = 1 !aero_particle_ptr_f%n_orig_part end subroutine @@ -486,7 +492,7 @@ subroutine f_aero_particle_id( & type(aero_particle_t), pointer :: aero_particle_ptr_f => null() type(c_ptr), intent(in) :: aero_particle_ptr_c - integer(c_int), intent(out) :: id + integer(c_int64_t), intent(out) :: id call c_f_pointer(aero_particle_ptr_c, aero_particle_ptr_f) @@ -496,12 +502,14 @@ subroutine f_aero_particle_id( & subroutine f_aero_particle_refract_shell( & aero_particle_ptr_c, & - refract_shell & + refract_shell, & + n_wavelengths & ) bind(C) type(aero_particle_t), pointer :: aero_particle_ptr_f => null() type(c_ptr), intent(in) :: aero_particle_ptr_c - complex(c_double_complex), intent(out) :: refract_shell + integer(c_int), intent(in) :: n_wavelengths + complex(c_double_complex), dimension(n_wavelengths), intent(out) :: refract_shell call c_f_pointer(aero_particle_ptr_c, aero_particle_ptr_f) @@ -511,12 +519,14 @@ subroutine f_aero_particle_refract_shell( & subroutine f_aero_particle_refract_core( & aero_particle_ptr_c, & - refract_core & + refract_core, & + n_wavelengths & ) bind(C) type(aero_particle_t), pointer :: aero_particle_ptr_f => null() type(c_ptr), intent(in) :: aero_particle_ptr_c - complex(c_double_complex), intent(out) :: refract_core + integer(c_int), intent(in) :: n_wavelengths + complex(c_double_complex), dimension(n_wavelengths), intent(out) :: refract_core call c_f_pointer(aero_particle_ptr_c, aero_particle_ptr_f) diff --git a/src/aero_particle.hpp b/src/aero_particle.hpp index f02458ff..344a516f 100644 --- a/src/aero_particle.hpp +++ b/src/aero_particle.hpp @@ -36,15 +36,15 @@ extern "C" void f_aero_particle_crit_diameter(const void *aero_particle_ptr, con extern "C" void f_aero_particle_coagulate(const void *aero_particle_1_ptr, const void *aero_particle_2_ptr, void *new_particle_ptr) noexcept; extern "C" void f_aero_particle_zero(void *aero_particle_ptr, const void *aero_data_ptr) noexcept; extern "C" void f_aero_particle_set_vols(void *aero_particle_ptr, const int *vol_size, const void *volumes) noexcept; -extern "C" void f_aero_particle_absorb_cross_sect(const void *aero_particle_ptr, double *val) noexcept; -extern "C" void f_aero_particle_scatter_cross_sect(const void *aero_particle_ptr, double *val) noexcept; -extern "C" void f_aero_particle_asymmetry(const void *aero_particle_ptr, double *val) noexcept; +extern "C" void f_aero_particle_absorb_cross_sect(const void *aero_particle_ptr, double *val, const int *arr_size) noexcept; +extern "C" void f_aero_particle_scatter_cross_sect(const void *aero_particle_ptr, double *val, const int *arr_size) noexcept; +extern "C" void f_aero_particle_asymmetry(const void *aero_particle_ptr, double *val, const int *arr_size) noexcept; extern "C" void f_aero_particle_greatest_create_time(const void *aero_particle_ptr, double *val) noexcept; extern "C" void f_aero_particle_least_create_time(const void *aero_particle_ptr, double *val) noexcept; extern "C" void f_aero_particle_n_orig_part(const void *aero_particle_ptr, void *arr_data, const int *arr_size) noexcept; extern "C" void f_aero_particle_id(const void *aero_particle_ptr, int *val) noexcept; -extern "C" void f_aero_particle_refract_shell(const void *aero_particle_ptr, std::complex *val) noexcept; -extern "C" void f_aero_particle_refract_core(const void *aero_particle_ptr, std::complex *val) noexcept; +extern "C" void f_aero_particle_refract_shell(const void *aero_particle_ptr, std::complex *val, const int *arr_size) noexcept; +extern "C" void f_aero_particle_refract_core(const void *aero_particle_ptr, std::complex *val, const int *arr_size) noexcept; namespace py = pybind11; struct AeroParticle { @@ -307,28 +307,34 @@ struct AeroParticle { } static auto scatter_cross_sect(const AeroParticle &self) { + int len = 1; double val; f_aero_particle_scatter_cross_sect( self.ptr.f_arg(), - &val + &val, + &len ); return val; } static auto absorb_cross_sect(const AeroParticle &self) { + int len = 1; double val; f_aero_particle_absorb_cross_sect( self.ptr.f_arg(), - &val + &val, + &len ); return val; } static auto asymmetry(const AeroParticle &self) { + int len = 1; double val; f_aero_particle_asymmetry( self.ptr.f_arg(), - &val + &val, + &len ); return val; } @@ -373,19 +379,23 @@ struct AeroParticle { } static auto refract_shell(const AeroParticle &self) { + int len = 1; std::complex refract_shell; f_aero_particle_refract_shell( self.ptr.f_arg(), - &refract_shell + &refract_shell, + &len ); return refract_shell; } static auto refract_core(const AeroParticle &self) { + int len = 1; std::complex refract_core; f_aero_particle_refract_core( self.ptr.f_arg(), - &refract_core + &refract_core, + &len ); return refract_core; } From 7193ee672078b412cca50803369dcf47f02cecc5 Mon Sep 17 00:00:00 2001 From: Jeffrey Curtis Date: Fri, 12 Apr 2024 15:15:56 -0500 Subject: [PATCH 07/31] add a copy_weight to sample_particles test --- tests/test_aero_state.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_aero_state.py b/tests/test_aero_state.py index f8b56210..85217bd6 100644 --- a/tests/test_aero_state.py +++ b/tests/test_aero_state.py @@ -432,6 +432,7 @@ def test_sample_particles(sut_minimal): # arrange aero_data = ppmc.AeroData(AERO_DATA_CTOR_ARG_MINIMAL) sut = ppmc.AeroState(aero_data, *AERO_STATE_CTOR_ARG_MINIMAL) + sut.copy_weight(sut_minimal) # act num_conc = sut_minimal.total_num_conc From 5f9cfa441ae5e133a8f366fc5fb48e066f24a93a Mon Sep 17 00:00:00 2001 From: Jeffrey Curtis Date: Fri, 12 Apr 2024 15:26:57 -0500 Subject: [PATCH 08/31] make length of optical arrays set to n_swbands from aero_data_parameters --- src/aero_particle.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/aero_particle.hpp b/src/aero_particle.hpp index 344a516f..35f9f66d 100644 --- a/src/aero_particle.hpp +++ b/src/aero_particle.hpp @@ -307,7 +307,7 @@ struct AeroParticle { } static auto scatter_cross_sect(const AeroParticle &self) { - int len = 1; + int len = n_swbands; double val; f_aero_particle_scatter_cross_sect( self.ptr.f_arg(), @@ -318,7 +318,7 @@ struct AeroParticle { } static auto absorb_cross_sect(const AeroParticle &self) { - int len = 1; + int len = n_swbands; double val; f_aero_particle_absorb_cross_sect( self.ptr.f_arg(), @@ -329,7 +329,7 @@ struct AeroParticle { } static auto asymmetry(const AeroParticle &self) { - int len = 1; + int len = n_swbands; double val; f_aero_particle_asymmetry( self.ptr.f_arg(), @@ -379,7 +379,7 @@ struct AeroParticle { } static auto refract_shell(const AeroParticle &self) { - int len = 1; + int len = n_swbands; std::complex refract_shell; f_aero_particle_refract_shell( self.ptr.f_arg(), @@ -390,7 +390,7 @@ struct AeroParticle { } static auto refract_core(const AeroParticle &self) { - int len = 1; + int len = n_swbands; std::complex refract_core; f_aero_particle_refract_core( self.ptr.f_arg(), From 6c1052baa806fa24627ec6c5bd62c6bd1ce9c9d8 Mon Sep 17 00:00:00 2001 From: Jeffrey Curtis Date: Fri, 12 Apr 2024 16:00:53 -0500 Subject: [PATCH 09/31] remove n_orig_part. replace with component sources --- src/aero_particle.F90 | 17 +++++++++++++++++ src/aero_particle.hpp | 6 +++--- src/pypartmc.cpp | 2 +- tests/test_aero_particle.py | 8 ++++---- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/aero_particle.F90 b/src/aero_particle.F90 index 9fe4657c..ce0c9c06 100644 --- a/src/aero_particle.F90 +++ b/src/aero_particle.F90 @@ -485,6 +485,23 @@ subroutine f_aero_particle_n_orig_part( & end subroutine + subroutine f_aero_particle_get_component_sources( & + aero_particle_ptr_c, & + source_list, & + n_sources & + ) bind(C) + + type(aero_particle_t), pointer :: aero_particle_ptr_f => null() + type(c_ptr), intent(in) :: aero_particle_ptr_c + integer(c_int), intent(in) :: n_sources + integer(c_int), dimension(n_sources), intent(inout) :: source_list + + call c_f_pointer(aero_particle_ptr_c, aero_particle_ptr_f) + + call aero_particle_get_component_sources(aero_particle_ptr_f, source_list) + + end subroutine + subroutine f_aero_particle_id( & aero_particle_ptr_c, & id & diff --git a/src/aero_particle.hpp b/src/aero_particle.hpp index 35f9f66d..d5d2ecb4 100644 --- a/src/aero_particle.hpp +++ b/src/aero_particle.hpp @@ -41,7 +41,7 @@ extern "C" void f_aero_particle_scatter_cross_sect(const void *aero_particle_ptr extern "C" void f_aero_particle_asymmetry(const void *aero_particle_ptr, double *val, const int *arr_size) noexcept; extern "C" void f_aero_particle_greatest_create_time(const void *aero_particle_ptr, double *val) noexcept; extern "C" void f_aero_particle_least_create_time(const void *aero_particle_ptr, double *val) noexcept; -extern "C" void f_aero_particle_n_orig_part(const void *aero_particle_ptr, void *arr_data, const int *arr_size) noexcept; +extern "C" void f_aero_particle_get_component_sources(const void *aero_particle_ptr, void *arr_data, const int *arr_size) noexcept; extern "C" void f_aero_particle_id(const void *aero_particle_ptr, int *val) noexcept; extern "C" void f_aero_particle_refract_shell(const void *aero_particle_ptr, std::complex *val, const int *arr_size) noexcept; extern "C" void f_aero_particle_refract_core(const void *aero_particle_ptr, std::complex *val, const int *arr_size) noexcept; @@ -339,11 +339,11 @@ struct AeroParticle { return val; } - static auto n_orig_part(const AeroParticle &self) { + static auto sources(const AeroParticle &self) { int len = AeroData::n_source(*self.aero_data); std::valarray data(len); - f_aero_particle_n_orig_part( + f_aero_particle_get_component_sources( self.ptr.f_arg(), begin(data), &len diff --git a/src/pypartmc.cpp b/src/pypartmc.cpp index 37e08dcc..900b4726 100644 --- a/src/pypartmc.cpp +++ b/src/pypartmc.cpp @@ -167,7 +167,7 @@ PYBIND11_MODULE(_PyPartMC, m) { "Refractive index of the shell (1).") .def_property_readonly("refract_core", AeroParticle::refract_core, "Refractive index of the core (1).") - .def_property_readonly("n_orig_part", AeroParticle::n_orig_part, + .def_property_readonly("sources", AeroParticle::sources, "Number of original particles from each source that coagulated to form particle.") .def_property_readonly("least_create_time", AeroParticle::least_create_time, "First time a constituent was created (s).") diff --git a/tests/test_aero_particle.py b/tests/test_aero_particle.py index 14cadbd6..eda32db3 100644 --- a/tests/test_aero_particle.py +++ b/tests/test_aero_particle.py @@ -570,7 +570,7 @@ def test_refract_core(): assert isinstance(value, complex) @staticmethod - def test_n_orig_part(): + def test_sources(): # arrange aero_data = ppmc.AeroData(AERO_DATA_CTOR_ARG_MINIMAL) aero_dist = ppmc.AeroDist(aero_data, AERO_DIST_CTOR_ARG_MINIMAL) @@ -578,11 +578,11 @@ def test_n_orig_part(): _ = aero_state.dist_sample(aero_dist, 1.0, 0.0) sut = aero_state.particle(0) # act - n_orig_part = sut.n_orig_part + sources = sut.sources # assert - assert len(n_orig_part) == aero_dist.n_mode - assert isinstance(n_orig_part[0], int) + assert len(sources) == aero_dist.n_mode + assert isinstance(sources[0], int) @staticmethod def test_least_create_time(): From 9b2ec3749ebbd568e354ace45071ff822b25fbbf Mon Sep 17 00:00:00 2001 From: Jeffrey Curtis Date: Fri, 12 Apr 2024 16:14:03 -0500 Subject: [PATCH 10/31] add missing file to CMakeList for fortran example --- readme_fortran/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/readme_fortran/CMakeLists.txt b/readme_fortran/CMakeLists.txt index d384ad77..962d8347 100644 --- a/readme_fortran/CMakeLists.txt +++ b/readme_fortran/CMakeLists.txt @@ -7,6 +7,7 @@ add_executable(main $ENV{PARTMC_HOME}/src/spec_line.F90 $ENV{PARTMC_HOME}/src/util.F90 $ENV{PARTMC_HOME}/src/constants.F90 + $ENV{PARTMC_HOME}/src/aero_component.F90 $ENV{PARTMC_HOME}/src/aero_data.F90 $ENV{PARTMC_HOME}/src/aero_mode.F90 $ENV{PARTMC_HOME}/src/aero_dist.F90 From 6f73902e181e4c75e72f797865d25022af3febd9 Mon Sep 17 00:00:00 2001 From: Jeffrey Curtis Date: Fri, 12 Apr 2024 18:48:02 -0500 Subject: [PATCH 11/31] fix readme_fortran example --- readme_fortran/main.f90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme_fortran/main.f90 b/readme_fortran/main.f90 index 0365b343..26eddfcb 100644 --- a/readme_fortran/main.f90 +++ b/readme_fortran/main.f90 @@ -20,7 +20,7 @@ program main call spec_file_close(f_aero_data) call spec_file_open("aero_dist.dat", f_aero_dist) - call spec_file_read_aero_dist(f_aero_dist, aero_data, aero_dist) + call spec_file_read_aero_dist(f_aero_dist, aero_data, .false., aero_dist) call spec_file_close(f_aero_dist) call aero_state_zero(aero_state) @@ -29,7 +29,7 @@ program main AERO_STATE_WEIGHT_NUMMASS_SOURCE) call aero_state_set_n_part_ideal(aero_state, dble(n_part)) call aero_state_add_aero_dist_sample(aero_state, aero_data, & - aero_dist, 1d0, 0d0, .true., .true., n_part_add) + aero_dist, 1d0, 1d0, 0d0, .true., .true., n_part_add) num_concs = aero_state_num_concs(aero_state, aero_data) masses = aero_state_masses(aero_state, aero_data) From 811e1eb5302ea0159e874ab358e83ac74a9d6332 Mon Sep 17 00:00:00 2001 From: Jeffrey Curtis Date: Sun, 14 Apr 2024 20:49:56 -0500 Subject: [PATCH 12/31] bump PartMC submodule --- gitmodules/partmc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitmodules/partmc b/gitmodules/partmc index afe7bed5..e9451e51 160000 --- a/gitmodules/partmc +++ b/gitmodules/partmc @@ -1 +1 @@ -Subproject commit afe7bed5ac2f0f2044b3d91f74b8158830d0d470 +Subproject commit e9451e51d845542bb4f586d2d194bf41e7f19af0 From 7e4bf735b644def62d63df170faa4c45170bfa1c Mon Sep 17 00:00:00 2001 From: Jeffrey Curtis Date: Sun, 14 Apr 2024 21:19:16 -0500 Subject: [PATCH 13/31] fix aero_particle id --- src/aero_particle.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aero_particle.hpp b/src/aero_particle.hpp index d5d2ecb4..eca3cc48 100644 --- a/src/aero_particle.hpp +++ b/src/aero_particle.hpp @@ -42,7 +42,7 @@ extern "C" void f_aero_particle_asymmetry(const void *aero_particle_ptr, double extern "C" void f_aero_particle_greatest_create_time(const void *aero_particle_ptr, double *val) noexcept; extern "C" void f_aero_particle_least_create_time(const void *aero_particle_ptr, double *val) noexcept; extern "C" void f_aero_particle_get_component_sources(const void *aero_particle_ptr, void *arr_data, const int *arr_size) noexcept; -extern "C" void f_aero_particle_id(const void *aero_particle_ptr, int *val) noexcept; +extern "C" void f_aero_particle_id(const void *aero_particle_ptr, int64_t *val) noexcept; extern "C" void f_aero_particle_refract_shell(const void *aero_particle_ptr, std::complex *val, const int *arr_size) noexcept; extern "C" void f_aero_particle_refract_core(const void *aero_particle_ptr, std::complex *val, const int *arr_size) noexcept; @@ -370,7 +370,7 @@ struct AeroParticle { } static auto id(const AeroParticle &self) { - int val; + int64_t val; f_aero_particle_id( self.ptr.f_arg(), &val From 260aa2585889f88990f72a9d510dcbdd88628431 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Wed, 17 Apr 2024 23:51:14 +0200 Subject: [PATCH 14/31] first steps toward enabling HDF5 in out local netCDF build (for int64 to be available) --- .gitmodules | 5 +++++ CMakeLists.txt | 2 ++ gitmodules/hdf5 | 1 + gitmodules/partmc | 2 +- 4 files changed, 9 insertions(+), 1 deletion(-) create mode 160000 gitmodules/hdf5 diff --git a/.gitmodules b/.gitmodules index a250302a..6e6650ca 100644 --- a/.gitmodules +++ b/.gitmodules @@ -53,3 +53,8 @@ [submodule "gitmodules/optional"] path = gitmodules/optional url = https://github.com/TartanLlama/optional.git + shallow = true +[submodule "gitmodules/hdf5"] + path = gitmodules/hdf5 + url = https://github.com/HDFGroup/hdf5.git + shallow = true diff --git a/CMakeLists.txt b/CMakeLists.txt index 3471225a..013d4106 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -380,10 +380,12 @@ target_include_directories(netcdf_clib PRIVATE ${CMAKE_BINARY_DIR}/netcdf ${CMAKE_SOURCE_DIR}/gitmodules/netcdf-c/include ${CMAKE_SOURCE_DIR}/gitmodules/netcdf-c/libsrc + ${CMAKE_SOURCE_DIR}/gitmodules/hdf5/src ) target_compile_definitions(netcdf_clib PRIVATE HAVE_CONFIG_H USE_NETCDF4 + USE_HDF5 ) target_compile_options(netcdf_clib PRIVATE $<$:-Wno-unused-result -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast> diff --git a/gitmodules/hdf5 b/gitmodules/hdf5 new file mode 160000 index 00000000..bbf1e26c --- /dev/null +++ b/gitmodules/hdf5 @@ -0,0 +1 @@ +Subproject commit bbf1e26cda5654fe9e6ea08c57029fa0c1007f53 diff --git a/gitmodules/partmc b/gitmodules/partmc index e9451e51..afe7bed5 160000 --- a/gitmodules/partmc +++ b/gitmodules/partmc @@ -1 +1 @@ -Subproject commit e9451e51d845542bb4f586d2d194bf41e7f19af0 +Subproject commit afe7bed5ac2f0f2044b3d91f74b8158830d0d470 From e00ca7ec9d526d56506213be0f79bec1c4a13a41 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Fri, 19 Apr 2024 20:34:19 +0200 Subject: [PATCH 15/31] compiling HDF5 using the HDF5_EXTERNALLY_CONFIGURED CMake option --- CMakeLists.txt | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 013d4106..1b6eee8b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -140,13 +140,21 @@ set(netcdf_c_dispatch_SOURCES dvar.c ddim.c dvarput.c dvarget.c ddispatch.c dcom ) add_prefix(gitmodules/netcdf-c/libdispatch/ netcdf_c_dispatch_SOURCES) +set(netcdf_c_libhdf5_SOURCES hdf5dispatch.c hdf5create.c hdf5open.c hdf5file.c + hdf5attr.c hdf5debug.c hdf5dim.c hdf5filter.c hdf5grp.c hdf5internal.c hdf5set_format_compatibility.c + hdf5type.c hdf5var.c +) +add_prefix(gitmodules/netcdf-c/libhdf5/ netcdf_c_libhdf5_SOURCES) + set(netcdf_c_lib_SOURCES nc_initialize.c) add_prefix(gitmodules/netcdf-c/liblib/ netcdf_c_lib_SOURCES) set(netcdf_c_src_SOURCES nc3dispatch.c nc3internal.c dim.c var.c ncio.c v1hpg.c memio.c posixio.c) add_prefix(gitmodules/netcdf-c/libsrc/ netcdf_c_src_SOURCES) -set(netcdf_c_src4_SOURCES nc4internal.c ncindex.c nc4cache.c nc4dispatch.c nc4type.c nc4grp.c nc4var.c) +set(netcdf_c_src4_SOURCES nc4internal.c ncindex.c nc4cache.c nc4dispatch.c nc4type.c nc4grp.c nc4var.c + ncfunc.c nc4dim.c nc4filters.c nc4attr.c +) add_prefix(gitmodules/netcdf-c/libsrc4/ netcdf_c_src4_SOURCES) set(netcdf_f_SOURCES typeSizes.F90 module_netcdf_nf_data.F90 module_netcdf_nc_data.F90 @@ -321,6 +329,16 @@ execute_process( OUTPUT_FILE ${CMAKE_BINARY_DIR}/include/camp/version.h ) +### HDF5 ########################################################################################### + +set(HDF5_EXTERNALLY_CONFIGURED 1) +set(H5_ENABLE_SHARED_LIB OFF) +set(BUILD_TESTING OFF) +set(HDF5_BUILD_TOOLS OFF) +set(HDF5_BUILD_EXAMPLES OFF) +set(HDF5_BUILD_HL_LIB ON) +add_subdirectory(${CMAKE_SOURCE_DIR}/gitmodules/hdf5) + ### netCDF ######################################################################################### file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/netcdf) @@ -349,6 +367,7 @@ LIST(APPEND netcdf_c_SOURCES ${netcdf_c_lib_SOURCES}) LIST(APPEND netcdf_c_SOURCES ${netcdf_c_dispatch_SOURCES}) LIST(APPEND netcdf_c_SOURCES ${netcdf_c_src_SOURCES}) LIST(APPEND netcdf_c_SOURCES ${netcdf_c_src4_SOURCES}) +LIST(APPEND netcdf_c_SOURCES ${netcdf_c_libhdf5_SOURCES}) foreach (f ${m4_SOURCES}) set(tmp ${CMAKE_BINARY_DIR}/netcdf/${f}.c) add_custom_command( @@ -381,6 +400,9 @@ target_include_directories(netcdf_clib PRIVATE ${CMAKE_SOURCE_DIR}/gitmodules/netcdf-c/include ${CMAKE_SOURCE_DIR}/gitmodules/netcdf-c/libsrc ${CMAKE_SOURCE_DIR}/gitmodules/hdf5/src + ${CMAKE_BINARY_DIR}/gitmodules/hdf5/src + ${CMAKE_SOURCE_DIR}/gitmodules/hdf5/hl/src + ${CMAKE_SOURCE_DIR}/gitmodules/hdf5/src/H5FDsubfiling ) target_compile_definitions(netcdf_clib PRIVATE HAVE_CONFIG_H @@ -391,6 +413,8 @@ target_compile_options(netcdf_clib PRIVATE $<$:-Wno-unused-result -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast> $<$:-Wno-pointer-to-int-cast -Wno-int-to-void-pointer-cast> ) +target_link_libraries(netcdf_clib PRIVATE ${HDF5_LIB_TARGET}) +target_link_libraries(netcdf_clib PRIVATE ${HDF5_HL_LIB_TARGET}) ### netCDF-FORTRAN ################################################################################# From d166a7aca0866b8fe1c85db86a9351eecefa5bb1 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Sat, 20 Apr 2024 03:23:27 +0200 Subject: [PATCH 16/31] fixes in build and link steps for hdf/netcdf --- CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b6eee8b..e5e56112 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,7 +142,7 @@ add_prefix(gitmodules/netcdf-c/libdispatch/ netcdf_c_dispatch_SOURCES) set(netcdf_c_libhdf5_SOURCES hdf5dispatch.c hdf5create.c hdf5open.c hdf5file.c hdf5attr.c hdf5debug.c hdf5dim.c hdf5filter.c hdf5grp.c hdf5internal.c hdf5set_format_compatibility.c - hdf5type.c hdf5var.c + hdf5type.c hdf5var.c nc4hdf.c nc4info.c nc4mem.c nc4memcb.c ) add_prefix(gitmodules/netcdf-c/libhdf5/ netcdf_c_libhdf5_SOURCES) @@ -346,6 +346,7 @@ string(CONCAT cmd "import sys; print(''.join([line for line in sys.stdin" " if ('VERSION' in line and line.strip().startswith('SET'))" " or 'CHUNK_' in line" + " or '_CHUNKS_' in line" " or line.strip().startswith('CHECK_INCLUDE_FILE')" " or line.strip().startswith('CHECK_TYPE_SIZE')" " or line.strip().startswith('CHECK_FUNCTION_EXISTS')" @@ -413,8 +414,8 @@ target_compile_options(netcdf_clib PRIVATE $<$:-Wno-unused-result -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast> $<$:-Wno-pointer-to-int-cast -Wno-int-to-void-pointer-cast> ) -target_link_libraries(netcdf_clib PRIVATE ${HDF5_LIB_TARGET}) -target_link_libraries(netcdf_clib PRIVATE ${HDF5_HL_LIB_TARGET}) +target_link_libraries(netcdf_clib PRIVATE ${CMAKE_BINARY_DIR}/gitmodules/hdf5/src/libhdf5.a) +target_link_libraries(netcdf_clib PRIVATE ${CMAKE_BINARY_DIR}/gitmodules/hdf5/hl/src/libhdf5_hl.a) ### netCDF-FORTRAN ################################################################################# From b1b0b624b55ced3710a221b9eb010a7459465249 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Sat, 20 Apr 2024 13:19:02 +0200 Subject: [PATCH 17/31] one more condition for netCDF CmakeLists extraction --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index e5e56112..7494d6ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -347,6 +347,7 @@ string(CONCAT cmd " if ('VERSION' in line and line.strip().startswith('SET'))" " or 'CHUNK_' in line" " or '_CHUNKS_' in line" + " or '_CACHE_SIZE' in line" " or line.strip().startswith('CHECK_INCLUDE_FILE')" " or line.strip().startswith('CHECK_TYPE_SIZE')" " or line.strip().startswith('CHECK_FUNCTION_EXISTS')" From b6e4b402b0be551e5681a8e834322897f2d846f8 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Sat, 20 Apr 2024 14:12:25 +0200 Subject: [PATCH 18/31] fix hdf5 linking --- CMakeLists.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7494d6ae..293010ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -332,11 +332,14 @@ execute_process( ### HDF5 ########################################################################################### set(HDF5_EXTERNALLY_CONFIGURED 1) -set(H5_ENABLE_SHARED_LIB OFF) +set(BUILD_STATIC_LIBS ON) +set(HDF5_BUILD_HL_LIB ON) +set(BUILD_SHARED_LIBS OFF) set(BUILD_TESTING OFF) set(HDF5_BUILD_TOOLS OFF) set(HDF5_BUILD_EXAMPLES OFF) -set(HDF5_BUILD_HL_LIB ON) +set(HDF5_ENABLE_Z_LIB_SUPPORT OFF) +set(HDF5_ENABLE_SZIP_SUPPORT OFF) add_subdirectory(${CMAKE_SOURCE_DIR}/gitmodules/hdf5) ### netCDF ######################################################################################### @@ -415,8 +418,8 @@ target_compile_options(netcdf_clib PRIVATE $<$:-Wno-unused-result -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast> $<$:-Wno-pointer-to-int-cast -Wno-int-to-void-pointer-cast> ) -target_link_libraries(netcdf_clib PRIVATE ${CMAKE_BINARY_DIR}/gitmodules/hdf5/src/libhdf5.a) -target_link_libraries(netcdf_clib PRIVATE ${CMAKE_BINARY_DIR}/gitmodules/hdf5/hl/src/libhdf5_hl.a) +target_link_libraries(netcdf_clib PRIVATE hdf5-static) +target_link_libraries(netcdf_clib PRIVATE hdf5_hl-static) ### netCDF-FORTRAN ################################################################################# From 5a5e874b1a5bff51179d14b06efc61b404c4eb84 Mon Sep 17 00:00:00 2001 From: Jeffrey Curtis Date: Sat, 20 Apr 2024 09:47:43 -0500 Subject: [PATCH 19/31] bump partmc to most recent bug fix for least_create_time --- gitmodules/partmc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitmodules/partmc b/gitmodules/partmc index afe7bed5..e9451e51 160000 --- a/gitmodules/partmc +++ b/gitmodules/partmc @@ -1 +1 @@ -Subproject commit afe7bed5ac2f0f2044b3d91f74b8158830d0d470 +Subproject commit e9451e51d845542bb4f586d2d194bf41e7f19af0 From a8a4d29525279d98cb8da022ababb386e8627618 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Sun, 21 Apr 2024 00:44:20 +0200 Subject: [PATCH 20/31] fix manifest --- MANIFEST.in | 18 ++++++++++++++++++ gitmodules/partmc | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index b922b931..7b347077 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -122,6 +122,9 @@ include gitmodules/netcdf-c/include/nchttp.h include gitmodules/netcdf-c/include/netcdf_mem.h include gitmodules/netcdf-c/include/netcdf_filter.h include gitmodules/netcdf-c/include/netcdf_aux.h +include gitmodules/netcdf-c/include/hdf5internal.h +include gitmodules/netcdf-c/include/hdf5dispatch.h +include gitmodules/netcdf-c/include/ncdimscale.h include gitmodules/netcdf-c/libdispatch/utf8proc_data.c include gitmodules/netcdf-c/libdispatch/utf8proc.h include gitmodules/netcdf-c/libdispatch/dvar.c @@ -188,6 +191,12 @@ include gitmodules/netcdf-c/libsrc4/nc4dispatch.c include gitmodules/netcdf-c/libsrc4/nc4type.c include gitmodules/netcdf-c/libsrc4/nc4grp.c include gitmodules/netcdf-c/libsrc4/nc4var.c +include gitmodules/netcdf-c/libsrc4/nc4dim.c +include gitmodules/netcdf-c/libsrc4/nc4attr.c +include gitmodules/netcdf-c/libsrc4/nc4filters.c +include gitmodules/netcdf-c/libsrc4/ncfunc.c + +graft gitmodules/netcdf-c/libhdf5 include gitmodules/netcdf-fortran/COPYRIGHT include gitmodules/netcdf-fortran/CMakeExtras/MatchNetCDFFortranTypes.cmake @@ -230,3 +239,12 @@ include gitmodules/netcdf-fortran/fortran/netcdf_text_variables.F90 include gitmodules/netcdf-fortran/fortran/netcdf_expanded.F90 include gitmodules/netcdf-fortran/fortran/netcdf4_eightbyte.F90 include gitmodules/netcdf-fortran/fortran/netcdf4_func.F90 + +include gitmodules/hdf5/COPYING +include gitmodules/hdf5/CMakeLists.txt +include gitmodules/hdf5/*.cmake +graft gitmodules/hdf5/src +graft gitmodules/hdf5/bin +graft gitmodules/hdf5/config +include gitmodules/hdf5/hl/CMakeLists.txt +graft gitmodules/hdf5/hl/src diff --git a/gitmodules/partmc b/gitmodules/partmc index e9451e51..afe7bed5 160000 --- a/gitmodules/partmc +++ b/gitmodules/partmc @@ -1 +1 @@ -Subproject commit e9451e51d845542bb4f586d2d194bf41e7f19af0 +Subproject commit afe7bed5ac2f0f2044b3d91f74b8158830d0d470 From db96e5e62aa161637bdfd5c4e3474b1e0977ee38 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Sun, 21 Apr 2024 00:51:46 +0200 Subject: [PATCH 21/31] bump partmc version again... --- gitmodules/partmc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitmodules/partmc b/gitmodules/partmc index afe7bed5..e9451e51 160000 --- a/gitmodules/partmc +++ b/gitmodules/partmc @@ -1 +1 @@ -Subproject commit afe7bed5ac2f0f2044b3d91f74b8158830d0d470 +Subproject commit e9451e51d845542bb4f586d2d194bf41e7f19af0 From f4069e5fd8e7a64ca091d48db711ec9567bfcafe Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Sun, 21 Apr 2024 01:31:38 +0200 Subject: [PATCH 22/31] sort out warning-detection false-positives --- .github/workflows/tests+pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests+pypi.yml b/.github/workflows/tests+pypi.yml index 00f936c9..8f832c7b 100644 --- a/.github/workflows/tests+pypi.yml +++ b/.github/workflows/tests+pypi.yml @@ -142,7 +142,7 @@ jobs: - run: | unset CI python -m build 2>&1 | tee build.log - exit `fgrep -i warning build.log | grep -v "WARNING setuptools_scm" | wc -l` + exit `fgrep -i warning build.log | grep -v "WARNING setuptools_scm" | grep -v "-warnings" | grep -v "All Warnings are enabled" | wc -l` - if: startsWith(matrix.platform, 'macos-') && matrix.python-version == '3.11' run: | From eed3bf3d3c1bf4e9b4279d549167a117d7834bf2 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Sun, 21 Apr 2024 15:42:54 +0200 Subject: [PATCH 23/31] workaround for CRLF issues on Windows in generated HDF5 headers; README FAQ updates --- CMakeLists.txt | 14 ++++++++++++++ README.md | 5 ++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 293010ae..3dd672d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -132,6 +132,9 @@ set(camp_SOURCES ) add_prefix(gitmodules/camp/src/ camp_SOURCES) +set(hdf5_GENERATED_HEADERS H5Edefin.h H5Einit.h H5Epubgen.h H5Eterm.h H5version.h H5overflow.h) +add_prefix(gitmodules/hdf5/src/ hdf5_GENERATED_HEADERS) + set(netcdf_c_dispatch_SOURCES dvar.c ddim.c dvarput.c dvarget.c ddispatch.c dcompound.c denum.c daux.c dvlen.c nc.c dfile.c dnotnc4.c dstring.c nclist.c nchashmap.c dinstance_intern.c dtype.c dgroup.c nclistmgr.c dattget.c dattinq.c dvarinq.c dfilter.c derror.c doffsets.c datt.c dattput.c dcopy.c drc.c @@ -340,8 +343,19 @@ set(HDF5_BUILD_TOOLS OFF) set(HDF5_BUILD_EXAMPLES OFF) set(HDF5_ENABLE_Z_LIB_SUPPORT OFF) set(HDF5_ENABLE_SZIP_SUPPORT OFF) + add_subdirectory(${CMAKE_SOURCE_DIR}/gitmodules/hdf5) +foreach(header ${hdf5_GENERATED_HEADERS}) + message(STATUS $CMAKE_SOURCE_DIR}/${header} "-->" ${CMAKE_BINARY_DIR}/${header}) + execute_process( + COMMAND ${PYTHON_EXECUTABLE} "-c" "print(''.join([l.replace('\\r','') for l in open(0,'r')]),end='')" + INPUT_FILE ${CMAKE_SOURCE_DIR}/${header} + OUTPUT_FILE ${CMAKE_BINARY_DIR}/${header} + ) + file(REMOVE ${CMAKE_SOURCE_DIR}/${header}) +endforeach() + ### netCDF ######################################################################################### file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/netcdf) diff --git a/README.md b/README.md index 38e378f4..b68732a7 100644 --- a/README.md +++ b/README.md @@ -289,7 +289,7 @@ MOSAIC_HOME=<> pip install --force-reinstall --no-binary=PyP ``` - Q: Why `pip install PyPartMC` triggers compilation on my brand new Apple machine, while it quickly downloads and installs binary packages when executed on older Macs, Windows or Linux? - A: We are not yet providing binary wheels on PyPI for Apple-silicon (arm64) machines. Cross-compilation with gfortran is only supported with experimental unofficial builds [and is tricky](https://github.com/iains/gcc-12-branch/issues/23), while Github Actions ARM64 virtual machines are [costly](https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#minute-multipliers). + A: We are providing binary wheels on PyPI for Apple-silicon (arm64) machines for selected macOS version made available by Github. In case the macOS version you are using is newer, compilation from source is triggered. - Q: Why some of the constructors expect data to be passed as **lists of single-entry dictionaries** instead of multi-element dictionaries? A: This is intentional and related with PartMC relying on the order of elements within spec-file input; while Python dictionaries preserve ordering (insertion order), JSON format does not, and we intend to make these data structures safe to be [de]serialized using JSON. @@ -301,6 +301,9 @@ import PyPartMC PyPartMC.__versions_of_build_time_dependencies__['PartMC'] ``` +- Q: Why m4 and perl are required at compile time? + A: PyPartMC includes parts of netCDF and HDF5 codebases which depend on m4 and perl, respectively, for generating source files before compilation. + ## Troubleshooting #### Common installation issues From bad418676e4147926e55b5244b3f01e62f5a44d6 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Sun, 21 Apr 2024 16:08:15 +0200 Subject: [PATCH 24/31] another try at workarounding Windows CRLF issue --- CMakeLists.txt | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3dd672d4..3c9d6e58 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -132,8 +132,8 @@ set(camp_SOURCES ) add_prefix(gitmodules/camp/src/ camp_SOURCES) -set(hdf5_GENERATED_HEADERS H5Edefin.h H5Einit.h H5Epubgen.h H5Eterm.h H5version.h H5overflow.h) -add_prefix(gitmodules/hdf5/src/ hdf5_GENERATED_HEADERS) +set(hdf5_INPUT_TXT_FILES H5err.txt H5overflow.txt H5vers.txt) +add_prefix(gitmodules/hdf5/src/ hdf5_INPUT_TXT_FILES) set(netcdf_c_dispatch_SOURCES dvar.c ddim.c dvarput.c dvarget.c ddispatch.c dcompound.c denum.c daux.c dvlen.c nc.c dfile.c dnotnc4.c dstring.c nclist.c nchashmap.c dinstance_intern.c dtype.c dgroup.c @@ -344,18 +344,17 @@ set(HDF5_BUILD_EXAMPLES OFF) set(HDF5_ENABLE_Z_LIB_SUPPORT OFF) set(HDF5_ENABLE_SZIP_SUPPORT OFF) -add_subdirectory(${CMAKE_SOURCE_DIR}/gitmodules/hdf5) - -foreach(header ${hdf5_GENERATED_HEADERS}) - message(STATUS $CMAKE_SOURCE_DIR}/${header} "-->" ${CMAKE_BINARY_DIR}/${header}) +foreach(file ${hdf5_INPUT_TXT_FILES}) execute_process( COMMAND ${PYTHON_EXECUTABLE} "-c" "print(''.join([l.replace('\\r','') for l in open(0,'r')]),end='')" - INPUT_FILE ${CMAKE_SOURCE_DIR}/${header} - OUTPUT_FILE ${CMAKE_BINARY_DIR}/${header} + INPUT_FILE ${CMAKE_SOURCE_DIR}/${file} + OUTPUT_FILE ${CMAKE_BINARY_DIR}/${file} ) - file(REMOVE ${CMAKE_SOURCE_DIR}/${header}) + file(RENAME ${CMAKE_BINARY_DIR}/${file} ${CMAKE_SOURCE_DIR}/${file}) endforeach() +add_subdirectory(${CMAKE_SOURCE_DIR}/gitmodules/hdf5) + ### netCDF ######################################################################################### file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/netcdf) From 4866520313ce298fe139754f7e2d5ec12ac453b6 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Sun, 21 Apr 2024 17:13:54 +0200 Subject: [PATCH 25/31] fix missing dir issue --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c9d6e58..aa17ec51 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -344,6 +344,8 @@ set(HDF5_BUILD_EXAMPLES OFF) set(HDF5_ENABLE_Z_LIB_SUPPORT OFF) set(HDF5_ENABLE_SZIP_SUPPORT OFF) +file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/gitmodules/hdf5) +file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/gitmodules/hdf5/src) foreach(file ${hdf5_INPUT_TXT_FILES}) execute_process( COMMAND ${PYTHON_EXECUTABLE} "-c" "print(''.join([l.replace('\\r','') for l in open(0,'r')]),end='')" From e7dc42c1391107b4edcbd2ef78f80e8f572cd75b Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Sun, 21 Apr 2024 18:06:12 +0200 Subject: [PATCH 26/31] tmate for debugging --- .github/workflows/tests+pypi.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests+pypi.yml b/.github/workflows/tests+pypi.yml index 8f832c7b..9bc2741a 100644 --- a/.github/workflows/tests+pypi.yml +++ b/.github/workflows/tests+pypi.yml @@ -206,10 +206,10 @@ jobs: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} python -m pytest --durations=10 -v -s -We -p no:unraisableexception gitmodules/devops_tests ### uncomment to gain ssh access in case of failure -# - if: ${{ failure() }} -# uses: mxschmitt/action-tmate@v3 -# with: -# limit-access-to-actor: true + - if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true dist_check: runs-on: ubuntu-latest From 52d6d4c2bfa451683a8b9c237dc65a8fb56c73a1 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Sun, 21 Apr 2024 22:29:34 +0200 Subject: [PATCH 27/31] try sorting out newline issue again --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index aa17ec51..2f2f8d39 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -348,7 +348,7 @@ file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/gitmodules/hdf5) file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/gitmodules/hdf5/src) foreach(file ${hdf5_INPUT_TXT_FILES}) execute_process( - COMMAND ${PYTHON_EXECUTABLE} "-c" "print(''.join([l.replace('\\r','') for l in open(0,'r')]),end='')" + COMMAND ${PYTHON_EXECUTABLE} "-c" "sys.stdout.reconfigure(newline='\\n');print(''.join([l.replace('\\r','') for l in open(0,'r')]),end='')" INPUT_FILE ${CMAKE_SOURCE_DIR}/${file} OUTPUT_FILE ${CMAKE_BINARY_DIR}/${file} ) From 2c64802ce24532d5c5499fff6b8b76508f5aa7a6 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Sun, 21 Apr 2024 23:21:27 +0200 Subject: [PATCH 28/31] comment out hacks --- CMakeLists.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f2f8d39..7f6e8c31 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -344,16 +344,16 @@ set(HDF5_BUILD_EXAMPLES OFF) set(HDF5_ENABLE_Z_LIB_SUPPORT OFF) set(HDF5_ENABLE_SZIP_SUPPORT OFF) -file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/gitmodules/hdf5) -file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/gitmodules/hdf5/src) -foreach(file ${hdf5_INPUT_TXT_FILES}) - execute_process( - COMMAND ${PYTHON_EXECUTABLE} "-c" "sys.stdout.reconfigure(newline='\\n');print(''.join([l.replace('\\r','') for l in open(0,'r')]),end='')" - INPUT_FILE ${CMAKE_SOURCE_DIR}/${file} - OUTPUT_FILE ${CMAKE_BINARY_DIR}/${file} - ) - file(RENAME ${CMAKE_BINARY_DIR}/${file} ${CMAKE_SOURCE_DIR}/${file}) -endforeach() +#file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/gitmodules/hdf5) +#file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/gitmodules/hdf5/src) +#foreach(file ${hdf5_INPUT_TXT_FILES}) +# execute_process( +# COMMAND ${PYTHON_EXECUTABLE} "-c" "sys.stdout.reconfigure(newline='\\n');print(''.join([l.replace('\\r','') for l in open(0,'r')]),end='')" +# INPUT_FILE ${CMAKE_SOURCE_DIR}/${file} +# OUTPUT_FILE ${CMAKE_BINARY_DIR}/${file} +# ) +# file(RENAME ${CMAKE_BINARY_DIR}/${file} ${CMAKE_SOURCE_DIR}/${file}) +#endforeach() add_subdirectory(${CMAKE_SOURCE_DIR}/gitmodules/hdf5) From e2dd51f8662b8232260b4b31e3eeb45901220871 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Mon, 22 Apr 2024 00:15:46 +0200 Subject: [PATCH 29/31] yet another try... --- CMakeLists.txt | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f6e8c31..2cd7da34 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -132,8 +132,8 @@ set(camp_SOURCES ) add_prefix(gitmodules/camp/src/ camp_SOURCES) -set(hdf5_INPUT_TXT_FILES H5err.txt H5overflow.txt H5vers.txt) -add_prefix(gitmodules/hdf5/src/ hdf5_INPUT_TXT_FILES) +set(hdf5_GENERATED_HEADERS H5Edefin.h H5Einit.h H5Epubgen.h H5Eterm.h H5version.h H5overflow.h) +add_prefix(gitmodules/hdf5/src/ hdf5_GENERATED_HEADERS) set(netcdf_c_dispatch_SOURCES dvar.c ddim.c dvarput.c dvarget.c ddispatch.c dcompound.c denum.c daux.c dvlen.c nc.c dfile.c dnotnc4.c dstring.c nclist.c nchashmap.c dinstance_intern.c dtype.c dgroup.c @@ -344,19 +344,17 @@ set(HDF5_BUILD_EXAMPLES OFF) set(HDF5_ENABLE_Z_LIB_SUPPORT OFF) set(HDF5_ENABLE_SZIP_SUPPORT OFF) -#file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/gitmodules/hdf5) -#file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/gitmodules/hdf5/src) -#foreach(file ${hdf5_INPUT_TXT_FILES}) -# execute_process( -# COMMAND ${PYTHON_EXECUTABLE} "-c" "sys.stdout.reconfigure(newline='\\n');print(''.join([l.replace('\\r','') for l in open(0,'r')]),end='')" -# INPUT_FILE ${CMAKE_SOURCE_DIR}/${file} -# OUTPUT_FILE ${CMAKE_BINARY_DIR}/${file} -# ) -# file(RENAME ${CMAKE_BINARY_DIR}/${file} ${CMAKE_SOURCE_DIR}/${file}) -#endforeach() - add_subdirectory(${CMAKE_SOURCE_DIR}/gitmodules/hdf5) +foreach(file ${hdf5_GENERATED_HEADERS}) + execute_process( + COMMAND ${PYTHON_EXECUTABLE} "-c" "import re;open(1,'wb').write(re.sub(b'\\r',b'',open(0,'rb').read()))" + INPUT_FILE ${CMAKE_SOURCE_DIR}/${file} + OUTPUT_FILE ${CMAKE_BINARY_DIR}/${file} + ) + file(REMOVE ${CMAKE_SOURCE_DIR}/${file}) +endforeach() + ### netCDF ######################################################################################### file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/netcdf) From 9f1d187d885d8c0a55621df955ac9fcd17908af8 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Tue, 23 Apr 2024 19:00:53 +0200 Subject: [PATCH 30/31] comment out tmate action --- .github/workflows/tests+pypi.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests+pypi.yml b/.github/workflows/tests+pypi.yml index 9bc2741a..8f832c7b 100644 --- a/.github/workflows/tests+pypi.yml +++ b/.github/workflows/tests+pypi.yml @@ -206,10 +206,10 @@ jobs: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} python -m pytest --durations=10 -v -s -We -p no:unraisableexception gitmodules/devops_tests ### uncomment to gain ssh access in case of failure - - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true +# - if: ${{ failure() }} +# uses: mxschmitt/action-tmate@v3 +# with: +# limit-access-to-actor: true dist_check: runs-on: ubuntu-latest From 3b2a97fd3ea3be407b891fe775c2df33cf47d622 Mon Sep 17 00:00:00 2001 From: Sylwester Arabas Date: Tue, 23 Apr 2024 19:03:25 +0200 Subject: [PATCH 31/31] comment out tmate action --- .github/workflows/tests+pypi.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests+pypi.yml b/.github/workflows/tests+pypi.yml index 9bc2741a..8f832c7b 100644 --- a/.github/workflows/tests+pypi.yml +++ b/.github/workflows/tests+pypi.yml @@ -206,10 +206,10 @@ jobs: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} python -m pytest --durations=10 -v -s -We -p no:unraisableexception gitmodules/devops_tests ### uncomment to gain ssh access in case of failure - - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true +# - if: ${{ failure() }} +# uses: mxschmitt/action-tmate@v3 +# with: +# limit-access-to-actor: true dist_check: runs-on: ubuntu-latest