From 1b19ebb53147b503236099f3f32f6259ef3e23bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Mon, 11 Oct 2021 07:07:50 +0200 Subject: [PATCH 01/42] CIBW: add support for py3.10; use CIBWv2 --- .github/workflows/main.yml | 6 +----- pyproject.toml | 7 +++++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b2321d97c..fb1ff70eb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -59,11 +59,7 @@ jobs: fetch-depth: '0' - name: Build wheels - uses: pypa/cibuildwheel@v1.11.1.post1 - env: - CIBW_BUILD: cp36-* cp37-* cp38-* cp39-* - CIBW_TEST_EXTRAS: test - CIBW_TEST_COMMAND: pytest -v {project}/tests + uses: pypa/cibuildwheel@v2.1.3 with: output-dir: dist diff --git a/pyproject.toml b/pyproject.toml index e701830c8..32dede29c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,3 +68,10 @@ target-version = [ max-statements = 80 max-attributes = 25 max-public-methods = 75 + +[tool.cibuildwheel] +# Disable building PyPy wheels on all platforms +skip = "pp*" +# Run the package tests using `pytest` +test-extras: "test" +test-command = "pytest -v {package}/tests" From 89a483d7b9298da4ceb9da20812f52879a43f2ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Mon, 11 Oct 2021 07:16:37 +0200 Subject: [PATCH 02/42] CI: fix old typo --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fb1ff70eb..2e399caca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python 3.8 - uses: actions\setup-python@v2 + uses: actions/setup-python@v2 with: python-version: 3.8 @@ -82,7 +82,7 @@ jobs: fetch-depth: '0' - name: Set up Python ${{ matrix.python-version }} - uses: actions\setup-python@v2 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} From 48457d7aed998de7b4c6b43e8532c374ce6b4579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Mon, 11 Oct 2021 07:21:37 +0200 Subject: [PATCH 03/42] pyproject: fix typo --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 32dede29c..5294fc59c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,5 +73,5 @@ target-version = [ # Disable building PyPy wheels on all platforms skip = "pp*" # Run the package tests using `pytest` -test-extras: "test" +test-extras = "test" test-command = "pytest -v {package}/tests" From acb521acc70d34f8521c5e24adf99f6ab3b843cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Wed, 20 Oct 2021 13:27:00 +0200 Subject: [PATCH 04/42] CI: add py3.10 to sdist test --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2e399caca..0b099b914 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,7 +74,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 From f3b8ec3e05a56f747f25f082376da68973be812e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Wed, 20 Oct 2021 15:29:00 +0200 Subject: [PATCH 05/42] add discussion paper --- CITATION.bib | 10 ++++++++++ README.md | 12 ++++++++---- docs/source/index.rst | 10 ++++++---- 3 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 CITATION.bib diff --git a/CITATION.bib b/CITATION.bib new file mode 100644 index 000000000..a511056e6 --- /dev/null +++ b/CITATION.bib @@ -0,0 +1,10 @@ +@Article{gmd-2021-301, +AUTHOR = {M\"uller, S. and Sch\"uler, L. and Zech, A. and He{\ss}e, F.}, +TITLE = {GSTools v1.3: A toolbox for geostatistical modelling in Python}, +JOURNAL = {Geoscientific Model Development Discussions}, +VOLUME = {2021}, +YEAR = {2021}, +PAGES = {1--33}, +URL = {https://gmd.copernicus.org/preprints/gmd-2021-301/}, +DOI = {10.5194/gmd-2021-301} +} diff --git a/README.md b/README.md index 9f6f87eb0..7ff21bf8f 100644 --- a/README.md +++ b/README.md @@ -66,13 +66,17 @@ To install the latest development version via pip, see the ## Citation -At the moment you can cite the Zenodo code publication of GSTools: +If you are using GSTools in your publication please cite our paper: -> Sebastian Müller & Lennart Schüler. GeoStat-Framework/GSTools. Zenodo. https://doi.org/10.5281/zenodo.1313628 +> Müller, S., Schüler, L., Zech, A., and Heße, F.: +> GSTools v1.3: A toolbox for geostatistical modelling in Python, +> Geosci. Model Dev. Discuss. [preprint], https://doi.org/10.5194/gmd-2021-301, in review, 2021. + +You can cite the Zenodo code publication of GSTools by: -If you want to cite a specific version, have a look at the Zenodo site. +> Sebastian Müller & Lennart Schüler. GeoStat-Framework/GSTools. Zenodo. https://doi.org/10.5281/zenodo.1313628 -A publication for the GeoStat-Framework is in preperation. +If you want to cite a specific version, have a look at the [Zenodo site](https://doi.org/10.5281/zenodo.1313628). ## Documentation for GSTools diff --git a/docs/source/index.rst b/docs/source/index.rst index ff7108f8f..f66ca738b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -93,13 +93,15 @@ For the development version, you can do almost the same: Citation ======== -At the moment you can cite the Zenodo code publication of GSTools: +If you are using GSTools in your publication please cite our paper: -| *Sebastian Müller & Lennart Schüler. GeoStat-Framework/GSTools. Zenodo. https://doi.org/10.5281/zenodo.1313628* +| *Müller, S., Schüler, L., Zech, A., and Heße, F.: GSTools v1.3: A toolbox for geostatistical modelling in Python, Geosci. Model Dev. Discuss. [preprint], https://doi.org/10.5194/gmd-2021-301, in review, 2021.* + +You can cite the Zenodo code publication of GSTools by: -If you want to cite a specific version, have a look at the Zenodo site. +| *Sebastian Müller & Lennart Schüler. GeoStat-Framework/GSTools. Zenodo. https://doi.org/10.5281/zenodo.1313628* -A publication for the GeoStat-Framework is in preperation. +If you want to cite a specific version, have a look at the `Zenodo site `__. Tutorials and Examples From 596ffd6557feeb5c23af1c325c611bd29dfb7f29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Thu, 21 Oct 2021 13:13:39 +0200 Subject: [PATCH 06/42] CIBW: skip 32bit builds for py3.10 like numpy --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5294fc59c..fa28a3758 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,8 +70,8 @@ target-version = [ max-public-methods = 75 [tool.cibuildwheel] -# Disable building PyPy wheels on all platforms -skip = "pp*" +# Disable building PyPy wheels on all platforms and 32bit for py3.10 +skip = "pp* cp310-win32 cp310-manylinux_i686" # Run the package tests using `pytest` test-extras = "test" test-command = "pytest -v {package}/tests" From c5aa154cdddd9da0bc0e407cc01943fc83ff830f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Thu, 21 Oct 2021 13:29:03 +0200 Subject: [PATCH 07/42] CI: update python version strings --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0b099b914..d531b5657 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,7 +74,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [3.6, 3.7, 3.8, 3.9, 3.10] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v2 From da4d7144ced2ea28738d3563772a2898303316f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Mon, 11 Oct 2021 07:16:37 +0200 Subject: [PATCH 08/42] CI: fix old typo --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b2321d97c..7da065cac 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python 3.8 - uses: actions\setup-python@v2 + uses: actions/setup-python@v2 with: python-version: 3.8 @@ -86,7 +86,7 @@ jobs: fetch-depth: '0' - name: Set up Python ${{ matrix.python-version }} - uses: actions\setup-python@v2 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} From 0cc899ae853b17362e689e2075626a8407050d93 Mon Sep 17 00:00:00 2001 From: LSchueler Date: Wed, 27 Oct 2021 17:29:29 +0200 Subject: [PATCH 09/42] Remove Cython code & replace it with GSTools-Core The Github actions still need to be updated. --- MANIFEST.in | 2 +- README.md | 1 + docs/source/index.rst | 1 + gstools/field/generator.py | 2 +- gstools/field/summator.pyx | 82 -------- gstools/krige/base.py | 4 +- gstools/krige/krigesum.pyx | 64 ------ gstools/variogram/estimator.pyx | 346 -------------------------------- gstools/variogram/variogram.py | 17 +- setup.cfg | 1 + setup.py | 167 +-------------- 11 files changed, 22 insertions(+), 665 deletions(-) delete mode 100644 gstools/field/summator.pyx delete mode 100644 gstools/krige/krigesum.pyx delete mode 100644 gstools/variogram/estimator.pyx diff --git a/MANIFEST.in b/MANIFEST.in index 71c3bb1d5..933629618 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ prune * graft tests -recursive-include gstools *.py *.pyx +recursive-include gstools *.py recursive-exclude gstools *.c *.cpp include LICENSE README.md pyproject.toml setup.py setup.cfg exclude CHANGELOG.md CONTRIBUTING.md AUTHORS.md diff --git a/README.md b/README.md index 7ff21bf8f..117ee2eaa 100644 --- a/README.md +++ b/README.md @@ -338,6 +338,7 @@ in memory for immediate 3D plotting in Python. - [NumPy >= 1.14.5](https://www.numpy.org) - [SciPy >= 1.1.0](https://www.scipy.org/scipylib) +- [GSTools-Core >= 0.1.0](https://github.com/GeoStat-Framework/GSTools-Core) - [hankel >= 1.0.2](https://github.com/steven-murray/hankel) - [emcee >= 3.0.0](https://github.com/dfm/emcee) - [pyevtk >= 1.1.1](https://github.com/pyscience-projects/pyevtk) diff --git a/docs/source/index.rst b/docs/source/index.rst index f66ca738b..1aa563d5f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -378,6 +378,7 @@ Requirements - `Numpy >= 1.14.5 `_ - `SciPy >= 1.1.0 `_ +- `GSTools-Core >= 0.1.0 `_ - `hankel >= 1.0.2 `_ - `emcee >= 3.0.0 `_ - `pyevtk >= 1.1.1 `_ diff --git a/gstools/field/generator.py b/gstools/field/generator.py index 66f11e99b..f8d7be315 100644 --- a/gstools/field/generator.py +++ b/gstools/field/generator.py @@ -16,8 +16,8 @@ import numpy as np +from gstools_core import summate, summate_incompr from gstools.covmodel.base import CovModel -from gstools.field.summator import summate, summate_incompr from gstools.random.rng import RNG __all__ = ["RandMeth", "IncomprRandMeth"] diff --git a/gstools/field/summator.pyx b/gstools/field/summator.pyx deleted file mode 100644 index ecd7ea58d..000000000 --- a/gstools/field/summator.pyx +++ /dev/null @@ -1,82 +0,0 @@ -#cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True -# -*- coding: utf-8 -*- -""" -This is the randomization method summator, implemented in cython. -""" - -import numpy as np - -cimport cython - -from cython.parallel import prange - -cimport numpy as np -from libc.math cimport cos, sin - - -def summate( - const double[:, :] cov_samples, - const double[:] z_1, - const double[:] z_2, - const double[:, :] pos - ): - cdef int i, j, d - cdef double phase - cdef int dim = pos.shape[0] - - cdef int X_len = pos.shape[1] - cdef int N = cov_samples.shape[1] - - cdef double[:] summed_modes = np.zeros(X_len, dtype=float) - - for i in prange(X_len, nogil=True): - for j in range(N): - phase = 0. - for d in range(dim): - phase += cov_samples[d, j] * pos[d, i] - summed_modes[i] += z_1[j] * cos(phase) + z_2[j] * sin(phase) - - return np.asarray(summed_modes) - - -cdef (double) abs_square(const double[:] vec) nogil: - cdef int i - cdef double r = 0. - - for i in range(vec.shape[0]): - r += vec[i]**2 - - return r - - -def summate_incompr( - const double[:, :] cov_samples, - const double[:] z_1, - const double[:] z_2, - const double[:, :] pos - ): - cdef int i, j, d - cdef double phase - cdef double k_2 - cdef int dim = pos.shape[0] - - cdef double[:] e1 = np.zeros(dim, dtype=float) - e1[0] = 1. - cdef double[:] proj = np.empty(dim) - - cdef int X_len = pos.shape[1] - cdef int N = cov_samples.shape[1] - - cdef double[:, :] summed_modes = np.zeros((dim, X_len), dtype=float) - - for i in range(X_len): - for j in range(N): - k_2 = abs_square(cov_samples[:, j]) - phase = 0. - for d in range(dim): - phase += cov_samples[d, j] * pos[d, i] - for d in range(dim): - proj[d] = e1[d] - cov_samples[d, j] * cov_samples[0, j] / k_2 - summed_modes[d, i] += proj[d] * (z_1[j] * cos(phase) + z_2[j] * sin(phase)) - - return np.asarray(summed_modes) diff --git a/gstools/krige/base.py b/gstools/krige/base.py index 1e2171856..5558ec7bc 100755 --- a/gstools/krige/base.py +++ b/gstools/krige/base.py @@ -16,11 +16,11 @@ import scipy.linalg as spl from scipy.spatial.distance import cdist -from gstools.field.base import Field -from gstools.krige.krigesum import ( +from gstools_core import ( calc_field_krige, calc_field_krige_and_variance, ) +from gstools.field.base import Field from gstools.krige.tools import get_drift_functions, set_condition from gstools.tools.geometric import rotated_main_axes from gstools.tools.misc import eval_func diff --git a/gstools/krige/krigesum.pyx b/gstools/krige/krigesum.pyx deleted file mode 100644 index 41911cef3..000000000 --- a/gstools/krige/krigesum.pyx +++ /dev/null @@ -1,64 +0,0 @@ -#cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True -# -*- coding: utf-8 -*- -""" -This is a summator for the kriging routines -""" - -import numpy as np - -cimport cython -from cython.parallel import prange -cimport numpy as np - - -def calc_field_krige_and_variance( - const double[:, :] krig_mat, - const double[:, :] krig_vecs, - const double[:] cond -): - - cdef int mat_i = krig_mat.shape[0] - cdef int res_i = krig_vecs.shape[1] - - cdef double[:] field = np.zeros(res_i) - cdef double[:] error = np.zeros(res_i) - cdef double krig_fac - - cdef int i, j, k - - # error = krig_vecs * krig_mat * krig_vecs - # field = cond * krig_mat * krig_vecs - for k in prange(res_i, nogil=True): - for i in range(mat_i): - krig_fac = 0.0 - for j in range(mat_i): - krig_fac += krig_mat[i, j] * krig_vecs[j, k] - error[k] += krig_vecs[i, k] * krig_fac - field[k] += cond[i] * krig_fac - - return np.asarray(field), np.asarray(error) - - -def calc_field_krige( - const double[:, :] krig_mat, - const double[:, :] krig_vecs, - const double[:] cond -): - - cdef int mat_i = krig_mat.shape[0] - cdef int res_i = krig_vecs.shape[1] - - cdef double[:] field = np.zeros(res_i) - cdef double krig_fac - - cdef int i, j, k - - # field = cond * krig_mat * krig_vecs - for k in prange(res_i, nogil=True): - for i in range(mat_i): - krig_fac = 0.0 - for j in range(mat_i): - krig_fac += krig_mat[i, j] * krig_vecs[j, k] - field[k] += cond[i] * krig_fac - - return np.asarray(field) diff --git a/gstools/variogram/estimator.pyx b/gstools/variogram/estimator.pyx deleted file mode 100644 index 8c149b41b..000000000 --- a/gstools/variogram/estimator.pyx +++ /dev/null @@ -1,346 +0,0 @@ -#cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True -# distutils: language = c++ -# -*- coding: utf-8 -*- -""" -This is the variogram estimater, implemented in cython. -""" - -import numpy as np - -cimport cython - -from cython.parallel import parallel, prange - -cimport numpy as np -from libc.math cimport M_PI, acos, atan2, cos, fabs, isnan, pow, sin, sqrt - - -cdef inline double dist_euclid( - const int dim, - const double[:,:] pos, - const int i, - const int j, -) nogil: - cdef int d - cdef double dist_squared = 0.0 - for d in range(dim): - dist_squared += ((pos[d,i] - pos[d,j]) * (pos[d,i] - pos[d,j])) - return sqrt(dist_squared) - - -cdef inline double dist_haversine( - const int dim, - const double[:,:] pos, - const int i, - const int j, -) nogil: - # pos holds lat-lon in deg - cdef double deg_2_rad = M_PI / 180.0 - cdef double diff_lat = (pos[0, j] - pos[0, i]) * deg_2_rad - cdef double diff_lon = (pos[1, j] - pos[1, i]) * deg_2_rad - cdef double arg = ( - pow(sin(diff_lat/2.0), 2) + - cos(pos[0, i]*deg_2_rad) * - cos(pos[0, j]*deg_2_rad) * - pow(sin(diff_lon/2.0), 2) - ) - return 2.0 * atan2(sqrt(arg), sqrt(1.0-arg)) - - -ctypedef double (*_dist_func)( - const int, - const double[:,:], - const int, - const int, -) nogil - - -cdef inline bint dir_test( - const int dim, - const double[:,:] pos, - const double dist, - const double[:,:] direction, - const double angles_tol, - const double bandwidth, - const int i, - const int j, - const int d, -) nogil: - cdef double s_prod = 0.0 # scalar product - cdef double b_dist = 0.0 # band-distance - cdef double tmp # temporary variable - cdef int k - cdef bint in_band = True - cdef bint in_angle = True - - # scalar-product calculation for bandwidth projection and angle calculation - for k in range(dim): - s_prod += (pos[k,i] - pos[k,j]) * direction[d,k] - - # calculate band-distance by projection of point-pair-vec to direction line - if bandwidth > 0.0: - for k in range(dim): - tmp = (pos[k,i] - pos[k,j]) - s_prod * direction[d,k] - b_dist += tmp * tmp - in_band = sqrt(b_dist) < bandwidth - - # allow repeating points (dist = 0) - if dist > 0.0: - # use smallest angle by taking absolute value for arccos angle formula - tmp = fabs(s_prod) / dist - if tmp < 1.0: # else same direction (prevent numerical errors) - in_angle = acos(tmp) < angles_tol - - return in_band and in_angle - - -cdef inline double estimator_matheron(const double f_diff) nogil: - return f_diff * f_diff - -cdef inline double estimator_cressie(const double f_diff) nogil: - return sqrt(fabs(f_diff)) - -ctypedef double (*_estimator_func)(const double) nogil - -cdef inline void normalization_matheron( - double[:] variogram, - long[:] counts, -): - cdef int i - for i in range(variogram.shape[0]): - # avoid division by zero - variogram[i] /= (2. * max(counts[i], 1)) - -cdef inline void normalization_cressie( - double[:] variogram, - long[:] counts, -): - cdef int i - cdef long cnt - for i in range(variogram.shape[0]): - # avoid division by zero - cnt = max(counts[i], 1) - variogram[i] = ( - 0.5 * (1./cnt * variogram[i])**4 / - (0.457 + 0.494 / cnt + 0.045 / cnt**2) - ) - -ctypedef void (*_normalization_func)( - double[:], - long[:], -) - -cdef inline void normalization_matheron_vec( - double[:,:] variogram, - long[:,:] counts, -): - cdef int d, i - for d in range(variogram.shape[0]): - normalization_matheron(variogram[d, :], counts[d, :]) - -cdef inline void normalization_cressie_vec( - double[:,:] variogram, - long[:,:] counts, -): - cdef int d, i - cdef long cnt - for d in range(variogram.shape[0]): - normalization_cressie(variogram[d, :], counts[d, :]) - -ctypedef void (*_normalization_func_vec)( - double[:,:], - long[:,:], -) - -cdef _estimator_func choose_estimator_func(str estimator_type): - cdef _estimator_func estimator_func - if estimator_type == 'm': - estimator_func = estimator_matheron - elif estimator_type == 'c': - estimator_func = estimator_cressie - return estimator_func - -cdef _normalization_func choose_estimator_normalization(str estimator_type): - cdef _normalization_func normalization_func - if estimator_type == 'm': - normalization_func = normalization_matheron - elif estimator_type == 'c': - normalization_func = normalization_cressie - return normalization_func - -cdef _normalization_func_vec choose_estimator_normalization_vec(str estimator_type): - cdef _normalization_func_vec normalization_func_vec - if estimator_type == 'm': - normalization_func_vec = normalization_matheron_vec - elif estimator_type == 'c': - normalization_func_vec = normalization_cressie_vec - return normalization_func_vec - - -def directional( - const int dim, - const double[:,:] f, - const double[:] bin_edges, - const double[:,:] pos, - const double[:,:] direction, # should be normed - const double angles_tol=M_PI/8.0, - const double bandwidth=-1.0, # negative values to turn of bandwidth search - const bint separate_dirs=False, # whether the direction bands don't overlap - str estimator_type='m', -): - if pos.shape[1] != f.shape[1]: - raise ValueError('len(pos) = {0} != len(f) = {1} '. - format(pos.shape[1], f.shape[1])) - - if bin_edges.shape[0] < 2: - raise ValueError('len(bin_edges) too small') - - if angles_tol <= 0: - raise ValueError('tolerance for angle search masks must be > 0') - - cdef _estimator_func estimator_func = choose_estimator_func(estimator_type) - cdef _normalization_func_vec normalization_func_vec = ( - choose_estimator_normalization_vec(estimator_type) - ) - - cdef int d_max = direction.shape[0] - cdef int i_max = bin_edges.shape[0] - 1 - cdef int j_max = pos.shape[1] - 1 - cdef int k_max = pos.shape[1] - cdef int f_max = f.shape[0] - - cdef double[:,:] variogram = np.zeros((d_max, len(bin_edges)-1)) - cdef long[:,:] counts = np.zeros((d_max, len(bin_edges)-1), dtype=long) - cdef int i, j, k, m, d - cdef double dist - - for i in prange(i_max, nogil=True): - for j in range(j_max): - for k in range(j+1, k_max): - dist = dist_euclid(dim, pos, j, k) - if dist < bin_edges[i] or dist >= bin_edges[i+1]: - continue # skip if not in current bin - for d in range(d_max): - if not dir_test(dim, pos, dist, direction, angles_tol, bandwidth, k, j, d): - continue # skip if not in current direction - for m in range(f_max): - # skip no data values - if not (isnan(f[m,k]) or isnan(f[m,j])): - counts[d, i] += 1 - variogram[d, i] += estimator_func(f[m,k] - f[m,j]) - # once we found a fitting direction - # break the search if directions are separated - if separate_dirs: - break - - normalization_func_vec(variogram, counts) - return np.asarray(variogram), np.asarray(counts) - -def unstructured( - const int dim, - const double[:,:] f, - const double[:] bin_edges, - const double[:,:] pos, - str estimator_type='m', - str distance_type='e', -): - cdef _dist_func distance - - if distance_type == 'e': - distance = dist_euclid - else: - distance = dist_haversine - if dim != 2: - raise ValueError('Haversine: dim = {0} != 2'.format(dim)) - - if pos.shape[1] != f.shape[1]: - raise ValueError('len(pos) = {0} != len(f) = {1} '. - format(pos.shape[1], f.shape[1])) - - if bin_edges.shape[0] < 2: - raise ValueError('len(bin_edges) too small') - - cdef _estimator_func estimator_func = choose_estimator_func(estimator_type) - cdef _normalization_func normalization_func = ( - choose_estimator_normalization(estimator_type) - ) - - cdef int i_max = bin_edges.shape[0] - 1 - cdef int j_max = pos.shape[1] - 1 - cdef int k_max = pos.shape[1] - cdef int f_max = f.shape[0] - - cdef double[:] variogram = np.zeros(len(bin_edges)-1) - cdef long[:] counts = np.zeros(len(bin_edges)-1, dtype=long) - cdef int i, j, k, m - cdef double dist - - for i in prange(i_max, nogil=True): - for j in range(j_max): - for k in range(j+1, k_max): - dist = distance(dim, pos, j, k) - if dist < bin_edges[i] or dist >= bin_edges[i+1]: - continue # skip if not in current bin - for m in range(f_max): - # skip no data values - if not (isnan(f[m,k]) or isnan(f[m,j])): - counts[i] += 1 - variogram[i] += estimator_func(f[m,k] - f[m,j]) - - normalization_func(variogram, counts) - return np.asarray(variogram), np.asarray(counts) - - -def structured(const double[:,:] f, str estimator_type='m'): - cdef _estimator_func estimator_func = choose_estimator_func(estimator_type) - cdef _normalization_func normalization_func = ( - choose_estimator_normalization(estimator_type) - ) - - cdef int i_max = f.shape[0] - 1 - cdef int j_max = f.shape[1] - cdef int k_max = i_max + 1 - - cdef double[:] variogram = np.zeros(k_max) - cdef long[:] counts = np.zeros(k_max, dtype=long) - cdef int i, j, k - - with nogil, parallel(): - for i in range(i_max): - for j in range(j_max): - for k in prange(1, k_max-i): - counts[k] += 1 - variogram[k] += estimator_func(f[i,j] - f[i+k,j]) - - normalization_func(variogram, counts) - return np.asarray(variogram) - - -def ma_structured( - const double[:,:] f, - const bint[:,:] mask, - str estimator_type='m', -): - cdef _estimator_func estimator_func = choose_estimator_func(estimator_type) - cdef _normalization_func normalization_func = ( - choose_estimator_normalization(estimator_type) - ) - - cdef int i_max = f.shape[0] - 1 - cdef int j_max = f.shape[1] - cdef int k_max = i_max + 1 - - cdef double[:] variogram = np.zeros(k_max) - cdef long[:] counts = np.zeros(k_max, dtype=long) - cdef int i, j, k - - with nogil, parallel(): - for i in range(i_max): - for j in range(j_max): - for k in prange(1, k_max-i): - if not mask[i,j] and not mask[i+k,j]: - counts[k] += 1 - variogram[k] += estimator_func(f[i,j] - f[i+k,j]) - - normalization_func(variogram, counts) - return np.asarray(variogram) diff --git a/gstools/variogram/variogram.py b/gstools/variogram/variogram.py index 7c1076959..d758e3b40 100644 --- a/gstools/variogram/variogram.py +++ b/gstools/variogram/variogram.py @@ -12,6 +12,12 @@ """ import numpy as np +from gstools_core import ( + variogram_directional as directional, + variogram_ma_structured as ma_structured, + variogram_structured as structured, + variogram_unstructured as unstructured, +) from gstools.normalizer.tools import remove_trend_norm_mean from gstools.tools.geometric import ( ang2dir, @@ -20,12 +26,6 @@ generate_grid, ) from gstools.variogram.binning import standard_bins -from gstools.variogram.estimator import ( - directional, - ma_structured, - structured, - unstructured, -) __all__ = [ "vario_estimate", @@ -59,7 +59,8 @@ def _separate_dirs_test(direction, angles_tol): for j in range(i + 1, direction.shape[0]): s_prod = np.minimum(np.abs(np.dot(direction[i], direction[j])), 1) separate_dirs &= np.arccos(s_prod) >= 2 * angles_tol - return separate_dirs + # gstools-core doesn't like the type `numpy.bool_` + return bool(separate_dirs) def vario_estimate( @@ -441,7 +442,7 @@ def vario_estimate_axis( field = np.ma.array(field, ndmin=1, dtype=np.double) if missing: field.mask = np.logical_or(field.mask, missing_mask) - mask = np.asarray(np.ma.getmaskarray(field), dtype=np.int32) + mask = np.ma.getmaskarray(field) else: field = np.array(field, ndmin=1, dtype=np.double, copy=False) missing_mask = None # free space diff --git a/setup.cfg b/setup.cfg index efc2fd4c5..d533120b0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,6 +39,7 @@ project_urls = packages = find: install_requires = emcee>=3.0.0,<4 + gstools_core>=0.1.2,<1 hankel>=1.0.2,<2 meshio>=4.0.3,<6 numpy>=1.14.5,<2 diff --git a/setup.py b/setup.py index 1bbc7fafa..291d34a54 100644 --- a/setup.py +++ b/setup.py @@ -1,169 +1,14 @@ # -*- coding: utf-8 -*- """GSTools: A geostatistical toolbox.""" -import glob import os -import subprocess -import sys -import tempfile -from distutils.ccompiler import new_compiler -from distutils.errors import CompileError, LinkError -from distutils.sysconfig import customize_compiler import numpy as np -from Cython.Build import cythonize -from setuptools import Extension, setup - -HERE = os.path.abspath(os.path.dirname(__file__)) - - -# openmp finder ############################################################### -# This code is adapted for a large part from the scikit-learn openmp_helpers.py -# which can be found at: -# https://github.com/scikit-learn/scikit-learn/blob/0.24.0/sklearn/_build_utils - - -CCODE = """ -#include -#include -int main(void) { -#pragma omp parallel -printf("nthreads=%d\\n", omp_get_num_threads()); -return 0; -} -""" - - -def get_openmp_flag(compiler): - """Get the compiler dependent openmp flag.""" - if hasattr(compiler, "compiler"): - compiler = compiler.compiler[0] - else: - compiler = compiler.__class__.__name__ - - if sys.platform == "win32" and ("icc" in compiler or "icl" in compiler): - return ["/Qopenmp"] - if sys.platform == "win32": - return ["/openmp"] - if sys.platform == "darwin" and ("icc" in compiler or "icl" in compiler): - return ["-openmp"] - if sys.platform == "darwin" and "openmp" in os.getenv("CPPFLAGS", ""): - return [] - # Default flag for GCC and clang: - return ["-fopenmp"] - - -def check_openmp_support(): - """Check whether OpenMP test code can be compiled and run.""" - ccompiler = new_compiler() - customize_compiler(ccompiler) - - with tempfile.TemporaryDirectory() as tmp_dir: - try: - os.chdir(tmp_dir) - # Write test program - with open("test_openmp.c", "w") as cfile: - cfile.write(CCODE) - os.mkdir("objects") - # Compile, test program - openmp_flags = get_openmp_flag(ccompiler) - ccompiler.compile( - ["test_openmp.c"], - output_dir="objects", - extra_postargs=openmp_flags, - ) - # Link test program - extra_preargs = os.getenv("LDFLAGS", None) - if extra_preargs is not None: - extra_preargs = extra_preargs.split(" ") - else: - extra_preargs = [] - objects = glob.glob( - os.path.join("objects", "*" + ccompiler.obj_extension) - ) - ccompiler.link_executable( - objects, - "test_openmp", - extra_preargs=extra_preargs, - extra_postargs=openmp_flags, - ) - # Run test program - output = subprocess.check_output("./test_openmp") - output = output.decode(sys.stdout.encoding or "utf-8").splitlines() - # Check test program output - if "nthreads=" in output[0]: - nthreads = int(output[0].strip().split("=")[1]) - openmp_supported = len(output) == nthreads - else: - openmp_supported = False - openmp_flags = [] - except (CompileError, LinkError, subprocess.CalledProcessError): - openmp_supported = False - openmp_flags = [] - finally: - os.chdir(HERE) - return openmp_supported, openmp_flags - - -# openmp ###################################################################### - - -# you can set GSTOOLS_BUILD_PARALLEL=0 or GSTOOLS_BUILD_PARALLEL=1 -GS_PARALLEL = os.getenv("GSTOOLS_BUILD_PARALLEL") -USE_OPENMP = bool(int(GS_PARALLEL)) if GS_PARALLEL else False - -if USE_OPENMP: - # just check if wanted - CAN_USE_OPENMP, FLAGS = check_openmp_support() - if CAN_USE_OPENMP: - print("## GSTOOLS setup: OpenMP found.") - print("## OpenMP flags:", FLAGS) - else: - print("## GSTOOLS setup: OpenMP not found.") -else: - print("## GSTOOLS setup: OpenMP not wanted by the user.") - FLAGS = [] - - -# cython extensions ########################################################### - - -CY_MODULES = [] -CY_MODULES.append( - Extension( - "gstools.field.summator", - [os.path.join("gstools", "field", "summator.pyx")], - include_dirs=[np.get_include()], - extra_compile_args=FLAGS, - extra_link_args=FLAGS, - ) -) -CY_MODULES.append( - Extension( - "gstools.variogram.estimator", - [os.path.join("gstools", "variogram", "estimator.pyx")], - language="c++", - include_dirs=[np.get_include()], - extra_compile_args=FLAGS, - extra_link_args=FLAGS, - ) -) -CY_MODULES.append( - Extension( - "gstools.krige.krigesum", - [os.path.join("gstools", "krige", "krigesum.pyx")], - include_dirs=[np.get_include()], - extra_compile_args=FLAGS, - extra_link_args=FLAGS, - ) -) -EXT_MODULES = cythonize(CY_MODULES) # annotate=True - -# embed signatures for sphinx -for ext_m in EXT_MODULES: - ext_m.cython_directives = {"embedsignature": True} - +from setuptools import setup +import sys +import site -# setup ####################################################################### +site.ENABLE_USER_SITE = "--user" in sys.argv[1:] +HERE = os.path.abspath(os.path.dirname(__file__)) -setup(ext_modules=EXT_MODULES, include_dirs=[np.get_include()]) +setup(include_dirs=[np.get_include()]) From 68ea3b5e4ddff872af461940f7ff85a2194fe9dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Mon, 8 Nov 2021 14:18:08 +0100 Subject: [PATCH 10/42] CI: use latest cibuildwheel version --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d531b5657..d1c9472b1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -59,7 +59,7 @@ jobs: fetch-depth: '0' - name: Build wheels - uses: pypa/cibuildwheel@v2.1.3 + uses: pypa/cibuildwheel@v2.2.2 with: output-dir: dist From 601459b91c264a32f7a898234ca505d07ce4e560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Mon, 8 Nov 2021 15:07:57 +0100 Subject: [PATCH 11/42] CI: manylinux2014 for cp310 (like scipy) --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index fa28a3758..7196c3dd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,3 +75,7 @@ skip = "pp* cp310-win32 cp310-manylinux_i686" # Run the package tests using `pytest` test-extras = "test" test-command = "pytest -v {package}/tests" + +[[tool.cibuildwheel.overrides]] +select = "cp310-*" +manylinux-x86_64-image = "manylinux2014" From 04531fac67cf3c3d0742fed33e5541a7e63f959c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Mon, 8 Nov 2021 20:45:44 +0100 Subject: [PATCH 12/42] CI: skip musllinux --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7196c3dd2..cc16563eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,12 +70,13 @@ target-version = [ max-public-methods = 75 [tool.cibuildwheel] -# Disable building PyPy wheels on all platforms and 32bit for py3.10 -skip = "pp* cp310-win32 cp310-manylinux_i686" +# Disable building PyPy wheels on all platforms, 32bit for py3.10 and musllinux builds +skip = "pp* cp310-win32 cp310-manylinux_i686 *-musllinux_*" # Run the package tests using `pytest` test-extras = "test" test-command = "pytest -v {package}/tests" [[tool.cibuildwheel.overrides]] +# use manylinux2014 for py3.10 select = "cp310-*" manylinux-x86_64-image = "manylinux2014" From 490ad203961f6087d2ab09435df5fd03409e38f0 Mon Sep 17 00:00:00 2001 From: LSchueler Date: Tue, 9 Nov 2021 17:42:41 +0100 Subject: [PATCH 13/42] Fix names in unit tests --- tests/test_latlon.py | 2 +- tests/{test_normalize.py => test_normalizer.py} | 0 tests/test_transform.py | 2 +- tests/test_variogram_structured.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename tests/{test_normalize.py => test_normalizer.py} (100%) diff --git a/tests/test_latlon.py b/tests/test_latlon.py index b51475052..b47b6794b 100644 --- a/tests/test_latlon.py +++ b/tests/test_latlon.py @@ -22,7 +22,7 @@ def fix_dim(self): return 2 -class TestCondition(unittest.TestCase): +class TestLatLon(unittest.TestCase): def setUp(self): self.cmod = gs.Gaussian( latlon=True, var=2, len_scale=777, rescale=gs.EARTH_RADIUS diff --git a/tests/test_normalize.py b/tests/test_normalizer.py similarity index 100% rename from tests/test_normalize.py rename to tests/test_normalizer.py diff --git a/tests/test_transform.py b/tests/test_transform.py index eb3cf13ec..ca196f66b 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -7,7 +7,7 @@ import gstools as gs -class TestSRF(unittest.TestCase): +class TestTransform(unittest.TestCase): def setUp(self): self.cov_model = gs.Gaussian(dim=2, var=1.5, len_scale=4.0) self.mean = 0.3 diff --git a/tests/test_variogram_structured.py b/tests/test_variogram_structured.py index 7aa181224..e1ad19086 100644 --- a/tests/test_variogram_structured.py +++ b/tests/test_variogram_structured.py @@ -10,7 +10,7 @@ import gstools as gs -class TestVariogramstructured(unittest.TestCase): +class TestVariogramStructured(unittest.TestCase): def setUp(self): pass From 41cb5d5aea03dfee001fcee36c18af1eeb20edb5 Mon Sep 17 00:00:00 2001 From: LSchueler Date: Tue, 9 Nov 2021 19:51:44 +0100 Subject: [PATCH 14/42] Restore Cython code and make Rust impl. optional Now, the GSTools-Core is used, if the package can be imported, but it can also be switched off, by setting the global var. `gstools.config.USE_RUST=False` during the runtime. --- MANIFEST.in | 2 +- README.md | 2 +- docs/source/index.rst | 2 +- gstools/__init__.py | 1 + gstools/config.py | 14 ++ gstools/field/generator.py | 11 +- gstools/field/summator.pyx | 82 ++++++++ gstools/krige/base.py | 16 +- gstools/krige/krigesum.pyx | 64 ++++++ gstools/variogram/estimator.pyx | 346 ++++++++++++++++++++++++++++++++ gstools/variogram/variogram.py | 25 ++- setup.cfg | 3 +- setup.py | 167 ++++++++++++++- 13 files changed, 712 insertions(+), 23 deletions(-) create mode 100644 gstools/config.py create mode 100644 gstools/field/summator.pyx create mode 100644 gstools/krige/krigesum.pyx create mode 100644 gstools/variogram/estimator.pyx diff --git a/MANIFEST.in b/MANIFEST.in index 933629618..71c3bb1d5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ prune * graft tests -recursive-include gstools *.py +recursive-include gstools *.py *.pyx recursive-exclude gstools *.c *.cpp include LICENSE README.md pyproject.toml setup.py setup.cfg exclude CHANGELOG.md CONTRIBUTING.md AUTHORS.md diff --git a/README.md b/README.md index 117ee2eaa..c0488480f 100644 --- a/README.md +++ b/README.md @@ -338,7 +338,6 @@ in memory for immediate 3D plotting in Python. - [NumPy >= 1.14.5](https://www.numpy.org) - [SciPy >= 1.1.0](https://www.scipy.org/scipylib) -- [GSTools-Core >= 0.1.0](https://github.com/GeoStat-Framework/GSTools-Core) - [hankel >= 1.0.2](https://github.com/steven-murray/hankel) - [emcee >= 3.0.0](https://github.com/dfm/emcee) - [pyevtk >= 1.1.1](https://github.com/pyscience-projects/pyevtk) @@ -346,6 +345,7 @@ in memory for immediate 3D plotting in Python. ### Optional +- [GSTools-Core >= 0.1.0](https://github.com/GeoStat-Framework/GSTools-Core) - [matplotlib](https://matplotlib.org) - [pyvista](https://docs.pyvista.org/) diff --git a/docs/source/index.rst b/docs/source/index.rst index 1aa563d5f..b36965352 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -378,7 +378,6 @@ Requirements - `Numpy >= 1.14.5 `_ - `SciPy >= 1.1.0 `_ -- `GSTools-Core >= 0.1.0 `_ - `hankel >= 1.0.2 `_ - `emcee >= 3.0.0 `_ - `pyevtk >= 1.1.1 `_ @@ -388,6 +387,7 @@ Requirements Optional -------- +- `GSTools-Core >= 0.1.0 `_ - `matplotlib `_ - `pyvista `_ diff --git a/gstools/__init__.py b/gstools/__init__.py index 77281d8d4..a82839cc7 100644 --- a/gstools/__init__.py +++ b/gstools/__init__.py @@ -127,6 +127,7 @@ """ # Hooray! from gstools import ( + config, covmodel, field, krige, diff --git a/gstools/config.py b/gstools/config.py new file mode 100644 index 000000000..ebbd571cd --- /dev/null +++ b/gstools/config.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +""" +GStools subpackage providing global variables. + +.. currentmodule:: gstools.config + +""" +# pylint: disable=W0611 +try: + import gstools_core + + USE_RUST = True +except ImportError: + USE_RUST = False diff --git a/gstools/field/generator.py b/gstools/field/generator.py index f8d7be315..91408e884 100644 --- a/gstools/field/generator.py +++ b/gstools/field/generator.py @@ -10,16 +10,23 @@ RandMeth IncomprRandMeth """ -# pylint: disable=C0103, W0222 +# pylint: disable=C0103, W0222, C0412 import warnings from copy import deepcopy as dcp import numpy as np -from gstools_core import summate, summate_incompr +from gstools import config from gstools.covmodel.base import CovModel from gstools.random.rng import RNG +if config.USE_RUST: + # pylint: disable=E0401 + from gstools_core import summate, summate_incompr +else: + # pylint: disable=C0412 + from gstools.field.summator import summate, summate_incompr + __all__ = ["RandMeth", "IncomprRandMeth"] diff --git a/gstools/field/summator.pyx b/gstools/field/summator.pyx new file mode 100644 index 000000000..ecd7ea58d --- /dev/null +++ b/gstools/field/summator.pyx @@ -0,0 +1,82 @@ +#cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True +# -*- coding: utf-8 -*- +""" +This is the randomization method summator, implemented in cython. +""" + +import numpy as np + +cimport cython + +from cython.parallel import prange + +cimport numpy as np +from libc.math cimport cos, sin + + +def summate( + const double[:, :] cov_samples, + const double[:] z_1, + const double[:] z_2, + const double[:, :] pos + ): + cdef int i, j, d + cdef double phase + cdef int dim = pos.shape[0] + + cdef int X_len = pos.shape[1] + cdef int N = cov_samples.shape[1] + + cdef double[:] summed_modes = np.zeros(X_len, dtype=float) + + for i in prange(X_len, nogil=True): + for j in range(N): + phase = 0. + for d in range(dim): + phase += cov_samples[d, j] * pos[d, i] + summed_modes[i] += z_1[j] * cos(phase) + z_2[j] * sin(phase) + + return np.asarray(summed_modes) + + +cdef (double) abs_square(const double[:] vec) nogil: + cdef int i + cdef double r = 0. + + for i in range(vec.shape[0]): + r += vec[i]**2 + + return r + + +def summate_incompr( + const double[:, :] cov_samples, + const double[:] z_1, + const double[:] z_2, + const double[:, :] pos + ): + cdef int i, j, d + cdef double phase + cdef double k_2 + cdef int dim = pos.shape[0] + + cdef double[:] e1 = np.zeros(dim, dtype=float) + e1[0] = 1. + cdef double[:] proj = np.empty(dim) + + cdef int X_len = pos.shape[1] + cdef int N = cov_samples.shape[1] + + cdef double[:, :] summed_modes = np.zeros((dim, X_len), dtype=float) + + for i in range(X_len): + for j in range(N): + k_2 = abs_square(cov_samples[:, j]) + phase = 0. + for d in range(dim): + phase += cov_samples[d, j] * pos[d, i] + for d in range(dim): + proj[d] = e1[d] - cov_samples[d, j] * cov_samples[0, j] / k_2 + summed_modes[d, i] += proj[d] * (z_1[j] * cos(phase) + z_2[j] * sin(phase)) + + return np.asarray(summed_modes) diff --git a/gstools/krige/base.py b/gstools/krige/base.py index 5558ec7bc..e3922d335 100755 --- a/gstools/krige/base.py +++ b/gstools/krige/base.py @@ -9,23 +9,29 @@ .. autosummary:: Krige """ -# pylint: disable=C0103, W0221, E1102, R0201 +# pylint: disable=C0103, W0221, E1102, R0201, C0412 import collections import numpy as np import scipy.linalg as spl from scipy.spatial.distance import cdist -from gstools_core import ( - calc_field_krige, - calc_field_krige_and_variance, -) +from gstools import config from gstools.field.base import Field from gstools.krige.tools import get_drift_functions, set_condition from gstools.tools.geometric import rotated_main_axes from gstools.tools.misc import eval_func from gstools.variogram import vario_estimate +if config.USE_RUST: + # pylint: disable=E0401 + from gstools_core import calc_field_krige, calc_field_krige_and_variance +else: + from gstools.krige.krigesum import ( + calc_field_krige, + calc_field_krige_and_variance, + ) + __all__ = ["Krige"] diff --git a/gstools/krige/krigesum.pyx b/gstools/krige/krigesum.pyx new file mode 100644 index 000000000..41911cef3 --- /dev/null +++ b/gstools/krige/krigesum.pyx @@ -0,0 +1,64 @@ +#cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True +# -*- coding: utf-8 -*- +""" +This is a summator for the kriging routines +""" + +import numpy as np + +cimport cython +from cython.parallel import prange +cimport numpy as np + + +def calc_field_krige_and_variance( + const double[:, :] krig_mat, + const double[:, :] krig_vecs, + const double[:] cond +): + + cdef int mat_i = krig_mat.shape[0] + cdef int res_i = krig_vecs.shape[1] + + cdef double[:] field = np.zeros(res_i) + cdef double[:] error = np.zeros(res_i) + cdef double krig_fac + + cdef int i, j, k + + # error = krig_vecs * krig_mat * krig_vecs + # field = cond * krig_mat * krig_vecs + for k in prange(res_i, nogil=True): + for i in range(mat_i): + krig_fac = 0.0 + for j in range(mat_i): + krig_fac += krig_mat[i, j] * krig_vecs[j, k] + error[k] += krig_vecs[i, k] * krig_fac + field[k] += cond[i] * krig_fac + + return np.asarray(field), np.asarray(error) + + +def calc_field_krige( + const double[:, :] krig_mat, + const double[:, :] krig_vecs, + const double[:] cond +): + + cdef int mat_i = krig_mat.shape[0] + cdef int res_i = krig_vecs.shape[1] + + cdef double[:] field = np.zeros(res_i) + cdef double krig_fac + + cdef int i, j, k + + # field = cond * krig_mat * krig_vecs + for k in prange(res_i, nogil=True): + for i in range(mat_i): + krig_fac = 0.0 + for j in range(mat_i): + krig_fac += krig_mat[i, j] * krig_vecs[j, k] + field[k] += cond[i] * krig_fac + + return np.asarray(field) diff --git a/gstools/variogram/estimator.pyx b/gstools/variogram/estimator.pyx new file mode 100644 index 000000000..8c149b41b --- /dev/null +++ b/gstools/variogram/estimator.pyx @@ -0,0 +1,346 @@ +#cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True +# distutils: language = c++ +# -*- coding: utf-8 -*- +""" +This is the variogram estimater, implemented in cython. +""" + +import numpy as np + +cimport cython + +from cython.parallel import parallel, prange + +cimport numpy as np +from libc.math cimport M_PI, acos, atan2, cos, fabs, isnan, pow, sin, sqrt + + +cdef inline double dist_euclid( + const int dim, + const double[:,:] pos, + const int i, + const int j, +) nogil: + cdef int d + cdef double dist_squared = 0.0 + for d in range(dim): + dist_squared += ((pos[d,i] - pos[d,j]) * (pos[d,i] - pos[d,j])) + return sqrt(dist_squared) + + +cdef inline double dist_haversine( + const int dim, + const double[:,:] pos, + const int i, + const int j, +) nogil: + # pos holds lat-lon in deg + cdef double deg_2_rad = M_PI / 180.0 + cdef double diff_lat = (pos[0, j] - pos[0, i]) * deg_2_rad + cdef double diff_lon = (pos[1, j] - pos[1, i]) * deg_2_rad + cdef double arg = ( + pow(sin(diff_lat/2.0), 2) + + cos(pos[0, i]*deg_2_rad) * + cos(pos[0, j]*deg_2_rad) * + pow(sin(diff_lon/2.0), 2) + ) + return 2.0 * atan2(sqrt(arg), sqrt(1.0-arg)) + + +ctypedef double (*_dist_func)( + const int, + const double[:,:], + const int, + const int, +) nogil + + +cdef inline bint dir_test( + const int dim, + const double[:,:] pos, + const double dist, + const double[:,:] direction, + const double angles_tol, + const double bandwidth, + const int i, + const int j, + const int d, +) nogil: + cdef double s_prod = 0.0 # scalar product + cdef double b_dist = 0.0 # band-distance + cdef double tmp # temporary variable + cdef int k + cdef bint in_band = True + cdef bint in_angle = True + + # scalar-product calculation for bandwidth projection and angle calculation + for k in range(dim): + s_prod += (pos[k,i] - pos[k,j]) * direction[d,k] + + # calculate band-distance by projection of point-pair-vec to direction line + if bandwidth > 0.0: + for k in range(dim): + tmp = (pos[k,i] - pos[k,j]) - s_prod * direction[d,k] + b_dist += tmp * tmp + in_band = sqrt(b_dist) < bandwidth + + # allow repeating points (dist = 0) + if dist > 0.0: + # use smallest angle by taking absolute value for arccos angle formula + tmp = fabs(s_prod) / dist + if tmp < 1.0: # else same direction (prevent numerical errors) + in_angle = acos(tmp) < angles_tol + + return in_band and in_angle + + +cdef inline double estimator_matheron(const double f_diff) nogil: + return f_diff * f_diff + +cdef inline double estimator_cressie(const double f_diff) nogil: + return sqrt(fabs(f_diff)) + +ctypedef double (*_estimator_func)(const double) nogil + +cdef inline void normalization_matheron( + double[:] variogram, + long[:] counts, +): + cdef int i + for i in range(variogram.shape[0]): + # avoid division by zero + variogram[i] /= (2. * max(counts[i], 1)) + +cdef inline void normalization_cressie( + double[:] variogram, + long[:] counts, +): + cdef int i + cdef long cnt + for i in range(variogram.shape[0]): + # avoid division by zero + cnt = max(counts[i], 1) + variogram[i] = ( + 0.5 * (1./cnt * variogram[i])**4 / + (0.457 + 0.494 / cnt + 0.045 / cnt**2) + ) + +ctypedef void (*_normalization_func)( + double[:], + long[:], +) + +cdef inline void normalization_matheron_vec( + double[:,:] variogram, + long[:,:] counts, +): + cdef int d, i + for d in range(variogram.shape[0]): + normalization_matheron(variogram[d, :], counts[d, :]) + +cdef inline void normalization_cressie_vec( + double[:,:] variogram, + long[:,:] counts, +): + cdef int d, i + cdef long cnt + for d in range(variogram.shape[0]): + normalization_cressie(variogram[d, :], counts[d, :]) + +ctypedef void (*_normalization_func_vec)( + double[:,:], + long[:,:], +) + +cdef _estimator_func choose_estimator_func(str estimator_type): + cdef _estimator_func estimator_func + if estimator_type == 'm': + estimator_func = estimator_matheron + elif estimator_type == 'c': + estimator_func = estimator_cressie + return estimator_func + +cdef _normalization_func choose_estimator_normalization(str estimator_type): + cdef _normalization_func normalization_func + if estimator_type == 'm': + normalization_func = normalization_matheron + elif estimator_type == 'c': + normalization_func = normalization_cressie + return normalization_func + +cdef _normalization_func_vec choose_estimator_normalization_vec(str estimator_type): + cdef _normalization_func_vec normalization_func_vec + if estimator_type == 'm': + normalization_func_vec = normalization_matheron_vec + elif estimator_type == 'c': + normalization_func_vec = normalization_cressie_vec + return normalization_func_vec + + +def directional( + const int dim, + const double[:,:] f, + const double[:] bin_edges, + const double[:,:] pos, + const double[:,:] direction, # should be normed + const double angles_tol=M_PI/8.0, + const double bandwidth=-1.0, # negative values to turn of bandwidth search + const bint separate_dirs=False, # whether the direction bands don't overlap + str estimator_type='m', +): + if pos.shape[1] != f.shape[1]: + raise ValueError('len(pos) = {0} != len(f) = {1} '. + format(pos.shape[1], f.shape[1])) + + if bin_edges.shape[0] < 2: + raise ValueError('len(bin_edges) too small') + + if angles_tol <= 0: + raise ValueError('tolerance for angle search masks must be > 0') + + cdef _estimator_func estimator_func = choose_estimator_func(estimator_type) + cdef _normalization_func_vec normalization_func_vec = ( + choose_estimator_normalization_vec(estimator_type) + ) + + cdef int d_max = direction.shape[0] + cdef int i_max = bin_edges.shape[0] - 1 + cdef int j_max = pos.shape[1] - 1 + cdef int k_max = pos.shape[1] + cdef int f_max = f.shape[0] + + cdef double[:,:] variogram = np.zeros((d_max, len(bin_edges)-1)) + cdef long[:,:] counts = np.zeros((d_max, len(bin_edges)-1), dtype=long) + cdef int i, j, k, m, d + cdef double dist + + for i in prange(i_max, nogil=True): + for j in range(j_max): + for k in range(j+1, k_max): + dist = dist_euclid(dim, pos, j, k) + if dist < bin_edges[i] or dist >= bin_edges[i+1]: + continue # skip if not in current bin + for d in range(d_max): + if not dir_test(dim, pos, dist, direction, angles_tol, bandwidth, k, j, d): + continue # skip if not in current direction + for m in range(f_max): + # skip no data values + if not (isnan(f[m,k]) or isnan(f[m,j])): + counts[d, i] += 1 + variogram[d, i] += estimator_func(f[m,k] - f[m,j]) + # once we found a fitting direction + # break the search if directions are separated + if separate_dirs: + break + + normalization_func_vec(variogram, counts) + return np.asarray(variogram), np.asarray(counts) + +def unstructured( + const int dim, + const double[:,:] f, + const double[:] bin_edges, + const double[:,:] pos, + str estimator_type='m', + str distance_type='e', +): + cdef _dist_func distance + + if distance_type == 'e': + distance = dist_euclid + else: + distance = dist_haversine + if dim != 2: + raise ValueError('Haversine: dim = {0} != 2'.format(dim)) + + if pos.shape[1] != f.shape[1]: + raise ValueError('len(pos) = {0} != len(f) = {1} '. + format(pos.shape[1], f.shape[1])) + + if bin_edges.shape[0] < 2: + raise ValueError('len(bin_edges) too small') + + cdef _estimator_func estimator_func = choose_estimator_func(estimator_type) + cdef _normalization_func normalization_func = ( + choose_estimator_normalization(estimator_type) + ) + + cdef int i_max = bin_edges.shape[0] - 1 + cdef int j_max = pos.shape[1] - 1 + cdef int k_max = pos.shape[1] + cdef int f_max = f.shape[0] + + cdef double[:] variogram = np.zeros(len(bin_edges)-1) + cdef long[:] counts = np.zeros(len(bin_edges)-1, dtype=long) + cdef int i, j, k, m + cdef double dist + + for i in prange(i_max, nogil=True): + for j in range(j_max): + for k in range(j+1, k_max): + dist = distance(dim, pos, j, k) + if dist < bin_edges[i] or dist >= bin_edges[i+1]: + continue # skip if not in current bin + for m in range(f_max): + # skip no data values + if not (isnan(f[m,k]) or isnan(f[m,j])): + counts[i] += 1 + variogram[i] += estimator_func(f[m,k] - f[m,j]) + + normalization_func(variogram, counts) + return np.asarray(variogram), np.asarray(counts) + + +def structured(const double[:,:] f, str estimator_type='m'): + cdef _estimator_func estimator_func = choose_estimator_func(estimator_type) + cdef _normalization_func normalization_func = ( + choose_estimator_normalization(estimator_type) + ) + + cdef int i_max = f.shape[0] - 1 + cdef int j_max = f.shape[1] + cdef int k_max = i_max + 1 + + cdef double[:] variogram = np.zeros(k_max) + cdef long[:] counts = np.zeros(k_max, dtype=long) + cdef int i, j, k + + with nogil, parallel(): + for i in range(i_max): + for j in range(j_max): + for k in prange(1, k_max-i): + counts[k] += 1 + variogram[k] += estimator_func(f[i,j] - f[i+k,j]) + + normalization_func(variogram, counts) + return np.asarray(variogram) + + +def ma_structured( + const double[:,:] f, + const bint[:,:] mask, + str estimator_type='m', +): + cdef _estimator_func estimator_func = choose_estimator_func(estimator_type) + cdef _normalization_func normalization_func = ( + choose_estimator_normalization(estimator_type) + ) + + cdef int i_max = f.shape[0] - 1 + cdef int j_max = f.shape[1] + cdef int k_max = i_max + 1 + + cdef double[:] variogram = np.zeros(k_max) + cdef long[:] counts = np.zeros(k_max, dtype=long) + cdef int i, j, k + + with nogil, parallel(): + for i in range(i_max): + for j in range(j_max): + for k in prange(1, k_max-i): + if not mask[i,j] and not mask[i+k,j]: + counts[k] += 1 + variogram[k] += estimator_func(f[i,j] - f[i+k,j]) + + normalization_func(variogram, counts) + return np.asarray(variogram) diff --git a/gstools/variogram/variogram.py b/gstools/variogram/variogram.py index d758e3b40..e24255ebd 100644 --- a/gstools/variogram/variogram.py +++ b/gstools/variogram/variogram.py @@ -10,14 +10,10 @@ vario_estimate vario_estimate_axis """ +# pylint: disable=C0412 import numpy as np -from gstools_core import ( - variogram_directional as directional, - variogram_ma_structured as ma_structured, - variogram_structured as structured, - variogram_unstructured as unstructured, -) +from gstools import config from gstools.normalizer.tools import remove_trend_norm_mean from gstools.tools.geometric import ( ang2dir, @@ -27,6 +23,21 @@ ) from gstools.variogram.binning import standard_bins +if config.USE_RUST: + # pylint: disable=E0401 + from gstools_core import variogram_directional as directional + from gstools_core import variogram_ma_structured as ma_structured + from gstools_core import variogram_structured as structured + from gstools_core import variogram_unstructured as unstructured +else: + # pylint: disable=C0412 + from gstools.variogram.estimator import ( + directional, + ma_structured, + structured, + unstructured, + ) + __all__ = [ "vario_estimate", "vario_estimate_axis", @@ -443,6 +454,8 @@ def vario_estimate_axis( if missing: field.mask = np.logical_or(field.mask, missing_mask) mask = np.ma.getmaskarray(field) + if not config.USE_RUST: + mask = np.asarray(mask, dtype=np.int32) else: field = np.array(field, ndmin=1, dtype=np.double, copy=False) missing_mask = None # free space diff --git a/setup.cfg b/setup.cfg index d533120b0..116b51ba9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,7 +39,6 @@ project_urls = packages = find: install_requires = emcee>=3.0.0,<4 - gstools_core>=0.1.2,<1 hankel>=1.0.2,<2 meshio>=4.0.3,<6 numpy>=1.14.5,<2 @@ -67,6 +66,8 @@ doc = plotting = matplotlib>=3,<4 pyvista>=0.29,<1 +rust = + gstools_core>=0.1.2,<1 test = coverage[toml]>=5.2.1,<6 pytest>=6.0,<7 diff --git a/setup.py b/setup.py index 291d34a54..1bbc7fafa 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,169 @@ # -*- coding: utf-8 -*- """GSTools: A geostatistical toolbox.""" +import glob import os - -import numpy as np -from setuptools import setup +import subprocess import sys -import site +import tempfile +from distutils.ccompiler import new_compiler +from distutils.errors import CompileError, LinkError +from distutils.sysconfig import customize_compiler -site.ENABLE_USER_SITE = "--user" in sys.argv[1:] +import numpy as np +from Cython.Build import cythonize +from setuptools import Extension, setup HERE = os.path.abspath(os.path.dirname(__file__)) -setup(include_dirs=[np.get_include()]) + +# openmp finder ############################################################### +# This code is adapted for a large part from the scikit-learn openmp_helpers.py +# which can be found at: +# https://github.com/scikit-learn/scikit-learn/blob/0.24.0/sklearn/_build_utils + + +CCODE = """ +#include +#include +int main(void) { +#pragma omp parallel +printf("nthreads=%d\\n", omp_get_num_threads()); +return 0; +} +""" + + +def get_openmp_flag(compiler): + """Get the compiler dependent openmp flag.""" + if hasattr(compiler, "compiler"): + compiler = compiler.compiler[0] + else: + compiler = compiler.__class__.__name__ + + if sys.platform == "win32" and ("icc" in compiler or "icl" in compiler): + return ["/Qopenmp"] + if sys.platform == "win32": + return ["/openmp"] + if sys.platform == "darwin" and ("icc" in compiler or "icl" in compiler): + return ["-openmp"] + if sys.platform == "darwin" and "openmp" in os.getenv("CPPFLAGS", ""): + return [] + # Default flag for GCC and clang: + return ["-fopenmp"] + + +def check_openmp_support(): + """Check whether OpenMP test code can be compiled and run.""" + ccompiler = new_compiler() + customize_compiler(ccompiler) + + with tempfile.TemporaryDirectory() as tmp_dir: + try: + os.chdir(tmp_dir) + # Write test program + with open("test_openmp.c", "w") as cfile: + cfile.write(CCODE) + os.mkdir("objects") + # Compile, test program + openmp_flags = get_openmp_flag(ccompiler) + ccompiler.compile( + ["test_openmp.c"], + output_dir="objects", + extra_postargs=openmp_flags, + ) + # Link test program + extra_preargs = os.getenv("LDFLAGS", None) + if extra_preargs is not None: + extra_preargs = extra_preargs.split(" ") + else: + extra_preargs = [] + objects = glob.glob( + os.path.join("objects", "*" + ccompiler.obj_extension) + ) + ccompiler.link_executable( + objects, + "test_openmp", + extra_preargs=extra_preargs, + extra_postargs=openmp_flags, + ) + # Run test program + output = subprocess.check_output("./test_openmp") + output = output.decode(sys.stdout.encoding or "utf-8").splitlines() + # Check test program output + if "nthreads=" in output[0]: + nthreads = int(output[0].strip().split("=")[1]) + openmp_supported = len(output) == nthreads + else: + openmp_supported = False + openmp_flags = [] + except (CompileError, LinkError, subprocess.CalledProcessError): + openmp_supported = False + openmp_flags = [] + finally: + os.chdir(HERE) + return openmp_supported, openmp_flags + + +# openmp ###################################################################### + + +# you can set GSTOOLS_BUILD_PARALLEL=0 or GSTOOLS_BUILD_PARALLEL=1 +GS_PARALLEL = os.getenv("GSTOOLS_BUILD_PARALLEL") +USE_OPENMP = bool(int(GS_PARALLEL)) if GS_PARALLEL else False + +if USE_OPENMP: + # just check if wanted + CAN_USE_OPENMP, FLAGS = check_openmp_support() + if CAN_USE_OPENMP: + print("## GSTOOLS setup: OpenMP found.") + print("## OpenMP flags:", FLAGS) + else: + print("## GSTOOLS setup: OpenMP not found.") +else: + print("## GSTOOLS setup: OpenMP not wanted by the user.") + FLAGS = [] + + +# cython extensions ########################################################### + + +CY_MODULES = [] +CY_MODULES.append( + Extension( + "gstools.field.summator", + [os.path.join("gstools", "field", "summator.pyx")], + include_dirs=[np.get_include()], + extra_compile_args=FLAGS, + extra_link_args=FLAGS, + ) +) +CY_MODULES.append( + Extension( + "gstools.variogram.estimator", + [os.path.join("gstools", "variogram", "estimator.pyx")], + language="c++", + include_dirs=[np.get_include()], + extra_compile_args=FLAGS, + extra_link_args=FLAGS, + ) +) +CY_MODULES.append( + Extension( + "gstools.krige.krigesum", + [os.path.join("gstools", "krige", "krigesum.pyx")], + include_dirs=[np.get_include()], + extra_compile_args=FLAGS, + extra_link_args=FLAGS, + ) +) +EXT_MODULES = cythonize(CY_MODULES) # annotate=True + +# embed signatures for sphinx +for ext_m in EXT_MODULES: + ext_m.cython_directives = {"embedsignature": True} + + +# setup ####################################################################### + + +setup(ext_modules=EXT_MODULES, include_dirs=[np.get_include()]) From e1bd45c1a09fdbb13393390676a99ef7ba455b67 Mon Sep 17 00:00:00 2001 From: LSchueler Date: Tue, 9 Nov 2021 20:12:46 +0100 Subject: [PATCH 15/42] Remove unnecessary `dim` arg. in Cython code This makes the Cython code compatible to GSTools-Core v0.1.2 again. --- gstools/variogram/estimator.pyx | 4 ++-- gstools/variogram/variogram.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/gstools/variogram/estimator.pyx b/gstools/variogram/estimator.pyx index 8c149b41b..6c5a67dff 100644 --- a/gstools/variogram/estimator.pyx +++ b/gstools/variogram/estimator.pyx @@ -178,7 +178,6 @@ cdef _normalization_func_vec choose_estimator_normalization_vec(str estimator_ty def directional( - const int dim, const double[:,:] f, const double[:] bin_edges, const double[:,:] pos, @@ -203,6 +202,7 @@ def directional( choose_estimator_normalization_vec(estimator_type) ) + cdef int dim = pos.shape[0] cdef int d_max = direction.shape[0] cdef int i_max = bin_edges.shape[0] - 1 cdef int j_max = pos.shape[1] - 1 @@ -237,13 +237,13 @@ def directional( return np.asarray(variogram), np.asarray(counts) def unstructured( - const int dim, const double[:,:] f, const double[:] bin_edges, const double[:,:] pos, str estimator_type='m', str distance_type='e', ): + cdef int dim = pos.shape[0] cdef _dist_func distance if distance_type == 'e': diff --git a/gstools/variogram/variogram.py b/gstools/variogram/variogram.py index e24255ebd..e9c00536b 100644 --- a/gstools/variogram/variogram.py +++ b/gstools/variogram/variogram.py @@ -353,7 +353,6 @@ def vario_estimate( # "h"aversine or "e"uclidean distance type distance_type = "h" if latlon else "e" estimates, counts = unstructured( - dim, field, bin_edges, pos, @@ -362,7 +361,6 @@ def vario_estimate( ) else: estimates, counts = directional( - dim, field, bin_edges, pos, From 872570fe537a0913106ac4d1de3298628c0008dc Mon Sep 17 00:00:00 2001 From: LSchueler Date: Tue, 9 Nov 2021 20:22:27 +0100 Subject: [PATCH 16/42] Satify coveralls --- gstools/config.py | 2 +- gstools/field/generator.py | 2 +- gstools/krige/base.py | 5 ++++- gstools/variogram/variogram.py | 16 ++++++++++++---- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/gstools/config.py b/gstools/config.py index ebbd571cd..090b6154f 100644 --- a/gstools/config.py +++ b/gstools/config.py @@ -9,6 +9,6 @@ try: import gstools_core - USE_RUST = True + USE_RUST = True #pragma: no cover except ImportError: USE_RUST = False diff --git a/gstools/field/generator.py b/gstools/field/generator.py index 91408e884..c13d71a79 100644 --- a/gstools/field/generator.py +++ b/gstools/field/generator.py @@ -22,7 +22,7 @@ if config.USE_RUST: # pylint: disable=E0401 - from gstools_core import summate, summate_incompr + from gstools_core import summate, summate_incompr # pragma: no cover else: # pylint: disable=C0412 from gstools.field.summator import summate, summate_incompr diff --git a/gstools/krige/base.py b/gstools/krige/base.py index e3922d335..d2ef1042c 100755 --- a/gstools/krige/base.py +++ b/gstools/krige/base.py @@ -25,7 +25,10 @@ if config.USE_RUST: # pylint: disable=E0401 - from gstools_core import calc_field_krige, calc_field_krige_and_variance + from gstools_core import ( + calc_field_krige, + calc_field_krige_and_variance, + ) # pragma: no cover else: from gstools.krige.krigesum import ( calc_field_krige, diff --git a/gstools/variogram/variogram.py b/gstools/variogram/variogram.py index e9c00536b..26f41054f 100644 --- a/gstools/variogram/variogram.py +++ b/gstools/variogram/variogram.py @@ -25,10 +25,18 @@ if config.USE_RUST: # pylint: disable=E0401 - from gstools_core import variogram_directional as directional - from gstools_core import variogram_ma_structured as ma_structured - from gstools_core import variogram_structured as structured - from gstools_core import variogram_unstructured as unstructured + from gstools_core import ( + variogram_directional as directional, + ) # pragma: no cover + from gstools_core import ( + variogram_ma_structured as ma_structured, + ) # pragma: no cover + from gstools_core import ( + variogram_structured as structured, + ) # pragma: no cover + from gstools_core import ( + variogram_unstructured as unstructured, + ) # pragma: no cover else: # pylint: disable=C0412 from gstools.variogram.estimator import ( From 1ff09ed620f1600e4de7fcf593f37deb576c01c1 Mon Sep 17 00:00:00 2001 From: LSchueler Date: Tue, 9 Nov 2021 20:39:15 +0100 Subject: [PATCH 17/42] Update changelog --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95ad2e2c6..82b9afd46 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,15 @@ All notable changes to **GSTools** will be documented in this file. +## [1.3.4] - Pure Pink ? + +### Enhancements +- add GStools-Core as optional dependency [#215](https://github.com/GeoStat-Framework/GSTools/pull/215) + +### Changes +- remove unnecessary `dim` argument in Cython code [#216](https://github.com/GeoStat-Framework/GSTools/issues/216) + + ## [1.3.3] - Pure Pink - 2021-08 ### Enhancements From 60325f0b798c50f856528fa5a396b414da9985b0 Mon Sep 17 00:00:00 2001 From: LSchueler Date: Tue, 9 Nov 2021 20:56:10 +0100 Subject: [PATCH 18/42] Update installation part of doc. --- docs/source/index.rst | 27 +++++++++++++++++++++++++++ gstools/config.py | 2 +- gstools/krige/base.py | 4 ++-- gstools/variogram/variogram.py | 16 ++++++++-------- 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index b36965352..28e6c34de 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -70,6 +70,8 @@ To get the latest development version you can install it directly from GitHub: If something went wrong during installation, try the :code:`-I` `flag from pip `_. +**Speeding up GSTools by parallelization** + To enable the OpenMP support, you have to provide a C compiler and OpenMP. Parallel support is controlled by an environment variable ``GSTOOLS_BUILD_PARALLEL``, that can be ``0`` or ``1`` (interpreted as ``0`` if not present). @@ -89,6 +91,31 @@ For the development version, you can do almost the same: export GSTOOLS_BUILD_PARALLEL=1 pip install git+git://github.com/GeoStat-Framework/GSTools.git@main +**Using experimental GSTools-Core for even more speed** + +You can install the optional dependency `GSTools-Core `_, +which is a re-implementation of the main algorithms used in GSTools. The new +package uses the language Rust and it should be faster (in some cases by orders +of magnitude), safer, and it will potentially completely replace the current +standard implementation in Cython. Once the package GSTools-Core is available +on your machine, it will be used by default. In case you want to switch back to +the Cython implementation, you can set :code:`gstools.config.USE_RUST=False` in +your code. This also works at runtime. You can install the optional dependency +e.g. by + +.. code-block:: none + + pip install gstools[rust] + +or by manually installing the package + +.. code-block:: none + + pip install gstools-core + +GSTools-Core will automatically use all your cores in parallel, without having +to use OpenMP or a local C compiler. + Citation ======== diff --git a/gstools/config.py b/gstools/config.py index 090b6154f..4186a259c 100644 --- a/gstools/config.py +++ b/gstools/config.py @@ -9,6 +9,6 @@ try: import gstools_core - USE_RUST = True #pragma: no cover + USE_RUST = True # pragma: no cover except ImportError: USE_RUST = False diff --git a/gstools/krige/base.py b/gstools/krige/base.py index d2ef1042c..0ffc482cb 100755 --- a/gstools/krige/base.py +++ b/gstools/krige/base.py @@ -25,10 +25,10 @@ if config.USE_RUST: # pylint: disable=E0401 - from gstools_core import ( + from gstools_core import ( # pragma: no cover calc_field_krige, calc_field_krige_and_variance, - ) # pragma: no cover + ) else: from gstools.krige.krigesum import ( calc_field_krige, diff --git a/gstools/variogram/variogram.py b/gstools/variogram/variogram.py index 26f41054f..77b9a2c20 100644 --- a/gstools/variogram/variogram.py +++ b/gstools/variogram/variogram.py @@ -26,17 +26,17 @@ if config.USE_RUST: # pylint: disable=E0401 from gstools_core import ( - variogram_directional as directional, - ) # pragma: no cover + variogram_directional as directional, # pragma: no cover + ) from gstools_core import ( - variogram_ma_structured as ma_structured, - ) # pragma: no cover + variogram_ma_structured as ma_structured, # pragma: no cover + ) from gstools_core import ( - variogram_structured as structured, - ) # pragma: no cover + variogram_structured as structured, # pragma: no cover + ) from gstools_core import ( - variogram_unstructured as unstructured, - ) # pragma: no cover + variogram_unstructured as unstructured, # pragma: no cover + ) else: # pylint: disable=C0412 from gstools.variogram.estimator import ( From 164b6872167ccb20c89b969c22ad35937a8be404 Mon Sep 17 00:00:00 2001 From: LSchueler Date: Tue, 9 Nov 2021 21:20:04 +0100 Subject: [PATCH 19/42] Mention RAYON_NUM_THREADS in readme --- docs/source/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index 28e6c34de..3880d412e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -115,6 +115,8 @@ or by manually installing the package GSTools-Core will automatically use all your cores in parallel, without having to use OpenMP or a local C compiler. +In case you want to restrict the number of threads used, you can set the +environment variable ``RAYON_NUM_THREADS`` to the desired amount. Citation From c0777e394c07c3cad60aaa682b031e6b2334dbdc Mon Sep 17 00:00:00 2001 From: LSchueler Date: Tue, 9 Nov 2021 21:27:03 +0100 Subject: [PATCH 20/42] Use correct version of GSTools-Core --- README.md | 2 +- docs/source/index.rst | 2 +- setup.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c0488480f..f68e49a91 100644 --- a/README.md +++ b/README.md @@ -345,7 +345,7 @@ in memory for immediate 3D plotting in Python. ### Optional -- [GSTools-Core >= 0.1.0](https://github.com/GeoStat-Framework/GSTools-Core) +- [GSTools-Core >= 0.2.0](https://github.com/GeoStat-Framework/GSTools-Core) - [matplotlib](https://matplotlib.org) - [pyvista](https://docs.pyvista.org/) diff --git a/docs/source/index.rst b/docs/source/index.rst index 3880d412e..8ead50563 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -416,7 +416,7 @@ Requirements Optional -------- -- `GSTools-Core >= 0.1.0 `_ +- `GSTools-Core >= 0.2.0 `_ - `matplotlib `_ - `pyvista `_ diff --git a/setup.cfg b/setup.cfg index 116b51ba9..5a59f87c9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,7 +67,7 @@ plotting = matplotlib>=3,<4 pyvista>=0.29,<1 rust = - gstools_core>=0.1.2,<1 + gstools_core>=0.2.0,<1 test = coverage[toml]>=5.2.1,<6 pytest>=6.0,<7 From 774c73034c41f47c0e4f97cfbdb13fbe57081472 Mon Sep 17 00:00:00 2001 From: LSchueler Date: Wed, 10 Nov 2021 16:58:03 +0100 Subject: [PATCH 21/42] Improve import situation --- gstools/variogram/variogram.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/gstools/variogram/variogram.py b/gstools/variogram/variogram.py index 77b9a2c20..7acce928a 100644 --- a/gstools/variogram/variogram.py +++ b/gstools/variogram/variogram.py @@ -23,22 +23,13 @@ ) from gstools.variogram.binning import standard_bins -if config.USE_RUST: +if config.USE_RUST: # pragma: no cover # pylint: disable=E0401 - from gstools_core import ( - variogram_directional as directional, # pragma: no cover - ) - from gstools_core import ( - variogram_ma_structured as ma_structured, # pragma: no cover - ) - from gstools_core import ( - variogram_structured as structured, # pragma: no cover - ) - from gstools_core import ( - variogram_unstructured as unstructured, # pragma: no cover - ) + from gstools_core import variogram_directional as directional + from gstools_core import variogram_ma_structured as ma_structured + from gstools_core import variogram_structured as structured + from gstools_core import variogram_unstructured as unstructured else: - # pylint: disable=C0412 from gstools.variogram.estimator import ( directional, ma_structured, From 0049a31e3fce1c866ea006e1be131ad5617a35eb Mon Sep 17 00:00:00 2001 From: LSchueler Date: Wed, 10 Nov 2021 17:04:18 +0100 Subject: [PATCH 22/42] Clean up some linter flags --- gstools/config.py | 4 ++-- gstools/field/generator.py | 5 ++--- gstools/krige/base.py | 7 ++----- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/gstools/config.py b/gstools/config.py index 4186a259c..38f3e7faa 100644 --- a/gstools/config.py +++ b/gstools/config.py @@ -6,9 +6,9 @@ """ # pylint: disable=W0611 -try: +try: # pragma: no cover import gstools_core - USE_RUST = True # pragma: no cover + USE_RUST = True except ImportError: USE_RUST = False diff --git a/gstools/field/generator.py b/gstools/field/generator.py index c13d71a79..31cc7eea2 100644 --- a/gstools/field/generator.py +++ b/gstools/field/generator.py @@ -20,11 +20,10 @@ from gstools.covmodel.base import CovModel from gstools.random.rng import RNG -if config.USE_RUST: +if config.USE_RUST: # pragma: no cover # pylint: disable=E0401 - from gstools_core import summate, summate_incompr # pragma: no cover + from gstools_core import summate, summate_incompr else: - # pylint: disable=C0412 from gstools.field.summator import summate, summate_incompr __all__ = ["RandMeth", "IncomprRandMeth"] diff --git a/gstools/krige/base.py b/gstools/krige/base.py index 0ffc482cb..8accf5080 100755 --- a/gstools/krige/base.py +++ b/gstools/krige/base.py @@ -23,12 +23,9 @@ from gstools.tools.misc import eval_func from gstools.variogram import vario_estimate -if config.USE_RUST: +if config.USE_RUST: # pragma: no cover # pylint: disable=E0401 - from gstools_core import ( # pragma: no cover - calc_field_krige, - calc_field_krige_and_variance, - ) + from gstools_core import calc_field_krige, calc_field_krige_and_variance else: from gstools.krige.krigesum import ( calc_field_krige, From 83ea2865a64d95834f1ad474a2594b1096533e66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Mon, 15 Nov 2021 12:39:07 +0100 Subject: [PATCH 23/42] CI: use manylinux1 for cp3.6 wheels --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index cc16563eb..11366ff1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,11 @@ skip = "pp* cp310-win32 cp310-manylinux_i686 *-musllinux_*" test-extras = "test" test-command = "pytest -v {package}/tests" +[[tool.cibuildwheel.overrides]] +# use manylinux1 for py3.6 +select = "cp36-*" +manylinux-x86_64-image = "manylinux1" + [[tool.cibuildwheel.overrides]] # use manylinux2014 for py3.10 select = "cp310-*" From 6be59f035d6936d6eedda0e086c7995fd58a8d4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Tue, 16 Nov 2021 16:52:20 +0100 Subject: [PATCH 24/42] CI: auditwheel 4 is smart enough to build manylinux1 compatible wheels for py3.6 --- pyproject.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 11366ff1a..cc16563eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,11 +76,6 @@ skip = "pp* cp310-win32 cp310-manylinux_i686 *-musllinux_*" test-extras = "test" test-command = "pytest -v {package}/tests" -[[tool.cibuildwheel.overrides]] -# use manylinux1 for py3.6 -select = "cp36-*" -manylinux-x86_64-image = "manylinux1" - [[tool.cibuildwheel.overrides]] # use manylinux2014 for py3.10 select = "cp310-*" From 8948f5f37cb3280951ba6fb7f17491c52ba6ecfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Wed, 17 Nov 2021 14:33:11 +0100 Subject: [PATCH 25/42] CI: separate jobs per arch for wheels; add universal2 wheel for macos --- .github/workflows/main.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d1c9472b1..a4de129a3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,12 +46,18 @@ jobs: python -m isort --check --diff --color . build_wheels: - name: wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} + name: wheels for ${{ matrix.cfg.os }} / ${{ matrix.cfg.arch }} + runs-on: ${{ matrix.cfg.os }} strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + cfg: + - { os: ubuntu-latest, arch: x86_64 } + - { os: ubuntu-latest, arch: i686 } + - { os: windows-latest, arch: AMD64 } + - { os: windows-latest, arch: x86 } + - { os: macos-latest, arch: x86_64 } + - { os: macos-latest, arch: universal2 } steps: - uses: actions/checkout@v2 @@ -60,6 +66,8 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.2.2 + env: + CIBW_ARCHS: ${{ matrix.cfg.arch }} with: output-dir: dist From ce25b326db16701273ae7e5e86c8685b6c9b0376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Wed, 17 Nov 2021 14:33:46 +0100 Subject: [PATCH 26/42] CI: also add arm64 wheels to be on the safe side --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a4de129a3..760d97bf1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,6 +57,7 @@ jobs: - { os: windows-latest, arch: AMD64 } - { os: windows-latest, arch: x86 } - { os: macos-latest, arch: x86_64 } + - { os: macos-latest, arch: arm64 } - { os: macos-latest, arch: universal2 } steps: From 5b97d674a713e754cd9c317798bfb197b2a31cdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Mon, 11 Oct 2021 07:07:50 +0200 Subject: [PATCH 27/42] CIBW: add support for py3.10; use CIBWv2 --- .github/workflows/main.yml | 6 +----- pyproject.toml | 7 +++++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7da065cac..2e399caca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -59,11 +59,7 @@ jobs: fetch-depth: '0' - name: Build wheels - uses: pypa/cibuildwheel@v1.11.1.post1 - env: - CIBW_BUILD: cp36-* cp37-* cp38-* cp39-* - CIBW_TEST_EXTRAS: test - CIBW_TEST_COMMAND: pytest -v {project}/tests + uses: pypa/cibuildwheel@v2.1.3 with: output-dir: dist diff --git a/pyproject.toml b/pyproject.toml index e701830c8..32dede29c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,3 +68,10 @@ target-version = [ max-statements = 80 max-attributes = 25 max-public-methods = 75 + +[tool.cibuildwheel] +# Disable building PyPy wheels on all platforms +skip = "pp*" +# Run the package tests using `pytest` +test-extras: "test" +test-command = "pytest -v {package}/tests" From 672b006766406dc38eaae9e78a729eb42fed82b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Mon, 11 Oct 2021 07:21:37 +0200 Subject: [PATCH 28/42] pyproject: fix typo --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 32dede29c..5294fc59c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,5 +73,5 @@ target-version = [ # Disable building PyPy wheels on all platforms skip = "pp*" # Run the package tests using `pytest` -test-extras: "test" +test-extras = "test" test-command = "pytest -v {package}/tests" From 4563341ab6b0982564efca3b32a685f66c585cbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Wed, 20 Oct 2021 13:27:00 +0200 Subject: [PATCH 29/42] CI: add py3.10 to sdist test --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2e399caca..0b099b914 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,7 +74,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 From d5ca1f23f7b1e502843ae6ff737465ece50e9067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Thu, 21 Oct 2021 13:13:39 +0200 Subject: [PATCH 30/42] CIBW: skip 32bit builds for py3.10 like numpy --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5294fc59c..fa28a3758 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,8 +70,8 @@ target-version = [ max-public-methods = 75 [tool.cibuildwheel] -# Disable building PyPy wheels on all platforms -skip = "pp*" +# Disable building PyPy wheels on all platforms and 32bit for py3.10 +skip = "pp* cp310-win32 cp310-manylinux_i686" # Run the package tests using `pytest` test-extras = "test" test-command = "pytest -v {package}/tests" From 44cd73cecbf0a0504166d20098cb30c3d3d76a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Thu, 21 Oct 2021 13:29:03 +0200 Subject: [PATCH 31/42] CI: update python version strings --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0b099b914..d531b5657 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,7 +74,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [3.6, 3.7, 3.8, 3.9, 3.10] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v2 From 57b550080cfc5af90197954992118a3ed198784f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Mon, 8 Nov 2021 14:18:08 +0100 Subject: [PATCH 32/42] CI: use latest cibuildwheel version --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d531b5657..d1c9472b1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -59,7 +59,7 @@ jobs: fetch-depth: '0' - name: Build wheels - uses: pypa/cibuildwheel@v2.1.3 + uses: pypa/cibuildwheel@v2.2.2 with: output-dir: dist From 332ec740b5172c530c1ba3e1da40488331b9c7bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Mon, 8 Nov 2021 15:07:57 +0100 Subject: [PATCH 33/42] CI: manylinux2014 for cp310 (like scipy) --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index fa28a3758..7196c3dd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,3 +75,7 @@ skip = "pp* cp310-win32 cp310-manylinux_i686" # Run the package tests using `pytest` test-extras = "test" test-command = "pytest -v {package}/tests" + +[[tool.cibuildwheel.overrides]] +select = "cp310-*" +manylinux-x86_64-image = "manylinux2014" From 9633c5899c5a5c22f056f9dd280f3c095c9ab399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Mon, 8 Nov 2021 20:45:44 +0100 Subject: [PATCH 34/42] CI: skip musllinux --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7196c3dd2..cc16563eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,12 +70,13 @@ target-version = [ max-public-methods = 75 [tool.cibuildwheel] -# Disable building PyPy wheels on all platforms and 32bit for py3.10 -skip = "pp* cp310-win32 cp310-manylinux_i686" +# Disable building PyPy wheels on all platforms, 32bit for py3.10 and musllinux builds +skip = "pp* cp310-win32 cp310-manylinux_i686 *-musllinux_*" # Run the package tests using `pytest` test-extras = "test" test-command = "pytest -v {package}/tests" [[tool.cibuildwheel.overrides]] +# use manylinux2014 for py3.10 select = "cp310-*" manylinux-x86_64-image = "manylinux2014" From 42d4f512c394523fca8debd339d1e3810aa8df9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Mon, 15 Nov 2021 12:39:07 +0100 Subject: [PATCH 35/42] CI: use manylinux1 for cp3.6 wheels --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index cc16563eb..11366ff1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,11 @@ skip = "pp* cp310-win32 cp310-manylinux_i686 *-musllinux_*" test-extras = "test" test-command = "pytest -v {package}/tests" +[[tool.cibuildwheel.overrides]] +# use manylinux1 for py3.6 +select = "cp36-*" +manylinux-x86_64-image = "manylinux1" + [[tool.cibuildwheel.overrides]] # use manylinux2014 for py3.10 select = "cp310-*" From 26debc50ea582cd8c960f1cc94c0c1b125e63755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Tue, 16 Nov 2021 16:52:20 +0100 Subject: [PATCH 36/42] CI: auditwheel 4 is smart enough to build manylinux1 compatible wheels for py3.6 --- pyproject.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 11366ff1a..cc16563eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,11 +76,6 @@ skip = "pp* cp310-win32 cp310-manylinux_i686 *-musllinux_*" test-extras = "test" test-command = "pytest -v {package}/tests" -[[tool.cibuildwheel.overrides]] -# use manylinux1 for py3.6 -select = "cp36-*" -manylinux-x86_64-image = "manylinux1" - [[tool.cibuildwheel.overrides]] # use manylinux2014 for py3.10 select = "cp310-*" From 9944cca139b77fa28aabf3993852009cc4f5a8f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Wed, 17 Nov 2021 14:33:11 +0100 Subject: [PATCH 37/42] CI: separate jobs per arch for wheels; add universal2 wheel for macos --- .github/workflows/main.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d1c9472b1..a4de129a3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,12 +46,18 @@ jobs: python -m isort --check --diff --color . build_wheels: - name: wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} + name: wheels for ${{ matrix.cfg.os }} / ${{ matrix.cfg.arch }} + runs-on: ${{ matrix.cfg.os }} strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + cfg: + - { os: ubuntu-latest, arch: x86_64 } + - { os: ubuntu-latest, arch: i686 } + - { os: windows-latest, arch: AMD64 } + - { os: windows-latest, arch: x86 } + - { os: macos-latest, arch: x86_64 } + - { os: macos-latest, arch: universal2 } steps: - uses: actions/checkout@v2 @@ -60,6 +66,8 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.2.2 + env: + CIBW_ARCHS: ${{ matrix.cfg.arch }} with: output-dir: dist From 0f87dd44e6969f1246ade4620add12bf33156825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Wed, 17 Nov 2021 14:33:46 +0100 Subject: [PATCH 38/42] CI: also add arm64 wheels to be on the safe side --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a4de129a3..760d97bf1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,6 +57,7 @@ jobs: - { os: windows-latest, arch: AMD64 } - { os: windows-latest, arch: x86 } - { os: macos-latest, arch: x86_64 } + - { os: macos-latest, arch: arm64 } - { os: macos-latest, arch: universal2 } steps: From 791203fe979ed112a1fb083f2c3370585ca1cc51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Wed, 17 Nov 2021 14:58:16 +0100 Subject: [PATCH 39/42] CI: skip testing arm64 on macos --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index cc16563eb..172ab8c70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,8 @@ skip = "pp* cp310-win32 cp310-manylinux_i686 *-musllinux_*" # Run the package tests using `pytest` test-extras = "test" test-command = "pytest -v {package}/tests" +# Skip trying to test arm64 builds on Intel Macs +test-skip = "*-macosx_arm64 *-macosx_universal2:arm64" [[tool.cibuildwheel.overrides]] # use manylinux2014 for py3.10 From 6bb10bc60690727d8480dc431602427b67ab276a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Wed, 17 Nov 2021 15:41:37 +0100 Subject: [PATCH 40/42] Setup.cfg: add more classifiers --- setup.cfg | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/setup.cfg b/setup.cfg index 5a59f87c9..eade3901a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,9 +16,12 @@ classifiers = Intended Audience :: Developers Intended Audience :: End Users/Desktop Intended Audience :: Science/Research + Intended Audience :: Education License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3) Natural Language :: English Operating System :: Unix + Operating System :: Microsoft + Operating System :: MacOS Programming Language :: Python Programming Language :: Python :: 3 Programming Language :: Python :: 3 :: Only @@ -26,9 +29,15 @@ classifiers = Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 Topic :: Scientific/Engineering + Topic :: Scientific/Engineering :: GIS + Topic :: Scientific/Engineering :: Hydrology + Topic :: Scientific/Engineering :: Mathematics + Topic :: Scientific/Engineering :: Physics Topic :: Utilities project_urls = + Homepage = https://geostat-framework.org/#gstools Documentation = https://gstools.readthedocs.io Source = https://github.com/GeoStat-Framework/GSTools Tracker = https://github.com/GeoStat-Framework/GSTools/issues From 3d6c7870122dbc68f470d491f9eba237248612f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Wed, 17 Nov 2021 15:42:03 +0100 Subject: [PATCH 41/42] CI: use build to build wheels --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 172ab8c70..219f0bb79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,8 @@ target-version = [ max-public-methods = 75 [tool.cibuildwheel] +# Switch to using build +build-frontend = "build" # Disable building PyPy wheels on all platforms, 32bit for py3.10 and musllinux builds skip = "pp* cp310-win32 cp310-manylinux_i686 *-musllinux_*" # Run the package tests using `pytest` From c462c5094740351b6040e89e60cb2da2739b1af4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20M=C3=BCller?= Date: Wed, 17 Nov 2021 15:42:13 +0100 Subject: [PATCH 42/42] update Changelog --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82b9afd46..2f017a28b 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,12 @@ All notable changes to **GSTools** will be documented in this file. -## [1.3.4] - Pure Pink ? +## [1.3.4] - Pure Pink - 2021-11 ### Enhancements - add GStools-Core as optional dependency [#215](https://github.com/GeoStat-Framework/GSTools/pull/215) +- provide wheels for Python 3.10 [#211](https://github.com/GeoStat-Framework/GSTools/pull/211) +- provide macOS wheels for Apple Silicon [#211](https://github.com/GeoStat-Framework/GSTools/pull/211) ### Changes - remove unnecessary `dim` argument in Cython code [#216](https://github.com/GeoStat-Framework/GSTools/issues/216) @@ -319,7 +321,8 @@ See: [#197](https://github.com/GeoStat-Framework/GSTools/issues/197) First release of GSTools. -[Unreleased]: https://github.com/GeoStat-Framework/gstools/compare/v1.3.3...HEAD +[Unreleased]: https://github.com/GeoStat-Framework/gstools/compare/v1.3.4...HEAD +[1.3.4]: https://github.com/GeoStat-Framework/gstools/compare/v1.3.3...v1.3.4 [1.3.3]: https://github.com/GeoStat-Framework/gstools/compare/v1.3.2...v1.3.3 [1.3.2]: https://github.com/GeoStat-Framework/gstools/compare/v1.3.1...v1.3.2 [1.3.1]: https://github.com/GeoStat-Framework/gstools/compare/v1.3.0...v1.3.1