Search for GDAL and PROJ data folders during startup

docs/source/install.md

@@ -107,3 +107,16 @@ Also see `.github/test-windows.yml` for additional ideas if you run into problem
 
 Windows is minimally tested; we are currently unable to get automated tests
 working on our Windows CI.
+
+## GDAL and PROJ data files
+
+GDAL requires certain files to be present within a GDAL data folder, as well
+as a PROJ data folder. These folders are normally detected automatically.
+
+If you have an unusual installation of GDAL and PROJ, you may need to set
+additional environment variables at **runtime** in order for these to be
+correctly detected by GDAL:
+
+-   set `GDAL_DATA` to the folder containing the GDAL data files (e.g., contains `header.dxf`)
+    within the installation of GDAL that is used by Pyogrio.
+-   set `PROJ_LIB` to the folder containing the PROJ data files (e.g., contains `proj.db`)

pyogrio/_err.pxd

@@ -1,3 +1,4 @@
 cdef object exc_check()
 cdef int exc_wrap_int(int retval) except -1
+cdef int exc_wrap_ogrerr(int retval) except -1
 cdef void *exc_wrap_pointer(void *ptr) except NULL

pyogrio/_err.pyx

@@ -1,7 +1,9 @@
 # ported from fiona::_err.pyx
 from enum import IntEnum
 
-from pyogrio._ogr cimport *
+from pyogrio._ogr cimport (
+    CE_None, CE_Debug, CE_Warning, CE_Failure, CE_Fatal, CPLErrorReset,
+    CPLGetLastErrorType, CPLGetLastErrorNo, CPLGetLastErrorMsg, OGRErr)
 
 
 # CPL Error types as an enum.
@@ -182,6 +184,8 @@ cdef void *exc_wrap_pointer(void *ptr) except NULL:
 cdef int exc_wrap_int(int err) except -1:
     """Wrap a GDAL/OGR function that returns CPLErr or OGRErr (int)
     Raises an exception if a non-fatal error has be set.
+
+    Copied from Fiona (_err.pyx).
     """
     if err:
         exc = exc_check()
@@ -191,3 +195,15 @@ cdef int exc_wrap_int(int err) except -1:
             # no error message from GDAL
             raise CPLE_BaseError(-1, -1, "Unspecified OGR / GDAL error")
     return err
+
+
+cdef int exc_wrap_ogrerr(int err) except -1:
+    """Wrap a function that returns OGRErr (int) but does not use the
+    CPL error stack.
+
+    Adapted from Fiona (_err.pyx).
+    """
+    if err != 0:
+        raise CPLE_BaseError(3, err, f"OGR Error code {err}")
+
+    return err

pyogrio/_ogr.pxd

@@ -8,6 +8,7 @@ cdef extern from "cpl_conv.h":
     void*   CPLMalloc(size_t)
     void    CPLFree(void *ptr)
 
+    const char* CPLFindFile(const char *pszClass, const char *filename)
     const char* CPLGetConfigOption(const char* key, const char* value)
     void        CPLSetConfigOption(const char* key, const char* value)
 
@@ -161,11 +162,13 @@ cdef extern from "ogr_srs_api.h":
     ctypedef void* OGRSpatialReferenceH
 
     int                     OSRAutoIdentifyEPSG(OGRSpatialReferenceH srs)
+    OGRErr                  OSRExportToWkt(OGRSpatialReferenceH srs, char **params)
     const char*             OSRGetAuthorityName(OGRSpatialReferenceH srs, const char *key)
     const char*             OSRGetAuthorityCode(OGRSpatialReferenceH srs, const char *key)
-    OGRErr                  OSRExportToWkt(OGRSpatialReferenceH srs, char **params)
+    OGRErr                  OSRImportFromEPSG(OGRSpatialReferenceH srs, int code)
 
     int                     OSRSetFromUserInput(OGRSpatialReferenceH srs, const char *pszDef)
+    void                    OSRSetPROJSearchPaths(const char *const *paths)
     OGRSpatialReferenceH    OSRNewSpatialReference(const char *wkt)
     void                    OSRRelease(OGRSpatialReferenceH srs)
 

pyogrio/_ogr.pyx

@@ -1,3 +1,11 @@
+import os
+import sys
+import warnings
+
+from pyogrio._err cimport exc_wrap_int, exc_wrap_ogrerr
+from pyogrio._err import CPLE_BaseError
+
+
 cdef get_string(const char *c_str, str encoding="UTF-8"):
     """Get Python string from a char *
 
@@ -113,3 +121,112 @@ def ogr_list_drivers():
 
     return drivers
 
+
+cdef void set_proj_search_path(str path):
+    """Set PROJ library data file search path for use in GDAL."""
+    cdef char **paths = NULL
+    cdef const char *path_c = NULL
+    path_b = path.encode("utf-8")
+    path_c = path_b
+    paths = CSLAddString(paths, path_c)
+    OSRSetPROJSearchPaths(<const char *const *>paths)
+
+
+cdef char has_gdal_data():
+    """Verify that GDAL library data files are correctly found.
+
+    Adapted from Fiona (_env.pyx).
+    """
+
+    if CPLFindFile("gdal", "header.dxf") != NULL:
+        return True
+
+    return False
+
+
+cdef char has_proj_data():
+    """Verify that PROJ library data files are correctly found.
+
+    Returns
+    -------
+    bool
+        True if a test spatial reference object could be created, which verifies
+        that data files are correctly loaded.
+
+    Adapted from Fiona (_env.pyx).
+    """
+    cdef OGRSpatialReferenceH srs = OSRNewSpatialReference(NULL)
+
+    try:
+        exc_wrap_ogrerr(exc_wrap_int(OSRImportFromEPSG(srs, 4326)))
+    except CPLE_BaseError:
+        return 0
+    else:
+        return 1
+    finally:
+        if srs != NULL:
+            OSRRelease(srs)
+
+
+def init_gdal_data():
+    """Set GDAL data search directories in the following precedence:
+    - wheel copy of gdal_data
+    - default detection by GDAL, including GDAL_DATA (detected automatically by GDAL)
+    - other well-known paths under sys.prefix
+
+    Adapted from Fiona (env.py, _env.pyx).
+    """
+
+    # wheels are packaged to include GDAL data files at pyogrio/gdal_data
+    wheel_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "gdal_data"))
+    if os.path.exists(wheel_path):
+        set_gdal_config_options({"GDAL_DATA": wheel_path})
+        if not has_gdal_data():
+            raise ValueError("Could not correctly detect GDAL data files installed by pyogrio wheel")
+        return
+
+    # GDAL correctly found data files from GDAL_DATA or compiled-in paths
+    if has_gdal_data():
+        return
+
+    wk_path = os.path.join(sys.prefix, 'share', 'gdal')
+    if os.path.exists(wk_path):
+        set_gdal_config_options({"GDAL_DATA": wk_path})
+        if not has_gdal_data():
+            raise ValueError(f"Found GDAL data directory at {wk_path} but it does not appear to correctly contain GDAL data files")
+        return
+
+    warnings.warn("Could not detect GDAL data files.  Set GDAL_DATA environment variable to the correct path.", RuntimeWarning)
+
+
+def init_proj_data():
+    """Set Proj search directories in the following precedence:
+    - wheel copy of proj_data
+    - default detection by PROJ, including PROJ_LIB (detected automatically by PROJ)
+    - search other well-known paths under sys.prefix
+
+    Adapted from Fiona (env.py, _env.pyx).
+    """
+
+    # wheels are packaged to include PROJ data files at pyogrio/proj_data
+    wheel_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "proj_data"))
+    if os.path.exists(wheel_path):
+        set_proj_search_path(wheel_path)
+        # verify that this now resolves
+        if not has_proj_data():
+            raise ValueError("Could not correctly detect PROJ data files installed by pyogrio wheel")
+        return
+
+    # PROJ correctly found data files from PROJ_LIB or compiled-in paths
+    if has_proj_data():
+        return
+
+    wk_path = os.path.join(sys.prefix, 'share', 'proj')
+    if os.path.exists(wk_path):
+        set_proj_search_path(wk_path)
+        # verify that this now resolves
+        if not has_proj_data():
+            raise ValueError(f"Found PROJ data directory at {wk_path} but it does not appear to correctly contain PROJ data files")
+        return
+
+    warnings.warn("Could not detect PROJ data files.  Set PROJ_LIB environment variable to the correct path.", RuntimeWarning)

pyogrio/core.py

@@ -9,9 +9,14 @@
         ogr_list_drivers,
         set_gdal_config_options as _set_gdal_config_options,
         get_gdal_config_option as _get_gdal_config_option,
+        init_gdal_data as _init_gdal_data,
+        init_proj_data as _init_proj_data,
     )
     from pyogrio._io import ogr_list_layers, ogr_read_bounds, ogr_read_info
 
+    _init_gdal_data()
+    _init_proj_data()
+
     __gdal_version__ = get_gdal_version()
     __gdal_version_string__ = get_gdal_version_string()