From 4ab5cb218b5752e47929df012e5be2ade9aa27de Mon Sep 17 00:00:00 2001 From: Nic Watson Date: Wed, 26 Jun 2024 00:17:18 -0400 Subject: [PATCH] Update liblmdb, add building for 3.12 (#361) * Pull lmdb 0.9.31 from upstream. * Get Python 3.12 building in CI. * Remove 2.7 builds as Github Actions removed support * Add `import setuptools` to CI since not in 3.12 venv anymore * Update supported versions, fix #358 * Update py-lmdb patch for lmdb 0.9.31 * Remove 2.7 cruft * Drop 3.7 building on MacOS since not easily supported in CI * Update changelog * Reduce tests a bit --- .github/workflows/python-package.yml | 73 ++++------- ChangeLog | 7 ++ README.md | 10 +- docs/index.rst | 34 +----- lib/lmdb.h | 4 +- lib/mdb.c | 174 ++++++++++++++++----------- lib/midl.c | 4 +- lib/midl.h | 4 +- lib/py-lmdb/env-copy-txn.patch | 16 +-- lmdb/__main__.py | 4 +- lmdb/cpython.c | 56 +-------- setup.py | 14 +-- 12 files changed, 165 insertions(+), 235 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 04d6c41d..3d8fb9cd 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,29 +16,22 @@ jobs: strategy: matrix: os: [ubuntu-20.04, macos-latest, windows-latest] - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11', pypy-2.7, pypy-3.9] + python-version: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12', pypy-3.10] impl: [cpython, cffi] purity: [pure, with-pylmdb-mods] exclude: - - python-version: pypy-2.7 + # Pypy doesn't work with cpython + - python-version: pypy-3.10 impl: cpython - - python-version: pypy-3.9 - impl: cpython - # CFFI has trouble building on Windows for 3.5 "self.find_available_vc_vers()[-1]" fails - - python-version: 3.5 - os: windows-latest - # Microsoft removed VC 9.0 installer so Python 2.7 modules can no longer be built on Windows - - python-version: 2.7 - os: windows-latest - # Doesn't seem to exist on github MacOS - - python-version: pypy-3.9 + # macos latest is now arm64 and 3.7 doesn't have arm64 build + - python-version: 3.7 os: macos-latest steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Set env vars Windows @@ -79,7 +72,7 @@ jobs: cffi=$LMDB_FORCE_CFFI pure=$LMDB_PURE system=$LMDB_FORCE_SYSTEM" echo "Windows: Envs are cpython=$Env:LMDB_FORCE_CPYTHON cffi=$Env:LMDB_FORCE_CFFI pure=$Env:LMDB_PURE system=$Env:LMDB_FORCE_SYSTEM" - python -m pip install wheel + python -m pip install setuptools wheel # Install this separately since sometimes Github Actions can't find it python -m pip install cffi python -m pip install flake8 pytest patch-ng @@ -94,7 +87,9 @@ jobs: # - name: Test with pytest # Limit the test cycle a little - if: matrix.python-version != '3.6' && matrix.python-version != '3.8' + if: >- + matrix.python-version != '3.7' && matrix.python-version != '3.9' && + matrix.python-version != '3.11' run: | echo "Envs are cpython=$LMDB_FORCE_CPYTHON cffi=$LMDB_FORCE_CFFI pure=$LMDB_PURE system=$LMDB_FORCE_SYSTEM" @@ -141,7 +136,7 @@ jobs: uses: RalfG/python-wheels-manylinux-build@v0.7.1 with: python-versions: >- - cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 + cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 build-requirements: 'patch-ng' - name: What do we have @@ -165,7 +160,7 @@ jobs: matrix: impl: [cpython, cffi] purity: [pure, with-pylmdb-mods] - PYTHON: ["cp36-cp36m", "cp37-cp37m" , "cp38-cp38" , "cp39-cp39", "cp310-cp310", "cp311-cp311"] + PYTHON: ["cp37-cp37m" , "cp38-cp38" , "cp39-cp39", "cp310-cp310", "cp311-cp311"] steps: - uses: actions/checkout@v2 - run: | @@ -179,7 +174,7 @@ jobs: python --version; /opt/python/${{ matrix.PYTHON }}/bin/python -m venv .venv; yum install -y libffi-devel; - .venv/bin/pip install -U pip wheel cffi six; + .venv/bin/pip install -U pip setuptools wheel cffi six; if \[ ${{ matrix.impl }} == cpython \] ; then echo LMDB_FORCE_CPYTHON=1 else @@ -194,7 +189,7 @@ jobs: cffi=$LMDB_FORCE_CFFI pure=$LMDB_PURE system=$LMDB_FORCE_SYSTEM\"; echo \"Windows: Envs are cpython=$Env:LMDB_FORCE_CPYTHON cffi=$Env:LMDB_FORCE_CFFI pure=$Env:LMDB_PURE system=$Env:LMDB_FORCE_SYSTEM\"; - .venv/bin/pip install flake8 pytest patch-ng; + .venv/bin/pip install setuptools flake8 pytest patch-ng; /opt/python/${{ matrix.PYTHON }}/bin/python setup.py develop bdist_wheel; ls dist; /opt/python/${{ matrix.PYTHON }}/bin/python -m pip install pytest; @@ -212,29 +207,29 @@ jobs: - uses: actions/upload-artifact@v2 if: >- - matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' && + matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' && matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython' with: path: vers.txt name: vers.txt - uses: actions/upload-artifact@v2 if: >- - matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' && + matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' && matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython' with: path: dist/lmdb*.tar.gz name: source - uses: RalfG/python-wheels-manylinux-build@v0.7.1-manylinux2014_aarch64 if: >- - matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' && + matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' && matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython' with: python-versions: >- - cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 + cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 build-requirements: 'patch-ng' - uses: actions/upload-artifact@v2 if: >- - matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' && + matrix.PYTHON == 'cp310-cp310' && runner.os == 'Linux' && matrix.purity == 'with-pylmdb-mods' && matrix.impl == 'cpython' with: path: dist/lmdb*manylinux*.whl @@ -248,41 +243,15 @@ jobs: # We publish a subset of the targets we test matrix: os: [macos-latest, windows-latest] - python-version: ['2.7', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', pypy-2.7, pypy-3.9] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', pypy-3.10] impl: [cpython, cffi] purity: [with-pylmdb-mods] exclude: - - python-version: 'pypy-2.7' - impl: cpython - - python-version: 'pypy-3.9' + - python-version: 'pypy-3.10' impl: cpython - - python-version: '2.7' - impl: cffi - - python-version: '3.5' - impl: cffi - - python-version: '3.6' - impl: cffi - python-version: '3.7' - impl: cffi - - python-version: '3.8' - impl: cffi - - python-version: '3.9' - impl: cffi - - python-version: '3.10' - impl: cffi - - python-version: '3.11' - impl: cffi - # CFFI has trouble building on Windows for 3.5 "self.find_available_vc_vers()[-1]" fails - - python-version: 3.5 - os: windows-latest - # Microsoft removed VC 9.0 installer so Python 2.7 modules can no longer be built on Windows - - python-version: '2.7' - os: windows-latest - # Doesn't seem to exist on github MacOS - - python-version: pypy-3.9 os: macos-latest - include: # Ubuntu artifacts apply to all python versions - os: ubuntu-20.04 diff --git a/ChangeLog b/ChangeLog index 9a10b229..7ae23362 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,10 @@ +2024-06-25 1.5.0 +* Add Python 3.12 binaries. + +* Update bundled LMDB to 0.9.31. + +* Remove Python 2.7 support. + 2022-04-04 v1.4.1 * Update CI to build manylinux binaries. diff --git a/README.md b/README.md index aa20fcb1..1377b2a2 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,7 @@ See [the documentation](https://lmdb.readthedocs.io) for more information. # Python Version Support Statement -This project has been around for a while. Previously, it supported all the way back to before 2.5. Currently py-lmdb -supports Python 2.7, Python >= 3.5, and pypy. - -Python 2.7 is now end-of-life. If you are still using Python 2.7, you should strongly considering porting to Python -3. - -That said, this project will continue to support running on Python 2.7 until Github Actions remove support for it. +This project has been around for a while. Previously, it supported all the way back to before 2.5. Currently, py-lmdb +supports Python >= 3.5 and pypy. +The last version of py-lmdb that supported Python 2.7 was 1.4.1. diff --git a/docs/index.rst b/docs/index.rst index 1e9e917d..fcd57f93 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,9 +11,8 @@ lmdb This is a universal Python binding for the `LMDB 'Lightning' Database `_. Two variants are provided and automatically selected during install: a `CFFI `_ variant -that supports `PyPy `_ and all versions of CPython >=2.7, -and a C extension that supports CPython >=2.7 and >=3.4. Both variants -provide the same interface. +that supports `PyPy `_ and all versions of CPython >=3.5, +and a C extension that supports >= 3.5. Both variants provide the same interface. LMDB is a tiny database with some excellent properties: @@ -41,21 +40,12 @@ to be installed via pip and easy_install without the need for a compiler to be present. The binary releases statically link against the bundled version of LMDB. -Initially 32-bit and 64-bit binaries are provided for Python 2.7; in future -binaries will be published for all supported versions of Python. - To install, use a command like: :: C:\Python27\python -mpip install lmdb -Or: - - :: - - C:\Python27\python -measy_install lmdb - Installation: UNIX ++++++++++++++++++ @@ -78,14 +68,12 @@ depend on the CPython development headers. On Debian/Ubuntu: apt-get install libffi-dev python-dev build-essential -To install the C extension, ensure a C compiler and `pip` or `easy_install` are +To install the C extension, ensure a C compiler and `pip` are available and type: :: pip install lmdb - # or - easy_install lmdb The CFFI variant may be used on CPython by setting the ``LMDB_FORCE_CFFI`` environment variable before installation, or before module import with an @@ -194,21 +182,11 @@ written the dirty pages to disk. Bytestrings +++++++++++ -This documentation uses `bytestring` to mean either the Python<=2.7 -:py:func:`str` type, or the Python>=3.0 :py:func:`bytes` type, depending on the -Python version in use. - -Due to the design of Python 2.x, LMDB will happily accept Unicode instances -where :py:func:`str` instances are expected, so long as they contain only ASCII -characters, in which case they are implicitly encoded to ASCII. You should not -rely on this behaviour! It results in brittle programs that often break the -moment they are deployed in production. Always explicitly encode and decode any -Unicode values before passing them to LMDB. +This documentation uses `bytestring` to mean the Python>=3.0 :py:func:`bytes` +type. This documentation uses :py:func:`bytes` in examples. In Python 3.x this is a -distinct type, whereas in Python 2.7 it is simply an alias for -:py:func:`str`. - +distinct type. Buffers +++++++ diff --git a/lib/lmdb.h b/lib/lmdb.h index 69aa2751..ff03c224 100644 --- a/lib/lmdb.h +++ b/lib/lmdb.h @@ -200,7 +200,7 @@ typedef int mdb_filehandle_t; /** Library minor version */ #define MDB_VERSION_MINOR 9 /** Library patch version */ -#define MDB_VERSION_PATCH 29 +#define MDB_VERSION_PATCH 31 /** Combine args a,b,c into a single integer for easy version comparisons */ #define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) @@ -210,7 +210,7 @@ typedef int mdb_filehandle_t; MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) /** The release date of this library version */ -#define MDB_VERSION_DATE "March 16, 2021" +#define MDB_VERSION_DATE "July 10, 2023" /** A stringifier for the version info */ #define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")" diff --git a/lib/mdb.c b/lib/mdb.c index 8cecdb2e..deb67796 100644 --- a/lib/mdb.c +++ b/lib/mdb.c @@ -77,8 +77,7 @@ #if defined(__mips) && defined(__linux) /* MIPS has cache coherency issues, requires explicit cache control */ -#include -extern int cacheflush(char *addr, int nbytes, int cache); +#include #define CACHEFLUSH(addr, bytes, cache) cacheflush(addr, bytes, cache) #else #define CACHEFLUSH(addr, bytes, cache) @@ -200,15 +199,21 @@ typedef SSIZE_T ssize_t; # error "Two's complement, reasonably sized integer types, please" #endif -#ifdef __GNUC__ -/** Put infrequently used env functions in separate section */ -# ifdef __APPLE__ -# define ESECT __attribute__ ((section("__TEXT,text_env"))) +#if (((__clang_major__ << 8) | __clang_minor__) >= 0x0302) || (((__GNUC__ << 8) | __GNUC_MINOR__) >= 0x0403) +/** Mark infrequently used env functions as cold. This puts them in a separate + * section, and optimizes them for size */ +#define ESECT __attribute__ ((cold)) +#else +/* On older compilers, use a separate section */ +# ifdef __GNUC__ +# ifdef __APPLE__ +# define ESECT __attribute__ ((section("__TEXT,text_env"))) +# else +# define ESECT __attribute__ ((section("text_env"))) +# endif # else -# define ESECT __attribute__ ((section("text_env"))) +# define ESECT # endif -#else -#define ESECT #endif #ifdef _WIN32 @@ -833,9 +838,26 @@ typedef struct MDB_page { } pb; uint32_t pb_pages; /**< number of overflow pages */ } mp_pb; - indx_t mp_ptrs[1]; /**< dynamic size */ + indx_t mp_ptrs[0]; /**< dynamic size */ } MDB_page; +/** Alternate page header, for 2-byte aligned access */ +typedef struct MDB_page2 { + uint16_t mp2_p[sizeof(pgno_t)/2]; + uint16_t mp2_pad; + uint16_t mp2_flags; + indx_t mp2_lower; + indx_t mp2_upper; + indx_t mp2_ptrs[0]; +} MDB_page2; + +#define MP_PGNO(p) (((MDB_page2 *)(void *)(p))->mp2_p) +#define MP_PAD(p) (((MDB_page2 *)(void *)(p))->mp2_pad) +#define MP_FLAGS(p) (((MDB_page2 *)(void *)(p))->mp2_flags) +#define MP_LOWER(p) (((MDB_page2 *)(void *)(p))->mp2_lower) +#define MP_UPPER(p) (((MDB_page2 *)(void *)(p))->mp2_upper) +#define MP_PTRS(p) (((MDB_page2 *)(void *)(p))->mp2_ptrs) + /** Size of the page header, excluding dynamic data at the end */ #define PAGEHDRSZ ((unsigned) offsetof(MDB_page, mp_ptrs)) @@ -846,10 +868,10 @@ typedef struct MDB_page { #define PAGEBASE ((MDB_DEVEL) ? PAGEHDRSZ : 0) /** Number of nodes on a page */ -#define NUMKEYS(p) (((p)->mp_lower - (PAGEHDRSZ-PAGEBASE)) >> 1) +#define NUMKEYS(p) ((MP_LOWER(p) - (PAGEHDRSZ-PAGEBASE)) >> 1) /** The amount of space remaining in the page */ -#define SIZELEFT(p) (indx_t)((p)->mp_upper - (p)->mp_lower) +#define SIZELEFT(p) (indx_t)(MP_UPPER(p) - MP_LOWER(p)) /** The percentage of space used in the page, in tenths of a percent. */ #define PAGEFILL(env, p) (1000L * ((env)->me_psize - PAGEHDRSZ - SIZELEFT(p)) / \ @@ -860,15 +882,15 @@ typedef struct MDB_page { #define FILL_THRESHOLD 250 /** Test if a page is a leaf page */ -#define IS_LEAF(p) F_ISSET((p)->mp_flags, P_LEAF) +#define IS_LEAF(p) F_ISSET(MP_FLAGS(p), P_LEAF) /** Test if a page is a LEAF2 page */ -#define IS_LEAF2(p) F_ISSET((p)->mp_flags, P_LEAF2) +#define IS_LEAF2(p) F_ISSET(MP_FLAGS(p), P_LEAF2) /** Test if a page is a branch page */ -#define IS_BRANCH(p) F_ISSET((p)->mp_flags, P_BRANCH) +#define IS_BRANCH(p) F_ISSET(MP_FLAGS(p), P_BRANCH) /** Test if a page is an overflow page */ -#define IS_OVERFLOW(p) F_ISSET((p)->mp_flags, P_OVERFLOW) +#define IS_OVERFLOW(p) F_ISSET(MP_FLAGS(p), P_OVERFLOW) /** Test if a page is a sub page */ -#define IS_SUBP(p) F_ISSET((p)->mp_flags, P_SUBP) +#define IS_SUBP(p) F_ISSET(MP_FLAGS(p), P_SUBP) /** The number of overflow pages needed to store the given size. */ #define OVPAGES(size, psize) ((PAGEHDRSZ-1 + (size)) / (psize) + 1) @@ -936,7 +958,7 @@ typedef struct MDB_node { #define LEAFSIZE(k, d) (NODESIZE + (k)->mv_size + (d)->mv_size) /** Address of node \b i in page \b p */ -#define NODEPTR(p, i) ((MDB_node *)((char *)(p) + (p)->mp_ptrs[i] + PAGEBASE)) +#define NODEPTR(p, i) ((MDB_node *)((char *)(p) + MP_PTRS(p)[i] + PAGEBASE)) /** Address of the key for the node */ #define NODEKEY(node) (void *)((node)->mn_data) @@ -964,6 +986,8 @@ typedef struct MDB_node { /** Copy a page number from src to dst */ #ifdef MISALIGNED_OK #define COPY_PGNO(dst,src) dst = src +#undef MP_PGNO +#define MP_PGNO(p) ((p)->mp_pgno) #else #if SIZE_MAX > 4294967295UL #define COPY_PGNO(dst,src) do { \ @@ -1517,6 +1541,8 @@ mdb_strerror(int err) NULL, err, 0, ptr, MSGSIZE, (va_list *)buf+MSGSIZE); return ptr; #else + if (err < 0) + return "Invalid error code"; return strerror(err); #endif } @@ -1554,7 +1580,7 @@ static pgno_t mdb_dbg_pgno(MDB_page *mp) { pgno_t ret; - COPY_PGNO(ret, mp->mp_pgno); + COPY_PGNO(ret, MP_PGNO(mp)); return ret; } @@ -1601,13 +1627,13 @@ void mdb_page_list(MDB_page *mp) { pgno_t pgno = mdb_dbg_pgno(mp); - const char *type, *state = (mp->mp_flags & P_DIRTY) ? ", dirty" : ""; + const char *type, *state = (MP_FLAGS(mp) & P_DIRTY) ? ", dirty" : ""; MDB_node *node; unsigned int i, nkeys, nsize, total = 0; MDB_val key; DKBUF; - switch (mp->mp_flags & (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP)) { + switch (MP_FLAGS(mp) & (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP)) { case P_BRANCH: type = "Branch page"; break; case P_LEAF: type = "Leaf page"; break; case P_LEAF|P_SUBP: type = "Sub-page"; break; @@ -1622,7 +1648,7 @@ mdb_page_list(MDB_page *mp) pgno, ((MDB_meta *)METADATA(mp))->mm_txnid); return; default: - fprintf(stderr, "Bad page %"Z"u flags 0x%X\n", pgno, mp->mp_flags); + fprintf(stderr, "Bad page %"Z"u flags 0x%X\n", pgno, MP_FLAGS(mp)); return; } @@ -1658,7 +1684,7 @@ mdb_page_list(MDB_page *mp) total = EVEN(total); } fprintf(stderr, "Total: header %d + contents %d + unused %d\n", - IS_LEAF2(mp) ? PAGEHDRSZ : PAGEBASE + mp->mp_lower, total, SIZELEFT(mp)); + IS_LEAF2(mp) ? PAGEHDRSZ : PAGEBASE + MP_LOWER(mp), total, SIZELEFT(mp)); } void @@ -2418,7 +2444,7 @@ mdb_page_touch(MDB_cursor *mc) pgno_t pgno; int rc; - if (!F_ISSET(mp->mp_flags, P_DIRTY)) { + if (!F_ISSET(MP_FLAGS(mp), P_DIRTY)) { if (txn->mt_flags & MDB_TXN_SPILLS) { np = NULL; rc = mdb_page_unspill(txn, mp, &np); @@ -6061,7 +6087,7 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, mc->mc_ki[mc->mc_top] = 0; return MDB_NOTFOUND; } - if (mp->mp_flags & P_LEAF2) { + if (MP_FLAGS(mp) & P_LEAF2) { nodekey.mv_size = mc->mc_db->md_pad; nodekey.mv_data = LEAF2KEY(mp, 0, nodekey.mv_size); } else { @@ -6082,7 +6108,7 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, unsigned int i; unsigned int nkeys = NUMKEYS(mp); if (nkeys > 1) { - if (mp->mp_flags & P_LEAF2) { + if (MP_FLAGS(mp) & P_LEAF2) { nodekey.mv_data = LEAF2KEY(mp, nkeys-1, nodekey.mv_size); } else { @@ -6100,7 +6126,7 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, if (rc < 0) { if (mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) { /* This is definitely the right page, skip search_page */ - if (mp->mp_flags & P_LEAF2) { + if (MP_FLAGS(mp) & P_LEAF2) { nodekey.mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], nodekey.mv_size); } else { @@ -6477,6 +6503,7 @@ mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data, rc = MDB_NOTFOUND; break; } + mc->mc_flags &= ~C_EOF; { MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { @@ -6663,7 +6690,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, *mc->mc_dbflag |= DB_DIRTY; if ((mc->mc_db->md_flags & (MDB_DUPSORT|MDB_DUPFIXED)) == MDB_DUPFIXED) - np->mp_flags |= P_LEAF2; + MP_FLAGS(np) |= P_LEAF2; mc->mc_flags |= C_INITIALIZED; } else { /* make sure all cursor pages are writable */ @@ -6685,7 +6712,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, fp_flags = P_LEAF|P_DIRTY; fp = env->me_pbuf; fp->mp_pad = data->mv_size; /* used if MDB_DUPFIXED */ - fp->mp_lower = fp->mp_upper = (PAGEHDRSZ-PAGEBASE); + MP_LOWER(fp) = MP_UPPER(fp) = (PAGEHDRSZ-PAGEBASE); olddata.mv_size = PAGEHDRSZ; goto prep_subDB; } @@ -6761,18 +6788,18 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, dkey.mv_data = memcpy(fp+1, olddata.mv_data, olddata.mv_size); /* Make sub-page header for the dup items, with dummy body */ - fp->mp_flags = P_LEAF|P_DIRTY|P_SUBP; - fp->mp_lower = (PAGEHDRSZ-PAGEBASE); + MP_FLAGS(fp) = P_LEAF|P_DIRTY|P_SUBP; + MP_LOWER(fp) = (PAGEHDRSZ-PAGEBASE); xdata.mv_size = PAGEHDRSZ + dkey.mv_size + data->mv_size; if (mc->mc_db->md_flags & MDB_DUPFIXED) { - fp->mp_flags |= P_LEAF2; + MP_FLAGS(fp) |= P_LEAF2; fp->mp_pad = data->mv_size; xdata.mv_size += 2 * data->mv_size; /* leave space for 2 more */ } else { xdata.mv_size += 2 * (sizeof(indx_t) + NODESIZE) + (dkey.mv_size & 1) + (data->mv_size & 1); } - fp->mp_upper = xdata.mv_size - PAGEBASE; + MP_UPPER(fp) = xdata.mv_size - PAGEBASE; olddata.mv_size = xdata.mv_size; /* pretend olddata is fp */ } else if (leaf->mn_flags & F_SUBDATA) { /* Data is on sub-DB, just store it */ @@ -6795,8 +6822,8 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, } /* FALLTHRU */ /* Big enough MDB_DUPFIXED sub-page */ case MDB_CURRENT: - fp->mp_flags |= P_DIRTY; - COPY_PGNO(fp->mp_pgno, mp->mp_pgno); + MP_FLAGS(fp) |= P_DIRTY; + COPY_PGNO(MP_PGNO(fp), MP_PGNO(mp)); mc->mc_xcursor->mx_cursor.mc_pg[0] = fp; flags |= F_DUPDATA; goto put_sub; @@ -6804,7 +6831,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, xdata.mv_size = olddata.mv_size + offset; } - fp_flags = fp->mp_flags; + fp_flags = MP_FLAGS(fp); if (NODESIZE + NODEKSZ(leaf) + xdata.mv_size > env->me_nodemax) { /* Too big for a sub-page, convert to sub-DB */ fp_flags &= ~P_SUBP; @@ -6834,16 +6861,16 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, sub_root = mp; } if (mp != fp) { - mp->mp_flags = fp_flags | P_DIRTY; - mp->mp_pad = fp->mp_pad; - mp->mp_lower = fp->mp_lower; - mp->mp_upper = fp->mp_upper + offset; + MP_FLAGS(mp) = fp_flags | P_DIRTY; + MP_PAD(mp) = MP_PAD(fp); + MP_LOWER(mp) = MP_LOWER(fp); + MP_UPPER(mp) = MP_UPPER(fp) + offset; if (fp_flags & P_LEAF2) { memcpy(METADATA(mp), METADATA(fp), NUMKEYS(fp) * fp->mp_pad); } else { - memcpy((char *)mp + mp->mp_upper + PAGEBASE, (char *)fp + fp->mp_upper + PAGEBASE, - olddata.mv_size - fp->mp_upper - PAGEBASE); - memcpy((char *)(&mp->mp_ptrs), (char *)(&fp->mp_ptrs), NUMKEYS(fp) * sizeof(mp->mp_ptrs[0])); + memcpy((char *)mp + MP_UPPER(mp) + PAGEBASE, (char *)fp + MP_UPPER(fp) + PAGEBASE, + olddata.mv_size - MP_UPPER(fp) - PAGEBASE); + memcpy((char *)MP_PTRS(mp), (char *)MP_PTRS(fp), NUMKEYS(fp) * sizeof(mp->mp_ptrs[0])); for (i=0; imp_ptrs[i] += offset; } @@ -6908,7 +6935,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, * Copy end of page, adjusting alignment so * compiler may copy words instead of bytes. */ - off = (PAGEHDRSZ + data->mv_size) & -sizeof(size_t); + off = (PAGEHDRSZ + data->mv_size) & -(int)sizeof(size_t); memcpy((size_t *)((char *)np + off), (size_t *)((char *)omp + off), sz - off); sz = PAGEHDRSZ; @@ -6936,11 +6963,14 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, else if (!(mc->mc_flags & C_SUB)) memcpy(olddata.mv_data, data->mv_data, data->mv_size); else { + if (key->mv_size != NODEKSZ(leaf)) + goto new_ksize; memcpy(NODEKEY(leaf), key->mv_data, key->mv_size); goto fix_parent; } return MDB_SUCCESS; } +new_ksize: mdb_node_del(mc, 0); } @@ -7291,7 +7321,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, void *ndata; DKBUF; - mdb_cassert(mc, mp->mp_upper >= mp->mp_lower); + mdb_cassert(mc, MP_UPPER(mp) >= MP_LOWER(mp)); DPRINTF(("add to %s %spage %"Z"u index %i, data size %"Z"u key size %"Z"u [%s]", IS_LEAF(mp) ? "leaf" : "branch", @@ -7310,8 +7340,8 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, memcpy(ptr, key->mv_data, ksize); /* Just using these for counting */ - mp->mp_lower += sizeof(indx_t); - mp->mp_upper -= ksize - sizeof(indx_t); + MP_LOWER(mp) += sizeof(indx_t); + MP_UPPER(mp) -= ksize - sizeof(indx_t); return MDB_SUCCESS; } @@ -7348,14 +7378,14 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, update: /* Move higher pointers up one slot. */ for (i = NUMKEYS(mp); i > indx; i--) - mp->mp_ptrs[i] = mp->mp_ptrs[i - 1]; + MP_PTRS(mp)[i] = MP_PTRS(mp)[i - 1]; /* Adjust free space offsets. */ - ofs = mp->mp_upper - node_size; - mdb_cassert(mc, ofs >= mp->mp_lower + sizeof(indx_t)); - mp->mp_ptrs[indx] = ofs; - mp->mp_upper = ofs; - mp->mp_lower += sizeof(indx_t); + ofs = MP_UPPER(mp) - node_size; + mdb_cassert(mc, ofs >= MP_LOWER(mp) + sizeof(indx_t)); + MP_PTRS(mp)[indx] = ofs; + MP_UPPER(mp) = ofs; + MP_LOWER(mp) += sizeof(indx_t); /* Write the node data. */ node = NODEPTR(mp, indx); @@ -7393,7 +7423,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, full: DPRINTF(("not enough room in page %"Z"u, got %u ptrs", mdb_dbg_pgno(mp), NUMKEYS(mp))); - DPRINTF(("upper-lower = %u - %u = %"Z"d", mp->mp_upper,mp->mp_lower,room)); + DPRINTF(("upper-lower = %u - %u = %"Z"d", MP_UPPER(mp),MP_LOWER(mp),room)); DPRINTF(("node size = %"Z"u", node_size)); mc->mc_txn->mt_flags |= MDB_TXN_ERROR; return MDB_PAGE_FULL; @@ -7424,8 +7454,8 @@ mdb_node_del(MDB_cursor *mc, int ksize) base = LEAF2KEY(mp, indx, ksize); if (x) memmove(base, base + ksize, x * ksize); - mp->mp_lower -= sizeof(indx_t); - mp->mp_upper += ksize - sizeof(indx_t); + MP_LOWER(mp) -= sizeof(indx_t); + MP_UPPER(mp) += ksize - sizeof(indx_t); return; } @@ -7439,21 +7469,21 @@ mdb_node_del(MDB_cursor *mc, int ksize) } sz = EVEN(sz); - ptr = mp->mp_ptrs[indx]; + ptr = MP_PTRS(mp)[indx]; for (i = j = 0; i < numkeys; i++) { if (i != indx) { - mp->mp_ptrs[j] = mp->mp_ptrs[i]; - if (mp->mp_ptrs[i] < ptr) - mp->mp_ptrs[j] += sz; + MP_PTRS(mp)[j] = MP_PTRS(mp)[i]; + if (MP_PTRS(mp)[i] < ptr) + MP_PTRS(mp)[j] += sz; j++; } } - base = (char *)mp + mp->mp_upper + PAGEBASE; - memmove(base + sz, base, ptr - mp->mp_upper); + base = (char *)mp + MP_UPPER(mp) + PAGEBASE; + memmove(base + sz, base, ptr - MP_UPPER(mp)); - mp->mp_lower -= sizeof(indx_t); - mp->mp_upper += sz; + MP_LOWER(mp) -= sizeof(indx_t); + MP_UPPER(mp) += sz; } /** Compact the main page after deleting a node on a subpage. @@ -7482,11 +7512,11 @@ mdb_node_shrink(MDB_page *mp, indx_t indx) } else { xp = (MDB_page *)((char *)sp + delta); /* destination subpage */ for (i = NUMKEYS(sp); --i >= 0; ) - xp->mp_ptrs[i] = sp->mp_ptrs[i] - delta; + MP_PTRS(xp)[i] = MP_PTRS(sp)[i] - delta; len = PAGEHDRSZ; } - sp->mp_upper = sp->mp_lower; - COPY_PGNO(sp->mp_pgno, mp->mp_pgno); + MP_UPPER(sp) = MP_LOWER(sp); + COPY_PGNO(MP_PGNO(sp), mp->mp_pgno); SETDSZ(node, nsize); /* Shift upward */ @@ -7557,7 +7587,7 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) mx->mx_db.md_leaf_pages = 1; mx->mx_db.md_overflow_pages = 0; mx->mx_db.md_entries = NUMKEYS(fp); - COPY_PGNO(mx->mx_db.md_root, fp->mp_pgno); + COPY_PGNO(mx->mx_db.md_root, MP_PGNO(fp)); mx->mx_cursor.mc_snum = 1; mx->mx_cursor.mc_top = 0; mx->mx_cursor.mc_flags = C_INITIALIZED|C_SUB; @@ -8729,9 +8759,13 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno mc->mc_ki[mc->mc_top] = x; } } else { - int psize, nsize, k; + int psize, nsize, k, keythresh; + /* Maximum free space in an empty page */ pmax = env->me_psize - PAGEHDRSZ; + /* Threshold number of keys considered "small" */ + keythresh = env->me_psize >> 7; + if (IS_LEAF(mp)) nsize = mdb_leaf_size(env, newkey, newdata); else @@ -8772,7 +8806,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno * the split so the new page is emptier than the old page. * This yields better packing during sequential inserts. */ - if (nkeys < 32 || nsize > pmax/16 || newindx >= nkeys) { + if (nkeys < keythresh || nsize > pmax/16 || newindx >= nkeys) { /* Find split point */ psize = 0; if (newindx <= split_indx || newindx >= nkeys) { diff --git a/lib/midl.c b/lib/midl.c index ab321291..b0ea5383 100644 --- a/lib/midl.c +++ b/lib/midl.c @@ -3,8 +3,8 @@ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * - * Copyright 2000-2020 The OpenLDAP Foundation. - * Portions Copyright 2001-2020 Howard Chu, Symas Corp. + * Copyright 2000-2021 The OpenLDAP Foundation. + * Portions Copyright 2001-2021 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/lib/midl.h b/lib/midl.h index 6bb9cf60..dd6ae77c 100644 --- a/lib/midl.h +++ b/lib/midl.h @@ -11,8 +11,8 @@ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * - * Copyright 2000-2020 The OpenLDAP Foundation. - * Portions Copyright 2001-2020 Howard Chu, Symas Corp. + * Copyright 2000-2021 The OpenLDAP Foundation. + * Portions Copyright 2001-2021 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/lib/py-lmdb/env-copy-txn.patch b/lib/py-lmdb/env-copy-txn.patch index 8ae79921..a118618d 100644 --- a/lib/py-lmdb/env-copy-txn.patch +++ b/lib/py-lmdb/env-copy-txn.patch @@ -1,5 +1,5 @@ diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h -index 69aa2751a2..ff7f77a10d 100644 +index ff03c22..3d6f4cc 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -682,9 +682,14 @@ int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); @@ -32,10 +32,10 @@ index 69aa2751a2..ff7f77a10d 100644 /** @brief Return statistics about the LMDB environment. * diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c -index 8cecdb2e69..9ede4183dd 100644 +index deb6779..b5d152c 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c -@@ -9332,12 +9332,12 @@ done: +@@ -9366,12 +9366,12 @@ done: /** Copy environment with compaction. */ static int ESECT @@ -50,7 +50,7 @@ index 8cecdb2e69..9ede4183dd 100644 pthread_t thr; pgno_t root, new_root; int rc = MDB_SUCCESS; -@@ -9383,9 +9383,11 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd) +@@ -9417,9 +9417,11 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd) if (rc) goto done; @@ -65,7 +65,7 @@ index 8cecdb2e69..9ede4183dd 100644 mp = (MDB_page *)my.mc_wbuf[0]; memset(mp, 0, NUM_METAS * env->me_psize); -@@ -9445,7 +9447,8 @@ finish: +@@ -9479,7 +9481,8 @@ finish: my.mc_error = rc; mdb_env_cthr_toggle(&my, 1 | MDB_EOF); rc = THREAD_FINISH(thr); @@ -75,7 +75,7 @@ index 8cecdb2e69..9ede4183dd 100644 done: #ifdef _WIN32 -@@ -9562,12 +9565,22 @@ leave: +@@ -9596,12 +9599,22 @@ leave: } int ESECT @@ -100,7 +100,7 @@ index 8cecdb2e69..9ede4183dd 100644 } int ESECT -@@ -9578,6 +9591,12 @@ mdb_env_copyfd(MDB_env *env, HANDLE fd) +@@ -9612,6 +9625,12 @@ mdb_env_copyfd(MDB_env *env, HANDLE fd) int ESECT mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) @@ -113,7 +113,7 @@ index 8cecdb2e69..9ede4183dd 100644 { int rc; MDB_name fname; -@@ -9589,7 +9608,7 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) +@@ -9623,7 +9642,7 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) mdb_fname_destroy(fname); } if (rc == MDB_SUCCESS) { diff --git a/lmdb/__main__.py b/lmdb/__main__.py index 4cb9dca6..934704f5 100644 --- a/lmdb/__main__.py +++ b/lmdb/__main__.py @@ -1,4 +1,4 @@ -# Copyright 2013 The py-lmdb authors, all rights reserved. +# Copyright 2013-2024 The py-lmdb authors, all rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted only as authorized by the OpenLDAP @@ -18,7 +18,7 @@ # Additional information about OpenLDAP can be obtained at # . -# Hack to support Python >=v2.6 'pythom -mlmdb' +# Hack to support Python >=v2.6 'python -mlmdb' from __future__ import absolute_import import lmdb.tool lmdb.tool.main() diff --git a/lmdb/cpython.c b/lmdb/cpython.c index ef304477..30308836 100644 --- a/lmdb/cpython.c +++ b/lmdb/cpython.c @@ -1,5 +1,5 @@ /* - * Copyright 2013 The py-lmdb authors, all rights reserved. + * Copyright 2013-2024 The py-lmdb authors, all rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted only as authorized by the OpenLDAP @@ -112,50 +112,12 @@ typedef struct IterObject IterObject; typedef struct TransObject TransObject; -/* ------------------------ */ -/* Python 3.x Compatibility */ -/* ------------------------ */ - -#if PY_MAJOR_VERSION >= 3 - # define MOD_RETURN(mod) return mod; # define MODINIT_NAME PyInit_cpython # define MAKE_ID(id) PyCapsule_New((void *) (1 + (id)), NULL, NULL) # define READ_ID(obj) (((int) (long) PyCapsule_GetPointer(obj, NULL)) - 1) -#else - -# define MOD_RETURN(mod) return -# define MODINIT_NAME initcpython - -# define MAKE_ID(id) PyInt_FromLong((long) id) -# define READ_ID(obj) PyInt_AS_LONG(obj) - -# define PyUnicode_InternFromString PyString_InternFromString -# define PyBytes_AS_STRING PyString_AS_STRING -# define PyBytes_GET_SIZE PyString_GET_SIZE -# define PyBytes_CheckExact PyString_CheckExact -# define PyBytes_FromStringAndSize PyString_FromStringAndSize -# define _PyBytes_Resize _PyString_Resize -# define PyMemoryView_FromMemory(x, y, z) PyBuffer_FromMemory(x, y) - -# ifndef PyBUF_READ -# define PyBUF_READ 0 -# endif - -/* Python 2.5 */ -# ifndef Py_TYPE -# define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) -# endif - -# ifndef PyVarObject_HEAD_INIT -# define PyVarObject_HEAD_INIT(x, y) \ - PyObject_HEAD_INIT(x) y, -# endif - -#endif - struct list_head { struct lmdb_object *prev; struct lmdb_object *next; @@ -649,11 +611,7 @@ parse_ulong(PyObject *obj, uint64_t *l, PyObject *max) PyErr_Format(PyExc_OverflowError, "Integer argument exceeds limit."); return -1; } -#if PY_MAJOR_VERSION >= 3 *l = PyLong_AsUnsignedLongLongMask(obj); -#else - *l = PyInt_AsUnsignedLongLongMask(obj); -#endif return 0; } @@ -2124,7 +2082,7 @@ cursor_get_multi(CursorObject *self, PyObject *args, PyObject *kwds) }; size_t buffer_pos = 0, buffer_size = 8; - size_t key_size, val_size, item_size = 0; + size_t key_size = 0, val_size, item_size = 0; char *buffer = NULL; static PyObject *cache = NULL; @@ -3797,11 +3755,7 @@ static PyTypeObject PyTransaction_Type = { static int append_string(PyObject *list, const char *s) { -#if PY_MAJOR_VERSION >= 3 PyObject *o = PyUnicode_FromString(s); -#else - PyObject *o = PyBytes_FromStringAndSize(s, strlen(s)); -#endif if(! o) { return -1; @@ -3862,7 +3816,6 @@ static struct PyMethodDef module_methods[] = { {0, 0, 0, 0} }; -#if PY_MAJOR_VERSION >= 3 static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "cpython", @@ -3874,7 +3827,6 @@ static struct PyModuleDef moduledef = { NULL, NULL }; -#endif /** * Initialize and publish the LMDB built-in types. @@ -3985,11 +3937,7 @@ PyMODINIT_FUNC MODINIT_NAME(void) { PyObject *__all__; -#if PY_MAJOR_VERSION >= 3 PyObject *mod = PyModule_Create(&moduledef); -#else - PyObject *mod = Py_InitModule3("cpython", module_methods, ""); -#endif if(! mod) { MOD_RETURN(NULL); } diff --git a/setup.py b/setup.py index 62d6384d..6d8709cc 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2020 The py-lmdb authors, all rights reserved. +# Copyright 2013-2024 The py-lmdb authors, all rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted only as authorized by the OpenLDAP @@ -45,8 +45,8 @@ if os.getenv('LMDB_FORCE_CFFI') is not None: use_cpython = False -if sys.version[:3] < '2.7' or (3, 0) < sys.version_info[:2] < (3, 4): - sys.stderr.write('Error: py-lmdb requires at least CPython 2.7 or 3.4\n') +if (3, 0) < sys.version_info[:2] < (3, 5): + sys.stderr.write('Error: py-lmdb requires at CPython 3.5\n') raise SystemExit(1) # @@ -110,11 +110,11 @@ if sys.platform.startswith('win'): patchfile = 'lib' + os.sep + 'py-lmdb' + os.sep + 'env-copy-txn.patch' patchset = patch.fromfile(patchfile) - rv = patchset.apply(3, root=dest) + rv = patchset.apply(2, root=dest) if not rv: raise Exception('Applying patch failed') else: - rv = os.system('/usr/bin/patch -N -p3 -d build/lib < lib/py-lmdb/env-copy-txn.patch') + rv = os.system('patch -N -p3 -d build/lib < lib/py-lmdb/env-copy-txn.patch') if rv: raise Exception('Applying patch failed') @@ -206,10 +206,7 @@ def grep_version(): "Programming Language :: Python", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", @@ -217,6 +214,7 @@ def grep_version(): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Database", "Topic :: Database :: Database Engines/Servers", ],