From 2f3ed1128d5bef7110be784513cd38f4e9340ab8 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 15 Sep 2020 09:11:16 -0600 Subject: [PATCH 1/7] removed bget --- CMakeLists.txt | 7 +-- src/clib/CMakeLists.txt | 2 +- src/clib/Makefile.am | 4 +- src/clib/pio_darray.c | 26 +++++------ src/clib/pio_darray_int.c | 96 ++++----------------------------------- src/clib/pio_file.c | 2 +- src/clib/pioc.c | 7 --- 7 files changed, 28 insertions(+), 116 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 983d689dab8..d12d2c5dda3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,7 +91,6 @@ option (PIO_INTERNAL_DOC "Enable PIO developer documentation" OFF) option (PIO_TEST_BIG_ENDIAN "Enable test to see if machine is big endian" ON) option (PIO_USE_MPIIO "Enable support for MPI-IO auto detect" ON) option (PIO_USE_MPISERIAL "Enable mpi-serial support (instead of MPI)" OFF) -option (PIO_USE_MALLOC "Use native malloc (instead of bget package)" ON) option (PIO_USE_PNETCDF_VARD "Use pnetcdf put_vard " OFF) option (WITH_PNETCDF "Require the use of PnetCDF" ON) @@ -103,11 +102,7 @@ else() endif() # Set a variable that appears in the config.h.in file. -if(PIO_USE_MALLOC) - set(USE_MALLOC 1) -else() - set(USE_MALLOC 0) -endif() +set(USE_MALLOC 1) # Set a variable that appears in the config.h.in file. if(PIO_ENABLE_LOGGING) diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt index 68d801907a0..8e3e63e6767 100644 --- a/src/clib/CMakeLists.txt +++ b/src/clib/CMakeLists.txt @@ -7,7 +7,7 @@ project (PIOC C) #============================================================================== set (src topology.c pio_file.c pioc_support.c pio_lists.c - pioc.c pioc_sc.c pio_spmd.c pio_rearrange.c pio_nc4.c bget.c + pioc.c pioc_sc.c pio_spmd.c pio_rearrange.c pio_nc4.c pio_nc.c pio_put_nc.c pio_get_nc.c pio_getput_int.c pio_msg.c pio_darray.c pio_darray_int.c pio_get_vard.c pio_put_vard.c pio_error.c) if (NETCDF_INTEGRATION) diff --git a/src/clib/Makefile.am b/src/clib/Makefile.am index bfcabe342ae..54c0d21bf73 100644 --- a/src/clib/Makefile.am +++ b/src/clib/Makefile.am @@ -19,10 +19,10 @@ libpioc_la_LDFLAGS = -version-info 4:1:3 include_HEADERS = pio.h uthash.h pio_meta.h # The library soure files. -libpioc_la_SOURCES = bget.c pioc_sc.c pio_darray.c pio_file.c \ +libpioc_la_SOURCES = pioc_sc.c pio_darray.c pio_file.c \ pio_getput_int.c pio_msg.c pio_nc.c pio_rearrange.c pioc.c \ pioc_support.c pio_darray_int.c pio_get_nc.c pio_lists.c pio_nc4.c \ pio_put_nc.c pio_spmd.c pio_get_vard.c pio_put_vard.c pio_error.c \ -pio_internal.h bget.h uthash.h pio_error.h +pio_internal.h uthash.h pio_error.h EXTRA_DIST = CMakeLists.txt topology.c pio_meta.h.in diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index 82d57b38514..fa91aeaca8a 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -274,7 +274,7 @@ PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, if (rlen > 0) { /* Allocate memory for the buffer for all vars/records. */ - if (!(file->iobuf = bget(iodesc->mpitype_size * (size_t)rlen))) + if (!(file->iobuf = malloc(iodesc->mpitype_size * (size_t)rlen))) return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); PLOG((3, "allocated %lld bytes for variable buffer", (size_t)rlen * iodesc->mpitype_size)); @@ -294,7 +294,7 @@ PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, /* this assures that iobuf is allocated on all iotasks thus assuring that the flush_output_buffer call above is called collectively (from all iotasks) */ - if (!(file->iobuf = bget(1))) + if (!(file->iobuf = malloc(1))) return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); PLOG((3, "allocated token for variable buffer")); } @@ -341,7 +341,7 @@ PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, if (file->iobuf) { PLOG((3,"freeing variable buffer in pio_darray")); - brel(file->iobuf); + free(file->iobuf); file->iobuf = NULL; } } @@ -365,9 +365,9 @@ PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, /* Get a buffer. */ if (ios->io_rank == 0) - vdesc0->fillbuf = bget(iodesc->maxholegridsize * iodesc->mpitype_size * nvars); + vdesc0->fillbuf = malloc(iodesc->maxholegridsize * iodesc->mpitype_size * nvars); else if (iodesc->holegridsize > 0) - vdesc0->fillbuf = bget(iodesc->holegridsize * iodesc->mpitype_size * nvars); + vdesc0->fillbuf = malloc(iodesc->holegridsize * iodesc->mpitype_size * nvars); /* copying the fill value into the data buffer for the box * rearranger. This will be overwritten with data where @@ -403,7 +403,7 @@ PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, /* Free resources. */ if (vdesc0->fillbuf) { - brel(vdesc0->fillbuf); + free(vdesc0->fillbuf); vdesc0->fillbuf = NULL; } } @@ -718,7 +718,7 @@ PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *arra if (!wmb) { /* Allocate a buffer. */ - if (!(wmb = bget((bufsize)sizeof(wmulti_buffer)))) + if (!(wmb = malloc((bufsize)sizeof(wmulti_buffer)))) return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); /* Set pointer to newly allocated buffer and initialize.*/ @@ -808,7 +808,7 @@ PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *arra /* Get memory for data. */ if (arraylen > 0) { - if (!(wmb->data = bgetr(wmb->data, (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size))) + if (!(wmb->data = realloc(wmb->data, (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size))) return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); PLOG((2, "got %ld bytes for data", (1 + wmb->num_arrays) * arraylen * iodesc->mpitype_size)); } @@ -816,14 +816,14 @@ PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *arra /* vid is an array of variable ids in the wmb list, grow the list * and add the new entry. */ - if (!(wmb->vid = bgetr(wmb->vid, sizeof(int) * (1 + wmb->num_arrays)))) + if (!(wmb->vid = realloc(wmb->vid, sizeof(int) * (1 + wmb->num_arrays)))) return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); /* wmb->frame is the record number, we assume that the variables * in the wmb list may not all have the same unlimited dimension * value although they usually do. */ if (vdesc->record >= 0) - if (!(wmb->frame = bgetr(wmb->frame, sizeof(int) * (1 + wmb->num_arrays)))) + if (!(wmb->frame = realloc(wmb->frame, sizeof(int) * (1 + wmb->num_arrays)))) return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); /* If we need a fill value, get it. If we are using the subset @@ -832,7 +832,7 @@ PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *arra if (iodesc->needsfill) { /* Get memory to hold fill value. */ - if (!(wmb->fillvalue = bgetr(wmb->fillvalue, iodesc->mpitype_size * (1 + wmb->num_arrays)))) + if (!(wmb->fillvalue = realloc(wmb->fillvalue, iodesc->mpitype_size * (1 + wmb->num_arrays)))) return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); memcpy((char *)wmb->fillvalue + iodesc->mpitype_size * wmb->num_arrays, @@ -961,7 +961,7 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, /* Allocate a buffer for one record. */ if (ios->ioproc && rlen > 0) - if (!(iobuf = bget(iodesc->mpitype_size * rlen))) + if (!(iobuf = malloc(iodesc->mpitype_size * rlen))) return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); /* Call the correct darray read function based on iotype. */ @@ -1018,7 +1018,7 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, /* Free the buffer. */ if (rlen > 0) - brel(iobuf); + free(iobuf); #ifdef USE_MPE pio_stop_mpe_log(DARRAY_READ, __func__); diff --git a/src/clib/pio_darray_int.c b/src/clib/pio_darray_int.c index 4a959ba6b6b..546a1d6b2ec 100644 --- a/src/clib/pio_darray_int.c +++ b/src/clib/pio_darray_int.c @@ -1339,8 +1339,8 @@ pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf) if (tmp_bufsize > 0) { - startlist[rrlen] = bget(fndims * sizeof(PIO_Offset)); - countlist[rrlen] = bget(fndims * sizeof(PIO_Offset)); + startlist[rrlen] = malloc(fndims * sizeof(PIO_Offset)); + countlist[rrlen] = malloc(fndims * sizeof(PIO_Offset)); for (int j = 0; j < fndims; j++) { @@ -1380,8 +1380,8 @@ pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf) /* Release the start and count arrays. */ for (int i = 0; i < rrlen; i++) { - brel(startlist[i]); - brel(countlist[i]); + free(startlist[i]); + free(countlist[i]); } } } @@ -1825,7 +1825,7 @@ flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) if (file->iobuf) { PLOG((3,"freeing variable buffer in flush_output_buffer")); - brel(file->iobuf); + free(file->iobuf); file->iobuf = NULL; } @@ -1835,7 +1835,7 @@ flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) return pio_err(NULL, file, ierr, __FILE__, __LINE__); if (vdesc->fillbuf) { - brel(vdesc->fillbuf); + free(vdesc->fillbuf); vdesc->fillbuf = NULL; } } @@ -1845,82 +1845,6 @@ flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) return ierr; } -/** - * Print out info about the buffer for debug purposes. This should - * only be called when logging is enabled. - * - * @param ios pointer to the IO system structure - * @param collective true if collective report is desired - * @ingroup PIO_write_darray_c - * @author Jim Edwards - */ -void -cn_buffer_report(iosystem_desc_t *ios, bool collective) -{ - int mpierr; /* Return code from MPI functions. */ - - PLOG((2, "cn_buffer_report ios->iossysid = %d collective = %d CN_bpool = %d", - ios->iosysid, collective, CN_bpool)); - if (CN_bpool) - { - long bget_stats[5]; - long bget_mins[5]; - long bget_maxs[5]; - - bstats(bget_stats, bget_stats+1,bget_stats+2,bget_stats+3,bget_stats+4); - if (collective) - { - PLOG((3, "cn_buffer_report calling MPI_Reduce ios->comp_comm = %d", ios->comp_comm)); - if ((mpierr = MPI_Reduce(bget_stats, bget_maxs, 5, MPI_LONG, MPI_MAX, 0, ios->comp_comm))) - check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); - PLOG((3, "cn_buffer_report calling MPI_Reduce")); - if ((mpierr = MPI_Reduce(bget_stats, bget_mins, 5, MPI_LONG, MPI_MIN, 0, ios->comp_comm))) - check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); - if (ios->compmaster == MPI_ROOT) - { - PLOG((1, "Currently allocated buffer space %ld %ld", bget_mins[0], bget_maxs[0])); - PLOG((1, "Currently available buffer space %ld %ld", bget_mins[1], bget_maxs[1])); - PLOG((1, "Current largest free block %ld %ld", bget_mins[2], bget_maxs[2])); - PLOG((1, "Number of successful bget calls %ld %ld", bget_mins[3], bget_maxs[3])); - PLOG((1, "Number of successful brel calls %ld %ld", bget_mins[4], bget_maxs[4])); - } - } - else - { - PLOG((1, "Currently allocated buffer space %ld", bget_stats[0])); - PLOG((1, "Currently available buffer space %ld", bget_stats[1])); - PLOG((1, "Current largest free block %ld", bget_stats[2])); - PLOG((1, "Number of successful bget calls %ld", bget_stats[3])); - PLOG((1, "Number of successful brel calls %ld", bget_stats[4])); - } - } -} - -/** - * Free the buffer pool. If malloc is used (that is, PIO_USE_MALLOC is - * non zero), this function does nothing. - * - * @param ios pointer to the IO system structure. - * @ingroup PIO_write_darray_c - * @author Jim Edwards - */ -void -free_cn_buffer_pool(iosystem_desc_t *ios) -{ -#if !PIO_USE_MALLOC - PLOG((2, "free_cn_buffer_pool CN_bpool = %d", CN_bpool)); - /* Note: it is possible that CN_bpool has been freed and set to NULL by bpool_free() */ - if (CN_bpool) - { - cn_buffer_report(ios, false); - bpoolrelease(CN_bpool); - PLOG((2, "free_cn_buffer_pool done!")); - free(CN_bpool); - CN_bpool = NULL; - } -#endif /* !PIO_USE_MALLOC */ -} - /** * Flush the buffer. * @@ -1958,21 +1882,21 @@ flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) wmb->num_arrays = 0; /* Release the list of variable IDs. */ - brel(wmb->vid); + free(wmb->vid); wmb->vid = NULL; /* Release the data memory. */ - brel(wmb->data); + free(wmb->data); wmb->data = NULL; /* If there is a fill value, release it. */ if (wmb->fillvalue) - brel(wmb->fillvalue); + free(wmb->fillvalue); wmb->fillvalue = NULL; /* Release the record number. */ if (wmb->frame) - brel(wmb->frame); + free(wmb->frame); wmb->frame = NULL; if (ret) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 2db64895539..1edf4f42ec6 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -419,7 +419,7 @@ PIOc_sync(int ncid) if (wmb->num_arrays > 0) flush_buffer(ncid, wmb, true); HASH_DEL(file->buffer, wmb); - brel(wmb); + free(wmb); } file->buffer = NULL; diff --git a/src/clib/pioc.c b/src/clib/pioc.c index 0f6e693b1d4..616377db97f 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -1243,13 +1243,6 @@ PIOc_free_iosystem(int iosysid) return pio_err(ios, NULL, ierr, __FILE__, __LINE__); PLOG((2, "%d iosystems are still open.", niosysid)); - /* Only free the buffer pool if this is the last open iosysid. */ - if (niosysid == 1) - { - free_cn_buffer_pool(ios); - PLOG((2, "Freed buffer pool.")); - } - /* Free the MPI communicators. my_comm is just a copy (but not an * MPI copy), so does not have to have an MPI_Comm_free() * call. comp_comm and io_comm are MPI duplicates of the comms From bad9cee8cf356b6b7c8dbd7f824678529196a823 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 15 Sep 2020 09:11:35 -0600 Subject: [PATCH 2/7] removed bget --- src/clib/bget.c | 1677 ----------------------------------------------- src/clib/bget.h | 41 -- 2 files changed, 1718 deletions(-) delete mode 100644 src/clib/bget.c delete mode 100644 src/clib/bget.h diff --git a/src/clib/bget.c b/src/clib/bget.c deleted file mode 100644 index d6e8b7bbbbb..00000000000 --- a/src/clib/bget.c +++ /dev/null @@ -1,1677 +0,0 @@ -/* - - B G E T - - Buffer allocator - - Designed and implemented in April of 1972 by John Walker, based on the - Case Algol OPRO$ algorithm implemented in 1966. - - Reimplemented in 1975 by John Walker for the Interdata 70. - Reimplemented in 1977 by John Walker for the Marinchip 9900. - Reimplemented in 1982 by Duff Kurland for the Intel 8080. - - Portable C version implemented in September of 1990 by an older, wiser - instance of the original implementor. - - Souped up and/or weighed down slightly shortly thereafter by Greg - Lutz. - - AMIX edition, including the new compaction call-back option, prepared - by John Walker in July of 1992. - - Bug in built-in test program fixed, ANSI compiler warnings eradicated, - buffer pool validator implemented, and guaranteed repeatable test - added by John Walker in October of 1995. - - This program is in the public domain. - - 1. This is the book of the generations of Adam. In the day that God - created man, in the likeness of God made he him; - 2. Male and female created he them; and blessed them, and called - their name Adam, in the day when they were created. - 3. And Adam lived an hundred and thirty years, and begat a son in - his own likeness, and after his image; and called his name Seth: - 4. And the days of Adam after he had begotten Seth were eight - hundred years: and he begat sons and daughters: - 5. And all the days that Adam lived were nine hundred and thirty - years: and he died. - 6. And Seth lived an hundred and five years, and begat Enos: - 7. And Seth lived after he begat Enos eight hundred and seven years, - and begat sons and daughters: - 8. And all the days of Seth were nine hundred and twelve years: and - he died. - 9. And Enos lived ninety years, and begat Cainan: - 10. And Enos lived after he begat Cainan eight hundred and fifteen - years, and begat sons and daughters: - 11. And all the days of Enos were nine hundred and five years: and - he died. - 12. And Cainan lived seventy years and begat Mahalaleel: - 13. And Cainan lived after he begat Mahalaleel eight hundred and - forty years, and begat sons and daughters: - 14. And all the days of Cainan were nine hundred and ten years: and - he died. - 15. And Mahalaleel lived sixty and five years, and begat Jared: - 16. And Mahalaleel lived after he begat Jared eight hundred and - thirty years, and begat sons and daughters: - 17. And all the days of Mahalaleel were eight hundred ninety and - five years: and he died. - 18. And Jared lived an hundred sixty and two years, and he begat - Enoch: - 19. And Jared lived after he begat Enoch eight hundred years, and - begat sons and daughters: - 20. And all the days of Jared were nine hundred sixty and two years: - and he died. - 21. And Enoch lived sixty and five years, and begat Methuselah: - 22. And Enoch walked with God after he begat Methuselah three - hundred years, and begat sons and daughters: - 23. And all the days of Enoch were three hundred sixty and five - years: - 24. And Enoch walked with God: and he was not; for God took him. - 25. And Methuselah lived an hundred eighty and seven years, and - begat Lamech. - 26. And Methuselah lived after he begat Lamech seven hundred eighty - and two years, and begat sons and daughters: - 27. And all the days of Methuselah were nine hundred sixty and nine - years: and he died. - 28. And Lamech lived an hundred eighty and two years, and begat a - son: - 29. And he called his name Noah, saying, This same shall comfort us - concerning our work and toil of our hands, because of the ground - which the LORD hath cursed. - 30. And Lamech lived after he begat Noah five hundred ninety and - five years, and begat sons and daughters: - 31. And all the days of Lamech were seven hundred seventy and seven - years: and he died. - 32. And Noah was five hundred years old: and Noah begat Shem, Ham, - and Japheth. - - And buffers begat buffers, and links begat links, and buffer pools - begat links to chains of buffer pools containing buffers, and lo the - buffers and links and pools of buffers and pools of links to chains of - pools of buffers were fruitful and they multiplied and the Operating - System looked down upon them and said that it was Good. - - - INTRODUCTION - ============ - - BGET is a comprehensive memory allocation package which is easily - configured to the needs of an application. BGET is efficient in - both the time needed to allocate and release buffers and in the - memory overhead required for buffer pool management. It - automatically consolidates contiguous space to minimise - fragmentation. BGET is configured by compile-time definitions, - Major options include: - - * A built-in test program to exercise BGET and - demonstrate how the various functions are used. - - * Allocation by either the "first fit" or "best fit" - method. - - * Wiping buffers at release time to catch code which - references previously released storage. - - * Built-in routines to dump individual buffers or the - entire buffer pool. - - * Retrieval of allocation and pool size statistics. - - * Quantisation of buffer sizes to a power of two to - satisfy hardware alignment constraints. - - * Automatic pool compaction, growth, and shrinkage by - means of call-backs to user defined functions. - - Applications of BGET can range from storage management in - ROM-based embedded programs to providing the framework upon which - a multitasking system incorporating garbage collection is - constructed. BGET incorporates extensive internal consistency - checking using the mechanism; all these checks can be - turned off by compiling with NDEBUG defined, yielding a version of - BGET with minimal size and maximum speed. - - The basic algorithm underlying BGET has withstood the test of - time; more than 25 years have passed since the first - implementation of this code. And yet, it is substantially more - efficient than the native allocation schemes of many operating - systems: the Macintosh and Microsoft Windows to name two, on which - programs have obtained substantial speed-ups by layering BGET as - an application level memory manager atop the underlying system's. - - BGET has been implemented on the largest mainframes and the lowest - of microprocessors. It has served as the core for multitasking - operating systems, multi-thread applications, embedded software in - data network switching processors, and a host of C programs. And - while it has accreted flexibility and additional options over the - years, it remains fast, memory efficient, portable, and easy to - integrate into your program. - - - BGET IMPLEMENTATION ASSUMPTIONS - =============================== - - BGET is written in as portable a dialect of C as possible. The - only fundamental assumption about the underlying hardware - architecture is that memory is allocated is a linear array which - can be addressed as a vector of C "char" objects. On segmented - address space architectures, this generally means that BGET should - be used to allocate storage within a single segment (although some - compilers simulate linear address spaces on segmented - architectures). On segmented architectures, then, BGET buffer - pools may not be larger than a segment, but since BGET allows any - number of separate buffer pools, there is no limit on the total - storage which can be managed, only on the largest individual - object which can be allocated. Machines with a linear address - architecture, such as the VAX, 680x0, Sparc, MIPS, or the Intel - 80386 and above in native mode, may use BGET without restriction. - - - GETTING STARTED WITH BGET - ========================= - - Although BGET can be configured in a multitude of fashions, there - are three basic ways of working with BGET. The functions - mentioned below are documented in the following section. Please - excuse the forward references which are made in the interest of - providing a roadmap to guide you to the BGET functions you're - likely to need. - - Embedded Applications - --------------------- - - Embedded applications typically have a fixed area of memory - dedicated to buffer allocation (often in a separate RAM address - space distinct from the ROM that contains the executable code). - To use BGET in such an environment, simply call bpool() with the - start address and length of the buffer pool area in RAM, then - allocate buffers with bget() and release them with brel(). - Embedded applications with very limited RAM but abundant CPU speed - may benefit by configuring BGET for BestFit allocation (which is - usually not worth it in other environments). - - Malloc() Emulation - ------------------ - - If the C library malloc() function is too slow, not present in - your development environment (for example, an a native Windows or - Macintosh program), or otherwise unsuitable, you can replace it - with BGET. Initially define a buffer pool of an appropriate size - with bpool()--usually obtained by making a call to the operating - system's low-level memory allocator. Then allocate buffers with - bget(), bgetz(), and bgetr() (the last two permit the allocation - of buffers initialised to zero and [inefficient] re-allocation of - existing buffers for compatibility with C library functions). - Release buffers by calling brel(). If a buffer allocation request - fails, obtain more storage from the underlying operating system, - add it to the buffer pool by another call to bpool(), and continue - execution. - - Automatic Storage Management - ---------------------------- - - You can use BGET as your application's native memory manager and - implement automatic storage pool expansion, contraction, and - optionally application-specific memory compaction by compiling - BGET with the BECtl variable defined, then calling bectl() and - supplying functions for storage compaction, acquisition, and - release, as well as a standard pool expansion increment. All of - these functions are optional (although it doesn't make much sense - to provide a release function without an acquisition function, - does it?). Once the call-back functions have been defined with - bectl(), you simply use bget() and brel() to allocate and release - storage as before. You can supply an initial buffer pool with - bpool() or rely on automatic allocation to acquire the entire - pool. When a call on bget() cannot be satisfied, BGET first - checks if a compaction function has been supplied. If so, it is - called (with the space required to satisfy the allocation request - and a sequence number to allow the compaction routine to be called - successively without looping). If the compaction function is able - to free any storage (it needn't know whether the storage it freed - was adequate) it should return a nonzero value, whereupon BGET - will retry the allocation request and, if it fails again, call the - compaction function again with the next-higher sequence number. - - If the compaction function returns zero, indicating failure to - free space, or no compaction function is defined, BGET next tests - whether a non-NULL allocation function was supplied to bectl(). - If so, that function is called with an argument indicating how - many bytes of additional space are required. This will be the - standard pool expansion increment supplied in the call to bectl() - unless the original bget() call requested a buffer larger than - this; buffers larger than the standard pool block can be managed - "off the books" by BGET in this mode. If the allocation function - succeeds in obtaining the storage, it returns a pointer to the new - block and BGET expands the buffer pool; if it fails, the - allocation request fails and returns NULL to the caller. If a - non-NULL release function is supplied, expansion blocks which - become totally empty are released to the global free pool by - passing their addresses to the release function. - - Equipped with appropriate allocation, release, and compaction - functions, BGET can be used as part of very sophisticated memory - management strategies, including garbage collection. (Note, - however, that BGET is *not* a garbage collector by itself, and - that developing such a system requires much additional logic and - careful design of the application's memory allocation strategy.) - - - BGET FUNCTION DESCRIPTIONS - ========================== - - Functions implemented in this file (some are enabled by certain of - the optional settings below): - - void bpool(void *buffer, bufsize len); - - Create a buffer pool of bytes, using the storage starting at - . You can call bpool() subsequently to contribute - additional storage to the overall buffer pool. - - void *bget(bufsize size); - - Allocate a buffer of bytes. The address of the buffer is - returned, or NULL if insufficient memory was available to allocate - the buffer. - - void *bgetz(bufsize size); - - Allocate a buffer of bytes and clear it to all zeroes. The - address of the buffer is returned, or NULL if insufficient memory - was available to allocate the buffer. - - void *bgetr(void *buffer, bufsize newsize); - - Reallocate a buffer previously allocated by bget(), changing its - size to and preserving all existing data. NULL is - returned if insufficient memory is available to reallocate the - buffer, in which case the original buffer remains intact. - - void brel(void *buf); - - Return the buffer , previously allocated by bget(), to the - free space pool. - - void bectl(int (*compact)(bufsize sizereq, int sequence), - void *(*acquire)(bufsize size), - void (*release)(void *buf), - bufsize pool_incr); - - Expansion control: specify functions through which the package may - compact storage (or take other appropriate action) when an - allocation request fails, and optionally automatically acquire - storage for expansion blocks when necessary, and release such - blocks when they become empty. If is non-NULL, whenever - a buffer allocation request fails, the function will be - called with arguments specifying the number of bytes (total buffer - size, including header overhead) required to satisfy the - allocation request, and a sequence number indicating the number of - consecutive calls on attempting to satisfy this - allocation request. The sequence number is 1 for the first call - on for a given allocation request, and increments on - subsequent calls, permitting the function to take - increasingly dire measures in an attempt to free up storage. If - the function returns a nonzero value, the allocation - attempt is re-tried. If returns 0 (as it must if it - isn't able to release any space or add storage to the buffer - pool), the allocation request fails, which can trigger automatic - pool expansion if the argument is non-NULL. At the time - the function is called, the state of the buffer - allocator is identical to that at the moment the allocation - request was made; consequently, the function may call - brel(), bpool(), bstats(), and/or directly manipulate the buffer - pool in any manner which would be valid were the application in - control. This does not, however, relieve the function - of the need to ensure that whatever actions it takes do not change - things underneath the application that made the allocation - request. For example, a function that released a buffer - in the process of being reallocated with bgetr() would lead to - disaster. Implementing a safe and effective mechanism - requires careful design of an application's memory architecture, - and cannot generally be easily retrofitted into existing code. - - If is non-NULL, that function will be called whenever an - allocation request fails. If the function succeeds in - allocating the requested space and returns a pointer to the new - area, allocation will proceed using the expanded buffer pool. If - cannot obtain the requested space, it should return NULL - and the entire allocation process will fail. - specifies the normal expansion block size. Providing an - function will cause subsequent bget() requests for buffers too - large to be managed in the linked-block scheme (in other words, - larger than minus the buffer overhead) to be satisfied - directly by calls to the function. Automatic release of - empty pool blocks will occur only if all pool blocks in the system - are the size given by . - - void bstats(bufsize *curalloc, bufsize *totfree, - bufsize *maxfree, long *nget, long *nrel); - - The amount of space currently allocated is stored into the - variable pointed to by . The total free space (sum of - all free blocks in the pool) is stored into the variable pointed - to by , and the size of the largest single block in the - free space pool is stored into the variable pointed to by - . The variables pointed to by and are - filled, respectively, with the number of successful (non-NULL - return) bget() calls and the number of brel() calls. - - void bstatse(bufsize *pool_incr, long *npool, - long *npget, long *nprel, - long *ndget, long *ndrel); - - Extended statistics: The expansion block size will be stored into - the variable pointed to by , or the negative thereof if - automatic expansion block releases are disabled. The number of - currently active pool blocks will be stored into the variable - pointed to by . The variables pointed to by and - will be filled with, respectively, the number of expansion - block acquisitions and releases which have occurred. The - variables pointed to by and will be filled with - the number of bget() and brel() calls, respectively, managed - through blocks directly allocated by the acquisition and release - functions. - - void bufdump(void *buf); - - The buffer pointed to by is dumped on standard output. - - void bpoold(void *pool, int dumpalloc, int dumpfree); - - All buffers in the buffer pool , previously initialised by a - call on bpool(), are listed in ascending memory address order. If - is nonzero, the contents of allocated buffers are - dumped; if is nonzero, the contents of free blocks are - dumped. - - int bpoolv(void *pool); - - The named buffer pool, previously initialised by a call on - bpool(), is validated for bad pointers, overwritten data, etc. If - compiled with NDEBUG not defined, any error generates an assertion - failure. Otherwise 1 is returned if the pool is valid, 0 if an - error is found. - - - BGET CONFIGURATION - ================== -*/ -#include -#include -#include -#if PIO_USE_MALLOC -#include -#endif - -#define TestProg 20000 /* Generate built-in test program - if defined. The value specifies - how many buffer allocation attempts - the test program should make. */ -#undef TestProg - -#define SizeQuant 4 /* Buffer allocation size quantum: - all buffers allocated are a - multiple of this size. This - MUST be a power of two. */ - -#define BufDump 1 /* Define this symbol to enable the - bpoold() function which dumps the - buffers in a buffer pool. */ - -#define BufValid 1 /* Define this symbol to enable the - bpoolv() function for validating - a buffer pool. */ - -#define DumpData 1 /* Define this symbol to enable the - bufdump() function which allows - dumping the contents of an allocated - or free buffer. */ - -#define BufStats 1 /* Define this symbol to enable the - bstats() function which calculates - the total free space in the buffer - pool, the largest available - buffer, and the total space - currently allocated. */ - -#define FreeWipe 1 /* Wipe free buffers to a guaranteed - pattern of garbage to trip up - miscreants who attempt to use - pointers into released buffers. */ - -//#define BestFit 1 -#undef BestFit -/* Use a best fit algorithm when - searching for space for an - allocation request. This uses - memory more efficiently, but - allocation will be much slower. */ - -#define BECtl 1 /* Define this symbol to enable the - bectl() function for automatic - pool space control. */ - -#include - -#ifdef lint -#define NDEBUG /* Exits in asserts confuse lint */ -/* LINTLIBRARY */ /* Don't complain about def, no ref */ -extern char *sprintf(); /* Sun includes don't define sprintf */ -#endif - -#include -#include - -#ifdef BufDump /* BufDump implies DumpData */ -#ifndef DumpData -#define DumpData 1 -#endif -#endif - -#ifdef DumpData -#include -#endif - -/* Declare the interface, including the requested buffer size type, - bufsize. */ - -#include "bget.h" - -#define MemSize size_t /* Type for size arguments to memxxx() - functions such as memcmp(). */ - -/* Queue links */ - -struct qlinks { - struct bfhead *flink; /* Forward link */ - struct bfhead *blink; /* Backward link */ -}; - -/* Header in allocated and free buffers */ - -struct bhead { - bufsize prevfree; /* Relative link back to previous - free buffer in memory or 0 if - previous buffer is allocated. */ - bufsize bsize; /* Buffer size: positive if free, - negative if allocated. */ -}; -#define BH(p) ((struct bhead *) (p)) - -/* Header in directly allocated buffers (by acqfcn) */ - -struct bdhead { - bufsize tsize; /* Total size, including overhead */ - struct bhead bh; /* Common header */ -}; -#define BDH(p) ((struct bdhead *) (p)) - -/* Header in free buffers */ - -struct bfhead { - struct bhead bh; /* Common allocated/free header */ - struct qlinks ql; /* Links on free list */ -}; -#define BFH(p) ((struct bfhead *) (p)) - -static struct bfhead freelist = { /* List of free buffers */ - {0, 0}, - {&freelist, &freelist} -}; - - -#ifdef BufStats -static bufsize totalloc = 0; /* Total space currently allocated */ -static long numget = 0, numrel = 0; /* Number of bget() and brel() calls */ -#ifdef BECtl -static long numpblk = 0; /* Number of pool blocks */ -static long numpget = 0, numprel = 0; /* Number of block gets and rels */ -static long numdget = 0, numdrel = 0; /* Number of direct gets and rels */ -#endif /* BECtl */ -#endif /* BufStats */ - -#ifdef BECtl - -/* Automatic expansion block management functions */ - -static int (*compfcn) _((bufsize sizereq, int sequence)) = NULL; -static void *(*acqfcn) _((bufsize size)) = NULL; -static void (*relfcn) _((void *buf)) = NULL; - -static bufsize exp_incr = 0; /* Expansion block size */ -static bufsize pool_len = 0; /* 0: no bpool calls have been made - -1: not all pool blocks are - the same size - >0: (common) block size for all - bpool calls made so far - */ -#endif - -/* Minimum allocation quantum: */ - -#define QLSize (sizeof(struct qlinks)) -#define SizeQ ((SizeQuant > QLSize) ? SizeQuant : QLSize) - -#define V (void) /* To denote unwanted returned values */ - -/* End sentinel: value placed in bsize field of dummy block delimiting - end of pool block. The most negative number which will fit in a - bufsize, defined in a way that the compiler will accept. */ - -#define ESent ((bufsize) (-(((1L << (sizeof(bufsize) * 8 - 2)) - 1) * 2) - 2)) - -/* added for PIO so that a bpool can be freed and another allocated */ -void bpoolrelease() -{ - PLOG((2, "bpoolrelease")); - freelist.bh.prevfree=0; - freelist.bh.bsize=0; - freelist.ql.flink=&freelist; - freelist.ql.blink=&freelist; - PLOG((2, "bpoolrelease")); - -#ifdef BufStats - totalloc = 0; /* Total space currently allocated */ - numget = 0; - numrel = 0; /* Number of bget() and brel() calls */ -#ifdef BECtl - numpblk = 0; /* Number of pool blocks */ - numpget = 0; - numprel = 0; /* Number of block gets and rels */ - numdget = 0; - numdrel = 0; /* Number of direct gets and rels */ -#endif /* BECtl */ -#endif /* BufStats */ - PLOG((2, "bpoolrelease")); - -#ifdef BECtl -/* Automatic expansion block management functions */ - compfcn = NULL; - acqfcn = NULL; - relfcn = NULL; - exp_incr = 0; - pool_len = 0; -#endif - PLOG((2, "bpoolrelease")); - -} - - - -/* BGET -- Allocate a buffer. */ - -void *bget(requested_size) -bufsize requested_size; -{ - bufsize size = requested_size; - struct bfhead *b; -#ifdef BestFit - struct bfhead *best; -#endif - void *buf; -#ifdef BECtl - int compactseq = 0; -#endif - -#if PIO_USE_MALLOC - // if(requested_size>maxsize){ - // maxsize=requested_size; - // printf("%s %d %d\n",__FILE__,__LINE__,maxsize); - // } - buf = malloc(requested_size); - // printf("bget allocate %ld %x\n",requested_size,buf); - return(buf); -#endif - - - if(size<=0) - print_trace(NULL); - - assert(size > 0); - - if (size < SizeQ) { /* Need at least room for the */ - size = SizeQ; /* queue links. */ - } -#ifdef SizeQuant -#if SizeQuant > 1 - size = (size + (SizeQuant - 1)) & (~(SizeQuant - 1)); -#endif -#endif - - size += sizeof(struct bhead); /* Add overhead in allocated buffer - to size required. */ - -#ifdef BECtl - /* If a compact function was provided in the call to bectl(), wrap - a loop around the allocation process to allow compaction to - intervene in case we don't find a suitable buffer in the chain. */ - - while (1) { -#endif - b = freelist.ql.flink; -#ifdef BestFit - best = &freelist; -#endif - - - /* Scan the free list searching for the first buffer big enough - to hold the requested size buffer. */ - -#ifdef BestFit - while (b != &freelist) { - // printf("%s %d %X %X %X %ld \n",__FILE__,__LINE__,b,&freelist,best,b->bh.bsize); - if (b->bh.bsize >= size) { - if ((best == &freelist) || (b->bh.bsize < best->bh.bsize)) { - best = b; - } - } - b = b->ql.flink; /* Link to next buffer */ - } - b = best; -#endif /* BestFit */ - - while (b != &freelist) { - if ((bufsize) b->bh.bsize >= size) { - - /* Buffer is big enough to satisfy the request. Allocate it - to the caller. We must decide whether the buffer is large - enough to split into the part given to the caller and a - free buffer that remains on the free list, or whether the - entire buffer should be removed from the free list and - given to the caller in its entirety. We only split the - buffer if enough room remains for a header plus the minimum - quantum of allocation. */ - - if ((b->bh.bsize - size) > (SizeQ + (sizeof(struct bhead)))) { - struct bhead *ba, *bn; - - ba = BH(((char *) b) + (b->bh.bsize - size)); - bn = BH(((char *) ba) + size); - assert(bn->prevfree == b->bh.bsize); - /* Subtract size from length of free block. */ - b->bh.bsize -= size; - /* Link allocated buffer to the previous free buffer. */ - ba->prevfree = b->bh.bsize; - /* Plug negative size into user buffer. */ - ba->bsize = -(bufsize) size; - /* Mark buffer after this one not preceded by free block. */ - bn->prevfree = 0; - -#ifdef BufStats - totalloc += size; - numget++; /* Increment number of bget() calls */ -#endif - buf = (void *) ((((char *) ba) + sizeof(struct bhead))); - return buf; - } else { - struct bhead *ba; - - ba = BH(((char *) b) + b->bh.bsize); - assert(ba->prevfree == b->bh.bsize); - - /* The buffer isn't big enough to split. Give the whole - shebang to the caller and remove it from the free list. */ - - assert(b->ql.blink->ql.flink == b); - assert(b->ql.flink->ql.blink == b); - b->ql.blink->ql.flink = b->ql.flink; - b->ql.flink->ql.blink = b->ql.blink; - -#ifdef BufStats - totalloc += b->bh.bsize; - numget++; /* Increment number of bget() calls */ -#endif - /* Negate size to mark buffer allocated. */ - b->bh.bsize = -(b->bh.bsize); - - /* Zero the back pointer in the next buffer in memory - to indicate that this buffer is allocated. */ - ba->prevfree = 0; - - /* Give user buffer starting at queue links. */ - buf = (void *) &(b->ql); - return buf; - } - } - b = b->ql.flink; /* Link to next buffer */ - } -#ifdef BECtl - - /* We failed to find a buffer. If there's a compact function - defined, notify it of the size requested. If it returns - TRUE, try the allocation again. */ - - if ((compfcn == NULL) || (!(*compfcn)(size, ++compactseq))) { - break; - } - } - - /* No buffer available with requested size free. */ - - /* Don't give up yet -- look in the reserve supply. */ - - if (acqfcn != NULL) { - if (size > exp_incr - sizeof(struct bhead)) { - - /* Request is too large to fit in a single expansion - block. Try to satisy it by a direct buffer acquisition. */ - - struct bdhead *bdh; - - size += sizeof(struct bdhead) - sizeof(struct bhead); - if ((bdh = BDH((*acqfcn)((bufsize) size))) != NULL) { - - /* Mark the buffer special by setting the size field - of its header to zero. */ - bdh->bh.bsize = 0; - bdh->bh.prevfree = 0; - bdh->tsize = size; -#ifdef BufStats - totalloc += size; - numget++; /* Increment number of bget() calls */ - numdget++; /* Direct bget() call count */ -#endif - buf = (void *) (bdh + 1); - - /*only let this happen once */ - printf("%s %d memory request exceeds block size %ld %ld\n",__FILE__,__LINE__, - size,exp_incr); - exp_incr = size+sizeof(struct bhead); - - return buf; - } - - } else { - - /* Try to obtain a new expansion block */ - - void *newpool; - - if ((newpool = (*acqfcn)((bufsize) exp_incr)) != NULL) { - bpool(newpool, exp_incr); - buf = bget(requested_size); /* This can't, I say, can't - get into a loop. */ - // printf("%s %d new memory block of size %d\n",__FILE__,__LINE__,exp_incr); - return buf; - } - } - } - - /* Still no buffer available */ - -#endif /* BECtl */ - - return NULL; -} - -/* BGETZ -- Allocate a buffer and clear its contents to zero. We clear - the entire contents of the buffer to zero, not just the - region requested by the caller. */ - -void *bgetz(size) -bufsize size; -{ - char *buf = (char *) bget(size); - - if (buf != NULL) { - struct bhead *b; - bufsize rsize; - - b = BH(buf - sizeof(struct bhead)); - rsize = -(b->bsize); - if (rsize == 0) { - struct bdhead *bd; - - bd = BDH(buf - sizeof(struct bdhead)); - rsize = bd->tsize - sizeof(struct bdhead); - } else { - rsize -= sizeof(struct bhead); - } - assert(rsize >= size); - V memset(buf, 0, (MemSize) rsize); - } - return ((void *) buf); -} - -/* BGETR -- Reallocate a buffer. This is a minimal implementation, - simply in terms of brel() and bget(). It could be - enhanced to allow the buffer to grow into adjacent free - blocks and to avoid moving data unnecessarily. */ - -void *bgetr(buf, size) -void *buf; -bufsize size; -{ - void *nbuf; - bufsize osize; /* Old size of buffer */ - struct bhead *b; - -#if PIO_USE_MALLOC - return(realloc(buf, size)); -#endif - if ((nbuf = bget(size)) == NULL) { /* Acquire new buffer */ - return NULL; - } - if (buf == NULL) { - return nbuf; - } - b = BH(((char *) buf) - sizeof(struct bhead)); - osize = -b->bsize; -#ifdef BECtl - if (osize == 0) { - /* Buffer acquired directly through acqfcn. */ - struct bdhead *bd; - - bd = BDH(((char *) buf) - sizeof(struct bdhead)); - osize = bd->tsize - sizeof(struct bdhead); - } else -#endif - osize -= sizeof(struct bhead); - assert(osize > 0); - V memcpy((char *) nbuf, (char *) buf, /* Copy the data */ - (MemSize) ((size < osize) ? size : osize)); - brel(buf); - return nbuf; -} - -/* BREL -- Release a buffer. */ - -void brel(buf) -void *buf; -{ - struct bfhead *b, *bn; - -#if PIO_USE_MALLOC - // printf("bget free %d %x\n",__LINE__,buf); - free(buf); - return; -#endif - - - if(buf==NULL) return; /* allow for null buffer */ - - b = BFH(((char *) buf) - sizeof(struct bhead)); -#ifdef BufStats - numrel++; /* Increment number of brel() calls */ -#endif - assert(buf != NULL); - -#ifdef BECtl - if (b->bh.bsize == 0) { /* Directly-acquired buffer? */ - struct bdhead *bdh; - - bdh = BDH(((char *) buf) - sizeof(struct bdhead)); - assert(b->bh.prevfree == 0); -#ifdef BufStats - totalloc -= bdh->tsize; - assert(totalloc >= 0); - numdrel++; /* Number of direct releases */ -#endif /* BufStats */ -#ifdef FreeWipe - V memset((char *) buf, 0x55, - (MemSize) (bdh->tsize - sizeof(struct bdhead))); -#endif /* FreeWipe */ - assert(relfcn != NULL); - (*relfcn)((void *) bdh); /* Release it directly. */ - return; - } -#endif /* BECtl */ - - /* Buffer size must be negative, indicating that the buffer is - allocated. */ - - if (b->bh.bsize >= 0) { - bn = NULL; - } - assert(b->bh.bsize < 0); - - /* Back pointer in next buffer must be zero, indicating the - same thing: */ - - assert(BH((char *) b - b->bh.bsize)->prevfree == 0); - -#ifdef BufStats - totalloc += b->bh.bsize; - assert(totalloc >= 0); -#endif - - /* If the back link is nonzero, the previous buffer is free. */ - - if (b->bh.prevfree != 0) { - - /* The previous buffer is free. Consolidate this buffer with it - by adding the length of this buffer to the previous free - buffer. Note that we subtract the size in the buffer being - released, since it's negative to indicate that the buffer is - allocated. */ - - register bufsize size = b->bh.bsize; - - /* Make the previous buffer the one we're working on. */ - assert(BH((char *) b - b->bh.prevfree)->bsize == b->bh.prevfree); - b = BFH(((char *) b) - b->bh.prevfree); - b->bh.bsize -= size; - } else { - - /* The previous buffer isn't allocated. Insert this buffer - on the free list as an isolated free block. */ - - assert(freelist.ql.blink->ql.flink == &freelist); - assert(freelist.ql.flink->ql.blink == &freelist); - b->ql.flink = &freelist; - b->ql.blink = freelist.ql.blink; - freelist.ql.blink = b; - b->ql.blink->ql.flink = b; - b->bh.bsize = -b->bh.bsize; - } - - /* Now we look at the next buffer in memory, located by advancing from - the start of this buffer by its size, to see if that buffer is - free. If it is, we combine this buffer with the next one in - memory, dechaining the second buffer from the free list. */ - - bn = BFH(((char *) b) + b->bh.bsize); - if (bn->bh.bsize > 0) { - - /* The buffer is free. Remove it from the free list and add - its size to that of our buffer. */ - - assert(BH((char *) bn + bn->bh.bsize)->prevfree == bn->bh.bsize); - assert(bn->ql.blink->ql.flink == bn); - assert(bn->ql.flink->ql.blink == bn); - bn->ql.blink->ql.flink = bn->ql.flink; - bn->ql.flink->ql.blink = bn->ql.blink; - b->bh.bsize += bn->bh.bsize; - - /* Finally, advance to the buffer that follows the newly - consolidated free block. We must set its backpointer to the - head of the consolidated free block. We know the next block - must be an allocated block because the process of recombination - guarantees that two free blocks will never be contiguous in - memory. */ - - bn = BFH(((char *) b) + b->bh.bsize); - } -#ifdef FreeWipe - V memset(((char *) b) + sizeof(struct bfhead), 0x55, - (MemSize) (b->bh.bsize - sizeof(struct bfhead))); -#endif - assert(bn->bh.bsize < 0); - - /* The next buffer is allocated. Set the backpointer in it to point - to this buffer; the previous free buffer in memory. */ - - bn->bh.prevfree = b->bh.bsize; - -#ifdef BECtl - - /* If a block-release function is defined, and this free buffer - constitutes the entire block, release it. Note that pool_len - is defined in such a way that the test will fail unless all - pool blocks are the same size. */ - - if (relfcn != NULL && - ((bufsize) b->bh.bsize) == (pool_len - sizeof(struct bhead))) { - - assert(b->bh.prevfree == 0); - assert(BH((char *) b + b->bh.bsize)->bsize == ESent); - assert(BH((char *) b + b->bh.bsize)->prevfree == b->bh.bsize); - /* Unlink the buffer from the free list */ - b->ql.blink->ql.flink = b->ql.flink; - b->ql.flink->ql.blink = b->ql.blink; - // printf("%s %d calling direct release for %x\n",__FILE__,__LINE__,b); - (*relfcn)(b); - // printf("%s %d completed direct release \n",__FILE__,__LINE__); -#ifdef BufStats - numprel++; /* Nr of expansion block releases */ - numpblk--; /* Total number of blocks */ - assert(numpblk == numpget - numprel); -#endif /* BufStats */ - } -#endif /* BECtl */ -} - -#ifdef BECtl - -/* BECTL -- Establish automatic pool expansion control */ - -void bectl(compact, acquire, release, pool_incr) - int (*compact) _((bufsize sizereq, int sequence)); -void *(*acquire) _((bufsize size)); -void (*release) _((void *buf)); -bufsize pool_incr; -{ - compfcn = compact; - acqfcn = acquire; - relfcn = release; - exp_incr = pool_incr; -} -#endif - -/* BPOOL -- Add a region of memory to the buffer pool. */ - -void bpool(buf, len) -void *buf; -bufsize len; -{ - struct bfhead *b = BFH(buf); - struct bhead *bn; - -#ifdef SizeQuant - len &= ~(SizeQuant - 1); -#endif -#ifdef BECtl - if (pool_len == 0) { - pool_len = len; - } else if (len != pool_len) { - pool_len = -1; - } -#ifdef BufStats - numpget++; /* Number of block acquisitions */ - numpblk++; /* Number of blocks total */ - assert(numpblk == numpget - numprel); -#endif /* BufStats */ -#endif /* BECtl */ - - /* Since the block is initially occupied by a single free buffer, - it had better not be (much) larger than the largest buffer - whose size we can store in bhead.bsize. */ - - assert(len - sizeof(struct bhead) <= -((bufsize) ESent + 1)); - - /* Clear the backpointer at the start of the block to indicate that - there is no free block prior to this one. That blocks - recombination when the first block in memory is released. */ - - b->bh.prevfree = 0; - - /* Chain the new block to the free list. */ - - assert(freelist.ql.blink->ql.flink == &freelist); - assert(freelist.ql.flink->ql.blink == &freelist); - b->ql.flink = &freelist; - b->ql.blink = freelist.ql.blink; - freelist.ql.blink = b; - b->ql.blink->ql.flink = b; - - /* Create a dummy allocated buffer at the end of the pool. This dummy - buffer is seen when a buffer at the end of the pool is released and - blocks recombination of the last buffer with the dummy buffer at - the end. The length in the dummy buffer is set to the largest - negative number to denote the end of the pool for diagnostic - routines (this specific value is not counted on by the actual - allocation and release functions). */ - - len -= sizeof(struct bhead); - b->bh.bsize = (bufsize) len; -#ifdef FreeWipe - V memset(((char *) b) + sizeof(struct bfhead), 0x55, - (MemSize) (len - sizeof(struct bfhead))); -#endif - bn = BH(((char *) b) + len); - bn->prevfree = (bufsize) len; - /* Definition of ESent assumes two's complement! */ - assert((~0) == -1); - bn->bsize = ESent; -} - -#ifdef BufStats - -void bfreespace(bufsize *totfree, bufsize *maxfree) -{ - struct bfhead *b = freelist.ql.flink; - *totfree = 0; - *maxfree = -1; - while (b != &freelist) { - assert(b->bh.bsize > 0); - *totfree += b->bh.bsize; - if (b->bh.bsize > *maxfree) { - *maxfree = b->bh.bsize; - } - b = b->ql.flink; /* Link to next buffer */ - } -} - -/* BSTATS -- Return buffer allocation free space statistics. */ - -void bstats(curalloc, totfree, maxfree, nget, nrel) - bufsize *curalloc, *totfree, *maxfree; -long *nget, *nrel; -{ - *nget = numget; - *nrel = numrel; - *curalloc = totalloc; - bfreespace(totfree, maxfree); -} - -#ifdef BECtl - -/* BSTATSE -- Return extended statistics */ - -void bstatse(pool_incr, npool, npget, nprel, ndget, ndrel) -bufsize *pool_incr; -long *npool, *npget, *nprel, *ndget, *ndrel; -{ - *pool_incr = (pool_len < 0) ? -exp_incr : exp_incr; - *npool = numpblk; - *npget = numpget; - *nprel = numprel; - *ndget = numdget; - *ndrel = numdrel; -} -#endif /* BECtl */ -#endif /* BufStats */ - -#ifdef DumpData - -/* BUFDUMP -- Dump the data in a buffer. This is called with the user - data pointer, and backs up to the buffer header. It will - dump either a free block or an allocated one. */ - -void bufdump(buf) -void *buf; -{ - struct bfhead *b; - unsigned char *bdump; - bufsize bdlen; - - b = BFH(((char *) buf) - sizeof(struct bhead)); - assert(b->bh.bsize != 0); - if (b->bh.bsize < 0) { - bdump = (unsigned char *) buf; - bdlen = (-b->bh.bsize) - sizeof(struct bhead); - } else { - bdump = (unsigned char *) (((char *) b) + sizeof(struct bfhead)); - bdlen = b->bh.bsize - sizeof(struct bfhead); - } - - while (bdlen > 0) { - int i, dupes = 0; - bufsize l = bdlen; - char bhex[50], bascii[20]; - - if (l > 16) { - l = 16; - } - - for (i = 0; i < l; i++) { - V sprintf(bhex + i * 3, "%02X ", bdump[i]); - bascii[i] = isprint(bdump[i]) ? bdump[i] : ' '; - } - bascii[i] = 0; - V printf("%-48s %s\n", bhex, bascii); - bdump += l; - bdlen -= l; - while ((bdlen > 16) && (memcmp((char *) (bdump - 16), - (char *) bdump, 16) == 0)) { - dupes++; - bdump += 16; - bdlen -= 16; - } - if (dupes > 1) { - V printf( - " (%d lines [%d bytes] identical to above line skipped)\n", - dupes, dupes * 16); - } else if (dupes == 1) { - bdump -= 16; - bdlen += 16; - } - } -} -#endif - -#ifdef BufDump - -/* BPOOLD -- Dump a buffer pool. The buffer headers are always listed. - If DUMPALLOC is nonzero, the contents of allocated buffers - are dumped. If DUMPFREE is nonzero, free blocks are - dumped as well. If FreeWipe checking is enabled, free - blocks which have been clobbered will always be dumped. */ - -void bpoold(buf, dumpalloc, dumpfree) -void *buf; -int dumpalloc, dumpfree; -{ - struct bfhead *b = BFH(buf); - - while (b->bh.bsize != ESent) { - bufsize bs = b->bh.bsize; - - if (bs < 0) { - bs = -bs; - V printf("Allocated buffer: size %6ld bytes.\n", (long) bs); - if (dumpalloc) { - bufdump((void *) (((char *) b) + sizeof(struct bhead))); - } - } else { - char *lerr = ""; - - assert(bs > 0); - if ((b->ql.blink->ql.flink != b) || - (b->ql.flink->ql.blink != b)) { - lerr = " (Bad free list links)"; - } - V printf("Free block: size %6ld bytes.%s\n", - (long) bs, lerr); -#ifdef FreeWipe - lerr = ((char *) b) + sizeof(struct bfhead); - if ((bs > sizeof(struct bfhead)) && ((*lerr != 0x55) || - (memcmp(lerr, lerr + 1, - (MemSize) (bs - (sizeof(struct bfhead) + 1))) != 0))) { - V printf( - "(Contents of above free block have been overstored.)\n"); - bufdump((void *) (((char *) b) + sizeof(struct bhead))); - } else -#endif - if (dumpfree) { - bufdump((void *) (((char *) b) + sizeof(struct bhead))); - } - } - b = BFH(((char *) b) + bs); - } -} -#endif /* BufDump */ - -#ifdef BufValid - -/* BPOOLV -- Validate a buffer pool. If NDEBUG isn't defined, - any error generates an assertion failure. */ - -int bpoolv(buf) -void *buf; -{ - struct bfhead *b = BFH(buf); - - while (b->bh.bsize != ESent) { - bufsize bs = b->bh.bsize; - - if (bs < 0) { - bs = -bs; - } else { - char *lerr = ""; - - assert(bs > 0); - if (bs <= 0) { - return 0; - } - if ((b->ql.blink->ql.flink != b) || - (b->ql.flink->ql.blink != b)) { - V printf("Free block: size %6ld bytes. (Bad free list links)\n", - (long) bs); - assert(0); - return 0; - } -#ifdef FreeWipe - lerr = ((char *) b) + sizeof(struct bfhead); - if ((bs > sizeof(struct bfhead)) && ((*lerr != 0x55) || - (memcmp(lerr, lerr + 1, - (MemSize) (bs - (sizeof(struct bfhead) + 1))) != 0))) { - V printf( - "(Contents of above free block have been overstored.)\n"); - bufdump((void *) (((char *) b) + sizeof(struct bhead))); - assert(0); - return 0; - } -#endif - } - b = BFH(((char *) b) + bs); - } - return 1; -} -#endif /* BufValid */ - -/***********************\ - * * - * Built-in test program * - * * - \***********************/ - -#ifdef TestProg - -#define Repeatable 1 /* Repeatable pseudorandom sequence */ - /* If Repeatable is not defined, a - time-seeded pseudorandom sequence - is generated, exercising BGET with - a different pattern of calls on each - run. */ -#define OUR_RAND /* Use our own built-in version of - rand() to guarantee the test is - 100% repeatable. */ - -#ifdef BECtl -#define PoolSize 300000 /* Test buffer pool size */ -#else -#define PoolSize 50000 /* Test buffer pool size */ -#endif -#define ExpIncr 32768 /* Test expansion block size */ -#define CompactTries 10 /* Maximum tries at compacting */ - -#define dumpAlloc 0 /* Dump allocated buffers ? */ -#define dumpFree 0 /* Dump free buffers ? */ - -#ifndef Repeatable -extern long time(); -#endif - -extern char *malloc(); -extern int free _((char *)); - -static char *bchain = NULL; /* Our private buffer chain */ -static char *bp = NULL; /* Our initial buffer pool */ - -#include - -#ifdef OUR_RAND - -static unsigned long int next = 1; - -/* Return next random integer */ - -int rand() -{ - next = next * 1103515245L + 12345; - return (unsigned int) (next / 65536L) % 32768L; -} - -/* Set seed for random generator */ - -void srand(seed) -unsigned int seed; -{ - next = seed; -} -#endif - -/* STATS -- Edit statistics returned by bstats() or bstatse(). */ - -static void stats(when) -char *when; -{ - bufsize cural, totfree, maxfree; - long nget, nfree; -#ifdef BECtl - bufsize pincr; - long totblocks, npget, nprel, ndget, ndrel; -#endif - - bstats(&cural, &totfree, &maxfree, &nget, &nfree); - V printf( - "%s: %ld gets, %ld releases. %ld in use, %ld free, largest = %ld\n", - when, nget, nfree, (long) cural, (long) totfree, (long) maxfree); -#ifdef BECtl - bstatse(&pincr, &totblocks, &npget, &nprel, &ndget, &ndrel); - V printf( - " Blocks: size = %ld, %ld (%ld bytes) in use, %ld gets, %ld frees\n", - (long)pincr, totblocks, pincr * totblocks, npget, nprel); - V printf(" %ld direct gets, %ld direct frees\n", ndget, ndrel); -#endif /* BECtl */ -} - -#ifdef BECtl -static int protect = 0; /* Disable compaction during bgetr() */ - -/* BCOMPACT -- Compaction call-back function. */ - -static int bcompact(bsize, seq) -bufsize bsize; -int seq; -{ -#ifdef CompactTries - char *bc = bchain; - int i = rand() & 0x3; - -#ifdef COMPACTRACE - V printf("Compaction requested. %ld bytes needed, sequence %d.\n", - (long) bsize, seq); -#endif - - if (protect || (seq > CompactTries)) { -#ifdef COMPACTRACE - V printf("Compaction gave up.\n"); -#endif - return 0; - } - - /* Based on a random cast, release a random buffer in the list - of allocated buffers. */ - - while (i > 0 && bc != NULL) { - bc = *((char **) bc); - i--; - } - if (bc != NULL) { - char *fb; - - fb = *((char **) bc); - if (fb != NULL) { - *((char **) bc) = *((char **) fb); - brel((void *) fb); - return 1; - } - } - -#ifdef COMPACTRACE - V printf("Compaction bailed out.\n"); -#endif -#endif /* CompactTries */ - return 0; -} - -/* BEXPAND -- Expand pool call-back function. */ - -static void *bexpand(size) -bufsize size; -{ - void *np = NULL; - bufsize cural, totfree, maxfree; - long nget, nfree; - - /* Don't expand beyond the total allocated size given by PoolSize. */ - - bstats(&cural, &totfree, &maxfree, &nget, &nfree); - - if (cural < PoolSize) { - np = (void *) malloc((unsigned) size); - } -#ifdef EXPTRACE - V printf("Expand pool by %ld -- %s.\n", (long) size, - np == NULL ? "failed" : "succeeded"); -#endif - return np; -} - -/* BSHRINK -- Shrink buffer pool call-back function. */ - -static void bshrink(buf) -void *buf; -{ - if (((char *) buf) == bp) { -#ifdef EXPTRACE - V printf("Initial pool released.\n"); -#endif - bp = NULL; - } -#ifdef EXPTRACE - V printf("Shrink pool.\n"); -#endif - free((char *) buf); -} - -#endif /* BECtl */ - -/* Restrict buffer requests to those large enough to contain our pointer and - small enough for the CPU architecture. */ - -static bufsize blimit(bs) -bufsize bs; -{ - if (bs < sizeof(char *)) { - bs = sizeof(char *); - } - - /* This is written out in this ugly fashion because the - cool expression in sizeof(int) that auto-configured - to any length int befuddled some compilers. */ - - if (sizeof(int) == 2) { - if (bs > 32767) { - bs = 32767; - } - } else { - if (bs > 200000) { - bs = 200000; - } - } - return bs; -} - -int main() -{ - int i; - double x; - - /* Seed the random number generator. If Repeatable is defined, we - always use the same seed. Otherwise, we seed from the clock to - shake things up from run to run. */ - -#ifdef Repeatable - V srand(1234); -#else - V srand((int) time((long *) NULL)); -#endif - - /* Compute x such that pow(x, p) ranges between 1 and 4*ExpIncr as - p ranges from 0 to ExpIncr-1, with a concentration in the lower - numbers. */ - - x = 4.0 * ExpIncr; - x = log(x); - x = exp(log(4.0 * ExpIncr) / (ExpIncr - 1.0)); - -#ifdef BECtl - bectl(bcompact, bexpand, bshrink, (bufsize) ExpIncr); - bp = malloc(ExpIncr); - assert(bp != NULL); - bpool((void *) bp, (bufsize) ExpIncr); -#else - bp = malloc(PoolSize); - assert(bp != NULL); - bpool((void *) bp, (bufsize) PoolSize); -#endif - - stats("Create pool"); - V bpoolv((void *) bp); - bpoold((void *) bp, dumpAlloc, dumpFree); - - for (i = 0; i < TestProg; i++) { - char *cb; - bufsize bs = pow(x, (double) (rand() & (ExpIncr - 1))); - - assert(bs <= (((bufsize) 4) * ExpIncr)); - bs = blimit(bs); - if (rand() & 0x400) { - cb = (char *) bgetz(bs); - } else { - cb = (char *) bget(bs); - } - if (cb == NULL) { -#ifdef EasyOut - break; -#else - char *bc = bchain; - - if (bc != NULL) { - char *fb; - - fb = *((char **) bc); - if (fb != NULL) { - *((char **) bc) = *((char **) fb); - brel((void *) fb); - } - continue; - } -#endif - } - *((char **) cb) = (char *) bchain; - bchain = cb; - - /* Based on a random cast, release a random buffer in the list - of allocated buffers. */ - - if ((rand() & 0x10) == 0) { - char *bc = bchain; - int i = rand() & 0x3; - - while (i > 0 && bc != NULL) { - bc = *((char **) bc); - i--; - } - if (bc != NULL) { - char *fb; - - fb = *((char **) bc); - if (fb != NULL) { - *((char **) bc) = *((char **) fb); - brel((void *) fb); - } - } - } - - /* Based on a random cast, reallocate a random buffer in the list - to a random size */ - - if ((rand() & 0x20) == 0) { - char *bc = bchain; - int i = rand() & 0x3; - - while (i > 0 && bc != NULL) { - bc = *((char **) bc); - i--; - } - if (bc != NULL) { - char *fb; - - fb = *((char **) bc); - if (fb != NULL) { - char *newb; - - bs = pow(x, (double) (rand() & (ExpIncr - 1))); - bs = blimit(bs); -#ifdef BECtl - protect = 1; /* Protect against compaction */ -#endif - newb = (char *) bgetr((void *) fb, bs); -#ifdef BECtl - protect = 0; -#endif - if (newb != NULL) { - *((char **) bc) = newb; - } - } - } - } - } - stats("\nAfter allocation"); - if (bp != NULL) { - V bpoolv((void *) bp); - bpoold((void *) bp, dumpAlloc, dumpFree); - } - - while (bchain != NULL) { - char *buf = bchain; - - bchain = *((char **) buf); - brel((void *) buf); - } - stats("\nAfter release"); -#ifndef BECtl - if (bp != NULL) { - V bpoolv((void *) bp); - bpoold((void *) bp, dumpAlloc, dumpFree); - } -#endif - - return 0; -} -#endif diff --git a/src/clib/bget.h b/src/clib/bget.h deleted file mode 100644 index 2b5be7dcdc0..00000000000 --- a/src/clib/bget.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - - Interface definitions for bget.c, the memory management package. - -*/ -#ifndef _BGET_H_ -#define _BGET_H_ - -/* in PIO we use DEBUG bget uses NDEBUG */ -//#ifndef DEBUG -//#undef NDEBUG -//#endif - -#ifndef _ -#ifdef PROTOTYPES -#define _(x) x /* If compiler knows prototypes */ -#else -#define _(x) () /* It it doesn't */ -#endif /* PROTOTYPES */ -#endif - -typedef long bufsize; -void bpool _((void *buffer, bufsize len)); -void *bget _((bufsize size)); -void *bgetz _((bufsize size)); -void *bgetr _((void *buffer, bufsize newsize)); -void brel _((void *buf)); -void bectl _((int (*compact)(bufsize sizereq, int sequence), - void *(*acquire)(bufsize size), - void (*release)(void *buf), bufsize pool_incr)); -void bstats _((bufsize *curalloc, bufsize *totfree, bufsize *maxfree, - long *nget, long *nrel)); -void bstatse _((bufsize *pool_incr, long *npool, long *npget, - long *nprel, long *ndget, long *ndrel)); -void bufdump _((void *buf)); -void bpoold _((void *pool, int dumpalloc, int dumpfree)); -int bpoolv _((void *pool)); -void bpoolrelease _(); -void bfreespace _((bufsize *maxfree, bufsize *totfree)); - -#endif From c9923271a04d7cdd9037db86eecc325c448c2199 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 15 Sep 2020 09:19:56 -0600 Subject: [PATCH 3/7] added cmake to test build --- .github/workflows/a4.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/a4.yml b/.github/workflows/a4.yml index a0e7ad96415..1723c37ecae 100644 --- a/.github/workflows/a4.yml +++ b/.github/workflows/a4.yml @@ -128,4 +128,19 @@ jobs: autoreconf -i ./configure --enable-fortran make -j check + - name: cmake build + run: | + set -x + gcc --version + echo 'export PATH=/home/runner/mpich/bin:$PATH' > .bashrc + source .bashrc + export CC=/home/runner/mpich/bin/mpicc + export FC=/home/runner/mpich/bin/mpifort + mkdir build + cd build + cmake -Wno-dev -DNetCDF_C_LIBRARY=/home/runner/netcdf-c/lib/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/home/runner/netcdf-c/include -DPnetCDF_PATH='/home/runner/pnetcdf' -DPIO_ENABLE_FORTRAN=Off -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off .. || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log) + make VERBOSE=1 + make tests VERBOSE=1 + ctest -VV + From 668bdd323d4e5968a59e14a84aa7ffce3994e3ea Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 15 Sep 2020 09:26:35 -0600 Subject: [PATCH 4/7] more bget fixes --- src/clib/pio_darray.c | 2 +- src/clib/pio_darray_int.c | 2 +- src/clib/pio_internal.h | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index fa91aeaca8a..61baa188110 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -718,7 +718,7 @@ PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *arra if (!wmb) { /* Allocate a buffer. */ - if (!(wmb = malloc((bufsize)sizeof(wmulti_buffer)))) + if (!(wmb = malloc(sizeof(wmulti_buffer)))) return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); /* Set pointer to newly allocated buffer and initialize.*/ diff --git a/src/clib/pio_darray_int.c b/src/clib/pio_darray_int.c index 546a1d6b2ec..d1ad415f302 100644 --- a/src/clib/pio_darray_int.c +++ b/src/clib/pio_darray_int.c @@ -23,7 +23,7 @@ extern PIO_Offset pio_buffer_size_limit; /** Initial size of compute buffer. */ -bufsize pio_cnbuffer_limit = 33554432; +long pio_cnbuffer_limit = 33554432; /** Global buffer pool pointer. */ extern void *CN_bpool; diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 5a365090ee5..196f3d7a2d4 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include From 14672d4d0bf2d853fe3446fc9f89ada74aaf1570 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 15 Sep 2020 09:57:41 -0600 Subject: [PATCH 5/7] working on cmake test build --- .github/workflows/a4.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/a4.yml b/.github/workflows/a4.yml index 1723c37ecae..353c18088c3 100644 --- a/.github/workflows/a4.yml +++ b/.github/workflows/a4.yml @@ -117,7 +117,7 @@ jobs: make sudo make install popd - - name: autotools build + - name: cmake build run: | set -x gcc --version @@ -125,10 +125,14 @@ jobs: source .bashrc export CC=/home/runner/mpich/bin/mpicc export FC=/home/runner/mpich/bin/mpifort - autoreconf -i - ./configure --enable-fortran - make -j check - - name: cmake build + export LD_LIBRARY_PATH="/home/runner/netcdf-c/lib:$LD_LIBRARY_PATH" + mkdir build + cd build + cmake -Wno-dev -DNetCDF_C_LIBRARY=/home/runner/netcdf-c/lib/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/home/runner/netcdf-c/include -DPnetCDF_PATH='/home/runner/pnetcdf' -DPIO_ENABLE_FORTRAN=Off -DPIO_HDF5_LOGGING=On -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off .. || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log) + make VERBOSE=1 + make tests VERBOSE=1 + ctest -VV + - name: autotools build run: | set -x gcc --version @@ -136,11 +140,8 @@ jobs: source .bashrc export CC=/home/runner/mpich/bin/mpicc export FC=/home/runner/mpich/bin/mpifort - mkdir build - cd build - cmake -Wno-dev -DNetCDF_C_LIBRARY=/home/runner/netcdf-c/lib/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/home/runner/netcdf-c/include -DPnetCDF_PATH='/home/runner/pnetcdf' -DPIO_ENABLE_FORTRAN=Off -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off .. || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log) - make VERBOSE=1 - make tests VERBOSE=1 - ctest -VV + autoreconf -i + ./configure --enable-fortran + make -j check From cbf9b1728158dc2f719c427eea31655091bdcd67 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 15 Sep 2020 10:14:49 -0600 Subject: [PATCH 6/7] working on cmake test build --- .github/workflows/a4.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/a4.yml b/.github/workflows/a4.yml index 353c18088c3..ccf44484080 100644 --- a/.github/workflows/a4.yml +++ b/.github/workflows/a4.yml @@ -125,10 +125,10 @@ jobs: source .bashrc export CC=/home/runner/mpich/bin/mpicc export FC=/home/runner/mpich/bin/mpifort - export LD_LIBRARY_PATH="/home/runner/netcdf-c/lib:$LD_LIBRARY_PATH" + export LD_LIBRARY_PATH="/home/runner/netcdf-c/lib:/home/runner/mpich/lib:/home/runner/hdf5/lib:/home/runner/netcdf-fortran/lib:/home/runner/pnetcdf/lib:$LD_LIBRARY_PATH" mkdir build cd build - cmake -Wno-dev -DNetCDF_C_LIBRARY=/home/runner/netcdf-c/lib/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/home/runner/netcdf-c/include -DPnetCDF_PATH='/home/runner/pnetcdf' -DPIO_ENABLE_FORTRAN=Off -DPIO_HDF5_LOGGING=On -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off .. || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log) + cmake -Wno-dev -DNetCDF_C_LIBRARY=/home/runner/netcdf-c/lib/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/home/runner/netcdf-c/include -DPnetCDF_PATH='/home/runner/pnetcdf' -DPIO_ENABLE_FORTRAN=Off -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off .. || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log) make VERBOSE=1 make tests VERBOSE=1 ctest -VV From d048ed581f49de97277b4910183391a821edc572 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 15 Sep 2020 10:56:25 -0600 Subject: [PATCH 7/7] working on cmake test build --- .github/workflows/a4.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/a4.yml b/.github/workflows/a4.yml index ccf44484080..3c625b33de4 100644 --- a/.github/workflows/a4.yml +++ b/.github/workflows/a4.yml @@ -131,7 +131,7 @@ jobs: cmake -Wno-dev -DNetCDF_C_LIBRARY=/home/runner/netcdf-c/lib/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/home/runner/netcdf-c/include -DPnetCDF_PATH='/home/runner/pnetcdf' -DPIO_ENABLE_FORTRAN=Off -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off .. || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log) make VERBOSE=1 make tests VERBOSE=1 - ctest -VV +# ctest -VV - name: autotools build run: | set -x