diff --git a/.travis.yml b/.travis.yml index 73dff72a1aa..6ebdd7167e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,46 +6,49 @@ language: cpp -compiler: - - gcc - - clang - -env: - - CONFIGURATION=Dev - - CONFIGURATION=Release - -before_install: - # we need at least g++-4.8 for c++11 features - - sudo add-apt-repository --yes ppa:ubuntu-toolchain-r/test - - sudo apt-get update --yes --quiet +sudo: false + +matrix: + include: + - os: linux + compiler: clang + env: BUILD_TYPE=Dev + - os: linux + compiler: clang + env: BUILD_TYPE=Release + - os: linux + compiler: gcc + env: BUILD_TYPE=Dev + - os: linux + compiler: gcc + env: BUILD_TYPE=Release + - os: osx + compiler: clang + env: BUILD_TYPE=Dev + - os: osx + compiler: clang + env: BUILD_TYPE=Release + +# http://docs.travis-ci.com/user/apt/ +addons: + apt: + sources: + - boost-latest + - ubuntu-toolchain-r-test + packages: + - g++-4.8 + - gcc-4.8 + - libboost1.55-dev + - libboost-program-options1.55-dev + - libgdal-dev + - libgeos++-dev + - libproj-dev + - libsparsehash-dev + - spatialite-bin install: - - cd .. - # upgrade compilers - - sudo apt-get install --yes gcc-4.8 g++-4.8 - # make sure 'cpp' is the just installed current one - - sudo rm /usr/bin/cpp - - sudo ln -s /usr/bin/cpp-4.8 /usr/bin/cpp - # upgrade libosmium dependencies - - sudo apt-get install --yes make libboost-dev libboost-program-options-dev libsparsehash-dev libprotobuf-dev protobuf-compiler libgeos++-dev libproj-dev libgdal1h libgdal-dev - - git clone https://github.com/osmcode/osm-testdata.git - # OSMPBF is too old, install from git - #- sudo apt-get install --yes libosmpbf-dev - - git clone https://github.com/scrosby/OSM-binary.git - - cd OSM-binary/src - - make - - sudo make install - - cd ../.. - - cd libosmium - -before_script: - - true + - scripts/travis_install.sh script: - - if [ "${CXX}" = 'g++' ]; then export CXX=g++-4.8; fi; - - mkdir build - - cd build - - cmake -LA -DCMAKE_BUILD_TYPE=${CONFIGURATION} .. - - make VERBOSE=1 - - ctest --output-on-failure + - scripts/travis_script.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c345a49817..22eb06aac56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,88 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [unreleased] - +### Added + +### Changed + +### Fixed + +## [2.3.0] - 2015-08-18 + +### Added + +- Allow instantiating osmium::geom::GEOSFactory with existing GEOS factory. +- Low-level functions to support generating a architecture- and endian- + independant CRC from OSM data. This is intended to be uses with boost::crc. +- Add new debug output format. This format is not intended to be read + automatically, but for human consumption. It formats the data nicely. +- Make writing of metadata configurable for XML and OPL output (use + `add_metadata=false` as file option). + +### Changed + +- Changed `add_user()` and `add_role()` in builders to use string length + without the 0-termination. +- Improved code setting file format from suffix/format argument. +- Memory mapping utility class now supports readonly, private writable or + shared writable operation. +- Allow empty version (0) in PBF files. +- Use utf8cpp header-only lib instead of boost for utf8 decoding. The library + is included in the libosmium distribution. +- New PBF reader and writer based on the protozero. A complete rewrite of the + code for reading and writing OSM PBF files. It doesn't use the Google + protobuf library and it doesn't use the OSMPBF/OSM-Binary library any more. + Instead is uses the protozero lightweight protobuf header library which is + included in the code. Not only does the new code have less dependencies, it + is faster and more robust. https://github.com/mapbox/protozero + +### Fixed + +- Various smaller bug fixes. +- Add encoding for relation member roles in OPL format. +- Change character encoding to new format in OPL: variable length hex code + between % characters instead of a % followed by 4-digit hex code. This is + necessary because unicode characters can be longer than the 4-digit hex + code. +- XML writer: The linefeed, carriage return, and tab characters are now + escaped properly. +- Reading large XML files could block. + +## [2.2.0] - 2015-07-04 + +### Added + +- Conversion functions for some low-level types. +- BoolVector index class. +- `min_op`/`max_op` utility functions. +- More tests here and there. +- Helper methods `is_between()` and `is_visible_at()` to DiffObject. +- GeoJSON factory using the RapidJSON library. +- Support for tile calculations. +- Create simple polygons from ways in geom factories. +- `MemoryMapping` and `TypedMemoryMapping` helper classes. +- `close()` function to `mmap_vector_base` class. +- Function on `Buffer` class to get iterator to specific offset. +- Explicit cast operator from `osmium::Timestamp` to `uint32_t`. + +### Changed + +- Throw exception on illegal values in functions parsing strings to get ids, + versions, etc. +- Improved error message for geometry exceptions. + +### Fixed + +- Throw exception from `dump_as_array()` and `dump_as_list()` functions if not + implemented in an index. +- After writing OSM files, program could stall up to a second. +- Dense location store was written out only partially. +- Use `uint64_t` as counter in benchmarks, so there can be no overflows. +- Example programs now read packed XML files, too. +- Refactoring of memory mapping code. Removes leak on Windows. +- Better check for invalid locations. +- Mark `cbegin()` and `cend()` of `mmap_vector_base` as const functions. + ## [2.1.0] - 2015-03-31 ### Added @@ -26,6 +108,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). Doxygen (up to version 1.8.8). This version contains a workaround to fix this. -[unreleased]: https://github.com/osmcode/libosmium/compare/v2.1.0...HEAD +[unreleased]: https://github.com/osmcode/libosmium/compare/v2.3.0...HEAD +[2.3.0]: https://github.com/osmcode/libosmium/compare/v2.3.0...v2.3.0 +[2.2.0]: https://github.com/osmcode/libosmium/compare/v2.1.0...v2.2.0 [2.1.0]: https://github.com/osmcode/libosmium/compare/v2.0.0...v2.1.0 diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e70a9935aa..fba967a4ed4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # #----------------------------------------------------------------------------- -set(CMAKE_CONFIGURATION_TYPES "Debug;Release;RelWithDebInfo;MinSizeRel;Dev" +set(CMAKE_CONFIGURATION_TYPES "Debug;Release;RelWithDebInfo;MinSizeRel;Dev;Coverage" CACHE STRING "List of available configuration types" FORCE) @@ -26,11 +26,13 @@ set(CMAKE_CONFIGURATION_TYPES "Debug;Release;RelWithDebInfo;MinSizeRel;Dev" project(libosmium) set(LIBOSMIUM_VERSION_MAJOR 2) -set(LIBOSMIUM_VERSION_MINOR 1) +set(LIBOSMIUM_VERSION_MINOR 3) set(LIBOSMIUM_VERSION_PATCH 0) set(LIBOSMIUM_VERSION - ${LIBOSMIUM_VERSION_MAJOR}.${LIBOSMIUM_VERSION_MINOR}.${LIBOSMIUM_VERSION_PATCH}) + "${LIBOSMIUM_VERSION_MAJOR}.${LIBOSMIUM_VERSION_MINOR}.${LIBOSMIUM_VERSION_PATCH}" + CACHE STRING + "Libosmium version") #----------------------------------------------------------------------------- @@ -55,6 +57,57 @@ option(BUILD_BENCHMARKS "compile benchmark programs" ${dev_build}) option(BUILD_DATA_TESTS "compile data tests, please run them with ctest" ${dev_build}) +#----------------------------------------------------------------------------- +# +# Coverage support +# +#----------------------------------------------------------------------------- + +include(CheckCXXCompilerFlag) +check_cxx_compiler_flag("-fkeep-inline-functions" HAS_KEEP_INLINE_FUNCTIONS) +if(HAS_KEEP_INLINE_FUNCTIONS) + set(extra_coverage_flags_ "-fkeep-inline-functions") +endif() + +set(CMAKE_CXX_FLAGS_COVERAGE + "-g -O0 -fno-inline-functions -fno-inline --coverage ${extra_coverage_flags_}" + CACHE STRING "Flags used by the compiler during coverage builds.") + +set(CMAKE_EXE_LINKER_FLAGS_COVERAGE + "--coverage" + CACHE STRING "Flags used by the linker during coverage builds.") + +if(CMAKE_BUILD_TYPE STREQUAL "Coverage") + if(BUILD_EXAMPLES OR BUILD_HEADERS OR BUILD_BENCHMARKS OR BUILD_DATA_TESTS) + message(WARNING "Coverage builds don't work for anything but the unit tests") + endif() + + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + string(REGEX REPLACE "^([0-9]+)\\.([0-9]+).*$" "llvm-cov-\\1.\\2" + gcov_ ${CMAKE_CXX_COMPILER_VERSION}) + else() + set(gcov_ "gcov") + endif() + + find_program(GCOV ${gcov_} DOC "Coverage tool") + find_program(GCOVR "gcovr" DOC "Coverage report tool") + + set(coverage_report_dir "${CMAKE_BINARY_DIR}/coverage") + file(MAKE_DIRECTORY ${coverage_report_dir}) + add_custom_target(coverage + ${GCOVR} + ${CMAKE_BINARY_DIR} + --root=${CMAKE_SOURCE_DIR} + --html --html-details + #--verbose + #--keep + '--filter=.*include/osmium.*' + --sort-percentage + --gcov-executable=${GCOV} + --output=${coverage_report_dir}/index.html) +endif() + + #----------------------------------------------------------------------------- # # Find external dependencies @@ -113,8 +166,10 @@ endif() #----------------------------------------------------------------------------- if(MSVC) set(USUAL_COMPILE_OPTIONS "/Ox") + set(USUAL_LINK_OPTIONS "/debug") else() set(USUAL_COMPILE_OPTIONS "-O3 -g") + set(USUAL_LINK_OPTIONS "") endif() if(WIN32) @@ -126,7 +181,7 @@ set(CMAKE_CXX_FLAGS_DEV "${USUAL_COMPILE_OPTIONS}" CACHE STRING "Flags used by the compiler during developer builds." FORCE) -set(CMAKE_EXE_LINKER_FLAGS_DEV "" +set(CMAKE_EXE_LINKER_FLAGS_DEV "${USUAL_LINK_OPTIONS}" CACHE STRING "Flags used by the linker during developer builds." FORCE) mark_as_advanced( diff --git a/EXTERNAL_LICENSES.txt b/EXTERNAL_LICENSES.txt new file mode 100644 index 00000000000..7b06fcf4110 --- /dev/null +++ b/EXTERNAL_LICENSES.txt @@ -0,0 +1,233 @@ + +==== For protozero from https://github.com/mapbox/protozero + +protozero copyright (c) Mapbox. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +==== For protozero from https://github.com/mapbox/protozero + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +==== For utf8.h + +Copyright 2006 Nemanja Trifunovic + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + diff --git a/README.md b/README.md index 503440e8ed8..9676d80b74b 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,10 @@ you need for your programs. For details see the [list of dependencies](https://github.com/osmcode/libosmium/wiki/Libosmium-dependencies). +The [protozero](https://github.com/mapbox/protozero) and +[utf8-cpp](http://utfcpp.sourceforge.net/) header-only libraries are included +in the libosmium repository. + ## Directories diff --git a/appveyor.yml b/appveyor.yml index 06c8e697496..a05c396cc3b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,6 +15,8 @@ branches: only: - master +shallow_clone: true + # Operating system (build VM template) os: Visual Studio 2014 CTP4 @@ -27,7 +29,7 @@ clone_folder: c:\projects\libosmium platform: x64 install: - # show all availble env vars + # show all available env vars - set - echo cmake on AppVeyor - cmake -version @@ -50,8 +52,8 @@ install: #cmake cannot find it otherwise - set LIBBZIP2=%LODEPSDIR%\bzip2\lib\libbz2.lib - set LIBBZIP2=%LIBBZIP2:\=/% - - ps: Start-FileDownload https://mapnik.s3.amazonaws.com/deps/cmake-3.1.0-win32-x86.7z -FileName cm.7z - - ps: Start-FileDownload https://mapnik.s3.amazonaws.com/dist/dev/libosmium-deps-win-14.0-x64.7z -FileName lodeps.7z + - ps: Start-FileDownload https://mapbox.s3.amazonaws.com/windows-builds/windows-build-deps/cmake-3.1.0-win32-x86.7z -FileName cm.7z + - ps: Start-FileDownload https://mapbox.s3.amazonaws.com/windows-builds/windows-build-deps/libosmium-deps-win-14.0-x64.7z -FileName lodeps.7z - 7z x cm.7z | %windir%\system32\find "ing archive" - 7z x lodeps.7z | %windir%\system32\find "ing archive" - echo %LODEPSDIR% @@ -59,19 +61,64 @@ install: - echo our own cmake - cmake -version - cd c:\projects - - git clone https://github.com/osmcode/osm-testdata.git + - git clone --depth 1 https://github.com/osmcode/osm-testdata.git build_script: - cd c:\projects\libosmium - mkdir build - cd build - echo %config% - - cmake .. -LA -G "Visual Studio 14 Win64" -DOsmium_DEBUG=TRUE -DCMAKE_BUILD_TYPE=%config% -DBUILD_BENCHMARKS=OFF -DBOOST_ROOT=%LODEPSDIR%\boost -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_57.lib -DOSMPBF_LIBRARY=%LODEPSDIR%\osmpbf\lib\osmpbf.lib -DOSMPBF_INCLUDE_DIR=%LODEPSDIR%\osmpbf\include -DPROTOBUF_LIBRARY=%LODEPSDIR%\protobuf\lib\libprotobuf.lib -DPROTOBUF_LITE_LIBRARY=%LODEPSDIR%\protobuf\lib\libprotobuf-lite.lib -DPROTOBUF_INCLUDE_DIR=%LODEPSDIR%\protobuf\include -DZLIB_LIBRARY=%LODEPSDIR%\zlib\lib\zlibwapi.lib -DZLIB_INCLUDE_DIR=%LODEPSDIR%\zlib\include -DEXPAT_LIBRARY=%LODEPSDIR%\expat\lib\libexpat.lib -DEXPAT_INCLUDE_DIR=%LODEPSDIR%\expat\include -DBZIP2_LIBRARIES=%LIBBZIP2% -DBZIP2_INCLUDE_DIR=%LODEPSDIR%\bzip2\include -DGDAL_LIBRARY=%LODEPSDIR%\gdal\lib\gdal_i.lib -DGDAL_INCLUDE_DIR=%LODEPSDIR%\gdal\include -DGEOS_LIBRARY=%LODEPSDIR%\geos\lib\geos.lib -DGEOS_INCLUDE_DIR=%LODEPSDIR%\geos\include -DPROJ_LIBRARY=%LODEPSDIR%\proj\lib\proj.lib -DPROJ_INCLUDE_DIR=%LODEPSDIR%\proj\include -DSPARSEHASH_INCLUDE_DIR=%LODEPSDIR%\sparsehash\include -DGETOPT_LIBRARY=%LODEPSDIR%\wingetopt\lib\wingetopt.lib -DGETOPT_INCLUDE_DIR=%LODEPSDIR%\wingetopt\include + # This will produce lots of LNK4099 warnings which can be ignored. + # Unfortunately they can't be disabled, see + # http://stackoverflow.com/questions/661606/visual-c-how-to-disable-specific-linker-warnings + - cmake .. -LA -G "Visual Studio 14 Win64" + -DOsmium_DEBUG=TRUE + -DCMAKE_BUILD_TYPE=%config% + -DBUILD_HEADERS=OFF + -DBOOST_ROOT=%LODEPSDIR%\boost + -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_57.lib + -DZLIB_LIBRARY=%LODEPSDIR%\zlib\lib\zlibwapi.lib + -DZLIB_INCLUDE_DIR=%LODEPSDIR%\zlib\include + -DEXPAT_LIBRARY=%LODEPSDIR%\expat\lib\libexpat.lib + -DEXPAT_INCLUDE_DIR=%LODEPSDIR%\expat\include + -DBZIP2_LIBRARIES=%LIBBZIP2% + -DBZIP2_INCLUDE_DIR=%LODEPSDIR%\bzip2\include + -DGDAL_LIBRARY=%LODEPSDIR%\gdal\lib\gdal_i.lib + -DGDAL_INCLUDE_DIR=%LODEPSDIR%\gdal\include + -DGEOS_LIBRARY=%LODEPSDIR%\geos\lib\geos.lib + -DGEOS_INCLUDE_DIR=%LODEPSDIR%\geos\include + -DPROJ_LIBRARY=%LODEPSDIR%\proj\lib\proj.lib + -DPROJ_INCLUDE_DIR=%LODEPSDIR%\proj\include + -DSPARSEHASH_INCLUDE_DIR=%LODEPSDIR%\sparsehash\include + -DGETOPT_LIBRARY=%LODEPSDIR%\wingetopt\lib\wingetopt.lib + -DGETOPT_INCLUDE_DIR=%LODEPSDIR%\wingetopt\include - msbuild libosmium.sln /p:Configuration=%config% /toolsversion:14.0 /p:Platform=x64 /p:PlatformToolset=v140 - #- cmake .. -LA -G "NMake Makefiles" -DOsmium_DEBUG=TRUE -DCMAKE_BUILD_TYPE=%config% -DBOOST_ROOT=%LODEPSDIR%\boost -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_57.lib -DOSMPBF_LIBRARY=%LODEPSDIR%\osmpbf\lib\osmpbf.lib -DOSMPBF_INCLUDE_DIR=%LODEPSDIR%\osmpbf\include -DPROTOBUF_LIBRARY=%LODEPSDIR%\protobuf\lib\libprotobuf.lib -DPROTOBUF_LITE_LIBRARY=%LODEPSDIR%\protobuf\lib\libprotobuf-lite.lib -DPROTOBUF_INCLUDE_DIR=%LODEPSDIR%\protobuf\include -DZLIB_LIBRARY=%LODEPSDIR%\zlib\lib\zlibwapi.lib -DZLIB_INCLUDE_DIR=%LODEPSDIR%\zlib\include -DEXPAT_LIBRARY=%LODEPSDIR%\expat\lib\libexpat.lib -DEXPAT_INCLUDE_DIR=%LODEPSDIR%\expat\include -DBZIP2_LIBRARIES=%LIBBZIP2% -DBZIP2_INCLUDE_DIR=%LODEPSDIR%\bzip2\include -DGDAL_LIBRARY=%LODEPSDIR%\gdal\lib\gdal_i.lib -DGDAL_INCLUDE_DIR=%LODEPSDIR%\gdal\include -DGEOS_LIBRARY=%LODEPSDIR%\geos\lib\geos.lib -DGEOS_INCLUDE_DIR=%LODEPSDIR%\geos\include -DPROJ_LIBRARY=%LODEPSDIR%\proj\lib\proj.lib -DPROJ_INCLUDE_DIR=%LODEPSDIR%\proj\include -DSPARSEHASH_INCLUDE_DIR=%LODEPSDIR%\sparsehash\include -DGETOPT_LIBRARY=%LODEPSDIR%\wingetopt\lib\wingetopt.lib -DGETOPT_INCLUDE_DIR=%LODEPSDIR%\wingetopt\include + #- cmake .. -LA -G "NMake Makefiles" + # -DOsmium_DEBUG=TRUE + # -DCMAKE_BUILD_TYPE=%config% + # -DBOOST_ROOT=%LODEPSDIR%\boost + # -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_57.lib + # -DZLIB_LIBRARY=%LODEPSDIR%\zlib\lib\zlibwapi.lib + # -DZLIB_INCLUDE_DIR=%LODEPSDIR%\zlib\include + # -DEXPAT_LIBRARY=%LODEPSDIR%\expat\lib\libexpat.lib + # -DEXPAT_INCLUDE_DIR=%LODEPSDIR%\expat\include + # -DBZIP2_LIBRARIES=%LIBBZIP2% + # -DBZIP2_INCLUDE_DIR=%LODEPSDIR%\bzip2\include + # -DGDAL_LIBRARY=%LODEPSDIR%\gdal\lib\gdal_i.lib + # -DGDAL_INCLUDE_DIR=%LODEPSDIR%\gdal\include + # -DGEOS_LIBRARY=%LODEPSDIR%\geos\lib\geos.lib + # -DGEOS_INCLUDE_DIR=%LODEPSDIR%\geos\include + # -DPROJ_LIBRARY=%LODEPSDIR%\proj\lib\proj.lib + # -DPROJ_INCLUDE_DIR=%LODEPSDIR%\proj\include + # -DSPARSEHASH_INCLUDE_DIR=%LODEPSDIR%\sparsehash\include + # -DGETOPT_LIBRARY=%LODEPSDIR%\wingetopt\lib\wingetopt.lib + # -DGETOPT_INCLUDE_DIR=%LODEPSDIR%\wingetopt\include #- nmake test_script: - # -LE fails_on_windows exempts tests we know will fail - - ctest --output-on-failure -C %config% -LE fails_on_windows + # "-E testdata-overview" exempts one test we know fails on Appveyor + # because we currently don't have spatialite support. + - ctest --output-on-failure + -C %config% + -E testdata-overview diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 6a4ca162dba..e46c833496d 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -13,6 +13,7 @@ set(BENCHMARKS count_tag index_map static_vs_dynamic_index + write_pbf CACHE STRING "Benchmark programs" ) diff --git a/benchmarks/osmium_benchmark_count.cpp b/benchmarks/osmium_benchmark_count.cpp index 701d6faec8c..d50c53dc814 100644 --- a/benchmarks/osmium_benchmark_count.cpp +++ b/benchmarks/osmium_benchmark_count.cpp @@ -4,6 +4,7 @@ */ +#include #include #include @@ -12,9 +13,9 @@ struct CountHandler : public osmium::handler::Handler { - int nodes = 0; - int ways = 0; - int relations = 0; + uint64_t nodes = 0; + uint64_t ways = 0; + uint64_t relations = 0; void node(osmium::Node&) { ++nodes; @@ -48,7 +49,5 @@ int main(int argc, char* argv[]) { std::cout << "Nodes: " << handler.nodes << "\n"; std::cout << "Ways: " << handler.ways << "\n"; std::cout << "Relations: " << handler.relations << "\n"; - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/benchmarks/osmium_benchmark_count_tag.cpp b/benchmarks/osmium_benchmark_count_tag.cpp index 4a77c34529b..8fa696a4ee6 100644 --- a/benchmarks/osmium_benchmark_count_tag.cpp +++ b/benchmarks/osmium_benchmark_count_tag.cpp @@ -4,6 +4,7 @@ */ +#include #include #include @@ -12,8 +13,8 @@ struct CountHandler : public osmium::handler::Handler { - int counter = 0; - int all = 0; + uint64_t counter = 0; + uint64_t all = 0; void node(osmium::Node& node) { ++all; @@ -49,7 +50,5 @@ int main(int argc, char* argv[]) { reader.close(); std::cout << "r_all=" << handler.all << " r_counter=" << handler.counter << "\n"; - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/benchmarks/osmium_benchmark_index_map.cpp b/benchmarks/osmium_benchmark_index_map.cpp index fa75fb2b86a..02578261808 100644 --- a/benchmarks/osmium_benchmark_index_map.cpp +++ b/benchmarks/osmium_benchmark_index_map.cpp @@ -35,7 +35,5 @@ int main(int argc, char* argv[]) { osmium::apply(reader, location_handler); reader.close(); - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/benchmarks/osmium_benchmark_static_vs_dynamic_index.cpp b/benchmarks/osmium_benchmark_static_vs_dynamic_index.cpp index 9c47c84499e..66e2a0bd918 100644 --- a/benchmarks/osmium_benchmark_static_vs_dynamic_index.cpp +++ b/benchmarks/osmium_benchmark_static_vs_dynamic_index.cpp @@ -46,7 +46,6 @@ int main(int argc, char* argv[]) { std::string input_filename = argv[1]; osmium::memory::Buffer buffer = osmium::io::read_file(input_filename); - google::protobuf::ShutdownProtobufLibrary(); const auto& map_factory = osmium::index::MapFactory::instance(); diff --git a/benchmarks/osmium_benchmark_write_pbf.cpp b/benchmarks/osmium_benchmark_write_pbf.cpp new file mode 100644 index 00000000000..869f3a8f849 --- /dev/null +++ b/benchmarks/osmium_benchmark_write_pbf.cpp @@ -0,0 +1,34 @@ +/* + + The code in this file is released into the Public Domain. + +*/ + +#include +#include + +#include +#include + +int main(int argc, char* argv[]) { + if (argc != 3) { + std::cerr << "Usage: " << argv[0] << " INPUT-FILE OUTPUT-FILE\n"; + exit(1); + } + + std::string input_filename = argv[1]; + std::string output_filename = argv[2]; + + osmium::io::Reader reader(input_filename); + osmium::io::File output_file(output_filename, "pbf"); + osmium::io::Header header; + osmium::io::Writer writer(output_file, header, osmium::io::overwrite::allow); + + while (osmium::memory::Buffer buffer = reader.read()) { + writer(std::move(buffer)); + } + + writer.close(); + reader.close(); +} + diff --git a/benchmarks/run_benchmark_write_pbf.sh b/benchmarks/run_benchmark_write_pbf.sh new file mode 100755 index 00000000000..8143097702a --- /dev/null +++ b/benchmarks/run_benchmark_write_pbf.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# +# run_benchmark_write_pbf.sh +# +# Will read the input file and after reading it into memory completely, +# write it to /dev/null. Because this will need the time to read *and* write +# the file, it will report the times for reading and writing. You can +# subtract the times needed for the "count" benchmark to (roughly) get the +# write times. +# + +set -e + +BENCHMARK_NAME=write_pbf + +. @CMAKE_BINARY_DIR@/benchmarks/setup.sh + +CMD=$OB_DIR/osmium_benchmark_$BENCHMARK_NAME + +echo "# file size num mem time cpu_kernel cpu_user cpu_percent cmd options" +for data in $OB_DATA_FILES; do + filename=`basename $data` + filesize=`stat --format="%s" --dereference $data` + for n in $OB_SEQ; do + $OB_TIME_CMD -f "$filename $filesize $n $OB_TIME_FORMAT" $CMD $data /dev/null 2>&1 >/dev/null | sed -e "s%$DATA_DIR/%%" | sed -e "s%$OB_DIR/%%" + done +done + diff --git a/cmake/FindOSMPBF.cmake b/cmake/FindOSMPBF.cmake deleted file mode 100644 index deeebd8b6c9..00000000000 --- a/cmake/FindOSMPBF.cmake +++ /dev/null @@ -1,50 +0,0 @@ -# -# Locate OSMPBF library -# -# This module defines -# OSMPBF_FOUND - if false, do not try to link to OSMPBF -# OSMPBF_LIBRARIES - full library path name -# OSMPBF_INCLUDE_DIRS - where to find OSMPBF.hpp -# -# Note that the expected include convention is -# #include -# and not -# #include -# - -find_path(OSMPBF_INCLUDE_DIR osmpbf/osmpbf.h - HINTS $ENV{OSMPBF_DIR} - PATH_SUFFIXES include - PATHS - ~/Library/Frameworks - /Library/Frameworks - /usr/local - /usr - /opt/local # DarwinPorts - /opt -) - -find_library(OSMPBF_LIBRARY - NAMES osmpbf - HINTS $ENV{OSMPBF_DIR} - PATH_SUFFIXES lib64 lib - PATHS - ~/Library/Frameworks - /Library/Frameworks - /usr/local - /usr - /opt/local - /opt -) - -# Handle the QUIETLY and REQUIRED arguments and set OSMPBF_FOUND to TRUE if -# all listed variables are TRUE. -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(OSMPBF DEFAULT_MSG OSMPBF_LIBRARY OSMPBF_INCLUDE_DIR) - -# Copy the results to the output variables. -if(OSMPBF_FOUND) - set(OSMPBF_INCLUDE_DIRS ${OSMPBF_INCLUDE_DIR}) - set(OSMPBF_LIBRARIES ${OSMPBF_LIBRARY}) -endif() - diff --git a/cmake/FindOsmium.cmake b/cmake/FindOsmium.cmake index 1de41a02268..bb140718bdf 100644 --- a/cmake/FindOsmium.cmake +++ b/cmake/FindOsmium.cmake @@ -110,15 +110,11 @@ endif() #---------------------------------------------------------------------- # Component 'pbf' if(Osmium_USE_PBF) - find_package(OSMPBF) - find_package(Protobuf) find_package(ZLIB) find_package(Threads) - if(OSMPBF_FOUND AND PROTOBUF_FOUND AND ZLIB_FOUND AND Threads_FOUND) + if(ZLIB_FOUND AND Threads_FOUND) list(APPEND OSMIUM_PBF_LIBRARIES - ${OSMPBF_LIBRARIES} - ${PROTOBUF_LITE_LIBRARY} ${ZLIB_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ) @@ -126,8 +122,6 @@ if(Osmium_USE_PBF) list(APPEND OSMIUM_PBF_LIBRARIES ws2_32) endif() list(APPEND OSMIUM_INCLUDE_DIRS - ${OSMPBF_INCLUDE_DIRS} - ${PROTOBUF_INCLUDE_DIR} ${ZLIB_INCLUDE_DIR} ) else() @@ -325,7 +319,7 @@ endif() if(MSVC) set(OSMIUM_WARNING_OPTIONS "/W3 /wd4514" CACHE STRING "Recommended warning options for libosmium") else() - set(OSMIUM_WARNING_OPTIONS "-Wall -Wextra -pedantic -Wredundant-decls -Wdisabled-optimization -Wctor-dtor-privacy -Wnon-virtual-dtor -Woverloaded-virtual -Wsign-promo -Wold-style-cast -Wno-return-type" CACHE STRING "Recommended warning options for libosmium") + set(OSMIUM_WARNING_OPTIONS "-Wall -Wextra -pedantic -Wredundant-decls -Wdisabled-optimization -Wctor-dtor-privacy -Wnon-virtual-dtor -Woverloaded-virtual -Wsign-promo -Wold-style-cast" CACHE STRING "Recommended warning options for libosmium") endif() set(OSMIUM_DRACONIC_CLANG_OPTIONS "-Wdocumentation -Wunused-exception-parameter -Wmissing-declarations -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-unused-macros -Wno-exit-time-destructors -Wno-global-constructors -Wno-padded -Wno-switch-enum -Wno-missing-prototypes -Wno-weak-vtables -Wno-cast-align -Wno-float-equal") diff --git a/cmake/iwyu.sh b/cmake/iwyu.sh index d2038449151..f7d8a15e809 100755 --- a/cmake/iwyu.sh +++ b/cmake/iwyu.sh @@ -10,12 +10,12 @@ cmdline="iwyu -Xiwyu --mapping_file=osmium.imp -std=c++11 -I include" log=build/iwyu.log +mkdir -p build/check_reports + echo "INCLUDE WHAT YOU USE REPORT:" >$log allok=yes -mkdir -p build/check_reports - for file in `find include/osmium -name \*.hpp`; do mkdir -p `dirname build/check_reports/$file` ifile="build/check_reports/${file%.hpp}.iwyu" diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 9d69a16bd9e..5ea819b9220 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -21,8 +21,6 @@ if(DOXYGEN_FOUND) WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generating API documentation with Doxygen" VERBATIM ) -# install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/html" -# DESTINATION "share/doc/libosmium-dev") else() message(STATUS "Looking for doxygen - not found") message(STATUS " Disabled making of documentation.") diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index c03e255e9ef..d5ed13d2e80 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -1128,7 +1128,7 @@ HTML_COLORSTYLE_GAMMA = 80 # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_TIMESTAMP = YES +HTML_TIMESTAMP = NO # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML # documentation will contain sections that can be hidden and shown after the diff --git a/doc/README.md b/doc/README.md index 7ca8e7cf73c..5e1cf4bd2e0 100644 --- a/doc/README.md +++ b/doc/README.md @@ -3,6 +3,6 @@ The `header.html` is created with: `doxygen -w html header.html footer.html stylesheet.css` -This might have to be rn again for newer Doxygen versions. After that add +This might have to be run again for newer Doxygen versions. After that add changes back in. diff --git a/examples/osmium_area_test.cpp b/examples/osmium_area_test.cpp index ee2ba129648..f072c5e546a 100644 --- a/examples/osmium_area_test.cpp +++ b/examples/osmium_area_test.cpp @@ -132,7 +132,5 @@ int main(int argc, char* argv[]) { } std::cerr << "\n"; } - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/examples/osmium_convert.cpp b/examples/osmium_convert.cpp index 7956e1181c1..4f2ba33185f 100644 --- a/examples/osmium_convert.cpp +++ b/examples/osmium_convert.cpp @@ -106,7 +106,6 @@ int main(int argc, char* argv[]) { exit_code = 1; } - google::protobuf::ShutdownProtobufLibrary(); return exit_code; } diff --git a/examples/osmium_count.cpp b/examples/osmium_count.cpp index dca18bfdc0a..baea153b88f 100644 --- a/examples/osmium_count.cpp +++ b/examples/osmium_count.cpp @@ -7,6 +7,7 @@ */ +#include #include #include @@ -15,9 +16,9 @@ struct CountHandler : public osmium::handler::Handler { - int nodes = 0; - int ways = 0; - int relations = 0; + uint64_t nodes = 0; + uint64_t ways = 0; + uint64_t relations = 0; void node(osmium::Node&) { ++nodes; @@ -51,7 +52,5 @@ int main(int argc, char* argv[]) { std::cout << "Nodes: " << handler.nodes << "\n"; std::cout << "Ways: " << handler.ways << "\n"; std::cout << "Relations: " << handler.relations << "\n"; - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/examples/osmium_create_node_cache.cpp b/examples/osmium_create_node_cache.cpp index 74f7596fe77..359fa1978a8 100644 --- a/examples/osmium_create_node_cache.cpp +++ b/examples/osmium_create_node_cache.cpp @@ -12,8 +12,7 @@ #include #include -#include -#include +#include #include #include @@ -51,8 +50,6 @@ int main(int argc, char* argv[]) { osmium::apply(reader, location_handler); reader.close(); - google::protobuf::ShutdownProtobufLibrary(); - return 0; } diff --git a/examples/osmium_debug.cpp b/examples/osmium_debug.cpp index 6878ed1e8d1..365fc729754 100644 --- a/examples/osmium_debug.cpp +++ b/examples/osmium_debug.cpp @@ -46,7 +46,5 @@ int main(int argc, char* argv[]) { } reader.close(); - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/examples/osmium_read.cpp b/examples/osmium_read.cpp index 1bb0299662c..653600684a1 100644 --- a/examples/osmium_read.cpp +++ b/examples/osmium_read.cpp @@ -26,7 +26,5 @@ int main(int argc, char* argv[]) { } reader.close(); - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/examples/osmium_serdump.cpp b/examples/osmium_serdump.cpp index a774a8dbd38..9ab26e4eeee 100644 --- a/examples/osmium_serdump.cpp +++ b/examples/osmium_serdump.cpp @@ -19,8 +19,7 @@ # include #endif -#include -#include +#include #include #include @@ -203,7 +202,5 @@ int main(int argc, char* argv[]) { map_relation2relation.dump_as_list(fd); close(fd); } - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/examples/osmium_toogr.cpp b/examples/osmium_toogr.cpp index 6d8ab8d940d..7c5a965c503 100644 --- a/examples/osmium_toogr.cpp +++ b/examples/osmium_toogr.cpp @@ -234,8 +234,6 @@ int main(int argc, char* argv[]) { osmium::apply(reader, location_handler, ogr_handler); reader.close(); - google::protobuf::ShutdownProtobufLibrary(); - int locations_fd = open("locations.dump", O_WRONLY | O_CREAT, 0644); if (locations_fd < 0) { throw std::system_error(errno, std::system_category(), "Open failed"); diff --git a/examples/osmium_toogr2.cpp b/examples/osmium_toogr2.cpp index a966c5edcca..e1b505688bd 100644 --- a/examples/osmium_toogr2.cpp +++ b/examples/osmium_toogr2.cpp @@ -327,7 +327,5 @@ int main(int argc, char* argv[]) { } std::cerr << "\n"; } - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/examples/osmium_toogr2_exp.cpp b/examples/osmium_toogr2_exp.cpp index 474da96d013..db8d5cf4ec3 100644 --- a/examples/osmium_toogr2_exp.cpp +++ b/examples/osmium_toogr2_exp.cpp @@ -301,7 +301,5 @@ int main(int argc, char* argv[]) { } std::cerr << "\n"; } - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/examples/osmium_use_node_cache.cpp b/examples/osmium_use_node_cache.cpp index 6b8f964abaa..cfee6df4683 100644 --- a/examples/osmium_use_node_cache.cpp +++ b/examples/osmium_use_node_cache.cpp @@ -12,8 +12,7 @@ #include #include -#include -#include +#include #include #include @@ -64,8 +63,6 @@ int main(int argc, char* argv[]) { osmium::apply(reader, location_handler, handler); reader.close(); - google::protobuf::ShutdownProtobufLibrary(); - return 0; } diff --git a/include/boost_unicode_iterator.hpp b/include/boost_unicode_iterator.hpp deleted file mode 100644 index 3a7b68fbe7a..00000000000 --- a/include/boost_unicode_iterator.hpp +++ /dev/null @@ -1,776 +0,0 @@ -/* - * - * Copyright (c) 2004 - * John Maddock - * - * Use, modification and distribution are subject to the - * Boost Software License, Version 1.0. (See accompanying file - * LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - * - */ - - /* - * LOCATION: see http://www.boost.org for most recent version. - * FILE unicode_iterator.hpp - * VERSION see - * DESCRIPTION: Iterator adapters for converting between different Unicode encodings. - */ - -/**************************************************************************** - -Contents: -~~~~~~~~~ - -1) Read Only, Input Adapters: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -template -class u32_to_u8_iterator; - -Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-8. - -template -class u8_to_u32_iterator; - -Adapts sequence of UTF-8 code points to "look like" a sequence of UTF-32. - -template -class u32_to_u16_iterator; - -Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-16. - -template -class u16_to_u32_iterator; - -Adapts sequence of UTF-16 code points to "look like" a sequence of UTF-32. - -2) Single pass output iterator adapters: - -template -class utf8_output_iterator; - -Accepts UTF-32 code points and forwards them on as UTF-8 code points. - -template -class utf16_output_iterator; - -Accepts UTF-32 code points and forwards them on as UTF-16 code points. - -****************************************************************************/ - -#ifndef BOOST_REGEX_UNICODE_ITERATOR_HPP -#define BOOST_REGEX_UNICODE_ITERATOR_HPP -#include -#include -#include -#include -#include -#include -#ifndef BOOST_NO_STD_LOCALE -#include -#include -#endif -#include // CHAR_BIT - -namespace boost{ - -namespace detail{ - -static const ::boost::uint16_t high_surrogate_base = 0xD7C0u; -static const ::boost::uint16_t low_surrogate_base = 0xDC00u; -static const ::boost::uint32_t ten_bit_mask = 0x3FFu; - -inline bool is_high_surrogate(::boost::uint16_t v) -{ - return (v & 0xFFFFFC00u) == 0xd800u; -} -inline bool is_low_surrogate(::boost::uint16_t v) -{ - return (v & 0xFFFFFC00u) == 0xdc00u; -} -template -inline bool is_surrogate(T v) -{ - return (v & 0xFFFFF800u) == 0xd800; -} - -inline unsigned utf8_byte_count(boost::uint8_t c) -{ - // if the most significant bit with a zero in it is in position - // 8-N then there are N bytes in this UTF-8 sequence: - boost::uint8_t mask = 0x80u; - unsigned result = 0; - while(c & mask) - { - ++result; - mask >>= 1; - } - return (result == 0) ? 1 : ((result > 4) ? 4 : result); -} - -inline unsigned utf8_trailing_byte_count(boost::uint8_t c) -{ - return utf8_byte_count(c) - 1; -} - -#ifdef BOOST_MSVC -#pragma warning(push) -#pragma warning(disable:4100) -#endif -inline void invalid_utf32_code_point(::boost::uint32_t val) -{ -#ifndef BOOST_NO_STD_LOCALE - std::stringstream ss; - ss << "Invalid UTF-32 code point U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-16 sequence"; - std::out_of_range e(ss.str()); -#else - std::out_of_range e("Invalid UTF-32 code point encountered while trying to encode UTF-16 sequence"); -#endif - boost::throw_exception(e); -} -#ifdef BOOST_MSVC -#pragma warning(pop) -#endif - - -} // namespace detail - -template -class u32_to_u16_iterator - : public boost::iterator_facade, U16Type, std::bidirectional_iterator_tag, const U16Type> -{ - typedef boost::iterator_facade, U16Type, std::bidirectional_iterator_tag, const U16Type> base_type; - -#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION) - typedef typename std::iterator_traits::value_type base_value_type; - - BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32); - BOOST_STATIC_ASSERT(sizeof(U16Type)*CHAR_BIT == 16); -#endif - -public: - typename base_type::reference - dereference()const - { - if(m_current == 2) - extract_current(); - return m_values[m_current]; - } - bool equal(const u32_to_u16_iterator& that)const - { - if(m_position == that.m_position) - { - // Both m_currents must be equal, or both even - // this is the same as saying their sum must be even: - return (m_current + that.m_current) & 1u ? false : true; - } - return false; - } - void increment() - { - // if we have a pending read then read now, so that we know whether - // to skip a position, or move to a low-surrogate: - if(m_current == 2) - { - // pending read: - extract_current(); - } - // move to the next surrogate position: - ++m_current; - // if we've reached the end skip a position: - if(m_values[m_current] == 0) - { - m_current = 2; - ++m_position; - } - } - void decrement() - { - if(m_current != 1) - { - // decrementing an iterator always leads to a valid position: - --m_position; - extract_current(); - m_current = m_values[1] ? 1 : 0; - } - else - { - m_current = 0; - } - } - BaseIterator base()const - { - return m_position; - } - // construct: - u32_to_u16_iterator() : m_position(), m_current(0) - { - m_values[0] = 0; - m_values[1] = 0; - m_values[2] = 0; - } - u32_to_u16_iterator(BaseIterator b) : m_position(b), m_current(2) - { - m_values[0] = 0; - m_values[1] = 0; - m_values[2] = 0; - } -private: - - void extract_current()const - { - // begin by checking for a code point out of range: - ::boost::uint32_t v = *m_position; - if(v >= 0x10000u) - { - if(v > 0x10FFFFu) - detail::invalid_utf32_code_point(*m_position); - // split into two surrogates: - m_values[0] = static_cast(v >> 10) + detail::high_surrogate_base; - m_values[1] = static_cast(v & detail::ten_bit_mask) + detail::low_surrogate_base; - m_current = 0; - BOOST_ASSERT(detail::is_high_surrogate(m_values[0])); - BOOST_ASSERT(detail::is_low_surrogate(m_values[1])); - } - else - { - // 16-bit code point: - m_values[0] = static_cast(*m_position); - m_values[1] = 0; - m_current = 0; - // value must not be a surrogate: - if(detail::is_surrogate(m_values[0])) - detail::invalid_utf32_code_point(*m_position); - } - } - BaseIterator m_position; - mutable U16Type m_values[3]; - mutable unsigned m_current; -}; - -template -class u16_to_u32_iterator - : public boost::iterator_facade, U32Type, std::bidirectional_iterator_tag, const U32Type> -{ - typedef boost::iterator_facade, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type; - // special values for pending iterator reads: - BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu); - -#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION) - typedef typename std::iterator_traits::value_type base_value_type; - - BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 16); - BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32); -#endif - -public: - typename base_type::reference - dereference()const - { - if(m_value == pending_read) - extract_current(); - return m_value; - } - bool equal(const u16_to_u32_iterator& that)const - { - return m_position == that.m_position; - } - void increment() - { - // skip high surrogate first if there is one: - if(detail::is_high_surrogate(*m_position)) ++m_position; - ++m_position; - m_value = pending_read; - } - void decrement() - { - --m_position; - // if we have a low surrogate then go back one more: - if(detail::is_low_surrogate(*m_position)) - --m_position; - m_value = pending_read; - } - BaseIterator base()const - { - return m_position; - } - // construct: - u16_to_u32_iterator() : m_position() - { - m_value = pending_read; - } - u16_to_u32_iterator(BaseIterator b) : m_position(b) - { - m_value = pending_read; - } - // - // Range checked version: - // - u16_to_u32_iterator(BaseIterator b, BaseIterator start, BaseIterator end) : m_position(b) - { - m_value = pending_read; - // - // The range must not start with a low surrogate, or end in a high surrogate, - // otherwise we run the risk of running outside the underlying input range. - // Likewise b must not be located at a low surrogate. - // - boost::uint16_t val; - if(start != end) - { - if((b != start) && (b != end)) - { - val = *b; - if(detail::is_surrogate(val) && ((val & 0xFC00u) == 0xDC00u)) - invalid_code_point(val); - } - val = *start; - if(detail::is_surrogate(val) && ((val & 0xFC00u) == 0xDC00u)) - invalid_code_point(val); - val = *--end; - if(detail::is_high_surrogate(val)) - invalid_code_point(val); - } - } -private: - static void invalid_code_point(::boost::uint16_t val) - { -#ifndef BOOST_NO_STD_LOCALE - std::stringstream ss; - ss << "Misplaced UTF-16 surrogate U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-32 sequence"; - std::out_of_range e(ss.str()); -#else - std::out_of_range e("Misplaced UTF-16 surrogate encountered while trying to encode UTF-32 sequence"); -#endif - boost::throw_exception(e); - } - void extract_current()const - { - m_value = static_cast(static_cast< ::boost::uint16_t>(*m_position)); - // if the last value is a high surrogate then adjust m_position and m_value as needed: - if(detail::is_high_surrogate(*m_position)) - { - // precondition; next value must have be a low-surrogate: - BaseIterator next(m_position); - ::boost::uint16_t t = *++next; - if((t & 0xFC00u) != 0xDC00u) - invalid_code_point(t); - m_value = (m_value - detail::high_surrogate_base) << 10; - m_value |= (static_cast(static_cast< ::boost::uint16_t>(t)) & detail::ten_bit_mask); - } - // postcondition; result must not be a surrogate: - if(detail::is_surrogate(m_value)) - invalid_code_point(static_cast< ::boost::uint16_t>(m_value)); - } - BaseIterator m_position; - mutable U32Type m_value; -}; - -template -class u32_to_u8_iterator - : public boost::iterator_facade, U8Type, std::bidirectional_iterator_tag, const U8Type> -{ - typedef boost::iterator_facade, U8Type, std::bidirectional_iterator_tag, const U8Type> base_type; - -#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION) - typedef typename std::iterator_traits::value_type base_value_type; - - BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32); - BOOST_STATIC_ASSERT(sizeof(U8Type)*CHAR_BIT == 8); -#endif - -public: - typename base_type::reference - dereference()const - { - if(m_current == 4) - extract_current(); - return m_values[m_current]; - } - bool equal(const u32_to_u8_iterator& that)const - { - if(m_position == that.m_position) - { - // either the m_current's must be equal, or one must be 0 and - // the other 4: which means neither must have bits 1 or 2 set: - return (m_current == that.m_current) - || (((m_current | that.m_current) & 3) == 0); - } - return false; - } - void increment() - { - // if we have a pending read then read now, so that we know whether - // to skip a position, or move to a low-surrogate: - if(m_current == 4) - { - // pending read: - extract_current(); - } - // move to the next surrogate position: - ++m_current; - // if we've reached the end skip a position: - if(m_values[m_current] == 0) - { - m_current = 4; - ++m_position; - } - } - void decrement() - { - if((m_current & 3) == 0) - { - --m_position; - extract_current(); - m_current = 3; - while(m_current && (m_values[m_current] == 0)) - --m_current; - } - else - --m_current; - } - BaseIterator base()const - { - return m_position; - } - // construct: - u32_to_u8_iterator() : m_position(), m_current(0) - { - m_values[0] = 0; - m_values[1] = 0; - m_values[2] = 0; - m_values[3] = 0; - m_values[4] = 0; - } - u32_to_u8_iterator(BaseIterator b) : m_position(b), m_current(4) - { - m_values[0] = 0; - m_values[1] = 0; - m_values[2] = 0; - m_values[3] = 0; - m_values[4] = 0; - } -private: - - void extract_current()const - { - boost::uint32_t c = *m_position; - if(c > 0x10FFFFu) - detail::invalid_utf32_code_point(c); - if(c < 0x80u) - { - m_values[0] = static_cast(c); - m_values[1] = static_cast(0u); - m_values[2] = static_cast(0u); - m_values[3] = static_cast(0u); - } - else if(c < 0x800u) - { - m_values[0] = static_cast(0xC0u + (c >> 6)); - m_values[1] = static_cast(0x80u + (c & 0x3Fu)); - m_values[2] = static_cast(0u); - m_values[3] = static_cast(0u); - } - else if(c < 0x10000u) - { - m_values[0] = static_cast(0xE0u + (c >> 12)); - m_values[1] = static_cast(0x80u + ((c >> 6) & 0x3Fu)); - m_values[2] = static_cast(0x80u + (c & 0x3Fu)); - m_values[3] = static_cast(0u); - } - else - { - m_values[0] = static_cast(0xF0u + (c >> 18)); - m_values[1] = static_cast(0x80u + ((c >> 12) & 0x3Fu)); - m_values[2] = static_cast(0x80u + ((c >> 6) & 0x3Fu)); - m_values[3] = static_cast(0x80u + (c & 0x3Fu)); - } - m_current= 0; - } - BaseIterator m_position; - mutable U8Type m_values[5]; - mutable unsigned m_current; -}; - -template -class u8_to_u32_iterator - : public boost::iterator_facade, U32Type, std::bidirectional_iterator_tag, const U32Type> -{ - typedef boost::iterator_facade, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type; - // special values for pending iterator reads: - BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu); - -#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION) - typedef typename std::iterator_traits::value_type base_value_type; - - BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 8); - BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32); -#endif - -public: - typename base_type::reference - dereference()const - { - if(m_value == pending_read) - extract_current(); - return m_value; - } - bool equal(const u8_to_u32_iterator& that)const - { - return m_position == that.m_position; - } - void increment() - { - // We must not start with a continuation character: - if((static_cast(*m_position) & 0xC0) == 0x80) - invalid_sequence(); - // skip high surrogate first if there is one: - unsigned c = detail::utf8_byte_count(*m_position); - if(m_value == pending_read) - { - // Since we haven't read in a value, we need to validate the code points: - for(unsigned i = 0; i < c; ++i) - { - ++m_position; - // We must have a continuation byte: - if((i != c - 1) && ((static_cast(*m_position) & 0xC0) != 0x80)) - invalid_sequence(); - } - } - else - { - std::advance(m_position, c); - } - m_value = pending_read; - } - void decrement() - { - // Keep backtracking until we don't have a trailing character: - unsigned count = 0; - while((*--m_position & 0xC0u) == 0x80u) ++count; - // now check that the sequence was valid: - if(count != detail::utf8_trailing_byte_count(*m_position)) - invalid_sequence(); - m_value = pending_read; - } - BaseIterator base()const - { - return m_position; - } - // construct: - u8_to_u32_iterator() : m_position() - { - m_value = pending_read; - } - u8_to_u32_iterator(BaseIterator b) : m_position(b) - { - m_value = pending_read; - } - // - // Checked constructor: - // - u8_to_u32_iterator(BaseIterator b, BaseIterator start, BaseIterator end) : m_position(b) - { - m_value = pending_read; - // - // We must not start with a continuation character, or end with a - // truncated UTF-8 sequence otherwise we run the risk of going past - // the start/end of the underlying sequence: - // - if(start != end) - { - unsigned char v = *start; - if((v & 0xC0u) == 0x80u) - invalid_sequence(); - if((b != start) && (b != end) && ((*b & 0xC0u) == 0x80u)) - invalid_sequence(); - BaseIterator pos = end; - do - { - v = *--pos; - } - while((start != pos) && ((v & 0xC0u) == 0x80u)); - std::ptrdiff_t extra = detail::utf8_byte_count(v); - if(std::distance(pos, end) < extra) - invalid_sequence(); - } - } -private: - static void invalid_sequence() - { - std::out_of_range e("Invalid UTF-8 sequence encountered while trying to encode UTF-32 character"); - boost::throw_exception(e); - } - void extract_current()const - { - m_value = static_cast(static_cast< ::boost::uint8_t>(*m_position)); - // we must not have a continuation character: - if((m_value & 0xC0u) == 0x80u) - invalid_sequence(); - // see how many extra bytes we have: - unsigned extra = detail::utf8_trailing_byte_count(*m_position); - // extract the extra bits, 6 from each extra byte: - BaseIterator next(m_position); - for(unsigned c = 0; c < extra; ++c) - { - ++next; - m_value <<= 6; - // We must have a continuation byte: - if((static_cast(*next) & 0xC0) != 0x80) - invalid_sequence(); - m_value += static_cast(*next) & 0x3Fu; - } - // we now need to remove a few of the leftmost bits, but how many depends - // upon how many extra bytes we've extracted: - static const boost::uint32_t masks[4] = - { - 0x7Fu, - 0x7FFu, - 0xFFFFu, - 0x1FFFFFu, - }; - m_value &= masks[extra]; - // check the result: - if(m_value > static_cast(0x10FFFFu)) - invalid_sequence(); - } - BaseIterator m_position; - mutable U32Type m_value; -}; - -template -class utf16_output_iterator -{ -public: - typedef void difference_type; - typedef void value_type; - typedef boost::uint32_t* pointer; - typedef boost::uint32_t& reference; - typedef std::output_iterator_tag iterator_category; - - utf16_output_iterator(const BaseIterator& b) - : m_position(b){} - utf16_output_iterator(const utf16_output_iterator& that) - : m_position(that.m_position){} - utf16_output_iterator& operator=(const utf16_output_iterator& that) - { - m_position = that.m_position; - return *this; - } - const utf16_output_iterator& operator*()const - { - return *this; - } - void operator=(boost::uint32_t val)const - { - push(val); - } - utf16_output_iterator& operator++() - { - return *this; - } - utf16_output_iterator& operator++(int) - { - return *this; - } - BaseIterator base()const - { - return m_position; - } -private: - void push(boost::uint32_t v)const - { - if(v >= 0x10000u) - { - // begin by checking for a code point out of range: - if(v > 0x10FFFFu) - detail::invalid_utf32_code_point(v); - // split into two surrogates: - *m_position++ = static_cast(v >> 10) + detail::high_surrogate_base; - *m_position++ = static_cast(v & detail::ten_bit_mask) + detail::low_surrogate_base; - } - else - { - // 16-bit code point: - // value must not be a surrogate: - if(detail::is_surrogate(v)) - detail::invalid_utf32_code_point(v); - *m_position++ = static_cast(v); - } - } - mutable BaseIterator m_position; -}; - -template -class utf8_output_iterator -{ -public: - typedef void difference_type; - typedef void value_type; - typedef boost::uint32_t* pointer; - typedef boost::uint32_t& reference; - typedef std::output_iterator_tag iterator_category; - - utf8_output_iterator(const BaseIterator& b) - : m_position(b){} - utf8_output_iterator(const utf8_output_iterator& that) - : m_position(that.m_position){} - utf8_output_iterator& operator=(const utf8_output_iterator& that) - { - m_position = that.m_position; - return *this; - } - const utf8_output_iterator& operator*()const - { - return *this; - } - void operator=(boost::uint32_t val)const - { - push(val); - } - utf8_output_iterator& operator++() - { - return *this; - } - utf8_output_iterator& operator++(int) - { - return *this; - } - BaseIterator base()const - { - return m_position; - } -private: - void push(boost::uint32_t c)const - { - if(c > 0x10FFFFu) - detail::invalid_utf32_code_point(c); - if(c < 0x80u) - { - *m_position++ = static_cast(c); - } - else if(c < 0x800u) - { - *m_position++ = static_cast(0xC0u + (c >> 6)); - *m_position++ = static_cast(0x80u + (c & 0x3Fu)); - } - else if(c < 0x10000u) - { - *m_position++ = static_cast(0xE0u + (c >> 12)); - *m_position++ = static_cast(0x80u + ((c >> 6) & 0x3Fu)); - *m_position++ = static_cast(0x80u + (c & 0x3Fu)); - } - else - { - *m_position++ = static_cast(0xF0u + (c >> 18)); - *m_position++ = static_cast(0x80u + ((c >> 12) & 0x3Fu)); - *m_position++ = static_cast(0x80u + ((c >> 6) & 0x3Fu)); - *m_position++ = static_cast(0x80u + (c & 0x3Fu)); - } - } - mutable BaseIterator m_position; -}; - -} // namespace boost - -#endif // BOOST_REGEX_UNICODE_ITERATOR_HPP - diff --git a/include/mmap_for_windows.hpp b/include/mmap_for_windows.hpp deleted file mode 100644 index abe62d64039..00000000000 --- a/include/mmap_for_windows.hpp +++ /dev/null @@ -1,103 +0,0 @@ -#ifndef MMAP_FOR_WINDOWS_HPP -#define MMAP_FOR_WINDOWS_HPP - -/* mmap() replacement for Windows - * - * Author: Mike Frysinger - * Placed into the public domain - */ - -/* References: - * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx - * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx - * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx - * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx - */ - -#include -#include -#include - -#define PROT_READ 0x1 -#define PROT_WRITE 0x2 -/* This flag is only available in WinXP+ */ -#ifdef FILE_MAP_EXECUTE -#define PROT_EXEC 0x4 -#else -#define PROT_EXEC 0x0 -#define FILE_MAP_EXECUTE 0 -#endif - -#define MAP_SHARED 0x01 -#define MAP_PRIVATE 0x02 -#define MAP_ANONYMOUS 0x20 -#define MAP_ANON MAP_ANONYMOUS -#define MAP_FAILED ((void *) -1) - -static DWORD dword_hi(uint64_t x) { - return static_cast(x >> 32); -} - -static DWORD dword_lo(uint64_t x) { - return static_cast(x & 0xffffffff); -} - -static void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset) -{ - if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC)) - return MAP_FAILED; - if (fd == -1) { - if (!(flags & MAP_ANON) || offset) - return MAP_FAILED; - } else if (flags & MAP_ANON) - return MAP_FAILED; - - DWORD flProtect; - if (prot & PROT_WRITE) { - if (prot & PROT_EXEC) - flProtect = PAGE_EXECUTE_READWRITE; - else - flProtect = PAGE_READWRITE; - } else if (prot & PROT_EXEC) { - if (prot & PROT_READ) - flProtect = PAGE_EXECUTE_READ; - else if (prot & PROT_EXEC) - flProtect = PAGE_EXECUTE; - } else - flProtect = PAGE_READONLY; - - uint64_t end = static_cast(length) + offset; - HANDLE mmap_fd; - if (fd == -1) - mmap_fd = INVALID_HANDLE_VALUE; - else - mmap_fd = (HANDLE)_get_osfhandle(fd); - - HANDLE h = CreateFileMapping(mmap_fd, NULL, flProtect, dword_hi(end), dword_lo(end), NULL); - if (h == NULL) - return MAP_FAILED; - - DWORD dwDesiredAccess; - if (prot & PROT_WRITE) - dwDesiredAccess = FILE_MAP_WRITE; - else - dwDesiredAccess = FILE_MAP_READ; - if (prot & PROT_EXEC) - dwDesiredAccess |= FILE_MAP_EXECUTE; - if (flags & MAP_PRIVATE) - dwDesiredAccess |= FILE_MAP_COPY; - void *ret = MapViewOfFile(h, dwDesiredAccess, dword_hi(offset), dword_lo(offset), length); - if (ret == NULL) { - CloseHandle(h); - ret = MAP_FAILED; - } - return ret; -} - -static int munmap(void *addr, size_t length) -{ - return UnmapViewOfFile(addr) ? 0 : -1; - /* ruh-ro, we leaked handle from CreateFileMapping() ... */ -} - -#endif diff --git a/include/osmium/area/assembler.hpp b/include/osmium/area/assembler.hpp index 0a5f12323e1..f1729910913 100644 --- a/include/osmium/area/assembler.hpp +++ b/include/osmium/area/assembler.hpp @@ -171,7 +171,7 @@ namespace osmium { } void add_tags_to_area(osmium::builder::AreaBuilder& builder, const osmium::Relation& relation) const { - auto count = std::count_if(relation.tags().begin(), relation.tags().end(), filter()); + const auto count = std::count_if(relation.tags().begin(), relation.tags().end(), filter()); if (debug()) { std::cerr << " found " << count << " tags on relation (without ignored ones)\n"; @@ -331,7 +331,7 @@ namespace osmium { if (debug()) { std::cerr << " has_closed_subring_back()\n"; } - auto end = ring.segments().end(); + const auto end = ring.segments().end(); for (auto it = ring.segments().begin() + 1; it != end - 1; ++it) { if (has_same_location(nr, it->first())) { split_off_subring(ring, it, it, end); @@ -348,7 +348,7 @@ namespace osmium { if (debug()) { std::cerr << " has_closed_subring_front()\n"; } - auto end = ring.segments().end(); + const auto end = ring.segments().end(); for (auto it = ring.segments().begin() + 1; it != end - 1; ++it) { if (has_same_location(nr, it->second())) { split_off_subring(ring, it, ring.segments().begin(), it+1); @@ -366,22 +366,22 @@ namespace osmium { osmium::area::detail::ProtoRing::segments_type segments(ring.segments().size()); std::copy(ring.segments().begin(), ring.segments().end(), segments.begin()); std::sort(segments.begin(), segments.end()); - auto it = std::adjacent_find(segments.begin(), segments.end(), [this](const osmium::area::detail::NodeRefSegment& s1, const osmium::area::detail::NodeRefSegment& s2) { + const auto it = std::adjacent_find(segments.begin(), segments.end(), [this](const osmium::area::detail::NodeRefSegment& s1, const osmium::area::detail::NodeRefSegment& s2) { return has_same_location(s1.first(), s2.first()); }); if (it == segments.end()) { return false; } - auto r1 = std::find_first_of(ring.segments().begin(), ring.segments().end(), it, it+1); + const auto r1 = std::find_first_of(ring.segments().begin(), ring.segments().end(), it, it+1); assert(r1 != ring.segments().end()); - auto r2 = std::find_first_of(ring.segments().begin(), ring.segments().end(), it+1, it+2); + const auto r2 = std::find_first_of(ring.segments().begin(), ring.segments().end(), it+1, it+2); assert(r2 != ring.segments().end()); if (debug()) { std::cerr << " found subring in ring " << ring << " at " << it->first() << "\n"; } - auto m = std::minmax(r1, r2); + const auto m = std::minmax(r1, r2); ProtoRing new_ring(m.first, m.second); ring.remove_segments(m.first, m.second); @@ -537,7 +537,7 @@ namespace osmium { } for (const auto ringptr : m_outer_rings) { - for (const auto segment : ringptr->segments()) { + for (const auto& segment : ringptr->segments()) { if (!segment.role_outer()) { ++m_inner_outer_mismatches; if (debug()) { @@ -550,7 +550,7 @@ namespace osmium { } } for (const auto ringptr : m_inner_rings) { - for (const auto segment : ringptr->segments()) { + for (const auto& segment : ringptr->segments()) { if (!segment.role_inner()) { ++m_inner_outer_mismatches; if (debug()) { diff --git a/include/osmium/area/multipolygon_collector.hpp b/include/osmium/area/multipolygon_collector.hpp index 84a52622b42..bf2a4ce86a1 100644 --- a/include/osmium/area/multipolygon_collector.hpp +++ b/include/osmium/area/multipolygon_collector.hpp @@ -41,6 +41,8 @@ DEALINGS IN THE SOFTWARE. #include #include +#include +#include #include #include #include @@ -49,8 +51,6 @@ DEALINGS IN THE SOFTWARE. namespace osmium { - struct invalid_location; - namespace relations { class RelationMeta; } @@ -107,8 +107,8 @@ namespace osmium { } /** - * We are interested in all relations tagged with type=multipolygon or - * type=boundary. + * We are interested in all relations tagged with type=multipolygon + * or type=boundary. * * Overwritten from the base class. */ @@ -142,15 +142,22 @@ namespace osmium { * Overwritten from the base class. */ void way_not_in_any_relation(const osmium::Way& way) { - if (way.nodes().size() > 3 && way.ends_have_same_location()) { - // way is closed and has enough nodes, build simple multipolygon - try { + // you need at least 4 nodes to make up a polygon + if (way.nodes().size() <= 3) { + return; + } + try { + if (!way.nodes().front().location() || !way.nodes().back().location()) { + throw osmium::invalid_location("invalid location"); + } + if (way.ends_have_same_location()) { + // way is closed and has enough nodes, build simple multipolygon TAssembler assembler(m_assembler_config); assembler(way, m_output_buffer); possibly_flush_output_buffer(); - } catch (osmium::invalid_location&) { - // XXX ignore } + } catch (osmium::invalid_location&) { + // XXX ignore } } diff --git a/include/osmium/builder/builder.hpp b/include/osmium/builder/builder.hpp index dcb95e2ac3c..4424d88e197 100644 --- a/include/osmium/builder/builder.hpp +++ b/include/osmium/builder/builder.hpp @@ -147,6 +147,7 @@ namespace osmium { * @param length Length of data in bytes. If data is a * \0-terminated string, length must contain the * \0 byte. + * @returns The number of bytes appended (length). */ osmium::memory::item_size_type append(const char* data, const osmium::memory::item_size_type length) { unsigned char* target = m_buffer.reserve_space(length); @@ -156,11 +157,24 @@ namespace osmium { /** * Append \0-terminated string to buffer. + * + * @param str \0-terminated string. + * @returns The number of bytes appended (strlen(str) + 1). */ osmium::memory::item_size_type append(const char* str) { return append(str, static_cast(std::strlen(str) + 1)); } + /** + * Append '\0' to the buffer. + * + * @returns The number of bytes appended (always 1). + */ + osmium::memory::item_size_type append_zero() { + *m_buffer.reserve_space(1) = '\0'; + return 1; + } + /// Return the buffer this builder is using. osmium::memory::Buffer& buffer() noexcept { return m_buffer; @@ -188,11 +202,11 @@ namespace osmium { * Add user name to buffer. * * @param user Pointer to user name. - * @param length Length of user name including \0 byte. + * @param length Length of user name (without \0 termination). */ void add_user(const char* user, const string_size_type length) { - object().set_user_size(length); - add_size(append(user, length)); + object().set_user_size(length + 1); + add_size(append(user, length) + append_zero()); add_padding(true); } @@ -202,7 +216,7 @@ namespace osmium { * @param user Pointer to \0-terminated user name. */ void add_user(const char* user) { - add_user(user, static_cast_with_assert(std::strlen(user) + 1)); + add_user(user, static_cast_with_assert(std::strlen(user))); } /** @@ -211,7 +225,7 @@ namespace osmium { * @param user User name. */ void add_user(const std::string& user) { - add_user(user.data(), static_cast_with_assert(user.size() + 1)); + add_user(user.data(), static_cast_with_assert(user.size())); } }; // class ObjectBuilder diff --git a/include/osmium/builder/osm_object_builder.hpp b/include/osmium/builder/osm_object_builder.hpp index 058f89e8ebc..074076ce6f5 100644 --- a/include/osmium/builder/osm_object_builder.hpp +++ b/include/osmium/builder/osm_object_builder.hpp @@ -72,13 +72,25 @@ namespace osmium { /** * Add tag to buffer. * - * @param key Tag key. - * @param value Tag value. + * @param key Tag key (0-terminated string). + * @param value Tag value (0-terminated string). */ void add_tag(const char* key, const char* value) { add_size(append(key) + append(value)); } + /** + * Add tag to buffer. + * + * @param key Pointer to tag key. + * @param key_length Length of key (not including the \0 byte). + * @param value Pointer to tag value. + * @param value_length Length of value (not including the \0 byte). + */ + void add_tag(const char* key, const string_size_type key_length, const char* value, const string_size_type value_length) { + add_size(append(key, key_length) + append_zero() + append(value, value_length) + append_zero()); + } + /** * Add tag to buffer. * @@ -128,11 +140,11 @@ namespace osmium { * @param member Relation member object where the length of the role * will be set. * @param role The role. - * @param length Length of role string including \0 termination. + * @param length Length of role (without \0 termination). */ void add_role(osmium::RelationMember& member, const char* role, const string_size_type length) { - member.set_role_size(length); - add_size(append(role, length)); + member.set_role_size(length + 1); + add_size(append(role, length) + append_zero()); add_padding(true); } @@ -144,7 +156,7 @@ namespace osmium { * @param role \0-terminated role. */ void add_role(osmium::RelationMember& member, const char* role) { - add_role(member, role, static_cast_with_assert(std::strlen(role) + 1)); + add_role(member, role, static_cast_with_assert(std::strlen(role))); } /** @@ -155,7 +167,7 @@ namespace osmium { * @param role Role. */ void add_role(osmium::RelationMember& member, const std::string& role) { - add_role(member, role.data(), static_cast_with_assert(role.size() + 1)); + add_role(member, role.data(), static_cast_with_assert(role.size())); } public: @@ -174,20 +186,35 @@ namespace osmium { * @param type The type (node, way, or relation). * @param ref The ID of the member. * @param role The role of the member. + * @param role_length Length of the role (without \0 termination). * @param full_member Optional pointer to the member object. If it * is available a copy will be added to the * relation. */ - void add_member(osmium::item_type type, object_id_type ref, const char* role, const osmium::OSMObject* full_member = nullptr) { + void add_member(osmium::item_type type, object_id_type ref, const char* role, const string_size_type role_length, const osmium::OSMObject* full_member = nullptr) { osmium::RelationMember* member = reserve_space_for(); new (member) osmium::RelationMember(ref, type, full_member != nullptr); add_size(sizeof(RelationMember)); - add_role(*member, role); + add_role(*member, role, role_length); if (full_member) { add_item(full_member); } } + /** + * Add a member to the relation. + * + * @param type The type (node, way, or relation). + * @param ref The ID of the member. + * @param role The role of the member (\0 terminated string). + * @param full_member Optional pointer to the member object. If it + * is available a copy will be added to the + * relation. + */ + void add_member(osmium::item_type type, object_id_type ref, const char* role, const osmium::OSMObject* full_member = nullptr) { + add_member(type, ref, role, strlen(role), full_member); + } + /** * Add a member to the relation. * @@ -199,13 +226,7 @@ namespace osmium { * relation. */ void add_member(osmium::item_type type, object_id_type ref, const std::string& role, const osmium::OSMObject* full_member = nullptr) { - osmium::RelationMember* member = reserve_space_for(); - new (member) osmium::RelationMember(ref, type, full_member != nullptr); - add_size(sizeof(RelationMember)); - add_role(*member, role); - if (full_member) { - add_item(full_member); - } + add_member(type, ref, role.data(), role.size(), full_member); } }; // class RelationMemberListBuilder diff --git a/include/osmium/geom/factory.hpp b/include/osmium/geom/factory.hpp index 518cbfbc265..9be050d2563 100644 --- a/include/osmium/geom/factory.hpp +++ b/include/osmium/geom/factory.hpp @@ -54,14 +54,43 @@ namespace osmium { * Exception thrown when an invalid geometry is encountered. An example * would be a linestring with less than two points. */ - struct geometry_error : public std::runtime_error { + class geometry_error : public std::runtime_error { + + std::string m_message; + osmium::object_id_type m_id; + + public: + + geometry_error(const std::string& message, const char* object_type = "", osmium::object_id_type id = 0) : + std::runtime_error(message), + m_message(message), + m_id(id) { + if (m_id != 0) { + m_message += " ("; + m_message += object_type; + m_message += "_id="; + m_message += std::to_string(m_id); + m_message += ")"; + } + } + + void set_id(const char* object_type, osmium::object_id_type id) { + if (m_id == 0 && id != 0) { + m_message += " ("; + m_message += object_type; + m_message += "_id="; + m_message += std::to_string(id); + m_message += ")"; + } + m_id = id; + } - geometry_error(const std::string& what) : - std::runtime_error(what) { + osmium::object_id_type id() const noexcept { + return m_id; } - geometry_error(const char* what) : - std::runtime_error(what) { + virtual const char* what() const noexcept override { + return m_message.c_str(); } }; // struct geometry_error @@ -174,11 +203,21 @@ namespace osmium { } point_type create_point(const osmium::Node& node) { - return create_point(node.location()); + try { + return create_point(node.location()); + } catch (osmium::geometry_error& e) { + e.set_id("node", node.id()); + throw; + } } point_type create_point(const osmium::NodeRef& node_ref) { - return create_point(node_ref.location()); + try { + return create_point(node_ref.location()); + } catch (osmium::geometry_error& e) { + e.set_id("node", node_ref.ref()); + throw; + } } /* LineString */ @@ -240,14 +279,19 @@ namespace osmium { } if (num_points < 2) { - throw osmium::geometry_error("not enough points for linestring"); + throw osmium::geometry_error("need at least two points for linestring"); } return linestring_finish(num_points); } linestring_type create_linestring(const osmium::Way& way, use_nodes un=use_nodes::unique, direction dir=direction::forward) { - return create_linestring(way.nodes(), un, dir); + try { + return create_linestring(way.nodes(), un, dir); + } catch (osmium::geometry_error& e) { + e.set_id("way", way.id()); + throw; + } } /* Polygon */ @@ -283,40 +327,86 @@ namespace osmium { return m_impl.polygon_finish(num_points); } + polygon_type create_polygon(const osmium::WayNodeList& wnl, use_nodes un = use_nodes::unique, direction dir = direction::forward) { + polygon_start(); + size_t num_points = 0; + + if (un == use_nodes::unique) { + osmium::Location last_location; + switch (dir) { + case direction::forward: + num_points = fill_polygon_unique(wnl.cbegin(), wnl.cend()); + break; + case direction::backward: + num_points = fill_polygon_unique(wnl.crbegin(), wnl.crend()); + break; + } + } else { + switch (dir) { + case direction::forward: + num_points = fill_polygon(wnl.cbegin(), wnl.cend()); + break; + case direction::backward: + num_points = fill_polygon(wnl.crbegin(), wnl.crend()); + break; + } + } + + if (num_points < 4) { + throw osmium::geometry_error("need at least four points for polygon"); + } + + return polygon_finish(num_points); + } + + polygon_type create_polygon(const osmium::Way& way, use_nodes un=use_nodes::unique, direction dir=direction::forward) { + try { + return create_polygon(way.nodes(), un, dir); + } catch (osmium::geometry_error& e) { + e.set_id("way", way.id()); + throw; + } + } + /* MultiPolygon */ multipolygon_type create_multipolygon(const osmium::Area& area) { - size_t num_polygons = 0; - size_t num_rings = 0; - m_impl.multipolygon_start(); - - for (auto it = area.cbegin(); it != area.cend(); ++it) { - const osmium::OuterRing& ring = static_cast(*it); - if (it->type() == osmium::item_type::outer_ring) { - if (num_polygons > 0) { - m_impl.multipolygon_polygon_finish(); + try { + size_t num_polygons = 0; + size_t num_rings = 0; + m_impl.multipolygon_start(); + + for (auto it = area.cbegin(); it != area.cend(); ++it) { + const osmium::OuterRing& ring = static_cast(*it); + if (it->type() == osmium::item_type::outer_ring) { + if (num_polygons > 0) { + m_impl.multipolygon_polygon_finish(); + } + m_impl.multipolygon_polygon_start(); + m_impl.multipolygon_outer_ring_start(); + add_points(ring); + m_impl.multipolygon_outer_ring_finish(); + ++num_rings; + ++num_polygons; + } else if (it->type() == osmium::item_type::inner_ring) { + m_impl.multipolygon_inner_ring_start(); + add_points(ring); + m_impl.multipolygon_inner_ring_finish(); + ++num_rings; } - m_impl.multipolygon_polygon_start(); - m_impl.multipolygon_outer_ring_start(); - add_points(ring); - m_impl.multipolygon_outer_ring_finish(); - ++num_rings; - ++num_polygons; - } else if (it->type() == osmium::item_type::inner_ring) { - m_impl.multipolygon_inner_ring_start(); - add_points(ring); - m_impl.multipolygon_inner_ring_finish(); - ++num_rings; } - } - // if there are no rings, this area is invalid - if (num_rings == 0) { - throw osmium::geometry_error("invalid area"); - } + // if there are no rings, this area is invalid + if (num_rings == 0) { + throw osmium::geometry_error("area contains no rings"); + } - m_impl.multipolygon_polygon_finish(); - return m_impl.multipolygon_finish(); + m_impl.multipolygon_polygon_finish(); + return m_impl.multipolygon_finish(); + } catch (osmium::geometry_error& e) { + e.set_id("area", area.id()); + throw; + } } }; // class GeometryFactory diff --git a/include/osmium/geom/geos.hpp b/include/osmium/geom/geos.hpp index 3c73637be05..771b08736fc 100644 --- a/include/osmium/geom/geos.hpp +++ b/include/osmium/geom/geos.hpp @@ -42,6 +42,8 @@ DEALINGS IN THE SOFTWARE. * @attention If you include this file, you'll need to link with `libgeos`. */ +#include +#include #include #include @@ -69,8 +71,8 @@ namespace osmium { struct geos_geometry_error : public geometry_error { - geos_geometry_error() : - geometry_error("geometry creation failed in GEOS library, see nested exception for details") { + geos_geometry_error(const char* message) : + geometry_error(std::string("geometry creation failed in GEOS library: ") + message) { } }; // struct geos_geometry_error @@ -81,8 +83,9 @@ namespace osmium { class GEOSFactoryImpl { - geos::geom::PrecisionModel m_precision_model; - geos::geom::GeometryFactory m_geos_factory; + std::unique_ptr m_precision_model; + std::unique_ptr m_our_geos_factory; + geos::geom::GeometryFactory* m_geos_factory; std::unique_ptr m_coordinate_sequence; std::vector> m_rings; @@ -96,18 +99,25 @@ namespace osmium { typedef std::unique_ptr multipolygon_type; typedef std::unique_ptr ring_type; + explicit GEOSFactoryImpl(geos::geom::GeometryFactory& geos_factory) : + m_precision_model(nullptr), + m_our_geos_factory(nullptr), + m_geos_factory(&geos_factory) { + } + explicit GEOSFactoryImpl(int srid = -1) : - m_precision_model(), - m_geos_factory(&m_precision_model, srid) { + m_precision_model(new geos::geom::PrecisionModel), + m_our_geos_factory(new geos::geom::GeometryFactory(m_precision_model.get(), srid)), + m_geos_factory(m_our_geos_factory.get()) { } /* Point */ point_type make_point(const osmium::geom::Coordinates& xy) const { try { - return point_type(m_geos_factory.createPoint(geos::geom::Coordinate(xy.x, xy.y))); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + return point_type(m_geos_factory->createPoint(geos::geom::Coordinate(xy.x, xy.y))); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } @@ -115,25 +125,25 @@ namespace osmium { void linestring_start() { try { - m_coordinate_sequence.reset(m_geos_factory.getCoordinateSequenceFactory()->create(static_cast(0), 2)); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + m_coordinate_sequence.reset(m_geos_factory->getCoordinateSequenceFactory()->create(static_cast(0), 2)); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } void linestring_add_location(const osmium::geom::Coordinates& xy) { try { m_coordinate_sequence->add(geos::geom::Coordinate(xy.x, xy.y)); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } linestring_type linestring_finish(size_t /* num_points */) { try { - return linestring_type(m_geos_factory.createLineString(m_coordinate_sequence.release())); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + return linestring_type(m_geos_factory->createLineString(m_coordinate_sequence.release())); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } @@ -154,50 +164,50 @@ namespace osmium { std::transform(std::next(m_rings.begin(), 1), m_rings.end(), std::back_inserter(*inner_rings), [](std::unique_ptr& r) { return r.release(); }); - m_polygons.emplace_back(m_geos_factory.createPolygon(m_rings[0].release(), inner_rings)); + m_polygons.emplace_back(m_geos_factory->createPolygon(m_rings[0].release(), inner_rings)); m_rings.clear(); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } void multipolygon_outer_ring_start() { try { - m_coordinate_sequence.reset(m_geos_factory.getCoordinateSequenceFactory()->create(static_cast(0), 2)); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + m_coordinate_sequence.reset(m_geos_factory->getCoordinateSequenceFactory()->create(static_cast(0), 2)); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } void multipolygon_outer_ring_finish() { try { - m_rings.emplace_back(m_geos_factory.createLinearRing(m_coordinate_sequence.release())); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + m_rings.emplace_back(m_geos_factory->createLinearRing(m_coordinate_sequence.release())); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } void multipolygon_inner_ring_start() { try { - m_coordinate_sequence.reset(m_geos_factory.getCoordinateSequenceFactory()->create(static_cast(0), 2)); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + m_coordinate_sequence.reset(m_geos_factory->getCoordinateSequenceFactory()->create(static_cast(0), 2)); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } void multipolygon_inner_ring_finish() { try { - m_rings.emplace_back(m_geos_factory.createLinearRing(m_coordinate_sequence.release())); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + m_rings.emplace_back(m_geos_factory->createLinearRing(m_coordinate_sequence.release())); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } void multipolygon_add_location(const osmium::geom::Coordinates& xy) { try { m_coordinate_sequence->add(geos::geom::Coordinate(xy.x, xy.y)); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } @@ -208,9 +218,9 @@ namespace osmium { return p.release(); }); m_polygons.clear(); - return multipolygon_type(m_geos_factory.createMultiPolygon(polygons)); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + return multipolygon_type(m_geos_factory->createMultiPolygon(polygons)); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } diff --git a/include/osmium/geom/mercator_projection.hpp b/include/osmium/geom/mercator_projection.hpp index a6d1d5742c9..22a0d646b00 100644 --- a/include/osmium/geom/mercator_projection.hpp +++ b/include/osmium/geom/mercator_projection.hpp @@ -47,6 +47,7 @@ namespace osmium { namespace detail { constexpr double earth_radius_for_epsg3857 = 6378137.0; + constexpr double max_coordinate_epsg3857 = 20037508.34; constexpr inline double lon_to_x(double lon) { return earth_radius_for_epsg3857 * deg_to_rad(lon); diff --git a/include/osmium/geom/rapid_geojson.hpp b/include/osmium/geom/rapid_geojson.hpp new file mode 100644 index 00000000000..a3d46870c67 --- /dev/null +++ b/include/osmium/geom/rapid_geojson.hpp @@ -0,0 +1,190 @@ +#ifndef OSMIUM_GEOM_RAPID_GEOJSON_HPP +#define OSMIUM_GEOM_RAPID_GEOJSON_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include + +namespace osmium { + + namespace geom { + + namespace detail { + + /** + * A geometry factory implementation that can be used with the + * RapidJSON (https://github.com/miloyip/rapidjson) JSON writer. + */ + template + class RapidGeoJSONFactoryImpl { + + TWriter* m_writer; + + public: + + typedef void point_type; + typedef void linestring_type; + typedef void polygon_type; + typedef void multipolygon_type; + typedef void ring_type; + + RapidGeoJSONFactoryImpl(TWriter& writer) : + m_writer(&writer) { + } + + /* Point */ + + // { "type": "Point", "coordinates": [100.0, 0.0] } + point_type make_point(const osmium::geom::Coordinates& xy) const { + m_writer->String("geometry"); + m_writer->StartObject(); + m_writer->String("type"); + m_writer->String("Point"); + m_writer->String("coordinates"); + m_writer->StartArray(); + m_writer->Double(xy.x); + m_writer->Double(xy.y); + m_writer->EndArray(); + m_writer->EndObject(); + } + + /* LineString */ + + // { "type": "LineString", "coordinates": [ [100.0, 0.0], [101.0, 1.0] ] } + void linestring_start() { + m_writer->String("geometry"); + m_writer->StartObject(); + m_writer->String("type"); + m_writer->String("LineString"); + m_writer->String("coordinates"); + m_writer->StartArray(); + } + + void linestring_add_location(const osmium::geom::Coordinates& xy) { + m_writer->StartArray(); + m_writer->Double(xy.x); + m_writer->Double(xy.y); + m_writer->EndArray(); + } + + linestring_type linestring_finish(size_t /* num_points */) { + m_writer->EndArray(); + m_writer->EndObject(); + } + + /* Polygon */ + + // { "type": "Polygon", "coordinates": [[[100.0, 0.0], [101.0, 1.0]]] } + void polygon_start() { + m_writer->String("geometry"); + m_writer->StartObject(); + m_writer->String("type"); + m_writer->String("Polygon"); + m_writer->String("coordinates"); + m_writer->StartArray(); + m_writer->StartArray(); + } + + void polygon_add_location(const osmium::geom::Coordinates& xy) { + m_writer->StartArray(); + m_writer->Double(xy.x); + m_writer->Double(xy.y); + m_writer->EndArray(); + } + + polygon_type polygon_finish(size_t /* num_points */) { + m_writer->EndArray(); + m_writer->EndArray(); + m_writer->EndObject(); + } + + /* MultiPolygon */ + + void multipolygon_start() { + m_writer->String("geometry"); + m_writer->StartObject(); + m_writer->String("type"); + m_writer->String("MultiPolygon"); + m_writer->String("coordinates"); + m_writer->StartArray(); + } + + void multipolygon_polygon_start() { + m_writer->StartArray(); + } + + void multipolygon_polygon_finish() { + m_writer->EndArray(); + } + + void multipolygon_outer_ring_start() { + m_writer->StartArray(); + } + + void multipolygon_outer_ring_finish() { + m_writer->EndArray(); + } + + void multipolygon_inner_ring_start() { + m_writer->StartArray(); + } + + void multipolygon_inner_ring_finish() { + m_writer->EndArray(); + } + + void multipolygon_add_location(const osmium::geom::Coordinates& xy) { + m_writer->StartArray(); + m_writer->Double(xy.x); + m_writer->Double(xy.y); + m_writer->EndArray(); + } + + multipolygon_type multipolygon_finish() { + m_writer->EndArray(); + m_writer->EndObject(); + } + + }; // class RapidGeoJSONFactoryImpl + + } // namespace detail + + template + using RapidGeoJSONFactory = GeometryFactory, TProjection>; + + } // namespace geom + +} // namespace osmium + +#endif // OSMIUM_GEOM_RAPID_GEOJSON_HPP diff --git a/include/osmium/geom/tile.hpp b/include/osmium/geom/tile.hpp new file mode 100644 index 00000000000..9cd0b282b7b --- /dev/null +++ b/include/osmium/geom/tile.hpp @@ -0,0 +1,101 @@ +#ifndef OSMIUM_GEOM_TILE_HPP +#define OSMIUM_GEOM_TILE_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include + +#include + +namespace osmium { + + namespace geom { + + namespace detail { + + template + inline T restrict_to_range(T value, T min, T max) { + if (value < min) return min; + if (value > max) return max; + return value; + } + + } // namespace detail + + /** + * A tile in the usual Mercator projection. + */ + struct Tile { + + uint32_t x; + uint32_t y; + uint32_t z; + + explicit Tile(uint32_t zoom, uint32_t tx, uint32_t ty) noexcept : x(tx), y(ty), z(zoom) { + } + + explicit Tile(uint32_t zoom, const osmium::Location& location) : + z(zoom) { + osmium::geom::Coordinates c = lonlat_to_mercator(location); + const int32_t n = 1LL << zoom; + const double scale = detail::max_coordinate_epsg3857 * 2 / n; + x = detail::restrict_to_range((c.x + detail::max_coordinate_epsg3857) / scale, 0, n-1); + y = detail::restrict_to_range((detail::max_coordinate_epsg3857 - c.y) / scale, 0, n-1); + } + + }; // struct Tile + + inline bool operator==(const Tile& a, const Tile& b) { + return a.z == b.z && a.x == b.x && a.y == b.y; + } + + inline bool operator!=(const Tile& a, const Tile& b) { + return ! (a == b); + } + + /** + * This defines an arbitrary order on tiles for use in std::map etc. + */ + inline bool operator<(const Tile& a, const Tile& b) { + if (a.z < b.z) return true; + if (a.z > b.z) return false; + if (a.x < b.x) return true; + if (a.x > b.x) return false; + return a.y < b.y; + } + + } // namespace geom + +} // namespace osmium + +#endif // OSMIUM_GEOM_TILE_HPP diff --git a/include/osmium/geom/wkb.hpp b/include/osmium/geom/wkb.hpp index 2f32fe33bea..a290c02d06d 100644 --- a/include/osmium/geom/wkb.hpp +++ b/include/osmium/geom/wkb.hpp @@ -37,18 +37,10 @@ DEALINGS IN THE SOFTWARE. #include #include -// Windows is only available for little endian architectures -// http://stackoverflow.com/questions/6449468/can-i-safely-assume-that-windows-installations-will-always-be-little-endian -#if !defined(_WIN32) && !defined(__APPLE__) -# include -#else -# define __LITTLE_ENDIAN 1234 -# define __BYTE_ORDER __LITTLE_ENDIAN -#endif - #include #include #include +#include namespace osmium { diff --git a/include/osmium/index/bool_vector.hpp b/include/osmium/index/bool_vector.hpp new file mode 100644 index 00000000000..94e1f726849 --- /dev/null +++ b/include/osmium/index/bool_vector.hpp @@ -0,0 +1,83 @@ +#ifndef OSMIUM_INDEX_BOOL_VECTOR_HPP +#define OSMIUM_INDEX_BOOL_VECTOR_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include + +namespace osmium { + + namespace index { + + /** + * Index storing one bit for each Id. The index automatically scales + * with the Ids stored. Default value is 'false'. Storage uses + * std::vector and needs a minimum of memory if the Ids are + * dense. + */ + template + class BoolVector { + + static_assert(std::is_unsigned::value, "Needs unsigned type"); + + std::vector m_bits; + + public: + + BoolVector() = default; + BoolVector(const BoolVector&) = default; + BoolVector(BoolVector&&) = default; + BoolVector& operator=(const BoolVector&) = default; + BoolVector& operator=(BoolVector&&) = default; + ~BoolVector() = default; + + void set(T id, bool value = true) { + if (m_bits.size() <= id) { + m_bits.resize(id + 1024 * 1024); + } + + m_bits[id] = value; + } + + bool get(T id) const { + return id < m_bits.size() && m_bits[id]; + } + + }; // class BoolVector + + } // namespace index + +} // namespace osmium + +#endif // OSMIUM_INDEX_BOOL_VECTOR_HPP diff --git a/include/osmium/index/detail/create_map_with_fd.hpp b/include/osmium/index/detail/create_map_with_fd.hpp index 29dc1dc675a..5ccbfc80922 100644 --- a/include/osmium/index/detail/create_map_with_fd.hpp +++ b/include/osmium/index/detail/create_map_with_fd.hpp @@ -39,8 +39,6 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include -#include #include namespace osmium { diff --git a/include/osmium/index/detail/mmap_vector_anon.hpp b/include/osmium/index/detail/mmap_vector_anon.hpp index 0ea4f9db657..fc016261e7c 100644 --- a/include/osmium/index/detail/mmap_vector_anon.hpp +++ b/include/osmium/index/detail/mmap_vector_anon.hpp @@ -35,9 +35,6 @@ DEALINGS IN THE SOFTWARE. #ifdef __linux__ -#include - -#include #include namespace osmium { @@ -45,26 +42,16 @@ namespace osmium { namespace detail { /** - * This class looks and behaves like STL vector, but uses mmap internally. + * This class looks and behaves like STL vector, but uses mmap + * internally. */ template - class mmap_vector_anon : public mmap_vector_base { + class mmap_vector_anon : public mmap_vector_base { public: mmap_vector_anon() : - mmap_vector_base( - -1, - osmium::detail::mmap_vector_size_increment, - 0, - osmium::detail::typed_mmap::map(osmium::detail::mmap_vector_size_increment)) { - } - - void reserve(size_t new_capacity) { - if (new_capacity > this->capacity()) { - this->data(osmium::detail::typed_mmap::remap(this->data(), this->capacity(), new_capacity)); - this->m_capacity = new_capacity; - } + mmap_vector_base() { } }; // class mmap_vector_anon diff --git a/include/osmium/index/detail/mmap_vector_base.hpp b/include/osmium/index/detail/mmap_vector_base.hpp index 3aff26d4103..9b647681203 100644 --- a/include/osmium/index/detail/mmap_vector_base.hpp +++ b/include/osmium/index/detail/mmap_vector_base.hpp @@ -34,11 +34,10 @@ DEALINGS IN THE SOFTWARE. */ #include -#include +#include // IWYU pragma: keep #include -#include -#include +#include namespace osmium { @@ -48,40 +47,29 @@ namespace osmium { /** * This is a base class for implementing classes that look like - * STL vector but use mmap internally. This class can not be used - * on it's own. Use the derived classes mmap_vector_anon or - * mmap_vector_file. + * STL vector but use mmap internally. Do not use this class itself, + * use the derived classes mmap_vector_anon or mmap_vector_file. */ - template class TDerived> + template class mmap_vector_base { protected: - int m_fd; - size_t m_capacity; size_t m_size; - T* m_data; + osmium::util::TypedMemoryMapping m_mapping; - explicit mmap_vector_base(int fd, size_t capacity, size_t size, T* data) noexcept : - m_fd(fd), - m_capacity(capacity), - m_size(size), - m_data(data) { - } + public: - explicit mmap_vector_base(int fd, size_t capacity, size_t size) : - m_fd(fd), - m_capacity(capacity), + explicit mmap_vector_base(int fd, size_t capacity, size_t size = 0) : m_size(size), - m_data(osmium::detail::typed_mmap::grow_and_map(capacity, m_fd)) { + m_mapping(capacity, osmium::util::MemoryMapping::mapping_mode::write_shared, fd) { } - void data(T* data) { - m_data = data; + explicit mmap_vector_base(size_t capacity = mmap_vector_size_increment) : + m_size(0), + m_mapping(capacity) { } - public: - typedef T value_type; typedef T& reference; typedef const T& const_reference; @@ -90,12 +78,14 @@ namespace osmium { typedef T* iterator; typedef const T* const_iterator; - ~mmap_vector_base() { - osmium::detail::typed_mmap::unmap(m_data, m_capacity); + ~mmap_vector_base() = default; + + void close() { + m_mapping.unmap(); } size_t capacity() const noexcept { - return m_capacity; + return m_mapping.size(); } size_t size() const noexcept { @@ -106,23 +96,23 @@ namespace osmium { return m_size == 0; } - const T* data() const noexcept { - return m_data; + const T* data() const { + return m_mapping.begin(); } - T* data() noexcept { - return m_data; + T* data() { + return m_mapping.begin(); } T& operator[](size_t n) { - return m_data[n]; + return data()[n]; } T at(size_t n) const { if (n >= m_size) { throw std::out_of_range("out of range"); } - return m_data[n]; + return data()[n]; } void clear() noexcept { @@ -134,16 +124,22 @@ namespace osmium { } void push_back(const T& value) { - if (m_size >= m_capacity) { + if (m_size >= capacity()) { resize(m_size+1); } - m_data[m_size] = value; + data()[m_size] = value; ++m_size; } + void reserve(size_t new_capacity) { + if (new_capacity > capacity()) { + m_mapping.resize(new_capacity); + } + } + void resize(size_t new_size) { if (new_size > capacity()) { - static_cast*>(this)->reserve(new_size + osmium::detail::mmap_vector_size_increment); + reserve(new_size + osmium::detail::mmap_vector_size_increment); } if (new_size > size()) { new (data() + size()) T[new_size - size()]; @@ -152,27 +148,27 @@ namespace osmium { } iterator begin() noexcept { - return m_data; + return data(); } iterator end() noexcept { - return m_data + m_size; + return data() + m_size; } const_iterator begin() const noexcept { - return m_data; + return data(); } const_iterator end() const noexcept { - return m_data + m_size; + return data() + m_size; } - const_iterator cbegin() noexcept { - return m_data; + const_iterator cbegin() const noexcept { + return data(); } - const_iterator cend() noexcept { - return m_data + m_size; + const_iterator cend() const noexcept { + return data() + m_size; } }; // class mmap_vector_base diff --git a/include/osmium/index/detail/mmap_vector_file.hpp b/include/osmium/index/detail/mmap_vector_file.hpp index 55077d18d45..1dadbcb45b3 100644 --- a/include/osmium/index/detail/mmap_vector_file.hpp +++ b/include/osmium/index/detail/mmap_vector_file.hpp @@ -33,11 +33,9 @@ DEALINGS IN THE SOFTWARE. */ -#include - -#include #include #include +#include namespace osmium { @@ -48,32 +46,19 @@ namespace osmium { * internally. */ template - class mmap_vector_file : public mmap_vector_base { + class mmap_vector_file : public mmap_vector_base { public: - explicit mmap_vector_file() : - mmap_vector_base( + explicit mmap_vector_file() : mmap_vector_base( osmium::detail::create_tmp_file(), - osmium::detail::mmap_vector_size_increment, - 0) { + osmium::detail::mmap_vector_size_increment) { } - explicit mmap_vector_file(int fd) : - mmap_vector_base( + explicit mmap_vector_file(int fd) : mmap_vector_base( fd, - osmium::detail::typed_mmap::file_size(fd) == 0 ? - osmium::detail::mmap_vector_size_increment : - osmium::detail::typed_mmap::file_size(fd), - osmium::detail::typed_mmap::file_size(fd)) { - } - - void reserve(size_t new_capacity) { - if (new_capacity > this->capacity()) { - typed_mmap::unmap(this->data(), this->capacity()); - this->data(typed_mmap::grow_and_map(new_capacity, this->m_fd)); - this->m_capacity = new_capacity; - } + osmium::util::file_size(fd) / sizeof(T), + osmium::util::file_size(fd) / sizeof(T)) { } }; // class mmap_vector_file diff --git a/include/osmium/index/detail/typed_mmap.hpp b/include/osmium/index/detail/typed_mmap.hpp deleted file mode 100644 index 77b065e8e95..00000000000 --- a/include/osmium/index/detail/typed_mmap.hpp +++ /dev/null @@ -1,229 +0,0 @@ -#ifndef OSMIUM_INDEX_DETAIL_TYPED_MMAP_HPP -#define OSMIUM_INDEX_DETAIL_TYPED_MMAP_HPP - -/* - -This file is part of Osmium (http://osmcode.org/libosmium). - -Copyright 2013-2015 Jochen Topf and others (see README). - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -*/ - -#include -#include -#include -#include - -#include - -#ifndef _WIN32 -# include -#else -# include -#endif - -#ifndef _MSC_VER -# include -#else -# define ftruncate _chsize -#endif - -// for bsd systems -#ifndef MAP_ANONYMOUS -# define MAP_ANONYMOUS MAP_ANON -#endif - -#include - -namespace osmium { - - /** - * @brief Namespace for Osmium internal use - */ - namespace detail { - - /** - * This is a helper class for working with memory mapped files and - * anonymous shared memory. It wraps the necessary system calls - * adding: - * - error checking: all functions throw exceptions where needed - * - internal casts and size calculations allow use with user defined - * type T instead of void* - * - * This class only contains static functions. It should never be - * instantiated. - * - * @tparam T Type of objects we want to store. - */ - template - class typed_mmap { - - public: - - /** - * Create anonymous private memory mapping with enough space for size - * objects of type T. - * - * Note that no constructor is called for any of the objects in this memory! - * - * @param size Number of objects of type T that should fit into this memory - * @returns Pointer to mapped memory - * @throws std::system_error If mmap(2) failed - */ - static T* map(size_t size) { - void* addr = ::mmap(nullptr, sizeof(T) * size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wold-style-cast" - if (addr == MAP_FAILED) { - throw std::system_error(errno, std::system_category(), "mmap failed"); - } -#pragma GCC diagnostic pop - return reinterpret_cast(addr); - } - - /** - * Create shared memory mapping of a file with enough space for size - * objects of type T. The file must already have at least the - * required size. - * - * Note that no constructor is called for any of the objects in this memory! - * - * @param size Number of objects of type T that should fit into this memory - * @param fd File descriptor - * @param write True if data should be writable - * @returns Pointer to mapped memory - * @throws std::system_error If mmap(2) failed - */ - static T* map(size_t size, int fd, bool write = false) { - int prot = PROT_READ; - if (write) { - prot |= PROT_WRITE; - } - void* addr = ::mmap(nullptr, sizeof(T) * size, prot, MAP_SHARED, fd, 0); -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wold-style-cast" - if (addr == MAP_FAILED) { - throw std::system_error(errno, std::system_category(), "mmap failed"); - } -#pragma GCC diagnostic pop - return reinterpret_cast(addr); - } - -// mremap(2) is only available on linux systems -#ifdef __linux__ - /** - * Grow memory mapping created with map(). - * - * Note that no constructor is called for any of the objects in this memory! - * - * @param data Pointer to current mapping (as returned by typed_mmap()) - * @param old_size Number of objects currently stored in this memory - * @param new_size Number of objects we want to have space for - * @throws std::system_error If mremap(2) call failed - */ - static T* remap(T* data, size_t old_size, size_t new_size) { - void* addr = ::mremap(reinterpret_cast(data), sizeof(T) * old_size, sizeof(T) * new_size, MREMAP_MAYMOVE); -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wold-style-cast" - if (addr == MAP_FAILED) { - throw std::system_error(errno, std::system_category(), "mremap failed"); - } -#pragma GCC diagnostic pop - return reinterpret_cast(addr); - } -#endif - - /** - * Release memory from map() call. - * - * Note that no destructor is called for the objects in this memory! - * - * @param data Pointer to the data - * @param size Number of objects of type T stored - * @throws std::system_error If munmap(2) call failed - */ - static void unmap(T* data, size_t size) { - if (::munmap(reinterpret_cast(data), sizeof(T) * size) != 0) { - throw std::system_error(errno, std::system_category(), "munmap failed"); - } - } - - /** - * Get number of objects of type T that would fit into a file. - * - * @param fd File descriptor - * @returns Number of objects of type T in this file - * @throws std::system_error If fstat(2) call failed - * @throws std::length_error If size of the file isn't a multiple of sizeof(T) - */ - static size_t file_size(int fd) { - struct stat s; - if (fstat(fd, &s) < 0) { - throw std::system_error(errno, std::system_category(), "fstat failed"); - } - if (static_cast(s.st_size) % sizeof(T) != 0) { - throw std::length_error("file size has to be multiple of object size"); - } - return static_cast(s.st_size) / sizeof(T); - } - - /** - * Grow file so there is enough space for at least new_size objects - * of type T. If the file is large enough already, nothing is done. - * The file is never shrunk. - * - * @param new_size Number of objects of type T that should fit into this file - * @param fd File descriptor - * @throws std::system_error If ftruncate(2) call failed - */ - static void grow_file(size_t new_size, int fd) { - if (file_size(fd) < new_size) { - if (::ftruncate(fd, static_cast_with_assert(sizeof(T) * new_size)) < 0) { - throw std::system_error(errno, std::system_category(), "ftruncate failed"); - } - } - } - - /** - * Grow file to given size (if it is smaller) and mmap it. - * - * @param size Number of objects of type T that should fit into this file - * @param fd File descriptor - * @throws Errors thrown by grow_file() or map() - */ - static T* grow_and_map(size_t size, int fd) { - grow_file(size, fd); - return map(size, fd, true); - } - - }; // class typed_mmap - - } // namespace detail - -} // namespace osmium - -#endif // OSMIUM_INDEX_DETAIL_TYPED_MMAP_HPP diff --git a/include/osmium/index/detail/vector_map.hpp b/include/osmium/index/detail/vector_map.hpp index 73c5a37a579..9c1cf4ed842 100644 --- a/include/osmium/index/detail/vector_map.hpp +++ b/include/osmium/index/detail/vector_map.hpp @@ -68,7 +68,7 @@ namespace osmium { m_vector(fd) { } - ~VectorBasedDenseMap() {} + ~VectorBasedDenseMap() = default; void reserve(const size_t size) override final { m_vector.reserve(size); @@ -97,6 +97,10 @@ namespace osmium { return m_vector.size(); } + size_t byte_size() const { + return m_vector.size() * sizeof(element_type); + } + size_t used_memory() const override final { return sizeof(TValue) * size(); } @@ -106,6 +110,10 @@ namespace osmium { m_vector.shrink_to_fit(); } + void dump_as_array(const int fd) override final { + osmium::io::detail::reliable_write(fd, reinterpret_cast(m_vector.data()), byte_size()); + } + iterator begin() { return m_vector.begin(); } diff --git a/include/osmium/index/detail/vector_multimap.hpp b/include/osmium/index/detail/vector_multimap.hpp index c2b2e1f2915..1d875fcba96 100644 --- a/include/osmium/index/detail/vector_multimap.hpp +++ b/include/osmium/index/detail/vector_multimap.hpp @@ -67,6 +67,16 @@ namespace osmium { public: + VectorBasedSparseMultimap() : + m_vector() { + } + + explicit VectorBasedSparseMultimap(int fd) : + m_vector(fd) { + } + + ~VectorBasedSparseMultimap() = default; + void set(const TId id, const TValue value) override final { m_vector.push_back(element_type(id, value)); } @@ -141,6 +151,30 @@ namespace osmium { osmium::io::detail::reliable_write(fd, reinterpret_cast(m_vector.data()), byte_size()); } + iterator begin() { + return m_vector.begin(); + } + + iterator end() { + return m_vector.end(); + } + + const_iterator cbegin() const { + return m_vector.cbegin(); + } + + const_iterator cend() const { + return m_vector.cend(); + } + + const_iterator begin() const { + return m_vector.cbegin(); + } + + const_iterator end() const { + return m_vector.cend(); + } + }; // class VectorBasedSparseMultimap } // namespace multimap diff --git a/include/osmium/index/index.hpp b/include/osmium/index/index.hpp index b73b319e87b..f415192d2db 100644 --- a/include/osmium/index/index.hpp +++ b/include/osmium/index/index.hpp @@ -67,7 +67,7 @@ namespace osmium { template OSMIUM_NORETURN void not_found_error(TKey key) { std::stringstream s; - s << "id " << key << " no found"; + s << "id " << key << " not found"; throw not_found(s.str()); } diff --git a/include/osmium/index/map.hpp b/include/osmium/index/map.hpp index 7b44b8e3377..61af672cdd7 100644 --- a/include/osmium/index/map.hpp +++ b/include/osmium/index/map.hpp @@ -148,7 +148,11 @@ namespace osmium { } virtual void dump_as_list(const int /*fd*/) { - std::runtime_error("can't dump as list"); + throw std::runtime_error("can't dump as list"); + } + + virtual void dump_as_array(const int /*fd*/) { + throw std::runtime_error("can't dump as array"); } }; // class Map @@ -195,6 +199,10 @@ namespace osmium { return m_callbacks.emplace(map_type_name, func).second; } + bool has_map_type(const std::string& map_type_name) const { + return m_callbacks.count(map_type_name); + } + std::vector map_types() const { std::vector result; @@ -242,9 +250,13 @@ namespace osmium { }); } +#define OSMIUM_CONCATENATE_DETAIL_(x, y) x##y +#define OSMIUM_CONCATENATE_(x, y) OSMIUM_CONCATENATE_DETAIL_(x, y) +#define OSMIUM_MAKE_UNIQUE_(x) OSMIUM_CONCATENATE_(x, __COUNTER__) + #define REGISTER_MAP(id, value, klass, name) \ namespace { \ - const bool registered_index_map_##name = osmium::index::register_map(#name); \ + const bool OSMIUM_MAKE_UNIQUE_(registered_index_map_##name) = osmium::index::register_map(#name); \ } } // namespace index diff --git a/include/osmium/index/map/dense_mmap_array.hpp b/include/osmium/index/map/dense_mmap_array.hpp index fc60a1ef21d..a912aebd1f0 100644 --- a/include/osmium/index/map/dense_mmap_array.hpp +++ b/include/osmium/index/map/dense_mmap_array.hpp @@ -35,7 +35,7 @@ DEALINGS IN THE SOFTWARE. #ifdef __linux__ -#include +#include // IWYU pragma: keep #include #define OSMIUM_HAS_INDEX_MAP_DENSE_MMAP_ARRAY diff --git a/include/osmium/index/map/sparse_mem_map.hpp b/include/osmium/index/map/sparse_mem_map.hpp index d053155f057..2b9048b3be2 100644 --- a/include/osmium/index/map/sparse_mem_map.hpp +++ b/include/osmium/index/map/sparse_mem_map.hpp @@ -33,7 +33,7 @@ DEALINGS IN THE SOFTWARE. */ -#include +#include // IWYU pragma: keep (for std::copy) #include #include #include diff --git a/include/osmium/io/any_input.hpp b/include/osmium/io/any_input.hpp index 633fab31392..d16d069a571 100644 --- a/include/osmium/io/any_input.hpp +++ b/include/osmium/io/any_input.hpp @@ -39,8 +39,8 @@ DEALINGS IN THE SOFTWARE. * Include this file if you want to read all kinds of OSM files. * * @attention If you include this file, you'll need to link with - * `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only), - * `libexpat`, `libz`, `libbz2`, and enable multithreading. + * `ws2_32` (Windows only), `libexpat`, `libz`, `libbz2`, + * and enable multithreading. */ #include // IWYU pragma: export diff --git a/include/osmium/io/any_output.hpp b/include/osmium/io/any_output.hpp index 63de3ff2ebf..990a27bacfd 100644 --- a/include/osmium/io/any_output.hpp +++ b/include/osmium/io/any_output.hpp @@ -39,12 +39,13 @@ DEALINGS IN THE SOFTWARE. * Include this file if you want to write all kinds of OSM files. * * @attention If you include this file, you'll need to link with - * `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only), - * `libz`, `libbz2`, and enable multithreading. + * `ws2_32` (Windows only), `libz`, `libbz2`, and enable + * multithreading. */ #include // IWYU pragma: export +#include // IWYU pragma: export #include // IWYU pragma: export #include // IWYU pragma: export #include // IWYU pragma: export diff --git a/include/osmium/io/bzip2_compression.hpp b/include/osmium/io/bzip2_compression.hpp index 7e86c154358..e961a87ab9a 100644 --- a/include/osmium/io/bzip2_compression.hpp +++ b/include/osmium/io/bzip2_compression.hpp @@ -274,11 +274,16 @@ namespace osmium { namespace { +// we want the register_compression() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_bzip2_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::bzip2, [](int fd) { return new osmium::io::Bzip2Compressor(fd); }, [](int fd) { return new osmium::io::Bzip2Decompressor(fd); }, [](const char* buffer, size_t size) { return new osmium::io::Bzip2BufferDecompressor(buffer, size); } ); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/include/osmium/io/compression.hpp b/include/osmium/io/compression.hpp index c1f8de2e71d..2529761810d 100644 --- a/include/osmium/io/compression.hpp +++ b/include/osmium/io/compression.hpp @@ -266,11 +266,16 @@ namespace osmium { namespace { +// we want the register_compression() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_no_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::none, [](int fd) { return new osmium::io::NoCompressor(fd); }, [](int fd) { return new osmium::io::NoDecompressor(fd); }, [](const char* buffer, size_t size) { return new osmium::io::NoDecompressor(buffer, size); } ); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/include/osmium/io/debug_output.hpp b/include/osmium/io/debug_output.hpp new file mode 100644 index 00000000000..2836f79879d --- /dev/null +++ b/include/osmium/io/debug_output.hpp @@ -0,0 +1,39 @@ +#ifndef OSMIUM_IO_DEBUG_OUTPUT_HPP +#define OSMIUM_IO_DEBUG_OUTPUT_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include // IWYU pragma: export +#include // IWYU pragma: export + +#endif // OSMIUM_IO_DEBUG_OUTPUT_HPP diff --git a/include/osmium/io/detail/debug_output_format.hpp b/include/osmium/io/detail/debug_output_format.hpp new file mode 100644 index 00000000000..efecc583313 --- /dev/null +++ b/include/osmium/io/detail/debug_output_format.hpp @@ -0,0 +1,482 @@ +#ifndef OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP +#define OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + namespace io { + + class File; + + namespace detail { + + constexpr const char* color_bold = "\x1b[1m"; + constexpr const char* color_black = "\x1b[30m"; + constexpr const char* color_gray = "\x1b[30;1m"; + constexpr const char* color_red = "\x1b[31m"; + constexpr const char* color_green = "\x1b[32m"; + constexpr const char* color_yellow = "\x1b[33m"; + constexpr const char* color_blue = "\x1b[34m"; + constexpr const char* color_magenta = "\x1b[35m"; + constexpr const char* color_cyan = "\x1b[36m"; + constexpr const char* color_white = "\x1b[37m"; + constexpr const char* color_reset = "\x1b[0m"; + + /** + * Writes out one buffer with OSM data in Debug format. + */ + class DebugOutputBlock : public osmium::handler::Handler { + + static constexpr size_t tmp_buffer_size = 50; + + std::shared_ptr m_input_buffer; + + std::shared_ptr m_out; + + char m_tmp_buffer[tmp_buffer_size+1]; + + bool m_add_metadata; + bool m_use_color; + + template + void output_formatted(const char* format, TArgs&&... args) { +#ifndef NDEBUG + int len = +#endif +#ifndef _MSC_VER + snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward(args)...); +#else + _snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward(args)...); +#endif + assert(len > 0 && static_cast(len) < tmp_buffer_size); + *m_out += m_tmp_buffer; + } + + void append_encoded_string(const char* data) { + const char* end = data + std::strlen(data); + + while (data != end) { + const char* last = data; + uint32_t c = utf8::next(data, end); + + // This is a list of Unicode code points that we let + // through instead of escaping them. It is incomplete + // and can be extended later. + // Generally we don't want to let through any + // non-printing characters. + if ((0x0020 <= c && c <= 0x0021) || + (0x0023 <= c && c <= 0x003b) || + (0x003d == c) || + (0x003f <= c && c <= 0x007e) || + (0x00a1 <= c && c <= 0x00ac) || + (0x00ae <= c && c <= 0x05ff)) { + m_out->append(last, data); + } else { + write_color(color_red); + output_formatted("", c); + write_color(color_blue); + } + } + } + + void write_color(const char* color) { + if (m_use_color) { + *m_out += color; + } + } + + void write_string(const char* string) { + *m_out += '"'; + write_color(color_blue); + append_encoded_string(string); + write_color(color_reset); + *m_out += '"'; + } + + void write_object_type(const char* object_type, bool visible = true) { + if (visible) { + write_color(color_bold); + } else { + write_color(color_white); + } + *m_out += object_type; + write_color(color_reset); + *m_out += ' '; + } + + void write_fieldname(const char* name) { + *m_out += " "; + write_color(color_cyan); + *m_out += name; + write_color(color_reset); + *m_out += ": "; + } + + void write_error(const char* msg) { + write_color(color_red); + *m_out += msg; + write_color(color_reset); + } + + void write_meta(const osmium::OSMObject& object) { + output_formatted("%" PRId64 "\n", object.id()); + if (m_add_metadata) { + write_fieldname("version"); + output_formatted(" %d", object.version()); + if (object.visible()) { + *m_out += " visible\n"; + } else { + write_error(" deleted\n"); + } + write_fieldname("changeset"); + output_formatted("%d\n", object.changeset()); + write_fieldname("timestamp"); + *m_out += object.timestamp().to_iso(); + output_formatted(" (%d)\n", object.timestamp()); + write_fieldname("user"); + output_formatted(" %d ", object.uid()); + write_string(object.user()); + *m_out += '\n'; + } + } + + void write_tags(const osmium::TagList& tags, const char* padding="") { + if (!tags.empty()) { + write_fieldname("tags"); + *m_out += padding; + output_formatted(" %d\n", tags.size()); + + osmium::max_op max; + for (const auto& tag : tags) { + max.update(std::strlen(tag.key())); + } + for (const auto& tag : tags) { + *m_out += " "; + write_string(tag.key()); + int spacing = max() - std::strlen(tag.key()); + while (spacing--) { + *m_out += " "; + } + *m_out += " = "; + write_string(tag.value()); + *m_out += '\n'; + } + } + } + + void write_location(const osmium::Location& location) { + write_fieldname("lon/lat"); + output_formatted(" %.7f,%.7f", location.lon_without_check(), location.lat_without_check()); + if (!location.valid()) { + write_error(" INVALID LOCATION!"); + } + *m_out += '\n'; + } + + void write_box(const osmium::Box& box) { + write_fieldname("box l/b/r/t"); + if (!box) { + write_error("BOX NOT SET!\n"); + return; + } + const auto& bl = box.bottom_left(); + const auto& tr = box.top_right(); + output_formatted("%.7f,%.7f %.7f,%.7f", bl.lon_without_check(), bl.lat_without_check(), tr.lon_without_check(), tr.lat_without_check()); + if (!box.valid()) { + write_error(" INVALID BOX!"); + } + *m_out += '\n'; + } + + public: + + explicit DebugOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool use_color) : + m_input_buffer(std::make_shared(std::move(buffer))), + m_out(std::make_shared()), + m_tmp_buffer(), + m_add_metadata(add_metadata), + m_use_color(use_color) { + } + + DebugOutputBlock(const DebugOutputBlock&) = default; + DebugOutputBlock& operator=(const DebugOutputBlock&) = default; + + DebugOutputBlock(DebugOutputBlock&&) = default; + DebugOutputBlock& operator=(DebugOutputBlock&&) = default; + + ~DebugOutputBlock() = default; + + std::string operator()() { + osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this); + + std::string out; + std::swap(out, *m_out); + return out; + } + + void node(const osmium::Node& node) { + write_object_type("node", node.visible()); + write_meta(node); + + if (node.visible()) { + write_location(node.location()); + } + + write_tags(node.tags()); + + *m_out += '\n'; + } + + void way(const osmium::Way& way) { + write_object_type("way", way.visible()); + write_meta(way); + write_tags(way.tags()); + + write_fieldname("nodes"); + + output_formatted(" %d", way.nodes().size()); + if (way.nodes().size() < 2) { + write_error(" LESS THAN 2 NODES!\n"); + } else if (way.nodes().size() > 2000) { + write_error(" MORE THAN 2000 NODES!\n"); + } else if (way.nodes().is_closed()) { + *m_out += " (closed)\n"; + } else { + *m_out += " (open)\n"; + } + + int width = int(log10(way.nodes().size())) + 1; + int n = 0; + for (const auto& node_ref : way.nodes()) { + output_formatted(" %0*d: %10" PRId64, width, n++, node_ref.ref()); + if (node_ref.location().valid()) { + output_formatted(" (%.7f,%.7f)", node_ref.location().lon_without_check(), node_ref.location().lat_without_check()); + } + *m_out += '\n'; + } + + *m_out += '\n'; + } + + void relation(const osmium::Relation& relation) { + static const char* short_typename[] = { "node", "way ", "rel " }; + write_object_type("relation", relation.visible()); + write_meta(relation); + write_tags(relation.tags()); + + write_fieldname("members"); + output_formatted(" %d\n", relation.members().size()); + + int width = int(log10(relation.members().size())) + 1; + int n = 0; + for (const auto& member : relation.members()) { + output_formatted(" %0*d: ", width, n++); + *m_out += short_typename[item_type_to_nwr_index(member.type())]; + output_formatted(" %10" PRId64 " ", member.ref()); + write_string(member.role()); + *m_out += '\n'; + } + + *m_out += '\n'; + } + + void changeset(const osmium::Changeset& changeset) { + write_object_type("changeset"); + output_formatted("%d\n", changeset.id()); + write_fieldname("num changes"); + output_formatted("%d", changeset.num_changes()); + if (changeset.num_changes() == 0) { + write_error(" NO CHANGES!"); + } + *m_out += '\n'; + write_fieldname("created at"); + *m_out += ' '; + *m_out += changeset.created_at().to_iso(); + output_formatted(" (%d)\n", changeset.created_at()); + write_fieldname("closed at"); + *m_out += " "; + if (changeset.closed()) { + *m_out += changeset.closed_at().to_iso(); + output_formatted(" (%d)\n", changeset.closed_at()); + } else { + write_error("OPEN!\n"); + } + write_fieldname("user"); + output_formatted(" %d ", changeset.uid()); + write_string(changeset.user()); + *m_out += '\n'; + + write_box(changeset.bounds()); + write_tags(changeset.tags(), " "); + + *m_out += '\n'; + } + + }; // DebugOutputBlock + + class DebugOutputFormat : public osmium::io::detail::OutputFormat { + + bool m_add_metadata; + bool m_use_color; + + public: + + DebugOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : + OutputFormat(file, output_queue), + m_add_metadata(file.get("add_metadata") != "false"), + m_use_color(file.get("color") == "true") { + } + + DebugOutputFormat(const DebugOutputFormat&) = delete; + DebugOutputFormat& operator=(const DebugOutputFormat&) = delete; + + void write_buffer(osmium::memory::Buffer&& buffer) override final { + m_output_queue.push(osmium::thread::Pool::instance().submit(DebugOutputBlock{std::move(buffer), m_add_metadata, m_use_color})); + } + + void write_fieldname(std::string& out, const char* name) { + out += " "; + if (m_use_color) { + out += color_cyan; + } + out += name; + if (m_use_color) { + out += color_reset; + } + out += ": "; + } + + void write_header(const osmium::io::Header& header) override final { + std::string out; + + if (m_use_color) { + out += color_bold; + } + out += "header\n"; + if (m_use_color) { + out += color_reset; + } + + write_fieldname(out, "multiple object versions"); + out += header.has_multiple_object_versions() ? "yes" : "no"; + out += '\n'; + write_fieldname(out, "bounding boxes"); + out += '\n'; + for (const auto& box : header.boxes()) { + out += " "; + box.bottom_left().as_string(std::back_inserter(out), ','); + out += " "; + box.top_right().as_string(std::back_inserter(out), ','); + out += '\n'; + } + write_fieldname(out, "options"); + out += '\n'; + for (const auto& opt : header) { + out += " "; + out += opt.first; + out += " = "; + out += opt.second; + out += '\n'; + } + out += "\n=============================================\n\n"; + + std::promise promise; + m_output_queue.push(promise.get_future()); + promise.set_value(std::move(out)); + } + + void close() override final { + std::string out; + std::promise promise; + m_output_queue.push(promise.get_future()); + promise.set_value(out); + } + + }; // class DebugOutputFormat + + namespace { + +// we want the register_output_format() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" + const bool registered_debug_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::debug, + [](const osmium::io::File& file, data_queue_type& output_queue) { + return new osmium::io::detail::DebugOutputFormat(file, output_queue); + }); +#pragma GCC diagnostic pop + + } // anonymous namespace + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP diff --git a/include/osmium/io/detail/opl_output_format.hpp b/include/osmium/io/detail/opl_output_format.hpp index cf92e1382d8..a3103d9bf67 100644 --- a/include/osmium/io/detail/opl_output_format.hpp +++ b/include/osmium/io/detail/opl_output_format.hpp @@ -46,23 +46,7 @@ DEALINGS IN THE SOFTWARE. #include #include -#include - -#ifdef __clang__ -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wmissing-noreturn" -# pragma clang diagnostic ignored "-Wsign-conversion" -#endif - -#if BOOST_VERSION >= 104800 -# include -#else -# include -#endif - -#ifdef __clang__ -# pragma clang diagnostic pop -#endif +#include #include #include @@ -103,6 +87,8 @@ namespace osmium { char m_tmp_buffer[tmp_buffer_size+1]; + bool m_add_metadata; + template void output_formatted(const char* format, TArgs&&... args) { #ifndef NDEBUG @@ -117,13 +103,12 @@ namespace osmium { *m_out += m_tmp_buffer; } - void append_encoded_string(const std::string& data) { - boost::u8_to_u32_iterator it(data.cbegin(), data.cbegin(), data.cend()); - boost::u8_to_u32_iterator end(data.cend(), data.cend(), data.cend()); - boost::utf8_output_iterator> oit(std::back_inserter(*m_out)); + void append_encoded_string(const char* data) { + const char* end = data + std::strlen(data); - for (; it != end; ++it) { - uint32_t c = *it; + while (data != end) { + const char* last = data; + uint32_t c = utf8::next(data, end); // This is a list of Unicode code points that we let // through instead of escaping them. It is incomplete @@ -138,21 +123,29 @@ namespace osmium { (0x0041 <= c && c <= 0x007e) || (0x00a1 <= c && c <= 0x00ac) || (0x00ae <= c && c <= 0x05ff)) { - *oit = c; + m_out->append(last, data); } else { *m_out += '%'; - output_formatted("%04x", c); + if (c <= 0xff) { + output_formatted("%02x", c); + } else { + output_formatted("%04x", c); + } + *m_out += '%'; } } } void write_meta(const osmium::OSMObject& object) { - output_formatted("%" PRId64 " v%d d", object.id(), object.version()); - *m_out += (object.visible() ? 'V' : 'D'); - output_formatted(" c%d t", object.changeset()); - *m_out += object.timestamp().to_iso(); - output_formatted(" i%d u", object.uid()); - append_encoded_string(object.user()); + output_formatted("%" PRId64, object.id()); + if (m_add_metadata) { + output_formatted(" v%d d", object.version()); + *m_out += (object.visible() ? 'V' : 'D'); + output_formatted(" c%d t", object.changeset()); + *m_out += object.timestamp().to_iso(); + output_formatted(" i%d u", object.uid()); + append_encoded_string(object.user()); + } *m_out += " T"; bool first = true; for (const auto& tag : object.tags()) { @@ -180,10 +173,11 @@ namespace osmium { public: - explicit OPLOutputBlock(osmium::memory::Buffer&& buffer) : + explicit OPLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata) : m_input_buffer(std::make_shared(std::move(buffer))), m_out(std::make_shared()), - m_tmp_buffer() { + m_tmp_buffer(), + m_add_metadata(add_metadata) { } OPLOutputBlock(const OPLOutputBlock&) = default; @@ -240,7 +234,7 @@ namespace osmium { } *m_out += item_type_to_char(member.type()); output_formatted("%" PRId64 "@", member.ref()); - *m_out += member.role(); + append_encoded_string(member.role()); } *m_out += '\n'; } @@ -274,17 +268,20 @@ namespace osmium { class OPLOutputFormat : public osmium::io::detail::OutputFormat { - OPLOutputFormat(const OPLOutputFormat&) = delete; - OPLOutputFormat& operator=(const OPLOutputFormat&) = delete; + bool m_add_metadata; public: OPLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : - OutputFormat(file, output_queue) { + OutputFormat(file, output_queue), + m_add_metadata(file.get("add_metadata") != "false") { } + OPLOutputFormat(const OPLOutputFormat&) = delete; + OPLOutputFormat& operator=(const OPLOutputFormat&) = delete; + void write_buffer(osmium::memory::Buffer&& buffer) override final { - m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer)})); + m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer), m_add_metadata})); } void close() override final { @@ -298,6 +295,8 @@ namespace osmium { namespace { +// we want the register_output_format() function to run, setting the variable +// is only a side-effect, it will never be used #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_opl_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::opl, diff --git a/include/osmium/io/detail/pbf.hpp b/include/osmium/io/detail/pbf.hpp index e64e51a57c0..15e457a1235 100644 --- a/include/osmium/io/detail/pbf.hpp +++ b/include/osmium/io/detail/pbf.hpp @@ -33,9 +33,7 @@ DEALINGS IN THE SOFTWARE. */ -#include - -#include +#include // needed for htonl and ntohl #ifndef _WIN32 @@ -45,38 +43,10 @@ DEALINGS IN THE SOFTWARE. #endif #include -#include +#include namespace osmium { -// avoid g++ false positive -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wreturn-type" - inline item_type osmpbf_membertype_to_item_type(const OSMPBF::Relation::MemberType mt) { - switch (mt) { - case OSMPBF::Relation::NODE: - return item_type::node; - case OSMPBF::Relation::WAY: - return item_type::way; - case OSMPBF::Relation::RELATION: - return item_type::relation; - } - } -#pragma GCC diagnostic pop - - inline OSMPBF::Relation::MemberType item_type_to_osmpbf_membertype(const item_type type) { - switch (type) { - case item_type::node: - return OSMPBF::Relation::NODE; - case item_type::way: - return OSMPBF::Relation::WAY; - case item_type::relation: - return OSMPBF::Relation::RELATION; - default: - throw std::runtime_error("Unknown relation member type"); - } - } - /** * Exception thrown when there was a problem with parsing the PBF format of * a file. @@ -93,6 +63,26 @@ namespace osmium { }; // struct pbf_error + namespace io { + + namespace detail { + + // the maximum size of a blob header in bytes + const int max_blob_header_size = 64 * 1024; // 64 kB + + // the maximum size of an uncompressed blob in bytes + const uint64_t max_uncompressed_blob_size = 32 * 1024 * 1024; // 32 MB + + // resolution for longitude/latitude used for conversion + // between representation as double and as int + const int64_t lonlat_resolution = 1000 * 1000 * 1000; + + const int64_t resolution_convert = lonlat_resolution / osmium::Location::coordinate_precision; + + } + + } + } // namespace osmium #endif // OSMIUM_IO_DETAIL_PBF_HPP diff --git a/include/osmium/io/detail/pbf_decoder.hpp b/include/osmium/io/detail/pbf_decoder.hpp new file mode 100644 index 00000000000..79e899ff85e --- /dev/null +++ b/include/osmium/io/detail/pbf_decoder.hpp @@ -0,0 +1,760 @@ +#ifndef OSMIUM_IO_DETAIL_PBF_DECODER_HPP +#define OSMIUM_IO_DETAIL_PBF_DECODER_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include // IWYU pragma: export +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + namespace io { + + namespace detail { + + using ptr_len_type = std::pair; + + class PBFPrimitiveBlockDecoder { + + static constexpr size_t initial_buffer_size = 2 * 1024 * 1024; + + ptr_len_type m_data; + std::vector m_stringtable; + + int64_t m_lon_offset = 0; + int64_t m_lat_offset = 0; + int64_t m_date_factor = 1000; + int32_t m_granularity = 100; + + osmium::osm_entity_bits::type m_read_types; + + osmium::memory::Buffer m_buffer { initial_buffer_size }; + + void decode_stringtable(const ptr_len_type& data) { + if (!m_stringtable.empty()) { + throw osmium::pbf_error("more than one stringtable in pbf file"); + } + + protozero::pbf_message pbf_string_table(data); + while (pbf_string_table.next(OSMFormat::StringTable::repeated_bytes_s)) { + m_stringtable.push_back(pbf_string_table.get_data()); + } + } + + void decode_primitive_block_metadata() { + protozero::pbf_message pbf_primitive_block(m_data); + while (pbf_primitive_block.next()) { + switch (pbf_primitive_block.tag()) { + case OSMFormat::PrimitiveBlock::required_StringTable_stringtable: + decode_stringtable(pbf_primitive_block.get_data()); + break; + case OSMFormat::PrimitiveBlock::optional_int32_granularity: + m_granularity = pbf_primitive_block.get_int32(); + break; + case OSMFormat::PrimitiveBlock::optional_int32_date_granularity: + m_date_factor = pbf_primitive_block.get_int32(); + break; + case OSMFormat::PrimitiveBlock::optional_int64_lat_offset: + m_lat_offset = pbf_primitive_block.get_int64(); + break; + case OSMFormat::PrimitiveBlock::optional_int64_lon_offset: + m_lon_offset = pbf_primitive_block.get_int64(); + break; + default: + pbf_primitive_block.skip(); + } + } + } + + void decode_primitive_block_data() { + protozero::pbf_message pbf_primitive_block(m_data); + while (pbf_primitive_block.next(OSMFormat::PrimitiveBlock::repeated_PrimitiveGroup_primitivegroup)) { + protozero::pbf_message pbf_primitive_group = pbf_primitive_block.get_message(); + while (pbf_primitive_group.next()) { + switch (pbf_primitive_group.tag()) { + case OSMFormat::PrimitiveGroup::repeated_Node_nodes: + if (m_read_types & osmium::osm_entity_bits::node) { + decode_node(pbf_primitive_group.get_data()); + } else { + pbf_primitive_group.skip(); + } + break; + case OSMFormat::PrimitiveGroup::optional_DenseNodes_dense: + if (m_read_types & osmium::osm_entity_bits::node) { + decode_dense_nodes(pbf_primitive_group.get_data()); + } else { + pbf_primitive_group.skip(); + } + break; + case OSMFormat::PrimitiveGroup::repeated_Way_ways: + if (m_read_types & osmium::osm_entity_bits::way) { + decode_way(pbf_primitive_group.get_data()); + } else { + pbf_primitive_group.skip(); + } + break; + case OSMFormat::PrimitiveGroup::repeated_Relation_relations: + if (m_read_types & osmium::osm_entity_bits::relation) { + decode_relation(pbf_primitive_group.get_data()); + } else { + pbf_primitive_group.skip(); + } + break; + default: + pbf_primitive_group.skip(); + } + } + } + } + + ptr_len_type decode_info(const ptr_len_type& data, osmium::OSMObject& object) { + ptr_len_type user = std::make_pair("", 0); + + protozero::pbf_message pbf_info(data); + while (pbf_info.next()) { + switch (pbf_info.tag()) { + case OSMFormat::Info::optional_int32_version: + { + auto version = pbf_info.get_int32(); + if (version < 0) { + throw osmium::pbf_error("object version must not be negative"); + } + object.set_version(static_cast_with_assert(version)); + } + break; + case OSMFormat::Info::optional_int64_timestamp: + object.set_timestamp(pbf_info.get_int64() * m_date_factor / 1000); + break; + case OSMFormat::Info::optional_int64_changeset: + { + auto changeset_id = pbf_info.get_int64(); + if (changeset_id < 0) { + throw osmium::pbf_error("object changeset_id must not be negative"); + } + object.set_changeset(static_cast_with_assert(changeset_id)); + } + break; + case OSMFormat::Info::optional_int32_uid: + object.set_uid_from_signed(pbf_info.get_int32()); + break; + case OSMFormat::Info::optional_uint32_user_sid: + user = m_stringtable.at(pbf_info.get_uint32()); + break; + case OSMFormat::Info::optional_bool_visible: + object.set_visible(pbf_info.get_bool()); + break; + default: + pbf_info.skip(); + } + } + + return user; + } + + using kv_type = std::pair; + + void build_tag_list(osmium::builder::Builder& builder, const kv_type& keys, const kv_type& vals) { + if (keys.first != keys.second) { + osmium::builder::TagListBuilder tl_builder(m_buffer, &builder); + auto kit = keys.first; + auto vit = vals.first; + while (kit != keys.second) { + if (vit == vals.second) { + // this is against the spec, must have same number of elements + throw osmium::pbf_error("PBF format error"); + } + const auto& k = m_stringtable.at(*kit++); + const auto& v = m_stringtable.at(*vit++); + tl_builder.add_tag(k.first, k.second, v.first, v.second); + } + } + } + + int32_t convert_pbf_coordinate(int64_t c) const { + return (c * m_granularity + m_lon_offset) / resolution_convert; + } + + void decode_node(const ptr_len_type& data) { + osmium::builder::NodeBuilder builder(m_buffer); + osmium::Node& node = builder.object(); + + kv_type keys; + kv_type vals; + int64_t lon = std::numeric_limits::max(); + int64_t lat = std::numeric_limits::max(); + + ptr_len_type user = { "", 0 }; + + protozero::pbf_message pbf_node(data); + while (pbf_node.next()) { + switch (pbf_node.tag()) { + case OSMFormat::Node::required_sint64_id: + node.set_id(pbf_node.get_sint64()); + break; + case OSMFormat::Node::packed_uint32_keys: + keys = pbf_node.get_packed_uint32(); + break; + case OSMFormat::Node::packed_uint32_vals: + vals = pbf_node.get_packed_uint32(); + break; + case OSMFormat::Node::optional_Info_info: + user = decode_info(pbf_node.get_data(), builder.object()); + break; + case OSMFormat::Node::required_sint64_lat: + lat = pbf_node.get_sint64(); + break; + case OSMFormat::Node::required_sint64_lon: + lon = pbf_node.get_sint64(); + break; + default: + pbf_node.skip(); + } + } + + if (node.visible()) { + if (lon == std::numeric_limits::max() || + lat == std::numeric_limits::max()) { + throw osmium::pbf_error("illegal coordinate format"); + } + node.set_location(osmium::Location( + convert_pbf_coordinate(lon), + convert_pbf_coordinate(lat) + )); + } + + builder.add_user(user.first, user.second); + + build_tag_list(builder, keys, vals); + + m_buffer.commit(); + } + + void decode_way(const ptr_len_type& data) { + osmium::builder::WayBuilder builder(m_buffer); + + kv_type keys; + kv_type vals; + std::pair refs; + + ptr_len_type user = { "", 0 }; + + protozero::pbf_message pbf_way(data); + while (pbf_way.next()) { + switch (pbf_way.tag()) { + case OSMFormat::Way::required_int64_id: + builder.object().set_id(pbf_way.get_int64()); + break; + case OSMFormat::Way::packed_uint32_keys: + keys = pbf_way.get_packed_uint32(); + break; + case OSMFormat::Way::packed_uint32_vals: + vals = pbf_way.get_packed_uint32(); + break; + case OSMFormat::Way::optional_Info_info: + user = decode_info(pbf_way.get_data(), builder.object()); + break; + case OSMFormat::Way::packed_sint64_refs: + refs = pbf_way.get_packed_sint64(); + break; + default: + pbf_way.skip(); + } + } + + builder.add_user(user.first, user.second); + + if (refs.first != refs.second) { + osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder); + osmium::util::DeltaDecode ref; + while (refs.first != refs.second) { + wnl_builder.add_node_ref(ref.update(*refs.first++)); + } + } + + build_tag_list(builder, keys, vals); + + m_buffer.commit(); + } + + void decode_relation(const ptr_len_type& data) { + osmium::builder::RelationBuilder builder(m_buffer); + + kv_type keys; + kv_type vals; + std::pair roles; + std::pair refs; + std::pair types; + + ptr_len_type user = { "", 0 }; + + protozero::pbf_message pbf_relation(data); + while (pbf_relation.next()) { + switch (pbf_relation.tag()) { + case OSMFormat::Relation::required_int64_id: + builder.object().set_id(pbf_relation.get_int64()); + break; + case OSMFormat::Relation::packed_uint32_keys: + keys = pbf_relation.get_packed_uint32(); + break; + case OSMFormat::Relation::packed_uint32_vals: + vals = pbf_relation.get_packed_uint32(); + break; + case OSMFormat::Relation::optional_Info_info: + user = decode_info(pbf_relation.get_data(), builder.object()); + break; + case OSMFormat::Relation::packed_int32_roles_sid: + roles = pbf_relation.get_packed_int32(); + break; + case OSMFormat::Relation::packed_sint64_memids: + refs = pbf_relation.get_packed_sint64(); + break; + case OSMFormat::Relation::packed_MemberType_types: + types = pbf_relation.get_packed_enum(); + break; + default: + pbf_relation.skip(); + } + } + + builder.add_user(user.first, user.second); + + if (refs.first != refs.second) { + osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder); + osmium::util::DeltaDecode ref; + while (roles.first != roles.second && refs.first != refs.second && types.first != types.second) { + const auto& r = m_stringtable.at(*roles.first++); + int type = *types.first++; + if (type < 0 || type > 2) { + throw osmium::pbf_error("unknown relation member type"); + } + rml_builder.add_member( + osmium::item_type(type + 1), + ref.update(*refs.first++), + r.first, + r.second + ); + } + } + + build_tag_list(builder, keys, vals); + + m_buffer.commit(); + } + + void decode_dense_nodes(const ptr_len_type& data) { + bool has_info = false; + bool has_visibles = false; + + std::pair ids; + std::pair lats; + std::pair lons; + + std::pair tags; + + std::pair versions; + std::pair timestamps; + std::pair changesets; + std::pair uids; + std::pair user_sids; + std::pair visibles; + + protozero::pbf_message pbf_dense_nodes(data); + while (pbf_dense_nodes.next()) { + switch (pbf_dense_nodes.tag()) { + case OSMFormat::DenseNodes::packed_sint64_id: + ids = pbf_dense_nodes.get_packed_sint64(); + break; + case OSMFormat::DenseNodes::optional_DenseInfo_denseinfo: + { + has_info = true; + protozero::pbf_message pbf_dense_info = pbf_dense_nodes.get_message(); + while (pbf_dense_info.next()) { + switch (pbf_dense_info.tag()) { + case OSMFormat::DenseInfo::packed_int32_version: + versions = pbf_dense_info.get_packed_int32(); + break; + case OSMFormat::DenseInfo::packed_sint64_timestamp: + timestamps = pbf_dense_info.get_packed_sint64(); + break; + case OSMFormat::DenseInfo::packed_sint64_changeset: + changesets = pbf_dense_info.get_packed_sint64(); + break; + case OSMFormat::DenseInfo::packed_sint32_uid: + uids = pbf_dense_info.get_packed_sint32(); + break; + case OSMFormat::DenseInfo::packed_sint32_user_sid: + user_sids = pbf_dense_info.get_packed_sint32(); + break; + case OSMFormat::DenseInfo::packed_bool_visible: + has_visibles = true; + visibles = pbf_dense_info.get_packed_bool(); + break; + default: + pbf_dense_info.skip(); + } + } + } + break; + case OSMFormat::DenseNodes::packed_sint64_lat: + lats = pbf_dense_nodes.get_packed_sint64(); + break; + case OSMFormat::DenseNodes::packed_sint64_lon: + lons = pbf_dense_nodes.get_packed_sint64(); + break; + case OSMFormat::DenseNodes::packed_int32_keys_vals: + tags = pbf_dense_nodes.get_packed_int32(); + break; + default: + pbf_dense_nodes.skip(); + } + } + + osmium::util::DeltaDecode dense_id; + osmium::util::DeltaDecode dense_latitude; + osmium::util::DeltaDecode dense_longitude; + osmium::util::DeltaDecode dense_uid; + osmium::util::DeltaDecode dense_user_sid; + osmium::util::DeltaDecode dense_changeset; + osmium::util::DeltaDecode dense_timestamp; + + auto tag_it = tags.first; + + while (ids.first != ids.second) { + if (lons.first == lons.second || + lats.first == lats.second) { + // this is against the spec, must have same number of elements + throw osmium::pbf_error("PBF format error"); + } + + bool visible = true; + + osmium::builder::NodeBuilder builder(m_buffer); + osmium::Node& node = builder.object(); + + node.set_id(dense_id.update(*ids.first++)); + + if (has_info) { + if (versions.first == versions.second || + changesets.first == changesets.second || + timestamps.first == timestamps.second || + uids.first == uids.second || + user_sids.first == user_sids.second) { + // this is against the spec, must have same number of elements + throw osmium::pbf_error("PBF format error"); + } + + auto version = *versions.first++; + if (version < 0) { + throw osmium::pbf_error("object version must not be negative"); + } + node.set_version(static_cast(version)); + + auto changeset_id = dense_changeset.update(*changesets.first++); + if (changeset_id < 0) { + throw osmium::pbf_error("object changeset_id must not be negative"); + } + node.set_changeset(static_cast(changeset_id)); + + node.set_timestamp(dense_timestamp.update(*timestamps.first++) * m_date_factor / 1000); + node.set_uid_from_signed(static_cast(dense_uid.update(*uids.first++))); + + if (has_visibles) { + if (visibles.first == visibles.second) { + // this is against the spec, must have same number of elements + throw osmium::pbf_error("PBF format error"); + } + visible = *visibles.first++; + } + node.set_visible(visible); + + const auto& u = m_stringtable.at(dense_user_sid.update(*user_sids.first++)); + builder.add_user(u.first, u.second); + } else { + builder.add_user(""); + } + + if (visible) { + builder.object().set_location(osmium::Location( + convert_pbf_coordinate(dense_longitude.update(*lons.first++)), + convert_pbf_coordinate(dense_latitude.update(*lats.first++)) + )); + } + + if (tag_it != tags.second) { + osmium::builder::TagListBuilder tl_builder(m_buffer, &builder); + while (tag_it != tags.second && *tag_it != 0) { + const auto& k = m_stringtable.at(*tag_it++); + if (tag_it == tags.second) { + throw osmium::pbf_error("PBF format error"); // this is against the spec, keys/vals must come in pairs + } + const auto& v = m_stringtable.at(*tag_it++); + tl_builder.add_tag(k.first, k.second, v.first, v.second); + } + + if (tag_it != tags.second) { + ++tag_it; + } + } + + m_buffer.commit(); + } + + } + + public: + + explicit PBFPrimitiveBlockDecoder(const ptr_len_type& data, osmium::osm_entity_bits::type read_types) : + m_data(data), + m_read_types(read_types) { + } + + PBFPrimitiveBlockDecoder(const PBFPrimitiveBlockDecoder&) = delete; + PBFPrimitiveBlockDecoder& operator=(const PBFPrimitiveBlockDecoder&) = delete; + + PBFPrimitiveBlockDecoder(PBFPrimitiveBlockDecoder&&) = delete; + PBFPrimitiveBlockDecoder& operator=(PBFPrimitiveBlockDecoder&&) = delete; + + ~PBFPrimitiveBlockDecoder() = default; + + osmium::memory::Buffer operator()() { + try { + decode_primitive_block_metadata(); + decode_primitive_block_data(); + } catch (std::out_of_range&) { + throw osmium::pbf_error("string id out of range"); + } + + return std::move(m_buffer); + } + + }; // class PBFPrimitiveBlockDecoder + + inline ptr_len_type decode_blob(const std::string& blob_data, std::string& output) { + int32_t raw_size; + std::pair zlib_data; + + protozero::pbf_message pbf_blob(blob_data); + while (pbf_blob.next()) { + switch (pbf_blob.tag()) { + case FileFormat::Blob::optional_bytes_raw: + { + auto data_len = pbf_blob.get_data(); + if (data_len.second > max_uncompressed_blob_size) { + throw osmium::pbf_error("illegal blob size"); + } + return data_len; + } + case FileFormat::Blob::optional_int32_raw_size: + raw_size = pbf_blob.get_int32(); + if (raw_size <= 0 || uint32_t(raw_size) > max_uncompressed_blob_size) { + throw osmium::pbf_error("illegal blob size"); + } + break; + case FileFormat::Blob::optional_bytes_zlib_data: + zlib_data = pbf_blob.get_data(); + break; + case FileFormat::Blob::optional_bytes_lzma_data: + throw osmium::pbf_error("lzma blobs not implemented"); + default: + throw osmium::pbf_error("unknown compression"); + } + } + + if (zlib_data.second != 0) { + return osmium::io::detail::zlib_uncompress_string( + zlib_data.first, + static_cast(zlib_data.second), + static_cast(raw_size), + output + ); + } + + throw osmium::pbf_error("blob contains no data"); + } + + inline osmium::Box decode_header_bbox(const ptr_len_type& data) { + int64_t left = std::numeric_limits::max(); + int64_t right = std::numeric_limits::max(); + int64_t top = std::numeric_limits::max(); + int64_t bottom = std::numeric_limits::max(); + + protozero::pbf_message pbf_header_bbox(data); + while (pbf_header_bbox.next()) { + switch (pbf_header_bbox.tag()) { + case OSMFormat::HeaderBBox::required_sint64_left: + left = pbf_header_bbox.get_sint64(); + break; + case OSMFormat::HeaderBBox::required_sint64_right: + right = pbf_header_bbox.get_sint64(); + break; + case OSMFormat::HeaderBBox::required_sint64_top: + top = pbf_header_bbox.get_sint64(); + break; + case OSMFormat::HeaderBBox::required_sint64_bottom: + bottom = pbf_header_bbox.get_sint64(); + break; + default: + pbf_header_bbox.skip(); + } + } + + if (left == std::numeric_limits::max() || + right == std::numeric_limits::max() || + top == std::numeric_limits::max() || + bottom == std::numeric_limits::max()) { + throw osmium::pbf_error("invalid bbox"); + } + + osmium::Box box; + box.extend(osmium::Location(left / resolution_convert, bottom / resolution_convert)); + box.extend(osmium::Location(right / resolution_convert, top / resolution_convert)); + + return box; + } + + inline osmium::io::Header decode_header_block(const ptr_len_type& data) { + osmium::io::Header header; + int i = 0; + + protozero::pbf_message pbf_header_block(data); + while (pbf_header_block.next()) { + switch (pbf_header_block.tag()) { + case OSMFormat::HeaderBlock::optional_HeaderBBox_bbox: + header.add_box(decode_header_bbox(pbf_header_block.get_data())); + break; + case OSMFormat::HeaderBlock::repeated_string_required_features: + { + auto feature = pbf_header_block.get_data(); + if (!strncmp("OsmSchema-V0.6", feature.first, feature.second)) { + // intentionally left blank + } else if (!strncmp("DenseNodes", feature.first, feature.second)) { + header.set("pbf_dense_nodes", true); + } else if (!strncmp("HistoricalInformation", feature.first, feature.second)) { + header.set_has_multiple_object_versions(true); + } else { + std::string msg("required feature not supported: "); + msg.append(feature.first, feature.second); + throw osmium::pbf_error(msg); + } + } + break; + case OSMFormat::HeaderBlock::repeated_string_optional_features: + header.set("pbf_optional_feature_" + std::to_string(i++), pbf_header_block.get_string()); + break; + case OSMFormat::HeaderBlock::optional_string_writingprogram: + header.set("generator", pbf_header_block.get_string()); + break; + case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp: + header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.get_int64()).to_iso()); + break; + case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_sequence_number: + header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.get_int64())); + break; + case OSMFormat::HeaderBlock::optional_string_osmosis_replication_base_url: + header.set("osmosis_replication_base_url", pbf_header_block.get_string()); + break; + default: + pbf_header_block.skip(); + } + } + + return header; + } + + /** + * Decode HeaderBlock. + * + * @param header_block_data Input data + * @returns Header object + * @throws osmium::pbf_error If there was a parsing error + */ + inline osmium::io::Header decode_header(const std::string& header_block_data) { + std::string output; + + return decode_header_block(decode_blob(header_block_data, output)); + } + + class PBFDataBlobDecoder { + + std::shared_ptr m_input_buffer; + osmium::osm_entity_bits::type m_read_types; + + public: + + PBFDataBlobDecoder(std::string&& input_buffer, osmium::osm_entity_bits::type read_types) : + m_input_buffer(std::make_shared(std::move(input_buffer))), + m_read_types(read_types) { + } + + PBFDataBlobDecoder(const PBFDataBlobDecoder&) = default; + PBFDataBlobDecoder& operator=(const PBFDataBlobDecoder&) = default; + + PBFDataBlobDecoder(PBFDataBlobDecoder&&) = default; + PBFDataBlobDecoder& operator=(PBFDataBlobDecoder&&) = default; + + ~PBFDataBlobDecoder() = default; + + osmium::memory::Buffer operator()() { + std::string output; + PBFPrimitiveBlockDecoder decoder(decode_blob(*m_input_buffer, output), m_read_types); + return decoder(); + } + + }; // class PBFDataBlobDecoder + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_PBF_DECODER_HPP diff --git a/include/osmium/io/detail/pbf_input_format.hpp b/include/osmium/io/detail/pbf_input_format.hpp index ba8fb42a2e6..7817d27d164 100644 --- a/include/osmium/io/detail/pbf_input_format.hpp +++ b/include/osmium/io/detail/pbf_input_format.hpp @@ -49,9 +49,12 @@ DEALINGS IN THE SOFTWARE. #include #include +#include + #include #include // IWYU pragma: export -#include +#include +#include #include #include #include @@ -76,13 +79,13 @@ namespace osmium { namespace detail { - typedef osmium::thread::Queue> queue_type; - /** * Class for parsing PBF files. */ class PBFInputFormat : public osmium::io::detail::InputFormat { + typedef osmium::thread::Queue> queue_type; + bool m_use_thread_pool; bool m_eof { false }; queue_type m_queue; @@ -115,15 +118,10 @@ namespace osmium { } /** - * Read BlobHeader by first reading the size and then the - * BlobHeader. The BlobHeader contains a type field (which is - * checked against the expected type) and a size field. - * - * @param expected_type Expected type of data ("OSMHeader" or - * "OSMData"). - * @returns Size of the data read from BlobHeader (0 on EOF). + * Read 4 bytes in network byte order from file. They contain + * the length of the following BlobHeader. */ - size_t read_blob_header(const char* expected_type) { + uint32_t read_blob_header_size_from_file() { uint32_t size_in_network_byte_order; try { @@ -133,37 +131,76 @@ namespace osmium { return 0; // EOF } - uint32_t size = ntohl(size_in_network_byte_order); - if (size > static_cast(OSMPBF::max_blob_header_size)) { + const uint32_t size = ntohl(size_in_network_byte_order); + if (size > static_cast(max_blob_header_size)) { throw osmium::pbf_error("invalid BlobHeader size (> max_blob_header_size)"); } - OSMPBF::BlobHeader blob_header; - if (!blob_header.ParseFromString(read_from_input_queue(size))) { - throw osmium::pbf_error("failed to parse BlobHeader"); + return size; + } + + /** + * Decode the BlobHeader. Make sure it contains the expected + * type. Return the size of the following Blob. + */ + size_t decode_blob_header(protozero::pbf_message&& pbf_blob_header, const char* expected_type) { + std::pair blob_header_type; + size_t blob_header_datasize = 0; + + while (pbf_blob_header.next()) { + switch (pbf_blob_header.tag()) { + case FileFormat::BlobHeader::required_string_type: + blob_header_type = pbf_blob_header.get_data(); + break; + case FileFormat::BlobHeader::required_int32_datasize: + blob_header_datasize = pbf_blob_header.get_int32(); + break; + default: + pbf_blob_header.skip(); + } } - if (blob_header.type() != expected_type) { + if (blob_header_datasize == 0) { + throw osmium::pbf_error("PBF format error: BlobHeader.datasize missing or zero."); + } + + if (strncmp(expected_type, blob_header_type.first, blob_header_type.second)) { throw osmium::pbf_error("blob does not have expected type (OSMHeader in first blob, OSMData in following blobs)"); } - return static_cast(blob_header.datasize()); + return blob_header_datasize; + } + + size_t check_type_and_get_blob_size(const char* expected_type) { + assert(expected_type); + + auto size = read_blob_header_size_from_file(); + if (size == 0) { // EOF + return 0; + } + + std::string blob_header = read_from_input_queue(size); + + return decode_blob_header(protozero::pbf_message(blob_header), expected_type); } void parse_osm_data(osmium::osm_entity_bits::type read_types) { osmium::thread::set_thread_name("_osmium_pbf_in"); - int n = 0; - while (auto size = read_blob_header("OSMData")) { + + while (auto size = check_type_and_get_blob_size("OSMData")) { + std::string input_buffer = read_from_input_queue(size); + if (input_buffer.size() > max_uncompressed_blob_size) { + throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size()))); + } if (m_use_thread_pool) { - m_queue.push(osmium::thread::Pool::instance().submit(DataBlobParser{read_from_input_queue(size), read_types})); + m_queue.push(osmium::thread::Pool::instance().submit(PBFDataBlobDecoder{ std::move(input_buffer), read_types })); } else { std::promise promise; m_queue.push(promise.get_future()); - DataBlobParser data_blob_parser{read_from_input_queue(size), read_types}; + PBFDataBlobDecoder data_blob_parser{ std::move(input_buffer), read_types }; promise.set_value(data_blob_parser()); } - ++n; if (m_quit_input_thread) { return; @@ -197,11 +234,10 @@ namespace osmium { m_quit_input_thread(false), m_input_queue(input_queue), m_input_buffer() { - GOOGLE_PROTOBUF_VERIFY_VERSION; // handle OSMHeader - auto size = read_blob_header("OSMHeader"); - m_header = parse_header_blob(read_from_input_queue(size)); + const auto size = check_type_and_get_blob_size("OSMHeader"); + m_header = decode_header(read_from_input_queue(size)); if (m_read_which_entities != osmium::osm_entity_bits::nothing) { m_reader = std::thread(&PBFInputFormat::parse_osm_data, this, m_read_which_entities); @@ -246,10 +282,15 @@ namespace osmium { namespace { +// we want the register_input_format() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_pbf_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::pbf, [](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) { return new osmium::io::detail::PBFInputFormat(file, read_which_entities, input_queue); }); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/include/osmium/io/detail/pbf_output_format.hpp b/include/osmium/io/detail/pbf_output_format.hpp index 288008f6ad6..8d8a079b494 100644 --- a/include/osmium/io/detail/pbf_output_format.hpp +++ b/include/osmium/io/detail/pbf_output_format.hpp @@ -33,75 +33,13 @@ DEALINGS IN THE SOFTWARE. */ -/* - -About the .osm.pbf file format -This is an excerpt of - -The .osm.pbf format and it's derived formats (.osh.pbf and .osc.pbf) are encoded -using googles protobuf library for the low-level storage. They are constructed -by nesting data on two levels: - -On the lower level the file is constructed using BlobHeaders and Blobs. A .osm.pbf -file contains multiple sequences of - 1. a 4-byte header size, stored in network-byte-order - 2. a BlobHeader of exactly this size - 3. a Blob - -The BlobHeader tells the reader about the type and size of the following Blob. The -Blob can contain data in raw or zlib-compressed form. After uncompressing the blob -it is treated differently depending on the type specified in the BlobHeader. - -The contents of the Blob belongs to the higher level. It contains either an HeaderBlock -(type="OSMHeader") or an PrimitiveBlock (type="OSMData"). The file needs to have -at least one HeaderBlock before the first PrimitiveBlock. - -The HeaderBlock contains meta-information like the writing program or a bbox. It may -also contain multiple "required features" that describe what kinds of input a -reading program needs to handle in order to fully understand the files' contents. - -The PrimitiveBlock can store multiple types of objects (i.e. 5 nodes, 2 ways and -1 relation). It contains one or more PrimitiveGroup which in turn contain multiple -nodes, ways or relations. A PrimitiveGroup should only contain one kind of object. - -There's a special kind of "object type" called dense-nodes. It is used to store nodes -in a very dense format, avoiding message overheads and using delta-encoding for nearly -all ids. - -All Strings are stored as indexes to rows in a StringTable. The StringTable contains -one row for each used string, so strings that are used multiple times need to be -stored only once. The StringTable is sorted by usage-count, so the most often used -string is stored at index 1. - -A simple outline of a .osm.pbf file could look like this: - - 4-bytes header size - BlobHeader - Blob - HeaderBlock - 4-bytes header size - BlobHeader - Blob - PrimitiveBlock - StringTable - PrimitiveGroup - 5 nodes - PrimitiveGroup - 2 ways - PrimitiveGroup - 1 relation - -More complete outlines of real .osm.pbf files can be created using the osmpbf-outline tool: - -*/ - #include #include #include #include #include #include -#include +#include #include #include #include @@ -109,10 +47,15 @@ More complete outlines of real .osm.pbf files can be created using the osmpbf-ou #include #include +#include + +#include + #include #include #include // IWYU pragma: export -#include +#include +#include #include #include #include @@ -129,6 +72,7 @@ More complete outlines of real .osm.pbf files can be created using the osmpbf-ou #include #include #include +#include #include namespace osmium { @@ -137,815 +81,493 @@ namespace osmium { namespace detail { - namespace { - - /** - * Serialize a protobuf message into a Blob, optionally apply compression - * and return it together with a BlobHeader ready to be written to a file. - * - * @param type Type-string used in the BlobHeader. - * @param msg Protobuf-message. - * @param use_compression Should the output be compressed using zlib? - */ - std::string serialize_blob(const std::string& type, const google::protobuf::MessageLite& msg, bool use_compression) { - OSMPBF::Blob pbf_blob; - - { - std::string content; - msg.SerializeToString(&content); - - pbf_blob.set_raw_size(static_cast_with_assert<::google::protobuf::int32>(content.size())); - - if (use_compression) { - pbf_blob.set_zlib_data(osmium::io::detail::zlib_compress(content)); - } else { - pbf_blob.set_raw(content); - } - } - - std::string blob_data; - pbf_blob.SerializeToString(&blob_data); - - OSMPBF::BlobHeader pbf_blob_header; - pbf_blob_header.set_type(type); - pbf_blob_header.set_datasize(static_cast_with_assert<::google::protobuf::int32>(blob_data.size())); - - std::string blob_header_data; - pbf_blob_header.SerializeToString(&blob_header_data); - - uint32_t sz = htonl(static_cast_with_assert(blob_header_data.size())); - - // write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob - std::string output; - output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size()); - output.append(reinterpret_cast(&sz), sizeof(sz)); - output.append(blob_header_data); - output.append(blob_data); - - return output; + /** + * Maximum number of items in a primitive block. + * + * The uncompressed length of a Blob *should* be less + * than 16 megabytes and *must* be less than 32 megabytes. + * + * A block may contain any number of entities, as long as + * the size limits for the surrounding blob are obeyed. + * However, for simplicity, the current Osmosis (0.38) + * as well as Osmium implementation always + * uses at most 8k entities in a block. + */ + constexpr int32_t max_entities_per_block = 8000; + + constexpr int location_granularity = 100; + + /** + * convert a double lat or lon value to an int, respecting the granularity + */ + inline int64_t lonlat2int(double lonlat) { + return static_cast(std::round(lonlat * lonlat_resolution / location_granularity)); + } + + /** + * Serialize a protobuf message into a Blob, optionally apply compression + * and return it together with a BlobHeader ready to be written to a file. + * + * @param type Type-string used in the BlobHeader. + * @param msg Protobuf-message. + * @param use_compression Should the output be compressed using zlib? + */ + inline std::string serialize_blob(const std::string& type, const std::string& msg, bool use_compression) { + std::string blob_data; + protozero::pbf_builder pbf_blob(blob_data); + + if (use_compression) { + pbf_blob.add_int32(FileFormat::Blob::optional_int32_raw_size, msg.size()); + pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_zlib_data, osmium::io::detail::zlib_compress(msg)); + } else { + pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_raw, msg); } - } // anonymous namespace + std::string blob_header_data; + protozero::pbf_builder pbf_blob_header(blob_header_data); - class PBFOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler { + pbf_blob_header.add_string(FileFormat::BlobHeader::required_string_type, type); + pbf_blob_header.add_int32(FileFormat::BlobHeader::required_int32_datasize, blob_data.size()); - /** - * This class models a variable that keeps track of the value - * it was last set to and returns the delta between old and - * new value from the update() call. - */ - template - class Delta { + uint32_t sz = htonl(static_cast_with_assert(blob_header_data.size())); - T m_value; + // write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob + std::string output; + output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size()); + output.append(reinterpret_cast(&sz), sizeof(sz)); + output.append(blob_header_data); + output.append(blob_data); - public: + return output; + } - Delta() : - m_value(0) { - } + class DenseNodes { - void clear() { - m_value = 0; - } + StringTable& m_stringtable; - T update(T new_value) { - using std::swap; - swap(m_value, new_value); - return m_value - new_value; - } + std::vector m_ids; - }; // class Delta + std::vector m_versions; + std::vector m_timestamps; + std::vector m_changesets; + std::vector m_uids; + std::vector m_user_sids; + std::vector m_visibles; - /** - * Maximum number of items in a primitive block. - * - * The uncompressed length of a Blob *should* be less - * than 16 megabytes and *must* be less than 32 megabytes. - * - * A block may contain any number of entities, as long as - * the size limits for the surrounding blob are obeyed. - * However, for simplicity, the current Osmosis (0.38) - * as well as Osmium implementation always - * uses at most 8k entities in a block. - */ - static constexpr uint32_t max_block_contents = 8000; + std::vector m_lats; + std::vector m_lons; + std::vector m_tags; - /** - * The output buffer (block) will be filled to about - * 95% and then written to disk. This leaves more than - * enough space for the string table (which typically - * needs about 0.1 to 0.3% of the block size). - */ - static constexpr int64_t buffer_fill_percent = 95; + osmium::util::DeltaEncode m_delta_id; - /** - * protobuf-struct of a HeaderBlock - */ - OSMPBF::HeaderBlock pbf_header_block; + osmium::util::DeltaEncode m_delta_timestamp; + osmium::util::DeltaEncode m_delta_changeset; + osmium::util::DeltaEncode m_delta_uid; + osmium::util::DeltaEncode m_delta_user_sid; - /** - * protobuf-struct of a PrimitiveBlock - */ - OSMPBF::PrimitiveBlock pbf_primitive_block; + osmium::util::DeltaEncode m_delta_lat; + osmium::util::DeltaEncode m_delta_lon; - /** - * pointer to PrimitiveGroups inside the current PrimitiveBlock, - * used for writing nodes, ways or relations - */ - OSMPBF::PrimitiveGroup* pbf_nodes; - OSMPBF::PrimitiveGroup* pbf_ways; - OSMPBF::PrimitiveGroup* pbf_relations; + bool m_add_metadata; + bool m_add_visible; - /** - * To flexibly handle multiple resolutions, the granularity, or - * resolution used for representing locations is adjustable in - * multiples of 1 nanodegree. The default scaling factor is 100 - * nanodegrees, corresponding to about ~1cm at the equator. - * This is the current resolution of the OSM database. - */ - int m_location_granularity; + public: - /** - * The granularity used for representing timestamps is also adjustable in - * multiples of 1 millisecond. The default scaling factor is 1000 - * milliseconds, which is the current resolution of the OSM database. - */ - int m_date_granularity; + DenseNodes(StringTable& stringtable, bool add_metadata, bool add_visible) : + m_stringtable(stringtable), + m_add_metadata(add_metadata), + m_add_visible(add_visible) { + } - /** - * should nodes be serialized into the dense format? - * - * nodes can be encoded one of two ways, as a Node - * (m_use_dense_nodes = false) and a special dense format. - * In the dense format, all information is stored 'column wise', - * as an array of ID's, array of latitudes, and array of - * longitudes. Each column is delta-encoded. This reduces - * header overheads and allows delta-coding to work very effectively. - */ - bool m_use_dense_nodes {true}; + void clear() { + m_ids.clear(); - /** - * should the PBF blobs contain zlib compressed data? - * - * the zlib compression is optional, it's possible to store the - * blobs in raw format. Disabling the compression can improve the - * writing speed a little but the output will be 2x to 3x bigger. - */ - bool m_use_compression {true}; + m_versions.clear(); + m_timestamps.clear(); + m_changesets.clear(); + m_uids.clear(); + m_user_sids.clear(); + m_visibles.clear(); - /** - * Should the string tables in the data blocks be sorted? - * - * Not sorting the string tables makes writing PBF files - * slightly faster. - */ - bool m_sort_stringtables { true }; + m_lats.clear(); + m_lons.clear(); + m_tags.clear(); - /** - * While the .osm.pbf-format is able to carry all meta information, it is - * also able to omit this information to reduce size. - */ - bool m_should_add_metadata {true}; - - /** - * Should the visible flag be added on objects? - */ - bool m_add_visible; - - /** - * counter used to quickly check the number of objects stored inside - * the current PrimitiveBlock. When the counter reaches max_block_contents - * the PrimitiveBlock is serialized into a Blob and flushed to the file. - * - * this check is performed in check_block_contents_counter() which is - * called once for each object. - */ - uint16_t primitive_block_contents; - int primitive_block_size; + m_delta_id.clear(); - // StringTable management - StringTable string_table; + m_delta_timestamp.clear(); + m_delta_changeset.clear(); + m_delta_uid.clear(); + m_delta_user_sid.clear(); - /** - * These variables are used to calculate the - * delta-encoding while storing dense-nodes. It holds the last seen values - * from which the difference is stored into the protobuf. - */ - Delta m_delta_id; - Delta m_delta_lat; - Delta m_delta_lon; - Delta m_delta_timestamp; - Delta m_delta_changeset; - Delta m_delta_uid; - Delta<::google::protobuf::int32> m_delta_user_sid; - - bool debug; - - bool has_debug_level(int) { - return false; + m_delta_lat.clear(); + m_delta_lon.clear(); } - ///// Blob writing ///// - - void delta_encode_string_ids() { - if (pbf_nodes && pbf_nodes->has_dense()) { - OSMPBF::DenseNodes* dense = pbf_nodes->mutable_dense(); - - if (dense->has_denseinfo()) { - OSMPBF::DenseInfo* denseinfo = dense->mutable_denseinfo(); - - for (int i = 0, l=denseinfo->user_sid_size(); iuser_sid(i); - denseinfo->set_user_sid(i, m_delta_user_sid.update(user_sid)); - } - } - } + size_t size() const { + return m_ids.size() * 3 * sizeof(int64_t); } - /** - * Before a PrimitiveBlock gets serialized, all interim StringTable-ids needs to be - * mapped to the associated real StringTable ids. This is done in this function. - * - * This function needs to know about the concrete structure of all item types to find - * all occurrences of string-ids. - */ - void map_string_ids() { - // test, if the node-block has been allocated - if (pbf_nodes) { - // iterate over all nodes, passing them to the map_common_string_ids function - for (int i = 0, l=pbf_nodes->nodes_size(); imutable_nodes(i)); - } - - // test, if the node-block has a densenodes structure - if (pbf_nodes->has_dense()) { - // get a pointer to the densenodes structure - OSMPBF::DenseNodes* dense = pbf_nodes->mutable_dense(); - - // in the densenodes structure keys and vals are encoded in an intermixed - // array, individual nodes are seperated by a value of 0 (0 in the StringTable - // is always unused). String-ids of 0 are thus kept alone. - for (int i = 0, l=dense->keys_vals_size(); i 0 to real string ids - auto sid = dense->keys_vals(i); - if (sid > 0) { - dense->set_keys_vals(i, string_table.map_string_id(sid)); - } - } - - // test if the densenodes block has meta infos - if (dense->has_denseinfo()) { - // get a pointer to the denseinfo structure - OSMPBF::DenseInfo* denseinfo = dense->mutable_denseinfo(); - - // iterate over all username string-ids - for (int i = 0, l=denseinfo->user_sid_size(); i 0 to real string ids - auto user_sid = string_table.map_string_id(denseinfo->user_sid(i)); - - // delta encode the string-id - denseinfo->set_user_sid(i, m_delta_user_sid.update(user_sid)); - } - } - } - } - - // test, if the ways-block has been allocated - if (pbf_ways) { - // iterate over all ways, passing them to the map_common_string_ids function - for (int i = 0, l=pbf_ways->ways_size(); imutable_ways(i)); - } - } + void add_node(const osmium::Node& node) { + m_ids.push_back(m_delta_id.update(node.id())); - // test, if the relations-block has been allocated - if (pbf_relations) { - // iterate over all relations - for (int i = 0, l=pbf_relations->relations_size(); imutable_relations(i); - - // pass them to the map_common_string_ids function - map_common_string_ids(relation); - - // iterate over all relation members, mapping the interim string-ids - // of the role to real string ids - for (int mi = 0; mi < relation->roles_sid_size(); ++mi) { - relation->set_roles_sid(mi, string_table.map_string_id(relation->roles_sid(mi))); - } + if (m_add_metadata) { + m_versions.push_back(node.version()); + m_timestamps.push_back(m_delta_timestamp.update(node.timestamp())); + m_changesets.push_back(m_delta_changeset.update(node.changeset())); + m_uids.push_back(m_delta_uid.update(node.uid())); + m_user_sids.push_back(m_delta_user_sid.update(m_stringtable.add(node.user()))); + if (m_add_visible) { + m_visibles.push_back(node.visible()); } } - } - /** - * a helper function used in map_string_ids to map common interim string-ids of the - * user name and all tags to real string ids. - * - * TPBFObject is either OSMPBF::Node, OSMPBF::Way or OSMPBF::Relation. - */ - template - void map_common_string_ids(TPBFObject* in) { - // if the object has meta-info attached - if (in->has_info()) { - // map the interim-id of the user name to a real id - OSMPBF::Info* info = in->mutable_info(); - info->set_user_sid(string_table.map_string_id(info->user_sid())); - } + m_lats.push_back(m_delta_lat.update(lonlat2int(node.location().lat_without_check()))); + m_lons.push_back(m_delta_lon.update(lonlat2int(node.location().lon_without_check()))); - // iterate over all tags and map the interim-ids of the key and the value to real ids - for (int i = 0, l=in->keys_size(); iset_keys(i, string_table.map_string_id(in->keys(i))); - in->set_vals(i, string_table.map_string_id(in->vals(i))); + for (const auto& tag : node.tags()) { + m_tags.push_back(m_stringtable.add(tag.key())); + m_tags.push_back(m_stringtable.add(tag.value())); } + m_tags.push_back(0); } + std::string serialize() const { + std::string data; + protozero::pbf_builder pbf_dense_nodes(data); - ///// MetaData helper ///// - - /** - * convert a double lat or lon value to an int, respecting the current blocks granularity - */ - int64_t lonlat2int(double lonlat) { - return static_cast(std::round(lonlat * OSMPBF::lonlat_resolution / location_granularity())); - } - - /** - * convert a timestamp to an int, respecting the current blocks granularity - */ - int64_t timestamp2int(time_t timestamp) { - return static_cast(std::round(timestamp * (1000.0 / date_granularity()))); - } + pbf_dense_nodes.add_packed_sint64(OSMFormat::DenseNodes::packed_sint64_id, m_ids.cbegin(), m_ids.cend()); - /** - * helper function used in the write()-calls to apply common information from an osmium-object - * onto a pbf-object. - * - * TPBFObject is either OSMPBF::Node, OSMPBF::Way or OSMPBF::Relation. - */ - template - void apply_common_info(const osmium::OSMObject& in, TPBFObject* out) { - // set the object-id - out->set_id(in.id()); - - // iterate over all tags and set the keys and vals, recording the strings in the - // interim StringTable and storing the interim ids - for (const auto& tag : in.tags()) { - out->add_keys(string_table.record_string(tag.key())); - out->add_vals(string_table.record_string(tag.value())); - } + if (m_add_metadata) { + protozero::pbf_builder pbf_dense_info(pbf_dense_nodes, OSMFormat::DenseNodes::optional_DenseInfo_denseinfo); + pbf_dense_info.add_packed_int32(OSMFormat::DenseInfo::packed_int32_version, m_versions.cbegin(), m_versions.cend()); + pbf_dense_info.add_packed_sint64(OSMFormat::DenseInfo::packed_sint64_timestamp, m_timestamps.cbegin(), m_timestamps.cend()); + pbf_dense_info.add_packed_sint64(OSMFormat::DenseInfo::packed_sint64_changeset, m_changesets.cbegin(), m_changesets.cend()); + pbf_dense_info.add_packed_sint32(OSMFormat::DenseInfo::packed_sint32_uid, m_uids.cbegin(), m_uids.cend()); + pbf_dense_info.add_packed_sint32(OSMFormat::DenseInfo::packed_sint32_user_sid, m_user_sids.cbegin(), m_user_sids.cend()); - if (m_should_add_metadata) { - // add an info-section to the pbf object and set the meta-info on it - OSMPBF::Info* out_info = out->mutable_info(); if (m_add_visible) { - out_info->set_visible(in.visible()); + pbf_dense_info.add_packed_bool(OSMFormat::DenseInfo::packed_bool_visible, m_visibles.cbegin(), m_visibles.cend()); } - out_info->set_version(static_cast<::google::protobuf::int32>(in.version())); - out_info->set_timestamp(timestamp2int(in.timestamp())); - out_info->set_changeset(in.changeset()); - out_info->set_uid(static_cast<::google::protobuf::int32>(in.uid())); - out_info->set_user_sid(string_table.record_string(in.user())); } + + pbf_dense_nodes.add_packed_sint64(OSMFormat::DenseNodes::packed_sint64_lat, m_lats.cbegin(), m_lats.cend()); + pbf_dense_nodes.add_packed_sint64(OSMFormat::DenseNodes::packed_sint64_lon, m_lons.cbegin(), m_lons.cend()); + + pbf_dense_nodes.add_packed_int32(OSMFormat::DenseNodes::packed_int32_keys_vals, m_tags.cbegin(), m_tags.cend()); + + return data; } + }; // class DenseNodes - ///// High-Level Block writing ///// + class PrimitiveBlock { - /** - * store the current pbf_header_block into a Blob and clear this struct afterwards. - */ - void store_header_block() { - if (debug && has_debug_level(1)) { - std::cerr << "storing header block" << std::endl; - } + std::string m_pbf_primitive_group_data; + protozero::pbf_builder m_pbf_primitive_group; + StringTable m_stringtable; + DenseNodes m_dense_nodes; + OSMFormat::PrimitiveGroup m_type; + int m_count; - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(serialize_blob("OSMHeader", pbf_header_block, m_use_compression)); + public: - pbf_header_block.Clear(); + PrimitiveBlock(bool add_metadata, bool add_visible) : + m_pbf_primitive_group_data(), + m_pbf_primitive_group(m_pbf_primitive_group_data), + m_stringtable(), + m_dense_nodes(m_stringtable, add_metadata, add_visible), + m_type(OSMFormat::PrimitiveGroup::unknown), + m_count(0) { } - /** - * store the interim StringTable to the current pbf_primitive_block, map all interim string ids - * to real StringTable ids and then store the current pbf_primitive_block into a Blob and clear - * this struct and all related pointers and maps afterwards. - */ - void store_primitive_block() { - if (debug && has_debug_level(1)) { - std::cerr << "storing primitive block with " << primitive_block_contents << " items" << std::endl; + const std::string& group_data() { + if (type() == OSMFormat::PrimitiveGroup::optional_DenseNodes_dense) { + m_pbf_primitive_group.add_message(OSMFormat::PrimitiveGroup::optional_DenseNodes_dense, m_dense_nodes.serialize()); } + return m_pbf_primitive_group_data; + } - // set the granularity - pbf_primitive_block.set_granularity(location_granularity()); - pbf_primitive_block.set_date_granularity(date_granularity()); - - string_table.store_stringtable(pbf_primitive_block.mutable_stringtable(), m_sort_stringtables); + void reset(OSMFormat::PrimitiveGroup type) { + m_pbf_primitive_group_data.clear(); + m_stringtable.clear(); + m_dense_nodes.clear(); + m_type = type; + m_count = 0; + } - if (m_sort_stringtables) { - map_string_ids(); - } else { - delta_encode_string_ids(); + void write_stringtable(protozero::pbf_builder& pbf_string_table) { + for (const char* s : m_stringtable) { + pbf_string_table.add_bytes(OSMFormat::StringTable::repeated_bytes_s, s); } + } - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(serialize_blob("OSMData", pbf_primitive_block, m_use_compression)); + protozero::pbf_builder& group() { + ++m_count; + return m_pbf_primitive_group; + } - // clear the PrimitiveBlock struct - pbf_primitive_block.Clear(); + void add_dense_node(const osmium::Node& node) { + m_dense_nodes.add_node(node); + ++m_count; + } - // clear the interim StringTable and its id map - string_table.clear(); + size_t add_string(const char* s) { + return m_stringtable.add(s); + } - // reset the delta variables - m_delta_id.clear(); - m_delta_lat.clear(); - m_delta_lon.clear(); - m_delta_timestamp.clear(); - m_delta_changeset.clear(); - m_delta_uid.clear(); - m_delta_user_sid.clear(); + int count() const { + return m_count; + } - // reset the contents-counter to zero - primitive_block_contents = 0; - primitive_block_size = 0; + OSMFormat::PrimitiveGroup type() const { + return m_type; + } - // reset the node/way/relation pointers to nullptr - pbf_nodes = nullptr; - pbf_ways = nullptr; - pbf_relations = nullptr; + size_t size() const { + return m_pbf_primitive_group_data.size() + m_stringtable.size() + m_dense_nodes.size(); } /** - * this little function checks primitive_block_contents counter against its maximum and calls - * store_primitive_block to flush the block to the disk when it's reached. It's also responsible - * for increasing this counter. - * - * this function also checks the estimated size of the current block and calls store_primitive_block - * when the estimated size reaches buffer_fill_percent of the maximum uncompressed blob size. + * The output buffer (block) will be filled to about + * 95% and then written to disk. This leaves more than + * enough space for the string table (which typically + * needs about 0.1 to 0.3% of the block size). */ - void check_block_contents_counter() { - if (primitive_block_contents >= max_block_contents) { - store_primitive_block(); - } else if (primitive_block_size > OSMPBF::max_uncompressed_blob_size * buffer_fill_percent / 100) { - if (debug && has_debug_level(1)) { - std::cerr << "storing primitive_block with only " << primitive_block_contents << " items, because its ByteSize (" << primitive_block_size << ") reached " << - (static_cast(primitive_block_size) / static_cast(OSMPBF::max_uncompressed_blob_size) * 100.0) << "% of the maximum blob-size" << std::endl; - } + constexpr static size_t max_used_blob_size = max_uncompressed_blob_size * 95 / 100; - store_primitive_block(); + bool can_add(OSMFormat::PrimitiveGroup type) const { + if (type != m_type) { + return false; } - - ++primitive_block_contents; + if (count() >= max_entities_per_block) { + return false; + } + return size() < max_used_blob_size; } + }; // class PrimitiveBlock - ///// Block content writing ///// - - /** - * Add a node to the block. - * - * @param node The node to add. - */ - void write_node(const osmium::Node& node) { - // add a way to the group - OSMPBF::Node* pbf_node = pbf_nodes->add_nodes(); - - // copy the common meta-info from the osmium-object to the pbf-object - apply_common_info(node, pbf_node); + class PBFOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler { - // modify lat & lon to integers, respecting the block's granularity and copy - // the ints to the pbf-object - pbf_node->set_lon(lonlat2int(node.location().lon_without_check())); - pbf_node->set_lat(lonlat2int(node.location().lat_without_check())); - } + /// Should nodes be encoded in DenseNodes? + bool m_use_dense_nodes; /** - * Add a node to the block using DenseNodes. + * Should the PBF blobs contain zlib compressed data? * - * @param node The node to add. + * The zlib compression is optional, it's possible to store the + * blobs in raw format. Disabling the compression can improve + * the writing speed a little but the output will be 2x to 3x + * bigger. */ - void write_dense_node(const osmium::Node& node) { - // add a DenseNodes-Section to the PrimitiveGroup - OSMPBF::DenseNodes* dense = pbf_nodes->mutable_dense(); + bool m_use_compression; - // copy the id, delta encoded - dense->add_id(m_delta_id.update(node.id())); + /// Should metadata of objects be written? + bool m_add_metadata; - // copy the longitude, delta encoded - dense->add_lon(m_delta_lon.update(lonlat2int(node.location().lon_without_check()))); + /// Should the visible flag be added to objects? + bool m_add_visible; - // copy the latitude, delta encoded - dense->add_lat(m_delta_lat.update(lonlat2int(node.location().lat_without_check()))); + PrimitiveBlock m_primitive_block; - // in the densenodes structure keys and vals are encoded in an intermixed - // array, individual nodes are seperated by a value of 0 (0 in the StringTable - // is always unused) - // so for three nodes the keys_vals array may look like this: 3 5 2 1 0 0 8 5 - // the first node has two tags (3=>5 and 2=>1), the second node does not - // have any tags and the third node has a single tag (8=>5) - for (const auto& tag : node.tags()) { - dense->add_keys_vals(string_table.record_string(tag.key())); - dense->add_keys_vals(string_table.record_string(tag.value())); + void store_primitive_block() { + if (m_primitive_block.count() == 0) { + return; } - dense->add_keys_vals(0); - - if (m_should_add_metadata) { - // add a DenseInfo-Section to the PrimitiveGroup - OSMPBF::DenseInfo* denseinfo = dense->mutable_denseinfo(); - - denseinfo->add_version(static_cast<::google::protobuf::int32>(node.version())); - - if (m_add_visible) { - denseinfo->add_visible(node.visible()); - } - - // copy the timestamp, delta encoded - denseinfo->add_timestamp(m_delta_timestamp.update(timestamp2int(node.timestamp()))); - // copy the changeset, delta encoded - denseinfo->add_changeset(m_delta_changeset.update(node.changeset())); + std::string primitive_block_data; + protozero::pbf_builder primitive_block(primitive_block_data); - // copy the user id, delta encoded - denseinfo->add_uid(static_cast<::google::protobuf::int32>(m_delta_uid.update(node.uid()))); - - // record the user-name to the interim stringtable and copy the - // interim string-id to the pbf-object - denseinfo->add_user_sid(string_table.record_string(node.user())); + { + protozero::pbf_builder pbf_string_table(primitive_block, OSMFormat::PrimitiveBlock::required_StringTable_stringtable); + m_primitive_block.write_stringtable(pbf_string_table); } - } - - /** - * Add a way to the block. - * - * @param way The way to add. - */ - void write_way(const osmium::Way& way) { - // add a way to the group - OSMPBF::Way* pbf_way = pbf_ways->add_ways(); - - // copy the common meta-info from the osmium-object to the pbf-object - apply_common_info(way, pbf_way); - // last way-node-id used for delta-encoding - Delta delta_id; + primitive_block.add_message(OSMFormat::PrimitiveBlock::repeated_PrimitiveGroup_primitivegroup, m_primitive_block.group_data()); - for (const auto& node_ref : way.nodes()) { - // copy the way-node-id, delta encoded - pbf_way->add_refs(delta_id.update(node_ref.ref())); - } - - // count up blob size by the size of the Way - primitive_block_size += pbf_way->ByteSize(); + std::promise promise; + m_output_queue.push(promise.get_future()); + promise.set_value(serialize_blob("OSMData", primitive_block_data, m_use_compression)); } - /** - * Add a relation to the block. - * - * @param relation The relation to add. - */ - void write_relation(const osmium::Relation& relation) { - // add a relation to the group - OSMPBF::Relation* pbf_relation = pbf_relations->add_relations(); - - // copy the common meta-info from the osmium-object to the pbf-object - apply_common_info(relation, pbf_relation); - - Delta delta_id; - - for (const auto& member : relation.members()) { - // record the relation-member role to the interim stringtable and copy the - // interim string-id to the pbf-object - pbf_relation->add_roles_sid(string_table.record_string(member.role())); - - // copy the relation-member-id, delta encoded - pbf_relation->add_memids(delta_id.update(member.ref())); - - // copy the relation-member-type, mapped to the OSMPBF enum - pbf_relation->add_types(item_type_to_osmpbf_membertype(member.type())); + template + void add_meta(const osmium::OSMObject& object, T& pbf_object) { + const osmium::TagList& tags = object.tags(); + + auto map_tag_key = [this](const osmium::Tag& tag) -> size_t { + return m_primitive_block.add_string(tag.key()); + }; + auto map_tag_value = [this](const osmium::Tag& tag) -> size_t { + return m_primitive_block.add_string(tag.value()); + }; + + pbf_object.add_packed_uint32(T::enum_type::packed_uint32_keys, + boost::make_transform_iterator(tags.begin(), map_tag_key), + boost::make_transform_iterator(tags.end(), map_tag_key)); + + pbf_object.add_packed_uint32(T::enum_type::packed_uint32_vals, + boost::make_transform_iterator(tags.begin(), map_tag_value), + boost::make_transform_iterator(tags.end(), map_tag_value)); + + if (m_add_metadata) { + protozero::pbf_builder pbf_info(pbf_object, T::enum_type::optional_Info_info); + + pbf_info.add_int32(OSMFormat::Info::optional_int32_version, object.version()); + pbf_info.add_int64(OSMFormat::Info::optional_int64_timestamp, object.timestamp()); + pbf_info.add_int64(OSMFormat::Info::optional_int64_changeset, object.changeset()); + pbf_info.add_int32(OSMFormat::Info::optional_int32_uid, object.uid()); + pbf_info.add_uint32(OSMFormat::Info::optional_uint32_user_sid, m_primitive_block.add_string(object.user())); + if (m_add_visible) { + pbf_info.add_bool(OSMFormat::Info::optional_bool_visible, object.visible()); + } } - - // count up blob size by the size of the Relation - primitive_block_size += pbf_relation->ByteSize(); } - // objects of this class can't be copied PBFOutputFormat(const PBFOutputFormat&) = delete; PBFOutputFormat& operator=(const PBFOutputFormat&) = delete; public: - /** - * Create PBFOutputFormat object from File. - */ explicit PBFOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : OutputFormat(file, output_queue), - pbf_header_block(), - pbf_primitive_block(), - pbf_nodes(nullptr), - pbf_ways(nullptr), - pbf_relations(nullptr), - m_location_granularity(pbf_primitive_block.granularity()), - m_date_granularity(pbf_primitive_block.date_granularity()), + m_use_dense_nodes(file.get("pbf_dense_nodes") != "false"), + m_use_compression(file.get("pbf_compression") != "none" && file.get("pbf_compression") != "false"), + m_add_metadata(file.get("pbf_add_metadata") != "false" && file.get("add_metadata") != "false"), m_add_visible(file.has_multiple_object_versions()), - primitive_block_contents(0), - primitive_block_size(0), - string_table(), - m_delta_id(), - m_delta_lat(), - m_delta_lon(), - m_delta_timestamp(), - m_delta_changeset(), - m_delta_uid(), - m_delta_user_sid(), - debug(true) { - GOOGLE_PROTOBUF_VERIFY_VERSION; - if (file.get("pbf_dense_nodes") == "false") { - m_use_dense_nodes = false; - } - if (file.get("pbf_compression") == "none" || file.get("pbf_compression") == "false") { - m_use_compression = false; - } - if (file.get("pbf_sort_stringtables") == "false") { - m_sort_stringtables = false; - } - if (file.get("pbf_add_metadata") == "false") { - m_should_add_metadata = false; - } + m_primitive_block(m_add_metadata, m_add_visible) { } void write_buffer(osmium::memory::Buffer&& buffer) override final { osmium::apply(buffer.cbegin(), buffer.cend(), *this); } + void write_header(const osmium::io::Header& header) override final { + std::string data; + protozero::pbf_builder pbf_header_block(data); - /** - * getter to access the granularity - */ - int location_granularity() const { - return m_location_granularity; - } - - /** - * setter to set the granularity - */ - PBFOutputFormat& location_granularity(int g) { - m_location_granularity = g; - return *this; - } - - - /** - * getter to access the date_granularity - */ - int date_granularity() const { - return m_date_granularity; - } - - /** - * Set date granularity. - */ - PBFOutputFormat& date_granularity(int g) { - m_date_granularity = g; - return *this; - } + if (!header.boxes().empty()) { + protozero::pbf_builder pbf_header_bbox(pbf_header_block, OSMFormat::HeaderBlock::optional_HeaderBBox_bbox); + osmium::Box box = header.joined_boxes(); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_left, box.bottom_left().lon() * lonlat_resolution); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_right, box.top_right().lon() * lonlat_resolution); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_top, box.top_right().lat() * lonlat_resolution); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_bottom, box.bottom_left().lat() * lonlat_resolution); + } - /** - * Initialize the writing process. - * - * This initializes the header-block, sets the required-features and - * the writing-program and adds the obligatory StringTable-Index 0. - */ - void write_header(const osmium::io::Header& header) override final { - // add the schema version as required feature to the HeaderBlock - pbf_header_block.add_required_features("OsmSchema-V0.6"); + pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "OsmSchema-V0.6"); - // when the densenodes-feature is used, add DenseNodes as required feature if (m_use_dense_nodes) { - pbf_header_block.add_required_features("DenseNodes"); + pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "DenseNodes"); } - // when the resulting file will carry history information, add - // HistoricalInformation as required feature if (m_file.has_multiple_object_versions()) { - pbf_header_block.add_required_features("HistoricalInformation"); + pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "HistoricalInformation"); } - // set the writing program - pbf_header_block.set_writingprogram(header.get("generator")); - - if (!header.boxes().empty()) { - OSMPBF::HeaderBBox* pbf_bbox = pbf_header_block.mutable_bbox(); - osmium::Box box = header.joined_boxes(); - pbf_bbox->set_left(static_cast<::google::protobuf::int64>(box.bottom_left().lon() * OSMPBF::lonlat_resolution)); - pbf_bbox->set_bottom(static_cast<::google::protobuf::int64>(box.bottom_left().lat() * OSMPBF::lonlat_resolution)); - pbf_bbox->set_right(static_cast<::google::protobuf::int64>(box.top_right().lon() * OSMPBF::lonlat_resolution)); - pbf_bbox->set_top(static_cast<::google::protobuf::int64>(box.top_right().lat() * OSMPBF::lonlat_resolution)); - } + pbf_header_block.add_string(OSMFormat::HeaderBlock::optional_string_writingprogram, header.get("generator")); std::string osmosis_replication_timestamp = header.get("osmosis_replication_timestamp"); if (!osmosis_replication_timestamp.empty()) { osmium::Timestamp ts(osmosis_replication_timestamp.c_str()); - pbf_header_block.set_osmosis_replication_timestamp(ts); + pbf_header_block.add_int64(OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp, ts); } std::string osmosis_replication_sequence_number = header.get("osmosis_replication_sequence_number"); if (!osmosis_replication_sequence_number.empty()) { - pbf_header_block.set_osmosis_replication_sequence_number(std::atoll(osmosis_replication_sequence_number.c_str())); + pbf_header_block.add_int64(OSMFormat::HeaderBlock::optional_int64_osmosis_replication_sequence_number, std::atoll(osmosis_replication_sequence_number.c_str())); } std::string osmosis_replication_base_url = header.get("osmosis_replication_base_url"); if (!osmosis_replication_base_url.empty()) { - pbf_header_block.set_osmosis_replication_base_url(osmosis_replication_base_url); + pbf_header_block.add_string(OSMFormat::HeaderBlock::optional_string_osmosis_replication_base_url, osmosis_replication_base_url); } - store_header_block(); + std::promise promise; + m_output_queue.push(promise.get_future()); + promise.set_value(serialize_blob("OSMHeader", data, m_use_compression)); } - /** - * Add a node to the pbf. - * - * A call to this method won't write the node to the file directly but - * cache it for later bulk-writing. Calling final() ensures that everything - * gets written and every file pointer is closed. - */ - void node(const osmium::Node& node) { - // first of we check the contents-counter which may flush the cached nodes to - // disk if the limit is reached. This call also increases the contents-counter - check_block_contents_counter(); - - if (debug && has_debug_level(2)) { - std::cerr << "node " << node.id() << " v" << node.version() << std::endl; - } - - // if no PrimitiveGroup for nodes has been added, add one and save the pointer - if (!pbf_nodes) { - pbf_nodes = pbf_primitive_block.add_primitivegroup(); + void switch_primitive_block_type(OSMFormat::PrimitiveGroup type) { + if (!m_primitive_block.can_add(type)) { + store_primitive_block(); + m_primitive_block.reset(type); } + } + void node(const osmium::Node& node) { if (m_use_dense_nodes) { - write_dense_node(node); - } else { - write_node(node); + switch_primitive_block_type(OSMFormat::PrimitiveGroup::optional_DenseNodes_dense); + m_primitive_block.add_dense_node(node); + return; } + + switch_primitive_block_type(OSMFormat::PrimitiveGroup::repeated_Node_nodes); + protozero::pbf_builder pbf_node{ m_primitive_block.group(), OSMFormat::PrimitiveGroup::repeated_Node_nodes }; + + pbf_node.add_sint64(OSMFormat::Node::required_sint64_id, node.id()); + add_meta(node, pbf_node); + + pbf_node.add_sint64(OSMFormat::Node::required_sint64_lat, lonlat2int(node.location().lat_without_check())); + pbf_node.add_sint64(OSMFormat::Node::required_sint64_lon, lonlat2int(node.location().lon_without_check())); } - /** - * Add a way to the pbf. - * - * A call to this method won't write the way to the file directly but - * cache it for later bulk-writing. Calling final() ensures that everything - * gets written and every file pointer is closed. - */ void way(const osmium::Way& way) { - // first of we check the contents-counter which may flush the cached ways to - // disk if the limit is reached. This call also increases the contents-counter - check_block_contents_counter(); + switch_primitive_block_type(OSMFormat::PrimitiveGroup::repeated_Way_ways); + protozero::pbf_builder pbf_way{ m_primitive_block.group(), OSMFormat::PrimitiveGroup::repeated_Way_ways }; - // if no PrimitiveGroup for nodes has been added, add one and save the pointer - if (!pbf_ways) { - pbf_ways = pbf_primitive_block.add_primitivegroup(); - } + pbf_way.add_int64(OSMFormat::Way::required_int64_id, way.id()); + add_meta(way, pbf_way); + + static auto map_node_ref = [](osmium::NodeRefList::const_iterator node_ref) noexcept -> osmium::object_id_type { + return node_ref->ref(); + }; + typedef osmium::util::DeltaEncodeIterator it_type; - write_way(way); + const auto& nodes = way.nodes(); + it_type first { nodes.cbegin(), nodes.cend(), map_node_ref }; + it_type last { nodes.cend(), nodes.cend(), map_node_ref }; + pbf_way.add_packed_sint64(OSMFormat::Way::packed_sint64_refs, first, last); } - /** - * Add a relation to the pbf. - * - * A call to this method won't write the way to the file directly but - * cache it for later bulk-writing. Calling final() ensures that everything - * gets written and every file pointer is closed. - */ void relation(const osmium::Relation& relation) { - // first of we check the contents-counter which may flush the cached relations to - // disk if the limit is reached. This call also increases the contents-counter - check_block_contents_counter(); - - // if no PrimitiveGroup for relations has been added, add one and save the pointer - if (!pbf_relations) { - pbf_relations = pbf_primitive_block.add_primitivegroup(); - } - - write_relation(relation); + switch_primitive_block_type(OSMFormat::PrimitiveGroup::repeated_Relation_relations); + protozero::pbf_builder pbf_relation { m_primitive_block.group(), OSMFormat::PrimitiveGroup::repeated_Relation_relations }; + + pbf_relation.add_int64(OSMFormat::Relation::required_int64_id, relation.id()); + add_meta(relation, pbf_relation); + + auto map_member_role = [this](const osmium::RelationMember& member) -> size_t { + return m_primitive_block.add_string(member.role()); + }; + pbf_relation.add_packed_int32(OSMFormat::Relation::packed_int32_roles_sid, + boost::make_transform_iterator(relation.members().begin(), map_member_role), + boost::make_transform_iterator(relation.members().end(), map_member_role)); + + static auto map_member_ref = [](osmium::RelationMemberList::const_iterator member) noexcept -> osmium::object_id_type { + return member->ref(); + }; + typedef osmium::util::DeltaEncodeIterator it_type; + const auto& members = relation.members(); + it_type first { members.cbegin(), members.cend(), map_member_ref }; + it_type last { members.cend(), members.cend(), map_member_ref }; + pbf_relation.add_packed_sint64(OSMFormat::Relation::packed_sint64_memids, first, last); + + static auto map_member_type = [](const osmium::RelationMember& member) noexcept -> int { + return osmium::item_type_to_nwr_index(member.type()); + }; + pbf_relation.add_packed_int32(OSMFormat::Relation::packed_MemberType_types, + boost::make_transform_iterator(relation.members().begin(), map_member_type), + boost::make_transform_iterator(relation.members().end(), map_member_type)); } /** - * Finalize the writing process, flush any open primitive blocks to the file and - * close the file. + * Finalize the writing process, flush any open primitive + * blocks to the file and close the file. */ void close() override final { - if (debug && has_debug_level(1)) { - std::cerr << "finishing" << std::endl; - } - - // if the current block contains any elements, flush it to the protobuf - if (primitive_block_contents > 0) { - store_primitive_block(); - } + store_primitive_block(); std::promise promise; m_output_queue.push(promise.get_future()); @@ -956,10 +578,15 @@ namespace osmium { namespace { +// we want the register_output_format() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_pbf_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::pbf, [](const osmium::io::File& file, data_queue_type& output_queue) { return new osmium::io::detail::PBFOutputFormat(file, output_queue); }); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/include/osmium/io/detail/pbf_parser.hpp b/include/osmium/io/detail/pbf_parser.hpp deleted file mode 100644 index 65a11e15581..00000000000 --- a/include/osmium/io/detail/pbf_parser.hpp +++ /dev/null @@ -1,455 +0,0 @@ -#ifndef OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP -#define OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP - -/* - -This file is part of Osmium (http://osmcode.org/libosmium). - -Copyright 2013-2015 Jochen Topf and others (see README). - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -*/ - -#include -#include -#include -#include - -#include - -#include -#include // IWYU pragma: export -#include -#include -#include -#include -#include -#include -#include -#include - -namespace osmium { - - namespace io { - - namespace detail { - - class PBFPrimitiveBlockParser { - - static constexpr size_t initial_buffer_size = 2 * 1024 * 1024; - - const std::string& m_data; - - const OSMPBF::StringTable* m_stringtable; - int64_t m_lon_offset; - int64_t m_lat_offset; - int64_t m_date_factor; - int32_t m_granularity; - - osmium::osm_entity_bits::type m_read_types; - - osmium::memory::Buffer m_buffer; - - PBFPrimitiveBlockParser(const PBFPrimitiveBlockParser&) = delete; - PBFPrimitiveBlockParser(PBFPrimitiveBlockParser&&) = delete; - - PBFPrimitiveBlockParser& operator=(const PBFPrimitiveBlockParser&) = delete; - PBFPrimitiveBlockParser& operator=(PBFPrimitiveBlockParser&&) = delete; - - public: - - explicit PBFPrimitiveBlockParser(const std::string& data, osmium::osm_entity_bits::type read_types) : - m_data(data), - m_stringtable(nullptr), - m_lon_offset(0), - m_lat_offset(0), - m_date_factor(1000), - m_granularity(100), - m_read_types(read_types), - m_buffer(initial_buffer_size) { - } - - ~PBFPrimitiveBlockParser() = default; - - osmium::memory::Buffer operator()() { - OSMPBF::PrimitiveBlock pbf_primitive_block; - if (!pbf_primitive_block.ParseFromString(m_data)) { - throw osmium::pbf_error("failed to parse PrimitiveBlock"); - } - - m_stringtable = &pbf_primitive_block.stringtable(); - m_lon_offset = pbf_primitive_block.lon_offset(); - m_lat_offset = pbf_primitive_block.lat_offset(); - m_date_factor = pbf_primitive_block.date_granularity() / 1000; - m_granularity = pbf_primitive_block.granularity(); - - for (int i = 0; i < pbf_primitive_block.primitivegroup_size(); ++i) { - const OSMPBF::PrimitiveGroup& group = pbf_primitive_block.primitivegroup(i); - - if (group.has_dense()) { - if (m_read_types & osmium::osm_entity_bits::node) parse_dense_node_group(group); - } else if (group.ways_size() != 0) { - if (m_read_types & osmium::osm_entity_bits::way) parse_way_group(group); - } else if (group.relations_size() != 0) { - if (m_read_types & osmium::osm_entity_bits::relation) parse_relation_group(group); - } else if (group.nodes_size() != 0) { - if (m_read_types & osmium::osm_entity_bits::node) parse_node_group(group); - } else { - throw osmium::pbf_error("group of unknown type"); - } - } - - return std::move(m_buffer); - } - - private: - - template - void parse_attributes(TBuilder& builder, const TPBFObject& pbf_object) { - auto& object = builder.object(); - - object.set_id(pbf_object.id()); - - if (pbf_object.has_info()) { - object.set_version(static_cast_with_assert(pbf_object.info().version())) - .set_changeset(static_cast_with_assert(pbf_object.info().changeset())) - .set_timestamp(pbf_object.info().timestamp() * m_date_factor) - .set_uid_from_signed(pbf_object.info().uid()); - if (pbf_object.info().has_visible()) { - object.set_visible(pbf_object.info().visible()); - } - builder.add_user(m_stringtable->s(static_cast_with_assert(pbf_object.info().user_sid()))); - } else { - builder.add_user("", 1); - } - } - - void parse_node_group(const OSMPBF::PrimitiveGroup& group) { - for (int i = 0; i < group.nodes_size(); ++i) { - osmium::builder::NodeBuilder builder(m_buffer); - const OSMPBF::Node& pbf_node = group.nodes(i); - parse_attributes(builder, pbf_node); - - if (builder.object().visible()) { - builder.object().set_location(osmium::Location( - (pbf_node.lon() * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision), - (pbf_node.lat() * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision))); - } - - if (pbf_node.keys_size() > 0) { - osmium::builder::TagListBuilder tl_builder(m_buffer, &builder); - for (int tag = 0; tag < pbf_node.keys_size(); ++tag) { - tl_builder.add_tag(m_stringtable->s(static_cast(pbf_node.keys(tag))), - m_stringtable->s(static_cast(pbf_node.vals(tag)))); - } - } - - m_buffer.commit(); - } - } - - void parse_way_group(const OSMPBF::PrimitiveGroup& group) { - for (int i = 0; i < group.ways_size(); ++i) { - osmium::builder::WayBuilder builder(m_buffer); - const OSMPBF::Way& pbf_way = group.ways(i); - parse_attributes(builder, pbf_way); - - if (pbf_way.refs_size() > 0) { - osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder); - int64_t ref = 0; - for (int n = 0; n < pbf_way.refs_size(); ++n) { - ref += pbf_way.refs(n); - wnl_builder.add_node_ref(ref); - } - } - - if (pbf_way.keys_size() > 0) { - osmium::builder::TagListBuilder tl_builder(m_buffer, &builder); - for (int tag = 0; tag < pbf_way.keys_size(); ++tag) { - tl_builder.add_tag(m_stringtable->s(static_cast(pbf_way.keys(tag))), - m_stringtable->s(static_cast(pbf_way.vals(tag)))); - } - } - - m_buffer.commit(); - } - } - - void parse_relation_group(const OSMPBF::PrimitiveGroup& group) { - for (int i = 0; i < group.relations_size(); ++i) { - osmium::builder::RelationBuilder builder(m_buffer); - const OSMPBF::Relation& pbf_relation = group.relations(i); - parse_attributes(builder, pbf_relation); - - if (pbf_relation.types_size() > 0) { - osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder); - int64_t ref = 0; - for (int n = 0; n < pbf_relation.types_size(); ++n) { - ref += pbf_relation.memids(n); - rml_builder.add_member(osmpbf_membertype_to_item_type(pbf_relation.types(n)), ref, m_stringtable->s(pbf_relation.roles_sid(n))); - } - } - - if (pbf_relation.keys_size() > 0) { - osmium::builder::TagListBuilder tl_builder(m_buffer, &builder); - for (int tag = 0; tag < pbf_relation.keys_size(); ++tag) { - tl_builder.add_tag(m_stringtable->s(static_cast(pbf_relation.keys(tag))), - m_stringtable->s(static_cast(pbf_relation.vals(tag)))); - } - } - - m_buffer.commit(); - } - } - - int add_tags(const OSMPBF::DenseNodes& dense, int n, osmium::builder::NodeBuilder* builder) { - if (n >= dense.keys_vals_size()) { - return n; - } - - if (dense.keys_vals(n) == 0) { - return n+1; - } - - osmium::builder::TagListBuilder tl_builder(m_buffer, builder); - - while (n < dense.keys_vals_size()) { - int tag_key_pos = dense.keys_vals(n++); - - if (tag_key_pos == 0) { - break; - } - - tl_builder.add_tag(m_stringtable->s(tag_key_pos), - m_stringtable->s(dense.keys_vals(n))); - - ++n; - } - - return n; - } - - void parse_dense_node_group(const OSMPBF::PrimitiveGroup& group) { - int64_t last_dense_id = 0; - int64_t last_dense_latitude = 0; - int64_t last_dense_longitude = 0; - int64_t last_dense_uid = 0; - int64_t last_dense_user_sid = 0; - int64_t last_dense_changeset = 0; - int64_t last_dense_timestamp = 0; - int last_dense_tag = 0; - - const OSMPBF::DenseNodes& dense = group.dense(); - - for (int i = 0; i < dense.id_size(); ++i) { - bool visible = true; - - last_dense_id += dense.id(i); - last_dense_latitude += dense.lat(i); - last_dense_longitude += dense.lon(i); - - if (dense.has_denseinfo()) { - last_dense_changeset += dense.denseinfo().changeset(i); - last_dense_timestamp += dense.denseinfo().timestamp(i); - last_dense_uid += dense.denseinfo().uid(i); - last_dense_user_sid += dense.denseinfo().user_sid(i); - if (dense.denseinfo().visible_size() > 0) { - visible = dense.denseinfo().visible(i); - } - assert(last_dense_changeset >= 0); - assert(last_dense_timestamp >= 0); - assert(last_dense_uid >= -1); - assert(last_dense_user_sid >= 0); - } - - osmium::builder::NodeBuilder builder(m_buffer); - osmium::Node& node = builder.object(); - - node.set_id(last_dense_id); - - if (dense.has_denseinfo()) { - auto v = dense.denseinfo().version(i); - assert(v > 0); - node.set_version(static_cast(v)); - node.set_changeset(static_cast(last_dense_changeset)); - node.set_timestamp(last_dense_timestamp * m_date_factor); - node.set_uid_from_signed(static_cast(last_dense_uid)); - node.set_visible(visible); - builder.add_user(m_stringtable->s(static_cast(last_dense_user_sid))); - } else { - builder.add_user("", 1); - } - - if (visible) { - builder.object().set_location(osmium::Location( - (last_dense_longitude * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision), - (last_dense_latitude * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision))); - } - - last_dense_tag = add_tags(dense, last_dense_tag, &builder); - m_buffer.commit(); - } - } - - }; // class PBFPrimitiveBlockParser - - /** - * PBF blobs can optionally be packed with the zlib algorithm. - * This function returns the raw data (if it was unpacked) or - * the unpacked data (if it was packed). - * - * @param input_data Reference to input data. - * @returns Unpacked data - * @throws osmium::pbf_error If there was a problem parsing the PBF - */ - inline std::unique_ptr unpack_blob(const std::string& input_data) { - OSMPBF::Blob pbf_blob; - if (!pbf_blob.ParseFromString(input_data)) { - throw osmium::pbf_error("failed to parse blob"); - } - - if (pbf_blob.has_raw()) { - return std::unique_ptr(pbf_blob.release_raw()); - } else if (pbf_blob.has_zlib_data()) { - auto raw_size = pbf_blob.raw_size(); - assert(raw_size >= 0); - assert(raw_size <= OSMPBF::max_uncompressed_blob_size); - return osmium::io::detail::zlib_uncompress(pbf_blob.zlib_data(), static_cast(raw_size)); - } else if (pbf_blob.has_lzma_data()) { - throw osmium::pbf_error("lzma blobs not implemented"); - } else { - throw osmium::pbf_error("blob contains no data"); - } - } - - /** - * Parse blob as a HeaderBlock. - * - * @param input_buffer Blob data - * @returns Header object - * @throws osmium::pbf_error If there was a parsing error - */ - inline osmium::io::Header parse_header_blob(const std::string& input_buffer) { - const std::unique_ptr data = unpack_blob(input_buffer); - - OSMPBF::HeaderBlock pbf_header_block; - if (!pbf_header_block.ParseFromString(*data)) { - throw osmium::pbf_error("failed to parse HeaderBlock"); - } - - osmium::io::Header header; - for (int i = 0; i < pbf_header_block.required_features_size(); ++i) { - const std::string& feature = pbf_header_block.required_features(i); - - if (feature == "OsmSchema-V0.6") continue; - if (feature == "DenseNodes") { - header.set("pbf_dense_nodes", true); - continue; - } - if (feature == "HistoricalInformation") { - header.set_has_multiple_object_versions(true); - continue; - } - - throw osmium::pbf_error(std::string("required feature not supported: ") + feature); - } - - for (int i = 0; i < pbf_header_block.optional_features_size(); ++i) { - const std::string& feature = pbf_header_block.optional_features(i); - header.set("pbf_optional_feature_" + std::to_string(i), feature); - } - - if (pbf_header_block.has_writingprogram()) { - header.set("generator", pbf_header_block.writingprogram()); - } - - if (pbf_header_block.has_bbox()) { - const OSMPBF::HeaderBBox& pbf_bbox = pbf_header_block.bbox(); - const int64_t resolution_convert = OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision; - osmium::Box box; - box.extend(osmium::Location(pbf_bbox.left() / resolution_convert, pbf_bbox.bottom() / resolution_convert)); - box.extend(osmium::Location(pbf_bbox.right() / resolution_convert, pbf_bbox.top() / resolution_convert)); - header.add_box(box); - } - - if (pbf_header_block.has_osmosis_replication_timestamp()) { - header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.osmosis_replication_timestamp()).to_iso()); - } - - if (pbf_header_block.has_osmosis_replication_sequence_number()) { - header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.osmosis_replication_sequence_number())); - } - - if (pbf_header_block.has_osmosis_replication_base_url()) { - header.set("osmosis_replication_base_url", pbf_header_block.osmosis_replication_base_url()); - } - - return header; - } - - class DataBlobParser { - - std::shared_ptr m_input_buffer; - osmium::osm_entity_bits::type m_read_types; - - public: - - DataBlobParser(std::string&& input_buffer, osmium::osm_entity_bits::type read_types) : - m_input_buffer(std::make_shared(std::move(input_buffer))), - m_read_types(read_types) { - if (input_buffer.size() > OSMPBF::max_uncompressed_blob_size) { - throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size()))); - } - } -/* - DataBlobParser(const DataBlobParser& other) : - m_input_buffer(std::move(other.m_input_buffer)), - m_read_types(other.m_read_types) { - }*/ - - DataBlobParser(const DataBlobParser&) = default; - DataBlobParser& operator=(const DataBlobParser&) = default; - - DataBlobParser(DataBlobParser&&) = default; - DataBlobParser& operator=(DataBlobParser&&) = default; - - ~DataBlobParser() = default; - - osmium::memory::Buffer operator()() { - const std::unique_ptr data = unpack_blob(*m_input_buffer); - PBFPrimitiveBlockParser parser(*data, m_read_types); - return parser(); - } - - }; // class DataBlobParser - - } // namespace detail - - } // namespace io - -} // namespace osmium - -#endif // OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP diff --git a/include/osmium/io/detail/pbf_stringtable.hpp b/include/osmium/io/detail/pbf_stringtable.hpp deleted file mode 100644 index 5f540f1271b..00000000000 --- a/include/osmium/io/detail/pbf_stringtable.hpp +++ /dev/null @@ -1,218 +0,0 @@ -#ifndef OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP -#define OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP - -/* - -This file is part of Osmium (http://osmcode.org/libosmium). - -Copyright 2013-2015 Jochen Topf and others (see README). - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -*/ - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace osmium { - - namespace io { - - namespace detail { - - /** - * StringTable management for PBF writer - * - * All strings are stored as indexes to rows in a StringTable. The StringTable contains - * one row for each used string, so strings that are used multiple times need to be - * stored only once. The StringTable is sorted by usage-count, so the most often used - * string is stored at index 1. - */ - class StringTable { - - public: - - /// type for string IDs (interim and final) - typedef uint16_t string_id_type; - - private: - - /** - * this is the struct used to build the StringTable. It is stored as - * the value-part in the strings-map. - * - * when a new string is added to the map, its count is set to 0 and - * the interim_id is set to the current size of the map. This interim_id - * is then stored into the pbf-objects. - * - * before the PrimitiveBlock is serialized, the map is sorted by count - * and stored into the pbf-StringTable. Afterwards the interim-ids are - * mapped to the "real" id in the StringTable. - * - * this way often used strings get lower ids in the StringTable. As the - * protobuf-serializer stores numbers in variable bit-lengths, lower - * IDs means less used space in the resulting file. - */ - struct string_info { - - /// number of occurrences of this string - uint16_t count; - - /// an intermediate-id - string_id_type interim_id; - - }; // struct string_info - - /** - * Interim StringTable, storing all strings that should be written to - * the StringTable once the block is written to disk. - */ - typedef std::map string2string_info_type; - string2string_info_type m_strings; - - /** - * This vector is used to map the interim IDs to real StringTable IDs after - * writing all strings to the StringTable. - */ - typedef std::vector interim_id2id_type; - interim_id2id_type m_id2id_map; - - size_t m_size = 0; - - public: - - StringTable() { - } - - friend bool operator<(const string_info& lhs, const string_info& rhs) { - return lhs.count > rhs.count; - } - - /** - * record a string in the interim StringTable if it's missing, otherwise just increase its counter, - * return the interim-id assigned to the string. - */ - string_id_type record_string(const std::string& string) { - string_info& info = m_strings[string]; - if (info.interim_id == 0) { - ++m_size; - info.interim_id = static_cast_with_assert(m_size); - } else { - info.count++; - } - return info.interim_id; - } - - /** - * Sort the interim StringTable and store it to the real protobuf StringTable. - * while storing to the real table, this function fills the id2id_map with - * pairs, mapping the interim-ids to final and real StringTable ids. - * - * Note that the m_strings table is a std::map and as such is sorted lexicographically. - * When the transformation into the sortedby multimap is done, it gets sorted by - * the count. The end result (at least with the glibc standard container/algorithm - * implementation) is that the string table is sorted first by reverse count (ie descending) - * and then by reverse lexicographic order. - */ - void store_stringtable(OSMPBF::StringTable* st, bool sort) { - // add empty StringTable entry at index 0 - // StringTable index 0 is reserved as delimiter in the densenodes key/value list - // this line also ensures that there's always a valid StringTable - st->add_s(""); - - if (sort) { - std::multimap sortedbycount; - - m_id2id_map.resize(m_size+1); - - std::transform(m_strings.begin(), m_strings.end(), - std::inserter(sortedbycount, sortedbycount.begin()), - [](const std::pair& p) { - return std::pair(p.second, p.first); - }); - - string_id_type n = 0; - - for (const auto& mapping : sortedbycount) { - // add the string of the current item to the pbf StringTable - st->add_s(mapping.second); - - // store the mapping from the interim-id to the real id - m_id2id_map[mapping.first.interim_id] = ++n; - } - } else { - std::vector> sortedbyid; - sortedbyid.reserve(m_strings.size()); - - for (const auto& p : m_strings) { - sortedbyid.emplace_back(p.second.interim_id, p.first.c_str()); - } - - std::sort(sortedbyid.begin(), sortedbyid.end()); - for (const auto& mapping : sortedbyid) { - st->add_s(mapping.second); - } - } - } - - /** - * Map from an interim ID to a real string ID. - */ - string_id_type map_string_id(const string_id_type interim_id) const { - return m_id2id_map[interim_id]; - } - - template - string_id_type map_string_id(const T interim_id) const { - return map_string_id(static_cast_with_assert(interim_id)); - } - - /** - * Clear the stringtable, preparing for the next block. - */ - void clear() { - m_strings.clear(); - m_id2id_map.clear(); - m_size = 0; - } - - }; // class StringTable - - } // namespace detail - - } // namespace io - -} // namespace osmium - -#endif // OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP diff --git a/include/osmium/io/detail/protobuf_tags.hpp b/include/osmium/io/detail/protobuf_tags.hpp new file mode 100644 index 00000000000..3f230876639 --- /dev/null +++ b/include/osmium/io/detail/protobuf_tags.hpp @@ -0,0 +1,170 @@ +#ifndef OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP +#define OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include + +namespace osmium { + + namespace io { + + namespace detail { + + // directly translated from + // https://github.com/scrosby/OSM-binary/blob/master/src/fileformat.proto + + namespace FileFormat { + + enum class Blob : protozero::pbf_tag_type { + optional_bytes_raw = 1, + optional_int32_raw_size = 2, + optional_bytes_zlib_data = 3, + optional_bytes_lzma_data = 4 + }; + + enum class BlobHeader : protozero::pbf_tag_type { + required_string_type = 1, + optional_bytes_indexdata = 2, + required_int32_datasize = 3 + }; + + } // namespace FileFormat + + // directly translated from + // https://github.com/scrosby/OSM-binary/blob/master/src/osmformat.proto + + namespace OSMFormat { + + enum class HeaderBlock : protozero::pbf_tag_type { + optional_HeaderBBox_bbox = 1, + repeated_string_required_features = 4, + repeated_string_optional_features = 5, + optional_string_writingprogram = 16, + optional_string_source = 17, + optional_int64_osmosis_replication_timestamp = 32, + optional_int64_osmosis_replication_sequence_number = 33, + optional_string_osmosis_replication_base_url = 34 + }; + + enum class HeaderBBox : protozero::pbf_tag_type { + required_sint64_left = 1, + required_sint64_right = 2, + required_sint64_top = 3, + required_sint64_bottom = 4 + }; + + enum class PrimitiveBlock : protozero::pbf_tag_type { + required_StringTable_stringtable = 1, + repeated_PrimitiveGroup_primitivegroup = 2, + optional_int32_granularity = 17, + optional_int32_date_granularity = 18, + optional_int64_lat_offset = 19, + optional_int64_lon_offset = 20 + }; + + enum class PrimitiveGroup : protozero::pbf_tag_type { + unknown = 0, + repeated_Node_nodes = 1, + optional_DenseNodes_dense = 2, + repeated_Way_ways = 3, + repeated_Relation_relations = 4, + repeated_ChangeSet_changesets = 5 + }; + + enum class StringTable : protozero::pbf_tag_type { + repeated_bytes_s = 1 + }; + + enum class Info : protozero::pbf_tag_type { + optional_int32_version = 1, + optional_int64_timestamp = 2, + optional_int64_changeset = 3, + optional_int32_uid = 4, + optional_uint32_user_sid = 5, + optional_bool_visible = 6 + }; + + enum class DenseInfo : protozero::pbf_tag_type { + packed_int32_version = 1, + packed_sint64_timestamp = 2, + packed_sint64_changeset = 3, + packed_sint32_uid = 4, + packed_sint32_user_sid = 5, + packed_bool_visible = 6 + }; + + enum class Node : protozero::pbf_tag_type { + required_sint64_id = 1, + packed_uint32_keys = 2, + packed_uint32_vals = 3, + optional_Info_info = 4, + required_sint64_lat = 8, + required_sint64_lon = 9 + }; + + enum class DenseNodes : protozero::pbf_tag_type { + packed_sint64_id = 1, + optional_DenseInfo_denseinfo = 5, + packed_sint64_lat = 8, + packed_sint64_lon = 9, + packed_int32_keys_vals = 10 + }; + + enum class Way : protozero::pbf_tag_type { + required_int64_id = 1, + packed_uint32_keys = 2, + packed_uint32_vals = 3, + optional_Info_info = 4, + packed_sint64_refs = 8 + }; + + enum class Relation : protozero::pbf_tag_type { + required_int64_id = 1, + packed_uint32_keys = 2, + packed_uint32_vals = 3, + optional_Info_info = 4, + packed_int32_roles_sid = 8, + packed_sint64_memids = 9, + packed_MemberType_types = 10 + }; + + } // namespace OSMFormat + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP diff --git a/include/osmium/io/detail/read_write.hpp b/include/osmium/io/detail/read_write.hpp index 6651cced9b7..9863bd719cc 100644 --- a/include/osmium/io/detail/read_write.hpp +++ b/include/osmium/io/detail/read_write.hpp @@ -122,7 +122,7 @@ namespace osmium { * @throws std::system_error On error. */ inline void reliable_write(const int fd, const unsigned char* output_buffer, const size_t size) { - constexpr size_t max_write = 100 * 1024 * 1024; // Max 100 MByte per write + constexpr size_t max_write = 100L * 1024L * 1024L; // Max 100 MByte per write size_t offset = 0; do { auto write_count = size - offset; diff --git a/include/osmium/io/detail/string_table.hpp b/include/osmium/io/detail/string_table.hpp new file mode 100644 index 00000000000..ae9d5f0ce3f --- /dev/null +++ b/include/osmium/io/detail/string_table.hpp @@ -0,0 +1,250 @@ +#ifndef OSMIUM_IO_DETAIL_STRING_TABLE_HPP +#define OSMIUM_IO_DETAIL_STRING_TABLE_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + namespace io { + + namespace detail { + + /** + * class StringStore + * + * Storage of lots of strings (const char *). Memory is allocated in chunks. + * If a string is added and there is no space in the current chunk, a new + * chunk will be allocated. Strings added to the store must not be larger + * than the chunk size. + * + * All memory is released when the destructor is called. There is no other way + * to release all or part of the memory. + * + */ + class StringStore { + + size_t m_chunk_size; + + std::list m_chunks; + + void add_chunk() { + m_chunks.push_front(std::string()); + m_chunks.front().reserve(m_chunk_size); + } + + public: + + StringStore(size_t chunk_size) : + m_chunk_size(chunk_size), + m_chunks() { + add_chunk(); + } + + void clear() noexcept { + m_chunks.erase(std::next(m_chunks.begin()), m_chunks.end()); + m_chunks.front().clear(); + } + + /** + * Add a null terminated string to the store. This will + * automatically get more memory if we are out. + * Returns a pointer to the copy of the string we have + * allocated. + */ + const char* add(const char* string) { + size_t len = std::strlen(string) + 1; + + assert(len <= m_chunk_size); + + size_t chunk_len = m_chunks.front().size(); + if (chunk_len + len > m_chunks.front().capacity()) { + add_chunk(); + chunk_len = 0; + } + + m_chunks.front().append(string); + m_chunks.front().append(1, '\0'); + + return m_chunks.front().c_str() + chunk_len; + } + + class const_iterator : public std::iterator { + + typedef std::list::const_iterator it_type; + it_type m_it; + const it_type m_last; + const char* m_pos; + + public: + + const_iterator(it_type it, it_type last) : + m_it(it), + m_last(last), + m_pos(it == last ? nullptr : m_it->c_str()) { + } + + const_iterator& operator++() { + assert(m_it != m_last); + auto last_pos = m_it->c_str() + m_it->size(); + while (m_pos != last_pos && *m_pos) ++m_pos; + if (m_pos != last_pos) ++m_pos; + if (m_pos == last_pos) { + ++m_it; + if (m_it != m_last) { + m_pos = m_it->c_str(); + } else { + m_pos = nullptr; + } + } + return *this; + } + + const_iterator operator++(int) { + const_iterator tmp(*this); + operator++(); + return tmp; + } + + bool operator==(const const_iterator& rhs) const { + return m_it == rhs.m_it && m_pos == rhs.m_pos; + } + + bool operator!=(const const_iterator& rhs) const { + return !(*this == rhs); + } + + const char* operator*() const { + assert(m_it != m_last); + assert(m_pos != nullptr); + return m_pos; + } + + }; // class const_iterator + + const_iterator begin() const { + if (m_chunks.front().empty()) { + return end(); + } + return const_iterator(m_chunks.begin(), m_chunks.end()); + } + + const_iterator end() const { + return const_iterator(m_chunks.end(), m_chunks.end()); + } + + // These functions get you some idea how much memory was + // used. + int get_chunk_size() const noexcept { + return m_chunk_size; + } + + int get_chunk_count() const noexcept { + return m_chunks.size(); + } + + int get_used_bytes_in_last_chunk() const noexcept { + return m_chunks.front().size(); + } + + }; // class StringStore + + struct StrComp { + + bool operator()(const char* lhs, const char* rhs) const { + return strcmp(lhs, rhs) < 0; + } + + }; // struct StrComp + + class StringTable { + + StringStore m_strings; + std::map m_index; + size_t m_size; + + public: + + StringTable() : + m_strings(1024 * 1024), + m_index(), + m_size(0) { + m_strings.add(""); + } + + void clear() { + m_strings.clear(); + m_index.clear(); + m_size = 0; + m_strings.add(""); + } + + size_t size() const noexcept { + return m_size + 1; + } + + size_t add(const char* s) { + auto f = m_index.find(s); + if (f != m_index.end()) { + return f->second; + } + + const char* cs = m_strings.add(s); + m_index[cs] = ++m_size; + return m_size; + } + + StringStore::const_iterator begin() const { + return m_strings.begin(); + } + + StringStore::const_iterator end() const { + return m_strings.end(); + } + + }; // class StringTable + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_STRING_TABLE_HPP diff --git a/include/osmium/io/detail/xml_input_format.hpp b/include/osmium/io/detail/xml_input_format.hpp index c03f84d9b49..b0f3da33927 100644 --- a/include/osmium/io/detail/xml_input_format.hpp +++ b/include/osmium/io/detail/xml_input_format.hpp @@ -66,6 +66,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include #include #include #include @@ -191,6 +192,8 @@ namespace osmium { std::atomic& m_done; + bool m_header_is_done; + /** * A C++ wrapper for the Expat parser that makes sure no memory is leaked. */ @@ -246,16 +249,25 @@ namespace osmium { T& m_data; std::promise& m_promise; + bool m_done; public: PromiseKeeper(T& data, std::promise& promise) : m_data(data), - m_promise(promise) { + m_promise(promise), + m_done(false) { + } + + void fullfill_promise() { + if (!m_done) { + m_promise.set_value(m_data); + m_done = true; + } } ~PromiseKeeper() { - m_promise.set_value(m_data); + fullfill_promise(); } }; // class PromiseKeeper @@ -279,7 +291,8 @@ namespace osmium { m_queue(queue), m_header_promise(header_promise), m_read_types(read_types), - m_done(done) { + m_done(done), + m_header_is_done(false) { } /** @@ -305,7 +318,8 @@ namespace osmium { m_queue(other.m_queue), m_header_promise(other.m_header_promise), m_read_types(other.m_read_types), - m_done(other.m_done) { + m_done(other.m_done), + m_header_is_done(other.m_header_is_done) { } XMLParser(XMLParser&&) = default; @@ -326,6 +340,9 @@ namespace osmium { last = data.empty(); try { parser(data, last); + if (m_header_is_done) { + promise_keeper.fullfill_promise(); + } } catch (ParserIsDone&) { return true; } catch (...) { @@ -343,8 +360,7 @@ namespace osmium { private: const char* init_object(osmium::OSMObject& object, const XML_Char** attrs) { - static const char* empty = ""; - const char* user = empty; + const char* user = ""; if (m_in_delete_section) { object.set_visible(false); @@ -371,8 +387,7 @@ namespace osmium { } void init_changeset(osmium::builder::ChangesetBuilder* builder, const XML_Char** attrs) { - static const char* empty = ""; - const char* user = empty; + const char* user = ""; osmium::Changeset& new_changeset = builder->object(); osmium::Location min; @@ -421,6 +436,7 @@ namespace osmium { } void header_is_done() { + m_header_is_done = true; if (m_read_types == osmium::osm_entity_bits::nothing) { throw ParserIsDone(); } @@ -722,10 +738,15 @@ namespace osmium { namespace { +// we want the register_input_format() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_xml_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::xml, [](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) { return new osmium::io::detail::XMLInputFormat(file, read_which_entities, input_queue); }); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/include/osmium/io/detail/xml_output_format.hpp b/include/osmium/io/detail/xml_output_format.hpp index 65ba171b8b1..2a381d5ab17 100644 --- a/include/osmium/io/detail/xml_output_format.hpp +++ b/include/osmium/io/detail/xml_output_format.hpp @@ -85,6 +85,9 @@ namespace osmium { case '\'': out += "'"; break; case '<': out += "<"; break; case '>': out += ">"; break; + case '\n': out += " "; break; + case '\r': out += " "; break; + case '\t': out += " "; break; default: out += *in; break; } } @@ -126,6 +129,7 @@ namespace osmium { operation m_last_op {operation::op_none}; + const bool m_add_metadata; const bool m_write_visible_flag; const bool m_write_change_ops; @@ -146,31 +150,33 @@ namespace osmium { void write_meta(const osmium::OSMObject& object) { oprintf(*m_out, " id=\"%" PRId64 "\"", object.id()); - if (object.version()) { - oprintf(*m_out, " version=\"%d\"", object.version()); - } + if (m_add_metadata) { + if (object.version()) { + oprintf(*m_out, " version=\"%d\"", object.version()); + } - if (object.timestamp()) { - *m_out += " timestamp=\""; - *m_out += object.timestamp().to_iso(); - *m_out += "\""; - } + if (object.timestamp()) { + *m_out += " timestamp=\""; + *m_out += object.timestamp().to_iso(); + *m_out += "\""; + } - if (!object.user_is_anonymous()) { - oprintf(*m_out, " uid=\"%d\" user=\"", object.uid()); - xml_string(*m_out, object.user()); - *m_out += "\""; - } + if (!object.user_is_anonymous()) { + oprintf(*m_out, " uid=\"%d\" user=\"", object.uid()); + xml_string(*m_out, object.user()); + *m_out += "\""; + } - if (object.changeset()) { - oprintf(*m_out, " changeset=\"%d\"", object.changeset()); - } + if (object.changeset()) { + oprintf(*m_out, " changeset=\"%d\"", object.changeset()); + } - if (m_write_visible_flag) { - if (object.visible()) { - *m_out += " visible=\"true\""; - } else { - *m_out += " visible=\"false\""; + if (m_write_visible_flag) { + if (object.visible()) { + *m_out += " visible=\"true\""; + } else { + *m_out += " visible=\"false\""; + } } } } @@ -224,9 +230,10 @@ namespace osmium { public: - explicit XMLOutputBlock(osmium::memory::Buffer&& buffer, bool write_visible_flag, bool write_change_ops) : + explicit XMLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool write_visible_flag, bool write_change_ops) : m_input_buffer(std::make_shared(std::move(buffer))), m_out(std::make_shared()), + m_add_metadata(add_metadata), m_write_visible_flag(write_visible_flag && !write_change_ops), m_write_change_ops(write_change_ops) { } @@ -392,12 +399,14 @@ namespace osmium { class XMLOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler { + bool m_add_metadata; bool m_write_visible_flag; public: XMLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : OutputFormat(file, output_queue), + m_add_metadata(file.get("add_metadata") != "false"), m_write_visible_flag(file.has_multiple_object_versions() || m_file.is_true("force_visible_flag")) { } @@ -408,7 +417,7 @@ namespace osmium { } void write_buffer(osmium::memory::Buffer&& buffer) override final { - m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_write_visible_flag, m_file.is_true("xml_change_format")})); + m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_add_metadata, m_write_visible_flag, m_file.is_true("xml_change_format")})); } void write_header(const osmium::io::Header& header) override final { @@ -468,10 +477,15 @@ namespace osmium { namespace { +// we want the register_output_format() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_xml_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::xml, [](const osmium::io::File& file, data_queue_type& output_queue) { return new osmium::io::detail::XMLOutputFormat(file, output_queue); }); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/include/osmium/io/detail/zlib.hpp b/include/osmium/io/detail/zlib.hpp index a402bf7e636..fc0baf344a4 100644 --- a/include/osmium/io/detail/zlib.hpp +++ b/include/osmium/io/detail/zlib.hpp @@ -85,23 +85,24 @@ namespace osmium { * * @param input Compressed input data. * @param raw_size Size of uncompressed data. - * @returns Uncompressed data. + * @param output Uncompressed result data. + * @returns Pointer and size to incompressed data. */ - inline std::unique_ptr zlib_uncompress(const std::string& input, unsigned long raw_size) { - auto output = std::unique_ptr(new std::string(raw_size, '\0')); + inline std::pair zlib_uncompress_string(const char* input, unsigned long input_size, unsigned long raw_size, std::string& output) { + output.resize(raw_size); auto result = ::uncompress( - reinterpret_cast(const_cast(output->data())), + reinterpret_cast(&*output.begin()), &raw_size, - reinterpret_cast(input.data()), - osmium::static_cast_with_assert(input.size()) + reinterpret_cast(input), + input_size ); if (result != Z_OK) { throw std::runtime_error(std::string("failed to uncompress data: ") + zError(result)); } - return output; + return std::make_pair(output.data(), output.size()); } } // namespace detail diff --git a/include/osmium/io/file.hpp b/include/osmium/io/file.hpp index 5b6c02f1324..3bbfacc6def 100644 --- a/include/osmium/io/file.hpp +++ b/include/osmium/io/file.hpp @@ -97,7 +97,9 @@ namespace osmium { * of the file will be taken from the suffix. * An empty filename or "-" means stdin or stdout. * @param format File format as string. See the description of the - * parse_format() function for details. + * parse_format() function for details. If this is + * empty the format will be deduced from the suffix + * of the filename. */ explicit File(const std::string& filename = "", const std::string& format = "") : Options(), @@ -107,20 +109,19 @@ namespace osmium { m_format_string(format) { // stdin/stdout - if (filename == "" || filename == "-") { + if (m_filename == "-") { m_filename = ""; - default_settings_for_stdinout(); } - // filename is actually a URL + // if filename is a URL, default to XML format std::string protocol = m_filename.substr(0, m_filename.find_first_of(':')); if (protocol == "http" || protocol == "https") { - default_settings_for_url(); + m_file_format = file_format::xml; } - detect_format_from_suffix(m_filename); - - if (format != "") { + if (format.empty()) { + detect_format_from_suffix(m_filename); + } else { parse_format(format); } } @@ -140,9 +141,6 @@ namespace osmium { m_buffer(buffer), m_buffer_size(size), m_format_string(format) { - - default_settings_for_stdinout(); - if (format != "") { parse_format(format); } @@ -220,6 +218,20 @@ namespace osmium { } else if (suffixes.back() == "opl") { m_file_format = file_format::opl; suffixes.pop_back(); + } else if (suffixes.back() == "json") { + m_file_format = file_format::json; + suffixes.pop_back(); + } else if (suffixes.back() == "o5m") { + m_file_format = file_format::o5m; + suffixes.pop_back(); + } else if (suffixes.back() == "o5c") { + m_file_format = file_format::o5m; + m_has_multiple_object_versions = true; + set("o5c_change_format", true); + suffixes.pop_back(); + } else if (suffixes.back() == "debug") { + m_file_format = file_format::debug; + suffixes.pop_back(); } if (suffixes.empty()) return; @@ -240,8 +252,8 @@ namespace osmium { } /** - * Check file format etc. for consistency and throw exception if there - * is a problem. + * Check file format etc. for consistency and throw exception if + * there is a problem. * * @throws std::runtime_error */ @@ -265,36 +277,6 @@ namespace osmium { } } - /** - * Set default settings for type and encoding when the filename is - * empty or "-". If you want to have a different default setting - * override this in a subclass. - */ - void default_settings_for_stdinout() { - m_file_format = file_format::unknown; - m_file_compression = file_compression::none; - } - - /** - * Set default settings for type and encoding when the filename is - * a normal file. If you want to have a different default setting - * override this in a subclass. - */ - void default_settings_for_file() { - m_file_format = file_format::unknown; - m_file_compression = file_compression::none; - } - - /** - * Set default settings for type and encoding when the filename is a URL. - * If you want to have a different default setting override this in a - * subclass. - */ - void default_settings_for_url() { - m_file_format = file_format::xml; - m_file_compression = file_compression::none; - } - file_format format() const noexcept { return m_file_format; } diff --git a/include/osmium/io/file_format.hpp b/include/osmium/io/file_format.hpp index 1a63a5e49d2..edfb1ff9f7d 100644 --- a/include/osmium/io/file_format.hpp +++ b/include/osmium/io/file_format.hpp @@ -44,7 +44,9 @@ namespace osmium { xml = 1, pbf = 2, opl = 3, - json = 4 + json = 4, + o5m = 5, + debug = 6 }; // avoid g++ false positive @@ -62,6 +64,10 @@ namespace osmium { return "OPL"; case file_format::json: return "JSON"; + case file_format::o5m: + return "O5M"; + case file_format::debug: + return "DEBUG"; } } #pragma GCC diagnostic pop diff --git a/include/osmium/io/gzip_compression.hpp b/include/osmium/io/gzip_compression.hpp index 272397707d9..204bfd52193 100644 --- a/include/osmium/io/gzip_compression.hpp +++ b/include/osmium/io/gzip_compression.hpp @@ -231,11 +231,16 @@ namespace osmium { namespace { +// we want the register_compression() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_gzip_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip, [](int fd) { return new osmium::io::GzipCompressor(fd); }, [](int fd) { return new osmium::io::GzipDecompressor(fd); }, [](const char* buffer, size_t size) { return new osmium::io::GzipBufferDecompressor(buffer, size); } ); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/include/osmium/io/pbf_input.hpp b/include/osmium/io/pbf_input.hpp index 766153edef9..d7f3787ae0e 100644 --- a/include/osmium/io/pbf_input.hpp +++ b/include/osmium/io/pbf_input.hpp @@ -39,7 +39,6 @@ DEALINGS IN THE SOFTWARE. * Include this file if you want to read OSM PBF files. * * @attention If you include this file, you'll need to link with - * `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only), * `libz`, and enable multithreading. */ diff --git a/include/osmium/io/pbf_output.hpp b/include/osmium/io/pbf_output.hpp index 5f46ede2b29..dad1013d7a9 100644 --- a/include/osmium/io/pbf_output.hpp +++ b/include/osmium/io/pbf_output.hpp @@ -39,7 +39,6 @@ DEALINGS IN THE SOFTWARE. * Include this file if you want to write OSM PBF files. * * @attention If you include this file, you'll need to link with - * `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only), * `libz`, and enable multithreading. */ diff --git a/include/osmium/memory/buffer.hpp b/include/osmium/memory/buffer.hpp index 85a3a46d4da..d800c685c2e 100644 --- a/include/osmium/memory/buffer.hpp +++ b/include/osmium/memory/buffer.hpp @@ -37,7 +37,6 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include #include #include #include @@ -83,7 +82,7 @@ namespace osmium { * Buffers exist in two flavours, those with external memory management and * those with internal memory management. If you already have some memory * with data in it (for instance read from disk), you create a Buffer with - * external memory managment. It is your job then to free the memory once + * external memory management. It is your job then to free the memory once * the buffer isn't used any more. If you don't have memory already, you can * create a Buffer object and have it manage the memory internally. It will * dynamically allocate memory and free it again after use. @@ -413,6 +412,15 @@ namespace osmium { return iterator(m_data, m_data + m_committed); } + template + t_iterator get_iterator(size_t offset) { + return t_iterator(m_data + offset, m_data + m_committed); + } + + iterator get_iterator(size_t offset) { + return iterator(m_data + offset, m_data + m_committed); + } + template t_iterator end() { return t_iterator(m_data + m_committed, m_data + m_committed); @@ -431,6 +439,15 @@ namespace osmium { return const_iterator(m_data, m_data + m_committed); } + template + t_const_iterator get_iterator(size_t offset) const { + return t_const_iterator(m_data + offset, m_data + m_committed); + } + + const_iterator get_iterator(size_t offset) const { + return const_iterator(m_data + offset, m_data + m_committed); + } + template t_const_iterator cend() const { return t_const_iterator(m_data + m_committed, m_data + m_committed); diff --git a/include/osmium/memory/collection.hpp b/include/osmium/memory/collection.hpp index 7deb88b458c..5cf3cc6e17a 100644 --- a/include/osmium/memory/collection.hpp +++ b/include/osmium/memory/collection.hpp @@ -38,7 +38,6 @@ DEALINGS IN THE SOFTWARE. #include #include -#include namespace osmium { diff --git a/include/osmium/memory/item.hpp b/include/osmium/memory/item.hpp index 2679ca6f20c..dc544049a6f 100644 --- a/include/osmium/memory/item.hpp +++ b/include/osmium/memory/item.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include diff --git a/include/osmium/osm/changeset.hpp b/include/osmium/osm/changeset.hpp index 0ab4e9bade8..07bc0dd95c1 100644 --- a/include/osmium/osm/changeset.hpp +++ b/include/osmium/osm/changeset.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include @@ -44,6 +43,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include namespace osmium { diff --git a/include/osmium/osm/crc.hpp b/include/osmium/osm/crc.hpp new file mode 100644 index 00000000000..eefa4a13ed8 --- /dev/null +++ b/include/osmium/osm/crc.hpp @@ -0,0 +1,223 @@ +#ifndef OSMIUM_OSM_CRC_HPP +#define OSMIUM_OSM_CRC_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + template + class CRC { + + static inline uint16_t byte_swap_16(uint16_t value) noexcept { +# if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap16(value); +# else + return (value >> 8) | (value << 8); +# endif + } + + static inline uint32_t byte_swap_32(uint32_t value) noexcept { +# if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap32(value); +# else + return (value >> 24) | + ((value >> 8) & 0x0000FF00) | + ((value << 8) & 0x00FF0000) | + (value << 24); +# endif + } + + static inline uint64_t byte_swap_64(uint64_t value) noexcept { +# if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(value); +# else + uint64_t val1 = byte_swap_32(value & 0xFFFFFFFF); + uint64_t val2 = byte_swap_32(value >> 32); + return (val1 << 32) & val2; +# endif + } + + TCRC m_crc; + + public: + + TCRC& operator()() { + return m_crc; + } + + const TCRC& operator()() const { + return m_crc; + } + + void update_bool(bool value) { + m_crc.process_byte(value); + } + + void update_int8(uint8_t value) { + m_crc.process_byte(value); + } + + void update_int16(uint16_t value) { +#if __BYTE_ORDER == __LITTLE_ENDIAN + m_crc.process_bytes(&value, sizeof(uint16_t)); +#else + uint16_t v = byte_swap_16(value); + m_crc.process_bytes(&v, sizeof(uint16_t)); +#endif + } + + void update_int32(uint32_t value) { +#if __BYTE_ORDER == __LITTLE_ENDIAN + m_crc.process_bytes(&value, sizeof(uint32_t)); +#else + uint32_t v = byte_swap_32(value); + m_crc.process_bytes(&v, sizeof(uint32_t)); +#endif + } + + void update_int64(uint64_t value) { +#if __BYTE_ORDER == __LITTLE_ENDIAN + m_crc.process_bytes(&value, sizeof(uint64_t)); +#else + uint64_t v = byte_swap_64(value); + m_crc.process_bytes(&v, sizeof(uint64_t)); +#endif + } + + void update_string(const char* str) { + while (*str) { + m_crc.process_byte(*str++); + } + } + + void update(const Timestamp& timestamp) { + update_int32(uint32_t(timestamp)); + } + + void update(const osmium::Location& location) { + update_int32(location.x()); + update_int32(location.y()); + } + + void update(const osmium::Box& box) { + update(box.bottom_left()); + update(box.top_right()); + } + + void update(const NodeRef& node_ref) { + update_int64(node_ref.ref()); + } + + void update(const NodeRefList& node_refs) { + for (const NodeRef& node_ref : node_refs) { + update(node_ref); + } + } + + void update(const TagList& tags) { + m_crc.process_bytes(tags.data(), tags.byte_size()); + } + + void update(const osmium::RelationMember& member) { + update_int64(member.ref()); + update_int16(uint16_t(member.type())); + update_string(member.role()); + } + + void update(const osmium::RelationMemberList& members) { + for (const RelationMember& member : members) { + update(member); + } + } + + void update(const osmium::OSMObject& object) { + update_int64(object.id()); + update_bool(object.visible()); + update_int32(object.version()); + update(object.timestamp()); + update_int32(object.uid()); + update_string(object.user()); + update(object.tags()); + } + + void update(const osmium::Node& node) { + update(static_cast(node)); + update(node.location()); + } + + void update(const osmium::Way& way) { + update(static_cast(way)); + update(way.nodes()); + } + + void update(const osmium::Relation& relation) { + update(static_cast(relation)); + update(relation.members()); + } + + void update(const osmium::Area& area) { + update(static_cast(area)); + for (auto it = area.cbegin(); it != area.cend(); ++it) { + if (it->type() == osmium::item_type::outer_ring || + it->type() == osmium::item_type::inner_ring) { + update(static_cast(*it)); + } + } + } + + void update(const osmium::Changeset& changeset) { + update_int64(changeset.id()); + update(changeset.created_at()); + update(changeset.closed_at()); + update(changeset.bounds()); + update_int32(changeset.num_changes()); + update_int32(changeset.uid()); + update_string(changeset.user()); + } + + }; // class CRC + +} // namespace osmium + +#endif // OSMIUM_OSM_CRC diff --git a/include/osmium/osm/diff_object.hpp b/include/osmium/osm/diff_object.hpp index 55a5cef68d6..1e053fdda7e 100644 --- a/include/osmium/osm/diff_object.hpp +++ b/include/osmium/osm/diff_object.hpp @@ -112,8 +112,35 @@ namespace osmium { return m_curr->timestamp(); } + /** + * Return the timestamp when the current version of the object is + * not valid any more, ie the time when the next version of the object + * is valid. If this is the last version of the object, this will + * return a special "end of time" timestamp that is guaranteed to + * be larger than any normal timestamp. + */ const osmium::Timestamp end_time() const noexcept { - return last() ? osmium::Timestamp() : m_next->timestamp(); + return last() ? osmium::end_of_time() : m_next->timestamp(); + } + + /** + * Current object version is valid between time "from" (inclusive) and + * time "to" (not inclusive). + * + * This is a bit more complex than you'd think, because we have to + * handle the case properly where the start_time() == end_time(). + */ + bool is_between(const osmium::Timestamp& from, const osmium::Timestamp& to) const noexcept { + return start_time() < to && + ((start_time() != end_time() && end_time() > from) || + (start_time() == end_time() && end_time() >= from)); + } + + /** + * Current object version is visible at the given timestamp. + */ + bool is_visible_at(const osmium::Timestamp& timestamp) const noexcept { + return start_time() <= timestamp && end_time() > timestamp && m_curr->visible(); } }; // class DiffObject diff --git a/include/osmium/osm/entity.hpp b/include/osmium/osm/entity.hpp index 14861a2d5c8..ce292c8d64b 100644 --- a/include/osmium/osm/entity.hpp +++ b/include/osmium/osm/entity.hpp @@ -35,6 +35,7 @@ DEALINGS IN THE SOFTWARE. #include #include +#include namespace osmium { diff --git a/include/osmium/osm/item_type.hpp b/include/osmium/osm/item_type.hpp index c2187a36aed..54975e326d7 100644 --- a/include/osmium/osm/item_type.hpp +++ b/include/osmium/osm/item_type.hpp @@ -33,6 +33,7 @@ DEALINGS IN THE SOFTWARE. */ +#include #include // IWYU pragma: keep #include #include @@ -56,6 +57,25 @@ namespace osmium { }; // enum class item_type + /** + * Return item_type for index: + * 0 -> node, 1 -> way, 2 -> relation + */ + inline item_type nwr_index_to_item_type(unsigned int i) noexcept { + assert(i <= 2); + return item_type(i+1); + } + + /** + * Return index for item_type: + * node -> 0, way -> 1, relation -> 2 + */ + inline unsigned int item_type_to_nwr_index(item_type type) noexcept { + unsigned int i = static_cast(type); + assert(i >= 1 && i <= 3); + return i - 1; + } + inline item_type char_to_item_type(const char c) noexcept { switch (c) { case 'X': diff --git a/include/osmium/osm/node_ref.hpp b/include/osmium/osm/node_ref.hpp index 76afa75fcbd..72359cd0fcf 100644 --- a/include/osmium/osm/node_ref.hpp +++ b/include/osmium/osm/node_ref.hpp @@ -33,11 +33,11 @@ DEALINGS IN THE SOFTWARE. */ +#include #include #include #include -#include #include #include diff --git a/include/osmium/osm/object.hpp b/include/osmium/osm/object.hpp index d5ae48aabe4..8c745ce9b3f 100644 --- a/include/osmium/osm/object.hpp +++ b/include/osmium/osm/object.hpp @@ -48,6 +48,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include namespace osmium { diff --git a/include/osmium/osm/relation.hpp b/include/osmium/osm/relation.hpp index e5b42fed56b..99a4f4cd169 100644 --- a/include/osmium/osm/relation.hpp +++ b/include/osmium/osm/relation.hpp @@ -120,6 +120,11 @@ namespace osmium { return static_cast(std::abs(m_ref)); } + RelationMember& set_ref(const osmium::object_id_type ref) noexcept { + m_ref = ref; + return *this; + } + item_type type() const noexcept { return m_type; } diff --git a/include/osmium/osm/timestamp.hpp b/include/osmium/osm/timestamp.hpp index 6b6a6e18988..390f0e74522 100644 --- a/include/osmium/osm/timestamp.hpp +++ b/include/osmium/osm/timestamp.hpp @@ -39,9 +39,9 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include #include +#include // IWYU pragma: keep namespace osmium { @@ -170,6 +170,16 @@ namespace osmium { return out; } + template <> + inline osmium::Timestamp min_op_start_value() { + return end_of_time(); + } + + template <> + inline osmium::Timestamp max_op_start_value() { + return start_of_time(); + } + } // namespace osmium #endif // OSMIUM_OSM_TIMESTAMP_HPP diff --git a/include/osmium/osm/types.hpp b/include/osmium/osm/types.hpp index aea61bd376f..b3414e59404 100644 --- a/include/osmium/osm/types.hpp +++ b/include/osmium/osm/types.hpp @@ -34,7 +34,6 @@ DEALINGS IN THE SOFTWARE. */ #include -#include namespace osmium { @@ -58,26 +57,6 @@ namespace osmium { */ typedef uint16_t string_size_type; - inline object_id_type string_to_object_id(const char* string) { - return std::atoll(string); - } - - inline object_version_type string_to_object_version(const char* string) { - return static_cast(std::atol(string)); - } - - inline changeset_id_type string_to_changeset_id(const char* string) { - return static_cast(std::atol(string)); - } - - inline signed_user_id_type string_to_user_id(const char* string) { - return static_cast(std::atol(string)); - } - - inline num_changes_type string_to_num_changes(const char* string) { - return static_cast(std::atol(string)); - } - } // namespace osmium #endif // OSMIUM_OSM_TYPES_HPP diff --git a/include/osmium/osm/types_from_string.hpp b/include/osmium/osm/types_from_string.hpp new file mode 100644 index 00000000000..b8de14c90f2 --- /dev/null +++ b/include/osmium/osm/types_from_string.hpp @@ -0,0 +1,116 @@ +#ifndef OSMIUM_OSM_TYPES_FROM_STRING_HPP +#define OSMIUM_OSM_TYPES_FROM_STRING_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace osmium { + + inline object_id_type string_to_object_id(const char* input) { + assert(input); + if (*input != '\0' && !std::isspace(*input)) { + char* end; + auto id = std::strtoll(input, &end, 10); + if (id != std::numeric_limits::min() && id != std::numeric_limits::max() && *end == '\0') { + return id; + } + } + throw std::range_error(std::string("illegal id: '") + input + "'"); + } + + inline std::pair string_to_object_id(const char* input, osmium::osm_entity_bits::type types) { + assert(input); + assert(types != osmium::osm_entity_bits::nothing); + if (*input != '\0') { + if (std::isdigit(*input)) { + return std::make_pair(osmium::item_type::undefined, string_to_object_id(input)); + } + osmium::item_type t = osmium::char_to_item_type(*input); + if (osmium::osm_entity_bits::from_item_type(t) & types) { + return std::make_pair(t, string_to_object_id(input+1)); + } + } + throw std::range_error(std::string("not a valid id: '") + input + "'"); + } + + namespace detail { + + inline long string_to_ulong(const char* input, const char *name) { + if (*input != '\0' && *input != '-' && !std::isspace(*input)) { + char* end; + auto value = std::strtoul(input, &end, 10); + if (value != std::numeric_limits::max() && *end == '\0') { + return value; + } + } + throw std::range_error(std::string("illegal ") + name + ": '" + input + "'"); + } + + } // namespace detail + + inline object_version_type string_to_object_version(const char* input) { + assert(input); + return static_cast(detail::string_to_ulong(input, "version")); + } + + inline changeset_id_type string_to_changeset_id(const char* input) { + assert(input); + return static_cast(detail::string_to_ulong(input, "changeset")); + } + + inline signed_user_id_type string_to_user_id(const char* input) { + assert(input); + if (input[0] == '-' && input[1] == '1' && input[2] == '\0') { + return -1; + } + return static_cast(detail::string_to_ulong(input, "user id")); + } + + inline num_changes_type string_to_num_changes(const char* input) { + assert(input); + return static_cast(detail::string_to_ulong(input, "value for num changes")); + } + +} // namespace osmium + +#endif // OSMIUM_OSM_TYPES_FROM_STRING_HPP diff --git a/include/osmium/relations/collector.hpp b/include/osmium/relations/collector.hpp index e3c4980870f..40e377393b2 100644 --- a/include/osmium/relations/collector.hpp +++ b/include/osmium/relations/collector.hpp @@ -512,7 +512,7 @@ namespace osmium { double percent = static_cast(size_before - size_after); percent /= size_before; percent *= 100; - std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast(percent) << "%)\n"; +// std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast(percent) << "%)\n"; m_count_complete = 0; } } diff --git a/include/osmium/thread/pool.hpp b/include/osmium/thread/pool.hpp index 87dd1fb933e..391603108ff 100644 --- a/include/osmium/thread/pool.hpp +++ b/include/osmium/thread/pool.hpp @@ -149,6 +149,7 @@ namespace osmium { ~Pool() { m_done = true; + m_work_queue.shutdown(); } size_t queue_size() const { diff --git a/include/osmium/thread/queue.hpp b/include/osmium/thread/queue.hpp index baaf2dc766b..76ad9a02031 100644 --- a/include/osmium/thread/queue.hpp +++ b/include/osmium/thread/queue.hpp @@ -41,9 +41,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include - -#include +#include // IWYU pragma: keep (for std::move) namespace osmium { @@ -71,6 +69,8 @@ namespace osmium { /// Used to signal readers when data is available in the queue. std::condition_variable m_data_available; + std::atomic m_done; + #ifdef OSMIUM_DEBUG_QUEUE_SIZE /// The largest size the queue has been so far. size_t m_largest_size; @@ -94,7 +94,8 @@ namespace osmium { m_name(name), m_mutex(), m_queue(), - m_data_available() + m_data_available(), + m_done(false) #ifdef OSMIUM_DEBUG_QUEUE_SIZE , m_largest_size(0), @@ -104,6 +105,7 @@ namespace osmium { } ~Queue() { + shutdown(); #ifdef OSMIUM_DEBUG_QUEUE_SIZE std::cerr << "queue '" << m_name << "' with max_size=" << m_max_size << " had largest size " << m_largest_size << " and was full " << m_full_counter << " times\n"; #endif @@ -132,24 +134,33 @@ namespace osmium { m_data_available.notify_one(); } + void shutdown() { + m_done = true; + m_data_available.notify_all(); + } + void wait_and_pop(T& value) { std::unique_lock lock(m_mutex); m_data_available.wait(lock, [this] { - return !m_queue.empty(); + return !m_queue.empty() || m_done; }); - value = std::move(m_queue.front()); - m_queue.pop(); + if (!m_queue.empty()) { + value = std::move(m_queue.front()); + m_queue.pop(); + } } void wait_and_pop_with_timeout(T& value) { std::unique_lock lock(m_mutex); if (!m_data_available.wait_for(lock, std::chrono::seconds(1), [this] { - return !m_queue.empty(); + return !m_queue.empty() || m_done; })) { return; } - value = std::move(m_queue.front()); - m_queue.pop(); + if (!m_queue.empty()) { + value = std::move(m_queue.front()); + m_queue.pop(); + } } bool try_pop(T& value) { diff --git a/include/osmium/thread/util.hpp b/include/osmium/thread/util.hpp index 62bb82ab502..ca4f6dd538e 100644 --- a/include/osmium/thread/util.hpp +++ b/include/osmium/thread/util.hpp @@ -58,7 +58,7 @@ namespace osmium { /** * Wait until the given future becomes ready. Will block if the future - * is not ready. Can be called more than once unless future.get(). + * is not ready. Can be called more than once unlike future.get(). */ template inline void wait_until_done(std::future& future) { diff --git a/include/osmium/util/data_file.hpp b/include/osmium/util/data_file.hpp new file mode 100644 index 00000000000..22bf1910a62 --- /dev/null +++ b/include/osmium/util/data_file.hpp @@ -0,0 +1,194 @@ +#ifndef OSMIUM_UTIL_DATA_FILE_HPP +#define OSMIUM_UTIL_DATA_FILE_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# include +# include +#endif + +#include + +namespace osmium { + + namespace util { + + /** + * Class wrapper for convenient access to some low-level file + * functions. + */ + class DataFile { + + FILE* m_file; + + public: + + /** + * Create and open a temporary file. It is removed after opening. + * + * @throws std::system_error if something went wrong. + */ + DataFile() : + m_file(::tmpfile()) { + if (!m_file) { + throw std::system_error(errno, std::system_category(), "tmpfile failed"); + } + } + + /** + * Create and open a temporary file with the specified size. It + * is removed after opening. + * + * @throws std::system_error if something went wrong. + */ + explicit DataFile(size_t size) : + DataFile() { + grow(size); + } + + /** + * Create and open a named file. + * + * @param filename the name of the file + * @param writable should the file be writable? + * @throws std::system_error if something went wrong. + */ + DataFile(const char* filename, bool writable) : + m_file(::fopen(filename, writable ? "wb+" : "rb" )) { + if (!m_file) { + throw std::system_error(errno, std::system_category(), "fopen failed"); + } + } + + /** + * Create and open a named file. + * + * @param filename the name of the file + * @param writable should the file be writable? + * @throws std::system_error if something went wrong. + */ + DataFile(const std::string& filename, bool writable) : + DataFile(filename.c_str(), writable) { + } + + /** + * In boolean context the DataFile class returns true if the file + * is open. + */ + operator bool() const noexcept { + return m_file != nullptr; + } + + /** + * Close the file. + * + * Does nothing if the file is already closed. + * + * @throws std::system_error if file could not be closed + */ + void close() { + if (m_file) { + if (::fclose(m_file) != 0) { + throw std::system_error(errno, std::system_category(), "fclose failed"); + } + m_file = nullptr; + } + } + + ~DataFile() noexcept { + try { + close(); + } catch (std::system_error&) { + // ignore + } + } + + /** + * Get file descriptor of underlying file. + * + * @throws std::runtime_errro if file is not open + * @throws std::system_error if fileno(3) call failed + */ + int fd() const { + if (!m_file) { + throw std::runtime_error("no open file"); + } + + int fd = ::fileno(m_file); + + if (fd == -1) { + throw std::system_error(errno, std::system_category(), "fileno failed"); + } + + return fd; + } + + /** + * Ask the operating system for the size of this file. + * + * @throws std::system_error if fstat(2) call failed + */ + size_t size() const { + return osmium::util::file_size(fd()); + } + + /** + * Grow file to given size. + * + * If the file is large enough already, nothing is done. + * The file is never shrunk. + * + * @throws std::system_error if ftruncate(2) call failed + */ + void grow(size_t new_size) const { + if (size() < new_size) { + osmium::util::resize_file(fd(), new_size); + } + } + + }; // class DataFile + + } // namespace util + +} // namespace osmium + + +#endif // OSMIUM_UTIL_DATA_FILE_HPP diff --git a/include/osmium/util/delta.hpp b/include/osmium/util/delta.hpp new file mode 100644 index 00000000000..0c77e52426f --- /dev/null +++ b/include/osmium/util/delta.hpp @@ -0,0 +1,147 @@ +#ifndef OSMIUM_UTIL_DELTA_HPP +#define OSMIUM_UTIL_DELTA_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include + +namespace osmium { + + namespace util { + + /** + * Helper class for delta encoding. + */ + template + class DeltaEncode { + + T m_value; + + public: + + DeltaEncode(T value = 0) : + m_value(value) { + } + + void clear() { + m_value = 0; + } + + T update(T new_value) { + using std::swap; + swap(m_value, new_value); + return m_value - new_value; + } + + }; // class DeltaEncode + + /** + * Helper class for delta decoding. + */ + template + class DeltaDecode { + + T m_value; + + public: + + DeltaDecode() : + m_value(0) { + } + + void clear() { + m_value = 0; + } + + T update(T delta) { + m_value += delta; + return m_value; + } + + }; // class DeltaDecode + + template + class DeltaEncodeIterator : public std::iterator { + + typedef TValue value_type; + + TBaseIterator m_it; + TBaseIterator m_end; + value_type m_delta; + DeltaEncode m_value; + TTransform m_trans; + + public: + + DeltaEncodeIterator(TBaseIterator first, TBaseIterator last, TTransform& trans) : + m_it(first), + m_end(last), + m_delta(m_trans(m_it)), + m_value(m_delta), + m_trans(trans) { + } + + DeltaEncodeIterator& operator++() { + if (m_it != m_end) { + m_delta = m_value.update(m_trans(++m_it)); + } + return *this; + } + + DeltaEncodeIterator operator++(int) { + DeltaEncodeIterator tmp(*this); + operator++(); + return tmp; + } + + value_type operator*() { + return m_delta; + } + + bool operator==(const DeltaEncodeIterator& rhs) const { + return m_it == rhs.m_it && m_end == rhs.m_end; + } + + bool operator!=(const DeltaEncodeIterator& rhs) const { + return !(*this == rhs); + } + + }; // class DeltaEncodeIterator + + } // namespace util + +} // namespace osmium + +#endif // OSMIUM_UTIL_DELTA_HPP diff --git a/include/osmium/util/endian.hpp b/include/osmium/util/endian.hpp new file mode 100644 index 00000000000..a5d91543ef7 --- /dev/null +++ b/include/osmium/util/endian.hpp @@ -0,0 +1,45 @@ +#ifndef OSMIUM_UTIL_ENDIAN_HPP +#define OSMIUM_UTIL_ENDIAN_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +// Windows is only available for little endian architectures +// http://stackoverflow.com/questions/6449468/can-i-safely-assume-that-windows-installations-will-always-be-little-endian +#if !defined(_WIN32) && !defined(__APPLE__) +# include +#else +# define __LITTLE_ENDIAN 1234 +# define __BYTE_ORDER __LITTLE_ENDIAN +#endif + +#endif // OSMIUM_UTIL_ENDIAN_HPP diff --git a/include/osmium/util/file.hpp b/include/osmium/util/file.hpp new file mode 100644 index 00000000000..461f4e6429b --- /dev/null +++ b/include/osmium/util/file.hpp @@ -0,0 +1,119 @@ +#ifndef OSMIUM_UTIL_FILE_HPP +#define OSMIUM_UTIL_FILE_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# include +# include +#endif + +#ifndef _MSC_VER +# include +#else +// https://msdn.microsoft.com/en-us/library/whx354w1.aspx +# define ftruncate _chsize_s +#endif + +namespace osmium { + + namespace util { + + /** + * Get file size. + * This is a small wrapper around a system call. + * + * @param fd File descriptor + * @returns file size + * @throws std::system_error If system call failed + */ + inline size_t file_size(int fd) { +#ifdef _MSC_VER + // Windows implementation + // https://msdn.microsoft.com/en-us/library/dfbc2kec.aspx + auto size = ::_filelengthi64(fd); + if (size == -1L) { + throw std::system_error(errno, std::system_category(), "_filelengthi64 failed"); + } + return size_t(size); +#else + // Unix implementation + struct stat s; + if (::fstat(fd, &s) != 0) { + throw std::system_error(errno, std::system_category(), "fstat failed"); + } + return size_t(s.st_size); +#endif + } + + /** + * Resize file. + * Small wrapper around ftruncate(2) system call. + * + * @param fd File descriptor + * @param new_size New size + * @throws std::system_error If ftruncate(2) call failed + */ + inline void resize_file(int fd, size_t new_size) { + if (::ftruncate(fd, new_size) != 0) { + throw std::system_error(errno, std::system_category(), "ftruncate failed"); + } + } + + /** + * Get the page size for this system. + */ + inline size_t get_pagesize() { +#ifdef _WIN32 + // Windows implementation + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#else + // Unix implementation + return ::sysconf(_SC_PAGESIZE); +#endif + } + + } // namespace util + +} // namespace osmium + +#endif // OSMIUM_UTIL_FILE_HPP diff --git a/include/osmium/util/memory_mapping.hpp b/include/osmium/util/memory_mapping.hpp new file mode 100644 index 00000000000..e48aff2825f --- /dev/null +++ b/include/osmium/util/memory_mapping.hpp @@ -0,0 +1,750 @@ +#ifndef OSMIUM_UTIL_MEMORY_MAPPING_HPP +#define OSMIUM_UTIL_MEMORY_MAPPING_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include + +#include + +#ifndef _WIN32 +# include +#else +# include +# include +# include +#endif + +namespace osmium { + + namespace util { + + /** + * Class for wrapping memory mapping system calls. + * + * Usage for anonymous mapping: + * @code + * MemoryMapping mapping(1024); // create anonymous mapping with size + * auto ptr = mapping.get_addr(); // get pointer to memory + * mapping.unmap(); // release mapping by calling unmap() (or at end of scope) + * @endcode + * + * Or for file-backed mapping: + * @code + * int fd = ::open(...); + * { + * MemoryMapping mapping(1024, MemoryMapping::mapping_mode::write_shared, fd, offset); + * // use mapping + * } + * ::close(fd); + * @endcode + * + * If the file backing a file-backed mapping is not large enough, it + * will be resized. This works, of course, only for writable files, + * so for read-only files you have to make sure they are large enough + * for any mapping you want. + * + * If you ask for a zero-sized mapping, a mapping of the systems page + * size will be created instead. For file-backed mapping this will only + * work if the file is writable. + * + * There are different implementations for Unix and Windows systems. + * On Unix systems this wraps the mmap(), munmap(), and the mremap() + * system calls. On Windows it wraps the CreateFileMapping(), + * CloseHandle(), MapViewOfFile(), and UnmapViewOfFile() functions. + */ + class MemoryMapping { + +public: + enum class mapping_mode { + readonly = 0, + write_private = 1, + write_shared = 2 + }; + +private: + + /// The size of the mapping + size_t m_size; + + /// Offset into the file + off_t m_offset; + + /// File handle we got the mapping from + int m_fd; + + /// Mapping mode + mapping_mode m_mapping_mode; + +#ifdef _WIN32 + HANDLE m_handle; +#endif + + /// The address where the memory is mapped + void* m_addr; + + bool is_valid() const noexcept; + + void make_invalid() noexcept; + +#ifdef _WIN32 + typedef DWORD flag_type; +#else + typedef int flag_type; +#endif + + flag_type get_protection() const noexcept; + + flag_type get_flags() const noexcept; + + // A zero-sized mapping is not allowed by the operating system. + // So if the user asks for a mapping of size 0, we map a full + // page instead. This way we don't have a special case in the rest + // of the code. + static size_t initial_size(size_t size) { + if (size == 0) { + return osmium::util::get_pagesize(); + } + return size; + } + +#ifdef _WIN32 + HANDLE get_handle() const noexcept; + HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept; + void* osmium::util::MemoryMapping::map_view_of_file() const noexcept; +#endif + + int resize_fd(int fd) { + // Anonymous mapping doesn't need resizing. + if (fd == -1) { + return -1; + } + + // Make sure the file backing this mapping is large enough. + if (osmium::util::file_size(fd) < m_size + m_offset) { + osmium::util::resize_file(fd, m_size + m_offset); + } + return fd; + } + + public: + + /** + * Create memory mapping of given size. + * + * If fd is not set (or fd == -1), an anonymous mapping will be + * created, otherwise a mapping based on the file descriptor will + * be created. + * + * @pre size > 0 or mode == write_shared oder write_private + * + * @param size Size of the mapping in bytes + * @param mode Mapping mode: readonly, or writable (shared or private) + * @param fd Open file descriptor of a file we want to map + * @param offset Offset into the file where the mapping should start + * @throws std::system_error if the mapping fails + */ + MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0); + + /// DEPRECATED: For backwards compatibility + MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0) : + MemoryMapping(size, writable ? mapping_mode::write_shared : mapping_mode::readonly, fd, offset) { + } + + /// You can not copy construct a MemoryMapping. + MemoryMapping(const MemoryMapping&) = delete; + + /// You can not copy a MemoryMapping. + MemoryMapping& operator=(const MemoryMapping&) = delete; + + /** + * Move construct a mapping from another one. The other mapping + * will be marked as invalid. + */ + MemoryMapping(MemoryMapping&& other); + + /** + * Move a mapping. The other mapping will be marked as invalid. + */ + MemoryMapping& operator=(MemoryMapping&& other); + + /** + * Releases the mapping by calling unmap(). Will never throw. + * Call unmap() instead if you want to be notified of any error. + */ + ~MemoryMapping() noexcept { + try { + unmap(); + } catch (std::system_error&) { + // ignore + } + } + + /** + * Unmap a mapping. If the mapping is not valid, it will do + * nothing. + * + * @throws std::system_error if the unmapping fails + */ + void unmap(); + + /** + * Resize a mapping to the given new size. + * + * On Linux systems this will use the mremap() function. On other + * systems it will unmap and remap the memory. This can only be + * done for file-based mappings, not anonymous mappings! + * + * @param new_size Number of bytes to resize to + * @throws std::system_error if the remapping fails + */ + void resize(size_t new_size); + + /** + * In a boolean context a MemoryMapping is true when it is a valid + * existing mapping. + */ + operator bool() const noexcept { + return is_valid(); + } + + /** + * The number of bytes mapped. This is the same size you created + * the mapping with. The actual mapping will probably be larger + * because the system will round it to the page size. + */ + size_t size() const noexcept { + return m_size; + } + + /** + * The file descriptor this mapping was created from. + * + * @returns file descriptor, -1 for anonymous mappings + */ + int fd() const noexcept { + return m_fd; + } + + /** + * Was this mapping created as a writable mapping? + */ + bool writable() const noexcept { + return m_mapping_mode != mapping_mode::readonly; + } + + /** + * Get the address of the mapping as any pointer type you like. + * + * @throws std::runtime_error if the mapping is invalid + */ + template + T* get_addr() const { + if (is_valid()) { + return reinterpret_cast(m_addr); + } + throw std::runtime_error("invalid memory mapping"); + } + + }; // class MemoryMapping + + /** + * Anonymous memory mapping. + * + * Usage for anonymous mapping: + * @code + * AnonymousMemoryMapping mapping(1024); // create anonymous mapping with size + * auto ptr = mapping.get_addr(); // get pointer to memory + * mapping.unmap(); // release mapping by calling unmap() (or at end of scope) + * @endcode + */ + class AnonymousMemoryMapping : public MemoryMapping { + + public: + + AnonymousMemoryMapping(size_t size) : + MemoryMapping(size, mapping_mode::write_private) { + } + +#ifndef __linux__ + /** + * On systems other than Linux anonymous mappings can not be + * resized! + */ + void resize(size_t) = delete; +#endif + + }; // class AnonymousMemoryMapping + + /** + * A thin wrapper around the MemoryMapping class used when all the + * data in the mapped memory is of the same type. Instead of thinking + * about the number of bytes mapped, this counts sizes in the number + * of objects of that type. + * + * Note that no effort is made to actually initialize the objects in + * this memory. This has to be done by the caller! + */ + template + class TypedMemoryMapping { + + MemoryMapping m_mapping; + + public: + + /** + * Create anonymous typed memory mapping of given size. + * + * @param size Number of objects of type T to be mapped + * @throws std::system_error if the mapping fails + */ + TypedMemoryMapping(size_t size) : + m_mapping(sizeof(T) * size, MemoryMapping::mapping_mode::write_private) { + } + + /** + * Create file-backed memory mapping of given size. The file must + * contain at least `sizeof(T) * size` bytes! + * + * @param size Number of objects of type T to be mapped + * @param mode Mapping mode: readonly, or writable (shared or private) + * @param fd Open file descriptor of a file we want to map + * @param offset Offset into the file where the mapping should start + * @throws std::system_error if the mapping fails + */ + TypedMemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset = 0) : + m_mapping(sizeof(T) * size, mode, fd, sizeof(T) * offset) { + } + + /// DEPRECATED: For backwards compatibility + TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset = 0) : + m_mapping(sizeof(T) * size, writable ? MemoryMapping::mapping_mode::write_shared : MemoryMapping::mapping_mode::readonly, fd, sizeof(T) * offset) { + } + + /// You can not copy construct a TypedMemoryMapping. + TypedMemoryMapping(const TypedMemoryMapping&) = delete; + + /// You can not copy a MemoryMapping. + TypedMemoryMapping& operator=(const TypedMemoryMapping&) = delete; + + /** + * Move construct a mapping from another one. The other mapping + * will be marked as invalid. + */ + TypedMemoryMapping(TypedMemoryMapping&& other) = default; + + /** + * Move a mapping. The other mapping will be marked as invalid. + */ + TypedMemoryMapping& operator=(TypedMemoryMapping&& other) = default; + + /** + * Releases the mapping by calling unmap(). Will never throw. + * Call unmap() instead if you want to be notified of any error. + */ + ~TypedMemoryMapping() = default; + + /** + * Unmap a mapping. If the mapping is not valid, it will do + * nothing. + * + * @throws std::system_error if the unmapping fails + */ + void unmap() { + m_mapping.unmap(); + } + + /** + * Resize a mapping to the given new size. + * + * On Linux systems this will use the mremap() function. On other + * systems it will unmap and remap the memory. This can only be + * done for file-based mappings, not anonymous mappings! + * + * @param new_size Number of objects of type T to resize to + * @throws std::system_error if the remapping fails + */ + void resize(size_t new_size) { + m_mapping.resize(sizeof(T) * new_size); + } + + /** + * In a boolean context a TypedMemoryMapping is true when it is + * a valid existing mapping. + */ + operator bool() const noexcept { + return !!m_mapping; + } + + /** + * The number of objects of class T mapped. This is the same size + * you created the mapping with. The actual mapping will probably + * be larger because the system will round it to the page size. + */ + size_t size() const noexcept { + assert(m_mapping.size() % sizeof(T) == 0); + return m_mapping.size() / sizeof(T); + } + + /** + * The file descriptor this mapping was created from. + * + * @returns file descriptor, -1 for anonymous mappings + */ + int fd() const noexcept { + return m_mapping.fd(); + } + + /** + * Was this mapping created as a writable mapping? + */ + bool writable() const noexcept { + return m_mapping.writable(); + } + + /** + * Get the address of the beginning of the mapping. + * + * @throws std::runtime_error if the mapping is invalid + */ + T* begin() { + return m_mapping.get_addr(); + } + + /** + * Get the address one past the end of the mapping. + * + * @throws std::runtime_error if the mapping is invalid + */ + T* end() { + return m_mapping.get_addr() + size(); + } + + const T* cbegin() const { + return m_mapping.get_addr(); + } + + const T* cend() const { + return m_mapping.get_addr() + size(); + } + + const T* begin() const { + return m_mapping.get_addr(); + } + + const T* end() const { + return m_mapping.get_addr() + size(); + } + + }; // class TypedMemoryMapping + + template + class AnonymousTypedMemoryMapping : public TypedMemoryMapping { + + public: + + AnonymousTypedMemoryMapping(size_t size) : + TypedMemoryMapping(size) { + } + +#ifndef __linux__ + /** + * On systems other than Linux anonymous mappings can not be + * resized! + */ + void resize(size_t) = delete; +#endif + + }; // class AnonymousTypedMemoryMapping + + } // namespace util + +} // namespace osmium + +#ifndef _WIN32 + +// =========== Unix implementation ============= + +// MAP_FAILED is often a macro containing an old style cast +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wold-style-cast" + +inline bool osmium::util::MemoryMapping::is_valid() const noexcept { + return m_addr != MAP_FAILED; +} + +inline void osmium::util::MemoryMapping::make_invalid() noexcept { + m_addr = MAP_FAILED; +} + +#pragma GCC diagnostic pop + +// for BSD systems +#ifndef MAP_ANONYMOUS +# define MAP_ANONYMOUS MAP_ANON +#endif + +inline int osmium::util::MemoryMapping::get_protection() const noexcept { + if (m_mapping_mode == mapping_mode::readonly) { + return PROT_READ; + } + return PROT_READ | PROT_WRITE; +} + +inline int osmium::util::MemoryMapping::get_flags() const noexcept { + if (m_fd == -1) { + return MAP_PRIVATE | MAP_ANONYMOUS; + } + if (m_mapping_mode == mapping_mode::write_shared) { + return MAP_SHARED; + } + return MAP_PRIVATE; +} + +inline osmium::util::MemoryMapping::MemoryMapping(size_t size, mapping_mode mode, int fd, off_t offset) : + m_size(initial_size(size)), + m_offset(offset), + m_fd(resize_fd(fd)), + m_mapping_mode(mode), + m_addr(::mmap(nullptr, m_size, get_protection(), get_flags(), m_fd, m_offset)) { + assert(!(fd == -1 && mode == mapping_mode::readonly)); + if (!is_valid()) { + throw std::system_error(errno, std::system_category(), "mmap failed"); + } +} + +inline osmium::util::MemoryMapping::MemoryMapping(MemoryMapping&& other) : + m_size(other.m_size), + m_offset(other.m_offset), + m_fd(other.m_fd), + m_mapping_mode(other.m_mapping_mode), + m_addr(other.m_addr) { + other.make_invalid(); +} + +inline osmium::util::MemoryMapping& osmium::util::MemoryMapping::operator=(osmium::util::MemoryMapping&& other) { + unmap(); + m_size = other.m_size; + m_offset = other.m_offset; + m_fd = other.m_fd; + m_mapping_mode = other.m_mapping_mode; + m_addr = other.m_addr; + other.make_invalid(); + return *this; +} + +inline void osmium::util::MemoryMapping::unmap() { + if (is_valid()) { + if (::munmap(m_addr, m_size) != 0) { + throw std::system_error(errno, std::system_category(), "munmap failed"); + } + make_invalid(); + } +} + +inline void osmium::util::MemoryMapping::resize(size_t new_size) { + assert(new_size > 0 && "can not resize to zero size"); + if (m_fd == -1) { // anonymous mapping +#ifdef __linux__ + m_addr = ::mremap(m_addr, m_size, new_size, MREMAP_MAYMOVE); + if (!is_valid()) { + throw std::system_error(errno, std::system_category(), "mremap failed"); + } + m_size = new_size; +#else + assert(false && "can't resize anonymous mappings on non-linux systems"); +#endif + } else { // file-based mapping + unmap(); + m_size = new_size; + resize_fd(m_fd); + m_addr = ::mmap(nullptr, new_size, get_protection(), get_flags(), m_fd, m_offset); + if (!is_valid()) { + throw std::system_error(errno, std::system_category(), "mmap (remap) failed"); + } + } +} + +#else + +// =========== Windows implementation ============= + +/* References: + * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx + * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx + * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx + * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx + */ + +namespace osmium { + + namespace util { + + inline DWORD dword_hi(uint64_t x) { + return static_cast(x >> 32); + } + + inline DWORD dword_lo(uint64_t x) { + return static_cast(x & 0xffffffff); + } + + } // namespace util + +} // namespace osmium + +inline DWORD osmium::util::MemoryMapping::get_protection() const noexcept { + switch (m_mapping_mode) { + case mapping_mode::readonly: + return PAGE_READONLY; + case mapping_mode::write_private: + return PAGE_WRITECOPY; + case mapping_mode::write_shared: + return PAGE_READWRITE; + } +} + +inline DWORD osmium::util::MemoryMapping::get_flags() const noexcept { + switch (m_mapping_mode) { + case mapping_mode::readonly: + return FILE_MAP_READ; + case mapping_mode::write_private: + return FILE_MAP_COPY; + case mapping_mode::write_shared: + return FILE_MAP_WRITE; + } +} + +inline HANDLE osmium::util::MemoryMapping::get_handle() const noexcept { + if (m_fd == -1) { + return INVALID_HANDLE_VALUE; + } + return reinterpret_cast(_get_osfhandle(m_fd)); +} + +inline HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept { + return CreateFileMapping(get_handle(), nullptr, get_protection(), osmium::util::dword_hi(static_cast(m_size) + m_offset), osmium::util::dword_lo(static_cast(m_size) + m_offset), nullptr); +} + +inline void* osmium::util::MemoryMapping::map_view_of_file() const noexcept { + return MapViewOfFile(m_handle, get_flags(), osmium::util::dword_hi(m_offset), osmium::util::dword_lo(m_offset), m_size); +} + +inline bool osmium::util::MemoryMapping::is_valid() const noexcept { + return m_addr != nullptr; +} + +inline void osmium::util::MemoryMapping::make_invalid() noexcept { + m_addr = nullptr; +} + +inline osmium::util::MemoryMapping::MemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset) : + m_size(initial_size(size)), + m_offset(offset), + m_fd(resize_fd(fd)), + m_mapping_mode(mode), + m_handle(create_file_mapping()), + m_addr(nullptr) { + + if (!m_handle) { + throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed"); + } + + m_addr = map_view_of_file(); + if (!is_valid()) { + throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed"); + } +} + +inline osmium::util::MemoryMapping::MemoryMapping(MemoryMapping&& other) : + m_size(other.m_size), + m_offset(other.m_offset), + m_fd(other.m_fd), + m_mapping_mode(other.m_mapping_mode), + m_handle(std::move(other.m_handle)), + m_addr(other.m_addr) { + other.make_invalid(); + other.m_handle = nullptr; +} + +inline osmium::util::MemoryMapping& osmium::util::MemoryMapping::operator=(osmium::util::MemoryMapping&& other) { + unmap(); + m_size = other.m_size; + m_offset = other.m_offset; + m_fd = other.m_fd; + m_mapping_mode = other.m_mapping_mode; + m_handle = std::move(other.m_handle); + m_addr = other.m_addr; + other.make_invalid(); + other.m_handle = nullptr; + return *this; +} + +inline void osmium::util::MemoryMapping::unmap() { + if (is_valid()) { + if (! UnmapViewOfFile(m_addr)) { + throw std::system_error(GetLastError(), std::system_category(), "UnmapViewOfFile failed"); + } + make_invalid(); + } + + if (m_handle) { + if (! CloseHandle(m_handle)) { + throw std::system_error(GetLastError(), std::system_category(), "CloseHandle failed"); + } + m_handle = nullptr; + } +} + +inline void osmium::util::MemoryMapping::resize(size_t new_size) { + unmap(); + + m_size = new_size; + resize_fd(m_fd); + + m_handle = create_file_mapping(); + if (!m_handle) { + throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed"); + } + + m_addr = map_view_of_file(); + if (!is_valid()) { + throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed"); + } +} + +#endif + +#endif // OSMIUM_UTIL_MEMORY_MAPPING_HPP diff --git a/include/osmium/util/minmax.hpp b/include/osmium/util/minmax.hpp new file mode 100644 index 00000000000..2eb601a245b --- /dev/null +++ b/include/osmium/util/minmax.hpp @@ -0,0 +1,120 @@ +#ifndef OSMIUM_UTIL_MINMAX_HPP +#define OSMIUM_UTIL_MINMAX_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include + +namespace osmium { + + template + inline T min_op_start_value() { + return std::numeric_limits::max(); + } + + /** + * Class for calculating the minimum of a bunch of values. + * Works with any numeric type. + * + * Usage: + * + * min_op x; + * x.update(27); + * x.update(12); + * auto min = x.get(); // 12 + */ + template + class min_op { + + T m_value; + + public: + + explicit min_op(T start_value = min_op_start_value()) : + m_value(start_value) { + } + + void update(T value) noexcept { + if (value < m_value) { + m_value = value; + } + } + + T operator()() const noexcept { + return m_value; + } + + }; + + template + inline T max_op_start_value() { + return std::numeric_limits::min(); + } + + /** + * Class for calculating the maximum of a bunch of values. + * Works with any numeric type. + * + * Usage: + * + * max_op x; + * x.update(27); + * x.update(12); + * auto max = x.get(); // 27 + */ + template + class max_op { + + T m_value; + + public: + + explicit max_op(T start_value = max_op_start_value()) : + m_value(start_value) { + } + + void update(T value) noexcept { + if (value > m_value) { + m_value = value; + } + } + + T operator()() const noexcept { + return m_value; + } + + }; + +} // namespace osmium + +#endif // OSMIUM_UTIL_MINMAX_HPP diff --git a/include/osmium/util/string.hpp b/include/osmium/util/string.hpp index 54eb3616b3b..55bfc6cd5e7 100644 --- a/include/osmium/util/string.hpp +++ b/include/osmium/util/string.hpp @@ -43,21 +43,55 @@ namespace osmium { * Split string on the separator character. * * @param str The string to be split. - * @param sep The separastor character. + * @param sep The separator character. + * @param compact Set this to true to remove empty strings from result * @returns Vector with the parts of the string split up. */ - inline std::vector split_string(const std::string& str, const char sep) { + inline std::vector split_string(const std::string& str, const char sep, bool compact = false) { std::vector tokens; if (!str.empty()) { size_t pos = 0; size_t nextpos = str.find_first_of(sep); while (nextpos != std::string::npos) { - tokens.push_back(str.substr(pos, nextpos-pos)); + if (!compact || (nextpos - pos != 0)) { + tokens.push_back(str.substr(pos, nextpos-pos)); + } pos = nextpos + 1; nextpos = str.find_first_of(sep, pos); } - tokens.push_back(str.substr(pos)); + if (!compact || pos != str.size()) { + tokens.push_back(str.substr(pos)); + } + } + + return tokens; + } + + /** + * Split string on the separator character(s). + * + * @param str The string to be split. + * @param sep The separator character(s). + * @param compact Set this to true to remove empty strings from result + * @returns Vector with the parts of the string split up. + */ + inline std::vector split_string(const std::string& str, const char* sep, bool compact = false) { + std::vector tokens; + + if (!str.empty()) { + size_t pos = 0; + size_t nextpos = str.find_first_of(sep); + while (nextpos != std::string::npos) { + if (!compact || (nextpos - pos != 0)) { + tokens.push_back(str.substr(pos, nextpos-pos)); + } + pos = nextpos + 1; + nextpos = str.find_first_of(sep, pos); + } + if (!compact || pos != str.size()) { + tokens.push_back(str.substr(pos)); + } } return tokens; diff --git a/include/protozero/byteswap.hpp b/include/protozero/byteswap.hpp new file mode 100644 index 00000000000..d019c28c5cb --- /dev/null +++ b/include/protozero/byteswap.hpp @@ -0,0 +1,49 @@ +#ifndef PROTOZERO_BYTESWAP_HPP +#define PROTOZERO_BYTESWAP_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#include + +namespace protozero { + +template +inline void byteswap(const char* /*data*/, char* /*result*/) { + assert(false); +} + +template <> +inline void byteswap<1>(const char* data, char* result) { + result[0] = data[0]; +} + +template <> +inline void byteswap<4>(const char* data, char* result) { + result[3] = data[0]; + result[2] = data[1]; + result[1] = data[2]; + result[0] = data[3]; +} + +template <> +inline void byteswap<8>(const char* data, char* result) { + result[7] = data[0]; + result[6] = data[1]; + result[5] = data[2]; + result[4] = data[3]; + result[3] = data[4]; + result[2] = data[5]; + result[1] = data[6]; + result[0] = data[7]; +} + +} // end namespace protozero + +#endif // PROTOZERO_BYTESWAP_HPP diff --git a/include/protozero/exception.hpp b/include/protozero/exception.hpp new file mode 100644 index 00000000000..1229f7dcbb6 --- /dev/null +++ b/include/protozero/exception.hpp @@ -0,0 +1,68 @@ +#ifndef PROTOZERO_EXCEPTION_HPP +#define PROTOZERO_EXCEPTION_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file exception.hpp + * + * @brief Contains the exceptions used in the protozero library. + */ + +#include + +/** + * @brief All parts of the protozero header-only library are in this namespace. + */ +namespace protozero { + +/** + * All exceptions explicitly thrown by the functions of the protozero library + * derive from this exception. + */ +struct exception : std::exception { + /// Returns the explanatory string. + const char *what() const noexcept { return "pbf exception"; } +}; + +/** + * This exception is thrown when parsing a varint thats larger than allowed. + * This should never happen unless the data is corrupted. + */ +struct varint_too_long_exception : exception { + /// Returns the explanatory string. + const char *what() const noexcept { return "varint too long exception"; } +}; + +/** + * This exception is thrown when the wire type of a pdf field is unknown. + * This should never happen unless the data is corrupted. + */ +struct unknown_pbf_wire_type_exception : exception { + /// Returns the explanatory string. + const char *what() const noexcept { return "unknown pbf field type exception"; } +}; + +/** + * This exception is thrown when we are trying to read a field and there + * are not enough bytes left in the buffer to read it. Almost all functions + * of the pbf_reader class can throw this exception. + * + * This should never happen unless the data is corrupted or you have + * initialized the pbf_reader object with incomplete data. + */ +struct end_of_buffer_exception : exception { + /// Returns the explanatory string. + const char *what() const noexcept { return "end of buffer exception"; } +}; + +} // end namespace protozero + +#endif // PROTOZERO_EXCEPTION_HPP diff --git a/include/protozero/pbf_builder.hpp b/include/protozero/pbf_builder.hpp new file mode 100644 index 00000000000..d49a7ba591c --- /dev/null +++ b/include/protozero/pbf_builder.hpp @@ -0,0 +1,111 @@ +#ifndef PROTOZERO_PBF_BUILDER_HPP +#define PROTOZERO_PBF_BUILDER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#include + +#include +#include + +namespace protozero { + +template +class pbf_builder : public pbf_writer { + + static_assert(std::is_same::type>::value, "T must be enum with underlying type protozero::pbf_tag_type"); + +public: + + using enum_type = T; + + pbf_builder(std::string& data) noexcept : + pbf_writer(data) { + } + + template + pbf_builder(pbf_writer& parent_writer, P tag) noexcept : + pbf_writer(parent_writer, pbf_tag_type(tag)) { + } + +#define PROTOZERO_WRITER_WRAP_ADD_SCALAR(name, type) \ + inline void add_##name(T tag, type value) { \ + pbf_writer::add_##name(pbf_tag_type(tag), value); \ + } + + PROTOZERO_WRITER_WRAP_ADD_SCALAR(bool, bool) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(enum, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(int32, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint32, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint32, uint32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(int64, int64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint64, int64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint64, uint64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed32, uint32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed32, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed64, uint64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed64, int64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(float, float) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(double, double) + + inline void add_bytes(T tag, const char* value, size_t size) { + pbf_writer::add_bytes(pbf_tag_type(tag), value, size); + } + + inline void add_bytes(T tag, const std::string& value) { + pbf_writer::add_bytes(pbf_tag_type(tag), value); + } + + inline void add_string(T tag, const char* value, size_t size) { + pbf_writer::add_string(pbf_tag_type(tag), value, size); + } + + inline void add_string(T tag, const std::string& value) { + pbf_writer::add_string(pbf_tag_type(tag), value); + } + + inline void add_string(T tag, const char* value) { + pbf_writer::add_string(pbf_tag_type(tag), value); + } + + inline void add_message(T tag, const char* value, size_t size) { + pbf_writer::add_message(pbf_tag_type(tag), value, size); + } + + inline void add_message(T tag, const std::string& value) { + pbf_writer::add_message(pbf_tag_type(tag), value); + } + +#define PROTOZERO_WRITER_WRAP_ADD_PACKED(name) \ + template \ + inline void add_packed_##name(T tag, InputIterator first, InputIterator last) { \ + pbf_writer::add_packed_##name(pbf_tag_type(tag), first, last); \ + } + + PROTOZERO_WRITER_WRAP_ADD_PACKED(bool) + PROTOZERO_WRITER_WRAP_ADD_PACKED(enum) + PROTOZERO_WRITER_WRAP_ADD_PACKED(int32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sint32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(uint32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(int64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sint64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(uint64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(float) + PROTOZERO_WRITER_WRAP_ADD_PACKED(double) + +}; + +} // end namespace protozero + +#endif // PROTOZERO_PBF_BUILDER_HPP diff --git a/include/protozero/pbf_message.hpp b/include/protozero/pbf_message.hpp new file mode 100644 index 00000000000..af29a00f443 --- /dev/null +++ b/include/protozero/pbf_message.hpp @@ -0,0 +1,50 @@ +#ifndef PROTOZERO_PBF_MESSAGE_HPP +#define PROTOZERO_PBF_MESSAGE_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#include + +#include +#include + +namespace protozero { + +template +class pbf_message : public pbf_reader { + + static_assert(std::is_same::type>::value, "T must be enum with underlying type protozero::pbf_tag_type"); + +public: + + using enum_type = T; + + template + pbf_message(Args&&... args) noexcept : + pbf_reader(std::forward(args)...) { + } + + inline bool next() { + return pbf_reader::next(); + } + + inline bool next(T tag) { + return pbf_reader::next(pbf_tag_type(tag)); + } + + inline T tag() const noexcept { + return T(pbf_reader::tag()); + } + +}; + +} // end namespace protozero + +#endif // PROTOZERO_PBF_MESSAGE_HPP diff --git a/include/protozero/pbf_reader.hpp b/include/protozero/pbf_reader.hpp new file mode 100644 index 00000000000..1c5ed0d7024 --- /dev/null +++ b/include/protozero/pbf_reader.hpp @@ -0,0 +1,1059 @@ +#ifndef PROTOZERO_PBF_READER_HPP +#define PROTOZERO_PBF_READER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_reader.hpp + * + * @brief Contains the pbf_reader class. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#if __BYTE_ORDER != __LITTLE_ENDIAN +# include +#endif + +/// Wrapper for assert() used for testing +#ifndef protozero_assert +# define protozero_assert(x) assert(x) +#endif + +namespace protozero { + +/** + * This class represents a protobuf message. Either a top-level message or + * a nested sub-message. Top-level messages can be created from any buffer + * with a pointer and length: + * + * @code + * std::string buffer; + * // fill buffer... + * pbf_reader message(buffer.data(), buffer.size()); + * @endcode + * + * Sub-messages are created using get_message(): + * + * @code + * pbf_reader message(...); + * message.next(); + * pbf_reader submessage = message.get_message(); + * @endcode + * + * All methods of the pbf_reader class except get_bytes() and get_string() + * provide the strong exception guarantee, ie they either succeed or do not + * change the pbf_reader object they are called on. Use the get_data() method + * instead of get_bytes() or get_string(), if you need this guarantee. + */ +class pbf_reader { + + // A pointer to the next unread data. + const char *m_data = nullptr; + + // A pointer to one past the end of data. + const char *m_end = nullptr; + + // The wire type of the current field. + pbf_wire_type m_wire_type = pbf_wire_type::unknown; + + // The tag of the current field. + pbf_tag_type m_tag = 0; + + template + inline T get_fixed() { + T result; + skip_bytes(sizeof(T)); +#if __BYTE_ORDER == __LITTLE_ENDIAN + memcpy(&result, m_data - sizeof(T), sizeof(T)); +#else + byteswap(m_data - sizeof(T), reinterpret_cast(&result)); +#endif + return result; + } + +#if __BYTE_ORDER == __LITTLE_ENDIAN + template + inline std::pair packed_fixed() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + protozero_assert(len % sizeof(T) == 0); + return std::make_pair(reinterpret_cast(m_data-len), reinterpret_cast(m_data)); + } + +#else + + template + class const_fixed_iterator : public std::iterator { + + const char* m_data; + const char* m_end; + + public: + + const_fixed_iterator() noexcept : + m_data(nullptr), + m_end(nullptr) { + } + + const_fixed_iterator(const char *data, const char* end) noexcept : + m_data(data), + m_end(end) { + } + + const_fixed_iterator(const const_fixed_iterator&) noexcept = default; + const_fixed_iterator(const_fixed_iterator&&) noexcept = default; + + const_fixed_iterator& operator=(const const_fixed_iterator&) noexcept = default; + const_fixed_iterator& operator=(const_fixed_iterator&&) noexcept = default; + + ~const_fixed_iterator() noexcept = default; + + T operator*() { + T result; + byteswap(m_data, reinterpret_cast(&result)); + return result; + } + + const_fixed_iterator& operator++() { + m_data += sizeof(T); + return *this; + } + + const_fixed_iterator operator++(int) { + const const_fixed_iterator tmp(*this); + ++(*this); + return tmp; + } + + bool operator==(const const_fixed_iterator& rhs) const noexcept { + return m_data == rhs.m_data && m_end == rhs.m_end; + } + + bool operator!=(const const_fixed_iterator& rhs) const noexcept { + return !(*this == rhs); + } + + }; // class const_fixed_iterator + + template + inline std::pair, const_fixed_iterator> packed_fixed() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + protozero_assert(len % sizeof(T) == 0); + return std::make_pair(const_fixed_iterator(m_data-len, m_data), + const_fixed_iterator(m_data, m_data)); + } +#endif + + template inline T get_varint(); + template inline T get_svarint(); + + inline pbf_length_type get_length() { return get_varint(); } + + inline void skip_bytes(pbf_length_type len); + + inline pbf_length_type get_len_and_skip(); + +public: + + /** + * Construct a pbf_reader message from a data pointer and a length. The pointer + * will be stored inside the pbf_reader object, no data is copied. So you must + * make sure the buffer stays valid as long as the pbf_reader object is used. + * + * The buffer must contain a complete protobuf message. + * + * @post There is no current field. + */ + inline pbf_reader(const char *data, size_t length) noexcept; + + /** + * Construct a pbf_reader message from a data pointer and a length. The pointer + * will be stored inside the pbf_reader object, no data is copied. So you must + * make sure the buffer stays valid as long as the pbf_reader object is used. + * + * The buffer must contain a complete protobuf message. + * + * @post There is no current field. + */ + inline pbf_reader(std::pair data) noexcept; + + /** + * Construct a pbf_reader message from a std::string. A pointer to the string + * internals will be stored inside the pbf_reader object, no data is copied. + * So you must make sure the string is unchanged as long as the pbf_reader + * object is used. + * + * The string must contain a complete protobuf message. + * + * @post There is no current field. + */ + inline pbf_reader(const std::string& data) noexcept; + + /** + * pbf_reader can be default constructed and behaves like it has an empty + * buffer. + */ + inline pbf_reader() noexcept = default; + + /// pbf_reader messages can be copied trivially. + inline pbf_reader(const pbf_reader&) noexcept = default; + + /// pbf_reader messages can be moved trivially. + inline pbf_reader(pbf_reader&&) noexcept = default; + + /// pbf_reader messages can be copied trivially. + inline pbf_reader& operator=(const pbf_reader& other) noexcept = default; + + /// pbf_reader messages can be moved trivially. + inline pbf_reader& operator=(pbf_reader&& other) noexcept = default; + + inline ~pbf_reader() = default; + + /** + * In a boolean context the pbf_reader class evaluates to `true` if there are + * still fields available and to `false` if the last field has been read. + */ + inline operator bool() const noexcept; + + /** + * Return the length in bytes of the current message. If you have + * already called next() and/or any of the get_*() functions, this will + * return the remaining length. + * + * This can, for instance, be used to estimate the space needed for a + * buffer. Of course you have to know reasonably well what data to expect + * and how it is encoded for this number to have any meaning. + */ + size_t length() const noexcept { + return size_t(m_end - m_data); + } + + /** + * Set next field in the message as the current field. This is usually + * called in a while loop: + * + * @code + * pbf_reader message(...); + * while (message.next()) { + * // handle field + * } + * @endcode + * + * @returns `true` if there is a next field, `false` if not. + * @pre There must be no current field. + * @post If it returns `true` there is a current field now. + */ + inline bool next(); + + /** + * Set next field with given tag in the message as the current field. + * Fields with other tags are skipped. This is usually called in a while + * loop for repeated fields: + * + * @code + * pbf_reader message(...); + * while (message.next(17)) { + * // handle field + * } + * @endcode + * + * or you can call it just once to get the one field with this tag: + * + * @code + * pbf_reader message(...); + * if (message.next(17)) { + * // handle field + * } + * @endcode + * + * @returns `true` if there is a next field with this tag. + * @pre There must be no current field. + * @post If it returns `true` there is a current field now with the given tag. + */ + inline bool next(pbf_tag_type tag); + + /** + * The tag of the current field. The tag is the field number from the + * description in the .proto file. + * + * Call next() before calling this function to set the current field. + * + * @returns tag of the current field. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + inline pbf_tag_type tag() const noexcept; + + /** + * Get the wire type of the current field. The wire types are: + * + * * 0 - varint + * * 1 - 64 bit + * * 2 - length-delimited + * * 5 - 32 bit + * + * All other types are illegal. + * + * Call next() before calling this function to set the current field. + * + * @returns wire type of the current field. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + inline pbf_wire_type wire_type() const noexcept; + + /** + * Check the wire type of the current field. + * + * @returns `true` if the current field has the given wire type. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + inline bool has_wire_type(pbf_wire_type type) const noexcept; + + /** + * Consume the current field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @post The current field was consumed and there is no current field now. + */ + inline void skip(); + + ///@{ + /** + * @name Scalar field accessor functions + */ + + /** + * Consume and return value of current "bool" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bool". + * @post The current field was consumed and there is no current field now. + */ + inline bool get_bool(); + + /** + * Consume and return value of current "enum" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "enum". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_enum() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "int32" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "int32". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_int32() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "sint32" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sint32". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_sint32() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_svarint(); + } + + /** + * Consume and return value of current "uint32" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "uint32". + * @post The current field was consumed and there is no current field now. + */ + inline uint32_t get_uint32() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "int64" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "int64". + * @post The current field was consumed and there is no current field now. + */ + inline int64_t get_int64() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "sint64" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sint64". + * @post The current field was consumed and there is no current field now. + */ + inline int64_t get_sint64() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_svarint(); + } + + /** + * Consume and return value of current "uint64" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "uint64". + * @post The current field was consumed and there is no current field now. + */ + inline uint64_t get_uint64() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "fixed32" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "fixed32". + * @post The current field was consumed and there is no current field now. + */ + inline uint32_t get_fixed32(); + + /** + * Consume and return value of current "sfixed32" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sfixed32". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_sfixed32(); + + /** + * Consume and return value of current "fixed64" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "fixed64". + * @post The current field was consumed and there is no current field now. + */ + inline uint64_t get_fixed64(); + + /** + * Consume and return value of current "sfixed64" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sfixed64". + * @post The current field was consumed and there is no current field now. + */ + inline int64_t get_sfixed64(); + + /** + * Consume and return value of current "float" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "float". + * @post The current field was consumed and there is no current field now. + */ + inline float get_float(); + + /** + * Consume and return value of current "double" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "double". + * @post The current field was consumed and there is no current field now. + */ + inline double get_double(); + + /** + * Consume and return value of current "bytes" or "string" field. + * + * @returns A pair with a pointer to the data and the length of the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bytes" or "string". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_data(); + + /** + * Consume and return value of current "bytes" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bytes". + * @post The current field was consumed and there is no current field now. + */ + inline std::string get_bytes(); + + /** + * Consume and return value of current "string" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "string". + * @post The current field was consumed and there is no current field now. + */ + inline std::string get_string(); + + /** + * Consume and return value of current "message" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "message". + * @post The current field was consumed and there is no current field now. + */ + inline pbf_reader get_message() { + return pbf_reader(get_data()); + } + + ///@} + +private: + + template + class const_varint_iterator : public std::iterator { + + protected: + + const char* m_data; + const char* m_end; + + public: + + const_varint_iterator() noexcept : + m_data(nullptr), + m_end(nullptr) { + } + + const_varint_iterator(const char *data, const char* end) noexcept : + m_data(data), + m_end(end) { + } + + const_varint_iterator(const const_varint_iterator&) noexcept = default; + const_varint_iterator(const_varint_iterator&&) noexcept = default; + + const_varint_iterator& operator=(const const_varint_iterator&) noexcept = default; + const_varint_iterator& operator=(const_varint_iterator&&) noexcept = default; + + ~const_varint_iterator() noexcept = default; + + T operator*() { + const char* d = m_data; // will be thrown away + return static_cast(decode_varint(&d, m_end)); + } + + const_varint_iterator& operator++() { + // Ignore the result, we call decode_varint() just for the + // side-effect of updating m_data. + decode_varint(&m_data, m_end); + return *this; + } + + const_varint_iterator operator++(int) { + const const_varint_iterator tmp(*this); + ++(*this); + return tmp; + } + + bool operator==(const const_varint_iterator& rhs) const noexcept { + return m_data == rhs.m_data && m_end == rhs.m_end; + } + + bool operator!=(const const_varint_iterator& rhs) const noexcept { + return !(*this == rhs); + } + + }; // class const_varint_iterator + + template + class const_svarint_iterator : public const_varint_iterator { + + public: + + const_svarint_iterator() noexcept : + const_varint_iterator() { + } + + const_svarint_iterator(const char *data, const char* end) noexcept : + const_varint_iterator(data, end) { + } + + const_svarint_iterator(const const_svarint_iterator&) = default; + const_svarint_iterator(const_svarint_iterator&&) = default; + + const_svarint_iterator& operator=(const const_svarint_iterator&) = default; + const_svarint_iterator& operator=(const_svarint_iterator&&) = default; + + ~const_svarint_iterator() = default; + + T operator*() { + const char* d = this->m_data; // will be thrown away + return static_cast(decode_zigzag64(decode_varint(&d, this->m_end))); + } + + const_svarint_iterator& operator++() { + // Ignore the result, we call decode_varint() just for the + // side-effect of updating m_data. + decode_varint(&this->m_data, this->m_end); + return *this; + } + + const_svarint_iterator operator++(int) { + const const_svarint_iterator tmp(*this); + ++(*this); + return tmp; + } + + }; // class const_svarint_iterator + +public: + + /// Forward iterator for iterating over bool (int32 varint) values. + typedef const_varint_iterator< int32_t> const_bool_iterator; + + /// Forward iterator for iterating over enum (int32 varint) values. + typedef const_varint_iterator< int32_t> const_enum_iterator; + + /// Forward iterator for iterating over int32 (varint) values. + typedef const_varint_iterator< int32_t> const_int32_iterator; + + /// Forward iterator for iterating over sint32 (varint) values. + typedef const_svarint_iterator const_sint32_iterator; + + /// Forward iterator for iterating over uint32 (varint) values. + typedef const_varint_iterator const_uint32_iterator; + + /// Forward iterator for iterating over int64 (varint) values. + typedef const_varint_iterator< int64_t> const_int64_iterator; + + /// Forward iterator for iterating over sint64 (varint) values. + typedef const_svarint_iterator const_sint64_iterator; + + /// Forward iterator for iterating over uint64 (varint) values. + typedef const_varint_iterator const_uint64_iterator; + + ///@{ + /** + * @name Repeated packed field accessor functions + */ + + /** + * Consume current "repeated packed bool" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed bool". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_bool(); + + /** + * Consume current "repeated packed enum" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed enum". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_enum(); + + /** + * Consume current "repeated packed int32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed int32". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_int32(); + + /** + * Consume current "repeated packed sint32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sint32". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_sint32(); + + /** + * Consume current "repeated packed uint32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed uint32". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_uint32(); + + /** + * Consume current "repeated packed int64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed int64". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_int64(); + + /** + * Consume current "repeated packed sint64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sint64". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_sint64(); + + /** + * Consume current "repeated packed uint64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed uint64". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_uint64(); + + /** + * Consume current "repeated packed fixed32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed fixed32". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_fixed32() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed sfixed32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sfixed32". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_sfixed32() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed fixed64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed fixed64". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_fixed64() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed sfixed64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sfixed64". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_sfixed64() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed float" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed float". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_float() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed double" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed double". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_double() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + ///@} + +}; // class pbf_reader + +pbf_reader::pbf_reader(const char *data, size_t length) noexcept + : m_data(data), + m_end(data + length), + m_wire_type(pbf_wire_type::unknown), + m_tag(0) { +} + +pbf_reader::pbf_reader(std::pair data) noexcept + : m_data(data.first), + m_end(data.first + data.second), + m_wire_type(pbf_wire_type::unknown), + m_tag(0) { +} + +pbf_reader::pbf_reader(const std::string& data) noexcept + : m_data(data.data()), + m_end(data.data() + data.size()), + m_wire_type(pbf_wire_type::unknown), + m_tag(0) { +} + +pbf_reader::operator bool() const noexcept { + return m_data < m_end; +} + +bool pbf_reader::next() { + if (m_data == m_end) { + return false; + } + + auto value = get_varint(); + m_tag = value >> 3; + + // tags 0 and 19000 to 19999 are not allowed as per + // https://developers.google.com/protocol-buffers/docs/proto + protozero_assert(((m_tag > 0 && m_tag < 19000) || (m_tag > 19999 && m_tag <= ((1 << 29) - 1))) && "tag out of range"); + + m_wire_type = pbf_wire_type(value & 0x07); +// XXX do we want this check? or should it throw an exception? +// protozero_assert((m_wire_type <=2 || m_wire_type == 5) && "illegal wire type"); + return true; +} + +bool pbf_reader::next(pbf_tag_type requested_tag) { + while (next()) { + if (m_tag == requested_tag) { + return true; + } else { + skip(); + } + } + return false; +} + +pbf_tag_type pbf_reader::tag() const noexcept { + return m_tag; +} + +pbf_wire_type pbf_reader::wire_type() const noexcept { + return m_wire_type; +} + +bool pbf_reader::has_wire_type(pbf_wire_type type) const noexcept { + return wire_type() == type; +} + +void pbf_reader::skip_bytes(pbf_length_type len) { + if (m_data + len > m_end) { + throw end_of_buffer_exception(); + } + m_data += len; + +// In debug builds reset the tag to zero so that we can detect (some) +// wrong code. +#ifndef NDEBUG + m_tag = 0; +#endif +} + +void pbf_reader::skip() { + protozero_assert(tag() != 0 && "call next() before calling skip()"); + switch (wire_type()) { + case pbf_wire_type::varint: + (void)get_uint32(); // called for the side-effect of skipping value + break; + case pbf_wire_type::fixed64: + skip_bytes(8); + break; + case pbf_wire_type::length_delimited: + skip_bytes(get_length()); + break; + case pbf_wire_type::fixed32: + skip_bytes(4); + break; + default: + throw unknown_pbf_wire_type_exception(); + } +} + +pbf_length_type pbf_reader::get_len_and_skip() { + auto len = get_length(); + skip_bytes(len); + return len; +} + +template +T pbf_reader::get_varint() { + return static_cast(decode_varint(&m_data, m_end)); +} + +template +T pbf_reader::get_svarint() { + protozero_assert((has_wire_type(pbf_wire_type::varint) || has_wire_type(pbf_wire_type::length_delimited)) && "not a varint"); + return static_cast(decode_zigzag64(decode_varint(&m_data, m_end))); +} + +uint32_t pbf_reader::get_fixed32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed"); + return get_fixed(); +} + +int32_t pbf_reader::get_sfixed32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed"); + return get_fixed(); +} + +uint64_t pbf_reader::get_fixed64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed"); + return get_fixed(); +} + +int64_t pbf_reader::get_sfixed64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed"); + return get_fixed(); +} + +float pbf_reader::get_float() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed"); + return get_fixed(); +} + +double pbf_reader::get_double() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed"); + return get_fixed(); +} + +bool pbf_reader::get_bool() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + protozero_assert((*m_data & 0x80) == 0 && "not a 1 byte varint"); + skip_bytes(1); + return m_data[-1] != 0; // -1 okay because we incremented m_data the line before +} + +std::pair pbf_reader::get_data() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::length_delimited) && "not of type string, bytes or message"); + auto len = get_len_and_skip(); + return std::make_pair(m_data-len, len); +} + +std::string pbf_reader::get_bytes() { + auto d = get_data(); + return std::string(d.first, d.second); +} + +std::string pbf_reader::get_string() { + return get_bytes(); +} + +std::pair pbf_reader::get_packed_bool() { + return get_packed_int32(); +} + +std::pair pbf_reader::get_packed_enum() { + return get_packed_int32(); +} + +std::pair pbf_reader::get_packed_int32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_int32_iterator(m_data-len, m_data), + pbf_reader::const_int32_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_uint32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_uint32_iterator(m_data-len, m_data), + pbf_reader::const_uint32_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_sint32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_sint32_iterator(m_data-len, m_data), + pbf_reader::const_sint32_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_int64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_int64_iterator(m_data-len, m_data), + pbf_reader::const_int64_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_uint64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_uint64_iterator(m_data-len, m_data), + pbf_reader::const_uint64_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_sint64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_sint64_iterator(m_data-len, m_data), + pbf_reader::const_sint64_iterator(m_data, m_data)); +} + +} // end namespace protozero + +#endif // PROTOZERO_PBF_READER_HPP diff --git a/include/protozero/pbf_types.hpp b/include/protozero/pbf_types.hpp new file mode 100644 index 00000000000..9f38584f439 --- /dev/null +++ b/include/protozero/pbf_types.hpp @@ -0,0 +1,49 @@ +#ifndef PROTOZERO_PBF_TYPES_HPP +#define PROTOZERO_PBF_TYPES_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_types.hpp + * + * @brief Contains the declaration of low-level types used in the pbf format. + */ + +#include + +namespace protozero { + + /** + * The type used for field tags (field numbers). + */ + typedef uint32_t pbf_tag_type; + + /** + * The type used to encode type information. + * See the table on + * https://developers.google.com/protocol-buffers/docs/encoding + */ + enum class pbf_wire_type : uint32_t { + varint = 0, // int32/64, uint32/64, sint32/64, bool, enum + fixed64 = 1, // fixed64, sfixed64, double + length_delimited = 2, // string, bytes, embedded messages, + // packed repeated fields + fixed32 = 5, // fixed32, sfixed32, float + unknown = 99 // used for default setting in this library + }; + + /** + * The type used for length values, such as the length of a field. + */ + typedef uint32_t pbf_length_type; + +} // end namespace protozero + +#endif // PROTOZERO_PBF_TYPES_HPP diff --git a/include/protozero/pbf_writer.hpp b/include/protozero/pbf_writer.hpp new file mode 100644 index 00000000000..53cbfdf1c49 --- /dev/null +++ b/include/protozero/pbf_writer.hpp @@ -0,0 +1,664 @@ +#ifndef PROTOZERO_PBF_WRITER_HPP +#define PROTOZERO_PBF_WRITER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_writer.hpp + * + * @brief Contains the pbf_writer class. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#if __BYTE_ORDER != __LITTLE_ENDIAN +# include +#endif + +/// Wrapper for assert() used for testing +#ifndef protozero_assert +# define protozero_assert(x) assert(x) +#endif + +namespace protozero { + +/** + * The pbf_writer is used to write PBF formatted messages into a buffer. + * + * Almost all methods in this class can throw an std::bad_alloc exception if + * the std::string used as a buffer wants to resize. + */ +class pbf_writer { + + std::string* m_data; + pbf_writer* m_parent_writer; + size_t m_pos = 0; + + inline void add_varint(uint64_t value) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); + protozero_assert(m_data); + write_varint(std::back_inserter(*m_data), value); + } + + inline void add_field(pbf_tag_type tag, pbf_wire_type type) { + protozero_assert(((tag > 0 && tag < 19000) || (tag > 19999 && tag <= ((1 << 29) - 1))) && "tag out of range"); + uint32_t b = (tag << 3) | uint32_t(type); + add_varint(b); + } + + inline void add_tagged_varint(pbf_tag_type tag, uint64_t value) { + add_field(tag, pbf_wire_type::varint); + add_varint(value); + } + + template + inline void add_fixed(T value) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); + protozero_assert(m_data); +#if __BYTE_ORDER == __LITTLE_ENDIAN + m_data->append(reinterpret_cast(&value), sizeof(T)); +#else + auto size = m_data->size(); + m_data->resize(size + sizeof(T)); + byteswap(reinterpret_cast(&value), const_cast(m_data->data() + size)); +#endif + } + + template + inline void add_packed_fixed(pbf_tag_type tag, It first, It last, std::input_iterator_tag) { + if (first == last) { + return; + } + + pbf_writer sw(*this, tag); + + while (first != last) { + sw.add_fixed(*first++); + } + } + + template + inline void add_packed_fixed(pbf_tag_type tag, It first, It last, std::forward_iterator_tag) { + if (first == last) { + return; + } + + add_length_varint(tag, sizeof(T) * pbf_length_type(std::distance(first, last))); + + while (first != last) { + add_fixed(*first++); + } + } + + template + inline void add_packed_varint(pbf_tag_type tag, It first, It last) { + if (first == last) { + return; + } + + pbf_writer sw(*this, tag); + + while (first != last) { + sw.add_varint(uint64_t(*first++)); + } + } + + template + inline void add_packed_svarint(pbf_tag_type tag, It first, It last) { + if (first == last) { + return; + } + + pbf_writer sw(*this, tag); + + while (first != last) { + sw.add_varint(encode_zigzag64(*first++)); + } + } + + // The number of bytes to reserve for the varint holding the length of + // a length-delimited field. The length has to fit into pbf_length_type, + // and a varint needs 8 bit for every 7 bit. + static const int reserve_bytes = sizeof(pbf_length_type) * 8 / 7 + 1; + + inline void open_submessage(pbf_tag_type tag) { + protozero_assert(m_pos == 0); + protozero_assert(m_data); + add_field(tag, pbf_wire_type::length_delimited); + m_data->append(size_t(reserve_bytes), '\0'); + m_pos = m_data->size(); + } + + inline void close_submessage() { + protozero_assert(m_pos != 0); + protozero_assert(m_data); + auto length = pbf_length_type(m_data->size() - m_pos); + + protozero_assert(m_data->size() >= m_pos - reserve_bytes); + auto n = write_varint(m_data->begin() + long(m_pos) - reserve_bytes, length); + + m_data->erase(m_data->begin() + long(m_pos) - reserve_bytes + n, m_data->begin() + long(m_pos)); + m_pos = 0; + } + + inline void add_length_varint(pbf_tag_type tag, pbf_length_type length) { + add_field(tag, pbf_wire_type::length_delimited); + add_varint(length); + } + +public: + + /** + * Create a writer using the given string as a data store. The pbf_writer + * stores a reference to that string and adds all data to it. + */ + inline explicit pbf_writer(std::string& data) noexcept : + m_data(&data), + m_parent_writer(nullptr), + m_pos(0) { + } + + /** + * Create a writer without a data store. In this form the writer can not + * be used! + */ + inline pbf_writer() noexcept : + m_data(nullptr), + m_parent_writer(nullptr), + m_pos(0) { + } + + /** + * Construct a pbf_writer for a submessage from the pbf_writer of the + * parent message. + * + * @param parent_writer The pbf_writer + * @param tag Tag (field number) of the field that will be written + */ + inline pbf_writer(pbf_writer& parent_writer, pbf_tag_type tag) : + m_data(parent_writer.m_data), + m_parent_writer(&parent_writer), + m_pos(0) { + m_parent_writer->open_submessage(tag); + } + + /// A pbf_writer object can be copied + pbf_writer(const pbf_writer&) noexcept = default; + + /// A pbf_writer object can be copied + pbf_writer& operator=(const pbf_writer&) noexcept = default; + + /// A pbf_writer object can be moved + inline pbf_writer(pbf_writer&&) noexcept = default; + + /// A pbf_writer object can be moved + inline pbf_writer& operator=(pbf_writer&&) noexcept = default; + + inline ~pbf_writer() { + if (m_parent_writer) { + m_parent_writer->close_submessage(); + } + } + + ///@{ + /** + * @name Scalar field writer functions + */ + + /** + * Add "bool" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_bool(pbf_tag_type tag, bool value) { + add_field(tag, pbf_wire_type::varint); + add_fixed(value); + } + + /** + * Add "enum" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_enum(pbf_tag_type tag, int32_t value) { + add_tagged_varint(tag, uint64_t(value)); + } + + /** + * Add "int32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_int32(pbf_tag_type tag, int32_t value) { + add_tagged_varint(tag, uint64_t(value)); + } + + /** + * Add "sint32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sint32(pbf_tag_type tag, int32_t value) { + add_tagged_varint(tag, encode_zigzag32(value)); + } + + /** + * Add "uint32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_uint32(pbf_tag_type tag, uint32_t value) { + add_tagged_varint(tag, value); + } + + /** + * Add "int64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_int64(pbf_tag_type tag, int64_t value) { + add_tagged_varint(tag, uint64_t(value)); + } + + /** + * Add "sint64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sint64(pbf_tag_type tag, int64_t value) { + add_tagged_varint(tag, encode_zigzag64(value)); + } + + /** + * Add "uint64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_uint64(pbf_tag_type tag, uint64_t value) { + add_tagged_varint(tag, value); + } + + /** + * Add "fixed32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_fixed32(pbf_tag_type tag, uint32_t value) { + add_field(tag, pbf_wire_type::fixed32); + add_fixed(value); + } + + /** + * Add "sfixed32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sfixed32(pbf_tag_type tag, int32_t value) { + add_field(tag, pbf_wire_type::fixed32); + add_fixed(value); + } + + /** + * Add "fixed64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_fixed64(pbf_tag_type tag, uint64_t value) { + add_field(tag, pbf_wire_type::fixed64); + add_fixed(value); + } + + /** + * Add "sfixed64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sfixed64(pbf_tag_type tag, int64_t value) { + add_field(tag, pbf_wire_type::fixed64); + add_fixed(value); + } + + /** + * Add "float" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_float(pbf_tag_type tag, float value) { + add_field(tag, pbf_wire_type::fixed32); + add_fixed(value); + } + + /** + * Add "double" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_double(pbf_tag_type tag, double value) { + add_field(tag, pbf_wire_type::fixed64); + add_fixed(value); + } + + /** + * Add "bytes" field to data. + * + * @param tag Tag (field number) of the field + * @param value Pointer to value to be written + * @param size Number of bytes to be written + */ + inline void add_bytes(pbf_tag_type tag, const char* value, size_t size) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); + protozero_assert(m_data); + assert(size <= std::numeric_limits::max()); + add_length_varint(tag, pbf_length_type(size)); + m_data->append(value, size); + } + + /** + * Add "bytes" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_bytes(pbf_tag_type tag, const std::string& value) { + add_bytes(tag, value.data(), value.size()); + } + + /** + * Add "string" field to data. + * + * @param tag Tag (field number) of the field + * @param value Pointer to value to be written + * @param size Number of bytes to be written + */ + inline void add_string(pbf_tag_type tag, const char* value, size_t size) { + add_bytes(tag, value, size); + } + + /** + * Add "string" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_string(pbf_tag_type tag, const std::string& value) { + add_bytes(tag, value.data(), value.size()); + } + + /** + * Add "string" field to data. Bytes from the value are written until + * a null byte is encountered. The null byte is not added. + * + * @param tag Tag (field number) of the field + * @param value Pointer to value to be written + */ + inline void add_string(pbf_tag_type tag, const char* value) { + add_bytes(tag, value, std::strlen(value)); + } + + /** + * Add "message" field to data. + * + * @param tag Tag (field number) of the field + * @param value Pointer to message to be written + * @param size Length of the message + */ + inline void add_message(pbf_tag_type tag, const char* value, size_t size) { + add_bytes(tag, value, size); + } + + /** + * Add "message" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written. The value must be a complete message. + */ + inline void add_message(pbf_tag_type tag, const std::string& value) { + add_bytes(tag, value.data(), value.size()); + } + + ///@} + + ///@{ + /** + * @name Repeated packed field writer functions + */ + + /** + * Add "repeated packed bool" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to bool. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_bool(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed enum" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_enum(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed int32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_int32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed sint32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sint32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_svarint(tag, first, last); + } + + /** + * Add "repeated packed uint32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_uint32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed int64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_int64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed sint64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sint64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_svarint(tag, first, last); + } + + /** + * Add "repeated packed uint64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_uint64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed fixed32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_fixed32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed sfixed32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sfixed32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed fixed64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_fixed64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed sfixed64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sfixed64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed float" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to float. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_float(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed double" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to double. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_double(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + ///@} + +}; // class pbf_writer + +} // end namespace protozero + +#endif // PROTOZERO_PBF_WRITER_HPP diff --git a/include/protozero/varint.hpp b/include/protozero/varint.hpp new file mode 100644 index 00000000000..bc9c3296d62 --- /dev/null +++ b/include/protozero/varint.hpp @@ -0,0 +1,136 @@ +#ifndef PROTOZERO_VARINT_HPP +#define PROTOZERO_VARINT_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file varint.hpp + * + * @brief Contains low-level varint and zigzag encoding and decoding functions. + */ + +#if __BYTE_ORDER != __LITTLE_ENDIAN +# error "This code only works on little endian machines." +#endif + +#include + +#include + +namespace protozero { + +/** + * The maximum length of a 64bit varint. + */ +const int8_t max_varint_length = sizeof(uint64_t) * 8 / 7 + 1; + +// from https://github.com/facebook/folly/blob/master/folly/Varint.h +/** + * Decode a 64bit varint. + * + * String exception guarantee: if there is an exception the data pointer will + * not be changed. + * + * @param[in,out] data Pointer to pointer to the input data. After the function + * returns this will point to the next data to be read. + * @param[in] end Pointer one past the end of the input data. + * @returns The decoded integer + * @throws varint_too_long_exception if the varint is longer then the maximum + * length that would fit in a 64bit int. Usually this means your data + * is corrupted or you are trying to read something as a varint that + * isn't. + * @throws end_of_buffer_exception if the *end* of the buffer was reached + * before the end of the varint. + */ +inline uint64_t decode_varint(const char** data, const char* end) { + const int8_t* begin = reinterpret_cast(*data); + const int8_t* iend = reinterpret_cast(end); + const int8_t* p = begin; + uint64_t val = 0; + + if (iend - begin >= max_varint_length) { // fast path + do { + int64_t b; + b = *p++; val = uint64_t((b & 0x7f) ); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 7); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 14); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 21); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 28); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 35); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 42); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 49); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 56); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 63); if (b >= 0) break; + throw varint_too_long_exception(); + } while (false); + } else { + int shift = 0; + while (p != iend && *p < 0) { + val |= uint64_t(*p++ & 0x7f) << shift; + shift += 7; + } + if (p == iend) { + throw end_of_buffer_exception(); + } + val |= uint64_t(*p++) << shift; + } + + *data = reinterpret_cast(p); + return val; +} + +/** + * Varint-encode a 64bit integer. + */ +template +inline int write_varint(OutputIterator data, uint64_t value) { + int n=1; + + while (value >= 0x80) { + *data++ = char((value & 0x7f) | 0x80); + value >>= 7; + ++n; + } + *data++ = char(value); + + return n; +} + +/** + * ZigZag encodes a 32 bit integer. + */ +inline uint32_t encode_zigzag32(int32_t value) noexcept { + return (static_cast(value) << 1) ^ (static_cast(value >> 31)); +} + +/** + * ZigZag encodes a 64 bit integer. + */ +inline uint64_t encode_zigzag64(int64_t value) noexcept { + return (static_cast(value) << 1) ^ (static_cast(value >> 63)); +} + +/** + * Decodes a 32 bit ZigZag-encoded integer. + */ +inline int32_t decode_zigzag32(uint32_t value) noexcept { + return int32_t(value >> 1) ^ -int32_t(value & 1); +} + +/** + * Decodes a 64 bit ZigZag-encoded integer. + */ +inline int64_t decode_zigzag64(uint64_t value) noexcept { + return int64_t(value >> 1) ^ -int64_t(value & 1); +} + +} // end namespace protozero + +#endif // PROTOZERO_VARINT_HPP diff --git a/include/utf8.h b/include/utf8.h new file mode 100644 index 00000000000..82b13f59f98 --- /dev/null +++ b/include/utf8.h @@ -0,0 +1,34 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "utf8/checked.h" +#include "utf8/unchecked.h" + +#endif // header guard diff --git a/include/utf8/checked.h b/include/utf8/checked.h new file mode 100644 index 00000000000..13311551381 --- /dev/null +++ b/include/utf8/checked.h @@ -0,0 +1,327 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "core.h" +#include + +namespace utf8 +{ + // Base for the exceptions that may be thrown from the library + class exception : public ::std::exception { + }; + + // Exceptions that may be thrown from the library functions. + class invalid_code_point : public exception { + uint32_t cp; + public: + invalid_code_point(uint32_t cp) : cp(cp) {} + virtual const char* what() const throw() { return "Invalid code point"; } + uint32_t code_point() const {return cp;} + }; + + class invalid_utf8 : public exception { + uint8_t u8; + public: + invalid_utf8 (uint8_t u) : u8(u) {} + virtual const char* what() const throw() { return "Invalid UTF-8"; } + uint8_t utf8_octet() const {return u8;} + }; + + class invalid_utf16 : public exception { + uint16_t u16; + public: + invalid_utf16 (uint16_t u) : u16(u) {} + virtual const char* what() const throw() { return "Invalid UTF-16"; } + uint16_t utf16_word() const {return u16;} + }; + + class not_enough_room : public exception { + public: + virtual const char* what() const throw() { return "Not enough space"; } + }; + + /// The library API - functions intended to be called by the users + + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (!utf8::internal::is_code_point_valid(cp)) + throw invalid_code_point(cp); + + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + + template + output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement) + { + while (start != end) { + octet_iterator sequence_start = start; + internal::utf_error err_code = utf8::internal::validate_next(start, end); + switch (err_code) { + case internal::UTF8_OK : + for (octet_iterator it = sequence_start; it != start; ++it) + *out++ = *it; + break; + case internal::NOT_ENOUGH_ROOM: + throw not_enough_room(); + case internal::INVALID_LEAD: + out = utf8::append (replacement, out); + ++start; + break; + case internal::INCOMPLETE_SEQUENCE: + case internal::OVERLONG_SEQUENCE: + case internal::INVALID_CODE_POINT: + out = utf8::append (replacement, out); + ++start; + // just one replacement mark for the sequence + while (start != end && utf8::internal::is_trail(*start)) + ++start; + break; + } + } + return out; + } + + template + inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out) + { + static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd); + return utf8::replace_invalid(start, end, out, replacement_marker); + } + + template + uint32_t next(octet_iterator& it, octet_iterator end) + { + uint32_t cp = 0; + internal::utf_error err_code = utf8::internal::validate_next(it, end, cp); + switch (err_code) { + case internal::UTF8_OK : + break; + case internal::NOT_ENOUGH_ROOM : + throw not_enough_room(); + case internal::INVALID_LEAD : + case internal::INCOMPLETE_SEQUENCE : + case internal::OVERLONG_SEQUENCE : + throw invalid_utf8(*it); + case internal::INVALID_CODE_POINT : + throw invalid_code_point(cp); + } + return cp; + } + + template + uint32_t peek_next(octet_iterator it, octet_iterator end) + { + return utf8::next(it, end); + } + + template + uint32_t prior(octet_iterator& it, octet_iterator start) + { + // can't do much if it == start + if (it == start) + throw not_enough_room(); + + octet_iterator end = it; + // Go back until we hit either a lead octet or start + while (utf8::internal::is_trail(*(--it))) + if (it == start) + throw invalid_utf8(*it); // error - no lead byte in the sequence + return utf8::peek_next(it, end); + } + + /// Deprecated in versions that include "prior" + template + uint32_t previous(octet_iterator& it, octet_iterator pass_start) + { + octet_iterator end = it; + while (utf8::internal::is_trail(*(--it))) + if (it == pass_start) + throw invalid_utf8(*it); // error - no lead byte in the sequence + octet_iterator temp = it; + return utf8::next(temp, end); + } + + template + void advance (octet_iterator& it, distance_type n, octet_iterator end) + { + for (distance_type i = 0; i < n; ++i) + utf8::next(it, end); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + utf8::next(first, last); + return dist; + } + + template + octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = utf8::internal::mask16(*start++); + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { + if (start != end) { + uint32_t trail_surrogate = utf8::internal::mask16(*start++); + if (utf8::internal::is_trail_surrogate(trail_surrogate)) + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; + else + throw invalid_utf16(static_cast(trail_surrogate)); + } + else + throw invalid_utf16(static_cast(cp)); + + } + // Lone trail surrogate + else if (utf8::internal::is_trail_surrogate(cp)) + throw invalid_utf16(static_cast(cp)); + + result = utf8::append(cp, result); + } + return result; + } + + template + u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start != end) { + uint32_t cp = utf8::next(start, end); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + return result; + } + + template + octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + result = utf8::append(*(start++), result); + + return result; + } + + template + u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start != end) + (*result++) = utf8::next(start, end); + + return result; + } + + // The iterator class + template + class iterator : public std::iterator { + octet_iterator it; + octet_iterator range_start; + octet_iterator range_end; + public: + iterator () {} + explicit iterator (const octet_iterator& octet_it, + const octet_iterator& range_start, + const octet_iterator& range_end) : + it(octet_it), range_start(range_start), range_end(range_end) + { + if (it < range_start || it > range_end) + throw std::out_of_range("Invalid utf-8 iterator position"); + } + // the default "big three" are OK + octet_iterator base () const { return it; } + uint32_t operator * () const + { + octet_iterator temp = it; + return utf8::next(temp, range_end); + } + bool operator == (const iterator& rhs) const + { + if (range_start != rhs.range_start || range_end != rhs.range_end) + throw std::logic_error("Comparing utf-8 iterators defined with different ranges"); + return (it == rhs.it); + } + bool operator != (const iterator& rhs) const + { + return !(operator == (rhs)); + } + iterator& operator ++ () + { + utf8::next(it, range_end); + return *this; + } + iterator operator ++ (int) + { + iterator temp = *this; + utf8::next(it, range_end); + return temp; + } + iterator& operator -- () + { + utf8::prior(it, range_start); + return *this; + } + iterator operator -- (int) + { + iterator temp = *this; + utf8::prior(it, range_start); + return temp; + } + }; // class iterator + +} // namespace utf8 + +#endif //header guard + + diff --git a/include/utf8/core.h b/include/utf8/core.h new file mode 100644 index 00000000000..693d388c078 --- /dev/null +++ b/include/utf8/core.h @@ -0,0 +1,329 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include + +namespace utf8 +{ + // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers + // You may need to change them to match your system. + // These typedefs have the same names as ones from cstdint, or boost/cstdint + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + +// Helper code - not intended to be directly called by the library users. May be changed at any time +namespace internal +{ + // Unicode constants + // Leading (high) surrogates: 0xd800 - 0xdbff + // Trailing (low) surrogates: 0xdc00 - 0xdfff + const uint16_t LEAD_SURROGATE_MIN = 0xd800u; + const uint16_t LEAD_SURROGATE_MAX = 0xdbffu; + const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u; + const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu; + const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10); + const uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN; + + // Maximum valid value for a Unicode code point + const uint32_t CODE_POINT_MAX = 0x0010ffffu; + + template + inline uint8_t mask8(octet_type oc) + { + return static_cast(0xff & oc); + } + template + inline uint16_t mask16(u16_type oc) + { + return static_cast(0xffff & oc); + } + template + inline bool is_trail(octet_type oc) + { + return ((utf8::internal::mask8(oc) >> 6) == 0x2); + } + + template + inline bool is_lead_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX); + } + + template + inline bool is_trail_surrogate(u16 cp) + { + return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + template + inline bool is_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + template + inline bool is_code_point_valid(u32 cp) + { + return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp)); + } + + template + inline typename std::iterator_traits::difference_type + sequence_length(octet_iterator lead_it) + { + uint8_t lead = utf8::internal::mask8(*lead_it); + if (lead < 0x80) + return 1; + else if ((lead >> 5) == 0x6) + return 2; + else if ((lead >> 4) == 0xe) + return 3; + else if ((lead >> 3) == 0x1e) + return 4; + else + return 0; + } + + template + inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length) + { + if (cp < 0x80) { + if (length != 1) + return true; + } + else if (cp < 0x800) { + if (length != 2) + return true; + } + else if (cp < 0x10000) { + if (length != 3) + return true; + } + + return false; + } + + enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT}; + + /// Helper for get_sequence_x + template + utf_error increase_safely(octet_iterator& it, octet_iterator end) + { + if (++it == end) + return NOT_ENOUGH_ROOM; + + if (!utf8::internal::is_trail(*it)) + return INCOMPLETE_SEQUENCE; + + return UTF8_OK; + } + + #define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;} + + /// get_sequence_x functions decode utf-8 sequences of the length x + template + utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + return UTF8_OK; + } + + template + utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f); + + return UTF8_OK; + } + + template + utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (*it) & 0x3f; + + return UTF8_OK; + } + + template + utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (utf8::internal::mask8(*it) << 6) & 0xfff; + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (*it) & 0x3f; + + return UTF8_OK; + } + + #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR + + template + utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + // Save the original value of it so we can go back in case of failure + // Of course, it does not make much sense with i.e. stream iterators + octet_iterator original_it = it; + + uint32_t cp = 0; + // Determine the sequence length based on the lead octet + typedef typename std::iterator_traits::difference_type octet_difference_type; + const octet_difference_type length = utf8::internal::sequence_length(it); + + // Get trail octets and calculate the code point + utf_error err = UTF8_OK; + switch (length) { + case 0: + return INVALID_LEAD; + case 1: + err = utf8::internal::get_sequence_1(it, end, cp); + break; + case 2: + err = utf8::internal::get_sequence_2(it, end, cp); + break; + case 3: + err = utf8::internal::get_sequence_3(it, end, cp); + break; + case 4: + err = utf8::internal::get_sequence_4(it, end, cp); + break; + } + + if (err == UTF8_OK) { + // Decoding succeeded. Now, security checks... + if (utf8::internal::is_code_point_valid(cp)) { + if (!utf8::internal::is_overlong_sequence(cp, length)){ + // Passed! Return here. + code_point = cp; + ++it; + return UTF8_OK; + } + else + err = OVERLONG_SEQUENCE; + } + else + err = INVALID_CODE_POINT; + } + + // Failure branch - restore the original value of the iterator + it = original_it; + return err; + } + + template + inline utf_error validate_next(octet_iterator& it, octet_iterator end) { + uint32_t ignored; + return utf8::internal::validate_next(it, end, ignored); + } + +} // namespace internal + + /// The library API - functions intended to be called by the users + + // Byte order mark + const uint8_t bom[] = {0xef, 0xbb, 0xbf}; + + template + octet_iterator find_invalid(octet_iterator start, octet_iterator end) + { + octet_iterator result = start; + while (result != end) { + utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end); + if (err_code != internal::UTF8_OK) + return result; + } + return result; + } + + template + inline bool is_valid(octet_iterator start, octet_iterator end) + { + return (utf8::find_invalid(start, end) == end); + } + + template + inline bool starts_with_bom (octet_iterator it, octet_iterator end) + { + return ( + ((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) && + ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) && + ((it != end) && (utf8::internal::mask8(*it)) == bom[2]) + ); + } + + //Deprecated in release 2.3 + template + inline bool is_bom (octet_iterator it) + { + return ( + (utf8::internal::mask8(*it++)) == bom[0] && + (utf8::internal::mask8(*it++)) == bom[1] && + (utf8::internal::mask8(*it)) == bom[2] + ); + } +} // namespace utf8 + +#endif // header guard + + diff --git a/include/utf8/unchecked.h b/include/utf8/unchecked.h new file mode 100644 index 00000000000..cb2427166b1 --- /dev/null +++ b/include/utf8/unchecked.h @@ -0,0 +1,228 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "core.h" + +namespace utf8 +{ + namespace unchecked + { + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f)| 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + + template + uint32_t next(octet_iterator& it) + { + uint32_t cp = utf8::internal::mask8(*it); + typename std::iterator_traits::difference_type length = utf8::internal::sequence_length(it); + switch (length) { + case 1: + break; + case 2: + it++; + cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); + break; + case 3: + ++it; + cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + ++it; + cp += (*it) & 0x3f; + break; + case 4: + ++it; + cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + ++it; + cp += (utf8::internal::mask8(*it) << 6) & 0xfff; + ++it; + cp += (*it) & 0x3f; + break; + } + ++it; + return cp; + } + + template + uint32_t peek_next(octet_iterator it) + { + return utf8::unchecked::next(it); + } + + template + uint32_t prior(octet_iterator& it) + { + while (utf8::internal::is_trail(*(--it))) ; + octet_iterator temp = it; + return utf8::unchecked::next(temp); + } + + // Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous) + template + inline uint32_t previous(octet_iterator& it) + { + return utf8::unchecked::prior(it); + } + + template + void advance (octet_iterator& it, distance_type n) + { + for (distance_type i = 0; i < n; ++i) + utf8::unchecked::next(it); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + utf8::unchecked::next(first); + return dist; + } + + template + octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = utf8::internal::mask16(*start++); + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { + uint32_t trail_surrogate = utf8::internal::mask16(*start++); + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; + } + result = utf8::unchecked::append(cp, result); + } + return result; + } + + template + u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start < end) { + uint32_t cp = utf8::unchecked::next(start); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + return result; + } + + template + octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + result = utf8::unchecked::append(*(start++), result); + + return result; + } + + template + u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start < end) + (*result++) = utf8::unchecked::next(start); + + return result; + } + + // The iterator class + template + class iterator : public std::iterator { + octet_iterator it; + public: + iterator () {} + explicit iterator (const octet_iterator& octet_it): it(octet_it) {} + // the default "big three" are OK + octet_iterator base () const { return it; } + uint32_t operator * () const + { + octet_iterator temp = it; + return utf8::unchecked::next(temp); + } + bool operator == (const iterator& rhs) const + { + return (it == rhs.it); + } + bool operator != (const iterator& rhs) const + { + return !(operator == (rhs)); + } + iterator& operator ++ () + { + ::std::advance(it, utf8::internal::sequence_length(it)); + return *this; + } + iterator operator ++ (int) + { + iterator temp = *this; + ::std::advance(it, utf8::internal::sequence_length(it)); + return temp; + } + iterator& operator -- () + { + utf8::unchecked::prior(it); + return *this; + } + iterator operator -- (int) + { + iterator temp = *this; + utf8::unchecked::prior(it); + return temp; + } + }; // class iterator + + } // namespace utf8::unchecked +} // namespace utf8 + + +#endif // header guard + diff --git a/scripts/travis_install.sh b/scripts/travis_install.sh new file mode 100755 index 00000000000..119e9fd15e3 --- /dev/null +++ b/scripts/travis_install.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# +# travis_install.sh +# + +if [ "$TRAVIS_OS_NAME" = "osx" ]; then + + brew install google-sparsehash || true + + brew install --without-python boost || true + + # workaround for gdal homebrew problem + brew remove gdal + brew install gdal + +fi + +cd .. +git clone --quiet --depth 1 https://github.com/osmcode/osm-testdata.git + diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh new file mode 100755 index 00000000000..75b3b365748 --- /dev/null +++ b/scripts/travis_script.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# +# travis_script.sh +# + +mkdir build +cd build + +# GCC ignores the pragmas in the code that disable the "return-type" warning +# selectively, so use this workaround. +if [ "${CXX}" = "g++" ]; then + WORKAROUND="-DCMAKE_CXX_FLAGS=-Wno-return-type" +else + WORKAROUND="" +fi + +if [ "${CXX}" = "g++" ]; then + CXX=g++-4.8 + CC=gcc-4.8 +fi + +cmake -LA \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + ${WORKAROUND} \ + .. + +make VERBOSE=1 +ctest --output-on-failure + diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7ba455b0295..00474577a2e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -15,6 +15,11 @@ add_library(testlib STATIC test_main.cpp) set(ALL_TESTS "") +# Otherwise GCC throws a lot of warnings for REQUIRE(...) from Catch v.1.2.1 +if(CMAKE_COMPILER_IS_GNUCXX) + add_definitions(-Wno-parentheses) +endif() + #----------------------------------------------------------------------------- # @@ -90,6 +95,7 @@ add_unit_test(area test_node_ref_segment) add_unit_test(basic test_box) add_unit_test(basic test_changeset) +add_unit_test(basic test_crc) add_unit_test(basic test_entity_bits) add_unit_test(basic test_location) add_unit_test(basic test_node) @@ -97,6 +103,7 @@ add_unit_test(basic test_node_ref) add_unit_test(basic test_object_comparisons) add_unit_test(basic test_relation) add_unit_test(basic test_timestamp) +add_unit_test(basic test_types_from_string) add_unit_test(basic test_way) add_unit_test(buffer test_buffer_node) @@ -111,23 +118,24 @@ add_unit_test(geom test_factory_with_projection ENABLE_IF ${GEOS_AND_PROJ_FOUND} LIBS ${GEOS_LIBRARY} ${PROJ_LIBRARY}) +add_unit_test(geom test_exception) add_unit_test(geom test_geojson) add_unit_test(geom test_geos ENABLE_IF ${GEOS_FOUND} LIBS ${GEOS_LIBRARY}) add_unit_test(geom test_geos_wkb ENABLE_IF ${GEOS_FOUND} LIBS ${GEOS_LIBRARY}) add_unit_test(geom test_mercator) add_unit_test(geom test_ogr ENABLE_IF ${GDAL_FOUND} LIBS ${GDAL_LIBRARY}) add_unit_test(geom test_projection ENABLE_IF ${PROJ_FOUND} LIBS ${PROJ_LIBRARY}) +add_unit_test(geom test_tile) add_unit_test(geom test_wkb) add_unit_test(geom test_wkt) add_unit_test(index test_id_to_location ENABLE_IF ${SPARSEHASH_FOUND}) -add_unit_test(index test_typed_mmap) -add_unit_test(index test_typed_mmap_grow LABELS "fails_on_windows") add_unit_test(io test_bzip2 ENABLE_IF ${BZIP2_FOUND} LIBS ${BZIP2_LIBRARIES}) add_unit_test(io test_file_formats) add_unit_test(io test_reader LIBS "${OSMIUM_XML_LIBRARIES}") add_unit_test(io test_output_iterator ENABLE_IF ${Threads_FOUND} LIBS ${CMAKE_THREAD_LIBS_INIT}) +add_unit_test(io test_string_table) add_unit_test(tags test_filter) add_unit_test(tags test_operators) @@ -136,7 +144,12 @@ add_unit_test(tags test_tag_list) add_unit_test(thread test_pool ENABLE_IF ${Threads_FOUND} LIBS ${CMAKE_THREAD_LIBS_INIT}) add_unit_test(util test_cast_with_assert) +add_unit_test(util test_data_file) +add_unit_test(util test_delta) add_unit_test(util test_double) +add_unit_test(util test_file) +add_unit_test(util test_memory_mapping) +add_unit_test(util test_minmax) add_unit_test(util test_options) add_unit_test(util test_string) diff --git a/test/data-tests/testdata-testcases.cpp b/test/data-tests/testdata-testcases.cpp index 6ed6ae91a2d..0ea7fc859f3 100644 --- a/test/data-tests/testdata-testcases.cpp +++ b/test/data-tests/testdata-testcases.cpp @@ -6,8 +6,6 @@ #include "testdata-testcases.hpp" -#include - std::string dirname; int main(int argc, char* argv[]) { diff --git a/test/data-tests/testdata-xml.cpp b/test/data-tests/testdata-xml.cpp index 5af4c4f27f6..8102759d1ec 100644 --- a/test/data-tests/testdata-xml.cpp +++ b/test/data-tests/testdata-xml.cpp @@ -12,6 +12,10 @@ #include #include +std::string S_(const char* s) { + return std::string(s); +} + std::string filename(const char* test_id, const char* suffix = "osm") { const char* testdir = getenv("TESTDIR"); if (!testdir) { @@ -286,12 +290,13 @@ TEST_CASE("Reading OSM XML 121") { } SECTION("Using Reader") { - REQUIRE_THROWS_AS({ + // can throw osmium::gzip_error or osmium::xml_error + REQUIRE_THROWS({ osmium::io::Reader reader(filename("121-truncated_gzip_file", "osm.gz")); osmium::io::Header header = reader.header(); osmium::memory::Buffer buffer = reader.read(); reader.close(); - }, osmium::gzip_error); + }); } } @@ -337,25 +342,25 @@ TEST_CASE("Reading OSM XML 140") { auto len = atoi(t["unicode_utf8_length"]); REQUIRE(len == strlen(uc)); - REQUIRE(!strcmp(uc, t["unicode_xml"])); + REQUIRE(S_(uc) == t["unicode_xml"]); // workaround for missing support for u8 string literals on Windows #if !defined(_MSC_VER) switch (count) { case 1: - REQUIRE(!strcmp(uc, u8"a")); + REQUIRE(S_(uc) == u8"a"); break; case 2: - REQUIRE(!strcmp(uc, u8"\u00e4")); + REQUIRE(S_(uc) == u8"\u00e4"); break; case 3: - REQUIRE(!strcmp(uc, u8"\u30dc")); + REQUIRE(S_(uc) == u8"\u30dc"); break; case 4: - REQUIRE(!strcmp(uc, u8"\U0001d11e")); + REQUIRE(S_(uc) == u8"\U0001d11e"); break; case 5: - REQUIRE(!strcmp(uc, u8"\U0001f6eb")); + REQUIRE(S_(uc) == u8"\U0001f6eb"); break; default: REQUIRE(false); // should not be here @@ -382,11 +387,100 @@ TEST_CASE("Reading OSM XML 141") { const osmium::Node& node = buffer.get(0); const osmium::TagList& tags = node.tags(); - REQUIRE(!strcmp(tags["less-than"], "<")); - REQUIRE(!strcmp(tags["greater-than"], ">")); - REQUIRE(!strcmp(tags["apostrophe"], "'")); - REQUIRE(!strcmp(tags["ampersand"], "&")); - REQUIRE(!strcmp(tags["quote"], "\"")); + REQUIRE(S_(tags["less-than"]) == "<"); + REQUIRE(S_(tags["greater-than"]) == ">"); + REQUIRE(S_(tags["apostrophe"]) == "'"); + REQUIRE(S_(tags["ampersand"]) == "&"); + REQUIRE(S_(tags["quote"]) == "\""); + } + +} + + +// ============================================= + +TEST_CASE("Reading OSM XML 142") { + + SECTION("Using Reader to read nodes") { + osmium::io::Reader reader(filename("142-whitespace")); + osmium::memory::Buffer buffer = reader.read(); + reader.close(); + + int count = 0; + for (auto it = buffer.begin(); it != buffer.end(); ++it) { + ++count; + REQUIRE(it->id() == count); + REQUIRE(it->tags().size() == 1); + const osmium::Tag& tag = *(it->tags().begin()); + + switch (count) { + case 1: + REQUIRE(S_(it->user()) == "user name"); + REQUIRE(S_(tag.key()) == "key with space"); + REQUIRE(S_(tag.value()) == "value with space"); + break; + case 2: + REQUIRE(S_(it->user()) == "line\nfeed"); + REQUIRE(S_(tag.key()) == "key with\nlinefeed"); + REQUIRE(S_(tag.value()) == "value with\nlinefeed"); + break; + case 3: + REQUIRE(S_(it->user()) == "carriage\rreturn"); + REQUIRE(S_(tag.key()) == "key with\rcarriage\rreturn"); + REQUIRE(S_(tag.value()) == "value with\rcarriage\rreturn"); + break; + case 4: + REQUIRE(S_(it->user()) == "tab\tulator"); + REQUIRE(S_(tag.key()) == "key with\ttab"); + REQUIRE(S_(tag.value()) == "value with\ttab"); + break; + case 5: + REQUIRE(S_(it->user()) == "unencoded linefeed"); + REQUIRE(S_(tag.key()) == "key with unencoded linefeed"); + REQUIRE(S_(tag.value()) == "value with unencoded linefeed"); + break; + default: + REQUIRE(false); // should not be here + } + } + REQUIRE(count == 5); + } + + SECTION("Using Reader to read relation") { + osmium::io::Reader reader(filename("142-whitespace")); + osmium::memory::Buffer buffer = reader.read(); + reader.close(); + + auto it = buffer.begin(); + REQUIRE(it != buffer.end()); + REQUIRE(it->id() == 21); + const auto& members = it->members(); + REQUIRE(members.size() == 5); + + int count = 0; + for (const auto& member : members) { + ++count; + switch (count) { + case 1: + REQUIRE(S_(member.role()) == "role with whitespace"); + break; + case 2: + REQUIRE(S_(member.role()) == "role with\nlinefeed"); + break; + case 3: + REQUIRE(S_(member.role()) == "role with\rcarriage\rreturn"); + break; + case 4: + REQUIRE(S_(member.role()) == "role with\ttab"); + break; + case 5: + REQUIRE(S_(member.role()) == "role with unencoded linefeed"); + break; + default: + REQUIRE(false); // should not be here + } + } + REQUIRE(count == 5); } } diff --git a/test/include/catch.hpp b/test/include/catch.hpp index bb87af296d9..73abfe8c6c2 100644 --- a/test/include/catch.hpp +++ b/test/include/catch.hpp @@ -1,10 +1,6 @@ - -// This is needed for Windows -#define CATCH_CONFIG_CPP11_NULLPTR - /* - * CATCH v1.0 build 53 (master branch) - * Generated: 2014-08-20 08:08:19.533804 + * Catch v1.2.1 + * Generated: 2015-06-30 18:23:27.961086 * ---------------------------------------------------------- * This file has been merged from multiple headers. Please don't edit it directly * Copyright (c) 2012 Two Blue Cubes Ltd. All rights reserved. @@ -17,33 +13,43 @@ #define TWOBLUECUBES_CATCH_HPP_INCLUDED +#ifdef __clang__ +# pragma clang system_header +#elif defined __GNUC__ +# pragma GCC system_header +#endif + // #included from: internal/catch_suppress_warnings.h #define TWOBLUECUBES_CATCH_SUPPRESS_WARNINGS_H_INCLUDED #ifdef __clang__ -#pragma clang diagnostic ignored "-Wglobal-constructors" -#pragma clang diagnostic ignored "-Wvariadic-macros" -#pragma clang diagnostic ignored "-Wc99-extensions" -#pragma clang diagnostic ignored "-Wunused-variable" -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wpadded" -#pragma clang diagnostic ignored "-Wc++98-compat" -#pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +# ifdef __ICC // icpc defines the __clang__ macro +# pragma warning(push) +# pragma warning(disable: 161 1682) +# else // __ICC +# pragma clang diagnostic ignored "-Wglobal-constructors" +# pragma clang diagnostic ignored "-Wvariadic-macros" +# pragma clang diagnostic ignored "-Wc99-extensions" +# pragma clang diagnostic ignored "-Wunused-variable" +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wpadded" +# pragma clang diagnostic ignored "-Wc++98-compat" +# pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +# pragma clang diagnostic ignored "-Wswitch-enum" +# endif #elif defined __GNUC__ -#pragma GCC diagnostic ignored "-Wvariadic-macros" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpadded" -#pragma GCC diagnostic ignored "-Wctor-dtor-privacy" -#pragma GCC diagnostic ignored "-Wsign-promo" +# pragma GCC diagnostic ignored "-Wvariadic-macros" +# pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wpadded" #endif -#ifdef CATCH_CONFIG_MAIN -# define CATCH_CONFIG_RUNNER +#if defined(CATCH_CONFIG_MAIN) || defined(CATCH_CONFIG_RUNNER) +# define CATCH_IMPL #endif -#ifdef CATCH_CONFIG_RUNNER +#ifdef CATCH_IMPL # ifndef CLARA_CONFIG_MAIN # define CLARA_CONFIG_MAIN_NOT_DEFINED # define CLARA_CONFIG_MAIN @@ -70,16 +76,34 @@ // #included from: catch_compiler_capabilities.h #define TWOBLUECUBES_CATCH_COMPILER_CAPABILITIES_HPP_INCLUDED -// Much of the following code is based on Boost (1.53) +// Detect a number of compiler features - mostly C++11/14 conformance - by compiler +// The following features are defined: +// +// CATCH_CONFIG_CPP11_NULLPTR : is nullptr supported? +// CATCH_CONFIG_CPP11_NOEXCEPT : is noexcept supported? +// CATCH_CONFIG_CPP11_GENERATED_METHODS : The delete and default keywords for compiler generated methods +// CATCH_CONFIG_CPP11_IS_ENUM : std::is_enum is supported? +// CATCH_CONFIG_CPP11_TUPLE : std::tuple is supported + +// CATCH_CONFIG_CPP11_OR_GREATER : Is C++11 supported? + +// CATCH_CONFIG_VARIADIC_MACROS : are variadic macros supported? + +// In general each macro has a _NO_ form +// (e.g. CATCH_CONFIG_CPP11_NO_NULLPTR) which disables the feature. +// Many features, at point of detection, define an _INTERNAL_ macro, so they +// can be combined, en-mass, with the _NO_ forms later. + +// All the C++11 features can be disabled with CATCH_CONFIG_NO_CPP11 #ifdef __clang__ # if __has_feature(cxx_nullptr) -# define CATCH_CONFIG_CPP11_NULLPTR +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR # endif # if __has_feature(cxx_noexcept) -# define CATCH_CONFIG_CPP11_NOEXCEPT +# define CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT # endif #endif // __clang__ @@ -88,51 +112,26 @@ // Borland #ifdef __BORLANDC__ -#if (__BORLANDC__ > 0x582 ) -//#define CATCH_CONFIG_SFINAE // Not confirmed -#endif - #endif // __BORLANDC__ //////////////////////////////////////////////////////////////////////////////// // EDG #ifdef __EDG_VERSION__ -#if (__EDG_VERSION__ > 238 ) -//#define CATCH_CONFIG_SFINAE // Not confirmed -#endif - #endif // __EDG_VERSION__ //////////////////////////////////////////////////////////////////////////////// // Digital Mars #ifdef __DMC__ -#if (__DMC__ > 0x840 ) -//#define CATCH_CONFIG_SFINAE // Not confirmed -#endif - #endif // __DMC__ //////////////////////////////////////////////////////////////////////////////// // GCC #ifdef __GNUC__ -#if __GNUC__ < 3 - -#if (__GNUC_MINOR__ >= 96 ) -//#define CATCH_CONFIG_SFINAE -#endif - -#elif __GNUC__ >= 3 - -// #define CATCH_CONFIG_SFINAE // Taking this out completely for now - -#endif // __GNUC__ < 3 - #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6 && defined(__GXX_EXPERIMENTAL_CXX0X__) ) - -#define CATCH_CONFIG_CPP11_NULLPTR +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR #endif #endif // __GNUC__ @@ -141,8 +140,13 @@ // Visual C++ #ifdef _MSC_VER -#if (_MSC_VER >= 1310 ) // (VC++ 7.0+) -//#define CATCH_CONFIG_SFINAE // Not confirmed +#if (_MSC_VER >= 1600) +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR +#endif + +#if (_MSC_VER >= 1900 ) // (VC++ 13 (VS2015)) +#define CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT +#define CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS #endif #endif // _MSC_VER @@ -153,21 +157,62 @@ ( defined __GNUC__ && __GNUC__ >= 3 ) || \ ( !defined __cplusplus && __STDC_VERSION__ >= 199901L || __cplusplus >= 201103L ) -#ifndef CATCH_CONFIG_NO_VARIADIC_MACROS -#define CATCH_CONFIG_VARIADIC_MACROS -#endif +#define CATCH_INTERNAL_CONFIG_VARIADIC_MACROS #endif //////////////////////////////////////////////////////////////////////////////// // C++ language feature support -// detect language version: -#if (__cplusplus == 201103L) -# define CATCH_CPP11 -# define CATCH_CPP11_OR_GREATER -#elif (__cplusplus >= 201103L) +// catch all support for C++11 +#if (__cplusplus >= 201103L) + # define CATCH_CPP11_OR_GREATER + +# if !defined(CATCH_INTERNAL_CONFIG_CPP11_NULLPTR) +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT +# define CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS +# define CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_IS_ENUM +# define CATCH_INTERNAL_CONFIG_CPP11_IS_ENUM +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_TUPLE +# define CATCH_INTERNAL_CONFIG_CPP11_TUPLE +# endif + +# ifndef CATCH_INTERNAL_CONFIG_VARIADIC_MACROS +# define CATCH_INTERNAL_CONFIG_VARIADIC_MACROS +# endif + +#endif // __cplusplus >= 201103L + +// Now set the actual defines based on the above + anything the user has configured +#if defined(CATCH_INTERNAL_CONFIG_CPP11_NULLPTR) && !defined(CATCH_CONFIG_CPP11_NO_NULLPTR) && !defined(CATCH_CONFIG_CPP11_NULLPTR) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_NULLPTR +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT) && !defined(CATCH_CONFIG_CPP11_NO_NOEXCEPT) && !defined(CATCH_CONFIG_CPP11_NOEXCEPT) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_NOEXCEPT +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS) && !defined(CATCH_CONFIG_CPP11_NO_GENERATED_METHODS) && !defined(CATCH_CONFIG_CPP11_GENERATED_METHODS) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_GENERATED_METHODS +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_IS_ENUM) && !defined(CATCH_CONFIG_CPP11_NO_IS_ENUM) && !defined(CATCH_CONFIG_CPP11_IS_ENUM) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_IS_ENUM +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_TUPLE) && !defined(CATCH_CONFIG_CPP11_NO_TUPLE) && !defined(CATCH_CONFIG_CPP11_TUPLE) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_TUPLE +#endif +#if defined(CATCH_INTERNAL_CONFIG_VARIADIC_MACROS) && !defined(CATCH_CONFIG_NO_VARIADIC_MACROS) && !defined(CATCH_CONFIG_VARIADIC_MACROS) +#define CATCH_CONFIG_VARIADIC_MACROS #endif // noexcept support: @@ -182,8 +227,16 @@ namespace Catch { class NonCopyable { - NonCopyable( NonCopyable const& ); - void operator = ( NonCopyable const& ); +#ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS + NonCopyable( NonCopyable const& ) = delete; + NonCopyable( NonCopyable && ) = delete; + NonCopyable& operator = ( NonCopyable const& ) = delete; + NonCopyable& operator = ( NonCopyable && ) = delete; +#else + NonCopyable( NonCopyable const& info ); + NonCopyable& operator = ( NonCopyable const& ); +#endif + protected: NonCopyable() {} virtual ~NonCopyable(); @@ -221,6 +274,7 @@ namespace Catch { void toLowerInPlace( std::string& s ); std::string toLower( std::string const& s ); std::string trim( std::string const& str ); + bool replaceInPlace( std::string& str, std::string const& replaceThis, std::string const& withThis ); struct pluralise { pluralise( std::size_t count, std::string const& label ); @@ -236,13 +290,14 @@ namespace Catch { SourceLineInfo(); SourceLineInfo( char const* _file, std::size_t _line ); SourceLineInfo( SourceLineInfo const& other ); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS SourceLineInfo( SourceLineInfo && ) = default; SourceLineInfo& operator = ( SourceLineInfo const& ) = default; SourceLineInfo& operator = ( SourceLineInfo && ) = default; # endif bool empty() const; bool operator == ( SourceLineInfo const& other ) const; + bool operator < ( SourceLineInfo const& other ) const; std::string file; std::size_t line; @@ -473,7 +528,7 @@ namespace Catch { struct ITestCaseRegistry { virtual ~ITestCaseRegistry(); virtual std::vector const& getAllTests() const = 0; - virtual void getFilteredTests( TestSpec const& testSpec, IConfig const& config, std::vector& matchingTestCases ) const = 0; + virtual void getFilteredTests( TestSpec const& testSpec, IConfig const& config, std::vector& matchingTestCases, bool negated = false ) const = 0; }; } @@ -609,7 +664,9 @@ namespace Catch { Exception = 0x100 | FailureBit, ThrewException = Exception | 1, - DidntThrowException = Exception | 2 + DidntThrowException = Exception | 2, + + FatalErrorCondition = 0x200 | FailureBit }; }; @@ -622,11 +679,11 @@ namespace Catch { // ResultDisposition::Flags enum struct ResultDisposition { enum Flags { - Normal = 0x00, + Normal = 0x01, - ContinueOnFailure = 0x01, // Failures fail test, but execution continues - FalseTest = 0x02, // Prefix expression with ! - SuppressFail = 0x04 // Failures are reported but do not fail the test + ContinueOnFailure = 0x02, // Failures fail test, but execution continues + FalseTest = 0x04, // Prefix expression with ! + SuppressFail = 0x08 // Failures are reported but do not fail the test }; }; inline ResultDisposition::Flags operator | ( ResultDisposition::Flags lhs, ResultDisposition::Flags rhs ) { @@ -674,7 +731,7 @@ namespace Catch { AssertionResult(); AssertionResult( AssertionInfo const& info, AssertionResultData const& data ); ~AssertionResult(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS AssertionResult( AssertionResult const& ) = default; AssertionResult( AssertionResult && ) = default; AssertionResult& operator = ( AssertionResult const& ) = default; @@ -730,8 +787,8 @@ namespace Catch { ResultDisposition::Flags resultDisposition ); template - ExpressionLhs operator->* ( T const& operand ); - ExpressionLhs operator->* ( bool value ); + ExpressionLhs operator <= ( T const& operand ); + ExpressionLhs operator <= ( bool value ); template ResultBuilder& operator << ( T const& value ) { @@ -954,40 +1011,6 @@ namespace Internal { // #included from: catch_tostring.h #define TWOBLUECUBES_CATCH_TOSTRING_H_INCLUDED -// #included from: catch_sfinae.hpp -#define TWOBLUECUBES_CATCH_SFINAE_HPP_INCLUDED - -// Try to detect if the current compiler supports SFINAE - -namespace Catch { - - struct TrueType { - static const bool value = true; - typedef void Enable; - char sizer[1]; - }; - struct FalseType { - static const bool value = false; - typedef void Disable; - char sizer[2]; - }; - -#ifdef CATCH_CONFIG_SFINAE - - template struct NotABooleanExpression; - - template struct If : NotABooleanExpression {}; - template<> struct If : TrueType {}; - template<> struct If : FalseType {}; - - template struct SizedIf; - template<> struct SizedIf : TrueType {}; - template<> struct SizedIf : FalseType {}; - -#endif // CATCH_CONFIG_SFINAE - -} // end namespace Catch - #include #include #include @@ -1040,35 +1063,59 @@ inline id performOptionalSelector( id obj, SEL sel ) { #endif +#ifdef CATCH_CONFIG_CPP11_TUPLE +#include +#endif + +#ifdef CATCH_CONFIG_CPP11_IS_ENUM +#include +#endif + namespace Catch { -namespace Detail { -// SFINAE is currently disabled by default for all compilers. -// If the non SFINAE version of IsStreamInsertable is ambiguous for you -// and your compiler supports SFINAE, try #defining CATCH_CONFIG_SFINAE -#ifdef CATCH_CONFIG_SFINAE +// Why we're here. +template +std::string toString( T const& value ); - template - class IsStreamInsertableHelper { - template struct TrueIfSizeable : TrueType {}; +// Built in overloads + +std::string toString( std::string const& value ); +std::string toString( std::wstring const& value ); +std::string toString( const char* const value ); +std::string toString( char* const value ); +std::string toString( const wchar_t* const value ); +std::string toString( wchar_t* const value ); +std::string toString( int value ); +std::string toString( unsigned long value ); +std::string toString( unsigned int value ); +std::string toString( const double value ); +std::string toString( const float value ); +std::string toString( bool value ); +std::string toString( char value ); +std::string toString( signed char value ); +std::string toString( unsigned char value ); - template - static TrueIfSizeable dummy(T2*); - static FalseType dummy(...); +#ifdef CATCH_CONFIG_CPP11_NULLPTR +std::string toString( std::nullptr_t ); +#endif - public: - typedef SizedIf type; - }; +#ifdef __OBJC__ + std::string toString( NSString const * const& nsstring ); + std::string toString( NSString * CATCH_ARC_STRONG const& nsstring ); + std::string toString( NSObject* const& nsObject ); +#endif - template - struct IsStreamInsertable : IsStreamInsertableHelper::type {}; +namespace Detail { -#else + extern std::string unprintableString; struct BorgType { template BorgType( T const& ); }; + struct TrueType { char sizer[1]; }; + struct FalseType { char sizer[2]; }; + TrueType& testStreamable( std::ostream& ); FalseType testStreamable( FalseType ); @@ -1081,12 +1128,38 @@ namespace Detail { enum { value = sizeof( testStreamable(s << t) ) == sizeof( TrueType ) }; }; -#endif +#if defined(CATCH_CONFIG_CPP11_IS_ENUM) + template::value + > + struct EnumStringMaker + { + static std::string convert( T const& ) { return unprintableString; } + }; + template + struct EnumStringMaker + { + static std::string convert( T const& v ) + { + return ::Catch::toString( + static_cast::type>(v) + ); + } + }; +#endif template struct StringMakerBase { +#if defined(CATCH_CONFIG_CPP11_IS_ENUM) + template + static std::string convert( T const& v ) + { + return EnumStringMaker::convert( v ); + } +#else template - static std::string convert( T const& ) { return "{?}"; } + static std::string convert( T const& ) { return unprintableString; } +#endif }; template<> @@ -1108,9 +1181,6 @@ namespace Detail { } // end namespace Detail -template -std::string toString( T const& value ); - template struct StringMaker : Detail::StringMakerBase::value> {}; @@ -1141,12 +1211,59 @@ namespace Detail { std::string rangeToString( InputIterator first, InputIterator last ); } +//template +//struct StringMaker > { +// static std::string convert( std::vector const& v ) { +// return Detail::rangeToString( v.begin(), v.end() ); +// } +//}; + template -struct StringMaker > { - static std::string convert( std::vector const& v ) { - return Detail::rangeToString( v.begin(), v.end() ); +std::string toString( std::vector const& v ) { + return Detail::rangeToString( v.begin(), v.end() ); +} + +#ifdef CATCH_CONFIG_CPP11_TUPLE + +// toString for tuples +namespace TupleDetail { + template< + typename Tuple, + std::size_t N = 0, + bool = (N < std::tuple_size::value) + > + struct ElementPrinter { + static void print( const Tuple& tuple, std::ostream& os ) + { + os << ( N ? ", " : " " ) + << Catch::toString(std::get(tuple)); + ElementPrinter::print(tuple,os); + } + }; + + template< + typename Tuple, + std::size_t N + > + struct ElementPrinter { + static void print( const Tuple&, std::ostream& ) {} + }; + +} + +template +struct StringMaker> { + + static std::string convert( const std::tuple& tuple ) + { + std::ostringstream os; + os << '{'; + TupleDetail::ElementPrinter>::print( tuple, os ); + os << " }"; + return os.str(); } }; +#endif // CATCH_CONFIG_CPP11_TUPLE namespace Detail { template @@ -1167,44 +1284,15 @@ std::string toString( T const& value ) { return StringMaker::convert( value ); } -// Built in overloads - -std::string toString( std::string const& value ); -std::string toString( std::wstring const& value ); -std::string toString( const char* const value ); -std::string toString( char* const value ); -std::string toString( const wchar_t* const value ); -std::string toString( wchar_t* const value ); -std::string toString( int value ); -std::string toString( unsigned long value ); -std::string toString( unsigned int value ); -std::string toString( const double value ); -std::string toString( const float value ); -std::string toString( bool value ); -std::string toString( char value ); -std::string toString( signed char value ); -std::string toString( unsigned char value ); - -#ifdef CATCH_CONFIG_CPP11_NULLPTR -std::string toString( std::nullptr_t ); -#endif - -#ifdef __OBJC__ - std::string toString( NSString const * const& nsstring ); - std::string toString( NSString * CATCH_ARC_STRONG const& nsstring ); - std::string toString( NSObject* const& nsObject ); -#endif - namespace Detail { template std::string rangeToString( InputIterator first, InputIterator last ) { std::ostringstream oss; oss << "{ "; if( first != last ) { - oss << toString( *first ); - for( ++first ; first != last ; ++first ) { - oss << ", " << toString( *first ); - } + oss << Catch::toString( *first ); + for( ++first ; first != last ; ++first ) + oss << ", " << Catch::toString( *first ); } oss << " }"; return oss.str(); @@ -1220,13 +1308,13 @@ namespace Catch { template class ExpressionLhs { ExpressionLhs& operator = ( ExpressionLhs const& ); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS ExpressionLhs& operator = ( ExpressionLhs && ) = delete; # endif public: ExpressionLhs( ResultBuilder& rb, T lhs ) : m_rb( rb ), m_lhs( lhs ) {} -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS ExpressionLhs( ExpressionLhs const& ) = default; ExpressionLhs( ExpressionLhs && ) = default; # endif @@ -1307,11 +1395,11 @@ class ExpressionLhs { namespace Catch { template - inline ExpressionLhs ResultBuilder::operator->* ( T const& operand ) { + inline ExpressionLhs ResultBuilder::operator <= ( T const& operand ) { return ExpressionLhs( *this, operand ); } - inline ExpressionLhs ResultBuilder::operator->* ( bool value ) { + inline ExpressionLhs ResultBuilder::operator <= ( bool value ) { return ExpressionLhs( *this, value ); } @@ -1401,6 +1489,8 @@ namespace Catch { virtual std::string getCurrentTestName() const = 0; virtual const AssertionResult* getLastResult() const = 0; + + virtual void handleFatalErrorCondition( std::string const& message ) = 0; }; IResultCapture& getResultCapture(); @@ -1481,7 +1571,7 @@ namespace Catch { do { \ Catch::ResultBuilder __catchResult( macroName, CATCH_INTERNAL_LINEINFO, #expr, resultDisposition ); \ try { \ - ( __catchResult->*expr ).endExpression(); \ + ( __catchResult <= expr ).endExpression(); \ } \ catch( ... ) { \ __catchResult.useActiveException( Catch::ResultDisposition::Normal ); \ @@ -1581,7 +1671,7 @@ namespace Catch { std::string matcherAsString = ::Catch::Matchers::matcher.toString(); \ __catchResult \ .setLhs( Catch::toString( arg ) ) \ - .setRhs( matcherAsString == "{?}" ? #matcher : matcherAsString ) \ + .setRhs( matcherAsString == Catch::Detail::unprintableString ? #matcher : matcherAsString ) \ .setOp( "matches" ) \ .setResultType( ::Catch::Matchers::matcher.match( arg ) ); \ __catchResult.captureExpression(); \ @@ -1642,6 +1732,9 @@ namespace Catch { bool allPassed() const { return failed == 0 && failedButOk == 0; } + bool allOk() const { + return failed == 0; + } std::size_t passed; std::size_t failed; @@ -1694,7 +1787,7 @@ namespace Catch { public: Timer() : m_ticks( 0 ) {} void start(); - unsigned int getElapsedNanoseconds() const; + unsigned int getElapsedMicroseconds() const; unsigned int getElapsedMilliseconds() const; double getElapsedSeconds() const; @@ -1708,7 +1801,7 @@ namespace Catch { namespace Catch { - class Section { + class Section : NonCopyable { public: Section( SectionInfo const& info ); ~Section(); @@ -1717,15 +1810,6 @@ namespace Catch { operator bool() const; private: -#ifdef CATCH_CPP11_OR_GREATER - Section( Section const& ) = delete; - Section( Section && ) = delete; - Section& operator = ( Section const& ) = delete; - Section& operator = ( Section && ) = delete; -#else - Section( Section const& info ); - Section& operator = ( Section const& ); -#endif SectionInfo m_info; std::string m_name; @@ -2700,7 +2784,7 @@ return @ desc; \ #endif -#ifdef CATCH_CONFIG_RUNNER +#ifdef CATCH_IMPL // #included from: internal/catch_impl.hpp #define TWOBLUECUBES_CATCH_IMPL_HPP_INCLUDED @@ -2712,7 +2796,7 @@ return @ desc; \ #pragma clang diagnostic ignored "-Wweak-vtables" #endif -// #included from: catch_runner.hpp +// #included from: ../catch_runner.hpp #define TWOBLUECUBES_CATCH_RUNNER_HPP_INCLUDED // #included from: internal/catch_commandline.hpp @@ -2968,6 +3052,11 @@ namespace Catch { Always, Never }; }; + struct RunTests { enum InWhatOrder { + InDeclarationOrder, + InLexicographicalOrder, + InRandomOrder + }; }; class TestSpec; @@ -2985,6 +3074,9 @@ namespace Catch { virtual bool showInvisibles() const = 0; virtual ShowDurations::OrNot showDurations() const = 0; virtual TestSpec const& testSpec() const = 0; + virtual RunTests::InWhatOrder runOrder() const = 0; + virtual unsigned int rngSeed() const = 0; + virtual bool forceColour() const = 0; }; } @@ -3010,12 +3102,16 @@ namespace Catch { private: bool isOwned; }; + + std::ostream& cout(); + std::ostream& cerr(); } #include #include #include #include +#include #ifndef CATCH_CONFIG_CONSOLE_WIDTH #define CATCH_CONFIG_CONSOLE_WIDTH 80 @@ -3035,10 +3131,13 @@ namespace Catch { noThrow( false ), showHelp( false ), showInvisibles( false ), + forceColour( false ), abortAfter( -1 ), + rngSeed( 0 ), verbosity( Verbosity::Normal ), warnings( WarnAbout::Nothing ), - showDurations( ShowDurations::DefaultForReporter ) + showDurations( ShowDurations::DefaultForReporter ), + runOrder( RunTests::InDeclarationOrder ) {} bool listTests; @@ -3051,12 +3150,15 @@ namespace Catch { bool noThrow; bool showHelp; bool showInvisibles; + bool forceColour; int abortAfter; + unsigned int rngSeed; Verbosity::Level verbosity; WarnAbout::What warnings; ShowDurations::OrNot showDurations; + RunTests::InWhatOrder runOrder; std::string reporterName; std::string outputFilename; @@ -3074,12 +3176,12 @@ namespace Catch { public: Config() - : m_os( std::cout.rdbuf() ) + : m_os( Catch::cout().rdbuf() ) {} Config( ConfigData const& data ) : m_data( data ), - m_os( std::cout.rdbuf() ) + m_os( Catch::cout().rdbuf() ) { if( !data.testsOrTags.empty() ) { TestSpecParser parser( ITagAliasRegistry::get() ); @@ -3090,7 +3192,7 @@ namespace Catch { } virtual ~Config() { - m_os.rdbuf( std::cout.rdbuf() ); + m_os.rdbuf( Catch::cout().rdbuf() ); m_stream.release(); } @@ -3112,7 +3214,7 @@ namespace Catch { bool shouldDebugBreak() const { return m_data.shouldDebugBreak; } void setStreamBuf( std::streambuf* buf ) { - m_os.rdbuf( buf ? buf : std::cout.rdbuf() ); + m_os.rdbuf( buf ? buf : Catch::cout().rdbuf() ); } void useStream( std::string const& streamName ) { @@ -3138,6 +3240,9 @@ namespace Catch { virtual bool includeSuccessfulResults() const { return m_data.showSuccessfulTests; } virtual bool warnAboutMissingAssertions() const { return m_data.warnings & WarnAbout::NoAssertions; } virtual ShowDurations::OrNot showDurations() const { return m_data.showDurations; } + virtual RunTests::InWhatOrder runOrder() const { return m_data.runOrder; } + virtual unsigned int rngSeed() const { return m_data.rngSeed; } + virtual bool forceColour() const { return m_data.forceColour; } private: ConfigData m_data; @@ -3399,7 +3504,7 @@ namespace Clara { template struct IArgFunction { virtual ~IArgFunction() {} -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS IArgFunction() = default; IArgFunction( IArgFunction const& ) = default; # endif @@ -3766,7 +3871,7 @@ namespace Clara { m_throwOnUnrecognisedTokens( other.m_throwOnUnrecognisedTokens ) { if( other.m_floatingArg.get() ) - m_floatingArg = ArgAutoPtr( new Arg( *other.m_floatingArg ) ); + m_floatingArg.reset( new Arg( *other.m_floatingArg ) ); } CommandLine& setThrowOnUnrecognisedTokens( bool shouldThrow = true ) { @@ -3794,7 +3899,7 @@ namespace Clara { ArgBuilder operator[]( UnpositionalTag ) { if( m_floatingArg.get() ) throw std::logic_error( "Only one unpositional argument can be added" ); - m_floatingArg = ArgAutoPtr( new Arg() ); + m_floatingArg.reset( new Arg() ); ArgBuilder builder( m_floatingArg.get() ); return builder; } @@ -3936,7 +4041,7 @@ namespace Clara { if( it == itEnd ) { if( token.type == Parser::Token::Positional || !m_throwOnUnrecognisedTokens ) unusedTokens.push_back( token ); - else if( m_throwOnUnrecognisedTokens ) + else if( errors.empty() && m_throwOnUnrecognisedTokens ) errors.push_back( "unrecognised option: " + token.data ); } } @@ -4034,7 +4139,28 @@ namespace Catch { config.warnings = static_cast( config.warnings | WarnAbout::NoAssertions ); else throw std::runtime_error( "Unrecognised warning: '" + _warning + "'" ); - + } + inline void setOrder( ConfigData& config, std::string const& order ) { + if( startsWith( "declared", order ) ) + config.runOrder = RunTests::InDeclarationOrder; + else if( startsWith( "lexical", order ) ) + config.runOrder = RunTests::InLexicographicalOrder; + else if( startsWith( "random", order ) ) + config.runOrder = RunTests::InRandomOrder; + else + throw std::runtime_error( "Unrecognised ordering: '" + order + "'" ); + } + inline void setRngSeed( ConfigData& config, std::string const& seed ) { + if( seed == "time" ) { + config.rngSeed = static_cast( std::time(0) ); + } + else { + std::stringstream ss; + ss << seed; + ss >> config.rngSeed; + if( ss.fail() ) + throw std::runtime_error( "Argment to --rng-seed should be the word 'time' or a number" ); + } } inline void setVerbosity( ConfigData& config, int level ) { // !TBD: accept strings? @@ -4146,6 +4272,18 @@ namespace Catch { .describe( "list all reporters" ) .bind( &ConfigData::listReporters ); + cli["--order"] + .describe( "test case order (defaults to decl)" ) + .bind( &setOrder, "decl|lex|rand" ); + + cli["--rng-seed"] + .describe( "set a specific seed for random numbers" ) + .bind( &setRngSeed, "'time'|number" ); + + cli["--force-colour"] + .describe( "force colourised output" ) + .bind( &ConfigData::forceColour ); + return cli; } @@ -4319,10 +4457,6 @@ namespace Catch { namespace Catch { - namespace Detail { - struct IColourImpl; - } - struct Colour { enum Code { None = 0, @@ -4368,7 +4502,6 @@ namespace Catch { static void use( Code _colourCode ); private: - static Detail::IColourImpl* impl(); bool m_moved; }; @@ -4462,7 +4595,7 @@ namespace Catch } virtual ~AssertionStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS AssertionStats( AssertionStats const& ) = default; AssertionStats( AssertionStats && ) = default; AssertionStats& operator = ( AssertionStats const& ) = default; @@ -4485,7 +4618,7 @@ namespace Catch missingAssertions( _missingAssertions ) {} virtual ~SectionStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS SectionStats( SectionStats const& ) = default; SectionStats( SectionStats && ) = default; SectionStats& operator = ( SectionStats const& ) = default; @@ -4512,7 +4645,7 @@ namespace Catch {} virtual ~TestCaseStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS TestCaseStats( TestCaseStats const& ) = default; TestCaseStats( TestCaseStats && ) = default; TestCaseStats& operator = ( TestCaseStats const& ) = default; @@ -4540,7 +4673,7 @@ namespace Catch {} virtual ~TestGroupStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS TestGroupStats( TestGroupStats const& ) = default; TestGroupStats( TestGroupStats && ) = default; TestGroupStats& operator = ( TestGroupStats const& ) = default; @@ -4562,7 +4695,7 @@ namespace Catch {} virtual ~TestRunStats(); -# ifndef CATCH_CPP11_OR_GREATER +# ifndef CATCH_CONFIG_CPP11_GENERATED_METHODS TestRunStats( TestRunStats const& _other ) : runInfo( _other.runInfo ), totals( _other.totals ), @@ -4598,11 +4731,14 @@ namespace Catch virtual void assertionStarting( AssertionInfo const& assertionInfo ) = 0; + // The return value indicates if the messages buffer should be cleared: virtual bool assertionEnded( AssertionStats const& assertionStats ) = 0; virtual void sectionEnded( SectionStats const& sectionStats ) = 0; virtual void testCaseEnded( TestCaseStats const& testCaseStats ) = 0; virtual void testGroupEnded( TestGroupStats const& testGroupStats ) = 0; virtual void testRunEnded( TestRunStats const& testRunStats ) = 0; + + virtual void skipTest( TestCaseInfo const& testInfo ) = 0; }; struct IReporterFactory { @@ -4630,9 +4766,9 @@ namespace Catch { TestSpec testSpec = config.testSpec(); if( config.testSpec().hasFilters() ) - std::cout << "Matching test cases:\n"; + Catch::cout() << "Matching test cases:\n"; else { - std::cout << "All available test cases:\n"; + Catch::cout() << "All available test cases:\n"; testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec(); } @@ -4653,15 +4789,15 @@ namespace Catch { : Colour::None; Colour colourGuard( colour ); - std::cout << Text( testCaseInfo.name, nameAttr ) << std::endl; + Catch::cout() << Text( testCaseInfo.name, nameAttr ) << std::endl; if( !testCaseInfo.tags.empty() ) - std::cout << Text( testCaseInfo.tagsAsString, tagsAttr ) << std::endl; + Catch::cout() << Text( testCaseInfo.tagsAsString, tagsAttr ) << std::endl; } if( !config.testSpec().hasFilters() ) - std::cout << pluralise( matchedTests, "test case" ) << "\n" << std::endl; + Catch::cout() << pluralise( matchedTests, "test case" ) << "\n" << std::endl; else - std::cout << pluralise( matchedTests, "matching test case" ) << "\n" << std::endl; + Catch::cout() << pluralise( matchedTests, "matching test case" ) << "\n" << std::endl; return matchedTests; } @@ -4677,7 +4813,7 @@ namespace Catch { ++it ) { matchedTests++; TestCaseInfo const& testCaseInfo = it->getTestCaseInfo(); - std::cout << testCaseInfo.name << std::endl; + Catch::cout() << testCaseInfo.name << std::endl; } return matchedTests; } @@ -4703,9 +4839,9 @@ namespace Catch { inline std::size_t listTags( Config const& config ) { TestSpec testSpec = config.testSpec(); if( config.testSpec().hasFilters() ) - std::cout << "Tags for matching test cases:\n"; + Catch::cout() << "Tags for matching test cases:\n"; else { - std::cout << "All available tags:\n"; + Catch::cout() << "All available tags:\n"; testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec(); } @@ -4739,14 +4875,14 @@ namespace Catch { .setInitialIndent( 0 ) .setIndent( oss.str().size() ) .setWidth( CATCH_CONFIG_CONSOLE_WIDTH-10 ) ); - std::cout << oss.str() << wrapper << "\n"; + Catch::cout() << oss.str() << wrapper << "\n"; } - std::cout << pluralise( tagCounts.size(), "tag" ) << "\n" << std::endl; + Catch::cout() << pluralise( tagCounts.size(), "tag" ) << "\n" << std::endl; return tagCounts.size(); } inline std::size_t listReporters( Config const& /*config*/ ) { - std::cout << "Available reports:\n"; + Catch::cout() << "Available reporters:\n"; IReporterRegistry::FactoryMap const& factories = getRegistryHub().getReporterRegistry().getFactories(); IReporterRegistry::FactoryMap::const_iterator itBegin = factories.begin(), itEnd = factories.end(), it; std::size_t maxNameLen = 0; @@ -4758,13 +4894,13 @@ namespace Catch { .setInitialIndent( 0 ) .setIndent( 7+maxNameLen ) .setWidth( CATCH_CONFIG_CONSOLE_WIDTH - maxNameLen-8 ) ); - std::cout << " " + Catch::cout() << " " << it->first << ":" << std::string( maxNameLen - it->first.size() + 2, ' ' ) << wrapper << "\n"; } - std::cout << std::endl; + Catch::cout() << std::endl; return factories.size(); } @@ -4814,32 +4950,15 @@ namespace SectionTracking { RunState runState() const { return m_runState; } - TrackedSection* findChild( std::string const& childName ) { - TrackedSections::iterator it = m_children.find( childName ); - return it != m_children.end() - ? &it->second - : NULL; - } - TrackedSection* acquireChild( std::string const& childName ) { - if( TrackedSection* child = findChild( childName ) ) - return child; - m_children.insert( std::make_pair( childName, TrackedSection( childName, this ) ) ); - return findChild( childName ); - } + TrackedSection* findChild( std::string const& childName ); + TrackedSection* acquireChild( std::string const& childName ); + void enter() { if( m_runState == NotStarted ) m_runState = Executing; } - void leave() { - for( TrackedSections::const_iterator it = m_children.begin(), itEnd = m_children.end(); - it != itEnd; - ++it ) - if( it->second.runState() != Completed ) { - m_runState = ExecutingChildren; - return; - } - m_runState = Completed; - } + void leave(); + TrackedSection* getParent() { return m_parent; } @@ -4852,9 +4971,31 @@ namespace SectionTracking { RunState m_runState; TrackedSections m_children; TrackedSection* m_parent; - }; + inline TrackedSection* TrackedSection::findChild( std::string const& childName ) { + TrackedSections::iterator it = m_children.find( childName ); + return it != m_children.end() + ? &it->second + : NULL; + } + inline TrackedSection* TrackedSection::acquireChild( std::string const& childName ) { + if( TrackedSection* child = findChild( childName ) ) + return child; + m_children.insert( std::make_pair( childName, TrackedSection( childName, this ) ) ); + return findChild( childName ); + } + inline void TrackedSection::leave() { + for( TrackedSections::const_iterator it = m_children.begin(), itEnd = m_children.end(); + it != itEnd; + ++it ) + if( it->second.runState() != Completed ) { + m_runState = ExecutingChildren; + return; + } + m_runState = Completed; + } + class TestCaseTracker { public: TestCaseTracker( std::string const& testCaseName ) @@ -4921,6 +5062,81 @@ using SectionTracking::TestCaseTracker; } // namespace Catch +// #included from: catch_fatal_condition.hpp +#define TWOBLUECUBES_CATCH_FATAL_CONDITION_H_INCLUDED + +namespace Catch { + + // Report the error condition then exit the process + inline void fatal( std::string const& message, int exitCode ) { + IContext& context = Catch::getCurrentContext(); + IResultCapture* resultCapture = context.getResultCapture(); + resultCapture->handleFatalErrorCondition( message ); + + if( Catch::alwaysTrue() ) // avoids "no return" warnings + exit( exitCode ); + } + +} // namespace Catch + +#if defined ( CATCH_PLATFORM_WINDOWS ) ///////////////////////////////////////// + +namespace Catch { + + struct FatalConditionHandler { + void reset() {} + }; + +} // namespace Catch + +#else // Not Windows - assumed to be POSIX compatible ////////////////////////// + +#include + +namespace Catch { + + struct SignalDefs { int id; const char* name; }; + extern SignalDefs signalDefs[]; + SignalDefs signalDefs[] = { + { SIGINT, "SIGINT - Terminal interrupt signal" }, + { SIGILL, "SIGILL - Illegal instruction signal" }, + { SIGFPE, "SIGFPE - Floating point error signal" }, + { SIGSEGV, "SIGSEGV - Segmentation violation signal" }, + { SIGTERM, "SIGTERM - Termination request signal" }, + { SIGABRT, "SIGABRT - Abort (abnormal termination) signal" } + }; + + struct FatalConditionHandler { + + static void handleSignal( int sig ) { + for( std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i ) + if( sig == signalDefs[i].id ) + fatal( signalDefs[i].name, -sig ); + fatal( "", -sig ); + } + + FatalConditionHandler() : m_isSet( true ) { + for( std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i ) + signal( signalDefs[i].id, handleSignal ); + } + ~FatalConditionHandler() { + reset(); + } + void reset() { + if( m_isSet ) { + for( std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i ) + signal( signalDefs[i].id, SIG_DFL ); + m_isSet = false; + } + } + + bool m_isSet; + }; + +} // namespace Catch + +#endif // not Windows + #include #include @@ -5076,7 +5292,7 @@ namespace Catch { } virtual void sectionEnded( SectionInfo const& info, Counts const& prevAssertions, double _durationInSeconds ) { - /*if( std::uncaught_exception() ) { // XXX Hack that makes Catch not run in loop in certain situations + /* if( std::uncaught_exception() ) { // XXX Hack that makes Catch not run in loop in certain situations m_unfinishedSections.push_back( UnfinishedSections( info, prevAssertions, _durationInSeconds ) ); return; }*/ @@ -5108,6 +5324,37 @@ namespace Catch { return &m_lastResult; } + virtual void handleFatalErrorCondition( std::string const& message ) { + ResultBuilder resultBuilder = makeUnexpectedResultBuilder(); + resultBuilder.setResultType( ResultWas::FatalErrorCondition ); + resultBuilder << message; + resultBuilder.captureExpression(); + + handleUnfinishedSections(); + + // Recreate section for test case (as we will lose the one that was in scope) + TestCaseInfo const& testCaseInfo = m_activeTestCase->getTestCaseInfo(); + SectionInfo testCaseSection( testCaseInfo.lineInfo, testCaseInfo.name, testCaseInfo.description ); + + Counts assertions; + assertions.failed = 1; + SectionStats testCaseSectionStats( testCaseSection, assertions, 0, false ); + m_reporter->sectionEnded( testCaseSectionStats ); + + TestCaseInfo testInfo = m_activeTestCase->getTestCaseInfo(); + + Totals deltaTotals; + deltaTotals.testCases.failed = 1; + m_reporter->testCaseEnded( TestCaseStats( testInfo, + deltaTotals, + "", + "", + false ) ); + m_totals.testCases.failed++; + testGroupEnded( "", m_totals, 1, 1 ); + m_reporter->testRunEnded( TestRunStats( m_runInfo, m_totals, false ) ); + } + public: // !TBD We need to do this another way! bool aborting() const { @@ -5129,12 +5376,12 @@ namespace Catch { Timer timer; timer.start(); if( m_reporter->getPreferences().shouldRedirectStdOut ) { - StreamRedirect coutRedir( std::cout, redirectedCout ); - StreamRedirect cerrRedir( std::cerr, redirectedCerr ); - m_activeTestCase->invoke(); + StreamRedirect coutRedir( Catch::cout(), redirectedCout ); + StreamRedirect cerrRedir( Catch::cerr(), redirectedCerr ); + invokeActiveTestCase(); } else { - m_activeTestCase->invoke(); + invokeActiveTestCase(); } duration = timer.getElapsedSeconds(); } @@ -5142,20 +5389,9 @@ namespace Catch { // This just means the test was aborted due to failure } catch(...) { - ResultBuilder exResult( m_lastAssertionInfo.macroName.c_str(), - m_lastAssertionInfo.lineInfo, - m_lastAssertionInfo.capturedExpression.c_str(), - m_lastAssertionInfo.resultDisposition ); - exResult.useActiveException(); + makeUnexpectedResultBuilder().useActiveException(); } - // If sections ended prematurely due to an exception we stored their - // infos here so we can tear them down outside the unwind process. - for( std::vector::const_reverse_iterator it = m_unfinishedSections.rbegin(), - itEnd = m_unfinishedSections.rend(); - it != itEnd; - ++it ) - sectionEnded( it->info, it->prevAssertions, it->durationInSeconds ); - m_unfinishedSections.clear(); + handleUnfinishedSections(); m_messages.clear(); Counts assertions = m_totals.assertions - prevAssertions; @@ -5171,7 +5407,32 @@ namespace Catch { m_reporter->sectionEnded( testCaseSectionStats ); } + void invokeActiveTestCase() { + FatalConditionHandler fatalConditionHandler; // Handle signals + m_activeTestCase->invoke(); + fatalConditionHandler.reset(); + } + private: + + ResultBuilder makeUnexpectedResultBuilder() const { + return ResultBuilder( m_lastAssertionInfo.macroName.c_str(), + m_lastAssertionInfo.lineInfo, + m_lastAssertionInfo.capturedExpression.c_str(), + m_lastAssertionInfo.resultDisposition ); + } + + void handleUnfinishedSections() { + // If sections ended prematurely due to an exception we stored their + // infos here so we can tear them down outside the unwind process. + for( std::vector::const_reverse_iterator it = m_unfinishedSections.rbegin(), + itEnd = m_unfinishedSections.rend(); + it != itEnd; + ++it ) + sectionEnded( it->info, it->prevAssertions, it->durationInSeconds ); + m_unfinishedSections.clear(); + } + struct UnfinishedSections { UnfinishedSections( SectionInfo const& _info, Counts const& _prevAssertions, double _durationInSeconds ) : info( _info ), prevAssertions( _prevAssertions ), durationInSeconds( _durationInSeconds ) @@ -5217,18 +5478,19 @@ namespace Catch { struct Version { Version( unsigned int _majorVersion, unsigned int _minorVersion, - unsigned int _buildNumber, - char const* const _branchName ) - : majorVersion( _majorVersion ), - minorVersion( _minorVersion ), - buildNumber( _buildNumber ), - branchName( _branchName ) - {} + unsigned int _patchNumber, + std::string const& _branchName, + unsigned int _buildNumber ); unsigned int const majorVersion; unsigned int const minorVersion; + unsigned int const patchNumber; + + // buildNumber is only used if branchName is not null + std::string const branchName; unsigned int const buildNumber; - char const* const branchName; + + friend std::ostream& operator << ( std::ostream& os, Version const& version ); private: void operator=( Version const& ); @@ -5259,7 +5521,7 @@ namespace Catch { Totals totals; - context.testGroupStarting( "", 1, 1 ); // deprecated? + context.testGroupStarting( "all tests", 1, 1 ); // deprecated? TestSpec testSpec = m_config->testSpec(); if( !testSpec.hasFilters() ) @@ -5282,7 +5544,15 @@ namespace Catch { m_testsAlreadyRun.insert( *it ); } } - context.testGroupEnded( "", totals, 1, 1 ); + std::vector skippedTestCases; + getRegistryHub().getTestCaseRegistry().getFilteredTests( testSpec, *m_config, skippedTestCases, true ); + + for( std::vector::const_iterator it = skippedTestCases.begin(), itEnd = skippedTestCases.end(); + it != itEnd; + ++it ) + m_reporter->skipTest( *it ); + + context.testGroupEnded( "all tests", totals, 1, 1 ); return totals; } @@ -5319,7 +5589,7 @@ namespace Catch { std::set m_testsAlreadyRun; }; - class Session { + class Session : NonCopyable { static bool alreadyInstantiated; public: @@ -5330,7 +5600,7 @@ namespace Catch { : m_cli( makeCommandLineParser() ) { if( alreadyInstantiated ) { std::string msg = "Only one instance of Catch::Session can ever be used"; - std::cerr << msg << std::endl; + Catch::cerr() << msg << std::endl; throw std::logic_error( msg ); } alreadyInstantiated = true; @@ -5340,15 +5610,10 @@ namespace Catch { } void showHelp( std::string const& processName ) { - std::cout << "\nCatch v" << libraryVersion.majorVersion << "." - << libraryVersion.minorVersion << " build " - << libraryVersion.buildNumber; - if( libraryVersion.branchName != std::string( "master" ) ) - std::cout << " (" << libraryVersion.branchName << " branch)"; - std::cout << "\n"; + Catch::cout() << "\nCatch v" << libraryVersion << "\n"; - m_cli.usage( std::cout, processName ); - std::cout << "For more detail usage please see the project docs\n" << std::endl; + m_cli.usage( Catch::cout(), processName ); + Catch::cout() << "For more detail usage please see the project docs\n" << std::endl; } int applyCommandLine( int argc, char* const argv[], OnUnusedOptions::DoWhat unusedOptionBehaviour = OnUnusedOptions::Fail ) { @@ -5362,11 +5627,12 @@ namespace Catch { catch( std::exception& ex ) { { Colour colourGuard( Colour::Red ); - std::cerr << "\nError(s) in input:\n" - << Text( ex.what(), TextAttributes().setIndent(2) ) - << "\n\n"; + Catch::cerr() + << "\nError(s) in input:\n" + << Text( ex.what(), TextAttributes().setIndent(2) ) + << "\n\n"; } - m_cli.usage( std::cout, m_configData.processName ); + m_cli.usage( Catch::cout(), m_configData.processName ); return (std::numeric_limits::max)(); } return 0; @@ -5392,6 +5658,9 @@ namespace Catch { try { config(); // Force config to be constructed + + std::srand( m_configData.rngSeed ); + Runner runner( m_config ); // Handle list request @@ -5401,7 +5670,7 @@ namespace Catch { return static_cast( runner.runTests().assertions.failed ); } catch( std::exception& ex ) { - std::cerr << ex.what() << std::endl; + Catch::cerr() << ex.what() << std::endl; return (std::numeric_limits::max)(); } } @@ -5442,10 +5711,18 @@ namespace Catch { #include #include #include +#include namespace Catch { class TestRegistry : public ITestCaseRegistry { + struct LexSort { + bool operator() (TestCase i,TestCase j) const { return (i& matchingTestCases ) const { + virtual void getFilteredTests( TestSpec const& testSpec, IConfig const& config, std::vector& matchingTestCases, bool negated = false ) const { + for( std::vector::const_iterator it = m_functionsInOrder.begin(), itEnd = m_functionsInOrder.end(); it != itEnd; ++it ) { - if( testSpec.matches( *it ) && ( config.allowThrows() || !it->throws() ) ) + bool includeTest = testSpec.matches( *it ) && ( config.allowThrows() || !it->throws() ); + if( includeTest != negated ) matchingTestCases.push_back( *it ); } + sortTests( config, matchingTestCases ); } private: + static void sortTests( IConfig const& config, std::vector& matchingTestCases ) { + + switch( config.runOrder() ) { + case RunTests::InLexicographicalOrder: + std::sort( matchingTestCases.begin(), matchingTestCases.end(), LexSort() ); + break; + case RunTests::InRandomOrder: + { + RandomNumberGenerator rng; + std::random_shuffle( matchingTestCases.begin(), matchingTestCases.end(), rng ); + } + break; + case RunTests::InDeclarationOrder: + // already in declaration order + break; + } + } std::set m_functions; std::vector m_functionsInOrder; std::vector m_nonHiddenFunctions; @@ -5619,7 +5916,7 @@ namespace Catch { throw; } @catch (NSException *exception) { - return toString( [exception description] ); + return Catch::toString( [exception description] ); } #else throw; @@ -5766,6 +6063,7 @@ namespace Catch { #include #include +#include namespace Catch { @@ -5829,6 +6127,15 @@ namespace Catch { isOwned = false; } } + +#ifndef CATCH_CONFIG_NOSTDOUT // If you #define this you must implement this functions + std::ostream& cout() { + return std::cout; + } + std::ostream& cerr() { + return std::cerr; + } +#endif } namespace Catch { @@ -5878,7 +6185,7 @@ namespace Catch { std::string testName = getResultCapture()->getCurrentTestName(); std::map::const_iterator it = - m_generatorsByTestName.find( testName ); + m_generatorsByTestName.find( testName ); return it != m_generatorsByTestName.end() ? it->second : NULL; @@ -5914,8 +6221,8 @@ namespace Catch { } Stream createStream( std::string const& streamName ) { - if( streamName == "stdout" ) return Stream( std::cout.rdbuf(), false ); - if( streamName == "stderr" ) return Stream( std::cerr.rdbuf(), false ); + if( streamName == "stdout" ) return Stream( Catch::cout().rdbuf(), false ); + if( streamName == "stderr" ) return Stream( Catch::cerr().rdbuf(), false ); if( streamName == "debug" ) return Stream( new StreamBufImpl, true ); throw std::domain_error( "Unknown stream: " + streamName ); @@ -5930,14 +6237,35 @@ namespace Catch { // #included from: catch_console_colour_impl.hpp #define TWOBLUECUBES_CATCH_CONSOLE_COLOUR_IMPL_HPP_INCLUDED -namespace Catch { namespace Detail { - struct IColourImpl { - virtual ~IColourImpl() {} - virtual void use( Colour::Code _colourCode ) = 0; - }; -}} +namespace Catch { + namespace { -#if defined ( CATCH_PLATFORM_WINDOWS ) ///////////////////////////////////////// + struct IColourImpl { + virtual ~IColourImpl() {} + virtual void use( Colour::Code _colourCode ) = 0; + }; + + struct NoColourImpl : IColourImpl { + void use( Colour::Code ) {} + + static IColourImpl* instance() { + static NoColourImpl s_instance; + return &s_instance; + } + }; + + } // anon namespace +} // namespace Catch + +#if !defined( CATCH_CONFIG_COLOUR_NONE ) && !defined( CATCH_CONFIG_COLOUR_WINDOWS ) && !defined( CATCH_CONFIG_COLOUR_ANSI ) +# ifdef CATCH_PLATFORM_WINDOWS +# define CATCH_CONFIG_COLOUR_WINDOWS +# else +# define CATCH_CONFIG_COLOUR_ANSI +# endif +#endif + +#if defined ( CATCH_CONFIG_COLOUR_WINDOWS ) ///////////////////////////////////////// #ifndef NOMINMAX #define NOMINMAX @@ -5952,7 +6280,7 @@ namespace Catch { namespace Detail { namespace Catch { namespace { - class Win32ColourImpl : public Detail::IColourImpl { + class Win32ColourImpl : public IColourImpl { public: Win32ColourImpl() : stdoutHandle( GetStdHandle(STD_OUTPUT_HANDLE) ) { @@ -5989,11 +6317,7 @@ namespace { WORD originalAttributes; }; - inline bool shouldUseColourForPlatform() { - return true; - } - - static Detail::IColourImpl* platformColourInstance() { + IColourImpl* platformColourInstance() { static Win32ColourImpl s_instance; return &s_instance; } @@ -6001,7 +6325,7 @@ namespace { } // end anon namespace } // end namespace Catch -#else // Not Windows - assumed to be POSIX compatible ////////////////////////// +#elif defined( CATCH_CONFIG_COLOUR_ANSI ) ////////////////////////////////////// #include @@ -6012,7 +6336,7 @@ namespace { // Thanks to Adam Strzelecki for original contribution // (http://github.com/nanoant) // https://github.com/philsquared/Catch/pull/131 - class PosixColourImpl : public Detail::IColourImpl { + class PosixColourImpl : public IColourImpl { public: virtual void use( Colour::Code _colourCode ) { switch( _colourCode ) { @@ -6033,53 +6357,48 @@ namespace { case Colour::Bright: throw std::logic_error( "not a colour" ); } } + static IColourImpl* instance() { + static PosixColourImpl s_instance; + return &s_instance; + } + private: void setColour( const char* _escapeCode ) { - std::cout << '\033' << _escapeCode; + Catch::cout() << '\033' << _escapeCode; } }; - inline bool shouldUseColourForPlatform() { - return isatty(STDOUT_FILENO); - } - - static Detail::IColourImpl* platformColourInstance() { - static PosixColourImpl s_instance; - return &s_instance; + IColourImpl* platformColourInstance() { + Ptr config = getCurrentContext().getConfig(); + return (config && config->forceColour()) || isatty(STDOUT_FILENO) + ? PosixColourImpl::instance() + : NoColourImpl::instance(); } } // end anon namespace } // end namespace Catch -#endif // not Windows +#else // not Windows or ANSI /////////////////////////////////////////////// namespace Catch { - namespace { - struct NoColourImpl : Detail::IColourImpl { - void use( Colour::Code ) {} + static IColourImpl* platformColourInstance() { return NoColourImpl::instance(); } - static IColourImpl* instance() { - static NoColourImpl s_instance; - return &s_instance; - } - }; - static bool shouldUseColour() { - return shouldUseColourForPlatform() && !isDebuggerActive(); - } - } +} // end namespace Catch + +#endif // Windows/ ANSI/ None + +namespace Catch { Colour::Colour( Code _colourCode ) : m_moved( false ) { use( _colourCode ); } Colour::Colour( Colour const& _other ) : m_moved( false ) { const_cast( _other ).m_moved = true; } Colour::~Colour(){ if( !m_moved ) use( None ); } - void Colour::use( Code _colourCode ) { - impl()->use( _colourCode ); - } - Detail::IColourImpl* Colour::impl() { - return shouldUseColour() - ? platformColourInstance() - : NoColourImpl::instance(); + void Colour::use( Code _colourCode ) { + static IColourImpl* impl = isDebuggerActive() + ? NoColourImpl::instance() + : platformColourInstance(); + impl->use( _colourCode ); } } // end namespace Catch @@ -6244,7 +6563,7 @@ namespace Catch { namespace Catch { inline TestCaseInfo::SpecialProperties parseSpecialTag( std::string const& tag ) { - if( tag == "." || + if( startsWith( tag, "." ) || tag == "hide" || tag == "!hide" ) return TestCaseInfo::IsHidden; @@ -6264,13 +6583,13 @@ namespace Catch { if( isReservedTag( tag ) ) { { Colour colourGuard( Colour::Red ); - std::cerr + Catch::cerr() << "Tag name [" << tag << "] not allowed.\n" << "Tag names starting with non alpha-numeric characters are reserved\n"; } { Colour colourGuard( Colour::FileName ); - std::cerr << _lineInfo << std::endl; + Catch::cerr() << _lineInfo << std::endl; } exit(1); } @@ -6298,14 +6617,15 @@ namespace Catch { } else { if( c == ']' ) { - enforceNotReservedTag( tag, _lineInfo ); - - inTag = false; - if( tag == "hide" || tag == "." ) + TestCaseInfo::SpecialProperties prop = parseSpecialTag( tag ); + if( prop == TestCaseInfo::IsHidden ) isHidden = true; - else - tags.insert( tag ); + else if( prop == TestCaseInfo::None ) + enforceNotReservedTag( tag, _lineInfo ); + + tags.insert( tag ); tag.clear(); + inTag = false; } else tag += c; @@ -6422,8 +6742,33 @@ namespace Catch { namespace Catch { - // These numbers are maintained by a script - Version libraryVersion( 1, 0, 53, "master" ); + Version::Version + ( unsigned int _majorVersion, + unsigned int _minorVersion, + unsigned int _patchNumber, + std::string const& _branchName, + unsigned int _buildNumber ) + : majorVersion( _majorVersion ), + minorVersion( _minorVersion ), + patchNumber( _patchNumber ), + branchName( _branchName ), + buildNumber( _buildNumber ) + {} + + std::ostream& operator << ( std::ostream& os, Version const& version ) { + os << version.majorVersion << "." + << version.minorVersion << "." + << version.patchNumber; + + if( !version.branchName.empty() ) { + os << "-" << version.branchName + << "." << version.buildNumber; + } + return os; + } + + Version libraryVersion( 1, 2, 1, "", 0 ); + } // #included from: catch_message.hpp @@ -6507,6 +6852,7 @@ namespace Catch virtual void testCaseEnded( TestCaseStats const& testCaseStats ); virtual void testGroupEnded( TestGroupStats const& testGroupStats ); virtual void testRunEnded( TestRunStats const& testRunStats ); + virtual void skipTest( TestCaseInfo const& ); private: Ptr m_legacyReporter; @@ -6580,6 +6926,8 @@ namespace Catch void LegacyReporterAdapter::testRunEnded( TestRunStats const& testRunStats ) { m_legacyReporter->EndTesting( testRunStats.totals ); } + void LegacyReporterAdapter::skipTest( TestCaseInfo const& ) { + } } // #included from: catch_timer.hpp @@ -6602,11 +6950,11 @@ namespace Catch { uint64_t getCurrentTicks() { static uint64_t hz=0, hzo=0; if (!hz) { - QueryPerformanceFrequency((LARGE_INTEGER*)&hz); - QueryPerformanceCounter((LARGE_INTEGER*)&hzo); + QueryPerformanceFrequency( reinterpret_cast( &hz ) ); + QueryPerformanceCounter( reinterpret_cast( &hzo ) ); } uint64_t t; - QueryPerformanceCounter((LARGE_INTEGER*)&t); + QueryPerformanceCounter( reinterpret_cast( &t ) ); return ((t-hzo)*1000000)/hz; } #else @@ -6621,14 +6969,14 @@ namespace Catch { void Timer::start() { m_ticks = getCurrentTicks(); } - unsigned int Timer::getElapsedNanoseconds() const { + unsigned int Timer::getElapsedMicroseconds() const { return static_cast(getCurrentTicks() - m_ticks); } unsigned int Timer::getElapsedMilliseconds() const { - return static_cast((getCurrentTicks() - m_ticks)/1000); + return static_cast(getElapsedMicroseconds()/1000); } double Timer::getElapsedSeconds() const { - return (getCurrentTicks() - m_ticks)/1000000.0; + return getElapsedMicroseconds()/1000000.0; } } // namespace Catch @@ -6666,6 +7014,20 @@ namespace Catch { return start != std::string::npos ? str.substr( start, 1+end-start ) : ""; } + bool replaceInPlace( std::string& str, std::string const& replaceThis, std::string const& withThis ) { + bool replaced = false; + std::size_t i = str.find( replaceThis ); + while( i != std::string::npos ) { + replaced = true; + str = str.substr( 0, i ) + withThis + str.substr( i+replaceThis.size() ); + if( i < str.size()-withThis.size() ) + i = str.find( replaceThis, i+withThis.size() ); + else + i = std::string::npos; + } + return replaced; + } + pluralise::pluralise( std::size_t count, std::string const& label ) : m_count( count ), m_label( label ) @@ -6693,6 +7055,9 @@ namespace Catch { bool SourceLineInfo::operator == ( SourceLineInfo const& other ) const { return line == other.line && file == other.file; } + bool SourceLineInfo::operator < ( SourceLineInfo const& other ) const { + return line < other.line || ( line == other.line && file < other.file ); + } std::ostream& operator << ( std::ostream& os, SourceLineInfo const& info ) { #ifndef __GNUG__ @@ -6787,7 +7152,7 @@ namespace Catch { size = sizeof(info); if( sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &size, NULL, 0) != 0 ) { - std::cerr << "\n** Call to sysctl failed - unable to determine if debugger is active **\n" << std::endl; + Catch::cerr() << "\n** Call to sysctl failed - unable to determine if debugger is active **\n" << std::endl; return false; } @@ -6828,7 +7193,7 @@ namespace Catch { namespace Catch { void writeToDebugConsole( std::string const& text ) { // !TBD: Need a version for Mac/ XCode and other IDEs - std::cout << text; + Catch::cout() << text; } } #endif // Platform @@ -6840,6 +7205,8 @@ namespace Catch { namespace Detail { + std::string unprintableString = "{?}"; + namespace { struct Endianness { enum Arch { Big, Little }; @@ -6898,7 +7265,7 @@ std::string toString( std::wstring const& value ) { s.reserve( value.size() ); for(size_t i = 0; i < value.size(); ++i ) s += value[i] <= 0xff ? static_cast( value[i] ) : '?'; - return toString( s ); + return Catch::toString( s ); } std::string toString( const char* const value ) { @@ -6922,20 +7289,21 @@ std::string toString( wchar_t* const value ) std::string toString( int value ) { std::ostringstream oss; oss << value; + if( value >= 255 ) + oss << " (0x" << std::hex << value << ")"; return oss.str(); } std::string toString( unsigned long value ) { std::ostringstream oss; - if( value > 8192 ) - oss << "0x" << std::hex << value; - else - oss << value; + oss << value; + if( value >= 255 ) + oss << " (0x" << std::hex << value << ")"; return oss.str(); } std::string toString( unsigned int value ) { - return toString( static_cast( value ) ); + return Catch::toString( static_cast( value ) ); } template @@ -7061,7 +7429,7 @@ namespace Catch { if( !result.isOk() ) { if( getCurrentContext().getConfig()->shouldDebugBreak() ) m_shouldDebugBreak = true; - if( getCurrentContext().getRunner()->aborting() || m_assertionInfo.resultDisposition == ResultDisposition::Normal ) + if( getCurrentContext().getRunner()->aborting() || (m_assertionInfo.resultDisposition & ResultDisposition::Normal) ) m_shouldThrow = true; } } @@ -7201,7 +7569,7 @@ namespace Catch { } catch( std::exception& ex ) { Colour colourGuard( Colour::Red ); - std::cerr << ex.what() << std::endl; + Catch::cerr() << ex.what() << std::endl; exit(1); } } @@ -7214,6 +7582,8 @@ namespace Catch { // #included from: catch_reporter_bases.hpp #define TWOBLUECUBES_CATCH_REPORTER_BASES_HPP_INCLUDED +#include + namespace Catch { struct StreamingReporterBase : SharedImpl { @@ -7246,7 +7616,6 @@ namespace Catch { } virtual void testCaseEnded( TestCaseStats const& /* _testCaseStats */ ) { currentTestCaseInfo.reset(); - assert( m_sectionStack.empty() ); } virtual void testGroupEnded( TestGroupStats const& /* _testGroupStats */ ) { currentGroupInfo.reset(); @@ -7257,6 +7626,11 @@ namespace Catch { currentTestRunInfo.reset(); } + virtual void skipTest( TestCaseInfo const& ) { + // Don't do anything with this by default. + // It can optionally be overridden in the derived class. + } + Ptr m_config; std::ostream& stream; @@ -7386,6 +7760,8 @@ namespace Catch { } virtual void testRunEndedCumulative() = 0; + virtual void skipTest( TestCaseInfo const& ) {} + Ptr m_config; std::ostream& stream; std::vector m_assertions; @@ -7401,6 +7777,16 @@ namespace Catch { }; + template + char const* getLineOfChars() { + static char line[CATCH_CONFIG_CONSOLE_WIDTH] = {0}; + if( !*line ) { + memset( line, C, CATCH_CONFIG_CONSOLE_WIDTH-1 ); + line[CATCH_CONFIG_CONSOLE_WIDTH-1] = 0; + } + return line; + } + } // end namespace Catch // #included from: ../internal/catch_reporter_registrars.hpp @@ -7470,7 +7856,6 @@ namespace Catch { #define TWOBLUECUBES_CATCH_XMLWRITER_HPP_INCLUDED #include -#include #include #include @@ -7513,7 +7898,7 @@ namespace Catch { XmlWriter() : m_tagIsOpen( false ), m_needsNewline( false ), - m_os( &std::cout ) + m_os( &Catch::cout() ) {} XmlWriter( std::ostream& os ) @@ -7527,27 +7912,6 @@ namespace Catch { endElement(); } -//# ifndef CATCH_CPP11_OR_GREATER -// XmlWriter& operator = ( XmlWriter const& other ) { -// XmlWriter temp( other ); -// swap( temp ); -// return *this; -// } -//# else -// XmlWriter( XmlWriter const& ) = default; -// XmlWriter( XmlWriter && ) = default; -// XmlWriter& operator = ( XmlWriter const& ) = default; -// XmlWriter& operator = ( XmlWriter && ) = default; -//# endif -// -// void swap( XmlWriter& other ) { -// std::swap( m_tagIsOpen, other.m_tagIsOpen ); -// std::swap( m_needsNewline, other.m_needsNewline ); -// std::swap( m_tags, other.m_tags ); -// std::swap( m_indent, other.m_indent ); -// std::swap( m_os, other.m_os ); -// } - XmlWriter& startElement( std::string const& name ) { ensureTagClosed(); newlineIfNecessary(); @@ -7683,81 +8047,90 @@ namespace Catch { } namespace Catch { - class XmlReporter : public SharedImpl { + class XmlReporter : public StreamingReporterBase { public: - XmlReporter( ReporterConfig const& config ) : m_config( config ), m_sectionDepth( 0 ) {} + XmlReporter( ReporterConfig const& _config ) + : StreamingReporterBase( _config ), + m_sectionDepth( 0 ) + {} + + virtual ~XmlReporter(); static std::string getDescription() { return "Reports test results as an XML document"; } - virtual ~XmlReporter(); - private: // IReporter - - virtual bool shouldRedirectStdout() const { - return true; + public: // StreamingReporterBase + virtual ReporterPreferences getPreferences() const { + ReporterPreferences prefs; + prefs.shouldRedirectStdOut = true; + return prefs; } - virtual void StartTesting() { - m_xml.setStream( m_config.stream() ); - m_xml.startElement( "Catch" ); - if( !m_config.fullConfig()->name().empty() ) - m_xml.writeAttribute( "name", m_config.fullConfig()->name() ); + virtual void noMatchingTestCases( std::string const& s ) { + StreamingReporterBase::noMatchingTestCases( s ); } - virtual void EndTesting( const Totals& totals ) { - m_xml.scopedElement( "OverallResults" ) - .writeAttribute( "successes", totals.assertions.passed ) - .writeAttribute( "failures", totals.assertions.failed ) - .writeAttribute( "expectedFailures", totals.assertions.failedButOk ); - m_xml.endElement(); + virtual void testRunStarting( TestRunInfo const& testInfo ) { + StreamingReporterBase::testRunStarting( testInfo ); + m_xml.setStream( stream ); + m_xml.startElement( "Catch" ); + if( !m_config->name().empty() ) + m_xml.writeAttribute( "name", m_config->name() ); } - virtual void StartGroup( const std::string& groupName ) { + virtual void testGroupStarting( GroupInfo const& groupInfo ) { + StreamingReporterBase::testGroupStarting( groupInfo ); m_xml.startElement( "Group" ) - .writeAttribute( "name", groupName ); + .writeAttribute( "name", groupInfo.name ); } - virtual void EndGroup( const std::string&, const Totals& totals ) { - m_xml.scopedElement( "OverallResults" ) - .writeAttribute( "successes", totals.assertions.passed ) - .writeAttribute( "failures", totals.assertions.failed ) - .writeAttribute( "expectedFailures", totals.assertions.failedButOk ); - m_xml.endElement(); + virtual void testCaseStarting( TestCaseInfo const& testInfo ) { + StreamingReporterBase::testCaseStarting(testInfo); + m_xml.startElement( "TestCase" ).writeAttribute( "name", trim( testInfo.name ) ); + + if ( m_config->showDurations() == ShowDurations::Always ) + m_testCaseTimer.start(); } - virtual void StartSection( const std::string& sectionName, const std::string& description ) { + virtual void sectionStarting( SectionInfo const& sectionInfo ) { + StreamingReporterBase::sectionStarting( sectionInfo ); if( m_sectionDepth++ > 0 ) { m_xml.startElement( "Section" ) - .writeAttribute( "name", trim( sectionName ) ) - .writeAttribute( "description", description ); + .writeAttribute( "name", trim( sectionInfo.name ) ) + .writeAttribute( "description", sectionInfo.description ); } } - virtual void NoAssertionsInSection( const std::string& ) {} - virtual void NoAssertionsInTestCase( const std::string& ) {} - virtual void EndSection( const std::string& /*sectionName*/, const Counts& assertions ) { - if( --m_sectionDepth > 0 ) { - m_xml.scopedElement( "OverallResults" ) - .writeAttribute( "successes", assertions.passed ) - .writeAttribute( "failures", assertions.failed ) - .writeAttribute( "expectedFailures", assertions.failedButOk ); - m_xml.endElement(); - } - } + virtual void assertionStarting( AssertionInfo const& ) { } - virtual void StartTestCase( const Catch::TestCaseInfo& testInfo ) { - m_xml.startElement( "TestCase" ).writeAttribute( "name", trim( testInfo.name ) ); - m_currentTestSuccess = true; - } + virtual bool assertionEnded( AssertionStats const& assertionStats ) { + const AssertionResult& assertionResult = assertionStats.assertionResult; - virtual void Result( const Catch::AssertionResult& assertionResult ) { - if( !m_config.fullConfig()->includeSuccessfulResults() && assertionResult.getResultType() == ResultWas::Ok ) - return; + // Print any info messages in tags. + if( assertionStats.assertionResult.getResultType() != ResultWas::Ok ) { + for( std::vector::const_iterator it = assertionStats.infoMessages.begin(), itEnd = assertionStats.infoMessages.end(); + it != itEnd; + ++it ) { + if( it->type == ResultWas::Info ) { + m_xml.scopedElement( "Info" ) + .writeText( it->message ); + } else if ( it->type == ResultWas::Warning ) { + m_xml.scopedElement( "Warning" ) + .writeText( it->message ); + } + } + } + // Drop out if result was successful but we're not printing them. + if( !m_config->includeSuccessfulResults() && isOk(assertionResult.getResultType()) ) + return true; + + // Print the expression if there is one. if( assertionResult.hasExpression() ) { m_xml.startElement( "Expression" ) .writeAttribute( "success", assertionResult.succeeded() ) + .writeAttribute( "type", assertionResult.getTestMacroName() ) .writeAttribute( "filename", assertionResult.getSourceInfo().file ) .writeAttribute( "line", assertionResult.getSourceInfo().line ); @@ -7765,58 +8138,96 @@ namespace Catch { .writeText( assertionResult.getExpression() ); m_xml.scopedElement( "Expanded" ) .writeText( assertionResult.getExpandedExpression() ); - m_currentTestSuccess &= assertionResult.succeeded(); } + // And... Print a result applicable to each result type. switch( assertionResult.getResultType() ) { case ResultWas::ThrewException: m_xml.scopedElement( "Exception" ) .writeAttribute( "filename", assertionResult.getSourceInfo().file ) .writeAttribute( "line", assertionResult.getSourceInfo().line ) .writeText( assertionResult.getMessage() ); - m_currentTestSuccess = false; + break; + case ResultWas::FatalErrorCondition: + m_xml.scopedElement( "Fatal Error Condition" ) + .writeAttribute( "filename", assertionResult.getSourceInfo().file ) + .writeAttribute( "line", assertionResult.getSourceInfo().line ) + .writeText( assertionResult.getMessage() ); break; case ResultWas::Info: m_xml.scopedElement( "Info" ) .writeText( assertionResult.getMessage() ); break; case ResultWas::Warning: - m_xml.scopedElement( "Warning" ) - .writeText( assertionResult.getMessage() ); + // Warning will already have been written break; case ResultWas::ExplicitFailure: m_xml.scopedElement( "Failure" ) .writeText( assertionResult.getMessage() ); - m_currentTestSuccess = false; break; - case ResultWas::Unknown: - case ResultWas::Ok: - case ResultWas::FailureBit: - case ResultWas::ExpressionFailed: - case ResultWas::Exception: - case ResultWas::DidntThrowException: + default: break; } + if( assertionResult.hasExpression() ) m_xml.endElement(); + + return true; } - virtual void Aborted() { - // !TBD + virtual void sectionEnded( SectionStats const& sectionStats ) { + StreamingReporterBase::sectionEnded( sectionStats ); + if( --m_sectionDepth > 0 ) { + XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResults" ); + e.writeAttribute( "successes", sectionStats.assertions.passed ); + e.writeAttribute( "failures", sectionStats.assertions.failed ); + e.writeAttribute( "expectedFailures", sectionStats.assertions.failedButOk ); + + if ( m_config->showDurations() == ShowDurations::Always ) + e.writeAttribute( "durationInSeconds", sectionStats.durationInSeconds ); + + m_xml.endElement(); + } } - virtual void EndTestCase( const Catch::TestCaseInfo&, const Totals&, const std::string&, const std::string& ) { - m_xml.scopedElement( "OverallResult" ).writeAttribute( "success", m_currentTestSuccess ); + virtual void testCaseEnded( TestCaseStats const& testCaseStats ) { + StreamingReporterBase::testCaseEnded( testCaseStats ); + XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResult" ); + e.writeAttribute( "success", testCaseStats.totals.assertions.allOk() ); + + if ( m_config->showDurations() == ShowDurations::Always ) + e.writeAttribute( "durationInSeconds", m_testCaseTimer.getElapsedSeconds() ); + + m_xml.endElement(); + } + + virtual void testGroupEnded( TestGroupStats const& testGroupStats ) { + StreamingReporterBase::testGroupEnded( testGroupStats ); + // TODO: Check testGroupStats.aborting and act accordingly. + m_xml.scopedElement( "OverallResults" ) + .writeAttribute( "successes", testGroupStats.totals.assertions.passed ) + .writeAttribute( "failures", testGroupStats.totals.assertions.failed ) + .writeAttribute( "expectedFailures", testGroupStats.totals.assertions.failedButOk ); + m_xml.endElement(); + } + + virtual void testRunEnded( TestRunStats const& testRunStats ) { + StreamingReporterBase::testRunEnded( testRunStats ); + m_xml.scopedElement( "OverallResults" ) + .writeAttribute( "successes", testRunStats.totals.assertions.passed ) + .writeAttribute( "failures", testRunStats.totals.assertions.failed ) + .writeAttribute( "expectedFailures", testRunStats.totals.assertions.failedButOk ); m_xml.endElement(); } private: - ReporterConfig m_config; - bool m_currentTestSuccess; + Timer m_testCaseTimer; XmlWriter m_xml; int m_sectionDepth; }; + INTERNAL_CATCH_REGISTER_REPORTER( "xml", XmlReporter ) + } // end namespace Catch // #included from: ../reporters/catch_reporter_junit.hpp @@ -7943,7 +8354,7 @@ namespace Catch { xml.writeAttribute( "classname", className ); xml.writeAttribute( "name", name ); } - xml.writeAttribute( "time", toString( sectionNode.stats.durationInSeconds ) ); + xml.writeAttribute( "time", Catch::toString( sectionNode.stats.durationInSeconds ) ); writeAssertions( sectionNode ); @@ -7976,6 +8387,7 @@ namespace Catch { std::string elementName; switch( result.getResultType() ) { case ResultWas::ThrewException: + case ResultWas::FatalErrorCondition: elementName = "error"; break; case ResultWas::ExplicitFailure: @@ -8034,8 +8446,6 @@ namespace Catch { // #included from: ../reporters/catch_reporter_console.hpp #define TWOBLUECUBES_CATCH_REPORTER_CONSOLE_HPP_INCLUDED -#include - namespace Catch { struct ConsoleReporter : StreamingReporterBase { @@ -8170,6 +8580,11 @@ namespace Catch { passOrFail = "FAILED"; messageLabel = "due to unexpected exception with message"; break; + case ResultWas::FatalErrorCondition: + colour = Colour::Error; + passOrFail = "FAILED"; + messageLabel = "due to a fatal error condition"; + break; case ResultWas::DidntThrowException: colour = Colour::Error; passOrFail = "FAILED"; @@ -8279,14 +8694,12 @@ namespace Catch { stream << "\n" << getLineOfChars<'~'>() << "\n"; Colour colour( Colour::SecondaryText ); stream << currentTestRunInfo->name - << " is a Catch v" << libraryVersion.majorVersion << "." - << libraryVersion.minorVersion << " b" - << libraryVersion.buildNumber; - if( libraryVersion.branchName != std::string( "master" ) ) - stream << " (" << libraryVersion.branchName << ")"; - stream << " host application.\n" + << " is a Catch v" << libraryVersion << " host application.\n" << "Run with -? for options\n\n"; + if( m_config->rngSeed() != 0 ) + stream << "Randomness seeded to: " << m_config->rngSeed() << "\n\n"; + currentTestRunInfo.used = true; } void lazyPrintGroupInfo() { @@ -8458,15 +8871,6 @@ namespace Catch { void printSummaryDivider() { stream << getLineOfChars<'-'>() << "\n"; } - template - static char const* getLineOfChars() { - static char line[CATCH_CONFIG_CONSOLE_WIDTH] = {0}; - if( !*line ) { - memset( line, C, CATCH_CONFIG_CONSOLE_WIDTH-1 ); - line[CATCH_CONFIG_CONSOLE_WIDTH-1] = 0; - } - return line; - } private: bool m_headerPrinted; @@ -8575,6 +8979,13 @@ namespace Catch { printExpressionWas(); printRemainingMessages(); break; + case ResultWas::FatalErrorCondition: + printResultType( Colour::Error, failedString() ); + printIssue( "fatal error condition with message:" ); + printMessage(); + printExpressionWas(); + printRemainingMessages(); + break; case ResultWas::DidntThrowException: printResultType( Colour::Error, failedString() ); printIssue( "expected exception, got none" ); @@ -8804,8 +9215,6 @@ namespace Catch { Matchers::Impl::StdString::EndsWith::~EndsWith() {} void Config::dummy() {} - - INTERNAL_CATCH_REGISTER_LEGACY_REPORTER( "xml", XmlReporter ) } #ifdef __clang__ @@ -8994,9 +9403,13 @@ using Catch::Detail::Approx; #define TWOBLUECUBES_CATCH_REENABLE_WARNINGS_H_INCLUDED #ifdef __clang__ -#pragma clang diagnostic pop +# ifdef __ICC // icpc defines the __clang__ macro +# pragma warning(pop) +# else +# pragma clang diagnostic pop +# endif #elif defined __GNUC__ -#pragma GCC diagnostic pop +# pragma GCC diagnostic pop #endif #endif // TWOBLUECUBES_SINGLE_INCLUDE_CATCH_HPP_INCLUDED diff --git a/test/include/catch_orig.hpp b/test/include/catch_orig.hpp index 6b8dfb5ebd0..de61226cf68 100644 --- a/test/include/catch_orig.hpp +++ b/test/include/catch_orig.hpp @@ -1,6 +1,6 @@ /* - * CATCH v1.0 build 53 (master branch) - * Generated: 2014-08-20 08:08:19.533804 + * Catch v1.2.1 + * Generated: 2015-06-30 18:23:27.961086 * ---------------------------------------------------------- * This file has been merged from multiple headers. Please don't edit it directly * Copyright (c) 2012 Two Blue Cubes Ltd. All rights reserved. @@ -13,31 +13,43 @@ #define TWOBLUECUBES_CATCH_HPP_INCLUDED +#ifdef __clang__ +# pragma clang system_header +#elif defined __GNUC__ +# pragma GCC system_header +#endif + // #included from: internal/catch_suppress_warnings.h #define TWOBLUECUBES_CATCH_SUPPRESS_WARNINGS_H_INCLUDED #ifdef __clang__ -#pragma clang diagnostic ignored "-Wglobal-constructors" -#pragma clang diagnostic ignored "-Wvariadic-macros" -#pragma clang diagnostic ignored "-Wc99-extensions" -#pragma clang diagnostic ignored "-Wunused-variable" -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wpadded" -#pragma clang diagnostic ignored "-Wc++98-compat" -#pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +# ifdef __ICC // icpc defines the __clang__ macro +# pragma warning(push) +# pragma warning(disable: 161 1682) +# else // __ICC +# pragma clang diagnostic ignored "-Wglobal-constructors" +# pragma clang diagnostic ignored "-Wvariadic-macros" +# pragma clang diagnostic ignored "-Wc99-extensions" +# pragma clang diagnostic ignored "-Wunused-variable" +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wpadded" +# pragma clang diagnostic ignored "-Wc++98-compat" +# pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +# pragma clang diagnostic ignored "-Wswitch-enum" +# endif #elif defined __GNUC__ -#pragma GCC diagnostic ignored "-Wvariadic-macros" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpadded" +# pragma GCC diagnostic ignored "-Wvariadic-macros" +# pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wpadded" #endif -#ifdef CATCH_CONFIG_MAIN -# define CATCH_CONFIG_RUNNER +#if defined(CATCH_CONFIG_MAIN) || defined(CATCH_CONFIG_RUNNER) +# define CATCH_IMPL #endif -#ifdef CATCH_CONFIG_RUNNER +#ifdef CATCH_IMPL # ifndef CLARA_CONFIG_MAIN # define CLARA_CONFIG_MAIN_NOT_DEFINED # define CLARA_CONFIG_MAIN @@ -64,16 +76,34 @@ // #included from: catch_compiler_capabilities.h #define TWOBLUECUBES_CATCH_COMPILER_CAPABILITIES_HPP_INCLUDED -// Much of the following code is based on Boost (1.53) +// Detect a number of compiler features - mostly C++11/14 conformance - by compiler +// The following features are defined: +// +// CATCH_CONFIG_CPP11_NULLPTR : is nullptr supported? +// CATCH_CONFIG_CPP11_NOEXCEPT : is noexcept supported? +// CATCH_CONFIG_CPP11_GENERATED_METHODS : The delete and default keywords for compiler generated methods +// CATCH_CONFIG_CPP11_IS_ENUM : std::is_enum is supported? +// CATCH_CONFIG_CPP11_TUPLE : std::tuple is supported + +// CATCH_CONFIG_CPP11_OR_GREATER : Is C++11 supported? + +// CATCH_CONFIG_VARIADIC_MACROS : are variadic macros supported? + +// In general each macro has a _NO_ form +// (e.g. CATCH_CONFIG_CPP11_NO_NULLPTR) which disables the feature. +// Many features, at point of detection, define an _INTERNAL_ macro, so they +// can be combined, en-mass, with the _NO_ forms later. + +// All the C++11 features can be disabled with CATCH_CONFIG_NO_CPP11 #ifdef __clang__ # if __has_feature(cxx_nullptr) -# define CATCH_CONFIG_CPP11_NULLPTR +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR # endif # if __has_feature(cxx_noexcept) -# define CATCH_CONFIG_CPP11_NOEXCEPT +# define CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT # endif #endif // __clang__ @@ -82,51 +112,26 @@ // Borland #ifdef __BORLANDC__ -#if (__BORLANDC__ > 0x582 ) -//#define CATCH_CONFIG_SFINAE // Not confirmed -#endif - #endif // __BORLANDC__ //////////////////////////////////////////////////////////////////////////////// // EDG #ifdef __EDG_VERSION__ -#if (__EDG_VERSION__ > 238 ) -//#define CATCH_CONFIG_SFINAE // Not confirmed -#endif - #endif // __EDG_VERSION__ //////////////////////////////////////////////////////////////////////////////// // Digital Mars #ifdef __DMC__ -#if (__DMC__ > 0x840 ) -//#define CATCH_CONFIG_SFINAE // Not confirmed -#endif - #endif // __DMC__ //////////////////////////////////////////////////////////////////////////////// // GCC #ifdef __GNUC__ -#if __GNUC__ < 3 - -#if (__GNUC_MINOR__ >= 96 ) -//#define CATCH_CONFIG_SFINAE -#endif - -#elif __GNUC__ >= 3 - -// #define CATCH_CONFIG_SFINAE // Taking this out completely for now - -#endif // __GNUC__ < 3 - #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6 && defined(__GXX_EXPERIMENTAL_CXX0X__) ) - -#define CATCH_CONFIG_CPP11_NULLPTR +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR #endif #endif // __GNUC__ @@ -135,8 +140,13 @@ // Visual C++ #ifdef _MSC_VER -#if (_MSC_VER >= 1310 ) // (VC++ 7.0+) -//#define CATCH_CONFIG_SFINAE // Not confirmed +#if (_MSC_VER >= 1600) +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR +#endif + +#if (_MSC_VER >= 1900 ) // (VC++ 13 (VS2015)) +#define CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT +#define CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS #endif #endif // _MSC_VER @@ -147,21 +157,62 @@ ( defined __GNUC__ && __GNUC__ >= 3 ) || \ ( !defined __cplusplus && __STDC_VERSION__ >= 199901L || __cplusplus >= 201103L ) -#ifndef CATCH_CONFIG_NO_VARIADIC_MACROS -#define CATCH_CONFIG_VARIADIC_MACROS -#endif +#define CATCH_INTERNAL_CONFIG_VARIADIC_MACROS #endif //////////////////////////////////////////////////////////////////////////////// // C++ language feature support -// detect language version: -#if (__cplusplus == 201103L) -# define CATCH_CPP11 -# define CATCH_CPP11_OR_GREATER -#elif (__cplusplus >= 201103L) +// catch all support for C++11 +#if (__cplusplus >= 201103L) + # define CATCH_CPP11_OR_GREATER + +# if !defined(CATCH_INTERNAL_CONFIG_CPP11_NULLPTR) +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT +# define CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS +# define CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_IS_ENUM +# define CATCH_INTERNAL_CONFIG_CPP11_IS_ENUM +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_TUPLE +# define CATCH_INTERNAL_CONFIG_CPP11_TUPLE +# endif + +# ifndef CATCH_INTERNAL_CONFIG_VARIADIC_MACROS +# define CATCH_INTERNAL_CONFIG_VARIADIC_MACROS +# endif + +#endif // __cplusplus >= 201103L + +// Now set the actual defines based on the above + anything the user has configured +#if defined(CATCH_INTERNAL_CONFIG_CPP11_NULLPTR) && !defined(CATCH_CONFIG_CPP11_NO_NULLPTR) && !defined(CATCH_CONFIG_CPP11_NULLPTR) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_NULLPTR +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT) && !defined(CATCH_CONFIG_CPP11_NO_NOEXCEPT) && !defined(CATCH_CONFIG_CPP11_NOEXCEPT) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_NOEXCEPT +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS) && !defined(CATCH_CONFIG_CPP11_NO_GENERATED_METHODS) && !defined(CATCH_CONFIG_CPP11_GENERATED_METHODS) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_GENERATED_METHODS +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_IS_ENUM) && !defined(CATCH_CONFIG_CPP11_NO_IS_ENUM) && !defined(CATCH_CONFIG_CPP11_IS_ENUM) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_IS_ENUM +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_TUPLE) && !defined(CATCH_CONFIG_CPP11_NO_TUPLE) && !defined(CATCH_CONFIG_CPP11_TUPLE) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_TUPLE +#endif +#if defined(CATCH_INTERNAL_CONFIG_VARIADIC_MACROS) && !defined(CATCH_CONFIG_NO_VARIADIC_MACROS) && !defined(CATCH_CONFIG_VARIADIC_MACROS) +#define CATCH_CONFIG_VARIADIC_MACROS #endif // noexcept support: @@ -176,8 +227,16 @@ namespace Catch { class NonCopyable { - NonCopyable( NonCopyable const& ); - void operator = ( NonCopyable const& ); +#ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS + NonCopyable( NonCopyable const& ) = delete; + NonCopyable( NonCopyable && ) = delete; + NonCopyable& operator = ( NonCopyable const& ) = delete; + NonCopyable& operator = ( NonCopyable && ) = delete; +#else + NonCopyable( NonCopyable const& info ); + NonCopyable& operator = ( NonCopyable const& ); +#endif + protected: NonCopyable() {} virtual ~NonCopyable(); @@ -215,6 +274,7 @@ namespace Catch { void toLowerInPlace( std::string& s ); std::string toLower( std::string const& s ); std::string trim( std::string const& str ); + bool replaceInPlace( std::string& str, std::string const& replaceThis, std::string const& withThis ); struct pluralise { pluralise( std::size_t count, std::string const& label ); @@ -230,13 +290,14 @@ namespace Catch { SourceLineInfo(); SourceLineInfo( char const* _file, std::size_t _line ); SourceLineInfo( SourceLineInfo const& other ); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS SourceLineInfo( SourceLineInfo && ) = default; SourceLineInfo& operator = ( SourceLineInfo const& ) = default; SourceLineInfo& operator = ( SourceLineInfo && ) = default; # endif bool empty() const; bool operator == ( SourceLineInfo const& other ) const; + bool operator < ( SourceLineInfo const& other ) const; std::string file; std::size_t line; @@ -467,7 +528,7 @@ namespace Catch { struct ITestCaseRegistry { virtual ~ITestCaseRegistry(); virtual std::vector const& getAllTests() const = 0; - virtual void getFilteredTests( TestSpec const& testSpec, IConfig const& config, std::vector& matchingTestCases ) const = 0; + virtual void getFilteredTests( TestSpec const& testSpec, IConfig const& config, std::vector& matchingTestCases, bool negated = false ) const = 0; }; } @@ -603,7 +664,9 @@ namespace Catch { Exception = 0x100 | FailureBit, ThrewException = Exception | 1, - DidntThrowException = Exception | 2 + DidntThrowException = Exception | 2, + + FatalErrorCondition = 0x200 | FailureBit }; }; @@ -616,11 +679,11 @@ namespace Catch { // ResultDisposition::Flags enum struct ResultDisposition { enum Flags { - Normal = 0x00, + Normal = 0x01, - ContinueOnFailure = 0x01, // Failures fail test, but execution continues - FalseTest = 0x02, // Prefix expression with ! - SuppressFail = 0x04 // Failures are reported but do not fail the test + ContinueOnFailure = 0x02, // Failures fail test, but execution continues + FalseTest = 0x04, // Prefix expression with ! + SuppressFail = 0x08 // Failures are reported but do not fail the test }; }; inline ResultDisposition::Flags operator | ( ResultDisposition::Flags lhs, ResultDisposition::Flags rhs ) { @@ -668,7 +731,7 @@ namespace Catch { AssertionResult(); AssertionResult( AssertionInfo const& info, AssertionResultData const& data ); ~AssertionResult(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS AssertionResult( AssertionResult const& ) = default; AssertionResult( AssertionResult && ) = default; AssertionResult& operator = ( AssertionResult const& ) = default; @@ -724,8 +787,8 @@ namespace Catch { ResultDisposition::Flags resultDisposition ); template - ExpressionLhs operator->* ( T const& operand ); - ExpressionLhs operator->* ( bool value ); + ExpressionLhs operator <= ( T const& operand ); + ExpressionLhs operator <= ( bool value ); template ResultBuilder& operator << ( T const& value ) { @@ -948,40 +1011,6 @@ namespace Internal { // #included from: catch_tostring.h #define TWOBLUECUBES_CATCH_TOSTRING_H_INCLUDED -// #included from: catch_sfinae.hpp -#define TWOBLUECUBES_CATCH_SFINAE_HPP_INCLUDED - -// Try to detect if the current compiler supports SFINAE - -namespace Catch { - - struct TrueType { - static const bool value = true; - typedef void Enable; - char sizer[1]; - }; - struct FalseType { - static const bool value = false; - typedef void Disable; - char sizer[2]; - }; - -#ifdef CATCH_CONFIG_SFINAE - - template struct NotABooleanExpression; - - template struct If : NotABooleanExpression {}; - template<> struct If : TrueType {}; - template<> struct If : FalseType {}; - - template struct SizedIf; - template<> struct SizedIf : TrueType {}; - template<> struct SizedIf : FalseType {}; - -#endif // CATCH_CONFIG_SFINAE - -} // end namespace Catch - #include #include #include @@ -1034,35 +1063,59 @@ inline id performOptionalSelector( id obj, SEL sel ) { #endif +#ifdef CATCH_CONFIG_CPP11_TUPLE +#include +#endif + +#ifdef CATCH_CONFIG_CPP11_IS_ENUM +#include +#endif + namespace Catch { -namespace Detail { -// SFINAE is currently disabled by default for all compilers. -// If the non SFINAE version of IsStreamInsertable is ambiguous for you -// and your compiler supports SFINAE, try #defining CATCH_CONFIG_SFINAE -#ifdef CATCH_CONFIG_SFINAE +// Why we're here. +template +std::string toString( T const& value ); - template - class IsStreamInsertableHelper { - template struct TrueIfSizeable : TrueType {}; +// Built in overloads - template - static TrueIfSizeable dummy(T2*); - static FalseType dummy(...); +std::string toString( std::string const& value ); +std::string toString( std::wstring const& value ); +std::string toString( const char* const value ); +std::string toString( char* const value ); +std::string toString( const wchar_t* const value ); +std::string toString( wchar_t* const value ); +std::string toString( int value ); +std::string toString( unsigned long value ); +std::string toString( unsigned int value ); +std::string toString( const double value ); +std::string toString( const float value ); +std::string toString( bool value ); +std::string toString( char value ); +std::string toString( signed char value ); +std::string toString( unsigned char value ); - public: - typedef SizedIf type; - }; +#ifdef CATCH_CONFIG_CPP11_NULLPTR +std::string toString( std::nullptr_t ); +#endif - template - struct IsStreamInsertable : IsStreamInsertableHelper::type {}; +#ifdef __OBJC__ + std::string toString( NSString const * const& nsstring ); + std::string toString( NSString * CATCH_ARC_STRONG const& nsstring ); + std::string toString( NSObject* const& nsObject ); +#endif -#else +namespace Detail { + + extern std::string unprintableString; struct BorgType { template BorgType( T const& ); }; + struct TrueType { char sizer[1]; }; + struct FalseType { char sizer[2]; }; + TrueType& testStreamable( std::ostream& ); FalseType testStreamable( FalseType ); @@ -1075,12 +1128,38 @@ namespace Detail { enum { value = sizeof( testStreamable(s << t) ) == sizeof( TrueType ) }; }; -#endif +#if defined(CATCH_CONFIG_CPP11_IS_ENUM) + template::value + > + struct EnumStringMaker + { + static std::string convert( T const& ) { return unprintableString; } + }; + template + struct EnumStringMaker + { + static std::string convert( T const& v ) + { + return ::Catch::toString( + static_cast::type>(v) + ); + } + }; +#endif template struct StringMakerBase { +#if defined(CATCH_CONFIG_CPP11_IS_ENUM) + template + static std::string convert( T const& v ) + { + return EnumStringMaker::convert( v ); + } +#else template - static std::string convert( T const& ) { return "{?}"; } + static std::string convert( T const& ) { return unprintableString; } +#endif }; template<> @@ -1102,9 +1181,6 @@ namespace Detail { } // end namespace Detail -template -std::string toString( T const& value ); - template struct StringMaker : Detail::StringMakerBase::value> {}; @@ -1135,12 +1211,59 @@ namespace Detail { std::string rangeToString( InputIterator first, InputIterator last ); } +//template +//struct StringMaker > { +// static std::string convert( std::vector const& v ) { +// return Detail::rangeToString( v.begin(), v.end() ); +// } +//}; + template -struct StringMaker > { - static std::string convert( std::vector const& v ) { - return Detail::rangeToString( v.begin(), v.end() ); +std::string toString( std::vector const& v ) { + return Detail::rangeToString( v.begin(), v.end() ); +} + +#ifdef CATCH_CONFIG_CPP11_TUPLE + +// toString for tuples +namespace TupleDetail { + template< + typename Tuple, + std::size_t N = 0, + bool = (N < std::tuple_size::value) + > + struct ElementPrinter { + static void print( const Tuple& tuple, std::ostream& os ) + { + os << ( N ? ", " : " " ) + << Catch::toString(std::get(tuple)); + ElementPrinter::print(tuple,os); + } + }; + + template< + typename Tuple, + std::size_t N + > + struct ElementPrinter { + static void print( const Tuple&, std::ostream& ) {} + }; + +} + +template +struct StringMaker> { + + static std::string convert( const std::tuple& tuple ) + { + std::ostringstream os; + os << '{'; + TupleDetail::ElementPrinter>::print( tuple, os ); + os << " }"; + return os.str(); } }; +#endif // CATCH_CONFIG_CPP11_TUPLE namespace Detail { template @@ -1161,44 +1284,15 @@ std::string toString( T const& value ) { return StringMaker::convert( value ); } -// Built in overloads - -std::string toString( std::string const& value ); -std::string toString( std::wstring const& value ); -std::string toString( const char* const value ); -std::string toString( char* const value ); -std::string toString( const wchar_t* const value ); -std::string toString( wchar_t* const value ); -std::string toString( int value ); -std::string toString( unsigned long value ); -std::string toString( unsigned int value ); -std::string toString( const double value ); -std::string toString( const float value ); -std::string toString( bool value ); -std::string toString( char value ); -std::string toString( signed char value ); -std::string toString( unsigned char value ); - -#ifdef CATCH_CONFIG_CPP11_NULLPTR -std::string toString( std::nullptr_t ); -#endif - -#ifdef __OBJC__ - std::string toString( NSString const * const& nsstring ); - std::string toString( NSString * CATCH_ARC_STRONG const& nsstring ); - std::string toString( NSObject* const& nsObject ); -#endif - namespace Detail { template std::string rangeToString( InputIterator first, InputIterator last ) { std::ostringstream oss; oss << "{ "; if( first != last ) { - oss << toString( *first ); - for( ++first ; first != last ; ++first ) { - oss << ", " << toString( *first ); - } + oss << Catch::toString( *first ); + for( ++first ; first != last ; ++first ) + oss << ", " << Catch::toString( *first ); } oss << " }"; return oss.str(); @@ -1214,13 +1308,13 @@ namespace Catch { template class ExpressionLhs { ExpressionLhs& operator = ( ExpressionLhs const& ); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS ExpressionLhs& operator = ( ExpressionLhs && ) = delete; # endif public: ExpressionLhs( ResultBuilder& rb, T lhs ) : m_rb( rb ), m_lhs( lhs ) {} -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS ExpressionLhs( ExpressionLhs const& ) = default; ExpressionLhs( ExpressionLhs && ) = default; # endif @@ -1301,11 +1395,11 @@ class ExpressionLhs { namespace Catch { template - inline ExpressionLhs ResultBuilder::operator->* ( T const& operand ) { + inline ExpressionLhs ResultBuilder::operator <= ( T const& operand ) { return ExpressionLhs( *this, operand ); } - inline ExpressionLhs ResultBuilder::operator->* ( bool value ) { + inline ExpressionLhs ResultBuilder::operator <= ( bool value ) { return ExpressionLhs( *this, value ); } @@ -1395,6 +1489,8 @@ namespace Catch { virtual std::string getCurrentTestName() const = 0; virtual const AssertionResult* getLastResult() const = 0; + + virtual void handleFatalErrorCondition( std::string const& message ) = 0; }; IResultCapture& getResultCapture(); @@ -1475,7 +1571,7 @@ namespace Catch { do { \ Catch::ResultBuilder __catchResult( macroName, CATCH_INTERNAL_LINEINFO, #expr, resultDisposition ); \ try { \ - ( __catchResult->*expr ).endExpression(); \ + ( __catchResult <= expr ).endExpression(); \ } \ catch( ... ) { \ __catchResult.useActiveException( Catch::ResultDisposition::Normal ); \ @@ -1575,7 +1671,7 @@ namespace Catch { std::string matcherAsString = ::Catch::Matchers::matcher.toString(); \ __catchResult \ .setLhs( Catch::toString( arg ) ) \ - .setRhs( matcherAsString == "{?}" ? #matcher : matcherAsString ) \ + .setRhs( matcherAsString == Catch::Detail::unprintableString ? #matcher : matcherAsString ) \ .setOp( "matches" ) \ .setResultType( ::Catch::Matchers::matcher.match( arg ) ); \ __catchResult.captureExpression(); \ @@ -1636,6 +1732,9 @@ namespace Catch { bool allPassed() const { return failed == 0 && failedButOk == 0; } + bool allOk() const { + return failed == 0; + } std::size_t passed; std::size_t failed; @@ -1688,7 +1787,7 @@ namespace Catch { public: Timer() : m_ticks( 0 ) {} void start(); - unsigned int getElapsedNanoseconds() const; + unsigned int getElapsedMicroseconds() const; unsigned int getElapsedMilliseconds() const; double getElapsedSeconds() const; @@ -1702,7 +1801,7 @@ namespace Catch { namespace Catch { - class Section { + class Section : NonCopyable { public: Section( SectionInfo const& info ); ~Section(); @@ -1711,15 +1810,6 @@ namespace Catch { operator bool() const; private: -#ifdef CATCH_CPP11_OR_GREATER - Section( Section const& ) = delete; - Section( Section && ) = delete; - Section& operator = ( Section const& ) = delete; - Section& operator = ( Section && ) = delete; -#else - Section( Section const& info ); - Section& operator = ( Section const& ); -#endif SectionInfo m_info; std::string m_name; @@ -2694,7 +2784,7 @@ return @ desc; \ #endif -#ifdef CATCH_CONFIG_RUNNER +#ifdef CATCH_IMPL // #included from: internal/catch_impl.hpp #define TWOBLUECUBES_CATCH_IMPL_HPP_INCLUDED @@ -2706,7 +2796,7 @@ return @ desc; \ #pragma clang diagnostic ignored "-Wweak-vtables" #endif -// #included from: catch_runner.hpp +// #included from: ../catch_runner.hpp #define TWOBLUECUBES_CATCH_RUNNER_HPP_INCLUDED // #included from: internal/catch_commandline.hpp @@ -2962,6 +3052,11 @@ namespace Catch { Always, Never }; }; + struct RunTests { enum InWhatOrder { + InDeclarationOrder, + InLexicographicalOrder, + InRandomOrder + }; }; class TestSpec; @@ -2979,6 +3074,9 @@ namespace Catch { virtual bool showInvisibles() const = 0; virtual ShowDurations::OrNot showDurations() const = 0; virtual TestSpec const& testSpec() const = 0; + virtual RunTests::InWhatOrder runOrder() const = 0; + virtual unsigned int rngSeed() const = 0; + virtual bool forceColour() const = 0; }; } @@ -3004,12 +3102,16 @@ namespace Catch { private: bool isOwned; }; + + std::ostream& cout(); + std::ostream& cerr(); } #include #include #include #include +#include #ifndef CATCH_CONFIG_CONSOLE_WIDTH #define CATCH_CONFIG_CONSOLE_WIDTH 80 @@ -3029,10 +3131,13 @@ namespace Catch { noThrow( false ), showHelp( false ), showInvisibles( false ), + forceColour( false ), abortAfter( -1 ), + rngSeed( 0 ), verbosity( Verbosity::Normal ), warnings( WarnAbout::Nothing ), - showDurations( ShowDurations::DefaultForReporter ) + showDurations( ShowDurations::DefaultForReporter ), + runOrder( RunTests::InDeclarationOrder ) {} bool listTests; @@ -3045,12 +3150,15 @@ namespace Catch { bool noThrow; bool showHelp; bool showInvisibles; + bool forceColour; int abortAfter; + unsigned int rngSeed; Verbosity::Level verbosity; WarnAbout::What warnings; ShowDurations::OrNot showDurations; + RunTests::InWhatOrder runOrder; std::string reporterName; std::string outputFilename; @@ -3068,12 +3176,12 @@ namespace Catch { public: Config() - : m_os( std::cout.rdbuf() ) + : m_os( Catch::cout().rdbuf() ) {} Config( ConfigData const& data ) : m_data( data ), - m_os( std::cout.rdbuf() ) + m_os( Catch::cout().rdbuf() ) { if( !data.testsOrTags.empty() ) { TestSpecParser parser( ITagAliasRegistry::get() ); @@ -3084,7 +3192,7 @@ namespace Catch { } virtual ~Config() { - m_os.rdbuf( std::cout.rdbuf() ); + m_os.rdbuf( Catch::cout().rdbuf() ); m_stream.release(); } @@ -3106,7 +3214,7 @@ namespace Catch { bool shouldDebugBreak() const { return m_data.shouldDebugBreak; } void setStreamBuf( std::streambuf* buf ) { - m_os.rdbuf( buf ? buf : std::cout.rdbuf() ); + m_os.rdbuf( buf ? buf : Catch::cout().rdbuf() ); } void useStream( std::string const& streamName ) { @@ -3132,6 +3240,9 @@ namespace Catch { virtual bool includeSuccessfulResults() const { return m_data.showSuccessfulTests; } virtual bool warnAboutMissingAssertions() const { return m_data.warnings & WarnAbout::NoAssertions; } virtual ShowDurations::OrNot showDurations() const { return m_data.showDurations; } + virtual RunTests::InWhatOrder runOrder() const { return m_data.runOrder; } + virtual unsigned int rngSeed() const { return m_data.rngSeed; } + virtual bool forceColour() const { return m_data.forceColour; } private: ConfigData m_data; @@ -3393,7 +3504,7 @@ namespace Clara { template struct IArgFunction { virtual ~IArgFunction() {} -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS IArgFunction() = default; IArgFunction( IArgFunction const& ) = default; # endif @@ -3760,7 +3871,7 @@ namespace Clara { m_throwOnUnrecognisedTokens( other.m_throwOnUnrecognisedTokens ) { if( other.m_floatingArg.get() ) - m_floatingArg = ArgAutoPtr( new Arg( *other.m_floatingArg ) ); + m_floatingArg.reset( new Arg( *other.m_floatingArg ) ); } CommandLine& setThrowOnUnrecognisedTokens( bool shouldThrow = true ) { @@ -3788,7 +3899,7 @@ namespace Clara { ArgBuilder operator[]( UnpositionalTag ) { if( m_floatingArg.get() ) throw std::logic_error( "Only one unpositional argument can be added" ); - m_floatingArg = ArgAutoPtr( new Arg() ); + m_floatingArg.reset( new Arg() ); ArgBuilder builder( m_floatingArg.get() ); return builder; } @@ -3930,7 +4041,7 @@ namespace Clara { if( it == itEnd ) { if( token.type == Parser::Token::Positional || !m_throwOnUnrecognisedTokens ) unusedTokens.push_back( token ); - else if( m_throwOnUnrecognisedTokens ) + else if( errors.empty() && m_throwOnUnrecognisedTokens ) errors.push_back( "unrecognised option: " + token.data ); } } @@ -4028,7 +4139,28 @@ namespace Catch { config.warnings = static_cast( config.warnings | WarnAbout::NoAssertions ); else throw std::runtime_error( "Unrecognised warning: '" + _warning + "'" ); - + } + inline void setOrder( ConfigData& config, std::string const& order ) { + if( startsWith( "declared", order ) ) + config.runOrder = RunTests::InDeclarationOrder; + else if( startsWith( "lexical", order ) ) + config.runOrder = RunTests::InLexicographicalOrder; + else if( startsWith( "random", order ) ) + config.runOrder = RunTests::InRandomOrder; + else + throw std::runtime_error( "Unrecognised ordering: '" + order + "'" ); + } + inline void setRngSeed( ConfigData& config, std::string const& seed ) { + if( seed == "time" ) { + config.rngSeed = static_cast( std::time(0) ); + } + else { + std::stringstream ss; + ss << seed; + ss >> config.rngSeed; + if( ss.fail() ) + throw std::runtime_error( "Argment to --rng-seed should be the word 'time' or a number" ); + } } inline void setVerbosity( ConfigData& config, int level ) { // !TBD: accept strings? @@ -4140,6 +4272,18 @@ namespace Catch { .describe( "list all reporters" ) .bind( &ConfigData::listReporters ); + cli["--order"] + .describe( "test case order (defaults to decl)" ) + .bind( &setOrder, "decl|lex|rand" ); + + cli["--rng-seed"] + .describe( "set a specific seed for random numbers" ) + .bind( &setRngSeed, "'time'|number" ); + + cli["--force-colour"] + .describe( "force colourised output" ) + .bind( &ConfigData::forceColour ); + return cli; } @@ -4313,10 +4457,6 @@ namespace Catch { namespace Catch { - namespace Detail { - struct IColourImpl; - } - struct Colour { enum Code { None = 0, @@ -4362,7 +4502,6 @@ namespace Catch { static void use( Code _colourCode ); private: - static Detail::IColourImpl* impl(); bool m_moved; }; @@ -4456,7 +4595,7 @@ namespace Catch } virtual ~AssertionStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS AssertionStats( AssertionStats const& ) = default; AssertionStats( AssertionStats && ) = default; AssertionStats& operator = ( AssertionStats const& ) = default; @@ -4479,7 +4618,7 @@ namespace Catch missingAssertions( _missingAssertions ) {} virtual ~SectionStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS SectionStats( SectionStats const& ) = default; SectionStats( SectionStats && ) = default; SectionStats& operator = ( SectionStats const& ) = default; @@ -4506,7 +4645,7 @@ namespace Catch {} virtual ~TestCaseStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS TestCaseStats( TestCaseStats const& ) = default; TestCaseStats( TestCaseStats && ) = default; TestCaseStats& operator = ( TestCaseStats const& ) = default; @@ -4534,7 +4673,7 @@ namespace Catch {} virtual ~TestGroupStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS TestGroupStats( TestGroupStats const& ) = default; TestGroupStats( TestGroupStats && ) = default; TestGroupStats& operator = ( TestGroupStats const& ) = default; @@ -4556,7 +4695,7 @@ namespace Catch {} virtual ~TestRunStats(); -# ifndef CATCH_CPP11_OR_GREATER +# ifndef CATCH_CONFIG_CPP11_GENERATED_METHODS TestRunStats( TestRunStats const& _other ) : runInfo( _other.runInfo ), totals( _other.totals ), @@ -4592,11 +4731,14 @@ namespace Catch virtual void assertionStarting( AssertionInfo const& assertionInfo ) = 0; + // The return value indicates if the messages buffer should be cleared: virtual bool assertionEnded( AssertionStats const& assertionStats ) = 0; virtual void sectionEnded( SectionStats const& sectionStats ) = 0; virtual void testCaseEnded( TestCaseStats const& testCaseStats ) = 0; virtual void testGroupEnded( TestGroupStats const& testGroupStats ) = 0; virtual void testRunEnded( TestRunStats const& testRunStats ) = 0; + + virtual void skipTest( TestCaseInfo const& testInfo ) = 0; }; struct IReporterFactory { @@ -4624,9 +4766,9 @@ namespace Catch { TestSpec testSpec = config.testSpec(); if( config.testSpec().hasFilters() ) - std::cout << "Matching test cases:\n"; + Catch::cout() << "Matching test cases:\n"; else { - std::cout << "All available test cases:\n"; + Catch::cout() << "All available test cases:\n"; testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec(); } @@ -4647,15 +4789,15 @@ namespace Catch { : Colour::None; Colour colourGuard( colour ); - std::cout << Text( testCaseInfo.name, nameAttr ) << std::endl; + Catch::cout() << Text( testCaseInfo.name, nameAttr ) << std::endl; if( !testCaseInfo.tags.empty() ) - std::cout << Text( testCaseInfo.tagsAsString, tagsAttr ) << std::endl; + Catch::cout() << Text( testCaseInfo.tagsAsString, tagsAttr ) << std::endl; } if( !config.testSpec().hasFilters() ) - std::cout << pluralise( matchedTests, "test case" ) << "\n" << std::endl; + Catch::cout() << pluralise( matchedTests, "test case" ) << "\n" << std::endl; else - std::cout << pluralise( matchedTests, "matching test case" ) << "\n" << std::endl; + Catch::cout() << pluralise( matchedTests, "matching test case" ) << "\n" << std::endl; return matchedTests; } @@ -4671,7 +4813,7 @@ namespace Catch { ++it ) { matchedTests++; TestCaseInfo const& testCaseInfo = it->getTestCaseInfo(); - std::cout << testCaseInfo.name << std::endl; + Catch::cout() << testCaseInfo.name << std::endl; } return matchedTests; } @@ -4697,9 +4839,9 @@ namespace Catch { inline std::size_t listTags( Config const& config ) { TestSpec testSpec = config.testSpec(); if( config.testSpec().hasFilters() ) - std::cout << "Tags for matching test cases:\n"; + Catch::cout() << "Tags for matching test cases:\n"; else { - std::cout << "All available tags:\n"; + Catch::cout() << "All available tags:\n"; testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec(); } @@ -4733,14 +4875,14 @@ namespace Catch { .setInitialIndent( 0 ) .setIndent( oss.str().size() ) .setWidth( CATCH_CONFIG_CONSOLE_WIDTH-10 ) ); - std::cout << oss.str() << wrapper << "\n"; + Catch::cout() << oss.str() << wrapper << "\n"; } - std::cout << pluralise( tagCounts.size(), "tag" ) << "\n" << std::endl; + Catch::cout() << pluralise( tagCounts.size(), "tag" ) << "\n" << std::endl; return tagCounts.size(); } inline std::size_t listReporters( Config const& /*config*/ ) { - std::cout << "Available reports:\n"; + Catch::cout() << "Available reporters:\n"; IReporterRegistry::FactoryMap const& factories = getRegistryHub().getReporterRegistry().getFactories(); IReporterRegistry::FactoryMap::const_iterator itBegin = factories.begin(), itEnd = factories.end(), it; std::size_t maxNameLen = 0; @@ -4752,13 +4894,13 @@ namespace Catch { .setInitialIndent( 0 ) .setIndent( 7+maxNameLen ) .setWidth( CATCH_CONFIG_CONSOLE_WIDTH - maxNameLen-8 ) ); - std::cout << " " + Catch::cout() << " " << it->first << ":" << std::string( maxNameLen - it->first.size() + 2, ' ' ) << wrapper << "\n"; } - std::cout << std::endl; + Catch::cout() << std::endl; return factories.size(); } @@ -4808,32 +4950,15 @@ namespace SectionTracking { RunState runState() const { return m_runState; } - TrackedSection* findChild( std::string const& childName ) { - TrackedSections::iterator it = m_children.find( childName ); - return it != m_children.end() - ? &it->second - : NULL; - } - TrackedSection* acquireChild( std::string const& childName ) { - if( TrackedSection* child = findChild( childName ) ) - return child; - m_children.insert( std::make_pair( childName, TrackedSection( childName, this ) ) ); - return findChild( childName ); - } + TrackedSection* findChild( std::string const& childName ); + TrackedSection* acquireChild( std::string const& childName ); + void enter() { if( m_runState == NotStarted ) m_runState = Executing; } - void leave() { - for( TrackedSections::const_iterator it = m_children.begin(), itEnd = m_children.end(); - it != itEnd; - ++it ) - if( it->second.runState() != Completed ) { - m_runState = ExecutingChildren; - return; - } - m_runState = Completed; - } + void leave(); + TrackedSection* getParent() { return m_parent; } @@ -4846,9 +4971,31 @@ namespace SectionTracking { RunState m_runState; TrackedSections m_children; TrackedSection* m_parent; - }; + inline TrackedSection* TrackedSection::findChild( std::string const& childName ) { + TrackedSections::iterator it = m_children.find( childName ); + return it != m_children.end() + ? &it->second + : NULL; + } + inline TrackedSection* TrackedSection::acquireChild( std::string const& childName ) { + if( TrackedSection* child = findChild( childName ) ) + return child; + m_children.insert( std::make_pair( childName, TrackedSection( childName, this ) ) ); + return findChild( childName ); + } + inline void TrackedSection::leave() { + for( TrackedSections::const_iterator it = m_children.begin(), itEnd = m_children.end(); + it != itEnd; + ++it ) + if( it->second.runState() != Completed ) { + m_runState = ExecutingChildren; + return; + } + m_runState = Completed; + } + class TestCaseTracker { public: TestCaseTracker( std::string const& testCaseName ) @@ -4915,6 +5062,81 @@ using SectionTracking::TestCaseTracker; } // namespace Catch +// #included from: catch_fatal_condition.hpp +#define TWOBLUECUBES_CATCH_FATAL_CONDITION_H_INCLUDED + +namespace Catch { + + // Report the error condition then exit the process + inline void fatal( std::string const& message, int exitCode ) { + IContext& context = Catch::getCurrentContext(); + IResultCapture* resultCapture = context.getResultCapture(); + resultCapture->handleFatalErrorCondition( message ); + + if( Catch::alwaysTrue() ) // avoids "no return" warnings + exit( exitCode ); + } + +} // namespace Catch + +#if defined ( CATCH_PLATFORM_WINDOWS ) ///////////////////////////////////////// + +namespace Catch { + + struct FatalConditionHandler { + void reset() {} + }; + +} // namespace Catch + +#else // Not Windows - assumed to be POSIX compatible ////////////////////////// + +#include + +namespace Catch { + + struct SignalDefs { int id; const char* name; }; + extern SignalDefs signalDefs[]; + SignalDefs signalDefs[] = { + { SIGINT, "SIGINT - Terminal interrupt signal" }, + { SIGILL, "SIGILL - Illegal instruction signal" }, + { SIGFPE, "SIGFPE - Floating point error signal" }, + { SIGSEGV, "SIGSEGV - Segmentation violation signal" }, + { SIGTERM, "SIGTERM - Termination request signal" }, + { SIGABRT, "SIGABRT - Abort (abnormal termination) signal" } + }; + + struct FatalConditionHandler { + + static void handleSignal( int sig ) { + for( std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i ) + if( sig == signalDefs[i].id ) + fatal( signalDefs[i].name, -sig ); + fatal( "", -sig ); + } + + FatalConditionHandler() : m_isSet( true ) { + for( std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i ) + signal( signalDefs[i].id, handleSignal ); + } + ~FatalConditionHandler() { + reset(); + } + void reset() { + if( m_isSet ) { + for( std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i ) + signal( signalDefs[i].id, SIG_DFL ); + m_isSet = false; + } + } + + bool m_isSet; + }; + +} // namespace Catch + +#endif // not Windows + #include #include @@ -5102,6 +5324,37 @@ namespace Catch { return &m_lastResult; } + virtual void handleFatalErrorCondition( std::string const& message ) { + ResultBuilder resultBuilder = makeUnexpectedResultBuilder(); + resultBuilder.setResultType( ResultWas::FatalErrorCondition ); + resultBuilder << message; + resultBuilder.captureExpression(); + + handleUnfinishedSections(); + + // Recreate section for test case (as we will lose the one that was in scope) + TestCaseInfo const& testCaseInfo = m_activeTestCase->getTestCaseInfo(); + SectionInfo testCaseSection( testCaseInfo.lineInfo, testCaseInfo.name, testCaseInfo.description ); + + Counts assertions; + assertions.failed = 1; + SectionStats testCaseSectionStats( testCaseSection, assertions, 0, false ); + m_reporter->sectionEnded( testCaseSectionStats ); + + TestCaseInfo testInfo = m_activeTestCase->getTestCaseInfo(); + + Totals deltaTotals; + deltaTotals.testCases.failed = 1; + m_reporter->testCaseEnded( TestCaseStats( testInfo, + deltaTotals, + "", + "", + false ) ); + m_totals.testCases.failed++; + testGroupEnded( "", m_totals, 1, 1 ); + m_reporter->testRunEnded( TestRunStats( m_runInfo, m_totals, false ) ); + } + public: // !TBD We need to do this another way! bool aborting() const { @@ -5123,12 +5376,12 @@ namespace Catch { Timer timer; timer.start(); if( m_reporter->getPreferences().shouldRedirectStdOut ) { - StreamRedirect coutRedir( std::cout, redirectedCout ); - StreamRedirect cerrRedir( std::cerr, redirectedCerr ); - m_activeTestCase->invoke(); + StreamRedirect coutRedir( Catch::cout(), redirectedCout ); + StreamRedirect cerrRedir( Catch::cerr(), redirectedCerr ); + invokeActiveTestCase(); } else { - m_activeTestCase->invoke(); + invokeActiveTestCase(); } duration = timer.getElapsedSeconds(); } @@ -5136,20 +5389,9 @@ namespace Catch { // This just means the test was aborted due to failure } catch(...) { - ResultBuilder exResult( m_lastAssertionInfo.macroName.c_str(), - m_lastAssertionInfo.lineInfo, - m_lastAssertionInfo.capturedExpression.c_str(), - m_lastAssertionInfo.resultDisposition ); - exResult.useActiveException(); + makeUnexpectedResultBuilder().useActiveException(); } - // If sections ended prematurely due to an exception we stored their - // infos here so we can tear them down outside the unwind process. - for( std::vector::const_reverse_iterator it = m_unfinishedSections.rbegin(), - itEnd = m_unfinishedSections.rend(); - it != itEnd; - ++it ) - sectionEnded( it->info, it->prevAssertions, it->durationInSeconds ); - m_unfinishedSections.clear(); + handleUnfinishedSections(); m_messages.clear(); Counts assertions = m_totals.assertions - prevAssertions; @@ -5165,7 +5407,32 @@ namespace Catch { m_reporter->sectionEnded( testCaseSectionStats ); } + void invokeActiveTestCase() { + FatalConditionHandler fatalConditionHandler; // Handle signals + m_activeTestCase->invoke(); + fatalConditionHandler.reset(); + } + private: + + ResultBuilder makeUnexpectedResultBuilder() const { + return ResultBuilder( m_lastAssertionInfo.macroName.c_str(), + m_lastAssertionInfo.lineInfo, + m_lastAssertionInfo.capturedExpression.c_str(), + m_lastAssertionInfo.resultDisposition ); + } + + void handleUnfinishedSections() { + // If sections ended prematurely due to an exception we stored their + // infos here so we can tear them down outside the unwind process. + for( std::vector::const_reverse_iterator it = m_unfinishedSections.rbegin(), + itEnd = m_unfinishedSections.rend(); + it != itEnd; + ++it ) + sectionEnded( it->info, it->prevAssertions, it->durationInSeconds ); + m_unfinishedSections.clear(); + } + struct UnfinishedSections { UnfinishedSections( SectionInfo const& _info, Counts const& _prevAssertions, double _durationInSeconds ) : info( _info ), prevAssertions( _prevAssertions ), durationInSeconds( _durationInSeconds ) @@ -5211,18 +5478,19 @@ namespace Catch { struct Version { Version( unsigned int _majorVersion, unsigned int _minorVersion, - unsigned int _buildNumber, - char const* const _branchName ) - : majorVersion( _majorVersion ), - minorVersion( _minorVersion ), - buildNumber( _buildNumber ), - branchName( _branchName ) - {} + unsigned int _patchNumber, + std::string const& _branchName, + unsigned int _buildNumber ); unsigned int const majorVersion; unsigned int const minorVersion; + unsigned int const patchNumber; + + // buildNumber is only used if branchName is not null + std::string const branchName; unsigned int const buildNumber; - char const* const branchName; + + friend std::ostream& operator << ( std::ostream& os, Version const& version ); private: void operator=( Version const& ); @@ -5253,7 +5521,7 @@ namespace Catch { Totals totals; - context.testGroupStarting( "", 1, 1 ); // deprecated? + context.testGroupStarting( "all tests", 1, 1 ); // deprecated? TestSpec testSpec = m_config->testSpec(); if( !testSpec.hasFilters() ) @@ -5276,7 +5544,15 @@ namespace Catch { m_testsAlreadyRun.insert( *it ); } } - context.testGroupEnded( "", totals, 1, 1 ); + std::vector skippedTestCases; + getRegistryHub().getTestCaseRegistry().getFilteredTests( testSpec, *m_config, skippedTestCases, true ); + + for( std::vector::const_iterator it = skippedTestCases.begin(), itEnd = skippedTestCases.end(); + it != itEnd; + ++it ) + m_reporter->skipTest( *it ); + + context.testGroupEnded( "all tests", totals, 1, 1 ); return totals; } @@ -5313,7 +5589,7 @@ namespace Catch { std::set m_testsAlreadyRun; }; - class Session { + class Session : NonCopyable { static bool alreadyInstantiated; public: @@ -5324,7 +5600,7 @@ namespace Catch { : m_cli( makeCommandLineParser() ) { if( alreadyInstantiated ) { std::string msg = "Only one instance of Catch::Session can ever be used"; - std::cerr << msg << std::endl; + Catch::cerr() << msg << std::endl; throw std::logic_error( msg ); } alreadyInstantiated = true; @@ -5334,15 +5610,10 @@ namespace Catch { } void showHelp( std::string const& processName ) { - std::cout << "\nCatch v" << libraryVersion.majorVersion << "." - << libraryVersion.minorVersion << " build " - << libraryVersion.buildNumber; - if( libraryVersion.branchName != std::string( "master" ) ) - std::cout << " (" << libraryVersion.branchName << " branch)"; - std::cout << "\n"; + Catch::cout() << "\nCatch v" << libraryVersion << "\n"; - m_cli.usage( std::cout, processName ); - std::cout << "For more detail usage please see the project docs\n" << std::endl; + m_cli.usage( Catch::cout(), processName ); + Catch::cout() << "For more detail usage please see the project docs\n" << std::endl; } int applyCommandLine( int argc, char* const argv[], OnUnusedOptions::DoWhat unusedOptionBehaviour = OnUnusedOptions::Fail ) { @@ -5356,11 +5627,12 @@ namespace Catch { catch( std::exception& ex ) { { Colour colourGuard( Colour::Red ); - std::cerr << "\nError(s) in input:\n" - << Text( ex.what(), TextAttributes().setIndent(2) ) - << "\n\n"; + Catch::cerr() + << "\nError(s) in input:\n" + << Text( ex.what(), TextAttributes().setIndent(2) ) + << "\n\n"; } - m_cli.usage( std::cout, m_configData.processName ); + m_cli.usage( Catch::cout(), m_configData.processName ); return (std::numeric_limits::max)(); } return 0; @@ -5386,6 +5658,9 @@ namespace Catch { try { config(); // Force config to be constructed + + std::srand( m_configData.rngSeed ); + Runner runner( m_config ); // Handle list request @@ -5395,7 +5670,7 @@ namespace Catch { return static_cast( runner.runTests().assertions.failed ); } catch( std::exception& ex ) { - std::cerr << ex.what() << std::endl; + Catch::cerr() << ex.what() << std::endl; return (std::numeric_limits::max)(); } } @@ -5436,10 +5711,18 @@ namespace Catch { #include #include #include +#include namespace Catch { class TestRegistry : public ITestCaseRegistry { + struct LexSort { + bool operator() (TestCase i,TestCase j) const { return (i& matchingTestCases ) const { + virtual void getFilteredTests( TestSpec const& testSpec, IConfig const& config, std::vector& matchingTestCases, bool negated = false ) const { + for( std::vector::const_iterator it = m_functionsInOrder.begin(), itEnd = m_functionsInOrder.end(); it != itEnd; ++it ) { - if( testSpec.matches( *it ) && ( config.allowThrows() || !it->throws() ) ) + bool includeTest = testSpec.matches( *it ) && ( config.allowThrows() || !it->throws() ); + if( includeTest != negated ) matchingTestCases.push_back( *it ); } + sortTests( config, matchingTestCases ); } private: + static void sortTests( IConfig const& config, std::vector& matchingTestCases ) { + + switch( config.runOrder() ) { + case RunTests::InLexicographicalOrder: + std::sort( matchingTestCases.begin(), matchingTestCases.end(), LexSort() ); + break; + case RunTests::InRandomOrder: + { + RandomNumberGenerator rng; + std::random_shuffle( matchingTestCases.begin(), matchingTestCases.end(), rng ); + } + break; + case RunTests::InDeclarationOrder: + // already in declaration order + break; + } + } std::set m_functions; std::vector m_functionsInOrder; std::vector m_nonHiddenFunctions; @@ -5613,7 +5916,7 @@ namespace Catch { throw; } @catch (NSException *exception) { - return toString( [exception description] ); + return Catch::toString( [exception description] ); } #else throw; @@ -5760,6 +6063,7 @@ namespace Catch { #include #include +#include namespace Catch { @@ -5823,6 +6127,15 @@ namespace Catch { isOwned = false; } } + +#ifndef CATCH_CONFIG_NOSTDOUT // If you #define this you must implement this functions + std::ostream& cout() { + return std::cout; + } + std::ostream& cerr() { + return std::cerr; + } +#endif } namespace Catch { @@ -5872,7 +6185,7 @@ namespace Catch { std::string testName = getResultCapture()->getCurrentTestName(); std::map::const_iterator it = - m_generatorsByTestName.find( testName ); + m_generatorsByTestName.find( testName ); return it != m_generatorsByTestName.end() ? it->second : NULL; @@ -5908,8 +6221,8 @@ namespace Catch { } Stream createStream( std::string const& streamName ) { - if( streamName == "stdout" ) return Stream( std::cout.rdbuf(), false ); - if( streamName == "stderr" ) return Stream( std::cerr.rdbuf(), false ); + if( streamName == "stdout" ) return Stream( Catch::cout().rdbuf(), false ); + if( streamName == "stderr" ) return Stream( Catch::cerr().rdbuf(), false ); if( streamName == "debug" ) return Stream( new StreamBufImpl, true ); throw std::domain_error( "Unknown stream: " + streamName ); @@ -5924,14 +6237,35 @@ namespace Catch { // #included from: catch_console_colour_impl.hpp #define TWOBLUECUBES_CATCH_CONSOLE_COLOUR_IMPL_HPP_INCLUDED -namespace Catch { namespace Detail { - struct IColourImpl { - virtual ~IColourImpl() {} - virtual void use( Colour::Code _colourCode ) = 0; - }; -}} +namespace Catch { + namespace { -#if defined ( CATCH_PLATFORM_WINDOWS ) ///////////////////////////////////////// + struct IColourImpl { + virtual ~IColourImpl() {} + virtual void use( Colour::Code _colourCode ) = 0; + }; + + struct NoColourImpl : IColourImpl { + void use( Colour::Code ) {} + + static IColourImpl* instance() { + static NoColourImpl s_instance; + return &s_instance; + } + }; + + } // anon namespace +} // namespace Catch + +#if !defined( CATCH_CONFIG_COLOUR_NONE ) && !defined( CATCH_CONFIG_COLOUR_WINDOWS ) && !defined( CATCH_CONFIG_COLOUR_ANSI ) +# ifdef CATCH_PLATFORM_WINDOWS +# define CATCH_CONFIG_COLOUR_WINDOWS +# else +# define CATCH_CONFIG_COLOUR_ANSI +# endif +#endif + +#if defined ( CATCH_CONFIG_COLOUR_WINDOWS ) ///////////////////////////////////////// #ifndef NOMINMAX #define NOMINMAX @@ -5946,7 +6280,7 @@ namespace Catch { namespace Detail { namespace Catch { namespace { - class Win32ColourImpl : public Detail::IColourImpl { + class Win32ColourImpl : public IColourImpl { public: Win32ColourImpl() : stdoutHandle( GetStdHandle(STD_OUTPUT_HANDLE) ) { @@ -5983,11 +6317,7 @@ namespace { WORD originalAttributes; }; - inline bool shouldUseColourForPlatform() { - return true; - } - - static Detail::IColourImpl* platformColourInstance() { + IColourImpl* platformColourInstance() { static Win32ColourImpl s_instance; return &s_instance; } @@ -5995,7 +6325,7 @@ namespace { } // end anon namespace } // end namespace Catch -#else // Not Windows - assumed to be POSIX compatible ////////////////////////// +#elif defined( CATCH_CONFIG_COLOUR_ANSI ) ////////////////////////////////////// #include @@ -6006,7 +6336,7 @@ namespace { // Thanks to Adam Strzelecki for original contribution // (http://github.com/nanoant) // https://github.com/philsquared/Catch/pull/131 - class PosixColourImpl : public Detail::IColourImpl { + class PosixColourImpl : public IColourImpl { public: virtual void use( Colour::Code _colourCode ) { switch( _colourCode ) { @@ -6027,53 +6357,48 @@ namespace { case Colour::Bright: throw std::logic_error( "not a colour" ); } } + static IColourImpl* instance() { + static PosixColourImpl s_instance; + return &s_instance; + } + private: void setColour( const char* _escapeCode ) { - std::cout << '\033' << _escapeCode; + Catch::cout() << '\033' << _escapeCode; } }; - inline bool shouldUseColourForPlatform() { - return isatty(STDOUT_FILENO); - } - - static Detail::IColourImpl* platformColourInstance() { - static PosixColourImpl s_instance; - return &s_instance; + IColourImpl* platformColourInstance() { + Ptr config = getCurrentContext().getConfig(); + return (config && config->forceColour()) || isatty(STDOUT_FILENO) + ? PosixColourImpl::instance() + : NoColourImpl::instance(); } } // end anon namespace } // end namespace Catch -#endif // not Windows +#else // not Windows or ANSI /////////////////////////////////////////////// namespace Catch { - namespace { - struct NoColourImpl : Detail::IColourImpl { - void use( Colour::Code ) {} + static IColourImpl* platformColourInstance() { return NoColourImpl::instance(); } - static IColourImpl* instance() { - static NoColourImpl s_instance; - return &s_instance; - } - }; - static bool shouldUseColour() { - return shouldUseColourForPlatform() && !isDebuggerActive(); - } - } +} // end namespace Catch + +#endif // Windows/ ANSI/ None + +namespace Catch { Colour::Colour( Code _colourCode ) : m_moved( false ) { use( _colourCode ); } Colour::Colour( Colour const& _other ) : m_moved( false ) { const_cast( _other ).m_moved = true; } Colour::~Colour(){ if( !m_moved ) use( None ); } - void Colour::use( Code _colourCode ) { - impl()->use( _colourCode ); - } - Detail::IColourImpl* Colour::impl() { - return shouldUseColour() - ? platformColourInstance() - : NoColourImpl::instance(); + void Colour::use( Code _colourCode ) { + static IColourImpl* impl = isDebuggerActive() + ? NoColourImpl::instance() + : platformColourInstance(); + impl->use( _colourCode ); } } // end namespace Catch @@ -6238,7 +6563,7 @@ namespace Catch { namespace Catch { inline TestCaseInfo::SpecialProperties parseSpecialTag( std::string const& tag ) { - if( tag == "." || + if( startsWith( tag, "." ) || tag == "hide" || tag == "!hide" ) return TestCaseInfo::IsHidden; @@ -6258,13 +6583,13 @@ namespace Catch { if( isReservedTag( tag ) ) { { Colour colourGuard( Colour::Red ); - std::cerr + Catch::cerr() << "Tag name [" << tag << "] not allowed.\n" << "Tag names starting with non alpha-numeric characters are reserved\n"; } { Colour colourGuard( Colour::FileName ); - std::cerr << _lineInfo << std::endl; + Catch::cerr() << _lineInfo << std::endl; } exit(1); } @@ -6292,14 +6617,15 @@ namespace Catch { } else { if( c == ']' ) { - enforceNotReservedTag( tag, _lineInfo ); - - inTag = false; - if( tag == "hide" || tag == "." ) + TestCaseInfo::SpecialProperties prop = parseSpecialTag( tag ); + if( prop == TestCaseInfo::IsHidden ) isHidden = true; - else - tags.insert( tag ); + else if( prop == TestCaseInfo::None ) + enforceNotReservedTag( tag, _lineInfo ); + + tags.insert( tag ); tag.clear(); + inTag = false; } else tag += c; @@ -6416,8 +6742,33 @@ namespace Catch { namespace Catch { - // These numbers are maintained by a script - Version libraryVersion( 1, 0, 53, "master" ); + Version::Version + ( unsigned int _majorVersion, + unsigned int _minorVersion, + unsigned int _patchNumber, + std::string const& _branchName, + unsigned int _buildNumber ) + : majorVersion( _majorVersion ), + minorVersion( _minorVersion ), + patchNumber( _patchNumber ), + branchName( _branchName ), + buildNumber( _buildNumber ) + {} + + std::ostream& operator << ( std::ostream& os, Version const& version ) { + os << version.majorVersion << "." + << version.minorVersion << "." + << version.patchNumber; + + if( !version.branchName.empty() ) { + os << "-" << version.branchName + << "." << version.buildNumber; + } + return os; + } + + Version libraryVersion( 1, 2, 1, "", 0 ); + } // #included from: catch_message.hpp @@ -6501,6 +6852,7 @@ namespace Catch virtual void testCaseEnded( TestCaseStats const& testCaseStats ); virtual void testGroupEnded( TestGroupStats const& testGroupStats ); virtual void testRunEnded( TestRunStats const& testRunStats ); + virtual void skipTest( TestCaseInfo const& ); private: Ptr m_legacyReporter; @@ -6574,6 +6926,8 @@ namespace Catch void LegacyReporterAdapter::testRunEnded( TestRunStats const& testRunStats ) { m_legacyReporter->EndTesting( testRunStats.totals ); } + void LegacyReporterAdapter::skipTest( TestCaseInfo const& ) { + } } // #included from: catch_timer.hpp @@ -6596,11 +6950,11 @@ namespace Catch { uint64_t getCurrentTicks() { static uint64_t hz=0, hzo=0; if (!hz) { - QueryPerformanceFrequency((LARGE_INTEGER*)&hz); - QueryPerformanceCounter((LARGE_INTEGER*)&hzo); + QueryPerformanceFrequency( reinterpret_cast( &hz ) ); + QueryPerformanceCounter( reinterpret_cast( &hzo ) ); } uint64_t t; - QueryPerformanceCounter((LARGE_INTEGER*)&t); + QueryPerformanceCounter( reinterpret_cast( &t ) ); return ((t-hzo)*1000000)/hz; } #else @@ -6615,14 +6969,14 @@ namespace Catch { void Timer::start() { m_ticks = getCurrentTicks(); } - unsigned int Timer::getElapsedNanoseconds() const { + unsigned int Timer::getElapsedMicroseconds() const { return static_cast(getCurrentTicks() - m_ticks); } unsigned int Timer::getElapsedMilliseconds() const { - return static_cast((getCurrentTicks() - m_ticks)/1000); + return static_cast(getElapsedMicroseconds()/1000); } double Timer::getElapsedSeconds() const { - return (getCurrentTicks() - m_ticks)/1000000.0; + return getElapsedMicroseconds()/1000000.0; } } // namespace Catch @@ -6660,6 +7014,20 @@ namespace Catch { return start != std::string::npos ? str.substr( start, 1+end-start ) : ""; } + bool replaceInPlace( std::string& str, std::string const& replaceThis, std::string const& withThis ) { + bool replaced = false; + std::size_t i = str.find( replaceThis ); + while( i != std::string::npos ) { + replaced = true; + str = str.substr( 0, i ) + withThis + str.substr( i+replaceThis.size() ); + if( i < str.size()-withThis.size() ) + i = str.find( replaceThis, i+withThis.size() ); + else + i = std::string::npos; + } + return replaced; + } + pluralise::pluralise( std::size_t count, std::string const& label ) : m_count( count ), m_label( label ) @@ -6687,6 +7055,9 @@ namespace Catch { bool SourceLineInfo::operator == ( SourceLineInfo const& other ) const { return line == other.line && file == other.file; } + bool SourceLineInfo::operator < ( SourceLineInfo const& other ) const { + return line < other.line || ( line == other.line && file < other.file ); + } std::ostream& operator << ( std::ostream& os, SourceLineInfo const& info ) { #ifndef __GNUG__ @@ -6781,7 +7152,7 @@ namespace Catch { size = sizeof(info); if( sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &size, NULL, 0) != 0 ) { - std::cerr << "\n** Call to sysctl failed - unable to determine if debugger is active **\n" << std::endl; + Catch::cerr() << "\n** Call to sysctl failed - unable to determine if debugger is active **\n" << std::endl; return false; } @@ -6822,7 +7193,7 @@ namespace Catch { namespace Catch { void writeToDebugConsole( std::string const& text ) { // !TBD: Need a version for Mac/ XCode and other IDEs - std::cout << text; + Catch::cout() << text; } } #endif // Platform @@ -6834,6 +7205,8 @@ namespace Catch { namespace Detail { + std::string unprintableString = "{?}"; + namespace { struct Endianness { enum Arch { Big, Little }; @@ -6892,7 +7265,7 @@ std::string toString( std::wstring const& value ) { s.reserve( value.size() ); for(size_t i = 0; i < value.size(); ++i ) s += value[i] <= 0xff ? static_cast( value[i] ) : '?'; - return toString( s ); + return Catch::toString( s ); } std::string toString( const char* const value ) { @@ -6916,20 +7289,21 @@ std::string toString( wchar_t* const value ) std::string toString( int value ) { std::ostringstream oss; oss << value; + if( value >= 255 ) + oss << " (0x" << std::hex << value << ")"; return oss.str(); } std::string toString( unsigned long value ) { std::ostringstream oss; - if( value > 8192 ) - oss << "0x" << std::hex << value; - else - oss << value; + oss << value; + if( value >= 255 ) + oss << " (0x" << std::hex << value << ")"; return oss.str(); } std::string toString( unsigned int value ) { - return toString( static_cast( value ) ); + return Catch::toString( static_cast( value ) ); } template @@ -7055,7 +7429,7 @@ namespace Catch { if( !result.isOk() ) { if( getCurrentContext().getConfig()->shouldDebugBreak() ) m_shouldDebugBreak = true; - if( getCurrentContext().getRunner()->aborting() || m_assertionInfo.resultDisposition == ResultDisposition::Normal ) + if( getCurrentContext().getRunner()->aborting() || (m_assertionInfo.resultDisposition & ResultDisposition::Normal) ) m_shouldThrow = true; } } @@ -7195,7 +7569,7 @@ namespace Catch { } catch( std::exception& ex ) { Colour colourGuard( Colour::Red ); - std::cerr << ex.what() << std::endl; + Catch::cerr() << ex.what() << std::endl; exit(1); } } @@ -7208,6 +7582,8 @@ namespace Catch { // #included from: catch_reporter_bases.hpp #define TWOBLUECUBES_CATCH_REPORTER_BASES_HPP_INCLUDED +#include + namespace Catch { struct StreamingReporterBase : SharedImpl { @@ -7240,7 +7616,6 @@ namespace Catch { } virtual void testCaseEnded( TestCaseStats const& /* _testCaseStats */ ) { currentTestCaseInfo.reset(); - assert( m_sectionStack.empty() ); } virtual void testGroupEnded( TestGroupStats const& /* _testGroupStats */ ) { currentGroupInfo.reset(); @@ -7251,6 +7626,11 @@ namespace Catch { currentTestRunInfo.reset(); } + virtual void skipTest( TestCaseInfo const& ) { + // Don't do anything with this by default. + // It can optionally be overridden in the derived class. + } + Ptr m_config; std::ostream& stream; @@ -7380,6 +7760,8 @@ namespace Catch { } virtual void testRunEndedCumulative() = 0; + virtual void skipTest( TestCaseInfo const& ) {} + Ptr m_config; std::ostream& stream; std::vector m_assertions; @@ -7395,6 +7777,16 @@ namespace Catch { }; + template + char const* getLineOfChars() { + static char line[CATCH_CONFIG_CONSOLE_WIDTH] = {0}; + if( !*line ) { + memset( line, C, CATCH_CONFIG_CONSOLE_WIDTH-1 ); + line[CATCH_CONFIG_CONSOLE_WIDTH-1] = 0; + } + return line; + } + } // end namespace Catch // #included from: ../internal/catch_reporter_registrars.hpp @@ -7464,7 +7856,6 @@ namespace Catch { #define TWOBLUECUBES_CATCH_XMLWRITER_HPP_INCLUDED #include -#include #include #include @@ -7507,7 +7898,7 @@ namespace Catch { XmlWriter() : m_tagIsOpen( false ), m_needsNewline( false ), - m_os( &std::cout ) + m_os( &Catch::cout() ) {} XmlWriter( std::ostream& os ) @@ -7521,27 +7912,6 @@ namespace Catch { endElement(); } -//# ifndef CATCH_CPP11_OR_GREATER -// XmlWriter& operator = ( XmlWriter const& other ) { -// XmlWriter temp( other ); -// swap( temp ); -// return *this; -// } -//# else -// XmlWriter( XmlWriter const& ) = default; -// XmlWriter( XmlWriter && ) = default; -// XmlWriter& operator = ( XmlWriter const& ) = default; -// XmlWriter& operator = ( XmlWriter && ) = default; -//# endif -// -// void swap( XmlWriter& other ) { -// std::swap( m_tagIsOpen, other.m_tagIsOpen ); -// std::swap( m_needsNewline, other.m_needsNewline ); -// std::swap( m_tags, other.m_tags ); -// std::swap( m_indent, other.m_indent ); -// std::swap( m_os, other.m_os ); -// } - XmlWriter& startElement( std::string const& name ) { ensureTagClosed(); newlineIfNecessary(); @@ -7677,81 +8047,90 @@ namespace Catch { } namespace Catch { - class XmlReporter : public SharedImpl { + class XmlReporter : public StreamingReporterBase { public: - XmlReporter( ReporterConfig const& config ) : m_config( config ), m_sectionDepth( 0 ) {} + XmlReporter( ReporterConfig const& _config ) + : StreamingReporterBase( _config ), + m_sectionDepth( 0 ) + {} + + virtual ~XmlReporter(); static std::string getDescription() { return "Reports test results as an XML document"; } - virtual ~XmlReporter(); - private: // IReporter - - virtual bool shouldRedirectStdout() const { - return true; + public: // StreamingReporterBase + virtual ReporterPreferences getPreferences() const { + ReporterPreferences prefs; + prefs.shouldRedirectStdOut = true; + return prefs; } - virtual void StartTesting() { - m_xml.setStream( m_config.stream() ); - m_xml.startElement( "Catch" ); - if( !m_config.fullConfig()->name().empty() ) - m_xml.writeAttribute( "name", m_config.fullConfig()->name() ); + virtual void noMatchingTestCases( std::string const& s ) { + StreamingReporterBase::noMatchingTestCases( s ); } - virtual void EndTesting( const Totals& totals ) { - m_xml.scopedElement( "OverallResults" ) - .writeAttribute( "successes", totals.assertions.passed ) - .writeAttribute( "failures", totals.assertions.failed ) - .writeAttribute( "expectedFailures", totals.assertions.failedButOk ); - m_xml.endElement(); + virtual void testRunStarting( TestRunInfo const& testInfo ) { + StreamingReporterBase::testRunStarting( testInfo ); + m_xml.setStream( stream ); + m_xml.startElement( "Catch" ); + if( !m_config->name().empty() ) + m_xml.writeAttribute( "name", m_config->name() ); } - virtual void StartGroup( const std::string& groupName ) { + virtual void testGroupStarting( GroupInfo const& groupInfo ) { + StreamingReporterBase::testGroupStarting( groupInfo ); m_xml.startElement( "Group" ) - .writeAttribute( "name", groupName ); + .writeAttribute( "name", groupInfo.name ); } - virtual void EndGroup( const std::string&, const Totals& totals ) { - m_xml.scopedElement( "OverallResults" ) - .writeAttribute( "successes", totals.assertions.passed ) - .writeAttribute( "failures", totals.assertions.failed ) - .writeAttribute( "expectedFailures", totals.assertions.failedButOk ); - m_xml.endElement(); + virtual void testCaseStarting( TestCaseInfo const& testInfo ) { + StreamingReporterBase::testCaseStarting(testInfo); + m_xml.startElement( "TestCase" ).writeAttribute( "name", trim( testInfo.name ) ); + + if ( m_config->showDurations() == ShowDurations::Always ) + m_testCaseTimer.start(); } - virtual void StartSection( const std::string& sectionName, const std::string& description ) { + virtual void sectionStarting( SectionInfo const& sectionInfo ) { + StreamingReporterBase::sectionStarting( sectionInfo ); if( m_sectionDepth++ > 0 ) { m_xml.startElement( "Section" ) - .writeAttribute( "name", trim( sectionName ) ) - .writeAttribute( "description", description ); + .writeAttribute( "name", trim( sectionInfo.name ) ) + .writeAttribute( "description", sectionInfo.description ); } } - virtual void NoAssertionsInSection( const std::string& ) {} - virtual void NoAssertionsInTestCase( const std::string& ) {} - virtual void EndSection( const std::string& /*sectionName*/, const Counts& assertions ) { - if( --m_sectionDepth > 0 ) { - m_xml.scopedElement( "OverallResults" ) - .writeAttribute( "successes", assertions.passed ) - .writeAttribute( "failures", assertions.failed ) - .writeAttribute( "expectedFailures", assertions.failedButOk ); - m_xml.endElement(); - } - } + virtual void assertionStarting( AssertionInfo const& ) { } - virtual void StartTestCase( const Catch::TestCaseInfo& testInfo ) { - m_xml.startElement( "TestCase" ).writeAttribute( "name", trim( testInfo.name ) ); - m_currentTestSuccess = true; - } + virtual bool assertionEnded( AssertionStats const& assertionStats ) { + const AssertionResult& assertionResult = assertionStats.assertionResult; - virtual void Result( const Catch::AssertionResult& assertionResult ) { - if( !m_config.fullConfig()->includeSuccessfulResults() && assertionResult.getResultType() == ResultWas::Ok ) - return; + // Print any info messages in tags. + if( assertionStats.assertionResult.getResultType() != ResultWas::Ok ) { + for( std::vector::const_iterator it = assertionStats.infoMessages.begin(), itEnd = assertionStats.infoMessages.end(); + it != itEnd; + ++it ) { + if( it->type == ResultWas::Info ) { + m_xml.scopedElement( "Info" ) + .writeText( it->message ); + } else if ( it->type == ResultWas::Warning ) { + m_xml.scopedElement( "Warning" ) + .writeText( it->message ); + } + } + } + // Drop out if result was successful but we're not printing them. + if( !m_config->includeSuccessfulResults() && isOk(assertionResult.getResultType()) ) + return true; + + // Print the expression if there is one. if( assertionResult.hasExpression() ) { m_xml.startElement( "Expression" ) .writeAttribute( "success", assertionResult.succeeded() ) + .writeAttribute( "type", assertionResult.getTestMacroName() ) .writeAttribute( "filename", assertionResult.getSourceInfo().file ) .writeAttribute( "line", assertionResult.getSourceInfo().line ); @@ -7759,58 +8138,96 @@ namespace Catch { .writeText( assertionResult.getExpression() ); m_xml.scopedElement( "Expanded" ) .writeText( assertionResult.getExpandedExpression() ); - m_currentTestSuccess &= assertionResult.succeeded(); } + // And... Print a result applicable to each result type. switch( assertionResult.getResultType() ) { case ResultWas::ThrewException: m_xml.scopedElement( "Exception" ) .writeAttribute( "filename", assertionResult.getSourceInfo().file ) .writeAttribute( "line", assertionResult.getSourceInfo().line ) .writeText( assertionResult.getMessage() ); - m_currentTestSuccess = false; + break; + case ResultWas::FatalErrorCondition: + m_xml.scopedElement( "Fatal Error Condition" ) + .writeAttribute( "filename", assertionResult.getSourceInfo().file ) + .writeAttribute( "line", assertionResult.getSourceInfo().line ) + .writeText( assertionResult.getMessage() ); break; case ResultWas::Info: m_xml.scopedElement( "Info" ) .writeText( assertionResult.getMessage() ); break; case ResultWas::Warning: - m_xml.scopedElement( "Warning" ) - .writeText( assertionResult.getMessage() ); + // Warning will already have been written break; case ResultWas::ExplicitFailure: m_xml.scopedElement( "Failure" ) .writeText( assertionResult.getMessage() ); - m_currentTestSuccess = false; break; - case ResultWas::Unknown: - case ResultWas::Ok: - case ResultWas::FailureBit: - case ResultWas::ExpressionFailed: - case ResultWas::Exception: - case ResultWas::DidntThrowException: + default: break; } + if( assertionResult.hasExpression() ) m_xml.endElement(); + + return true; } - virtual void Aborted() { - // !TBD + virtual void sectionEnded( SectionStats const& sectionStats ) { + StreamingReporterBase::sectionEnded( sectionStats ); + if( --m_sectionDepth > 0 ) { + XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResults" ); + e.writeAttribute( "successes", sectionStats.assertions.passed ); + e.writeAttribute( "failures", sectionStats.assertions.failed ); + e.writeAttribute( "expectedFailures", sectionStats.assertions.failedButOk ); + + if ( m_config->showDurations() == ShowDurations::Always ) + e.writeAttribute( "durationInSeconds", sectionStats.durationInSeconds ); + + m_xml.endElement(); + } } - virtual void EndTestCase( const Catch::TestCaseInfo&, const Totals&, const std::string&, const std::string& ) { - m_xml.scopedElement( "OverallResult" ).writeAttribute( "success", m_currentTestSuccess ); + virtual void testCaseEnded( TestCaseStats const& testCaseStats ) { + StreamingReporterBase::testCaseEnded( testCaseStats ); + XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResult" ); + e.writeAttribute( "success", testCaseStats.totals.assertions.allOk() ); + + if ( m_config->showDurations() == ShowDurations::Always ) + e.writeAttribute( "durationInSeconds", m_testCaseTimer.getElapsedSeconds() ); + + m_xml.endElement(); + } + + virtual void testGroupEnded( TestGroupStats const& testGroupStats ) { + StreamingReporterBase::testGroupEnded( testGroupStats ); + // TODO: Check testGroupStats.aborting and act accordingly. + m_xml.scopedElement( "OverallResults" ) + .writeAttribute( "successes", testGroupStats.totals.assertions.passed ) + .writeAttribute( "failures", testGroupStats.totals.assertions.failed ) + .writeAttribute( "expectedFailures", testGroupStats.totals.assertions.failedButOk ); + m_xml.endElement(); + } + + virtual void testRunEnded( TestRunStats const& testRunStats ) { + StreamingReporterBase::testRunEnded( testRunStats ); + m_xml.scopedElement( "OverallResults" ) + .writeAttribute( "successes", testRunStats.totals.assertions.passed ) + .writeAttribute( "failures", testRunStats.totals.assertions.failed ) + .writeAttribute( "expectedFailures", testRunStats.totals.assertions.failedButOk ); m_xml.endElement(); } private: - ReporterConfig m_config; - bool m_currentTestSuccess; + Timer m_testCaseTimer; XmlWriter m_xml; int m_sectionDepth; }; + INTERNAL_CATCH_REGISTER_REPORTER( "xml", XmlReporter ) + } // end namespace Catch // #included from: ../reporters/catch_reporter_junit.hpp @@ -7937,7 +8354,7 @@ namespace Catch { xml.writeAttribute( "classname", className ); xml.writeAttribute( "name", name ); } - xml.writeAttribute( "time", toString( sectionNode.stats.durationInSeconds ) ); + xml.writeAttribute( "time", Catch::toString( sectionNode.stats.durationInSeconds ) ); writeAssertions( sectionNode ); @@ -7970,6 +8387,7 @@ namespace Catch { std::string elementName; switch( result.getResultType() ) { case ResultWas::ThrewException: + case ResultWas::FatalErrorCondition: elementName = "error"; break; case ResultWas::ExplicitFailure: @@ -8028,8 +8446,6 @@ namespace Catch { // #included from: ../reporters/catch_reporter_console.hpp #define TWOBLUECUBES_CATCH_REPORTER_CONSOLE_HPP_INCLUDED -#include - namespace Catch { struct ConsoleReporter : StreamingReporterBase { @@ -8164,6 +8580,11 @@ namespace Catch { passOrFail = "FAILED"; messageLabel = "due to unexpected exception with message"; break; + case ResultWas::FatalErrorCondition: + colour = Colour::Error; + passOrFail = "FAILED"; + messageLabel = "due to a fatal error condition"; + break; case ResultWas::DidntThrowException: colour = Colour::Error; passOrFail = "FAILED"; @@ -8273,14 +8694,12 @@ namespace Catch { stream << "\n" << getLineOfChars<'~'>() << "\n"; Colour colour( Colour::SecondaryText ); stream << currentTestRunInfo->name - << " is a Catch v" << libraryVersion.majorVersion << "." - << libraryVersion.minorVersion << " b" - << libraryVersion.buildNumber; - if( libraryVersion.branchName != std::string( "master" ) ) - stream << " (" << libraryVersion.branchName << ")"; - stream << " host application.\n" + << " is a Catch v" << libraryVersion << " host application.\n" << "Run with -? for options\n\n"; + if( m_config->rngSeed() != 0 ) + stream << "Randomness seeded to: " << m_config->rngSeed() << "\n\n"; + currentTestRunInfo.used = true; } void lazyPrintGroupInfo() { @@ -8452,15 +8871,6 @@ namespace Catch { void printSummaryDivider() { stream << getLineOfChars<'-'>() << "\n"; } - template - static char const* getLineOfChars() { - static char line[CATCH_CONFIG_CONSOLE_WIDTH] = {0}; - if( !*line ) { - memset( line, C, CATCH_CONFIG_CONSOLE_WIDTH-1 ); - line[CATCH_CONFIG_CONSOLE_WIDTH-1] = 0; - } - return line; - } private: bool m_headerPrinted; @@ -8569,6 +8979,13 @@ namespace Catch { printExpressionWas(); printRemainingMessages(); break; + case ResultWas::FatalErrorCondition: + printResultType( Colour::Error, failedString() ); + printIssue( "fatal error condition with message:" ); + printMessage(); + printExpressionWas(); + printRemainingMessages(); + break; case ResultWas::DidntThrowException: printResultType( Colour::Error, failedString() ); printIssue( "expected exception, got none" ); @@ -8798,8 +9215,6 @@ namespace Catch { Matchers::Impl::StdString::EndsWith::~EndsWith() {} void Config::dummy() {} - - INTERNAL_CATCH_REGISTER_LEGACY_REPORTER( "xml", XmlReporter ) } #ifdef __clang__ @@ -8988,9 +9403,13 @@ using Catch::Detail::Approx; #define TWOBLUECUBES_CATCH_REENABLE_WARNINGS_H_INCLUDED #ifdef __clang__ -#pragma clang diagnostic pop +# ifdef __ICC // icpc defines the __clang__ macro +# pragma warning(pop) +# else +# pragma clang diagnostic pop +# endif #elif defined __GNUC__ -#pragma GCC diagnostic pop +# pragma GCC diagnostic pop #endif #endif // TWOBLUECUBES_SINGLE_INCLUDE_CATCH_HPP_INCLUDED diff --git a/test/t/basic/helper.hpp b/test/t/basic/helper.hpp index 61e076934dc..5a2130e4eca 100644 --- a/test/t/basic/helper.hpp +++ b/test/t/basic/helper.hpp @@ -9,7 +9,7 @@ inline void add_tags(osmium::memory::Buffer& buffer, osmium::builder::Builder& builder, const std::vector>& tags) { osmium::builder::TagListBuilder tl_builder(buffer, &builder); - for (auto& tag : tags) { + for (const auto& tag : tags) { tl_builder.add_tag(tag.first, tag.second); } } @@ -26,9 +26,11 @@ inline osmium::Way& buffer_add_way(osmium::memory::Buffer& buffer, const char* u osmium::builder::WayBuilder builder(buffer); builder.add_user(user); add_tags(buffer, builder, tags); - osmium::builder::WayNodeListBuilder wnl_builder(buffer, &builder); - for (const osmium::object_id_type ref : nodes) { - wnl_builder.add_node_ref(ref); + { + osmium::builder::WayNodeListBuilder wnl_builder(buffer, &builder); + for (const osmium::object_id_type ref : nodes) { + wnl_builder.add_node_ref(ref); + } } buffer.commit(); return builder.object(); @@ -38,9 +40,11 @@ inline osmium::Way& buffer_add_way(osmium::memory::Buffer& buffer, const char* u osmium::builder::WayBuilder builder(buffer); builder.add_user(user); add_tags(buffer, builder, tags); - osmium::builder::WayNodeListBuilder wnl_builder(buffer, &builder); - for (auto& p : nodes) { - wnl_builder.add_node_ref(p.first, p.second); + { + osmium::builder::WayNodeListBuilder wnl_builder(buffer, &builder); + for (const auto& p : nodes) { + wnl_builder.add_node_ref(p.first, p.second); + } } buffer.commit(); return builder.object(); @@ -53,9 +57,11 @@ inline osmium::Relation& buffer_add_relation( osmium::builder::RelationBuilder builder(buffer); builder.add_user(user); add_tags(buffer, builder, tags); - osmium::builder::RelationMemberListBuilder rml_builder(buffer, &builder); - for (const auto& member : members) { - rml_builder.add_member(osmium::char_to_item_type(std::get<0>(member)), std::get<1>(member), std::get<2>(member)); + { + osmium::builder::RelationMemberListBuilder rml_builder(buffer, &builder); + for (const auto& member : members) { + rml_builder.add_member(osmium::char_to_item_type(std::get<0>(member)), std::get<1>(member), std::get<2>(member)); + } } buffer.commit(); return builder.object(); @@ -69,15 +75,15 @@ inline osmium::Area& buffer_add_area(osmium::memory::Buffer& buffer, const char* builder.add_user(user); add_tags(buffer, builder, tags); - for (auto& ring : rings) { + for (const auto& ring : rings) { if (ring.first) { osmium::builder::OuterRingBuilder ring_builder(buffer, &builder); - for (auto& p : ring.second) { + for (const auto& p : ring.second) { ring_builder.add_node_ref(p.first, p.second); } } else { osmium::builder::InnerRingBuilder ring_builder(buffer, &builder); - for (auto& p : ring.second) { + for (const auto& p : ring.second) { ring_builder.add_node_ref(p.first, p.second); } } diff --git a/test/t/basic/test_box.cpp b/test/t/basic/test_box.cpp index 8182fbf353c..768cf41f31a 100644 --- a/test/t/basic/test_box.cpp +++ b/test/t/basic/test_box.cpp @@ -2,7 +2,10 @@ #include +#include + #include +#include #include TEST_CASE("Box") { @@ -48,6 +51,10 @@ TEST_CASE("Box") { REQUIRE(b.contains(loc1)); REQUIRE(b.contains(loc2)); REQUIRE(b.contains(loc3)); + + osmium::CRC crc32; + crc32.update(b); + REQUIRE(crc32().checksum() == 0xd381a838); } SECTION("output_defined") { diff --git a/test/t/basic/test_changeset.cpp b/test/t/basic/test_changeset.cpp index 2549c1e6767..fc9f1bdee99 100644 --- a/test/t/basic/test_changeset.cpp +++ b/test/t/basic/test_changeset.cpp @@ -1,12 +1,16 @@ #include "catch.hpp" +#include + #include +#include #include "helper.hpp" -TEST_CASE("Basic_Changeset") { +TEST_CASE("Basic Changeset") { + + osmium::CRC crc32; -SECTION("changeset_builder") { osmium::memory::Buffer buffer(10 * 1000); osmium::Changeset& cs1 = buffer_add_changeset(buffer, @@ -28,6 +32,9 @@ SECTION("changeset_builder") { REQUIRE(1 == cs1.tags().size()); REQUIRE(std::string("user") == cs1.user()); + crc32.update(cs1); + REQUIRE(crc32().checksum() == 0xf44aff25); + osmium::Changeset& cs2 = buffer_add_changeset(buffer, "user", {{"comment", "foo"}, {"foo", "bar"}}); @@ -52,6 +59,5 @@ SECTION("changeset_builder") { REQUIRE(cs1 <= cs2); REQUIRE(false == (cs1 > cs2)); REQUIRE(false == (cs1 >= cs2)); -} } diff --git a/test/t/basic/test_crc.cpp b/test/t/basic/test_crc.cpp new file mode 100644 index 00000000000..aab1013f4cf --- /dev/null +++ b/test/t/basic/test_crc.cpp @@ -0,0 +1,49 @@ +#include "catch.hpp" + +#include + +#include + +#include "helper.hpp" + +TEST_CASE("CRC of basic datatypes") { + + osmium::CRC crc32; + + SECTION("Bool") { + crc32.update_bool(true); + crc32.update_bool(false); + + REQUIRE(crc32().checksum() == 0x58c223be); + } + + SECTION("Char") { + crc32.update_int8('x'); + crc32.update_int8('y'); + + REQUIRE(crc32().checksum() == 0x8fe62899); + } + + SECTION("String") { + const char* str = "foobar"; + crc32.update_string(str); + + REQUIRE(crc32().checksum() == 0x9ef61f95); + } + + SECTION("Timestamp") { + osmium::Timestamp t("2015-07-12T13:10:46Z"); + crc32.update(t); + + REQUIRE(crc32().checksum() == 0x58a29d7); + } + + SECTION("Location") { + osmium::Location loc { 3.46, 2.001 }; + crc32.update(loc); + + REQUIRE(crc32().checksum() == 0xddee042c); + } + +} + diff --git a/test/t/basic/test_node.cpp b/test/t/basic/test_node.cpp index 6c2c899bd25..db5b4cd53ce 100644 --- a/test/t/basic/test_node.cpp +++ b/test/t/basic/test_node.cpp @@ -1,11 +1,16 @@ #include "catch.hpp" +#include + +#include #include #include "helper.hpp" TEST_CASE("Basic_Node") { + osmium::CRC crc32; + SECTION("node_builder") { osmium::memory::Buffer buffer(10000); @@ -36,6 +41,9 @@ SECTION("node_builder") { REQUIRE(osmium::Location(3.5, 4.7) == node.location()); REQUIRE(2 == node.tags().size()); + crc32.update(node); + REQUIRE(crc32().checksum() == 0xc696802f); + node.set_visible(false); REQUIRE(false == node.visible()); REQUIRE(true == node.deleted()); diff --git a/test/t/basic/test_relation.cpp b/test/t/basic/test_relation.cpp index 4c62a4189c1..fd5c7b4ad29 100644 --- a/test/t/basic/test_relation.cpp +++ b/test/t/basic/test_relation.cpp @@ -1,12 +1,16 @@ #include "catch.hpp" +#include + +#include #include #include "helper.hpp" -TEST_CASE("Basic_Relation") { +TEST_CASE("Build relation") { + + osmium::CRC crc32; -SECTION("relation_builder") { osmium::memory::Buffer buffer(10000); osmium::Relation& relation = buffer_add_relation(buffer, @@ -55,6 +59,7 @@ SECTION("relation_builder") { } ++n; } -} + crc32.update(relation); + REQUIRE(crc32().checksum() == 0xebcd836d); } diff --git a/test/t/basic/test_timestamp.cpp b/test/t/basic/test_timestamp.cpp index 6a04a4d0957..f015730ea00 100644 --- a/test/t/basic/test_timestamp.cpp +++ b/test/t/basic/test_timestamp.cpp @@ -29,6 +29,10 @@ TEST_CASE("Timestamp") { REQUIRE("2000-01-01T00:00:00Z" == t.to_iso()); } + SECTION("throws if initialized from bad string") { + REQUIRE_THROWS_AS(osmium::Timestamp("x"), std::invalid_argument); + } + SECTION("can be implicitly cast to time_t") { osmium::Timestamp t(4242); time_t x = t; diff --git a/test/t/basic/test_types_from_string.cpp b/test/t/basic/test_types_from_string.cpp new file mode 100644 index 00000000000..2481ae87649 --- /dev/null +++ b/test/t/basic/test_types_from_string.cpp @@ -0,0 +1,90 @@ +#include "catch.hpp" + +#include +#include + +TEST_CASE("set ID from string") { + REQUIRE(osmium::string_to_object_id("0") == 0); + REQUIRE(osmium::string_to_object_id("17") == 17); + REQUIRE(osmium::string_to_object_id("-17") == -17); + REQUIRE(osmium::string_to_object_id("01") == 1); + + REQUIRE_THROWS_AS(osmium::string_to_object_id(""), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id(" "), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id(" 22"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("x"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("0x1"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("12a"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("12345678901234567890"), std::range_error); +} + +TEST_CASE("set type and ID from string") { + auto n17 = osmium::string_to_object_id("n17", osmium::osm_entity_bits::nwr); + REQUIRE(n17.first == osmium::item_type::node); + REQUIRE(n17.second == 17); + + auto w42 = osmium::string_to_object_id("w42", osmium::osm_entity_bits::nwr); + REQUIRE(w42.first == osmium::item_type::way); + REQUIRE(w42.second == 42); + + auto r_2 = osmium::string_to_object_id("r-2", osmium::osm_entity_bits::nwr); + REQUIRE(r_2.first == osmium::item_type::relation); + REQUIRE(r_2.second == -2); + + auto x3 = osmium::string_to_object_id("3", osmium::osm_entity_bits::nwr); + REQUIRE(x3.first == osmium::item_type::undefined); + REQUIRE(x3.second == 3); + + REQUIRE_THROWS_AS(osmium::string_to_object_id("", osmium::osm_entity_bits::nwr), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("n", osmium::osm_entity_bits::nwr), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("x3", osmium::osm_entity_bits::nwr), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("nx3", osmium::osm_entity_bits::nwr), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("n3", osmium::osm_entity_bits::way), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("n3a", osmium::osm_entity_bits::nwr), std::range_error); +} + +TEST_CASE("set object version from string") { + REQUIRE(osmium::string_to_object_version("0") == 0); + REQUIRE(osmium::string_to_object_version("1") == 1); + + REQUIRE_THROWS_AS(osmium::string_to_object_version("-1"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_version(""), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_version(" "), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_version(" 22"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_version("x"), std::range_error); +} + +TEST_CASE("set changeset id from string") { + REQUIRE(osmium::string_to_changeset_id("0") == 0); + REQUIRE(osmium::string_to_changeset_id("1") == 1); + + REQUIRE_THROWS_AS(osmium::string_to_changeset_id("-1"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_changeset_id(""), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_changeset_id(" "), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_changeset_id(" 22"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_changeset_id("x"), std::range_error); +} + +TEST_CASE("set user id from string") { + REQUIRE(osmium::string_to_user_id("0") == 0); + REQUIRE(osmium::string_to_user_id("1") == 1); + REQUIRE(osmium::string_to_user_id("-1") == -1); + + REQUIRE_THROWS_AS(osmium::string_to_user_id("-2"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_user_id(""), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_user_id(" "), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_user_id(" 22"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_user_id("x"), std::range_error); +} + +TEST_CASE("set num changes from string") { + REQUIRE(osmium::string_to_num_changes("0") == 0); + REQUIRE(osmium::string_to_num_changes("1") == 1); + + REQUIRE_THROWS_AS(osmium::string_to_num_changes("-1"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_num_changes(""), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_num_changes(" "), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_num_changes(" 22"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_num_changes("x"), std::range_error); +} + diff --git a/test/t/basic/test_way.cpp b/test/t/basic/test_way.cpp index 9d2ba0691ef..7c7bc214841 100644 --- a/test/t/basic/test_way.cpp +++ b/test/t/basic/test_way.cpp @@ -1,11 +1,16 @@ #include "catch.hpp" +#include + #include +#include #include #include "helper.hpp" -TEST_CASE("Basic_Way") { +TEST_CASE("Build way") { + + osmium::CRC crc32; SECTION("way_builder") { osmium::memory::Buffer buffer(10000); @@ -38,6 +43,9 @@ SECTION("way_builder") { REQUIRE(3 == way.nodes()[1].ref()); REQUIRE(2 == way.nodes()[2].ref()); REQUIRE(! way.is_closed()); + + crc32.update(way); + REQUIRE(crc32().checksum() == 0x20fe7a30); } SECTION("closed_way") { diff --git a/test/t/buffer/test_buffer_purge.cpp b/test/t/buffer/test_buffer_purge.cpp index 10cdfe72cea..a72db1b5c1e 100644 --- a/test/t/buffer/test_buffer_purge.cpp +++ b/test/t/buffer/test_buffer_purge.cpp @@ -108,7 +108,7 @@ TEST_CASE("Purge data from buffer") { node_builder.object().set_removed(true); } buffer.commit(); - size_t size2 = buffer.committed() - size1; + REQUIRE(std::distance(buffer.begin(), buffer.end()) == 2); CallbackClass callback; @@ -127,20 +127,20 @@ TEST_CASE("Purge data from buffer") { node_builder.add_user("testuser_longer_name"); } buffer.commit(); - size_t size1 = buffer.committed(); + { osmium::builder::NodeBuilder node_builder(buffer); node_builder.add_user("testuser"); node_builder.object().set_removed(true); } buffer.commit(); - size_t size2 = buffer.committed() - size1; + { osmium::builder::NodeBuilder node_builder(buffer); node_builder.add_user("sn"); } buffer.commit(); - size_t size3 = buffer.committed() - (size1 + size2); + REQUIRE(std::distance(buffer.begin(), buffer.end()) == 3); CallbackClass callback; @@ -159,21 +159,21 @@ TEST_CASE("Purge data from buffer") { node_builder.object().set_removed(true); } buffer.commit(); - size_t size1 = buffer.committed(); + { osmium::builder::NodeBuilder node_builder(buffer); node_builder.add_user("testuser"); node_builder.object().set_removed(true); } buffer.commit(); - size_t size2 = buffer.committed() - size1; + { osmium::builder::NodeBuilder node_builder(buffer); node_builder.add_user("sn"); node_builder.object().set_removed(true); } buffer.commit(); - size_t size3 = buffer.committed() - (size1 + size2); + REQUIRE(std::distance(buffer.begin(), buffer.end()) == 3); CallbackClass callback; diff --git a/test/t/geom/test_exception.cpp b/test/t/geom/test_exception.cpp new file mode 100644 index 00000000000..fe950434b54 --- /dev/null +++ b/test/t/geom/test_exception.cpp @@ -0,0 +1,16 @@ +#include "catch.hpp" + +#include + +#include + +TEST_CASE("Geometry exception") { + + SECTION("geometry_error") { + osmium::geometry_error e("some error message", "node", 17); + REQUIRE(e.id() == 17); + REQUIRE(std::string(e.what()) == "some error message (node_id=17)"); + } + +} + diff --git a/test/t/geom/test_geos.cpp b/test/t/geom/test_geos.cpp index e93228b4cc7..d849e3b1603 100644 --- a/test/t/geom/test_geos.cpp +++ b/test/t/geom/test_geos.cpp @@ -5,9 +5,7 @@ #include "../basic/helper.hpp" -TEST_CASE("GEOS_Geometry") { - -SECTION("point") { +TEST_CASE("GEOS geometry factory - create point") { osmium::geom::GEOSFactory<> factory; std::unique_ptr point {factory.create_point(osmium::Location(3.2, 4.2))}; @@ -16,7 +14,7 @@ SECTION("point") { REQUIRE(-1 == point->getSRID()); } -SECTION("non_default_srid") { +TEST_CASE("GEOS geometry factory - create point with non-default srid") { osmium::geom::GEOSFactory<> factory(4326); std::unique_ptr point {factory.create_point(osmium::Location(3.2, 4.2))}; @@ -25,13 +23,23 @@ SECTION("non_default_srid") { REQUIRE(4326 == point->getSRID()); } -SECTION("empty_point") { +TEST_CASE("GEOS geometry factory - create point with externally created GEOS factory") { + geos::geom::GeometryFactory geos_factory; + osmium::geom::GEOSFactory<> factory(geos_factory); + + std::unique_ptr point {factory.create_point(osmium::Location(3.2, 4.2))}; + REQUIRE(3.2 == point->getX()); + REQUIRE(4.2 == point->getY()); + REQUIRE(0 == point->getSRID()); +} + +TEST_CASE("GEOS geometry factory - can not create from invalid location") { osmium::geom::GEOSFactory<> factory; REQUIRE_THROWS_AS(factory.create_point(osmium::Location()), osmium::invalid_location); } -SECTION("linestring") { +TEST_CASE("GEOS geometry factory - create linestring") { osmium::geom::GEOSFactory<> factory; osmium::memory::Buffer buffer(10000); @@ -42,7 +50,7 @@ SECTION("linestring") { {2, {3.6, 4.9}} }); - { + SECTION("from way node list") { std::unique_ptr linestring {factory.create_linestring(wnl)}; REQUIRE(3 == linestring->getNumPoints()); @@ -52,7 +60,7 @@ SECTION("linestring") { REQUIRE(3.6 == p2->getX()); } - { + SECTION("without duplicates and backwards") { std::unique_ptr linestring {factory.create_linestring(wnl, osmium::geom::use_nodes::unique, osmium::geom::direction::backward)}; REQUIRE(3 == linestring->getNumPoints()); std::unique_ptr p0 = std::unique_ptr(linestring->getPointN(0)); @@ -61,14 +69,14 @@ SECTION("linestring") { REQUIRE(3.2 == p2->getX()); } - { + SECTION("with duplicates") { std::unique_ptr linestring {factory.create_linestring(wnl, osmium::geom::use_nodes::all)}; REQUIRE(4 == linestring->getNumPoints()); std::unique_ptr p0 = std::unique_ptr(linestring->getPointN(0)); REQUIRE(3.2 == p0->getX()); } - { + SECTION("with duplicates and backwards") { std::unique_ptr linestring {factory.create_linestring(wnl, osmium::geom::use_nodes::all, osmium::geom::direction::backward)}; REQUIRE(4 == linestring->getNumPoints()); std::unique_ptr p0 = std::unique_ptr(linestring->getPointN(0)); @@ -76,7 +84,7 @@ SECTION("linestring") { } } -SECTION("area_1outer_0inner") { +TEST_CASE("GEOS geometry factory - create area with one outer and no inner rings") { osmium::geom::GEOSFactory<> factory; osmium::memory::Buffer buffer(10000); @@ -105,7 +113,7 @@ SECTION("area_1outer_0inner") { REQUIRE(3.5 == l0e_p0->getX()); } -SECTION("area_1outer_1inner") { +TEST_CASE("GEOS geometry factory - create area with one outer and one inner ring") { osmium::geom::GEOSFactory<> factory; osmium::memory::Buffer buffer(10000); @@ -142,7 +150,7 @@ SECTION("area_1outer_1inner") { REQUIRE(5 == l0i0->getNumPoints()); } -SECTION("area_2outer_2inner") { +TEST_CASE("GEOS geometry factory - create area with two outer and two inner rings") { osmium::geom::GEOSFactory<> factory; osmium::memory::Buffer buffer(10000); @@ -195,4 +203,3 @@ SECTION("area_2outer_2inner") { REQUIRE(5 == l1e->getNumPoints()); } -} diff --git a/test/t/geom/test_projection.cpp b/test/t/geom/test_projection.cpp index 2257d7f0669..58854109e42 100644 --- a/test/t/geom/test_projection.cpp +++ b/test/t/geom/test_projection.cpp @@ -1,5 +1,7 @@ #include "catch.hpp" +#include + #include #include #include @@ -128,4 +130,20 @@ SECTION("compare_mercators") { } } +SECTION("compare_mercators") { + osmium::geom::MercatorProjection projection_merc; + osmium::geom::Projection projection_3857(3857); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution<> dis_x(-180.0, 180.0); + std::uniform_real_distribution<> dis_y(-90.0, 90.0); + + for (int n = 0; n < 100000; ++n) { + const osmium::Location loc(dis_x(gen), dis_y(gen)); + REQUIRE(projection_merc(loc).x == Approx(projection_3857(loc).x).epsilon(0.1)); + REQUIRE(projection_merc(loc).y == Approx(projection_3857(loc).y).epsilon(0.1)); + } +} + } diff --git a/test/t/geom/test_tile.cpp b/test/t/geom/test_tile.cpp new file mode 100644 index 00000000000..e80cb960410 --- /dev/null +++ b/test/t/geom/test_tile.cpp @@ -0,0 +1,93 @@ +#include "catch.hpp" + +#include + +#include + +#include "helper.hpp" + +#include "test_tile_data.hpp" + +TEST_CASE("Tile") { + + SECTION("x0.0 y0.0 zoom 0") { + osmium::Location l(0.0, 0.0); + + osmium::geom::Tile t(0, l); + + REQUIRE(t.x == 0); + REQUIRE(t.y == 0); + REQUIRE(t.z == 0); + } + + SECTION("x180.0 y90.0 zoom 0") { + osmium::Location l(180.0, 90.0); + + osmium::geom::Tile t(0, l); + + REQUIRE(t.x == 0); + REQUIRE(t.y == 0); + REQUIRE(t.z == 0); + } + + SECTION("x180.0 y90.0 zoom 4") { + osmium::Location l(180.0, 90.0); + + osmium::geom::Tile t(4, l); + + REQUIRE(t.x == (1 << 4) - 1); + REQUIRE(t.y == 0); + REQUIRE(t.z == 4); + } + + SECTION("x0.0 y0.0 zoom 4") { + osmium::Location l(0.0, 0.0); + + osmium::geom::Tile t(4, l); + + auto n = 1 << (4-1); + REQUIRE(t.x == n); + REQUIRE(t.y == n); + REQUIRE(t.z == 4); + } + + SECTION("equality") { + osmium::geom::Tile a(4, 3, 4); + osmium::geom::Tile b(4, 3, 4); + osmium::geom::Tile c(4, 4, 3); + REQUIRE(a == b); + REQUIRE(a != c); + REQUIRE(b != c); + } + + SECTION("order") { + osmium::geom::Tile a(2, 3, 4); + osmium::geom::Tile b(4, 3, 4); + osmium::geom::Tile c(4, 4, 3); + osmium::geom::Tile d(4, 4, 2); + REQUIRE(a < b); + REQUIRE(a < c); + REQUIRE(b < c); + REQUIRE(d < c); + } + + SECTION("tilelist") { + std::istringstream input_data(s); + while (input_data) { + double lon, lat; + uint32_t x, y, zoom; + input_data >> lon; + input_data >> lat; + input_data >> x; + input_data >> y; + input_data >> zoom; + + osmium::Location l(lon, lat); + osmium::geom::Tile t(zoom, l); + REQUIRE(t.x == x); + REQUIRE(t.y == y); + } + } + +} + diff --git a/test/t/geom/test_tile_data.hpp b/test/t/geom/test_tile_data.hpp new file mode 100644 index 00000000000..e5c0953c050 --- /dev/null +++ b/test/t/geom/test_tile_data.hpp @@ -0,0 +1,475 @@ + +std::string s = R"(127.4864358 16.8380041 223904 118630 18 +163.1103174 39.4760232 121 48 7 +-4.1372725 -22.5105386 31 36 6 +98.7193066 -36.2312406 1 1 1 +63.5773661 -13.47636 21 17 5 +-88.4518148 37.9805485 260 395 10 +-14.5903133 -28.2989812 3763 4767 13 +-13.4971239 -34.4080035 14 19 5 +-169.156223 -64.0900356 3 93 7 +27.1473191 -4.1993125 4713 4191 13 +-160.9733129 54.3684314 13 81 8 +129.0194139 14.2576156 439 235 9 +-69.5085993 -56.8253221 10057 22700 15 +-77.8387486 18.1961517 1162 1837 12 +-76.2695325 -18.2494296 147 282 9 +-91.594905 7.6698071 1 3 3 +-116.7926741 -20.6060813 179 571 10 +109.0552776 -1.9947569 52620 33131 16 +-156.1846426 -79.3817554 33 449 9 +-95.3403755 -27.8978407 1 4 3 +-55.1827573 -73.2293796 44 103 7 +-108.5207885 -48.0099293 50 167 8 +23.7540108 -15.3395164 9273 8898 14 +-155.6662842 -68.3295899 0 0 0 +75.8139119 30.9914252 363 209 9 +-135.8034544 64.7242469 0 1 2 +-48.743352 70.9392876 23 13 6 +-38.6968026 7.7867812 0 0 0 +-18.5234838 11.8557704 29395 30594 16 +-152.5632568 19.4069834 78 455 10 +-63.2089431 -80.5909713 0 0 0 +-94.1255611 -81.2028822 244 930 10 +175.0862205 -33.0865142 3 2 2 +-179.6241926 -37.0256609 1 625 10 +135.6783824 -38.6011643 459739 323170 19 +-139.6407533 -83.2495209 0 7 3 +-14.1336447 -56.5465949 58 88 7 +-30.7414568 -32.9543731 26 38 6 +10.3306861 73.2444693 1082 399 11 +-84.8379263 29.2363222 16 26 6 +-94.0685822 -39.5503996 7821 20309 15 +-86.5944356 -41.7491891 265 642 10 +11.9182172 -80.5613703 34937 58784 16 +91.8773752 -32.1741317 0 0 0 +126.2879157 20.5759564 1742 904 11 +-160.7743029 -47.3192128 27999 340565 19 +-4.2449045 -50.3288332 31 42 6 +-66.6857158 61.4380757 10 9 5 +169.7372317 -74.3365704 3 3 2 +87.4815328 75.6218888 95 21 7 +60.3544927 28.3165267 0 0 0 +-48.9614619 -59.3292497 2 5 3 +-123.2935018 -59.5454886 80 362 9 +-31.5909316 -14.8985476 13 17 5 +58.1862173 59.0957666 2710 1209 12 +-72.8881665 -2.2648849 1218 2073 12 +-33.7267461 8.6006817 106512 124785 18 +175.723181 46.4929742 7 2 3 +-127.1963326 76.0328786 0 0 0 +-161.7444367 -26.7634497 13293 151310 18 +-163.976298 -8.1169845 91 1070 11 +63.7757109 5.6049418 2 1 2 +12.7012434 22.1157713 70160 57276 17 +19.5832849 -25.0284049 1135 1171 11 +-22.6619642 54.8831276 111 81 8 +78.7736907 24.0919863 5888 3530 13 +121.9003045 -64.6256685 107 94 7 +-64.8766793 61.6655916 81 71 8 +113.0498445 -70.0016518 416 397 9 +-51.5116259 68.1532424 46781 31217 17 +-89.9005747 87.9523248 512 0 11 +59.2536822 -75.1520493 10 13 4 +17.375372 74.9259262 287448 93371 19 +75.6426945 8.4808632 186153 124873 18 +144.89589 75.1647661 14786 2875 14 +174.4350898 -29.268169 258091 153376 18 +-91.8773113 50.6182166 0 0 0 +5.3822308 45.1391794 134991 94156 18 +43.0978373 -10.0764237 324909 276895 19 +55.6682917 -40.9015342 21451 20470 15 +-37.584032 52.253723 6 5 4 +131.0141997 29.0271798 28309 13621 15 +69.8830721 -86.8548645 363918 524287 19 +-107.7917548 -76.9626654 26290 110787 17 +-57.1736642 49.9345991 2 2 3 +165.5170226 -84.9735363 982 1021 10 +-139.7208984 -73.8754873 1 12 4 +-154.5959463 -10.596718 4 33 6 +-106.4164918 36.5042686 3348 6405 14 +-92.9688259 -11.2262505 0 2 2 +138.2722283 8.1109779 7 3 3 +123.1389319 -40.3505677 862 637 10 +-171.4780435 24.9112482 0 13 5 +89.3668763 -63.1809434 392293 381781 19 +-79.6906176 -13.376312 9130 17612 15 +133.1329522 -28.9032634 445 298 9 +115.0369496 70.6965823 6 1 3 +-74.8926697 -78.9770185 2391 7144 13 +51.9175766 73.0184277 164 50 8 +178.1319784 43.7111421 509 186 9 +162.4098389 61.1595443 0 0 0 +7.6241126 -75.6182944 2 3 2 +172.8902767 28.5068027 125 53 7 +156.4030739 -76.7309238 478 431 9 +-131.3963189 62.2039684 1 2 3 +34.6057088 -36.2933264 2441 2491 12 +-3.3896413 -48.2734347 15 20 5 +83.528842 -48.219886 2998 2675 12 +16.6000512 -31.6322244 17894 19421 15 +-18.2425749 21.0749052 14 14 5 +35.6035336 46.9916228 78498 46105 17 +-109.3453091 -34.2312523 12 38 6 +88.4909962 58.6104749 47 19 6 +-129.7372783 21.7408241 571 1794 12 +25.7269392 -40.1240139 1 1 1 +26.3829579 5.248998 37570 31811 16 +141.1768247 -41.4653038 14617 10269 14 +151.3788752 -38.0160512 241302 161042 18 +-92.7471483 -35.391745 15883 39664 16 +142.5902623 -14.1023743 28 17 5 +179.3461485 -86.7573357 3 3 2 +-40.9746194 61.5689546 790 576 11 +128.6399735 -54.8895009 56186 44772 16 +-141.391583 -63.8272 0 2 2 +-3.5004218 19.3089998 4016 3648 13 +16.138208 -42.7868627 1 1 1 +78.502315 -18.91667 2 2 2 +-44.6311826 -15.7483106 98572 142686 18 +-120.431941 -39.3431529 0 0 0 +-70.4915024 25.4763412 9967 13984 15 +118.0711114 -66.5691073 211 192 8 +-114.945538 38.1764389 0 0 0 +142.991315 -46.3378741 14699 10577 14 +34.2770562 -84.7173756 312063 518834 19 +109.5563415 -85.9138266 105424 131071 17 +-171.8032643 70.1948223 0 3 4 +-90.8434294 89.7252122 126 0 9 +163.5677082 -53.9885419 3 2 2 +128.884016 -63.0732584 112461 95358 17 +116.5348575 -56.1616143 843 705 10 +-171.5770602 37.9225943 11 197 9 +48.1396653 -49.3932234 5191 5392 13 +-157.6786731 15.3895763 32507 239456 19 +15.7385145 -37.1364397 1113 1251 11 +137.5179466 61.2025671 1 0 1 +-70.4767722 42.9798424 19938 24086 16 +27.8867901 43.4642562 9461 5991 14 +68.5677462 -84.9560937 11312 16334 14 +-56.790151 -67.124147 179437 395476 19 +108.850832 43.3493384 26291 11996 15 +-139.0655153 8.2673118 1 7 4 +59.3726661 83.3857699 21788 1515 15 +-158.4718709 -12.2313178 1 17 5 +30.358316 -83.020501 2393 3871 12 +-169.9667863 71.1152107 1 13 6 +-89.9418297 -7.7258969 131156 273429 19 +-16.5050312 47.1843923 116 89 8 +-151.9641886 -36.3579042 5103 39881 16 +169.7406916 60.0950674 62 18 6 +32.6694543 -1.8435981 2 2 2 +97.7880811 77.5295169 50569 9674 16 +87.7554889 -85.6741368 389947 524287 19 +-19.0174909 29.1940122 0 0 1 +-86.6180019 82.6445919 1 0 2 +-50.1997802 -21.1913243 1476 2294 12 +73.8710121 75.2201385 5 1 3 +41.7434624 79.1410045 322937 65770 19 +-122.4312562 -68.4513916 10 48 6 +-54.6116448 -23.1761137 1 2 2 +-35.8774263 -51.5317442 102 170 8 +179.2976644 -76.2729885 127 107 7 +-111.6934402 -85.2696836 1 7 3 +-24.2137947 28.4102025 3 3 3 +-168.0816395 -64.0317696 16 375 9 +43.2514876 6.4266793 2540 1974 12 +25.5877932 49.3486828 4 2 3 +-133.3671657 -25.0795973 8 36 6 +-140.2002274 -37.3713396 452 2507 12 +73.6326557 -21.9898207 360 288 9 +-83.0901448 -69.1893461 2205 6305 13 +-32.83227 -11.1061854 3348 4350 13 +-151.8897244 14.6891958 0 0 0 +-118.1125151 82.3085937 704 288 12 +119.9903922 1.4884267 53 31 6 +-116.9455865 -48.1971821 0 2 2 +111.4279587 -52.6024326 828 688 10 +-78.9207774 28.6183104 147207 218615 19 +45.1367631 24.1416251 40 27 6 +115.0905685 7.2971256 6714 3929 13 +-58.0837371 -55.4033522 43 87 7 +-44.6785779 83.0751777 6158 877 14 +80.2002966 84.3087089 2960 91 12 +-167.3118039 -59.2698371 72 1445 11 +139.9974902 -74.5994693 455 419 9 +-27.5858357 -36.1890547 6936 9960 14 +68.8572424 20.2096749 353 226 9 +-168.7172825 69.1607462 2053 15104 16 +62.1361934 74.6007777 44079 11896 16 +-102.9645124 65.3735325 112191 135144 19 +178.8434887 18.6231394 65325 29316 16 +-138.0494072 -80.3786289 29 228 8 +33.3858934 62.9029909 155382 71699 18 +-1.6915643 74.737228 1 0 2 +18.2680756 6.6441482 1 0 1 +34.0511738 -28.6862516 2 2 2 +-24.0705994 -0.6026568 28386 32877 16 +-176.8765092 -14.3252022 568 35403 16 +154.6553417 -79.7390666 1903 1809 11 +151.6055263 59.552346 14 4 4 +-101.7681729 45.3498263 113933 187876 19 +-52.3703658 -89.8260406 181 511 9 +-85.7937259 16.5632413 2143 3713 13 +114.7030999 6.6846014 104 61 7 +13.6027861 88.6863877 2 0 2 +60.5726928 38.256536 43794 25219 16 +141.0903381 -35.4468007 7306 4959 13 +-38.8182454 -55.2332792 200 350 9 +179.7818018 -79.6600052 523970 462627 19 +-45.2102175 82.7381225 196301 32058 19 +-36.5811826 -35.3991346 0 0 0 +-26.7719575 -75.7014102 223154 435372 19 +77.3451937 -65.4613594 1 1 1 +-37.3286225 38.3250599 51945 50407 17 +70.8235495 35.3870419 365288 206979 19 +99.8381373 -83.4101655 1 1 1 +25.8851923 67.415543 9370 3991 14 +-12.3393758 68.5972027 7 3 4 +-142.9400273 -36.2904852 1 9 4 +110.085689 -50.9514972 422467 348655 19 +121.5093657 53.9939447 54888 21044 16 +151.1307841 -33.0798435 58 38 6 +100.4346499 -57.0129759 6 5 3 +114.0564882 63.4987339 428250 141474 19 +-17.3978586 19.2981593 57 57 7 +-129.101039 -66.3423574 579 3067 12 +117.6639917 14.0568892 433504 241463 19 +97.7014577 69.1261732 202216 60490 18 +-174.7931333 81.4174709 7583 46045 19 +47.205341 -48.8608019 20680 21495 15 +167.416796 -7.5063098 247 133 8 +63.6744589 21.3392888 0 0 0 +-106.0719817 33.8353547 53832 104862 18 +-48.4051268 27.0438152 191648 221209 19 +64.5675545 -27.0683253 5 4 3 +110.7800249 -35.8217841 0 0 0 +164.8954138 81.9558801 31393 2538 15 +-54.4595565 -23.0128432 357 579 10 +43.2542709 67.694011 2540 989 12 +137.6537233 -78.7052092 28913 28450 15 +76.9271563 -60.0777348 0 0 0 +-145.0240216 47.3869213 6367 22947 16 +136.2328853 -73.4534547 3598 3304 12 +48.8047148 74.3001023 81 23 7 +-144.6978756 33.6270048 200 820 11 +64.6723407 -37.196427 21 19 5 +-58.3356482 -19.0812366 22148 36307 16 +19.611535 -31.3013119 0 0 0 +-63.8923439 -25.9560287 1 2 2 +-152.5288047 -30.9747477 0 1 1 +-99.8509683 -69.0258965 7295 25181 15 +-22.9015207 -21.0915082 13 17 5 +161.4402438 61.1430241 15539 4652 14 +-102.9499976 -1.2629414 1 4 3 +-159.7995801 51.1570142 14 85 8 +10.5265485 -86.8526122 67 127 7 +47.1581041 -73.1628608 646 823 10 +2.7577906 -66.3257104 66540 98133 17 +96.9653593 -9.8258675 6 4 3 +-55.4199846 25.3325116 2834 3499 13 +-47.3590106 59.5085133 2 2 3 +179.1441501 22.1324479 7 3 3 +16.2649007 47.8480398 4 2 3 +-27.8373963 -0.3926597 27700 32839 16 +-99.7938802 -48.8013068 912 2685 12 +133.4858024 28.457146 3 1 2 +-174.7621207 -25.8318084 238 9409 14 +106.856378 19.640472 203 113 8 +31.4361995 -80.2488406 76981 116886 17 +148.6539554 -70.0347611 116 99 7 +45.1821773 17.5961595 80 57 7 +-30.3630114 71.3238998 54481 27877 17 +106.547704 44.9731358 6520 2947 13 +-132.8057564 23.3928795 67 221 9 +116.6455816 -10.1097592 52 33 6 +172.5512559 -53.2307289 3 2 2 +-114.2084519 42.834332 2994 6030 14 +91.0172087 87.6025 1 0 1 +-101.666812 -84.7858729 7130 32495 15 +-14.8074931 68.3277393 0 0 0 +140.3354277 -7.8709618 14 8 4 +-130.4961987 -69.4777523 9011 50594 16 +-121.239479 1.0896367 1 3 3 +-141.2241052 3.4495348 56471 257117 19 +42.6041377 -19.1328846 2532 2269 12 +141.3750885 -1.8110503 468036 264781 19 +-26.545021 -30.9641274 218 302 9 +87.6154866 10.1978751 5 3 3 +88.002559 -33.4108714 762 612 10 +170.9910642 -7.1925019 3993 2130 12 +-66.5680487 -88.1144993 165197 524287 19 +-71.5925378 45.720316 0 0 0 +-16.457642 57.1688038 930 625 11 +20.8379624 -11.1485568 35 33 6 +-72.9399538 -16.6504502 1 2 2 +52.8746845 -71.5213577 10598 12927 14 +93.2392918 12.5073156 48 29 6 +-3.7803834 73.7748237 128319 49794 18 +93.5713795 -38.8308882 24 19 5 +87.3474619 -6.9970464 23 16 5 +161.2199467 48.5612779 30 11 5 +87.343969 -31.3153618 47 37 6 +60.5318627 58.0869948 42 19 6 +161.4388458 42.7612385 1 0 1 +14.1262999 -26.2101142 4 4 3 +-135.776346 -21.4248586 503 2297 12 +-100.3980364 82.0236388 1 0 3 +-55.8093833 -87.2455194 0 0 0 +-15.6397352 64.454076 935 540 11 +131.5974423 39.1251372 110 48 7 +56.8045509 27.5658629 168 107 8 +146.3634996 14.9287416 3 1 2 +51.5699041 74.0370231 82 23 7 +29.3100901 33.7208834 152414 104963 18 +19.0622442 1.6781772 141 126 8 +-22.2575472 73.6467471 114864 50126 18 +125.0208495 47.8533914 3470 1426 12 +-92.4804503 29.8409387 995 1691 12 +113.6362202 86.1158625 0 0 0 +113.6203756 28.4267778 26 13 5 +124.472202 25.8347062 13856 6974 14 +79.3654306 -33.7864728 46 38 6 +-121.169233 -66.7642843 2 12 4 +179.5816311 1.0182064 0 0 0 +179.1123794 53.367624 0 0 0 +-25.6611689 32.6575586 14048 13236 15 +92.3370554 58.8306651 12 4 4 +142.2756448 63.6603101 3 1 2 +-162.8914282 -84.54406 6229 129034 17 +-54.0759935 53.1086102 0 0 0 +9.1647045 -39.4876873 34436 40604 16 +59.3708822 -43.9789068 10894 10425 14 +-16.6397572 46.5985252 929 723 11 +-36.1053642 -67.6882841 0 1 1 +-82.5851985 -69.5277766 4 12 4 +117.0059969 -63.2697116 432546 382068 19 +127.9740251 85.5341901 112129 0 17 +-13.9255265 59.7720207 120931 76457 18 +-167.7778382 41.7974194 69 761 11 +-100.1217856 -70.5624949 454 1599 11 +-42.4790343 -2.0016191 25034 33132 16 +-8.0782317 65.9680074 30 16 6 +-37.0481466 -55.2856125 203 350 9 +149.074382 16.7121888 58 28 6 +-53.1010518 -15.8492068 0 0 0 +50.8517919 -30.8366257 42025 38674 16 +-175.0355994 77.1929422 0 4 5 +-33.3221217 72.5636809 6 3 4 +-139.9201802 52.3400823 7296 21546 16 +86.0299659 23.4535286 1513 886 11 +105.6297238 -50.5521342 25998 21733 15 +104.2700533 36.5417507 206999 102450 18 +58.5238253 84.0320186 86843 3912 17 +-121.9177826 65.4393267 1321 2108 13 +-152.689211 57.4112922 39774 159515 19 +94.2583831 80.8625455 6240 801 13 +118.8199874 -37.6116567 53 39 6 +5.787144 7.0599251 4227 3934 13 +152.1682213 11.778732 236 119 8 +37.4503636 64.311498 2474 1084 12 +35.2616139 -59.8196291 38 45 6 +-20.2808572 -26.9983008 29075 37875 16 +-88.8541607 -20.4896703 66370 146320 18 +125.7726709 41.2603793 27 11 5 +-55.4322696 22.1767236 11338 14313 15 +-38.2393439 -56.4681037 6 11 4 +-139.4517643 -81.3621632 3 29 5 +-146.7954207 -74.6050826 6044 53642 16 +161.6654898 -71.7313805 15549 12957 14 +-85.4720514 -73.177675 2 6 3 +-25.7457381 -42.9842376 13 20 5 +-2.0766311 51.0142455 0 0 1 +-82.9179334 1.4865326 8836 16248 15 +140.5661302 61.5056993 28 9 5 +-30.2614099 35.5786378 54518 51659 17 +-49.2072142 -38.6965571 0 1 1 +124.6587534 9.5039576 433 242 9 +-113.4516741 81.4585593 24229 11410 17 +-4.5134723 68.229217 3 1 3 +75.4838529 -44.997406 2906 2622 12 +6.4715239 28.6900832 8486 6828 14 +91.9187555 -24.8192643 6187 4679 13 +-28.2510181 -42.6238777 26 40 6 +-151.1042237 -21.4553189 2630 18384 15 +-149.6272551 0.4235911 1382 8172 14 +-51.9171488 74.630423 0 0 0 +-36.5660117 37.242858 101 99 8 +91.7136677 -30.9077617 772 604 10 +61.6846009 -85.9378164 5 7 3 +11.3772008 69.40183 34839 14980 16 +82.6825938 9.4496443 11954 7759 14 +61.8446231 -40.0114106 2751 2545 12 +-51.6223196 -11.7880324 5 8 4 +113.0309076 -73.4376173 13336 13217 14 +-169.3808275 -72.7209175 0 25 5 +-98.5653414 -80.0893122 231 910 10 +-20.4653707 29.9801495 3 3 3 +78.9156686 2.599349 0 0 0 +76.0635255 -71.0823347 2913 3216 12 +-26.1185551 22.3616029 28013 28589 16 +177.8803853 -56.3662368 4071 2828 12 +-157.7926463 78.4998022 15 34 8 +168.6834344 -34.5535211 7934 4934 13 +-77.208013 -44.0964079 0 1 1 +-56.6162078 28.1240365 10 13 5 +8.6548899 72.178831 137374 53767 18 +-27.9342497 8.2525327 1 1 2 +91.6356971 -13.5230128 6181 4406 13 +-161.9980398 -75.4443511 0 13 4 +46.8556576 -27.1078679 5162 4737 13 +147.2806954 -48.1491071 465 334 9 +-168.2679875 -29.0171568 0 2 2 +10.0251187 -3.144812 8 8 4 +109.0281873 81.9713348 26307 2528 15 +-129.6281276 -36.9614028 73359 320148 19 +7.3831244 -18.3270273 2132 2260 12 +-34.4625217 53.2837646 52988 42524 17 +129.8855275 -26.30807 3525 2358 12 +-69.6374195 -45.7769025 10045 21081 15 +59.9868336 50.3716565 43688 22120 16 +126.4653338 -75.607391 1743 1698 11 +-34.6459616 53.2502443 51 41 7 +152.2415169 -71.7528837 1 1 1 +-83.2126752 32.6685119 35239 52939 17 +178.6388805 70.5727365 31 7 5 +-153.5646223 -81.9491561 2 29 5 +178.2668159 11.8222247 0 0 0 +-27.136902 18.4287365 6 7 4 +-104.7744923 -64.7769211 54777 193540 18 +58.2318889 9.5897974 2710 1938 12 +138.9707487 -65.2734433 14516 12149 14 +-115.29331 -84.3402223 2944 16033 14 +-66.3487123 -81.8340211 20 58 6 +76.1802855 40.4007156 364 193 9 +-77.5026834 -30.653231 145 301 9 +-52.6095007 -79.7155889 11595 28942 15 +-2.5594899 -31.6276806 7 9 4 +-58.6267217 41.4851391 2 2 3 +-0.7245205 -70.7329433 509 801 10 +161.2822996 -79.0311957 124257 114418 17 +144.8720197 80.1059269 14785 1811 14 +159.3764075 -23.5100054 0 0 0 +82.3109493 -7.0673699 95504 68115 17 +94.3931949 63.0474034 97 34 7 +87.4523391 -73.2690527 3043 3297 12 +101.4256455 -0.8267694 12 8 4 +-112.7666152 -82.3670281 12239 61007 16 +-82.0948447 -38.0810449 8 19 5 +113.2906728 -19.193243 104 70 7 +16.953131 78.402684 35 8 6 +-81.7974685 -53.9596787 34 86 7 +69.8051889 58.1416894 181902 78760 18 +-9.2435464 -53.4296594 3 5 3 +30.0415066 11.4884737 4 3 3 +125.2704157 -69.6324197 54 49 6 +-41.2060435 63.8501787 789 548 11 +120.407662 -4.9889504 6835 4209 13 +92.0345558 -8.0809262 24761 17121 15 +127.1061722 83.1311204 6988 428 13 +-178.9414629 64.0086678 770 69897 18 +49.0743035 -4.3000419 10425 8387 14 +-45.109002 -55.1435498 1534 2803 12 +3.2795498 75.95918 32 10 6 +)"; + diff --git a/test/t/geom/test_wkt.cpp b/test/t/geom/test_wkt.cpp index ff1417c72e9..53830740938 100644 --- a/test/t/geom/test_wkt.cpp +++ b/test/t/geom/test_wkt.cpp @@ -74,6 +74,14 @@ SECTION("linestring_with_two_same_locations") { }); REQUIRE_THROWS_AS(factory.create_linestring(wnl), osmium::geometry_error); + + try { + factory.create_linestring(wnl); + } catch (osmium::geometry_error& e) { + REQUIRE(e.id() == 0); + REQUIRE(std::string(e.what()) == "need at least two points for linestring"); + } + REQUIRE_THROWS_AS(factory.create_linestring(wnl, osmium::geom::use_nodes::unique, osmium::geom::direction::backward), osmium::geometry_error); { diff --git a/test/t/index/test_typed_mmap.cpp b/test/t/index/test_typed_mmap.cpp deleted file mode 100644 index bcc17bd2d24..00000000000 --- a/test/t/index/test_typed_mmap.cpp +++ /dev/null @@ -1,76 +0,0 @@ -#include "catch.hpp" - -#include - -#if defined(_MSC_VER) || (defined(__GNUC__) && defined(_WIN32)) -#include "win_mkstemp.hpp" -#endif - -TEST_CASE("TypedMmap") { - - SECTION("Mmap") { - uint64_t* data = osmium::detail::typed_mmap::map(10); - - data[0] = 4ul; - data[3] = 9ul; - data[9] = 25ul; - - REQUIRE(4ul == data[0]); - REQUIRE(9ul == data[3]); - REQUIRE(25ul == data[9]); - - osmium::detail::typed_mmap::unmap(data, 10); - } - - SECTION("MmapSizeZero") { - REQUIRE_THROWS_AS(osmium::detail::typed_mmap::map(0), std::system_error); - } - - SECTION("MmapHugeSize") { - // this is a horrible hack to only run the test on 64bit machines. - if (sizeof(size_t) >= 8) { - REQUIRE_THROWS_AS(osmium::detail::typed_mmap::map(1ULL << (sizeof(size_t) * 6)), std::system_error); - } - } - -#ifdef __linux__ - SECTION("Remap") { - uint64_t* data = osmium::detail::typed_mmap::map(10); - - data[0] = 4ul; - data[3] = 9ul; - data[9] = 25ul; - - uint64_t* new_data = osmium::detail::typed_mmap::remap(data, 10, 1000); - - REQUIRE(4ul == new_data[0]); - REQUIRE(9ul == new_data[3]); - REQUIRE(25ul == new_data[9]); - } -#else -# pragma message("not running 'Remap' test case on this machine") -#endif - - SECTION("FileSize") { - const int size = 100; - char filename[] = "test_mmap_file_size_XXXXXX"; - const int fd = mkstemp(filename); - REQUIRE(fd > 0); - REQUIRE(0 == osmium::detail::typed_mmap::file_size(fd)); - REQUIRE(0 == ftruncate(fd, size * sizeof(uint64_t))); - REQUIRE(size == osmium::detail::typed_mmap::file_size(fd)); - - osmium::detail::typed_mmap::grow_file(size / 2, fd); - REQUIRE(size == osmium::detail::typed_mmap::file_size(fd)); - - osmium::detail::typed_mmap::grow_file(size, fd); - REQUIRE(size == osmium::detail::typed_mmap::file_size(fd)); - - osmium::detail::typed_mmap::grow_file(size * 2, fd); - REQUIRE((size * 2) == osmium::detail::typed_mmap::file_size(fd)); - - REQUIRE(0 == close(fd)); - REQUIRE(0 == unlink(filename)); - } - -} diff --git a/test/t/index/test_typed_mmap_grow.cpp b/test/t/index/test_typed_mmap_grow.cpp deleted file mode 100644 index 92ee0b40ea4..00000000000 --- a/test/t/index/test_typed_mmap_grow.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "catch.hpp" - -#include - -#if defined(_MSC_VER) || (defined(__GNUC__) && defined(_WIN32)) -#include "win_mkstemp.hpp" -#endif - -TEST_CASE("TypedMmapGrow") { - - SECTION("GrowAndMap") { - const int size = 100; - char filename[] = "test_mmap_grow_and_map_XXXXXX"; - const int fd = mkstemp(filename); - REQUIRE(fd > 0); - - uint64_t* data = osmium::detail::typed_mmap::grow_and_map(size, fd); - REQUIRE(size == osmium::detail::typed_mmap::file_size(fd)); - - data[0] = 1ul; - data[1] = 8ul; - data[99] = 27ul; - - REQUIRE(1ul == data[0]); - REQUIRE(8ul == data[1]); - REQUIRE(27ul == data[99]); - - osmium::detail::typed_mmap::unmap(data, size); - - REQUIRE(0 == close(fd)); - REQUIRE(0 == unlink(filename)); - } - -} diff --git a/test/t/io/test_file_formats.cpp b/test/t/io/test_file_formats.cpp index e8785d6a411..f0ba0c66f9d 100644 --- a/test/t/io/test_file_formats.cpp +++ b/test/t/io/test_file_formats.cpp @@ -247,5 +247,29 @@ TEST_CASE("FileFormats") { REQUIRE_THROWS_AS(f.check(), std::runtime_error); } + SECTION("url without format") { + osmium::io::File f {"http://www.example.com/api"}; + REQUIRE(osmium::io::file_format::xml == f.format()); + REQUIRE(osmium::io::file_compression::none == f.compression()); + REQUIRE(false == f.has_multiple_object_versions()); + f.check(); + } + + SECTION("url without format and filename") { + osmium::io::File f {"http://planet.osm.org/pbf/planet-latest.osm.pbf"}; + REQUIRE(osmium::io::file_format::pbf == f.format()); + REQUIRE(osmium::io::file_compression::none == f.compression()); + REQUIRE(false == f.has_multiple_object_versions()); + f.check(); + } + + SECTION("url with format") { + osmium::io::File f {"http://www.example.com/api", "osh"}; + REQUIRE(osmium::io::file_format::xml == f.format()); + REQUIRE(osmium::io::file_compression::none == f.compression()); + REQUIRE(true == f.has_multiple_object_versions()); + f.check(); + } + } diff --git a/test/t/io/test_string_table.cpp b/test/t/io/test_string_table.cpp new file mode 100644 index 00000000000..7fedfcfcac0 --- /dev/null +++ b/test/t/io/test_string_table.cpp @@ -0,0 +1,94 @@ +#include "catch.hpp" + +#include + +TEST_CASE("String store") { + osmium::io::detail::StringStore ss(100); + + SECTION("empty") { + REQUIRE(ss.begin() == ss.end()); + } + + SECTION("add zero-length string") { + const char* s1 = ss.add(""); + REQUIRE(std::string(s1) == ""); + + auto it = ss.begin(); + REQUIRE(s1 == *it); + REQUIRE(std::string(*it) == ""); + REQUIRE(++it == ss.end()); + } + + SECTION("add strings") { + const char* s1 = ss.add("foo"); + const char* s2 = ss.add("bar"); + REQUIRE(s1 != s2); + REQUIRE(std::string(s1) == "foo"); + REQUIRE(std::string(s2) == "bar"); + + auto it = ss.begin(); + REQUIRE(s1 == *it++); + REQUIRE(s2 == *it++); + REQUIRE(it == ss.end()); + } + + SECTION("add zero-length string and longer strings") { + const char* s1 = ss.add(""); + const char* s2 = ss.add("xxx"); + const char* s3 = ss.add("yyyyy"); + + auto it = ss.begin(); + REQUIRE(std::string(*it++) == ""); + REQUIRE(std::string(*it++) == "xxx"); + REQUIRE(std::string(*it++) == "yyyyy"); + REQUIRE(it == ss.end()); + } + + SECTION("add many strings") { + for (const char* teststring : {"a", "abc", "abcd", "abcde"}) { + int i = 0; + for (; i < 100; ++i) { + ss.add(teststring); + } + + for (const char* s : ss) { + REQUIRE(std::string(s) == teststring); + --i; + } + + REQUIRE(i == 0); + ss.clear(); + } + } + +} + +TEST_CASE("String table") { + osmium::io::detail::StringTable st; + + SECTION("empty") { + REQUIRE(st.size() == 1); + REQUIRE(std::next(st.begin()) == st.end()); + } + + SECTION("add strings") { + REQUIRE(st.add("foo") == 1); + REQUIRE(st.add("bar") == 2); + REQUIRE(st.add("bar") == 2); + REQUIRE(st.add("baz") == 3); + REQUIRE(st.add("foo") == 1); + REQUIRE(st.size() == 4); + + auto it = st.begin(); + REQUIRE(std::string("") == *it++); + REQUIRE(std::string("foo") == *it++); + REQUIRE(std::string("bar") == *it++); + REQUIRE(std::string("baz") == *it++); + REQUIRE(it == st.end()); + + st.clear(); + REQUIRE(st.size() == 1); + } + +} + diff --git a/test/t/tags/test_tag_list.cpp b/test/t/tags/test_tag_list.cpp index c2512d12510..77523e7965d 100644 --- a/test/t/tags/test_tag_list.cpp +++ b/test/t/tags/test_tag_list.cpp @@ -4,73 +4,99 @@ #include #include -TEST_CASE("tag_list") { +TEST_CASE("create tag list") { + osmium::memory::Buffer buffer(10240); + + SECTION("with TagListBuilder from char*") { + { + osmium::builder::TagListBuilder builder(buffer); + builder.add_tag("highway", "primary"); + builder.add_tag("name", "Main Street"); + } + buffer.commit(); + } - SECTION("can_be_created_from_initializer_list") { - osmium::memory::Buffer buffer(10240); + SECTION("with TagListBuilder from char* with length") { + { + osmium::builder::TagListBuilder builder(buffer); + builder.add_tag("highway", strlen("highway"), "primary", strlen("primary")); + builder.add_tag("nameXX", 4, "Main Street", 11); + } + buffer.commit(); + } - const osmium::TagList& tl = osmium::builder::build_tag_list(buffer, { + SECTION("with TagListBuilder from std::string") { + { + osmium::builder::TagListBuilder builder(buffer); + builder.add_tag(std::string("highway"), std::string("primary")); + const std::string source = "name"; + std::string gps = "Main Street"; + builder.add_tag(source, gps); + } + buffer.commit(); + } + + SECTION("with build_tag_list from initializer list") { + osmium::builder::build_tag_list(buffer, { { "highway", "primary" }, - { "name", "Main Street" }, - { "source", "GPS" } + { "name", "Main Street" } }); - - REQUIRE(osmium::item_type::tag_list == tl.type()); - REQUIRE(3 == tl.size()); - REQUIRE(std::string("highway") == tl.begin()->key()); - REQUIRE(std::string("primary") == tl.begin()->value()); } - SECTION("can_be_created_from_map") { - osmium::memory::Buffer buffer(10240); - - const osmium::TagList& tl = osmium::builder::build_tag_list_from_map(buffer, std::map({ + SECTION("with build_tag_list_from_map") { + osmium::builder::build_tag_list_from_map(buffer, std::map({ { "highway", "primary" }, { "name", "Main Street" } })); - - REQUIRE(osmium::item_type::tag_list == tl.type()); - REQUIRE(2 == tl.size()); - - if (std::string("highway") == tl.begin()->key()) { - REQUIRE(std::string("primary") == tl.begin()->value()); - REQUIRE(std::string("name") == std::next(tl.begin(), 1)->key()); - REQUIRE(std::string("Main Street") == std::next(tl.begin(), 1)->value()); - } else { - REQUIRE(std::string("highway") == std::next(tl.begin(), 1)->key()); - REQUIRE(std::string("primary") == std::next(tl.begin(), 1)->value()); - REQUIRE(std::string("name") == tl.begin()->key()); - REQUIRE(std::string("Main Street") == tl.begin()->value()); - } } - SECTION("can_be_created_with_callback") { - osmium::memory::Buffer buffer(10240); - - const osmium::TagList& tl = osmium::builder::build_tag_list_from_func(buffer, [](osmium::builder::TagListBuilder& tlb) { + SECTION("with build_tag_list_from_func") { + osmium::builder::build_tag_list_from_func(buffer, [](osmium::builder::TagListBuilder& tlb) { tlb.add_tag("highway", "primary"); - tlb.add_tag("bridge", "true"); + tlb.add_tag("name", "Main Street"); }); - - REQUIRE(osmium::item_type::tag_list == tl.type()); - REQUIRE(2 == tl.size()); - REQUIRE(std::string("bridge") == std::next(tl.begin(), 1)->key()); - REQUIRE(std::string("true") == std::next(tl.begin(), 1)->value()); } - SECTION("returns_value_by_key") { - osmium::memory::Buffer buffer(10240); + const osmium::TagList& tl = *buffer.begin(); + REQUIRE(osmium::item_type::tag_list == tl.type()); + REQUIRE(2 == tl.size()); - const osmium::TagList& tl = osmium::builder::build_tag_list_from_func(buffer, [](osmium::builder::TagListBuilder& tlb) { - tlb.add_tag("highway", "primary"); - tlb.add_tag("bridge", "true"); - }); + auto it = tl.begin(); + REQUIRE(std::string("highway") == it->key()); + REQUIRE(std::string("primary") == it->value()); + ++it; + REQUIRE(std::string("name") == it->key()); + REQUIRE(std::string("Main Street") == it->value()); + ++it; + REQUIRE(it == tl.end()); - REQUIRE(std::string("primary") == tl.get_value_by_key("highway")); - REQUIRE(nullptr == tl.get_value_by_key("name")); - REQUIRE(std::string("foo") == tl.get_value_by_key("name", "foo")); + REQUIRE(std::string("primary") == tl.get_value_by_key("highway")); + REQUIRE(nullptr == tl.get_value_by_key("foo")); + REQUIRE(std::string("default") == tl.get_value_by_key("foo", "default")); - REQUIRE(std::string("true") == tl["bridge"]); - } + REQUIRE(std::string("Main Street") == tl["name"]); +} + +TEST_CASE("empty keys and values are okay") { + osmium::memory::Buffer buffer(10240); + + const osmium::TagList& tl = osmium::builder::build_tag_list(buffer, { + { "empty value", "" }, + { "", "empty key" } + }); + + REQUIRE(osmium::item_type::tag_list == tl.type()); + REQUIRE(2 == tl.size()); + + auto it = tl.begin(); + REQUIRE(std::string("empty value") == it->key()); + REQUIRE(std::string("") == it->value()); + ++it; + REQUIRE(std::string("") == it->key()); + REQUIRE(std::string("empty key") == it->value()); + ++it; + REQUIRE(it == tl.end()); + REQUIRE(std::string("") == tl.get_value_by_key("empty value")); + REQUIRE(std::string("empty key") == tl.get_value_by_key("")); } diff --git a/test/t/thread/test_pool.cpp b/test/t/thread/test_pool.cpp index 66bb3734a8f..5fa6bbab5ae 100644 --- a/test/t/thread/test_pool.cpp +++ b/test/t/thread/test_pool.cpp @@ -59,7 +59,6 @@ TEST_CASE("thread") { auto& pool = osmium::thread::Pool::instance(); result = 0; - bool got_exception = false; auto future = pool.submit(test_job_throw {}); REQUIRE_THROWS_AS(future.get(), std::runtime_error); diff --git a/test/t/util/test_data_file.cpp b/test/t/util/test_data_file.cpp new file mode 100644 index 00000000000..3f432f94376 --- /dev/null +++ b/test/t/util/test_data_file.cpp @@ -0,0 +1,81 @@ +#include "catch.hpp" + +#include + +#include + +TEST_CASE("temporary file") { + + SECTION("create/open") { + osmium::util::DataFile file; + + REQUIRE(!!file); + int fd = file.fd(); + + REQUIRE(fd > 0); + + const char buf[] = "foobar"; + REQUIRE(::write(fd, buf, sizeof(buf)) == sizeof(buf)); + + file.close(); + + REQUIRE(!file); + } + +} + +TEST_CASE("named file") { + + SECTION("create/open") { + { + osmium::util::DataFile file("test.data", true); + + REQUIRE(!!file); + int fd = file.fd(); + + REQUIRE(fd > 0); + + REQUIRE(file.size() == 0); + + const char buf[] = "foobar"; + REQUIRE(::write(fd, buf, sizeof(buf) - 1) == sizeof(buf) - 1); + + file.close(); + + REQUIRE(!file); + } + { + osmium::util::DataFile file("test.data", false); + + REQUIRE(!!file); + int fd = file.fd(); + + REQUIRE(fd > 0); + + REQUIRE(file.size() == 6); + + char buf[10]; + int len = ::read(fd, buf, sizeof(buf)); + + REQUIRE(len == 6); + REQUIRE(!strncmp(buf, "foobar", 6)); + + file.close(); + + REQUIRE(!file); + REQUIRE(unlink("test.data") == 0); + } + } + + SECTION("grow file") { + osmium::util::DataFile file("test.data", true); + + REQUIRE(!!file); + + REQUIRE(file.size() == 0); + file.grow(10); + REQUIRE(file.size() == 10); + } + +} + diff --git a/test/t/util/test_delta.cpp b/test/t/util/test_delta.cpp new file mode 100644 index 00000000000..cebcca8ed06 --- /dev/null +++ b/test/t/util/test_delta.cpp @@ -0,0 +1,68 @@ +#include "catch.hpp" + +#include + +#include + +TEST_CASE("delta encode") { + + osmium::util::DeltaEncode x; + + SECTION("int") { + REQUIRE(x.update(17) == 17); + REQUIRE(x.update(10) == -7); + } + +} + +TEST_CASE("delta decode") { + + osmium::util::DeltaDecode x; + + SECTION("int") { + REQUIRE(x.update(17) == 17); + REQUIRE(x.update(10) == 27); + } + +} + +TEST_CASE("delta encode and decode") { + + std::vector a = { 5, -9, 22, 13, 0, 23 }; + + osmium::util::DeltaEncode de; + std::vector b; + for (int x : a) { + b.push_back(de.update(x)); + } + + osmium::util::DeltaDecode dd; + std::vector c; + for (int x : b) { + c.push_back(dd.update(x)); + } + +} + +TEST_CASE("delta encode iterator") { + std::vector data = { 4, 5, 13, 22, 12 }; + + auto l = [](std::vector::const_iterator it) -> int { + return *it; + }; + + typedef osmium::util::DeltaEncodeIterator::const_iterator, decltype(l), int> it_type; + it_type it(data.begin(), data.end(), l); + it_type end(data.end(), data.end(), l); + + REQUIRE(*it == 4); + ++it; + REQUIRE(*it++ == 1); + REQUIRE(*it == 8); + ++it; + REQUIRE(*it++ == 9); + REQUIRE(*it == -10); + ++it; + REQUIRE(it == end); +} + diff --git a/test/t/util/test_file.cpp b/test/t/util/test_file.cpp new file mode 100644 index 00000000000..2787261a54d --- /dev/null +++ b/test/t/util/test_file.cpp @@ -0,0 +1,69 @@ +#include "catch.hpp" + +#include + +#ifdef _WIN32 +// https://msdn.microsoft.com/en-us/library/ksazx244.aspx +// https://msdn.microsoft.com/en-us/library/a9yf33zb.aspx +class DoNothingInvalidParameterHandler { + + static void invalid_parameter_handler( + const wchar_t* expression, + const wchar_t* function, + const wchar_t* file, + unsigned int line, + uintptr_t pReserved + ) { + // do nothing + } + + _invalid_parameter_handler old_handler; + +public: + + DoNothingInvalidParameterHandler() : + old_handler(_set_invalid_parameter_handler(invalid_parameter_handler)) { + } + + ~DoNothingInvalidParameterHandler() { + _set_invalid_parameter_handler(old_handler); + } + +}; // class InvalidParameterHandler +#endif + + +TEST_CASE("file_size") { + +#ifdef _WIN32 + DoNothingInvalidParameterHandler handler; +#endif + + SECTION("illegal fd should throw") { + REQUIRE_THROWS_AS(osmium::util::file_size(-1), std::system_error); + } + + SECTION("unused fd should throw") { + // its unlikely that fd 1000 is open... + REQUIRE_THROWS_AS(osmium::util::file_size(1000), std::system_error); + } + +} + +TEST_CASE("resize_file") { + +#ifdef _WIN32 + DoNothingInvalidParameterHandler handler; +#endif + + SECTION("illegal fd should throw") { + REQUIRE_THROWS_AS(osmium::util::resize_file(-1, 10), std::system_error); + } + + SECTION("unused fd should throw") { + // its unlikely that fd 1000 is open... + REQUIRE_THROWS_AS(osmium::util::resize_file(1000, 10), std::system_error); + } + +} + diff --git a/test/t/util/test_memory_mapping.cpp b/test/t/util/test_memory_mapping.cpp new file mode 100644 index 00000000000..29893f7c70b --- /dev/null +++ b/test/t/util/test_memory_mapping.cpp @@ -0,0 +1,419 @@ +#include "catch.hpp" + +#include +#include + +#include +#include + +#if defined(_MSC_VER) || (defined(__GNUC__) && defined(_WIN32)) +#include "win_mkstemp.hpp" +#endif + +static const size_t huge = std::numeric_limits::max(); + +TEST_CASE("anonymous mapping") { + + SECTION("simple memory mapping should work") { + osmium::util::MemoryMapping mapping(1000, osmium::util::MemoryMapping::mapping_mode::write_private); + REQUIRE(mapping.get_addr() != nullptr); + + REQUIRE(mapping.size() >= 1000); + + volatile int* addr = mapping.get_addr(); + + REQUIRE(mapping.writable()); + + *addr = 42; + REQUIRE(*addr == 42); + + REQUIRE(!!mapping); + mapping.unmap(); + REQUIRE(!mapping); + mapping.unmap(); // second unmap is okay + } + + SECTION("memory mapping of zero length should work") { + osmium::util::MemoryMapping mapping(0, osmium::util::MemoryMapping::mapping_mode::write_private); + REQUIRE(mapping.get_addr() != nullptr); + + REQUIRE(mapping.size() == osmium::util::get_pagesize()); + + REQUIRE(!!mapping); + mapping.unmap(); + REQUIRE(!mapping); + } + + SECTION("moving a memory mapping should work") { + osmium::util::MemoryMapping mapping1(1000, osmium::util::MemoryMapping::mapping_mode::write_private); + int* addr1 = mapping1.get_addr(); + *addr1 = 42; + + REQUIRE(!!mapping1); + osmium::util::MemoryMapping mapping2(std::move(mapping1)); + REQUIRE(!!mapping2); + REQUIRE(!mapping1); + mapping1.unmap(); + + int* addr2 = mapping2.get_addr(); + REQUIRE(*addr2 == 42); + + mapping2.unmap(); + REQUIRE(!mapping2); + } + + SECTION("move assignment should work") { + osmium::util::MemoryMapping mapping1(1000, osmium::util::MemoryMapping::mapping_mode::write_private); + osmium::util::MemoryMapping mapping2(1000, osmium::util::MemoryMapping::mapping_mode::write_private); + + REQUIRE(!!mapping1); + REQUIRE(!!mapping2); + + int* addr1 = mapping1.get_addr(); + *addr1 = 42; + + mapping2 = std::move(mapping1); + REQUIRE(!!mapping2); + REQUIRE(!mapping1); + + int* addr2 = mapping2.get_addr(); + REQUIRE(*addr2 == 42); + + mapping2.unmap(); + REQUIRE(!mapping2); + } + +#ifdef __linux__ + SECTION("remapping to larger size should work") { + osmium::util::MemoryMapping mapping(1000, osmium::util::MemoryMapping::mapping_mode::write_private); + REQUIRE(mapping.size() >= 1000); + + size_t size1 = mapping.size(); + + int* addr1 = mapping.get_addr(); + *addr1 = 42; + + mapping.resize(8000); + REQUIRE(mapping.size() > size1); + + int* addr2 = mapping.get_addr(); + REQUIRE(*addr2 == 42); + } + + SECTION("remapping to smaller size should work") { + osmium::util::MemoryMapping mapping(8000, osmium::util::MemoryMapping::mapping_mode::write_private); + REQUIRE(mapping.size() >= 1000); + + size_t size1 = mapping.size(); + + int* addr1 = mapping.get_addr(); + *addr1 = 42; + + mapping.resize(500); + REQUIRE(mapping.size() < size1); + + int* addr2 = mapping.get_addr(); + REQUIRE(*addr2 == 42); + } +#endif + +} + +TEST_CASE("file-based mapping") { + + SECTION("writing to a mapped file should work") { + char filename[] = "test_mmap_write_XXXXXX"; + const int fd = mkstemp(filename); + REQUIRE(fd > 0); + + osmium::util::resize_file(fd, 100); + + { + osmium::util::MemoryMapping mapping(100, osmium::util::MemoryMapping::mapping_mode::write_shared, fd); + REQUIRE(mapping.writable()); + + REQUIRE(!!mapping); + REQUIRE(mapping.size() >= 100); + + *mapping.get_addr() = 1234; + + mapping.unmap(); + } + + REQUIRE(osmium::util::file_size(fd) == 100); + + { + osmium::util::MemoryMapping mapping(100, osmium::util::MemoryMapping::mapping_mode::readonly, fd); + REQUIRE(!mapping.writable()); + + REQUIRE(!!mapping); + REQUIRE(mapping.size() >= 100); + REQUIRE(*mapping.get_addr() == 1234); + + mapping.unmap(); + } + + REQUIRE(0 == close(fd)); + REQUIRE(0 == unlink(filename)); + } + + SECTION("writing to a privately mapped file should work") { + char filename[] = "test_mmap_write_XXXXXX"; + const int fd = mkstemp(filename); + REQUIRE(fd > 0); + + osmium::util::resize_file(fd, 100); + + { + osmium::util::MemoryMapping mapping(100, osmium::util::MemoryMapping::mapping_mode::write_private, fd); + REQUIRE(mapping.writable()); + + REQUIRE(!!mapping); + REQUIRE(mapping.size() >= 100); + + *mapping.get_addr() = 1234; + + mapping.unmap(); + } + + REQUIRE(osmium::util::file_size(fd) == 100); + + { + osmium::util::MemoryMapping mapping(100, osmium::util::MemoryMapping::mapping_mode::readonly, fd); + REQUIRE(!mapping.writable()); + + REQUIRE(!!mapping); + REQUIRE(mapping.size() >= 100); + REQUIRE(*mapping.get_addr() == 0); // should not see the value set above + + mapping.unmap(); + } + + REQUIRE(0 == close(fd)); + REQUIRE(0 == unlink(filename)); + } + + SECTION("remapping to larger size should work") { + char filename[] = "test_mmap_grow_XXXXXX"; + const int fd = mkstemp(filename); + REQUIRE(fd > 0); + + osmium::util::MemoryMapping mapping(100, osmium::util::MemoryMapping::mapping_mode::write_shared, fd); + REQUIRE(mapping.size() >= 100); + size_t size1 = mapping.size(); + + int* addr1 = mapping.get_addr(); + *addr1 = 42; + + mapping.resize(8000); + REQUIRE(mapping.size() >= 8000); + REQUIRE(mapping.size() > size1); + + int* addr2 = mapping.get_addr(); + REQUIRE(*addr2 == 42); + + mapping.unmap(); + + REQUIRE(0 == close(fd)); + REQUIRE(0 == unlink(filename)); + } + + SECTION("remapping to smaller size should work") { + char filename[] = "test_mmap_shrink_XXXXXX"; + const int fd = mkstemp(filename); + REQUIRE(fd > 0); + + { + osmium::util::MemoryMapping mapping(8000, osmium::util::MemoryMapping::mapping_mode::write_shared, fd); + REQUIRE(mapping.size() >= 8000); + size_t size1 = mapping.size(); + + int* addr1 = mapping.get_addr(); + *addr1 = 42; + + mapping.resize(50); + REQUIRE(mapping.size() >= 50); + REQUIRE(mapping.size() < size1); + + int* addr2 = mapping.get_addr(); + REQUIRE(*addr2 == 42); + } + + REQUIRE(0 == close(fd)); + REQUIRE(0 == unlink(filename)); + } +} + +TEST_CASE("typed anonymous mapping") { + + SECTION("simple memory mapping should work") { + osmium::util::TypedMemoryMapping mapping(1000); + volatile uint32_t* addr = mapping.begin(); + + REQUIRE(mapping.writable()); + + *addr = 42; + REQUIRE(*addr == 42); + + REQUIRE(!!mapping); + mapping.unmap(); + REQUIRE(!mapping); + mapping.unmap(); // second unmap is okay + } + + SECTION("moving a memory mapping should work") { + osmium::util::TypedMemoryMapping mapping1(1000); + uint32_t* addr1 = mapping1.begin(); + *addr1 = 42; + + REQUIRE(!!mapping1); + osmium::util::TypedMemoryMapping mapping2(std::move(mapping1)); + REQUIRE(!!mapping2); + REQUIRE(!mapping1); + mapping1.unmap(); + + auto addr2 = mapping2.begin(); + REQUIRE(*addr2 == 42); + + mapping2.unmap(); + REQUIRE(!mapping2); + } + + SECTION("move assignment should work") { + osmium::util::TypedMemoryMapping mapping1(1000); + osmium::util::TypedMemoryMapping mapping2(1000); + + REQUIRE(!!mapping1); + REQUIRE(!!mapping2); + + auto addr1 = mapping1.begin(); + *addr1 = 42; + + mapping2 = std::move(mapping1); + REQUIRE(!!mapping2); + REQUIRE(!mapping1); + + auto addr2 = mapping2.begin(); + REQUIRE(*addr2 == 42); + + mapping2.unmap(); + REQUIRE(!mapping2); + } + +#ifdef __linux__ + SECTION("remapping to larger size should work") { + osmium::util::TypedMemoryMapping mapping(1000); + REQUIRE(mapping.size() >= 1000); + + auto addr1 = mapping.begin(); + *addr1 = 42; + + mapping.resize(8000); + + auto addr2 = mapping.begin(); + REQUIRE(*addr2 == 42); + } + + SECTION("remapping to smaller size should work") { + osmium::util::TypedMemoryMapping mapping(8000); + REQUIRE(mapping.size() >= 8000); + + auto addr1 = mapping.begin(); + *addr1 = 42; + + mapping.resize(500); + + auto addr2 = mapping.begin(); + REQUIRE(*addr2 == 42); + } +#endif + +} + +TEST_CASE("typed file-based mapping") { + + SECTION("writing to a mapped file should work") { + char filename[] = "test_mmap_file_size_XXXXXX"; + const int fd = mkstemp(filename); + REQUIRE(fd > 0); + + osmium::util::resize_file(fd, 100); + + { + osmium::util::TypedMemoryMapping mapping(100, osmium::util::MemoryMapping::mapping_mode::write_shared, fd); + REQUIRE(mapping.writable()); + + REQUIRE(!!mapping); + REQUIRE(mapping.size() >= 100); + + *mapping.begin() = 1234; + + mapping.unmap(); + } + + { + osmium::util::TypedMemoryMapping mapping(100, osmium::util::MemoryMapping::mapping_mode::readonly, fd); + REQUIRE(!mapping.writable()); + + REQUIRE(!!mapping); + REQUIRE(mapping.size() >= 100); + REQUIRE(*mapping.begin() == 1234); + + mapping.unmap(); + } + + REQUIRE(0 == close(fd)); + REQUIRE(0 == unlink(filename)); + } + +} + +TEST_CASE("anonymous memory mapping class") { + + SECTION("simple memory mapping should work") { + osmium::util::AnonymousMemoryMapping mapping(1000); + REQUIRE(mapping.get_addr() != nullptr); + + volatile int* addr = mapping.get_addr(); + + REQUIRE(mapping.writable()); + + *addr = 42; + REQUIRE(*addr == 42); + + REQUIRE(!!mapping); + mapping.unmap(); + REQUIRE(!mapping); + mapping.unmap(); // second unmap is okay + } + +#ifdef __linux__ + SECTION("remapping to larger size should work") { + osmium::util::AnonymousMemoryMapping mapping(1000); + REQUIRE(mapping.size() >= 1000); + + int* addr1 = mapping.get_addr(); + *addr1 = 42; + + mapping.resize(2000); + + int* addr2 = mapping.get_addr(); + REQUIRE(*addr2 == 42); + } + + SECTION("remapping to smaller size should work") { + osmium::util::AnonymousMemoryMapping mapping(2000); + REQUIRE(mapping.size() >= 2000); + + int* addr1 = mapping.get_addr(); + *addr1 = 42; + + mapping.resize(500); + + int* addr2 = mapping.get_addr(); + REQUIRE(*addr2 == 42); + } +#endif + +} + diff --git a/test/t/util/test_minmax.cpp b/test/t/util/test_minmax.cpp new file mode 100644 index 00000000000..8b40f856e84 --- /dev/null +++ b/test/t/util/test_minmax.cpp @@ -0,0 +1,68 @@ +#include "catch.hpp" + +#include +#include + +TEST_CASE("minmax numeric") { + + SECTION("min") { + osmium::min_op x; + REQUIRE(x() == std::numeric_limits::max()); + + x.update(17); + REQUIRE(x() == 17); + + x.update(10); + REQUIRE(x() == 10); + + x.update(22); + REQUIRE(x() == 10); + } + + SECTION("max") { + osmium::max_op x; + REQUIRE(x() == 0); + + x.update(17); + REQUIRE(x() == 17); + + x.update(10); + REQUIRE(x() == 17); + + x.update(22); + REQUIRE(x() == 22); + } + +} + +TEST_CASE("minmax timestamp") { + + SECTION("min") { + osmium::min_op x; + + x.update(osmium::Timestamp("2010-01-01T00:00:00Z")); + REQUIRE(x().to_iso() == "2010-01-01T00:00:00Z"); + + x.update(osmium::Timestamp("2015-01-01T00:00:00Z")); + REQUIRE(x().to_iso() == "2010-01-01T00:00:00Z"); + + x.update(osmium::Timestamp("2000-01-01T00:00:00Z")); + REQUIRE(x().to_iso() == "2000-01-01T00:00:00Z"); + } + + SECTION("max") { + osmium::max_op x; + + x.update(osmium::Timestamp("2010-01-01T00:00:00Z")); + REQUIRE(x().to_iso() == "2010-01-01T00:00:00Z"); + + x.update(osmium::Timestamp("2015-01-01T00:00:00Z")); + REQUIRE(x().to_iso() == "2015-01-01T00:00:00Z"); + + x.update(osmium::Timestamp("2000-01-01T00:00:00Z")); + REQUIRE(x().to_iso() == "2015-01-01T00:00:00Z"); + + } + +} + diff --git a/test/t/util/test_string.cpp b/test/t/util/test_string.cpp index fa49787791f..0960afe9a3b 100644 --- a/test/t/util/test_string.cpp +++ b/test/t/util/test_string.cpp @@ -9,6 +9,7 @@ TEST_CASE("split_string") { std::vector result = {"foo", "baramba", "baz"}; REQUIRE(result == osmium::split_string(str, ',')); + REQUIRE(result == osmium::split_string(str, ',', true)); } SECTION("split_string string without sep") { @@ -16,34 +17,43 @@ TEST_CASE("split_string") { std::vector result = {"foo"}; REQUIRE(result == osmium::split_string(str, ',')); + REQUIRE(result == osmium::split_string(str, ',', true)); } SECTION("split_string string with empty at end") { std::string str { "foo,bar," }; std::vector result = {"foo", "bar", ""}; + std::vector resultc = {"foo", "bar"}; REQUIRE(result == osmium::split_string(str, ',')); + REQUIRE(resultc == osmium::split_string(str, ',', true)); } SECTION("split_string string with empty in middle") { std::string str { "foo,,bar" }; std::vector result = {"foo", "", "bar"}; + std::vector resultc = {"foo", "bar"}; REQUIRE(result == osmium::split_string(str, ',')); + REQUIRE(resultc == osmium::split_string(str, ',', true)); } SECTION("split_string string with empty at start") { std::string str { ",bar,baz" }; std::vector result = {"", "bar", "baz"}; + std::vector resultc = {"bar", "baz"}; REQUIRE(result == osmium::split_string(str, ',')); + REQUIRE(resultc == osmium::split_string(str, ',', true)); } SECTION("split_string sep") { std::string str { "," }; std::vector result = {"", ""}; + std::vector resultc; REQUIRE(result == osmium::split_string(str, ',')); + REQUIRE(resultc == osmium::split_string(str, ',', true)); } SECTION("split_string empty string") { @@ -51,6 +61,7 @@ TEST_CASE("split_string") { std::vector result; REQUIRE(result == osmium::split_string(str, ',')); + REQUIRE(result == osmium::split_string(str, ',', true)); } }