From 2d0b2ca98749021e3fb738b16ddd087c21f96221 Mon Sep 17 00:00:00 2001 From: Denis Chaplygin Date: Wed, 17 Jun 2020 13:16:21 +0300 Subject: [PATCH] Removed STXXL support. --- .gitignore | 6 --- .travis.yml | 9 ---- CHANGELOG.md | 3 +- CMakeLists.txt | 32 ------------ cmake/FindSTXXL.cmake | 51 ------------------- features/support/env.js | 7 +-- include/extractor/internal_extractor_edge.hpp | 27 ---------- include/storage/serialization.hpp | 22 -------- include/util/meminfo.hpp | 18 ------- include/util/vector_view.hpp | 12 +---- scripts/travis/before_install.armhf.sh | 8 --- scripts/travis/before_install.i686.sh | 2 +- src/tools/contract.cpp | 3 -- src/tools/extract.cpp | 4 -- test/.stxxl | 1 - test/data/.stxxl | 1 - 16 files changed, 5 insertions(+), 201 deletions(-) delete mode 100644 cmake/FindSTXXL.cmake delete mode 100644 test/.stxxl delete mode 100644 test/data/.stxxl diff --git a/.gitignore b/.gitignore index 9fca65f0c0f..21bf08310cb 100644 --- a/.gitignore +++ b/.gitignore @@ -72,12 +72,6 @@ Thumbs.db ########################### .idea/ -# stxxl related files # -####################### -.stxxl -stxxl.log -stxxl.errlog - # Compiled Binary Files # #################################### /osrm-extract diff --git a/.travis.yml b/.travis.yml index 69530f94cd7..8a62697ae71 100644 --- a/.travis.yml +++ b/.travis.yml @@ -151,14 +151,6 @@ matrix: TARGET_ARCH='i686' CCOMPILER='gcc-7' CXXCOMPILER='g++-7' BUILD_TYPE='Release' CFLAGS='-m32 -msse2 -mfpmath=sse' CXXFLAGS='-m32 -msse2 -mfpmath=sse' - - os: linux - compiler: "gcc-7-stxxl" - addons: &gcc7 - apt: - sources: ['ubuntu-toolchain-r-test'] - packages: ['g++-7', 'libbz2-dev', 'libstxxl-dev', 'libxml2-dev', 'libzip-dev', 'liblua5.2-dev', 'libtbb-dev', 'libboost-all-dev'] - env: CCOMPILER='gcc-7' CXXCOMPILER='g++-7' BUILD_TYPE='Release' ENABLE_STXXL=On - - os: linux compiler: "gcc-5-release" addons: &gcc49 @@ -483,7 +475,6 @@ install: -DENABLE_COVERAGE=${ENABLE_COVERAGE:-OFF} \ -DENABLE_NODE_BINDINGS=${ENABLE_NODE_BINDINGS:-OFF} \ -DENABLE_SANITIZER=${ENABLE_SANITIZER:-OFF} \ - -DENABLE_STXXL=${ENABLE_STXXL:-OFF} \ -DBUILD_TOOLS=ON \ -DENABLE_CCACHE=ON \ -DCMAKE_INSTALL_PREFIX=${OSRM_INSTALL_DIR} \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 60d63db66de..3a73315d641 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,8 @@ - Misc: - CHANGED: Reduce memory usage for raster source handling. [#5572](https://github.com/Project-OSRM/osrm-backend/pull/5572) - CHANGED: Add cmake option `ENABLE_DEBUG_LOGGING` to control whether output debug logging. [#3427](https://github.com/Project-OSRM/osrm-backend/issues/3427) - + - Infrastructure + - REMOVED: STXXL support removed as STXXL became abandonware. [#5760](https://github.com/Project-OSRM/osrm-backend/pull/5760) # 5.21.0 - Changes from 5.20.0 - Features: diff --git a/CMakeLists.txt b/CMakeLists.txt index 9425cf55131..e6feb6de9aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,6 @@ option(ENABLE_ASSERTIONS "Use assertions in release mode" OFF) option(ENABLE_DEBUG_LOGGING "Use debug logging in release mode" OFF) option(ENABLE_COVERAGE "Build with coverage instrumentalisation" OFF) option(ENABLE_SANITIZER "Use memory sanitizer for Debug build" OFF) -option(ENABLE_STXXL "Use STXXL library" OFF) option(ENABLE_LTO "Use LTO if available" OFF) option(ENABLE_FUZZING "Fuzz testing using LLVM's libFuzzer" OFF) option(ENABLE_GOLD_LINKER "Use GNU gold linker if available" ON) @@ -38,7 +37,6 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") if(ENABLE_MASON) # versions in use set(MASON_BOOST_VERSION "1.65.1") - set(MASON_STXXL_VERSION "1.4.1-1") set(MASON_EXPAT_VERSION "2.2.0") set(MASON_LUA_VERSION "5.2.4") set(MASON_BZIP2_VERSION "1.0.6") @@ -480,13 +478,6 @@ if(ENABLE_MASON) mason_use(boost_libsystem VERSION ${MASON_BOOST_VERSION}) set(Boost_SYSTEM_LIBRARY ${MASON_PACKAGE_boost_libsystem_STATIC_LIBS}) - if (ENABLE_STXXL) - mason_use(stxxl VERSION ${MASON_STXXL_VERSION}) - add_dependency_includes(${MASON_PACKAGE_stxxl_INCLUDE_DIRS}) - set(MAYBE_STXXL_LIBRARY ${MASON_PACKAGE_stxxl_STATIC_LIBS}) - add_definitions(-DUSE_STXXL_LIBRARY) - endif() - mason_use(expat VERSION ${MASON_EXPAT_VERSION}) add_dependency_includes(${MASON_PACKAGE_expat_INCLUDE_DIRS}) set(EXPAT_LIBRARIES ${MASON_PACKAGE_expat_STATIC_LIBS}) @@ -542,17 +533,6 @@ else() find_package(EXPAT REQUIRED) add_dependency_includes(${EXPAT_INCLUDE_DIRS}) - if (ENABLE_STXXL) - find_package(STXXL) - if (STXXL_FOUND) - add_dependency_includes(${STXXL_INCLUDE_DIR}) - set(MAYBE_STXXL_LIBRARY ${STXXL_LIBRARY}) - add_definitions(-DUSE_STXXL_LIBRARY) - else() - MESSAGE(STATUS "STXXL was requested but not found, default STL will be used") - endif() - endif() - find_package(BZip2 REQUIRED) add_dependency_includes(${BZIP2_INCLUDE_DIR}) @@ -612,15 +592,6 @@ add_dependency_defines(-DBOOST_SPIRIT_USE_PHOENIX_V3) add_dependency_defines(-DBOOST_RESULT_OF_USE_DECLTYPE) add_dependency_defines(-DBOOST_FILESYSTEM_NO_DEPRECATED) -if (ENABLE_STXXL) - set(OpenMP_FIND_QUIETLY ON) - find_package(OpenMP) - if(OPENMP_FOUND) - message(STATUS "OpenMP support found. Linking just in case for stxxl") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") - endif() -endif() - add_definitions(${OSRM_DEFINES}) include_directories(SYSTEM ${DEPENDENCIES_INCLUDE_DIRS}) @@ -653,7 +624,6 @@ set(EXTRACTOR_LIBRARIES ${EXPAT_LIBRARIES} ${USED_LUA_LIBRARIES} ${OSMIUM_LIBRARIES} - ${MAYBE_STXXL_LIBRARY} ${TBB_LIBRARIES} ${ZLIB_LIBRARY} ${MAYBE_COVERAGE_LIBRARIES}) @@ -687,7 +657,6 @@ set(CONTRACTOR_LIBRARIES ${BOOST_BASE_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${USED_LUA_LIBRARIES} - ${MAYBE_STXXL_LIBRARY} ${TBB_LIBRARIES} ${MAYBE_RT_LIBRARY} ${MAYBE_COVERAGE_LIBRARIES}) @@ -707,7 +676,6 @@ set(STORAGE_LIBRARIES set(UTIL_LIBRARIES ${BOOST_BASE_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} - ${MAYBE_STXXL_LIBRARY} ${TBB_LIBRARIES} ${MAYBE_COVERAGE_LIBRARIES} ${ZLIB_LIBRARY}) diff --git a/cmake/FindSTXXL.cmake b/cmake/FindSTXXL.cmake deleted file mode 100644 index 473fb6a29db..00000000000 --- a/cmake/FindSTXXL.cmake +++ /dev/null @@ -1,51 +0,0 @@ -# Locate STXXL library -# This module defines -# STXXL_FOUND, if false, do not try to link to libstxxl -# STXXL_LIBRARY -# STXXL_INCLUDE_DIR, where to find stxxl.h -# - - -IF( NOT STXXL_FIND_QUIETLY ) - MESSAGE(STATUS "Looking for STXXL...") -ENDIF() - -FIND_PATH(STXXL_INCLUDE_DIR stxxl.h - HINTS - $ENV{STXXL_DIR} - PATH_SUFFIXES stxxl include/stxxl/stxxl include/stxxl include - PATHS - ~/Library/Frameworks - /Library/Frameworks - /usr/local - /usr - /opt/local # DarwinPorts - /opt -) - -FIND_LIBRARY(STXXL_LIBRARY - NAMES stxxl stxxl_debug - HINTS - $ENV{STXXL_DIR} - PATH_SUFFIXES lib64 lib - PATHS - ~/Library/Frameworks - /Library/Frameworks - /usr/local - /usr - /opt/local - /opt -) - -INCLUDE(FindPackageHandleStandardArgs) -# handle the QUIETLY and REQUIRED arguments and set STXXL_FOUND to TRUE if -# all listed variables are TRUE -FIND_PACKAGE_HANDLE_STANDARD_ARGS(STXXL DEFAULT_MSG STXXL_LIBRARY STXXL_INCLUDE_DIR) - -IF( NOT STXXL_FIND_QUIETLY ) - IF( STXXL_FOUND ) - MESSAGE(STATUS "Found STXXL: ${STXXL_LIBRARY}" ) - ENDIF() -ENDIF() - -MARK_AS_ADVANCED(STXXL_INCLUDE_DIR STXXL_LIBRARY) diff --git a/features/support/env.js b/features/support/env.js index a76938c16f1..9b4c5acc959 100644 --- a/features/support/env.js +++ b/features/support/env.js @@ -22,14 +22,9 @@ module.exports = function () { this.PROFILES_PATH = path.resolve(this.ROOT_PATH, 'profiles'); this.FIXTURES_PATH = path.resolve(this.ROOT_PATH, 'unit_tests/fixtures'); this.BIN_PATH = process.env.OSRM_BUILD_DIR && process.env.OSRM_BUILD_DIR || path.resolve(this.ROOT_PATH, 'build'); - var stxxl_config = path.resolve(this.ROOT_PATH, 'test/.stxxl'); - if (!fs.existsSync(stxxl_config)) { - return callback(new Error('*** '+stxxl_config+ 'does not exist')); - } - this.DATASET_NAME = 'cucumber'; this.PLATFORM_WINDOWS = process.platform.match(/^win.*/); - this.DEFAULT_ENVIRONMENT = Object.assign({STXXLCFG: stxxl_config}, process.env); + this.DEFAULT_ENVIRONMENT = process.env; this.DEFAULT_PROFILE = 'bicycle'; this.DEFAULT_INPUT_FORMAT = 'osm'; this.DEFAULT_LOAD_METHOD = process.argv[process.argv.indexOf('-m') +1].match('mmap') ? 'mmap' : 'datastore'; diff --git a/include/extractor/internal_extractor_edge.hpp b/include/extractor/internal_extractor_edge.hpp index 7ba4a9bf22f..5b2007fa763 100644 --- a/include/extractor/internal_extractor_edge.hpp +++ b/include/extractor/internal_extractor_edge.hpp @@ -85,33 +85,6 @@ struct InternalExtractorEdge DurationData duration_data; // coordinate of the source node util::Coordinate source_coordinate; - - // necessary static util functions for stxxl's sorting - static InternalExtractorEdge min_osm_value() - { - return InternalExtractorEdge( - MIN_OSM_NODEID, MIN_OSM_NODEID, WeightData(), DurationData(), util::Coordinate()); - } - static InternalExtractorEdge max_osm_value() - { - return InternalExtractorEdge( - MAX_OSM_NODEID, MAX_OSM_NODEID, WeightData(), DurationData(), util::Coordinate()); - } - - static InternalExtractorEdge min_internal_value() - { - auto v = min_osm_value(); - v.result.source = 0; - v.result.target = 0; - return v; - } - static InternalExtractorEdge max_internal_value() - { - auto v = max_osm_value(); - v.result.source = std::numeric_limits::max(); - v.result.target = std::numeric_limits::max(); - return v; - } }; } } diff --git a/include/storage/serialization.hpp b/include/storage/serialization.hpp index 97cba94efe4..07655ba4d3c 100644 --- a/include/storage/serialization.hpp +++ b/include/storage/serialization.hpp @@ -17,10 +17,6 @@ #include #include -#if USE_STXXL_LIBRARY -#include -#endif - namespace osrm { namespace storage @@ -134,24 +130,6 @@ inline void write(storage::tar::FileWriter &writer, writer.WriteStreaming(name, vec.begin(), vec.size()); } -#if USE_STXXL_LIBRARY -template -inline void read(storage::tar::FileReader &reader, const std::string &name, stxxl::vector &vec) -{ - auto size = reader.ReadElementCount64(name); - vec.reserve(size); - reader.ReadStreaming(name, std::back_inserter(vec), size); -} - -template -inline void -write(storage::tar::FileWriter &writer, const std::string &name, const stxxl::vector &vec) -{ - writer.WriteElementCount64(name, vec.size()); - writer.WriteStreaming(name, vec.begin(), vec.size()); -} -#endif - template void read(io::BufferReader &reader, std::vector &data) { const auto count = reader.ReadElementCount64(); diff --git a/include/util/meminfo.hpp b/include/util/meminfo.hpp index 40abce33e1c..1fe9d4d28b9 100644 --- a/include/util/meminfo.hpp +++ b/include/util/meminfo.hpp @@ -7,29 +7,11 @@ #include #endif -#if USE_STXXL_LIBRARY -#include -#endif - namespace osrm { namespace util { -inline void DumpSTXXLStats() -{ -#if USE_STXXL_LIBRARY -#if STXXL_VERSION_MAJOR > 1 || (STXXL_VERSION_MAJOR == 1 && STXXL_VERSION_MINOR >= 4) - auto manager = stxxl::block_manager::get_instance(); - util::Log() << "STXXL: peak bytes used: " << manager->get_maximum_allocation(); - util::Log() << "STXXL: total disk allocated: " << manager->get_total_bytes(); -#else -#warning STXXL 1.4+ recommended - STXXL memory summary will not be available - util::Log() << "STXXL: memory summary not available, needs STXXL 1.4 or higher"; -#endif -#endif -} - inline void DumpMemoryStats() { #ifndef _WIN32 diff --git a/include/util/vector_view.hpp b/include/util/vector_view.hpp index 260cc511eb3..288029a7083 100644 --- a/include/util/vector_view.hpp +++ b/include/util/vector_view.hpp @@ -19,10 +19,6 @@ #include #include -#if USE_STXXL_LIBRARY -#include -#endif - namespace osrm { namespace util @@ -253,16 +249,10 @@ template void swap(vector_view &lhs, vector_view std::swap(lhs.m_size, rhs.m_size); } -#if USE_STXXL_LIBRARY -template using ExternalVector = stxxl::vector; -#else -template using ExternalVector = std::vector; -#endif - template using InternalOrExternalVector = typename std::conditional, + std::vector, std::vector>::type; template diff --git a/scripts/travis/before_install.armhf.sh b/scripts/travis/before_install.armhf.sh index 2744742d47e..069ee1628b0 100644 --- a/scripts/travis/before_install.armhf.sh +++ b/scripts/travis/before_install.armhf.sh @@ -18,11 +18,3 @@ sudo apt-get update -qq --yes || true sudo apt-get install -qq --yes --force-yes g++-4.8-arm-linux-gnueabihf g++-4.8-multilib-arm-linux-gnueabihf gcc-4.8-arm-linux-gnueabihf gcc-4.8-multilib-arm-linux-gnueabihf sudo apt-get install -qq --yes --force-yes libexpat1-dev:armhf zlib1g-dev:armhf libbz2-dev:armhf libboost-date-time-dev:armhf libboost-filesystem-dev:armhf libboost-iostreams-dev:armhf libboost-program-options-dev:armhf libboost-regex-dev:armhf libboost-system-dev:armhf libboost-thread-dev:armhf libtbb-dev:armhf libboost-test-dev:armhf - -## build libstxxl1v5:armhf from sources, no package in trusty -if [ $UBUNTU_RELEASE == trusty ] ; then - ( cd /tmp && wget http://sourceforge.net/projects/stxxl/files/stxxl/1.4.1/stxxl-1.4.1.tar.gz && tar xf stxxl-1.4.1.tar.gz ) - ( cd /tmp/stxxl-1.4.1 && mkdir build && cd build && CC=arm-linux-gnueabihf-gcc-4.8 CXX=arm-linux-gnueabihf-g++-4.8 cmake .. && make && sudo make install ) -else - sudo apt-get install -qq --yes --force-yes libstxxl1v5:armhf -fi diff --git a/scripts/travis/before_install.i686.sh b/scripts/travis/before_install.i686.sh index 214d778e927..ce0e23a2d1e 100644 --- a/scripts/travis/before_install.i686.sh +++ b/scripts/travis/before_install.i686.sh @@ -3,4 +3,4 @@ sudo dpkg --add-architecture i386 sudo add-apt-repository --yes ppa:ubuntu-toolchain-r/test && ( sudo apt-get update -qq --yes || true ) -sudo apt-get install -qq --yes --force-yes g++-7-multilib libxml2-dev:i386 libexpat1-dev:i386 libzip-dev:i386 libbz2-dev:i386 libstxxl-dev:i386 libtbb-dev:i386 lua5.2:i386 liblua5.2-dev:i386 libboost-date-time-dev:i386 libboost-filesystem-dev:i386 libboost-iostreams-dev:i386 libboost-program-options-dev:i386 libboost-regex-dev:i386 libboost-system-dev:i386 libboost-thread-dev:i386 libboost-test-dev:i386 +sudo apt-get install -qq --yes --force-yes g++-7-multilib libxml2-dev:i386 libexpat1-dev:i386 libzip-dev:i386 libbz2-dev:i386 libtbb-dev:i386 lua5.2:i386 liblua5.2-dev:i386 libboost-date-time-dev:i386 libboost-filesystem-dev:i386 libboost-iostreams-dev:i386 libboost-program-options-dev:i386 libboost-regex-dev:i386 libboost-system-dev:i386 libboost-thread-dev:i386 libboost-test-dev:i386 diff --git a/src/tools/contract.cpp b/src/tools/contract.cpp index 258e202f838..244840bebe9 100644 --- a/src/tools/contract.cpp +++ b/src/tools/contract.cpp @@ -189,21 +189,18 @@ int main(int argc, char *argv[]) try osrm::contract(contractor_config); - util::DumpSTXXLStats(); util::DumpMemoryStats(); return EXIT_SUCCESS; } catch (const osrm::RuntimeError &e) { - util::DumpSTXXLStats(); util::DumpMemoryStats(); util::Log(logERROR) << e.what(); return e.GetCode(); } catch (const std::bad_alloc &e) { - util::DumpSTXXLStats(); util::DumpMemoryStats(); util::Log(logERROR) << e.what(); util::Log(logERROR) << "Please provide more memory or consider using a larger swapfile"; diff --git a/src/tools/extract.cpp b/src/tools/extract.cpp index f44744bfee3..6daf2495ec8 100644 --- a/src/tools/extract.cpp +++ b/src/tools/extract.cpp @@ -186,28 +186,24 @@ int main(int argc, char *argv[]) try osrm::extract(extractor_config); - util::DumpSTXXLStats(); util::DumpMemoryStats(); return EXIT_SUCCESS; } catch (const osrm::RuntimeError &e) { - util::DumpSTXXLStats(); util::DumpMemoryStats(); util::Log(logERROR) << e.what(); return e.GetCode(); } catch (const std::system_error &e) { - util::DumpSTXXLStats(); util::DumpMemoryStats(); util::Log(logERROR) << e.what(); return e.code().value(); } catch (const std::bad_alloc &e) { - util::DumpSTXXLStats(); util::DumpMemoryStats(); util::Log(logERROR) << "[exception] " << e.what(); util::Log(logERROR) << "Please provide more memory or consider using a larger swapfile"; diff --git a/test/.stxxl b/test/.stxxl deleted file mode 100644 index 61cbf03777f..00000000000 --- a/test/.stxxl +++ /dev/null @@ -1 +0,0 @@ -disk=###.stxxl,20,memory diff --git a/test/data/.stxxl b/test/data/.stxxl deleted file mode 100644 index 4b2217b2c0d..00000000000 --- a/test/data/.stxxl +++ /dev/null @@ -1 +0,0 @@ -disk=/tmp/stxxl_###,200M,syscall autogrow