Skip to content

Commit

Permalink
Removed STXXL support.
Browse files Browse the repository at this point in the history
  • Loading branch information
Denis Chaplygin committed Jun 30, 2020
1 parent 365121d commit a30c016
Show file tree
Hide file tree
Showing 16 changed files with 5 additions and 201 deletions.
6 changes: 0 additions & 6 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,6 @@ Thumbs.db
###########################
.idea/

# stxxl related files #
#######################
.stxxl
stxxl.log
stxxl.errlog

# Compiled Binary Files #
####################################
/osrm-extract
Expand Down
9 changes: 0 additions & 9 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -151,14 +151,6 @@ matrix:
TARGET_ARCH='i686' CCOMPILER='gcc-7' CXXCOMPILER='g++-7' BUILD_TYPE='Release'
CFLAGS='-m32 -msse2 -mfpmath=sse' CXXFLAGS='-m32 -msse2 -mfpmath=sse'
- os: linux
compiler: "gcc-7-stxxl"
addons: &gcc7
apt:
sources: ['ubuntu-toolchain-r-test']
packages: ['g++-7', 'libbz2-dev', 'libstxxl-dev', 'libxml2-dev', 'libzip-dev', 'liblua5.2-dev', 'libtbb-dev', 'libboost-all-dev']
env: CCOMPILER='gcc-7' CXXCOMPILER='g++-7' BUILD_TYPE='Release' ENABLE_STXXL=On

- os: linux
compiler: "gcc-5-release"
addons: &gcc49
Expand Down Expand Up @@ -483,7 +475,6 @@ install:
-DENABLE_COVERAGE=${ENABLE_COVERAGE:-OFF} \
-DENABLE_NODE_BINDINGS=${ENABLE_NODE_BINDINGS:-OFF} \
-DENABLE_SANITIZER=${ENABLE_SANITIZER:-OFF} \
-DENABLE_STXXL=${ENABLE_STXXL:-OFF} \
-DBUILD_TOOLS=ON \
-DENABLE_CCACHE=ON \
-DCMAKE_INSTALL_PREFIX=${OSRM_INSTALL_DIR} \
Expand Down
3 changes: 2 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
- FIXED: treat `bicycle=use_sidepath` as no access on the tagged way. [#5622](https://github.com/Project-OSRM/osrm-backend/pull/5622)
- Misc:
- CHANGED: Reduce memory usage for raster source handling. [#5572](https://github.com/Project-OSRM/osrm-backend/pull/5572)

- Infrastructure
- REMOVED: STXXL support removed as STXXL became abandonware. [#5760](https://github.com/Project-OSRM/osrm-backend/pull/5760)
# 5.21.0
- Changes from 5.20.0
- Features:
Expand Down
32 changes: 0 additions & 32 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ option(BUILD_PACKAGE "Build OSRM package" OFF)
option(ENABLE_ASSERTIONS "Use assertions in release mode" OFF)
option(ENABLE_COVERAGE "Build with coverage instrumentalisation" OFF)
option(ENABLE_SANITIZER "Use memory sanitizer for Debug build" OFF)
option(ENABLE_STXXL "Use STXXL library" OFF)
option(ENABLE_LTO "Use LTO if available" OFF)
option(ENABLE_FUZZING "Fuzz testing using LLVM's libFuzzer" OFF)
option(ENABLE_GOLD_LINKER "Use GNU gold linker if available" ON)
Expand All @@ -37,7 +36,6 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
if(ENABLE_MASON)
# versions in use
set(MASON_BOOST_VERSION "1.65.1")
set(MASON_STXXL_VERSION "1.4.1-1")
set(MASON_EXPAT_VERSION "2.2.0")
set(MASON_LUA_VERSION "5.2.4")
set(MASON_BZIP2_VERSION "1.0.6")
Expand Down Expand Up @@ -472,13 +470,6 @@ if(ENABLE_MASON)
mason_use(boost_libsystem VERSION ${MASON_BOOST_VERSION})
set(Boost_SYSTEM_LIBRARY ${MASON_PACKAGE_boost_libsystem_STATIC_LIBS})

if (ENABLE_STXXL)
mason_use(stxxl VERSION ${MASON_STXXL_VERSION})
add_dependency_includes(${MASON_PACKAGE_stxxl_INCLUDE_DIRS})
set(MAYBE_STXXL_LIBRARY ${MASON_PACKAGE_stxxl_STATIC_LIBS})
add_definitions(-DUSE_STXXL_LIBRARY)
endif()

mason_use(expat VERSION ${MASON_EXPAT_VERSION})
add_dependency_includes(${MASON_PACKAGE_expat_INCLUDE_DIRS})
set(EXPAT_LIBRARIES ${MASON_PACKAGE_expat_STATIC_LIBS})
Expand Down Expand Up @@ -537,17 +528,6 @@ else()
find_package(EXPAT REQUIRED)
add_dependency_includes(${EXPAT_INCLUDE_DIRS})

if (ENABLE_STXXL)
find_package(STXXL)
if (STXXL_FOUND)
add_dependency_includes(${STXXL_INCLUDE_DIR})
set(MAYBE_STXXL_LIBRARY ${STXXL_LIBRARY})
add_definitions(-DUSE_STXXL_LIBRARY)
else()
MESSAGE(STATUS "STXXL was requested but not found, default STL will be used")
endif()
endif()

find_package(BZip2 REQUIRED)
add_dependency_includes(${BZIP2_INCLUDE_DIR})

Expand Down Expand Up @@ -607,15 +587,6 @@ add_dependency_defines(-DBOOST_SPIRIT_USE_PHOENIX_V3)
add_dependency_defines(-DBOOST_RESULT_OF_USE_DECLTYPE)
add_dependency_defines(-DBOOST_FILESYSTEM_NO_DEPRECATED)

if (ENABLE_STXXL)
set(OpenMP_FIND_QUIETLY ON)
find_package(OpenMP)
if(OPENMP_FOUND)
message(STATUS "OpenMP support found. Linking just in case for stxxl")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
endif()
endif()

add_definitions(${OSRM_DEFINES})
include_directories(SYSTEM ${DEPENDENCIES_INCLUDE_DIRS})

Expand Down Expand Up @@ -648,7 +619,6 @@ set(EXTRACTOR_LIBRARIES
${EXPAT_LIBRARIES}
${USED_LUA_LIBRARIES}
${OSMIUM_LIBRARIES}
${MAYBE_STXXL_LIBRARY}
${TBB_LIBRARIES}
${ZLIB_LIBRARY}
${MAYBE_COVERAGE_LIBRARIES})
Expand Down Expand Up @@ -682,7 +652,6 @@ set(CONTRACTOR_LIBRARIES
${BOOST_BASE_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT}
${USED_LUA_LIBRARIES}
${MAYBE_STXXL_LIBRARY}
${TBB_LIBRARIES}
${MAYBE_RT_LIBRARY}
${MAYBE_COVERAGE_LIBRARIES})
Expand All @@ -702,7 +671,6 @@ set(STORAGE_LIBRARIES
set(UTIL_LIBRARIES
${BOOST_BASE_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT}
${MAYBE_STXXL_LIBRARY}
${TBB_LIBRARIES}
${MAYBE_COVERAGE_LIBRARIES}
${ZLIB_LIBRARY})
Expand Down
51 changes: 0 additions & 51 deletions cmake/FindSTXXL.cmake

This file was deleted.

7 changes: 1 addition & 6 deletions features/support/env.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,9 @@ module.exports = function () {
this.PROFILES_PATH = path.resolve(this.ROOT_PATH, 'profiles');
this.FIXTURES_PATH = path.resolve(this.ROOT_PATH, 'unit_tests/fixtures');
this.BIN_PATH = process.env.OSRM_BUILD_DIR && process.env.OSRM_BUILD_DIR || path.resolve(this.ROOT_PATH, 'build');
var stxxl_config = path.resolve(this.ROOT_PATH, 'test/.stxxl');
if (!fs.existsSync(stxxl_config)) {
return callback(new Error('*** '+stxxl_config+ 'does not exist'));
}

this.DATASET_NAME = 'cucumber';
this.PLATFORM_WINDOWS = process.platform.match(/^win.*/);
this.DEFAULT_ENVIRONMENT = Object.assign({STXXLCFG: stxxl_config}, process.env);
this.DEFAULT_ENVIRONMENT = process.env;
this.DEFAULT_PROFILE = 'bicycle';
this.DEFAULT_INPUT_FORMAT = 'osm';
this.DEFAULT_LOAD_METHOD = process.argv[process.argv.indexOf('-m') +1].match('mmap') ? 'mmap' : 'datastore';
Expand Down
27 changes: 0 additions & 27 deletions include/extractor/internal_extractor_edge.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -85,33 +85,6 @@ struct InternalExtractorEdge
DurationData duration_data;
// coordinate of the source node
util::Coordinate source_coordinate;

// necessary static util functions for stxxl's sorting
static InternalExtractorEdge min_osm_value()
{
return InternalExtractorEdge(
MIN_OSM_NODEID, MIN_OSM_NODEID, WeightData(), DurationData(), util::Coordinate());
}
static InternalExtractorEdge max_osm_value()
{
return InternalExtractorEdge(
MAX_OSM_NODEID, MAX_OSM_NODEID, WeightData(), DurationData(), util::Coordinate());
}

static InternalExtractorEdge min_internal_value()
{
auto v = min_osm_value();
v.result.source = 0;
v.result.target = 0;
return v;
}
static InternalExtractorEdge max_internal_value()
{
auto v = max_osm_value();
v.result.source = std::numeric_limits<NodeID>::max();
v.result.target = std::numeric_limits<NodeID>::max();
return v;
}
};
}
}
Expand Down
22 changes: 0 additions & 22 deletions include/storage/serialization.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,6 @@
#include <cstdint>
#include <tuple>

#if USE_STXXL_LIBRARY
#include <stxxl/vector>
#endif

namespace osrm
{
namespace storage
Expand Down Expand Up @@ -134,24 +130,6 @@ inline void write(storage::tar::FileWriter &writer,
writer.WriteStreaming<T>(name, vec.begin(), vec.size());
}

#if USE_STXXL_LIBRARY
template <typename T>
inline void read(storage::tar::FileReader &reader, const std::string &name, stxxl::vector<T> &vec)
{
auto size = reader.ReadElementCount64(name);
vec.reserve(size);
reader.ReadStreaming<T>(name, std::back_inserter(vec), size);
}

template <typename T>
inline void
write(storage::tar::FileWriter &writer, const std::string &name, const stxxl::vector<T> &vec)
{
writer.WriteElementCount64(name, vec.size());
writer.WriteStreaming<T>(name, vec.begin(), vec.size());
}
#endif

template <typename T> void read(io::BufferReader &reader, std::vector<T> &data)
{
const auto count = reader.ReadElementCount64();
Expand Down
18 changes: 0 additions & 18 deletions include/util/meminfo.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,29 +7,11 @@
#include <sys/resource.h>
#endif

#if USE_STXXL_LIBRARY
#include <stxxl/mng>
#endif

namespace osrm
{
namespace util
{

inline void DumpSTXXLStats()
{
#if USE_STXXL_LIBRARY
#if STXXL_VERSION_MAJOR > 1 || (STXXL_VERSION_MAJOR == 1 && STXXL_VERSION_MINOR >= 4)
auto manager = stxxl::block_manager::get_instance();
util::Log() << "STXXL: peak bytes used: " << manager->get_maximum_allocation();
util::Log() << "STXXL: total disk allocated: " << manager->get_total_bytes();
#else
#warning STXXL 1.4+ recommended - STXXL memory summary will not be available
util::Log() << "STXXL: memory summary not available, needs STXXL 1.4 or higher";
#endif
#endif
}

inline void DumpMemoryStats()
{
#ifndef _WIN32
Expand Down
12 changes: 1 addition & 11 deletions include/util/vector_view.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,6 @@
#include <utility>
#include <vector>

#if USE_STXXL_LIBRARY
#include <stxxl/vector>
#endif

namespace osrm
{
namespace util
Expand Down Expand Up @@ -253,16 +249,10 @@ template <typename DataT> void swap(vector_view<DataT> &lhs, vector_view<DataT>
std::swap(lhs.m_size, rhs.m_size);
}

#if USE_STXXL_LIBRARY
template <typename T> using ExternalVector = stxxl::vector<T>;
#else
template <typename T> using ExternalVector = std::vector<T>;
#endif

template <typename DataT, storage::Ownership Ownership>
using InternalOrExternalVector =
typename std::conditional<Ownership == storage::Ownership::External,
ExternalVector<DataT>,
std::vector<DataT>,
std::vector<DataT>>::type;

template <typename DataT, storage::Ownership Ownership>
Expand Down
8 changes: 0 additions & 8 deletions scripts/travis/before_install.armhf.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,3 @@ sudo apt-get update -qq --yes || true

sudo apt-get install -qq --yes --force-yes g++-4.8-arm-linux-gnueabihf g++-4.8-multilib-arm-linux-gnueabihf gcc-4.8-arm-linux-gnueabihf gcc-4.8-multilib-arm-linux-gnueabihf
sudo apt-get install -qq --yes --force-yes libexpat1-dev:armhf zlib1g-dev:armhf libbz2-dev:armhf libboost-date-time-dev:armhf libboost-filesystem-dev:armhf libboost-iostreams-dev:armhf libboost-program-options-dev:armhf libboost-regex-dev:armhf libboost-system-dev:armhf libboost-thread-dev:armhf libtbb-dev:armhf libboost-test-dev:armhf

## build libstxxl1v5:armhf from sources, no package in trusty
if [ $UBUNTU_RELEASE == trusty ] ; then
( cd /tmp && wget http://sourceforge.net/projects/stxxl/files/stxxl/1.4.1/stxxl-1.4.1.tar.gz && tar xf stxxl-1.4.1.tar.gz )
( cd /tmp/stxxl-1.4.1 && mkdir build && cd build && CC=arm-linux-gnueabihf-gcc-4.8 CXX=arm-linux-gnueabihf-g++-4.8 cmake .. && make && sudo make install )
else
sudo apt-get install -qq --yes --force-yes libstxxl1v5:armhf
fi
2 changes: 1 addition & 1 deletion scripts/travis/before_install.i686.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@
sudo dpkg --add-architecture i386
sudo add-apt-repository --yes ppa:ubuntu-toolchain-r/test && ( sudo apt-get update -qq --yes || true )

sudo apt-get install -qq --yes --force-yes g++-7-multilib libxml2-dev:i386 libexpat1-dev:i386 libzip-dev:i386 libbz2-dev:i386 libstxxl-dev:i386 libtbb-dev:i386 lua5.2:i386 liblua5.2-dev:i386 libboost-date-time-dev:i386 libboost-filesystem-dev:i386 libboost-iostreams-dev:i386 libboost-program-options-dev:i386 libboost-regex-dev:i386 libboost-system-dev:i386 libboost-thread-dev:i386 libboost-test-dev:i386
sudo apt-get install -qq --yes --force-yes g++-7-multilib libxml2-dev:i386 libexpat1-dev:i386 libzip-dev:i386 libbz2-dev:i386 libtbb-dev:i386 lua5.2:i386 liblua5.2-dev:i386 libboost-date-time-dev:i386 libboost-filesystem-dev:i386 libboost-iostreams-dev:i386 libboost-program-options-dev:i386 libboost-regex-dev:i386 libboost-system-dev:i386 libboost-thread-dev:i386 libboost-test-dev:i386
3 changes: 0 additions & 3 deletions src/tools/contract.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -189,21 +189,18 @@ int main(int argc, char *argv[]) try

osrm::contract(contractor_config);

util::DumpSTXXLStats();
util::DumpMemoryStats();

return EXIT_SUCCESS;
}
catch (const osrm::RuntimeError &e)
{
util::DumpSTXXLStats();
util::DumpMemoryStats();
util::Log(logERROR) << e.what();
return e.GetCode();
}
catch (const std::bad_alloc &e)
{
util::DumpSTXXLStats();
util::DumpMemoryStats();
util::Log(logERROR) << e.what();
util::Log(logERROR) << "Please provide more memory or consider using a larger swapfile";
Expand Down
4 changes: 0 additions & 4 deletions src/tools/extract.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -186,28 +186,24 @@ int main(int argc, char *argv[]) try

osrm::extract(extractor_config);

util::DumpSTXXLStats();
util::DumpMemoryStats();

return EXIT_SUCCESS;
}
catch (const osrm::RuntimeError &e)
{
util::DumpSTXXLStats();
util::DumpMemoryStats();
util::Log(logERROR) << e.what();
return e.GetCode();
}
catch (const std::system_error &e)
{
util::DumpSTXXLStats();
util::DumpMemoryStats();
util::Log(logERROR) << e.what();
return e.code().value();
}
catch (const std::bad_alloc &e)
{
util::DumpSTXXLStats();
util::DumpMemoryStats();
util::Log(logERROR) << "[exception] " << e.what();
util::Log(logERROR) << "Please provide more memory or consider using a larger swapfile";
Expand Down
Loading

0 comments on commit a30c016

Please sign in to comment.