Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Save unprojected coordinates in node caches and use osmium dense file array #668

Merged
merged 9 commits into from
Jan 9, 2017
Merged

Save unprojected coordinates in node caches and use osmium dense file array #668

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
456f197
store unprojected coordinates in node caches
lonvia Jan 3, 2017
b9330ad
remove scale option
lonvia Jan 3, 2017
d9951a5
replace flatnode store with osmium dense file array
lonvia Jan 4, 2017
52e941f
allow concurrent reads on flatnode file
lonvia Jan 4, 2017
1b52e50
use iterator instead of indexing when getting node list
lonvia Jan 5, 2017
7a081d4
remove tags column from _nodes table
lonvia Jan 7, 2017
8f41f12
clang-format of node-ram-cache
lonvia Jan 7, 2017
8bbef6a
persistent store: catch invalid location exception
lonvia Jan 7, 2017
a940c98
persistent cache: move try-catch further out
lonvia Jan 7, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .clang-format
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ BreakBeforeBraces: Mozilla
IndentWidth: 4
ConstructorInitializerIndentWidth: 0
ReflowComments: false
AlwaysBreakTemplateDeclarations: true
12 changes: 7 additions & 5 deletions geometry-processor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,13 @@ way_helper::~way_helper()
{
}
size_t way_helper::set(osmium::WayNodeList const &node_ids,
const middle_query_t *mid)
middle_query_t const *mid, reprojection const *proj)
{
node_cache.clear();
mid->nodes_get_list(node_cache, node_ids);
mid->nodes_get_list(node_cache, node_ids, proj);

// equivalent to returning node_count for complete ways, different for partial extracts
// equivalent to returning node_count for complete ways, different for partial
// extracts
return node_cache.size();
}

Expand Down Expand Up @@ -152,13 +153,14 @@ multitaglist_t relation_helper::get_filtered_tags(tagtransform *transform, expor
return filtered;
}

multinodelist_t relation_helper::get_nodes(middle_t const *mid) const
multinodelist_t relation_helper::get_nodes(middle_t const *mid,
reprojection const *proj) const
{
multinodelist_t nodes(roles.size());

size_t i = 0;
for (auto const &w : data.select<osmium::Way>()) {
mid->nodes_get_list(nodes[i++], w.nodes());
mid->nodes_get_list(nodes[i++], w.nodes(), proj);
}

return nodes;
Expand Down
7 changes: 5 additions & 2 deletions geometry-processor.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
struct middle_query_t;
struct middle_t;
struct options_t;
class reprojection;

struct geometry_processor {
// factory method for creating various types of geometry processors either by name or by geometry column type
Expand Down Expand Up @@ -79,7 +80,8 @@ struct way_helper
{
way_helper();
~way_helper();
size_t set(osmium::WayNodeList const &node_ids, const middle_query_t *mid);
size_t set(osmium::WayNodeList const &node_ids, middle_query_t const *mid,
reprojection const *proj);

nodelist_t node_cache;
};
Expand All @@ -91,7 +93,8 @@ struct relation_helper
~relation_helper();
size_t set(osmium::RelationMemberList const &member_list, middle_t const *mid);
multitaglist_t get_filtered_tags(tagtransform *transform, export_list const &el) const;
multinodelist_t get_nodes(middle_t const *mid) const;
multinodelist_t get_nodes(middle_t const *mid,
reprojection const *proj) const;

osmium::memory::ItemIteratorRange<const osmium::Way> way_iterator() const
{ return data.select<osmium::Way>(); }
Expand Down
141 changes: 57 additions & 84 deletions middle-pgsql.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,90 +256,76 @@ void middle_pgsql_t::buffer_correct_params(char const **param, size_t size)
}
}

void middle_pgsql_t::local_nodes_set(osmium::Node const &node,
double lat, double lon)
void middle_pgsql_t::local_nodes_set(osmium::Node const &node)
{
copy_buffer.reserve(node.tags().byte_size() + 100);

bool copy = node_table->copyMode;
char delim = copy ? '\t' : '\0';
const char *paramValues[4] = { copy_buffer.c_str(), };
const char *paramValues[4] = {
copy_buffer.c_str(),
};

copy_buffer = std::to_string(node.id());
copy_buffer += delim;

#ifdef FIXED_POINT
ramNode n(lon, lat);
paramValues[1] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(n.int_lat());
copy_buffer += std::to_string(node.location().y());
copy_buffer += delim;

paramValues[2] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(n.int_lon());
copy_buffer += delim;
#else
paramValues[1] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(lat);
copy_buffer += delim;

paramValues[2] = paramValues[0] + copy_buffer.size();
copy_buffer += std::to_string(lon);
copy_buffer += delim;
#endif

if (node.tags().empty() && !out_options->extra_attributes) {
paramValues[3] = nullptr;
copy_buffer += "\\N";
} else {
paramValues[3] = paramValues[0] + copy_buffer.size();
buffer_store_tags(node, out_options->extra_attributes, copy);
}
copy_buffer += std::to_string(node.location().x());

if (copy) {
copy_buffer += '\n';
pgsql_CopyData(__FUNCTION__, node_table->sql_conn, copy_buffer);
} else {
buffer_correct_params(paramValues, 4);
pgsql_execPrepared(node_table->sql_conn, "insert_node", 4,
(const char * const *)paramValues, PGRES_COMMAND_OK);
pgsql_execPrepared(node_table->sql_conn, "insert_node", 3,
(const char *const *)paramValues, PGRES_COMMAND_OK);
}
}

// This should be made more efficient by using an IN(ARRAY[]) construct */
size_t middle_pgsql_t::local_nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds) const
size_t middle_pgsql_t::local_nodes_get_list(nodelist_t &out,
osmium::WayNodeList const &nds,
reprojection const *proj) const
{
assert(out.empty());

char tmp[16];

char *tmp2 = static_cast<char *>(malloc(sizeof(char) * nds.size() * 16));
if (tmp2 == nullptr) return 0; //failed to allocate memory, return */

if (tmp2 == nullptr) {
return 0; // failed to allocate memory, return
}

// create a list of ids in tmp2 to query the database */
// create a list of ids in tmp2 to query the database
sprintf(tmp2, "{");
int countDB = 0;
out.reserve(nds.size());
for (auto const &n : nds) {
// Check cache first */
osmNode loc;
if (cache->get(&loc, n.ref()) == 0) {
out.push_back(loc);
auto loc = cache->get(n.ref());
if (loc.valid()) {
auto coord = proj->reproject(loc);
out.push_back(osmNode(coord.x, coord.y));
continue;
}

countDB++;
// Mark nodes as needing to be fetched from the DB */
// Mark nodes as needing to be fetched from the DB
out.push_back(osmNode());

snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", n.ref());
strncat(tmp2, tmp, sizeof(char)*(nds.size()*16 - 2));
strncat(tmp2, tmp, sizeof(char) * (nds.size() * 16 - 2));
}
tmp2[strlen(tmp2) - 1] = '}'; // replace last , with } to complete list of ids*/
// replace last , with } to complete list of ids
tmp2[strlen(tmp2) - 1] = '}';

if (countDB == 0) {
free(tmp2);
return nds.size(); // All ids where in cache, so nothing more to do */
return nds.size(); // All ids where in cache, so nothing more to do
}

pgsql_endCopy(node_table);
Expand All @@ -348,26 +334,20 @@ size_t middle_pgsql_t::local_nodes_get_list(nodelist_t &out, osmium::WayNodeList

char const *paramValues[1];
paramValues[0] = tmp2;
PGresult *res = pgsql_execPrepared(sql_conn, "get_node_list", 1, paramValues, PGRES_TUPLES_OK);
PGresult *res = pgsql_execPrepared(sql_conn, "get_node_list", 1,
paramValues, PGRES_TUPLES_OK);
int countPG = PQntuples(res);

//store the pg results in a hashmap and telling it how many we expect
// store the pg results in a hashmap and telling it how many we expect
std::unordered_map<osmid_t, osmNode> pg_nodes(countPG);

for (int i = 0; i < countPG; i++) {
osmid_t id = strtoosmid(PQgetvalue(res, i, 0), nullptr, 10);
osmNode node;
#ifdef FIXED_POINT
ramNode n((int) strtol(PQgetvalue(res, i, 2), nullptr, 10),
(int) strtol(PQgetvalue(res, i, 1), nullptr, 10));

node.lat = n.lat();
node.lon = n.lon();
#else
node.lat = strtod(PQgetvalue(res, i, 1), nullptr);
node.lon = strtod(PQgetvalue(res, i, 2), nullptr);
#endif
pg_nodes.emplace(id, node);
osmium::Location n((int)strtol(PQgetvalue(res, i, 2), nullptr, 10),
(int)strtol(PQgetvalue(res, i, 1), nullptr, 10));

auto coord = proj->reproject(n);
pg_nodes.emplace(id, osmNode(coord.x, coord.y));
}

PQclear(res);
Expand All @@ -378,8 +358,9 @@ size_t middle_pgsql_t::local_nodes_get_list(nodelist_t &out, osmium::WayNodeList
size_t wrtidx = 0;
for (size_t i = 0; i < nds.size(); ++i) {
if (std::isnan(out[i].lat)) {
std::unordered_map<osmid_t, osmNode>::iterator found = pg_nodes.find(nds[i].ref());
if(found != pg_nodes.end()) {
std::unordered_map<osmid_t, osmNode>::iterator found =
pg_nodes.find(nds[i].ref());
if (found != pg_nodes.end()) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could this entire loop be an auto nd:nds?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is part of the code that was just formatted by clang-format. I don't want to touch it at the moment.

out[wrtidx] = found->second;
++wrtidx;
}
Expand All @@ -394,23 +375,24 @@ size_t middle_pgsql_t::local_nodes_get_list(nodelist_t &out, osmium::WayNodeList
return wrtidx;
}


void middle_pgsql_t::nodes_set(osmium::Node const &node, double lat, double lon)
void middle_pgsql_t::nodes_set(osmium::Node const &node)
{
cache->set(node.id(), lat, lon);
cache->set(node.id(), node.location());

if (out_options->flat_node_cache_enabled) {
persistent_cache->set(node.id(), lat, lon);
persistent_cache->set(node.id(), node.location());
} else {
local_nodes_set(node, lat, lon);
local_nodes_set(node);
}
}

size_t middle_pgsql_t::nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds) const
size_t middle_pgsql_t::nodes_get_list(nodelist_t &out,
osmium::WayNodeList const &nds,
reprojection const *proj) const
{
return (out_options->flat_node_cache_enabled)
? persistent_cache->get_list(out, nds)
: local_nodes_get_list(out, nds);
? persistent_cache->get_list(out, nds, proj)
: local_nodes_get_list(out, nds, proj);
}

void middle_pgsql_t::local_nodes_delete(osmid_t osm_id)
Expand All @@ -428,7 +410,7 @@ void middle_pgsql_t::local_nodes_delete(osmid_t osm_id)
void middle_pgsql_t::nodes_delete(osmid_t osm_id)
{
if (out_options->flat_node_cache_enabled) {
persistent_cache->set(osm_id, NAN, NAN);
persistent_cache->set(osm_id, osmium::Location());
} else {
local_nodes_delete(osm_id);
}
Expand Down Expand Up @@ -1021,10 +1003,14 @@ void middle_pgsql_t::start(const options_t *out_options_)
// staying set for the second.
build_indexes = !append && !out_options->droptemp;

cache.reset(new node_ram_cache( out_options->alloc_chunkwise | ALLOC_LOSSY, out_options->cache, out_options->scale));
if (out_options->flat_node_cache_enabled) persistent_cache.reset(new node_persistent_cache(out_options, out_options->append, false, cache));
cache.reset(new node_ram_cache(out_options->alloc_chunkwise | ALLOC_LOSSY,
out_options->cache));

fprintf(stderr, "Mid: pgsql, scale=%d cache=%d\n", out_options->scale, out_options->cache);
if (out_options->flat_node_cache_enabled) {
persistent_cache.reset(new node_persistent_cache(out_options, cache));
}

fprintf(stderr, "Mid: pgsql, cache=%d\n", out_options->cache);

// We use a connection per table to enable the use of COPY */
for (auto& table: tables) {
Expand Down Expand Up @@ -1089,9 +1075,6 @@ void middle_pgsql_t::commit(void) {
table.transactionMode = 0;
}
}
// Make sure the flat nodes are committed to disk or there will be
// surprises later.
if (out_options->flat_node_cache_enabled) persistent_cache.reset();
}

void middle_pgsql_t::pgsql_stop_one(table_desc *table)
Expand Down Expand Up @@ -1138,24 +1121,17 @@ void middle_pgsql_t::stop(void)
}

middle_pgsql_t::middle_pgsql_t()
: tables(), num_tables(0), node_table(nullptr), way_table(nullptr), rel_table(nullptr),
append(false), mark_pending(true), cache(), persistent_cache(), build_indexes(true)
: num_tables(0), node_table(nullptr), way_table(nullptr), rel_table(nullptr),
append(false), mark_pending(true), build_indexes(true)
{
// clang-format off
/*table = t_node,*/
tables.push_back(table_desc(
/*name*/ "%p_nodes",
/*start*/ "BEGIN;\n",
#ifdef FIXED_POINT
/*create*/ "CREATE %m TABLE %p_nodes (id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, lat int4 not null, lon int4 not null, tags text[]) {TABLESPACE %t};\n",
/*create_index*/ nullptr,
/*prepare*/ "PREPARE insert_node (" POSTGRES_OSMID_TYPE ", int4, int4, text[]) AS INSERT INTO %p_nodes VALUES ($1,$2,$3,$4);\n"
#else
/*create*/ "CREATE %m TABLE %p_nodes (id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, lat double precision not null, lon double precision not null, tags text[]) {TABLESPACE %t};\n",
/*create*/ "CREATE %m TABLE %p_nodes (id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, lat int4 not null, lon int4 not null) {TABLESPACE %t};\n",
/*create_index*/ nullptr,
/*prepare*/ "PREPARE insert_node (" POSTGRES_OSMID_TYPE ", double precision, double precision, text[]) AS INSERT INTO %p_nodes VALUES ($1,$2,$3,$4);\n"
#endif
"PREPARE get_node (" POSTGRES_OSMID_TYPE ") AS SELECT lat,lon,tags FROM %p_nodes WHERE id = $1 LIMIT 1;\n"
/*prepare*/ "PREPARE insert_node (" POSTGRES_OSMID_TYPE ", int4, int4) AS INSERT INTO %p_nodes VALUES ($1,$2,$3);\n"
"PREPARE get_node_list(" POSTGRES_OSMID_TYPE "[]) AS SELECT id, lat, lon FROM %p_nodes WHERE id = ANY($1::" POSTGRES_OSMID_TYPE "[]);\n"
"PREPARE delete_node (" POSTGRES_OSMID_TYPE ") AS DELETE FROM %p_nodes WHERE id = $1;\n",
/*prepare_intarray*/ nullptr,
Expand Down Expand Up @@ -1231,10 +1207,7 @@ std::shared_ptr<const middle_query_t> middle_pgsql_t::get_instance() const {
//NOTE: this is thread safe for use in pending async processing only because
//during that process they are only read from
mid->cache = cache;
// The persistent cache on the other hand is not thread-safe for reading,
// so we create one per instance.
if (out_options->flat_node_cache_enabled)
mid->persistent_cache.reset(new node_persistent_cache(out_options, 1, true, cache));
mid->persistent_cache = persistent_cache;

// We use a connection per table to enable the use of COPY */
for(int i=0; i<num_tables; i++) {
Expand Down
10 changes: 6 additions & 4 deletions middle-pgsql.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,9 @@ struct middle_pgsql_t : public slim_middle_t {
void end(void) override;
void commit(void) override;

void nodes_set(osmium::Node const &node, double lat, double lon) override;
size_t nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds) const override;
void nodes_set(osmium::Node const &node) override;
size_t nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds,
reprojection const *proj) const override;
void nodes_delete(osmid_t id) override;
void node_changed(osmid_t id) override;

Expand Down Expand Up @@ -86,8 +87,9 @@ struct middle_pgsql_t : public slim_middle_t {
* Sets up sql_conn for the table
*/
void connect(table_desc& table);
void local_nodes_set(osmium::Node const &node, double lat, double lon);
size_t local_nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds) const;
void local_nodes_set(osmium::Node const &node);
size_t local_nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds,
reprojection const *proj) const;
void local_nodes_delete(osmid_t osm_id);

std::vector<table_desc> tables;
Expand Down
Loading