VladPolskiy/osm2pgsql
1
0
Fork 0
mirror of https://github.com/osm2pgsql-dev/osm2pgsql.git synced 2025-08-19 16:28:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Split middle interface between insert query and insert

Browse Source 
Middle has three interface classes: middle_query_t, middle_t
and slim_middle_t. They inherit from each other. This is
problematic when one wants to supply a middle implementation
that follows a similar inheritence structure.

There is actualy no need that the interfaces for inserting
(middle_t and slim_middle_t) inherit from middle_query_t.
The code is very clear about which interface is needed by
which class.

This change splits off the middle_query_t interface and
cleans up the code where the clear distinction was still
missing.

Needs some larger rework of the middle test which did rely
on being able to access both interface types. There is now
a template class for testing middle that also contains
some more common boilerplate code.
This commit is contained in:
Sarah Hoffmann
2018-12-16 18:14:35 +01:00
parent 56c8f56b43
commit b7ec16d858
26 changed files with 465 additions and 506 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -173,7 +173,6 @@ set(osm2pgsql_lib_SOURCES
id-tracker.cpp
middle-pgsql.cpp
middle-ram.cpp
middle.cpp
node-persistent-cache.cpp
node-ram-cache.cpp
options.cpp

5
geometry-processor.cpp
View File

@ -81,7 +81,8 @@ relation_helper::relation_helper()
: data(1024, osmium::memory::Buffer::auto_grow::yes)
{}
size_t relation_helper::set(osmium::Relation const &rel, middle_t const *mid)
size_t relation_helper::set(osmium::Relation const &rel,
middle_query_t const *mid)
{
// cleanup
data.clear();
@ -93,7 +94,7 @@ size_t relation_helper::set(osmium::Relation const &rel, middle_t const *mid)
return num_ways;
}
void relation_helper::add_way_locations(middle_t const *mid)
void relation_helper::add_way_locations(middle_query_t const *mid)
{
for (auto &w : data.select<osmium::Way>()) {
mid->nodes_get_list(&(w.nodes()));

5
geometry-processor.hpp
View File

@ -13,7 +13,6 @@
#include "tagtransform.hpp"
struct middle_query_t;
struct middle_t;
struct options_t;
class reprojection;
@ -86,8 +85,8 @@ class relation_helper
public:
relation_helper();
size_t set(osmium::Relation const &rel, middle_t const *mid);
void add_way_locations(middle_t const *mid);
size_t set(osmium::Relation const &rel, middle_query_t const *mid);
void add_way_locations(middle_query_t const *mid);
rolelist_t roles;
osmium::memory::Buffer data;

9
middle-pgsql.cpp
View File

@ -949,10 +949,9 @@ void middle_pgsql_t::analyze()
}
}
void middle_pgsql_t::start(const options_t *out_options_)
void middle_pgsql_t::start(const options_t *options)
{
out_options = out_options_;
bool dropcreate = !out_options->append; ///< If tables need to be dropped and created anew
out_options = options;
ways_pending_tracker.reset(new id_tracker());
rels_pending_tracker.reset(new id_tracker());
@ -983,13 +982,13 @@ void middle_pgsql_t::start(const options_t *out_options_)
for (auto &table : tables) {
table.connect(out_options);
if (dropcreate) {
if (!append) {
pgsql_exec(table.sql_conn, PGRES_COMMAND_OK,
"DROP TABLE IF EXISTS %s CASCADE", table.name());
}
table.begin();
if (dropcreate) {
if (!append) {
table.create();
}
table.prepare_queries(append);

4
middle-pgsql.hpp
View File

@ -17,7 +17,8 @@
#include "node-ram-cache.hpp"
#include "pgsql.hpp"
struct middle_pgsql_t : public slim_middle_t {
struct middle_pgsql_t : public slim_middle_t, public middle_query_t
{
middle_pgsql_t();
void start(const options_t *out_options_) override;
@ -115,6 +116,7 @@ private:
bool append;
bool mark_pending;
options_t const *out_options;
std::shared_ptr<node_ram_cache> cache;
std::shared_ptr<node_persistent_cache> persistent_cache;

15
middle-ram.cpp
View File

@ -40,12 +40,12 @@ void middle_ram_t::nodes_set(osmium::Node const &node)
void middle_ram_t::ways_set(osmium::Way const &way)
{
ways.set(way.id(), new ramWay(way, out_options->extra_attributes));
ways.set(way.id(), new ramWay(way, extra_attributes));
}
void middle_ram_t::relations_set(osmium::Relation const &rel)
{
rels.set(rel.id(), new ramRel(rel, out_options->extra_attributes));
rels.set(rel.id(), new ramRel(rel, extra_attributes));
}
size_t middle_ram_t::nodes_get_list(osmium::WayNodeList *nodes) const
@ -155,11 +155,10 @@ void middle_ram_t::analyze(void)
/* No need */
}
void middle_ram_t::start(const options_t *out_options_)
void middle_ram_t::start(const options_t *options)
{
out_options = out_options_;
cache.reset(
new node_ram_cache(out_options->alloc_chunkwise, out_options->cache));
extra_attributes = options->extra_attributes;
cache.reset(new node_ram_cache(options->alloc_chunkwise, options->cache));
}
void middle_ram_t::stop(osmium::thread::Pool &pool)
@ -196,6 +195,8 @@ idlist_t middle_ram_t::relations_using_way(osmid_t) const
std::shared_ptr<middle_query_t>
middle_ram_t::get_query_instance(std::shared_ptr<middle_t> const &mid) const
{
auto me = std::dynamic_pointer_cast<middle_ram_t>(mid);
assert(me);
// No copy here because readonly access is thread safe.
return std::static_pointer_cast<middle_query_t>(mid);
return std::static_pointer_cast<middle_query_t>(me);
}

10
middle-ram.hpp
View File

@ -9,11 +9,11 @@
#ifndef MIDDLE_RAM_H
#define MIDDLE_RAM_H
#include <array>
#include <memory>
#include <vector>
#include "middle.hpp"
#include <vector>
#include <array>
struct node_ram_cache;
struct options_t;
@ -80,11 +80,12 @@ public:
}
};
struct middle_ram_t : public middle_t {
struct middle_ram_t : public middle_t, public middle_query_t
{
middle_ram_t();
virtual ~middle_ram_t();
void start(const options_t *out_options_) override;
void start(const options_t *options) override;
void stop(osmium::thread::Pool &pool) override;
void analyze(void) override;
void commit(void) override;
@ -152,6 +153,7 @@ private:
elem_cache_t<ramRel, 10> rels;
std::unique_ptr<node_ram_cache> cache;
bool extra_attributes;
/* the previous behaviour of iterate_ways was to delete all ways as they
* were being iterated. this doesn't work now that the output handles its

14
middle.cpp
View File

@ -1,14 +0,0 @@
#include "middle.hpp"
#include "middle-pgsql.hpp"
#include "middle-ram.hpp"
#include <memory>
std::shared_ptr<middle_t> middle_t::create_middle(const bool slim)
{
if(slim)
return std::make_shared<middle_pgsql_t>();
else
return std::make_shared<middle_ram_t>();
}

29
middle.hpp
View File

@ -20,10 +20,10 @@
struct options_t;
/**
* object which stores OSM node/ways/relations from the input file
* Infterface for returning information about raw OSM input data from a cache.
*/
struct middle_query_t {
virtual ~middle_query_t() {}
virtual ~middle_query_t() = 0;
/**
* Retrives node locations for the given node list.
@ -75,15 +75,16 @@ struct middle_query_t {
virtual idlist_t relations_using_way(osmid_t way_id) const = 0;
};
inline middle_query_t::~middle_query_t() = default;
/**
* A specialized middle backend which is persistent, and supports updates
* Interface for storing raw OSM data in an intermediate cache.
*/
struct middle_t : public middle_query_t {
static std::shared_ptr<middle_t> create_middle(bool slim);
struct middle_t
{
virtual ~middle_t() = 0;
virtual ~middle_t() {}
virtual void start(const options_t *out_options_) = 0;
virtual void start(options_t const *out_options) = 0;
virtual void stop(osmium::thread::Pool &pool) = 0;
virtual void analyze(void) = 0;
virtual void commit(void) = 0;
@ -110,12 +111,16 @@ struct middle_t : public middle_query_t {
virtual std::shared_ptr<middle_query_t>
get_query_instance(std::shared_ptr<middle_t> const &mid) const = 0;
const options_t* out_options;
};
inline middle_t::~middle_t() = default;
/**
* Extended interface for permanent caching of raw OSM data.
* It also allows updates.
*/
struct slim_middle_t : public middle_t {
virtual ~slim_middle_t() {}
virtual ~slim_middle_t() = 0;
virtual void nodes_delete(osmid_t id) = 0;
virtual void node_changed(osmid_t id) = 0;
@ -127,4 +132,6 @@ struct slim_middle_t : public middle_t {
virtual void relation_changed(osmid_t id) = 0;
};
inline slim_middle_t::~slim_middle_t() = default;
#endif

10
options.cpp
View File

@ -7,13 +7,13 @@
#else
#define basename /*SKIP IT*/
#endif
#include <stdio.h>
#include <string.h>
#include <stdexcept>
#include <sstream>
#include <thread> // for number of threads
#include <boost/format.hpp>
#include <cstdio>
#include <cstring>
#include <osmium/version.hpp>
#include <sstream>
#include <stdexcept>
#include <thread> // for number of threads
#ifdef HAVE_LUA
extern "C" {

27
osm2pgsql.cpp
View File

@ -24,13 +24,14 @@
*/
#include "config.h"
#include "osmtypes.hpp"
#include "reprojection.hpp"
#include "middle-pgsql.hpp"
#include "middle-ram.hpp"
#include "options.hpp"
#include "parse-osmium.hpp"
#include "middle.hpp"
#include "output.hpp"
#include "osmdata.hpp"
#include "osmtypes.hpp"
#include "output.hpp"
#include "parse-osmium.hpp"
#include "reprojection.hpp"
#include "util.hpp"
#include <time.h>
@ -53,11 +54,19 @@ int main(int argc, char *argv[])
if(options.long_usage_bool)
return 0;
//setup the middle
std::shared_ptr<middle_t> middle = middle_t::create_middle(options.slim);
//setup the middle and backend (output)
std::shared_ptr<middle_t> middle;
std::vector<std::shared_ptr<output_t>> outputs;
//setup the backend (output)
std::vector<std::shared_ptr<output_t> > outputs = output_t::create_outputs(middle.get(), options);
if (options.slim) {
auto mid = std::make_shared<middle_pgsql_t>();
outputs = output_t::create_outputs(mid.get(), options);
middle = std::static_pointer_cast<middle_t>(mid);
} else {
auto mid = std::make_shared<middle_ram_t>();
outputs = output_t::create_outputs(mid.get(), options);
middle = std::static_pointer_cast<middle_t>(mid);
}
//let osmdata orchestrate between the middle and the outs
osmdata_t osmdata(middle, outputs, options.projection);

4
output-multi.cpp
View File

@ -365,7 +365,7 @@ int output_multi_t::process_relation(osmium::Relation const &rel,
if (!filter) {
//TODO: move this into geometry processor, figure a way to come back for tag transform
//grab ways/nodes of the members in the relation, bail if none were used
if (m_relation_helper.set(rel, (middle_t *)m_mid) < 1)
if (m_relation_helper.set(rel, m_mid) < 1)
return 0;
//NOTE: make_polygon is preset here this is to force the tag matching
@ -383,7 +383,7 @@ int output_multi_t::process_relation(osmium::Relation const &rel,
outtags, true);
if (!filter)
{
m_relation_helper.add_way_locations((middle_t *)m_mid);
m_relation_helper.add_way_locations(m_mid);
auto geoms = m_processor->process_relation(
rel, m_relation_helper.data, &m_builder);
for (const auto geom : geoms) {

2
output.cpp
View File

@ -5,7 +5,7 @@
#include "output-multi.hpp"
#include "taginfo_impl.hpp"
#include <string.h>
#include <cstring>
#include <stdexcept>
#include <boost/format.hpp>

3
tests/CMakeLists.txt
View File

@ -2,7 +2,6 @@ add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} ${ARGS})
add_dependencies(check osm2pgsql)
add_library(common-pg STATIC common-pg.cpp common-pg.hpp common-cleanup.cpp common-cleanup.hpp)
add_library(middle-tests STATIC middle-tests.cpp middle-tests.hpp)
set(TESTS
test-expire-tiles.cpp
@ -37,7 +36,7 @@ set(TESTS
foreach (test ${TESTS})
get_filename_component(test_name ${test} NAME_WE)
add_executable(${test_name} ${test})
target_link_libraries(${test_name} osm2pgsql_lib common-pg middle-tests)
target_link_libraries(${test_name} osm2pgsql_lib common-pg)
add_test(NAME ${test_name} COMMAND ${test_name}
WORKING_DIRECTORY ${osm2pgsql_SOURCE_DIR})
add_dependencies(check ${test_name})

286
tests/middle-tests.cpp
View File

@ -1,286 +0,0 @@
#include <iostream>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <cassert>
#include <list>
#include <tuple>
#include "osmtypes.hpp"
#include "reprojection.hpp"
#include "tests/middle-tests.hpp"
#include <osmium/builder/attr.hpp>
#include <osmium/memory/buffer.hpp>
#define BLOCK_SHIFT 13
#define PER_BLOCK (((osmid_t)1) << BLOCK_SHIFT)
// simple osmium buffer to store all the objects in
osmium::memory::Buffer buffer(4096, osmium::memory::Buffer::auto_grow::yes);
// do not project anything
std::shared_ptr<reprojection>
proj(reprojection::create_projection(PROJ_LATLONG));
#define ALLOWED_ERROR 10e-9
bool node_okay(osmium::Location loc, osmium::Node const &expected)
{
if ((loc.lat() > expected.location().lat() + ALLOWED_ERROR) ||
(loc.lat() < expected.location().lat() - ALLOWED_ERROR)) {
std::cerr << "ERROR: Node should have lat=" << expected.location().lat()
<< ", but got back " << loc.lat() << " from middle.\n";
return false;
}
if ((loc.lon() > expected.location().lon() + ALLOWED_ERROR) ||
(loc.lon() < expected.location().lon() - ALLOWED_ERROR)) {
std::cerr << "ERROR: Node should have lon=" << expected.location().lon()
<< ", but got back " << loc.lon() << " from middle.\n";
return false;
}
return true;
}
size_t add_node(osmid_t id, double lat, double lon)
{
using namespace osmium::builder::attr;
return osmium::builder::add_node(buffer, _id(id), _location(lon, lat));
}
size_t way_with_nodes(std::vector<osmid_t> const &ids)
{
using namespace osmium::builder::attr;
return osmium::builder::add_way(buffer, _nodes(ids));
}
int test_node_set(middle_t *mid)
{
buffer.clear();
auto const &node = buffer.get<osmium::Node>(add_node(1234, 12.3456789, 98.7654321));
auto &way = buffer.get<osmium::Way>(way_with_nodes({node.id()}));
// set the node
mid->nodes_set(node);
// get it back
if (mid->nodes_get_list(&(way.nodes())) != way.nodes().size()) {
std::cerr << "ERROR: Unable to get node list.\n";
return 1;
}
// check that it's the same
if (!node_okay(way.nodes()[0].location(), node)) {
return 1;
}
return 0;
}
inline double test_lat(osmid_t id) {
return 1 + 1e-5 * id;
}
int test_nodes_comprehensive_set(middle_t *mid)
{
std::vector<size_t> expected_nodes;
expected_nodes.reserve(PER_BLOCK*8+1);
buffer.clear();
// 2 dense blocks, the second partially filled at the star
for (osmid_t id = 0; id < (PER_BLOCK+(PER_BLOCK >> 1) + 1); ++id)
{
expected_nodes.emplace_back(add_node(id, test_lat(id), 0.0));
}
// 1 dense block, 75% filled
for (osmid_t id = PER_BLOCK*2; id < PER_BLOCK*3; ++id)
{
if ((id % 4 == 0) || (id % 4 == 1) || (id % 4 == 2))
expected_nodes.emplace_back(add_node(id, test_lat(id), 0.0));
}
// 1 dense block, sparsly filled
for (osmid_t id = PER_BLOCK*3; id < PER_BLOCK*4; ++id)
{
if (id % 4 == 0)
expected_nodes.emplace_back(add_node(id, test_lat(id), 0.0));
}
// A lone sparse node
expected_nodes.emplace_back(add_node(PER_BLOCK*5, test_lat(PER_BLOCK*5), 0.0));
// A dense block of alternating positions of zero/non-zero
for (osmid_t id = PER_BLOCK*6; id < PER_BLOCK*7; ++id)
{
if (id % 2 == 0)
expected_nodes.emplace_back(add_node(id, 0.0, 0.0));
else
expected_nodes.emplace_back(add_node(id, test_lat(id), 0.0));
}
expected_nodes.emplace_back(add_node(PER_BLOCK*8, 0.0, 0.0));
expected_nodes.emplace_back(add_node(PER_BLOCK*8+1, 0.0, 0.0));
// Load up the nodes into the middle
std::vector<osmid_t> ids;
for (auto pos : expected_nodes)
{
auto const &node = buffer.get<osmium::Node>(pos);
mid->nodes_set(node);
ids.push_back(node.id());
}
auto &way = buffer.get<osmium::Way>(way_with_nodes(ids));
if (mid->nodes_get_list(&(way.nodes())) != ids.size()) {
std::cerr << "ERROR: Unable to get node list.\n";
return 1;
}
for (size_t i = 0; i < ids.size(); ++i) {
auto const &node = buffer.get<osmium::Node>(expected_nodes[i]);
if (!node_okay(way.nodes()[i].location(), node)) {
return 1;
}
}
return 0;
}
struct test_pending_processor : public middle_t::pending_processor {
test_pending_processor(): pending_ways(), pending_rels() {}
~test_pending_processor() {}
void enqueue_ways(osmid_t id) override
{
pending_ways.push_back(id);
}
void process_ways() override
{
pending_ways.clear();
}
void enqueue_relations(osmid_t id) override
{
pending_rels.push_back(id);
}
void process_relations() override
{
pending_rels.clear();
}
std::list<osmid_t> pending_ways;
std::list<osmid_t> pending_rels;
};
int test_way_set(middle_t *mid)
{
buffer.clear();
osmid_t way_id = 1;
double lat = 12.3456789;
double lon = 98.7654321;
idlist_t nds;
std::vector<size_t> nodes;
// set nodes
for (osmid_t i = 1; i <= 10; ++i) {
nds.push_back(i);
nodes.push_back(add_node(i, lat, lon));
auto const &node = buffer.get<osmium::Node>(nodes.back());
mid->nodes_set(node);
}
// set the way
{
using namespace osmium::builder::attr;
auto pos = osmium::builder::add_way(buffer, _id(way_id), _nodes(nds));
auto const &way = buffer.get<osmium::Way>(pos);
mid->ways_set(way);
}
// commit the setup data
mid->commit();
// get it back
osmium::memory::Buffer relbuf(4096, osmium::memory::Buffer::auto_grow::yes);
{
using namespace osmium::builder::attr;
osmium::builder::add_relation(
relbuf, _id(123), _member(osmium::item_type::node, 132),
_member(osmium::item_type::way, way_id, "outer"));
}
auto const &rel = relbuf.get<osmium::Relation>(0);
auto buf_pos = buffer.committed();
rolelist_t roles;
size_t way_count = mid->rel_way_members_get(rel, &roles, buffer);
if (way_count != 1) { std::cerr << "ERROR: Unable to get way list.\n"; return 1; }
if (roles.size() != 1) {
std::cerr << "Bad length of role ist. Expected 1, got " << roles.size()
<< ".\n";
return 1;
}
if (strcmp(roles[0], "outer") != 0) {
std::cerr << "Bad role. Expected 'outer', got '" << roles[0] << "'.\n";
return 1;
}
auto &way = buffer.get<osmium::Way>(buf_pos);
// check that it's the same
if (way.nodes().size() != nds.size()) {
std::cerr << "ERROR: Way should have " << nds.size() << " nodes, but got back "
<< way.nodes().size() << " from middle.\n";
return 1;
}
if (way.id() != way_id) {
std::cerr << "ERROR: Way should have id=" << way_id << ", but got back "
<< way.id() << " from middle.\n";
return 1;
}
mid->nodes_get_list(&(way.nodes()));
for (size_t i = 0; i < nds.size(); ++i) {
if (way.nodes()[i].location().lon() != lon) {
std::cerr << "ERROR: Way node should have lon=" << lon
<< ", but got back " << way.nodes()[i].location().lon()
<< " from middle.\n";
return 1;
}
if (way.nodes()[i].location().lat() != lat) {
std::cerr << "ERROR: Way node should have lat=" << lat
<< ", but got back " << way.nodes()[i].location().lat()
<< " from middle.\n";
return 1;
}
}
// the way we just inserted should not be pending
test_pending_processor tpp;
mid->iterate_ways(tpp);
if (mid->pending_count() != 0) {
std::cerr << "ERROR: Was expecting no pending ways, but got "
<< mid->pending_count() << " from middle.\n";
return 1;
}
// some middles don't support changing the nodes - they
// don't have diff update ability. here, we will just
// skip the test for that.
if (dynamic_cast<slim_middle_t *>(mid)) {
slim_middle_t *slim = dynamic_cast<slim_middle_t *>(mid);
// finally, try touching a node on a non-pending way. that should
// make it become pending. we just checked that the way is not
// pending, so any change must be due to the node changing.
slim->node_changed(nds[0]);
slim->iterate_ways(tpp);
if (slim->pending_count() != 1) {
std::cerr << "ERROR: Was expecting a single pending way from node update, but got "
<< slim->pending_count() << " from middle.\n";
return 1;
}
}
return 0;
}

317
tests/middle-tests.hpp
View File

@ -1,16 +1,319 @@
#ifndef TESTS_MIDDLE_TEST_HPP
#define TESTS_MIDDLE_TEST_HPP
#include <vector>
#include "middle.hpp"
#include "options.hpp"
#include "output-null.hpp"
#include "reprojection.hpp"
// tests that a single node can be set and retrieved. returns 0 on success.
int test_node_set(middle_t *mid);
#include <osmium/builder/attr.hpp>
#include <osmium/memory/buffer.hpp>
// tests various combinations of nodes being set and retrieved to trigger different cache strategies. returns 0 on success.
int test_nodes_comprehensive_set(middle_t *mid);
template <typename MID>
class test_middle_helper
{
enum
{
BLOCK_SHIFT = 13,
PER_BLOCK = (1ULL << BLOCK_SHIFT)
};
// tests that a single way and supporting nodes can be set and retrieved.
// returns 0 on success.
int test_way_set(middle_t *mid);
struct test_pending_processor : public middle_t::pending_processor
{
void enqueue_ways(osmid_t id) override { pending_ways.push_back(id); }
void process_ways() override { pending_ways.clear(); }
void enqueue_relations(osmid_t id) override
{
pending_rels.push_back(id);
}
void process_relations() override { pending_rels.clear(); }
std::vector<osmid_t> pending_ways;
std::vector<osmid_t> pending_rels;
};
public:
test_middle_helper(options_t const &options)
: m_output(&m_mid, options),
m_buffer(4096, osmium::memory::Buffer::auto_grow::yes),
m_proj(reprojection::create_projection(PROJ_LATLONG))
{
m_mid.start(&options);
}
~test_middle_helper() { commit_and_stop(); }
void start(options_t const *options) { m_mid.start(options); }
void commit_and_stop()
{
osmium::thread::Pool pool(1);
m_mid.commit();
m_mid.stop(pool);
}
// tests that a single node can be set and retrieved. returns 0 on success.
int test_node_set()
{
m_buffer.clear();
auto const &node =
m_buffer.get<osmium::Node>(add_node(1234, 12.3456789, 98.7654321));
auto &way = m_buffer.get<osmium::Way>(way_with_nodes({node.id()}));
// set the node
m_mid.nodes_set(node);
// get it back
if (m_mid.nodes_get_list(&(way.nodes())) != way.nodes().size()) {
std::cerr << "ERROR: Unable to get node list.\n";
return 1;
}
// check that it's the same
if (!node_okay(way.nodes()[0].location(), node)) {
return 1;
}
return 0;
}
// tests various combinations of nodes being set and retrieved to trigger different cache strategies. returns 0 on success.
int test_nodes_comprehensive_set()
{
std::vector<size_t> expected_nodes;
expected_nodes.reserve(PER_BLOCK * 8 + 1);
m_buffer.clear();
// 2 dense blocks, the second partially filled at the star
for (osmid_t id = 0; id < (PER_BLOCK + (PER_BLOCK >> 1) + 1); ++id) {
expected_nodes.emplace_back(add_node(id, test_lat(id), 0.0));
}
// 1 dense block, 75% filled
for (osmid_t id = PER_BLOCK * 2; id < PER_BLOCK * 3; ++id) {
if ((id % 4 == 0) || (id % 4 == 1) || (id % 4 == 2))
expected_nodes.emplace_back(add_node(id, test_lat(id), 0.0));
}
// 1 dense block, sparsly filled
for (osmid_t id = PER_BLOCK * 3; id < PER_BLOCK * 4; ++id) {
if (id % 4 == 0)
expected_nodes.emplace_back(add_node(id, test_lat(id), 0.0));
}
// A lone sparse node
expected_nodes.emplace_back(
add_node(PER_BLOCK * 5, test_lat(PER_BLOCK * 5), 0.0));
// A dense block of alternating positions of zero/non-zero
for (osmid_t id = PER_BLOCK * 6; id < PER_BLOCK * 7; ++id) {
if (id % 2 == 0)
expected_nodes.emplace_back(add_node(id, 0.0, 0.0));
else
expected_nodes.emplace_back(add_node(id, test_lat(id), 0.0));
}
expected_nodes.emplace_back(add_node(PER_BLOCK * 8, 0.0, 0.0));
expected_nodes.emplace_back(add_node(PER_BLOCK * 8 + 1, 0.0, 0.0));
// Load up the nodes into the middle
std::vector<osmid_t> ids;
for (auto pos : expected_nodes) {
auto const &node = m_buffer.get<osmium::Node>(pos);
m_mid.nodes_set(node);
ids.push_back(node.id());
}
auto &way = m_buffer.get<osmium::Way>(way_with_nodes(ids));
if (m_mid.nodes_get_list(&(way.nodes())) != ids.size()) {
std::cerr << "ERROR: Unable to get node list.\n";
return 1;
}
for (size_t i = 0; i < ids.size(); ++i) {
auto const &node = m_buffer.get<osmium::Node>(expected_nodes[i]);
if (!node_okay(way.nodes()[i].location(), node)) {
return 1;
}
}
return 0;
}
// tests that a single way and supporting nodes can be set and retrieved.
// returns 0 on success.
int test_way_set()
{
m_buffer.clear();
osmid_t way_id = 1;
double lat = 12.3456789;
double lon = 98.7654321;
idlist_t nds;
std::vector<size_t> nodes;
// set nodes
for (osmid_t i = 1; i <= 10; ++i) {
nds.push_back(i);
nodes.push_back(add_node(i, lat, lon));
auto const &node = m_buffer.get<osmium::Node>(nodes.back());
m_mid.nodes_set(node);
}
// set the way
{
using namespace osmium::builder::attr;
auto pos =
osmium::builder::add_way(m_buffer, _id(way_id), _nodes(nds));
auto const &way = m_buffer.get<osmium::Way>(pos);
m_mid.ways_set(way);
}
// commit the setup data
m_mid.commit();
// get it back
osmium::memory::Buffer relbuf(4096,
osmium::memory::Buffer::auto_grow::yes);
{
using namespace osmium::builder::attr;
osmium::builder::add_relation(
relbuf, _id(123), _member(osmium::item_type::node, 132),
_member(osmium::item_type::way, way_id, "outer"));
}
auto const &rel = relbuf.get<osmium::Relation>(0);
auto buf_pos = m_buffer.committed();
rolelist_t roles;
size_t way_count = m_mid.rel_way_members_get(rel, &roles, m_buffer);
if (way_count != 1) {
std::cerr << "ERROR: Unable to get way list.\n";
return 1;
}
if (roles.size() != 1) {
std::cerr << "Bad length of role ist. Expected 1, got "
<< roles.size() << ".\n";
return 1;
}
if (strcmp(roles[0], "outer") != 0) {
std::cerr << "Bad role. Expected 'outer', got '" << roles[0]
<< "'.\n";
return 1;
}
auto &way = m_buffer.get<osmium::Way>(buf_pos);
// check that it's the same
if (way.nodes().size() != nds.size()) {
std::cerr << "ERROR: Way should have " << nds.size()
<< " nodes, but got back " << way.nodes().size()
<< " from middle.\n";
return 1;
}
if (way.id() != way_id) {
std::cerr << "ERROR: Way should have id=" << way_id
<< ", but got back " << way.id() << " from middle.\n";
return 1;
}
m_mid.nodes_get_list(&(way.nodes()));
for (size_t i = 0; i < nds.size(); ++i) {
if (way.nodes()[i].location().lon() != lon) {
std::cerr << "ERROR: Way node should have lon=" << lon
<< ", but got back "
<< way.nodes()[i].location().lon()
<< " from middle.\n";
return 1;
}
if (way.nodes()[i].location().lat() != lat) {
std::cerr << "ERROR: Way node should have lat=" << lat
<< ", but got back "
<< way.nodes()[i].location().lat()
<< " from middle.\n";
return 1;
}
}
// the way we just inserted should not be pending
test_pending_processor tpp;
m_mid.iterate_ways(tpp);
if (m_mid.pending_count() != 0) {
std::cerr << "ERROR: Was expecting no pending ways, but got "
<< m_mid.pending_count() << " from middle.\n";
return 1;
}
// some middles don't support changing the nodes - they
// don't have diff update ability. here, we will just
// skip the test for that.
if (dynamic_cast<slim_middle_t *>(&m_mid)) {
slim_middle_t *slim = dynamic_cast<slim_middle_t *>(&m_mid);
// finally, try touching a node on a non-pending way. that should
// make it become pending. we just checked that the way is not
// pending, so any change must be due to the node changing.
slim->node_changed(nds[0]);
slim->iterate_ways(tpp);
if (slim->pending_count() != 1) {
std::cerr << "ERROR: Was expecting a single pending way from "
"node update, but got "
<< slim->pending_count() << " from middle.\n";
return 1;
}
}
return 0;
}
private:
static constexpr double allowed_error() { return 10e-9; }
static bool node_okay(osmium::Location loc, osmium::Node const &expected)
{
if ((loc.lat() > expected.location().lat() + allowed_error()) ||
(loc.lat() < expected.location().lat() - allowed_error())) {
std::cerr << "ERROR: Node should have lat="
<< expected.location().lat() << ", but got back "
<< loc.lat() << " from middle.\n";
return false;
}
if ((loc.lon() > expected.location().lon() + allowed_error()) ||
(loc.lon() < expected.location().lon() - allowed_error())) {
std::cerr << "ERROR: Node should have lon="
<< expected.location().lon() << ", but got back "
<< loc.lon() << " from middle.\n";
return false;
}
return true;
}
size_t add_node(osmid_t id, double lat, double lon)
{
using namespace osmium::builder::attr;
return osmium::builder::add_node(m_buffer, _id(id),
_location(lon, lat));
}
size_t way_with_nodes(std::vector<osmid_t> const &ids)
{
using namespace osmium::builder::attr;
return osmium::builder::add_way(m_buffer, _nodes(ids));
}
static constexpr double test_lat(osmid_t id) { return 1 + 1e-5 * id; }
MID m_mid;
output_null_t m_output;
// simple osmium buffer to store all the objects in
osmium::memory::Buffer m_buffer;
// 1:1 projection
std::shared_ptr<reprojection> m_proj;
};
#endif /* TESTS_MIDDLE_TEST_HPP */

20
tests/mockups.hpp
View File

@ -15,26 +15,16 @@ struct dummy_middle_t : public middle_t {
void commit(void) override { }
void nodes_set(osmium::Node const &) override {}
size_t nodes_get_list(osmium::WayNodeList *) const override { return 0; }
void ways_set(osmium::Way const &) override { }
bool ways_get(osmid_t, osmium::memory::Buffer &) const override { return true; }
size_t rel_way_members_get(osmium::Relation const &, rolelist_t *,
osmium::memory::Buffer &) const override
{
return 0;
}
void relations_set(osmium::Relation const &) override { }
bool relations_get(osmid_t, osmium::memory::Buffer &) const override { return 0; }
void iterate_ways(pending_processor&) override { }
void iterate_relations(pending_processor&) override { }
virtual size_t pending_count() const override { return 0; }
idlist_t relations_using_way(osmid_t) const override { return idlist_t(); }
std::shared_ptr<middle_query_t>
get_query_instance(std::shared_ptr<middle_t> const &mid) const override
{
@ -53,26 +43,16 @@ struct dummy_slim_middle_t : public slim_middle_t {
void commit(void) override { }
void nodes_set(osmium::Node const &) override {}
size_t nodes_get_list(osmium::WayNodeList *) const override { return 0; }
void ways_set(osmium::Way const &) override { }
bool ways_get(osmid_t, osmium::memory::Buffer &) const override { return true; }
size_t rel_way_members_get(osmium::Relation const &, rolelist_t *,
osmium::memory::Buffer &) const override
{
return 0;
}
void relations_set(osmium::Relation const &) override { }
bool relations_get(osmid_t, osmium::memory::Buffer &) const override { return 0; }
void iterate_ways(pending_processor&) override { }
void iterate_relations(pending_processor&) override { }
size_t pending_count() const override { return 0; }
idlist_t relations_using_way(osmid_t) const override { return idlist_t(); }
std::shared_ptr<middle_query_t>
get_query_instance(std::shared_ptr<middle_t> const &mid) const override
{

54
tests/test-middle-flat.cpp
View File

@ -31,48 +31,38 @@ void run_tests(options_t options, const std::string cache_type) {
options.flat_node_file = boost::optional<std::string>(FLAT_NODES_FILE_NAME);
{
middle_pgsql_t mid_pgsql;
output_null_t out_test(&mid_pgsql, options);
test_middle_helper<middle_pgsql_t> t(options);
mid_pgsql.start(&options);
if (test_node_set(&mid_pgsql) != 0) { throw std::runtime_error("test_node_set failed."); }
osmium::thread::Pool pool(1);
mid_pgsql.commit();
mid_pgsql.stop(pool);
if (t.test_node_set() != 0) {
throw std::runtime_error("test_node_set failed.");
}
}
{
middle_pgsql_t mid_pgsql;
output_null_t out_test(&mid_pgsql, options);
test_middle_helper<middle_pgsql_t> t(options);
mid_pgsql.start(&options);
if (test_nodes_comprehensive_set(&mid_pgsql) != 0) { throw std::runtime_error("test_nodes_comprehensive_set failed."); }
osmium::thread::Pool pool(1);
mid_pgsql.commit();
mid_pgsql.stop(pool);
if (t.test_nodes_comprehensive_set() != 0) {
throw std::runtime_error("test_nodes_comprehensive_set failed.");
}
}
/* This should work, but doesn't. More tests are needed that look at updates
without the complication of ways.
*/
/* {
middle_pgsql_t mid_pgsql;
output_null_t out_test(&mid_pgsql, options);
{
test_middle_helper<middle_pgsql_t> t(options);
mid_pgsql.start(&options);
mid_pgsql.commit();
mid_pgsql.stop();
// Switch to append mode because this tests updates
options.append = true;
options.create = false;
mid_pgsql.start(&options);
if (test_way_set(&mid_pgsql) != 0) { throw std::runtime_error("test_way_set failed."); }
t.commit_and_stop();
mid_pgsql.commit();
mid_pgsql.stop();
}*/
// Switch to append mode because this tests updates
options.append = true;
options.create = false;
t.start(&options);
if (t.test_way_set() != 0) {
throw std::runtime_error("test_way_set failed.");
}
}
}
int main(int argc, char *argv[]) {
std::unique_ptr<pg::tempdb> db;

55
tests/test-middle-pgsql.cpp
View File

@ -22,51 +22,34 @@ void run_tests(options_t options, const std::string cache_type) {
options.append = false;
options.create = true;
{
middle_pgsql_t mid_pgsql;
output_null_t out_test(&mid_pgsql, options);
test_middle_helper<middle_pgsql_t> t(options);
mid_pgsql.start(&options);
if (test_node_set(&mid_pgsql) != 0) { throw std::runtime_error("test_node_set failed."); }
osmium::thread::Pool pool(1);
mid_pgsql.commit();
mid_pgsql.stop(pool);
if (t.test_node_set() != 0) {
throw std::runtime_error("test_node_set failed.");
}
}
{
middle_pgsql_t mid_pgsql;
output_null_t out_test(&mid_pgsql, options);
test_middle_helper<middle_pgsql_t> t(options);
mid_pgsql.start(&options);
if (test_nodes_comprehensive_set(&mid_pgsql) != 0) { throw std::runtime_error("test_nodes_comprehensive_set failed."); }
osmium::thread::Pool pool(1);
mid_pgsql.commit();
mid_pgsql.stop(pool);
if (t.test_nodes_comprehensive_set() != 0) {
throw std::runtime_error("test_nodes_comprehensive_set failed.");
}
}
{
middle_pgsql_t mid_pgsql;
output_null_t out_test(&mid_pgsql, options);
test_middle_helper<middle_pgsql_t> t(options);
mid_pgsql.start(&options);
{
osmium::thread::Pool pool(1);
mid_pgsql.commit();
mid_pgsql.stop(pool);
}
t.commit_and_stop();
// Switch to append mode because this tests updates
options.append = true;
options.create = false;
mid_pgsql.start(&options);
if (test_way_set(&mid_pgsql) != 0) { throw std::runtime_error("test_way_set failed."); }
// Switch to append mode because this tests updates
options.append = true;
options.create = false;
t.start(&options);
{
osmium::thread::Pool pool(1);
mid_pgsql.commit();
mid_pgsql.stop(pool);
}
if (t.test_way_set() != 0) {
throw std::runtime_error("test_way_set failed.");
}
}
}
int main(int argc, char *argv[]) {

53
tests/test-middle-ram.cpp
View File

@ -12,40 +12,35 @@
#include "tests/middle-tests.hpp"
void run_tests(const options_t options, const std::string cache_type) {
{
middle_ram_t mid_ram;
output_null_t out_test(&mid_ram, options);
void run_tests(const options_t options, const std::string cache_type)
{
{
test_middle_helper<middle_ram_t> t(options);
mid_ram.start(&options);
if (t.test_node_set() != 0) {
throw std::runtime_error("test_node_set failed with " + cache_type +
" cache.");
}
}
if (test_node_set(&mid_ram) != 0) { throw std::runtime_error("test_node_set failed with " + cache_type + " cache."); }
osmium::thread::Pool pool(1);
mid_ram.commit();
mid_ram.stop(pool);
}
{
middle_ram_t mid_ram;
output_null_t out_test(&mid_ram, options);
{
test_middle_helper<middle_ram_t> t(options);
mid_ram.start(&options);
if (t.test_nodes_comprehensive_set() != 0) {
throw std::runtime_error(
"test_nodes_comprehensive_set failed with " + cache_type +
" cache.");
}
}
if (test_nodes_comprehensive_set(&mid_ram) != 0) { throw std::runtime_error("test_nodes_comprehensive_set failed with " + cache_type + " cache."); }
osmium::thread::Pool pool(1);
mid_ram.commit();
mid_ram.stop(pool);
}
{
middle_ram_t mid_ram;
output_null_t out_test(&mid_ram, options);
{
test_middle_helper<middle_ram_t> t(options);
mid_ram.start(&options);
if (test_way_set(&mid_ram) != 0) { throw std::runtime_error("test_way_set failed with " + cache_type + " cache."); }
osmium::thread::Pool pool(1);
mid_ram.commit();
mid_ram.stop(pool);
}
if (t.test_way_set() != 0) {
throw std::runtime_error("test_way_set failed with " + cache_type +
" cache.");
}
}
}
int main(int argc, char *argv[]) {

17
tests/test-options-parse.cpp
View File

@ -69,18 +69,14 @@ void test_middles()
{
const char* a1[] = {"osm2pgsql", "--slim", "tests/liechtenstein-2013-08-03.osm.pbf"};
options_t options = options_t(len(a1), const_cast<char **>(a1));
std::shared_ptr<middle_t> mid = middle_t::create_middle(options.slim);
if(dynamic_cast<middle_pgsql_t *>(mid.get()) == nullptr)
{
throw std::logic_error("Using slim mode we expected a pgsql middle");
if (!options.slim) {
throw std::logic_error("Using slim mode expected");
}
const char* a2[] = {"osm2pgsql", "tests/liechtenstein-2013-08-03.osm.pbf"};
options = options_t(len(a2), const_cast<char **>(a2));
mid = middle_t::create_middle(options.slim);
if(dynamic_cast<middle_ram_t *>(mid.get()) == nullptr)
{
throw std::logic_error("Using without slim mode we expected a ram middle");
if (options.slim) {
throw std::logic_error("Using without slim mode expected");
}
}
@ -88,7 +84,7 @@ void test_outputs()
{
const char* a1[] = {"osm2pgsql", "-O", "pgsql", "--style", "default.style", "tests/liechtenstein-2013-08-03.osm.pbf"};
options_t options = options_t(len(a1), const_cast<char **>(a1));
std::shared_ptr<middle_t> mid = middle_t::create_middle(options.slim);
auto mid = std::make_shared<middle_ram_t>();
std::vector<std::shared_ptr<output_t> > outs = output_t::create_outputs(mid.get(), options);
output_t* out = outs.front().get();
if(dynamic_cast<output_pgsql_t *>(out) == nullptr)
@ -98,7 +94,6 @@ void test_outputs()
const char* a2[] = {"osm2pgsql", "-O", "gazetteer", "--style", "tests/gazetteer-test.style", "tests/liechtenstein-2013-08-03.osm.pbf"};
options = options_t(len(a2), const_cast<char **>(a2));
mid = middle_t::create_middle(options.slim);
outs = output_t::create_outputs(mid.get(), options);
out = outs.front().get();
if(dynamic_cast<output_gazetteer_t *>(out) == nullptr)
@ -108,7 +103,6 @@ void test_outputs()
const char* a3[] = {"osm2pgsql", "-O", "null", "--style", "default.style", "tests/liechtenstein-2013-08-03.osm.pbf"};
options = options_t(len(a3), const_cast<char **>(a3));
mid = middle_t::create_middle(options.slim);
outs = output_t::create_outputs(mid.get(), options);
out = outs.front().get();
if(dynamic_cast<output_null_t *>(out) == nullptr)
@ -118,7 +112,6 @@ void test_outputs()
const char* a4[] = {"osm2pgsql", "-O", "keine_richtige_ausgabe", "--style", "default.style", "tests/liechtenstein-2013-08-03.osm.pbf"};
options = options_t(len(a4), const_cast<char **>(a4));
mid = middle_t::create_middle(options.slim);
try
{
outs = output_t::create_outputs(mid.get(), options);

10
tests/test-output-multi-line-storage.cpp
View File

@ -7,11 +7,11 @@
#include <stdexcept>
#include <memory>
#include "osmtypes.hpp"
#include "osmdata.hpp"
#include "middle.hpp"
#include "output-multi.hpp"
#include "middle-pgsql.hpp"
#include "options.hpp"
#include "osmdata.hpp"
#include "osmtypes.hpp"
#include "output-multi.hpp"
#include "taginfo_impl.hpp"
#include <sys/types.h>
@ -45,7 +45,7 @@ int main(int argc, char *argv[]) {
options.style = "tests/test_output_multi_line_trivial.style.json";
//setup the middle
std::shared_ptr<middle_t> middle = middle_t::create_middle(options.slim);
auto middle = std::make_shared<middle_pgsql_t>();
//setup the backend (output)
std::vector<std::shared_ptr<output_t> > outputs = output_t::create_outputs(middle.get(), options);

1
tests/test-output-multi-point.cpp
View File

@ -19,7 +19,6 @@
#include <boost/lexical_cast.hpp>
#include "tests/middle-tests.hpp"
#include "tests/common-pg.hpp"
#include "tests/common.hpp"

11
tests/test-output-multi-poly-trivial.cpp
View File

@ -7,11 +7,11 @@
#include <stdexcept>
#include <memory>
#include "osmtypes.hpp"
#include "osmdata.hpp"
#include "middle.hpp"
#include "output-multi.hpp"
#include "middle-pgsql.hpp"
#include "options.hpp"
#include "osmdata.hpp"
#include "osmtypes.hpp"
#include "output-multi.hpp"
#include "taginfo_impl.hpp"
#include <sys/types.h>
@ -19,13 +19,12 @@
#include <boost/lexical_cast.hpp>
#include "tests/middle-tests.hpp"
#include "tests/common-pg.hpp"
#include "tests/common.hpp"
void run_osm2pgsql(options_t &options) {
//setup the middle
std::shared_ptr<middle_t> middle = middle_t::create_middle(options.slim);
auto middle = std::make_shared<middle_pgsql_t>();
//setup the backend (output)
std::vector<std::shared_ptr<output_t> > outputs = output_t::create_outputs(middle.get(), options);

8
tests/test-output-multi-tags.cpp
View File

@ -7,11 +7,11 @@
#include <stdexcept>
#include <memory>
#include "osmtypes.hpp"
#include "middle.hpp"
#include "output-multi.hpp"
#include "middle-pgsql.hpp"
#include "options.hpp"
#include "osmdata.hpp"
#include "osmtypes.hpp"
#include "output-multi.hpp"
#include "taginfo_impl.hpp"
#include <sys/types.h>
@ -45,7 +45,7 @@ int main(int argc, char *argv[]) {
options.style = "tests/test_output_multi_tags.json";
//setup the middle
std::shared_ptr<middle_t> middle = middle_t::create_middle(options.slim);
auto middle = std::make_shared<middle_pgsql_t>();
//setup the backend (output)
std::vector<std::shared_ptr<output_t> > outputs = output_t::create_outputs(middle.get(), options);

1
tests/test-output-pgsql.cpp
View File

@ -20,7 +20,6 @@
#include <boost/lexical_cast.hpp>
#include "tests/middle-tests.hpp"
#include "tests/common-pg.hpp"
#include "tests/common-cleanup.hpp"
#include "tests/common.hpp"