From 34382ba8bdaa886b63ec5f8b0ddc1e928ca50063 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 18 Aug 2015 23:20:48 +0200 Subject: [PATCH] update to libosmium 2.3.0 - remove protobuf dependencies as libosmium now comes with its own pbf parser - fix remaining uses of libxml2 as file format in tests --- .gitignore | 7 - .travis.yml | 4 +- Makefile.am | 32 +- README.md | 7 +- configure.ac | 8 - .../osmium/area/multipolygon_collector.hpp | 1 + contrib/libosmium/osmium/builder/builder.hpp | 24 +- .../osmium/builder/osm_object_builder.hpp | 63 +- contrib/libosmium/osmium/geom/geos.hpp | 35 +- contrib/libosmium/osmium/geom/wkb.hpp | 10 +- .../index/detail/create_map_with_fd.hpp | 2 - .../osmium/index/detail/mmap_vector_base.hpp | 4 +- .../osmium/index/detail/mmap_vector_file.hpp | 3 +- .../osmium/index/map/dense_mmap_array.hpp | 2 +- .../osmium/index/map/sparse_mem_map.hpp | 2 +- contrib/libosmium/osmium/io/any_input.hpp | 4 +- contrib/libosmium/osmium/io/any_output.hpp | 5 +- .../libosmium/osmium/io/bzip2_compression.hpp | 5 + contrib/libosmium/osmium/io/compression.hpp | 5 + .../pbf_type_conv.hpp => debug_output.hpp} | 44 +- .../osmium/io/detail/debug_output_format.hpp | 482 +++++++ .../osmium/io/detail/opl_output_format.hpp | 75 +- contrib/libosmium/osmium/io/detail/pbf.hpp | 23 +- .../osmium/io/detail/pbf_decoder.hpp | 760 +++++++++++ .../osmium/io/detail/pbf_input_format.hpp | 94 +- .../osmium/io/detail/pbf_output_format.hpp | 1143 ++++++----------- .../libosmium/osmium/io/detail/pbf_parser.hpp | 456 ------- .../osmium/io/detail/pbf_stringtable.hpp | 218 ---- .../osmium/io/detail/protobuf_tags.hpp | 170 +++ .../osmium/io/detail/string_table.hpp | 250 ++++ .../osmium/io/detail/xml_input_format.hpp | 30 +- .../osmium/io/detail/xml_output_format.hpp | 60 +- contrib/libosmium/osmium/io/detail/zlib.hpp | 15 +- contrib/libosmium/osmium/io/file.hpp | 68 +- contrib/libosmium/osmium/io/file_format.hpp | 8 +- .../libosmium/osmium/io/gzip_compression.hpp | 5 + contrib/libosmium/osmium/io/pbf_input.hpp | 1 - contrib/libosmium/osmium/io/pbf_output.hpp | 1 - contrib/libosmium/osmium/memory/buffer.hpp | 1 - .../libosmium/osmium/memory/collection.hpp | 1 - contrib/libosmium/osmium/memory/item.hpp | 1 - contrib/libosmium/osmium/osm/changeset.hpp | 1 - contrib/libosmium/osmium/osm/crc.hpp | 223 ++++ contrib/libosmium/osmium/osm/entity.hpp | 1 + contrib/libosmium/osmium/osm/node_ref.hpp | 2 +- contrib/libosmium/osmium/osm/timestamp.hpp | 3 +- contrib/libosmium/osmium/thread/queue.hpp | 4 +- contrib/libosmium/osmium/thread/util.hpp | 2 +- contrib/libosmium/osmium/util/data_file.hpp | 2 + contrib/libosmium/osmium/util/delta.hpp | 54 +- contrib/libosmium/osmium/util/endian.hpp | 45 + contrib/libosmium/osmium/util/file.hpp | 1 + .../libosmium/osmium/util/memory_mapping.hpp | 113 +- contrib/libosmium/osmium/util/minmax.hpp | 3 - contrib/libosmium/protozero/byteswap.hpp | 49 + contrib/libosmium/protozero/exception.hpp | 68 + contrib/libosmium/protozero/pbf_builder.hpp | 111 ++ contrib/libosmium/protozero/pbf_message.hpp | 50 + contrib/libosmium/protozero/pbf_reader.hpp | 1059 +++++++++++++++ contrib/libosmium/protozero/pbf_types.hpp | 49 + contrib/libosmium/protozero/pbf_writer.hpp | 664 ++++++++++ contrib/libosmium/protozero/varint.hpp | 136 ++ contrib/protobuf/fileformat.proto | 54 - contrib/protobuf/osmformat.proto | 260 ---- contrib/protobuf/osmpbf/osmpbf.h | 26 - tests/test-hstore-match-only.cpp | 2 +- tests/test-options-parse.cpp | 2 +- tests/test-output-multi-line-storage.cpp | 2 +- tests/test-output-multi-poly-trivial.cpp | 2 +- tests/test-output-multi-tags.cpp | 2 +- tests/test-output-pgsql-schema.cpp | 2 +- tests/test-output-pgsql-z_order.cpp | 2 +- tests/test-output-pgsql.cpp | 4 +- 73 files changed, 5004 insertions(+), 2123 deletions(-) rename contrib/libosmium/osmium/io/{detail/pbf_type_conv.hpp => debug_output.hpp} (54%) create mode 100644 contrib/libosmium/osmium/io/detail/debug_output_format.hpp create mode 100644 contrib/libosmium/osmium/io/detail/pbf_decoder.hpp delete mode 100644 contrib/libosmium/osmium/io/detail/pbf_parser.hpp delete mode 100644 contrib/libosmium/osmium/io/detail/pbf_stringtable.hpp create mode 100644 contrib/libosmium/osmium/io/detail/protobuf_tags.hpp create mode 100644 contrib/libosmium/osmium/io/detail/string_table.hpp create mode 100644 contrib/libosmium/osmium/osm/crc.hpp create mode 100644 contrib/libosmium/osmium/util/endian.hpp create mode 100644 contrib/libosmium/protozero/byteswap.hpp create mode 100644 contrib/libosmium/protozero/exception.hpp create mode 100644 contrib/libosmium/protozero/pbf_builder.hpp create mode 100644 contrib/libosmium/protozero/pbf_message.hpp create mode 100644 contrib/libosmium/protozero/pbf_reader.hpp create mode 100644 contrib/libosmium/protozero/pbf_types.hpp create mode 100644 contrib/libosmium/protozero/pbf_writer.hpp create mode 100644 contrib/libosmium/protozero/varint.hpp delete mode 100644 contrib/protobuf/fileformat.proto delete mode 100644 contrib/protobuf/osmformat.proto delete mode 100644 contrib/protobuf/osmpbf/osmpbf.h diff --git a/.gitignore b/.gitignore index 414c2644..c498acea 100644 --- a/.gitignore +++ b/.gitignore @@ -20,13 +20,6 @@ missing nodecachefilereader osm2pgsql -contrib/protobuf/osmpbf/fileformat.pb.h -contrib/protobuf/osmpbf/osmformat.pb.h -fileformat.pb.cc -fileformat.pb.h -osmformat.pb.cc -osmformat.pb.h - Makefile config.h config.log diff --git a/.travis.yml b/.travis.yml index 48b08ca7..7b79af49 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,6 @@ addons: - g++-4.8 - libtool - libexpat1-dev - - libprotobuf-dev - - protobuf-compiler - libgeos-dev - libgeos++-dev - libpq-dev @@ -35,7 +33,7 @@ matrix: env: BUILDCXXFLAGS='-pedantic -Wall -std=c++11' before_install: - if [[ $TRAVIS_OS_NAME == 'osx' ]]; then - brew install protobuf lua; + brew install lua; fi # update versions install: diff --git a/Makefile.am b/Makefile.am index 6b26e455..844902b0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -112,36 +112,12 @@ TESTS = $(check_PROGRAMS) tests/regression-test.sh TEST_EXTENSIONS = .sh SH_LOG_COMPILER = sh -libosm2pgsql_la_SOURCES += fileformat.pb.cc osmformat.pb.cc - -fileformat.pb.cc: contrib/protobuf/fileformat.proto - $(AM_V_GEN) $(PROTOC) --proto_path=contrib/protobuf --cpp_out=. $< - -contrib/protobuf/osmpbf/fileformat.pb.h: fileformat.pb.cc - cp fileformat.pb.h $@ - -osmformat.pb.cc: contrib/protobuf/osmformat.proto - $(AM_V_GEN) $(PROTOC) --proto_path=contrib/protobuf --cpp_out=. $< - -contrib/protobuf/osmpbf/osmformat.pb.h: osmformat.pb.cc - cp osmformat.pb.h $@ - -parse-osmium.cpp: contrib/protobuf/osmpbf/fileformat.pb.h contrib/protobuf/osmpbf/osmformat.pb.h - -BUILT_SOURCES = \ - fileformat.pb.cc contrib/protobuf/osmpbf/fileformat.pb.h \ - osmformat.pb.cc contrib/protobuf/osmpbf/osmformat.pb.h - -CLEANFILES = \ - fileformat.pb.cc fileformat.pb.h contrib/protobuf/osmpbf/fileformat.pb.h \ - osmformat.pb.cc osmformat.pb.h contrib/protobuf/osmpbf/osmformat.pb.h - osm2pgsqldir = $(datadir)/osm2pgsql -AM_CFLAGS = @PTHREAD_CFLAGS@ @LFS_CFLAGS@ @POSTGRESQL_CPPFLAGS@ @EXPAT_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ @PROTOBUF_CFLAGS@ @ZLIB_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -DVERSION='"@PACKAGE_VERSION@"' @LUA_INCLUDE@ -AM_CPPFLAGS = -isystem contrib/protobuf -isystem contrib/libosmium @PTHREAD_CFLAGS@ @POSTGRESQL_CPPFLAGS@ @EXPAT_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -isystem geos-fallback @LUA_INCLUDE@ @BOOST_CPPFLAGS@ +AM_CFLAGS = @PTHREAD_CFLAGS@ @LFS_CFLAGS@ @POSTGRESQL_CPPFLAGS@ @EXPAT_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ @ZLIB_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -DVERSION='"@PACKAGE_VERSION@"' @LUA_INCLUDE@ +AM_CPPFLAGS = -isystem contrib/libosmium @PTHREAD_CFLAGS@ @POSTGRESQL_CPPFLAGS@ @EXPAT_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -isystem geos-fallback @LUA_INCLUDE@ @BOOST_CPPFLAGS@ -GLOBAL_LDFLAGS = @PTHREAD_CFLAGS@ @ZLIB_LDFLAGS@ @ZLIB_LIBS@ @POSTGRESQL_LDFLAGS@ @POSTGRESQL_LIBS@ @EXPAT_LDFLAGS@ @BZIP2_LDFLAGS@ @BZIP2_LIBS@ @GEOS_LDFLAGS@ @GEOS_LIBS@ @PROJ_LDFLAGS@ @PROJ_LIBS@ @PROTOBUF_LIBS@ -L/usr/lib/x86_64-linux-gnu @EXPAT_LIBS@ @LUA_LIB@ @BOOST_LDFLAGS@ @BOOST_FILESYSTEM_LIB@ @BOOST_SYSTEM_LIB@ @BOOST_THREAD_LIB@ +GLOBAL_LDFLAGS = @PTHREAD_CFLAGS@ @ZLIB_LDFLAGS@ @ZLIB_LIBS@ @POSTGRESQL_LDFLAGS@ @POSTGRESQL_LIBS@ @EXPAT_LDFLAGS@ @BZIP2_LDFLAGS@ @BZIP2_LIBS@ @GEOS_LDFLAGS@ @GEOS_LIBS@ @PROJ_LDFLAGS@ @PROJ_LIBS@ -L/usr/lib/x86_64-linux-gnu @EXPAT_LIBS@ @LUA_LIB@ @BOOST_LDFLAGS@ @BOOST_FILESYSTEM_LIB@ @BOOST_SYSTEM_LIB@ @BOOST_THREAD_LIB@ osm2pgsql_LDADD += $(GLOBAL_LDFLAGS) tests_test_parse_xml2_LDADD += $(GLOBAL_LDFLAGS) tests_test_middle_ram_LDADD += $(GLOBAL_LDFLAGS) @@ -171,8 +147,6 @@ man1_MANS = docs/osm2pgsql.1 docs/nodecachefilereader.1 EXTRA_DIST = osm2pgsql.spec.in \ osm2pgsql.spec \ - contrib/protobuf/fileformat.proto \ - contrib/protobuf/osmformat.proto \ debian \ $(osm2pgsql_DATA) diff --git a/README.md b/README.md index 6d18a536..ffdf904e 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,6 @@ to configure and build itself and requires * [proj](http://proj.osgeo.org/) * [bzip2](http://www.bzip.org/) * [zlib](http://www.zlib.net/) -* [Protocol Buffers](https://developers.google.com/protocol-buffers/) * [PostgreSQL](http://www.postgresql.org/) client libraries * [Lua](http://www.lua.org/) (Optional, used for [Lua tag transforms](docs/lua.md)) @@ -52,7 +51,7 @@ To install on a Debian or Ubuntu system, first install the prerequisites: sudo apt-get install autoconf automake libtool make g++ pkg-config libboost-dev \ libboost-system-dev libboost-filesystem-dev libboost-thread-dev libexpat1-dev \ libgeos-dev libgeos++-dev libpq-dev libbz2-dev libproj-dev zlib1g-dev \ - protobuf-compiler libprotobuf-dev lua5.2 liblua5.2-dev + lua5.2 liblua5.2-dev ``` To install on a Fedora system, use @@ -60,7 +59,7 @@ To install on a Fedora system, use ```sh sudo yum install gcc-c++ automake libtool pkgconfig boost-devel \ expat-devel bzip2-devel postgresql-devel geos-devel proj-devel \ - lua-devel protobuf-devel protobuf-lite-devel + lua-devel ``` To install on a FreeBSD system, use @@ -68,7 +67,7 @@ To install on a FreeBSD system, use ```sh pkg install devel/git devel/autoconf devel/automake devel/gmake devel/libtool \ textproc/expat2 graphics/geos graphics/proj databases/postgresql94-client \ - devel/boost-libs devel/protobuf lang/lua52 devel/pkgconf + devel/boost-libs lang/lua52 devel/pkgconf ``` Then you should be able to bootstrap the build system: diff --git a/configure.ac b/configure.ac index f7c777a0..2665cb5a 100644 --- a/configure.ac +++ b/configure.ac @@ -70,14 +70,6 @@ then AC_MSG_ERROR([required library not found]); fi -dnl Check for protobuf library and protoc binary -PKG_CHECK_MODULES(PROTOBUF, protobuf-lite >= 2.4.0, - [ AC_CHECK_PROG([PROTOC], [protoc], [protoc]) - AS_IF([test "x${PROTOC}" == "x"], - [AC_MSG_ERROR([protoc compiler not found.])]) - ], [AC_MSG_ERROR([protobuf development libraries not found.])] -) - dnl Check for PostgresSQL client library AX_LIB_POSTGRESQL if test "x$POSTGRESQL_VERSION" = "x" diff --git a/contrib/libosmium/osmium/area/multipolygon_collector.hpp b/contrib/libosmium/osmium/area/multipolygon_collector.hpp index 2cb8fe13..bf2a4ce8 100644 --- a/contrib/libosmium/osmium/area/multipolygon_collector.hpp +++ b/contrib/libosmium/osmium/area/multipolygon_collector.hpp @@ -42,6 +42,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include #include #include #include diff --git a/contrib/libosmium/osmium/builder/builder.hpp b/contrib/libosmium/osmium/builder/builder.hpp index dcb95e2a..4424d88e 100644 --- a/contrib/libosmium/osmium/builder/builder.hpp +++ b/contrib/libosmium/osmium/builder/builder.hpp @@ -147,6 +147,7 @@ namespace osmium { * @param length Length of data in bytes. If data is a * \0-terminated string, length must contain the * \0 byte. + * @returns The number of bytes appended (length). */ osmium::memory::item_size_type append(const char* data, const osmium::memory::item_size_type length) { unsigned char* target = m_buffer.reserve_space(length); @@ -156,11 +157,24 @@ namespace osmium { /** * Append \0-terminated string to buffer. + * + * @param str \0-terminated string. + * @returns The number of bytes appended (strlen(str) + 1). */ osmium::memory::item_size_type append(const char* str) { return append(str, static_cast(std::strlen(str) + 1)); } + /** + * Append '\0' to the buffer. + * + * @returns The number of bytes appended (always 1). + */ + osmium::memory::item_size_type append_zero() { + *m_buffer.reserve_space(1) = '\0'; + return 1; + } + /// Return the buffer this builder is using. osmium::memory::Buffer& buffer() noexcept { return m_buffer; @@ -188,11 +202,11 @@ namespace osmium { * Add user name to buffer. * * @param user Pointer to user name. - * @param length Length of user name including \0 byte. + * @param length Length of user name (without \0 termination). */ void add_user(const char* user, const string_size_type length) { - object().set_user_size(length); - add_size(append(user, length)); + object().set_user_size(length + 1); + add_size(append(user, length) + append_zero()); add_padding(true); } @@ -202,7 +216,7 @@ namespace osmium { * @param user Pointer to \0-terminated user name. */ void add_user(const char* user) { - add_user(user, static_cast_with_assert(std::strlen(user) + 1)); + add_user(user, static_cast_with_assert(std::strlen(user))); } /** @@ -211,7 +225,7 @@ namespace osmium { * @param user User name. */ void add_user(const std::string& user) { - add_user(user.data(), static_cast_with_assert(user.size() + 1)); + add_user(user.data(), static_cast_with_assert(user.size())); } }; // class ObjectBuilder diff --git a/contrib/libosmium/osmium/builder/osm_object_builder.hpp b/contrib/libosmium/osmium/builder/osm_object_builder.hpp index 058f89e8..074076ce 100644 --- a/contrib/libosmium/osmium/builder/osm_object_builder.hpp +++ b/contrib/libosmium/osmium/builder/osm_object_builder.hpp @@ -72,13 +72,25 @@ namespace osmium { /** * Add tag to buffer. * - * @param key Tag key. - * @param value Tag value. + * @param key Tag key (0-terminated string). + * @param value Tag value (0-terminated string). */ void add_tag(const char* key, const char* value) { add_size(append(key) + append(value)); } + /** + * Add tag to buffer. + * + * @param key Pointer to tag key. + * @param key_length Length of key (not including the \0 byte). + * @param value Pointer to tag value. + * @param value_length Length of value (not including the \0 byte). + */ + void add_tag(const char* key, const string_size_type key_length, const char* value, const string_size_type value_length) { + add_size(append(key, key_length) + append_zero() + append(value, value_length) + append_zero()); + } + /** * Add tag to buffer. * @@ -128,11 +140,11 @@ namespace osmium { * @param member Relation member object where the length of the role * will be set. * @param role The role. - * @param length Length of role string including \0 termination. + * @param length Length of role (without \0 termination). */ void add_role(osmium::RelationMember& member, const char* role, const string_size_type length) { - member.set_role_size(length); - add_size(append(role, length)); + member.set_role_size(length + 1); + add_size(append(role, length) + append_zero()); add_padding(true); } @@ -144,7 +156,7 @@ namespace osmium { * @param role \0-terminated role. */ void add_role(osmium::RelationMember& member, const char* role) { - add_role(member, role, static_cast_with_assert(std::strlen(role) + 1)); + add_role(member, role, static_cast_with_assert(std::strlen(role))); } /** @@ -155,7 +167,7 @@ namespace osmium { * @param role Role. */ void add_role(osmium::RelationMember& member, const std::string& role) { - add_role(member, role.data(), static_cast_with_assert(role.size() + 1)); + add_role(member, role.data(), static_cast_with_assert(role.size())); } public: @@ -174,18 +186,33 @@ namespace osmium { * @param type The type (node, way, or relation). * @param ref The ID of the member. * @param role The role of the member. + * @param role_length Length of the role (without \0 termination). + * @param full_member Optional pointer to the member object. If it + * is available a copy will be added to the + * relation. + */ + void add_member(osmium::item_type type, object_id_type ref, const char* role, const string_size_type role_length, const osmium::OSMObject* full_member = nullptr) { + osmium::RelationMember* member = reserve_space_for(); + new (member) osmium::RelationMember(ref, type, full_member != nullptr); + add_size(sizeof(RelationMember)); + add_role(*member, role, role_length); + if (full_member) { + add_item(full_member); + } + } + + /** + * Add a member to the relation. + * + * @param type The type (node, way, or relation). + * @param ref The ID of the member. + * @param role The role of the member (\0 terminated string). * @param full_member Optional pointer to the member object. If it * is available a copy will be added to the * relation. */ void add_member(osmium::item_type type, object_id_type ref, const char* role, const osmium::OSMObject* full_member = nullptr) { - osmium::RelationMember* member = reserve_space_for(); - new (member) osmium::RelationMember(ref, type, full_member != nullptr); - add_size(sizeof(RelationMember)); - add_role(*member, role); - if (full_member) { - add_item(full_member); - } + add_member(type, ref, role, strlen(role), full_member); } /** @@ -199,13 +226,7 @@ namespace osmium { * relation. */ void add_member(osmium::item_type type, object_id_type ref, const std::string& role, const osmium::OSMObject* full_member = nullptr) { - osmium::RelationMember* member = reserve_space_for(); - new (member) osmium::RelationMember(ref, type, full_member != nullptr); - add_size(sizeof(RelationMember)); - add_role(*member, role); - if (full_member) { - add_item(full_member); - } + add_member(type, ref, role.data(), role.size(), full_member); } }; // class RelationMemberListBuilder diff --git a/contrib/libosmium/osmium/geom/geos.hpp b/contrib/libosmium/osmium/geom/geos.hpp index d4105b0e..771b0873 100644 --- a/contrib/libosmium/osmium/geom/geos.hpp +++ b/contrib/libosmium/osmium/geom/geos.hpp @@ -42,6 +42,7 @@ DEALINGS IN THE SOFTWARE. * @attention If you include this file, you'll need to link with `libgeos`. */ +#include #include #include @@ -82,8 +83,9 @@ namespace osmium { class GEOSFactoryImpl { - geos::geom::PrecisionModel m_precision_model; - geos::geom::GeometryFactory m_geos_factory; + std::unique_ptr m_precision_model; + std::unique_ptr m_our_geos_factory; + geos::geom::GeometryFactory* m_geos_factory; std::unique_ptr m_coordinate_sequence; std::vector> m_rings; @@ -97,16 +99,23 @@ namespace osmium { typedef std::unique_ptr multipolygon_type; typedef std::unique_ptr ring_type; + explicit GEOSFactoryImpl(geos::geom::GeometryFactory& geos_factory) : + m_precision_model(nullptr), + m_our_geos_factory(nullptr), + m_geos_factory(&geos_factory) { + } + explicit GEOSFactoryImpl(int srid = -1) : - m_precision_model(), - m_geos_factory(&m_precision_model, srid) { + m_precision_model(new geos::geom::PrecisionModel), + m_our_geos_factory(new geos::geom::GeometryFactory(m_precision_model.get(), srid)), + m_geos_factory(m_our_geos_factory.get()) { } /* Point */ point_type make_point(const osmium::geom::Coordinates& xy) const { try { - return point_type(m_geos_factory.createPoint(geos::geom::Coordinate(xy.x, xy.y))); + return point_type(m_geos_factory->createPoint(geos::geom::Coordinate(xy.x, xy.y))); } catch (geos::util::GEOSException& e) { THROW(osmium::geos_geometry_error(e.what())); } @@ -116,7 +125,7 @@ namespace osmium { void linestring_start() { try { - m_coordinate_sequence.reset(m_geos_factory.getCoordinateSequenceFactory()->create(static_cast(0), 2)); + m_coordinate_sequence.reset(m_geos_factory->getCoordinateSequenceFactory()->create(static_cast(0), 2)); } catch (geos::util::GEOSException& e) { THROW(osmium::geos_geometry_error(e.what())); } @@ -132,7 +141,7 @@ namespace osmium { linestring_type linestring_finish(size_t /* num_points */) { try { - return linestring_type(m_geos_factory.createLineString(m_coordinate_sequence.release())); + return linestring_type(m_geos_factory->createLineString(m_coordinate_sequence.release())); } catch (geos::util::GEOSException& e) { THROW(osmium::geos_geometry_error(e.what())); } @@ -155,7 +164,7 @@ namespace osmium { std::transform(std::next(m_rings.begin(), 1), m_rings.end(), std::back_inserter(*inner_rings), [](std::unique_ptr& r) { return r.release(); }); - m_polygons.emplace_back(m_geos_factory.createPolygon(m_rings[0].release(), inner_rings)); + m_polygons.emplace_back(m_geos_factory->createPolygon(m_rings[0].release(), inner_rings)); m_rings.clear(); } catch (geos::util::GEOSException& e) { THROW(osmium::geos_geometry_error(e.what())); @@ -164,7 +173,7 @@ namespace osmium { void multipolygon_outer_ring_start() { try { - m_coordinate_sequence.reset(m_geos_factory.getCoordinateSequenceFactory()->create(static_cast(0), 2)); + m_coordinate_sequence.reset(m_geos_factory->getCoordinateSequenceFactory()->create(static_cast(0), 2)); } catch (geos::util::GEOSException& e) { THROW(osmium::geos_geometry_error(e.what())); } @@ -172,7 +181,7 @@ namespace osmium { void multipolygon_outer_ring_finish() { try { - m_rings.emplace_back(m_geos_factory.createLinearRing(m_coordinate_sequence.release())); + m_rings.emplace_back(m_geos_factory->createLinearRing(m_coordinate_sequence.release())); } catch (geos::util::GEOSException& e) { THROW(osmium::geos_geometry_error(e.what())); } @@ -180,7 +189,7 @@ namespace osmium { void multipolygon_inner_ring_start() { try { - m_coordinate_sequence.reset(m_geos_factory.getCoordinateSequenceFactory()->create(static_cast(0), 2)); + m_coordinate_sequence.reset(m_geos_factory->getCoordinateSequenceFactory()->create(static_cast(0), 2)); } catch (geos::util::GEOSException& e) { THROW(osmium::geos_geometry_error(e.what())); } @@ -188,7 +197,7 @@ namespace osmium { void multipolygon_inner_ring_finish() { try { - m_rings.emplace_back(m_geos_factory.createLinearRing(m_coordinate_sequence.release())); + m_rings.emplace_back(m_geos_factory->createLinearRing(m_coordinate_sequence.release())); } catch (geos::util::GEOSException& e) { THROW(osmium::geos_geometry_error(e.what())); } @@ -209,7 +218,7 @@ namespace osmium { return p.release(); }); m_polygons.clear(); - return multipolygon_type(m_geos_factory.createMultiPolygon(polygons)); + return multipolygon_type(m_geos_factory->createMultiPolygon(polygons)); } catch (geos::util::GEOSException& e) { THROW(osmium::geos_geometry_error(e.what())); } diff --git a/contrib/libosmium/osmium/geom/wkb.hpp b/contrib/libosmium/osmium/geom/wkb.hpp index 2f32fe33..a290c02d 100644 --- a/contrib/libosmium/osmium/geom/wkb.hpp +++ b/contrib/libosmium/osmium/geom/wkb.hpp @@ -37,18 +37,10 @@ DEALINGS IN THE SOFTWARE. #include #include -// Windows is only available for little endian architectures -// http://stackoverflow.com/questions/6449468/can-i-safely-assume-that-windows-installations-will-always-be-little-endian -#if !defined(_WIN32) && !defined(__APPLE__) -# include -#else -# define __LITTLE_ENDIAN 1234 -# define __BYTE_ORDER __LITTLE_ENDIAN -#endif - #include #include #include +#include namespace osmium { diff --git a/contrib/libosmium/osmium/index/detail/create_map_with_fd.hpp b/contrib/libosmium/osmium/index/detail/create_map_with_fd.hpp index 29dc1dc6..5ccbfc80 100644 --- a/contrib/libosmium/osmium/index/detail/create_map_with_fd.hpp +++ b/contrib/libosmium/osmium/index/detail/create_map_with_fd.hpp @@ -39,8 +39,6 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include -#include #include namespace osmium { diff --git a/contrib/libosmium/osmium/index/detail/mmap_vector_base.hpp b/contrib/libosmium/osmium/index/detail/mmap_vector_base.hpp index 2400f89d..9b647681 100644 --- a/contrib/libosmium/osmium/index/detail/mmap_vector_base.hpp +++ b/contrib/libosmium/osmium/index/detail/mmap_vector_base.hpp @@ -34,7 +34,7 @@ DEALINGS IN THE SOFTWARE. */ #include -#include +#include // IWYU pragma: keep #include #include @@ -62,7 +62,7 @@ namespace osmium { explicit mmap_vector_base(int fd, size_t capacity, size_t size = 0) : m_size(size), - m_mapping(capacity, true, fd) { + m_mapping(capacity, osmium::util::MemoryMapping::mapping_mode::write_shared, fd) { } explicit mmap_vector_base(size_t capacity = mmap_vector_size_increment) : diff --git a/contrib/libosmium/osmium/index/detail/mmap_vector_file.hpp b/contrib/libosmium/osmium/index/detail/mmap_vector_file.hpp index 1336003a..1dadbcb4 100644 --- a/contrib/libosmium/osmium/index/detail/mmap_vector_file.hpp +++ b/contrib/libosmium/osmium/index/detail/mmap_vector_file.hpp @@ -33,10 +33,9 @@ DEALINGS IN THE SOFTWARE. */ -#include - #include #include +#include namespace osmium { diff --git a/contrib/libosmium/osmium/index/map/dense_mmap_array.hpp b/contrib/libosmium/osmium/index/map/dense_mmap_array.hpp index fc60a1ef..a912aebd 100644 --- a/contrib/libosmium/osmium/index/map/dense_mmap_array.hpp +++ b/contrib/libosmium/osmium/index/map/dense_mmap_array.hpp @@ -35,7 +35,7 @@ DEALINGS IN THE SOFTWARE. #ifdef __linux__ -#include +#include // IWYU pragma: keep #include #define OSMIUM_HAS_INDEX_MAP_DENSE_MMAP_ARRAY diff --git a/contrib/libosmium/osmium/index/map/sparse_mem_map.hpp b/contrib/libosmium/osmium/index/map/sparse_mem_map.hpp index d053155f..2b9048b3 100644 --- a/contrib/libosmium/osmium/index/map/sparse_mem_map.hpp +++ b/contrib/libosmium/osmium/index/map/sparse_mem_map.hpp @@ -33,7 +33,7 @@ DEALINGS IN THE SOFTWARE. */ -#include +#include // IWYU pragma: keep (for std::copy) #include #include #include diff --git a/contrib/libosmium/osmium/io/any_input.hpp b/contrib/libosmium/osmium/io/any_input.hpp index 633fab31..d16d069a 100644 --- a/contrib/libosmium/osmium/io/any_input.hpp +++ b/contrib/libosmium/osmium/io/any_input.hpp @@ -39,8 +39,8 @@ DEALINGS IN THE SOFTWARE. * Include this file if you want to read all kinds of OSM files. * * @attention If you include this file, you'll need to link with - * `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only), - * `libexpat`, `libz`, `libbz2`, and enable multithreading. + * `ws2_32` (Windows only), `libexpat`, `libz`, `libbz2`, + * and enable multithreading. */ #include // IWYU pragma: export diff --git a/contrib/libosmium/osmium/io/any_output.hpp b/contrib/libosmium/osmium/io/any_output.hpp index 63de3ff2..990a27ba 100644 --- a/contrib/libosmium/osmium/io/any_output.hpp +++ b/contrib/libosmium/osmium/io/any_output.hpp @@ -39,12 +39,13 @@ DEALINGS IN THE SOFTWARE. * Include this file if you want to write all kinds of OSM files. * * @attention If you include this file, you'll need to link with - * `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only), - * `libz`, `libbz2`, and enable multithreading. + * `ws2_32` (Windows only), `libz`, `libbz2`, and enable + * multithreading. */ #include // IWYU pragma: export +#include // IWYU pragma: export #include // IWYU pragma: export #include // IWYU pragma: export #include // IWYU pragma: export diff --git a/contrib/libosmium/osmium/io/bzip2_compression.hpp b/contrib/libosmium/osmium/io/bzip2_compression.hpp index 7e86c154..e961a87a 100644 --- a/contrib/libosmium/osmium/io/bzip2_compression.hpp +++ b/contrib/libosmium/osmium/io/bzip2_compression.hpp @@ -274,11 +274,16 @@ namespace osmium { namespace { +// we want the register_compression() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_bzip2_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::bzip2, [](int fd) { return new osmium::io::Bzip2Compressor(fd); }, [](int fd) { return new osmium::io::Bzip2Decompressor(fd); }, [](const char* buffer, size_t size) { return new osmium::io::Bzip2BufferDecompressor(buffer, size); } ); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/contrib/libosmium/osmium/io/compression.hpp b/contrib/libosmium/osmium/io/compression.hpp index c1f8de2e..25297618 100644 --- a/contrib/libosmium/osmium/io/compression.hpp +++ b/contrib/libosmium/osmium/io/compression.hpp @@ -266,11 +266,16 @@ namespace osmium { namespace { +// we want the register_compression() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_no_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::none, [](int fd) { return new osmium::io::NoCompressor(fd); }, [](int fd) { return new osmium::io::NoDecompressor(fd); }, [](const char* buffer, size_t size) { return new osmium::io::NoDecompressor(buffer, size); } ); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/contrib/libosmium/osmium/io/detail/pbf_type_conv.hpp b/contrib/libosmium/osmium/io/debug_output.hpp similarity index 54% rename from contrib/libosmium/osmium/io/detail/pbf_type_conv.hpp rename to contrib/libosmium/osmium/io/debug_output.hpp index 799869a7..2836f798 100644 --- a/contrib/libosmium/osmium/io/detail/pbf_type_conv.hpp +++ b/contrib/libosmium/osmium/io/debug_output.hpp @@ -1,5 +1,5 @@ -#ifndef OSMIUM_IO_DETAIL_PBF_TYPE_CONV_HPP -#define OSMIUM_IO_DETAIL_PBF_TYPE_CONV_HPP +#ifndef OSMIUM_IO_DEBUG_OUTPUT_HPP +#define OSMIUM_IO_DEBUG_OUTPUT_HPP /* @@ -33,41 +33,7 @@ DEALINGS IN THE SOFTWARE. */ -#include +#include // IWYU pragma: export +#include // IWYU pragma: export -#include - -namespace osmium { - -// avoid g++ false positive -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wreturn-type" - inline item_type osmpbf_membertype_to_item_type(const OSMPBF::Relation::MemberType mt) { - switch (mt) { - case OSMPBF::Relation::NODE: - return item_type::node; - case OSMPBF::Relation::WAY: - return item_type::way; - case OSMPBF::Relation::RELATION: - return item_type::relation; - } - } -#pragma GCC diagnostic pop - - inline OSMPBF::Relation::MemberType item_type_to_osmpbf_membertype(const item_type type) { - switch (type) { - case item_type::node: - return OSMPBF::Relation::NODE; - case item_type::way: - return OSMPBF::Relation::WAY; - case item_type::relation: - return OSMPBF::Relation::RELATION; - default: - throw std::runtime_error("Unknown relation member type"); - } - } - - -} // namespace osmium - -#endif // OSMIUM_IO_DETAIL_PBF_TYPE_CONV_HPP +#endif // OSMIUM_IO_DEBUG_OUTPUT_HPP diff --git a/contrib/libosmium/osmium/io/detail/debug_output_format.hpp b/contrib/libosmium/osmium/io/detail/debug_output_format.hpp new file mode 100644 index 00000000..efecc583 --- /dev/null +++ b/contrib/libosmium/osmium/io/detail/debug_output_format.hpp @@ -0,0 +1,482 @@ +#ifndef OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP +#define OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + namespace io { + + class File; + + namespace detail { + + constexpr const char* color_bold = "\x1b[1m"; + constexpr const char* color_black = "\x1b[30m"; + constexpr const char* color_gray = "\x1b[30;1m"; + constexpr const char* color_red = "\x1b[31m"; + constexpr const char* color_green = "\x1b[32m"; + constexpr const char* color_yellow = "\x1b[33m"; + constexpr const char* color_blue = "\x1b[34m"; + constexpr const char* color_magenta = "\x1b[35m"; + constexpr const char* color_cyan = "\x1b[36m"; + constexpr const char* color_white = "\x1b[37m"; + constexpr const char* color_reset = "\x1b[0m"; + + /** + * Writes out one buffer with OSM data in Debug format. + */ + class DebugOutputBlock : public osmium::handler::Handler { + + static constexpr size_t tmp_buffer_size = 50; + + std::shared_ptr m_input_buffer; + + std::shared_ptr m_out; + + char m_tmp_buffer[tmp_buffer_size+1]; + + bool m_add_metadata; + bool m_use_color; + + template + void output_formatted(const char* format, TArgs&&... args) { +#ifndef NDEBUG + int len = +#endif +#ifndef _MSC_VER + snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward(args)...); +#else + _snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward(args)...); +#endif + assert(len > 0 && static_cast(len) < tmp_buffer_size); + *m_out += m_tmp_buffer; + } + + void append_encoded_string(const char* data) { + const char* end = data + std::strlen(data); + + while (data != end) { + const char* last = data; + uint32_t c = utf8::next(data, end); + + // This is a list of Unicode code points that we let + // through instead of escaping them. It is incomplete + // and can be extended later. + // Generally we don't want to let through any + // non-printing characters. + if ((0x0020 <= c && c <= 0x0021) || + (0x0023 <= c && c <= 0x003b) || + (0x003d == c) || + (0x003f <= c && c <= 0x007e) || + (0x00a1 <= c && c <= 0x00ac) || + (0x00ae <= c && c <= 0x05ff)) { + m_out->append(last, data); + } else { + write_color(color_red); + output_formatted("", c); + write_color(color_blue); + } + } + } + + void write_color(const char* color) { + if (m_use_color) { + *m_out += color; + } + } + + void write_string(const char* string) { + *m_out += '"'; + write_color(color_blue); + append_encoded_string(string); + write_color(color_reset); + *m_out += '"'; + } + + void write_object_type(const char* object_type, bool visible = true) { + if (visible) { + write_color(color_bold); + } else { + write_color(color_white); + } + *m_out += object_type; + write_color(color_reset); + *m_out += ' '; + } + + void write_fieldname(const char* name) { + *m_out += " "; + write_color(color_cyan); + *m_out += name; + write_color(color_reset); + *m_out += ": "; + } + + void write_error(const char* msg) { + write_color(color_red); + *m_out += msg; + write_color(color_reset); + } + + void write_meta(const osmium::OSMObject& object) { + output_formatted("%" PRId64 "\n", object.id()); + if (m_add_metadata) { + write_fieldname("version"); + output_formatted(" %d", object.version()); + if (object.visible()) { + *m_out += " visible\n"; + } else { + write_error(" deleted\n"); + } + write_fieldname("changeset"); + output_formatted("%d\n", object.changeset()); + write_fieldname("timestamp"); + *m_out += object.timestamp().to_iso(); + output_formatted(" (%d)\n", object.timestamp()); + write_fieldname("user"); + output_formatted(" %d ", object.uid()); + write_string(object.user()); + *m_out += '\n'; + } + } + + void write_tags(const osmium::TagList& tags, const char* padding="") { + if (!tags.empty()) { + write_fieldname("tags"); + *m_out += padding; + output_formatted(" %d\n", tags.size()); + + osmium::max_op max; + for (const auto& tag : tags) { + max.update(std::strlen(tag.key())); + } + for (const auto& tag : tags) { + *m_out += " "; + write_string(tag.key()); + int spacing = max() - std::strlen(tag.key()); + while (spacing--) { + *m_out += " "; + } + *m_out += " = "; + write_string(tag.value()); + *m_out += '\n'; + } + } + } + + void write_location(const osmium::Location& location) { + write_fieldname("lon/lat"); + output_formatted(" %.7f,%.7f", location.lon_without_check(), location.lat_without_check()); + if (!location.valid()) { + write_error(" INVALID LOCATION!"); + } + *m_out += '\n'; + } + + void write_box(const osmium::Box& box) { + write_fieldname("box l/b/r/t"); + if (!box) { + write_error("BOX NOT SET!\n"); + return; + } + const auto& bl = box.bottom_left(); + const auto& tr = box.top_right(); + output_formatted("%.7f,%.7f %.7f,%.7f", bl.lon_without_check(), bl.lat_without_check(), tr.lon_without_check(), tr.lat_without_check()); + if (!box.valid()) { + write_error(" INVALID BOX!"); + } + *m_out += '\n'; + } + + public: + + explicit DebugOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool use_color) : + m_input_buffer(std::make_shared(std::move(buffer))), + m_out(std::make_shared()), + m_tmp_buffer(), + m_add_metadata(add_metadata), + m_use_color(use_color) { + } + + DebugOutputBlock(const DebugOutputBlock&) = default; + DebugOutputBlock& operator=(const DebugOutputBlock&) = default; + + DebugOutputBlock(DebugOutputBlock&&) = default; + DebugOutputBlock& operator=(DebugOutputBlock&&) = default; + + ~DebugOutputBlock() = default; + + std::string operator()() { + osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this); + + std::string out; + std::swap(out, *m_out); + return out; + } + + void node(const osmium::Node& node) { + write_object_type("node", node.visible()); + write_meta(node); + + if (node.visible()) { + write_location(node.location()); + } + + write_tags(node.tags()); + + *m_out += '\n'; + } + + void way(const osmium::Way& way) { + write_object_type("way", way.visible()); + write_meta(way); + write_tags(way.tags()); + + write_fieldname("nodes"); + + output_formatted(" %d", way.nodes().size()); + if (way.nodes().size() < 2) { + write_error(" LESS THAN 2 NODES!\n"); + } else if (way.nodes().size() > 2000) { + write_error(" MORE THAN 2000 NODES!\n"); + } else if (way.nodes().is_closed()) { + *m_out += " (closed)\n"; + } else { + *m_out += " (open)\n"; + } + + int width = int(log10(way.nodes().size())) + 1; + int n = 0; + for (const auto& node_ref : way.nodes()) { + output_formatted(" %0*d: %10" PRId64, width, n++, node_ref.ref()); + if (node_ref.location().valid()) { + output_formatted(" (%.7f,%.7f)", node_ref.location().lon_without_check(), node_ref.location().lat_without_check()); + } + *m_out += '\n'; + } + + *m_out += '\n'; + } + + void relation(const osmium::Relation& relation) { + static const char* short_typename[] = { "node", "way ", "rel " }; + write_object_type("relation", relation.visible()); + write_meta(relation); + write_tags(relation.tags()); + + write_fieldname("members"); + output_formatted(" %d\n", relation.members().size()); + + int width = int(log10(relation.members().size())) + 1; + int n = 0; + for (const auto& member : relation.members()) { + output_formatted(" %0*d: ", width, n++); + *m_out += short_typename[item_type_to_nwr_index(member.type())]; + output_formatted(" %10" PRId64 " ", member.ref()); + write_string(member.role()); + *m_out += '\n'; + } + + *m_out += '\n'; + } + + void changeset(const osmium::Changeset& changeset) { + write_object_type("changeset"); + output_formatted("%d\n", changeset.id()); + write_fieldname("num changes"); + output_formatted("%d", changeset.num_changes()); + if (changeset.num_changes() == 0) { + write_error(" NO CHANGES!"); + } + *m_out += '\n'; + write_fieldname("created at"); + *m_out += ' '; + *m_out += changeset.created_at().to_iso(); + output_formatted(" (%d)\n", changeset.created_at()); + write_fieldname("closed at"); + *m_out += " "; + if (changeset.closed()) { + *m_out += changeset.closed_at().to_iso(); + output_formatted(" (%d)\n", changeset.closed_at()); + } else { + write_error("OPEN!\n"); + } + write_fieldname("user"); + output_formatted(" %d ", changeset.uid()); + write_string(changeset.user()); + *m_out += '\n'; + + write_box(changeset.bounds()); + write_tags(changeset.tags(), " "); + + *m_out += '\n'; + } + + }; // DebugOutputBlock + + class DebugOutputFormat : public osmium::io::detail::OutputFormat { + + bool m_add_metadata; + bool m_use_color; + + public: + + DebugOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : + OutputFormat(file, output_queue), + m_add_metadata(file.get("add_metadata") != "false"), + m_use_color(file.get("color") == "true") { + } + + DebugOutputFormat(const DebugOutputFormat&) = delete; + DebugOutputFormat& operator=(const DebugOutputFormat&) = delete; + + void write_buffer(osmium::memory::Buffer&& buffer) override final { + m_output_queue.push(osmium::thread::Pool::instance().submit(DebugOutputBlock{std::move(buffer), m_add_metadata, m_use_color})); + } + + void write_fieldname(std::string& out, const char* name) { + out += " "; + if (m_use_color) { + out += color_cyan; + } + out += name; + if (m_use_color) { + out += color_reset; + } + out += ": "; + } + + void write_header(const osmium::io::Header& header) override final { + std::string out; + + if (m_use_color) { + out += color_bold; + } + out += "header\n"; + if (m_use_color) { + out += color_reset; + } + + write_fieldname(out, "multiple object versions"); + out += header.has_multiple_object_versions() ? "yes" : "no"; + out += '\n'; + write_fieldname(out, "bounding boxes"); + out += '\n'; + for (const auto& box : header.boxes()) { + out += " "; + box.bottom_left().as_string(std::back_inserter(out), ','); + out += " "; + box.top_right().as_string(std::back_inserter(out), ','); + out += '\n'; + } + write_fieldname(out, "options"); + out += '\n'; + for (const auto& opt : header) { + out += " "; + out += opt.first; + out += " = "; + out += opt.second; + out += '\n'; + } + out += "\n=============================================\n\n"; + + std::promise promise; + m_output_queue.push(promise.get_future()); + promise.set_value(std::move(out)); + } + + void close() override final { + std::string out; + std::promise promise; + m_output_queue.push(promise.get_future()); + promise.set_value(out); + } + + }; // class DebugOutputFormat + + namespace { + +// we want the register_output_format() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" + const bool registered_debug_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::debug, + [](const osmium::io::File& file, data_queue_type& output_queue) { + return new osmium::io::detail::DebugOutputFormat(file, output_queue); + }); +#pragma GCC diagnostic pop + + } // anonymous namespace + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP diff --git a/contrib/libosmium/osmium/io/detail/opl_output_format.hpp b/contrib/libosmium/osmium/io/detail/opl_output_format.hpp index cf92e138..a3103d9b 100644 --- a/contrib/libosmium/osmium/io/detail/opl_output_format.hpp +++ b/contrib/libosmium/osmium/io/detail/opl_output_format.hpp @@ -46,23 +46,7 @@ DEALINGS IN THE SOFTWARE. #include #include -#include - -#ifdef __clang__ -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wmissing-noreturn" -# pragma clang diagnostic ignored "-Wsign-conversion" -#endif - -#if BOOST_VERSION >= 104800 -# include -#else -# include -#endif - -#ifdef __clang__ -# pragma clang diagnostic pop -#endif +#include #include #include @@ -103,6 +87,8 @@ namespace osmium { char m_tmp_buffer[tmp_buffer_size+1]; + bool m_add_metadata; + template void output_formatted(const char* format, TArgs&&... args) { #ifndef NDEBUG @@ -117,13 +103,12 @@ namespace osmium { *m_out += m_tmp_buffer; } - void append_encoded_string(const std::string& data) { - boost::u8_to_u32_iterator it(data.cbegin(), data.cbegin(), data.cend()); - boost::u8_to_u32_iterator end(data.cend(), data.cend(), data.cend()); - boost::utf8_output_iterator> oit(std::back_inserter(*m_out)); + void append_encoded_string(const char* data) { + const char* end = data + std::strlen(data); - for (; it != end; ++it) { - uint32_t c = *it; + while (data != end) { + const char* last = data; + uint32_t c = utf8::next(data, end); // This is a list of Unicode code points that we let // through instead of escaping them. It is incomplete @@ -138,21 +123,29 @@ namespace osmium { (0x0041 <= c && c <= 0x007e) || (0x00a1 <= c && c <= 0x00ac) || (0x00ae <= c && c <= 0x05ff)) { - *oit = c; + m_out->append(last, data); } else { *m_out += '%'; - output_formatted("%04x", c); + if (c <= 0xff) { + output_formatted("%02x", c); + } else { + output_formatted("%04x", c); + } + *m_out += '%'; } } } void write_meta(const osmium::OSMObject& object) { - output_formatted("%" PRId64 " v%d d", object.id(), object.version()); - *m_out += (object.visible() ? 'V' : 'D'); - output_formatted(" c%d t", object.changeset()); - *m_out += object.timestamp().to_iso(); - output_formatted(" i%d u", object.uid()); - append_encoded_string(object.user()); + output_formatted("%" PRId64, object.id()); + if (m_add_metadata) { + output_formatted(" v%d d", object.version()); + *m_out += (object.visible() ? 'V' : 'D'); + output_formatted(" c%d t", object.changeset()); + *m_out += object.timestamp().to_iso(); + output_formatted(" i%d u", object.uid()); + append_encoded_string(object.user()); + } *m_out += " T"; bool first = true; for (const auto& tag : object.tags()) { @@ -180,10 +173,11 @@ namespace osmium { public: - explicit OPLOutputBlock(osmium::memory::Buffer&& buffer) : + explicit OPLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata) : m_input_buffer(std::make_shared(std::move(buffer))), m_out(std::make_shared()), - m_tmp_buffer() { + m_tmp_buffer(), + m_add_metadata(add_metadata) { } OPLOutputBlock(const OPLOutputBlock&) = default; @@ -240,7 +234,7 @@ namespace osmium { } *m_out += item_type_to_char(member.type()); output_formatted("%" PRId64 "@", member.ref()); - *m_out += member.role(); + append_encoded_string(member.role()); } *m_out += '\n'; } @@ -274,17 +268,20 @@ namespace osmium { class OPLOutputFormat : public osmium::io::detail::OutputFormat { - OPLOutputFormat(const OPLOutputFormat&) = delete; - OPLOutputFormat& operator=(const OPLOutputFormat&) = delete; + bool m_add_metadata; public: OPLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : - OutputFormat(file, output_queue) { + OutputFormat(file, output_queue), + m_add_metadata(file.get("add_metadata") != "false") { } + OPLOutputFormat(const OPLOutputFormat&) = delete; + OPLOutputFormat& operator=(const OPLOutputFormat&) = delete; + void write_buffer(osmium::memory::Buffer&& buffer) override final { - m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer)})); + m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer), m_add_metadata})); } void close() override final { @@ -298,6 +295,8 @@ namespace osmium { namespace { +// we want the register_output_format() function to run, setting the variable +// is only a side-effect, it will never be used #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_opl_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::opl, diff --git a/contrib/libosmium/osmium/io/detail/pbf.hpp b/contrib/libosmium/osmium/io/detail/pbf.hpp index f2975c58..15e457a1 100644 --- a/contrib/libosmium/osmium/io/detail/pbf.hpp +++ b/contrib/libosmium/osmium/io/detail/pbf.hpp @@ -33,7 +33,7 @@ DEALINGS IN THE SOFTWARE. */ -#include +#include // needed for htonl and ntohl #ifndef _WIN32 @@ -43,6 +43,7 @@ DEALINGS IN THE SOFTWARE. #endif #include +#include namespace osmium { @@ -62,6 +63,26 @@ namespace osmium { }; // struct pbf_error + namespace io { + + namespace detail { + + // the maximum size of a blob header in bytes + const int max_blob_header_size = 64 * 1024; // 64 kB + + // the maximum size of an uncompressed blob in bytes + const uint64_t max_uncompressed_blob_size = 32 * 1024 * 1024; // 32 MB + + // resolution for longitude/latitude used for conversion + // between representation as double and as int + const int64_t lonlat_resolution = 1000 * 1000 * 1000; + + const int64_t resolution_convert = lonlat_resolution / osmium::Location::coordinate_precision; + + } + + } + } // namespace osmium #endif // OSMIUM_IO_DETAIL_PBF_HPP diff --git a/contrib/libosmium/osmium/io/detail/pbf_decoder.hpp b/contrib/libosmium/osmium/io/detail/pbf_decoder.hpp new file mode 100644 index 00000000..79e899ff --- /dev/null +++ b/contrib/libosmium/osmium/io/detail/pbf_decoder.hpp @@ -0,0 +1,760 @@ +#ifndef OSMIUM_IO_DETAIL_PBF_DECODER_HPP +#define OSMIUM_IO_DETAIL_PBF_DECODER_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include // IWYU pragma: export +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + namespace io { + + namespace detail { + + using ptr_len_type = std::pair; + + class PBFPrimitiveBlockDecoder { + + static constexpr size_t initial_buffer_size = 2 * 1024 * 1024; + + ptr_len_type m_data; + std::vector m_stringtable; + + int64_t m_lon_offset = 0; + int64_t m_lat_offset = 0; + int64_t m_date_factor = 1000; + int32_t m_granularity = 100; + + osmium::osm_entity_bits::type m_read_types; + + osmium::memory::Buffer m_buffer { initial_buffer_size }; + + void decode_stringtable(const ptr_len_type& data) { + if (!m_stringtable.empty()) { + throw osmium::pbf_error("more than one stringtable in pbf file"); + } + + protozero::pbf_message pbf_string_table(data); + while (pbf_string_table.next(OSMFormat::StringTable::repeated_bytes_s)) { + m_stringtable.push_back(pbf_string_table.get_data()); + } + } + + void decode_primitive_block_metadata() { + protozero::pbf_message pbf_primitive_block(m_data); + while (pbf_primitive_block.next()) { + switch (pbf_primitive_block.tag()) { + case OSMFormat::PrimitiveBlock::required_StringTable_stringtable: + decode_stringtable(pbf_primitive_block.get_data()); + break; + case OSMFormat::PrimitiveBlock::optional_int32_granularity: + m_granularity = pbf_primitive_block.get_int32(); + break; + case OSMFormat::PrimitiveBlock::optional_int32_date_granularity: + m_date_factor = pbf_primitive_block.get_int32(); + break; + case OSMFormat::PrimitiveBlock::optional_int64_lat_offset: + m_lat_offset = pbf_primitive_block.get_int64(); + break; + case OSMFormat::PrimitiveBlock::optional_int64_lon_offset: + m_lon_offset = pbf_primitive_block.get_int64(); + break; + default: + pbf_primitive_block.skip(); + } + } + } + + void decode_primitive_block_data() { + protozero::pbf_message pbf_primitive_block(m_data); + while (pbf_primitive_block.next(OSMFormat::PrimitiveBlock::repeated_PrimitiveGroup_primitivegroup)) { + protozero::pbf_message pbf_primitive_group = pbf_primitive_block.get_message(); + while (pbf_primitive_group.next()) { + switch (pbf_primitive_group.tag()) { + case OSMFormat::PrimitiveGroup::repeated_Node_nodes: + if (m_read_types & osmium::osm_entity_bits::node) { + decode_node(pbf_primitive_group.get_data()); + } else { + pbf_primitive_group.skip(); + } + break; + case OSMFormat::PrimitiveGroup::optional_DenseNodes_dense: + if (m_read_types & osmium::osm_entity_bits::node) { + decode_dense_nodes(pbf_primitive_group.get_data()); + } else { + pbf_primitive_group.skip(); + } + break; + case OSMFormat::PrimitiveGroup::repeated_Way_ways: + if (m_read_types & osmium::osm_entity_bits::way) { + decode_way(pbf_primitive_group.get_data()); + } else { + pbf_primitive_group.skip(); + } + break; + case OSMFormat::PrimitiveGroup::repeated_Relation_relations: + if (m_read_types & osmium::osm_entity_bits::relation) { + decode_relation(pbf_primitive_group.get_data()); + } else { + pbf_primitive_group.skip(); + } + break; + default: + pbf_primitive_group.skip(); + } + } + } + } + + ptr_len_type decode_info(const ptr_len_type& data, osmium::OSMObject& object) { + ptr_len_type user = std::make_pair("", 0); + + protozero::pbf_message pbf_info(data); + while (pbf_info.next()) { + switch (pbf_info.tag()) { + case OSMFormat::Info::optional_int32_version: + { + auto version = pbf_info.get_int32(); + if (version < 0) { + throw osmium::pbf_error("object version must not be negative"); + } + object.set_version(static_cast_with_assert(version)); + } + break; + case OSMFormat::Info::optional_int64_timestamp: + object.set_timestamp(pbf_info.get_int64() * m_date_factor / 1000); + break; + case OSMFormat::Info::optional_int64_changeset: + { + auto changeset_id = pbf_info.get_int64(); + if (changeset_id < 0) { + throw osmium::pbf_error("object changeset_id must not be negative"); + } + object.set_changeset(static_cast_with_assert(changeset_id)); + } + break; + case OSMFormat::Info::optional_int32_uid: + object.set_uid_from_signed(pbf_info.get_int32()); + break; + case OSMFormat::Info::optional_uint32_user_sid: + user = m_stringtable.at(pbf_info.get_uint32()); + break; + case OSMFormat::Info::optional_bool_visible: + object.set_visible(pbf_info.get_bool()); + break; + default: + pbf_info.skip(); + } + } + + return user; + } + + using kv_type = std::pair; + + void build_tag_list(osmium::builder::Builder& builder, const kv_type& keys, const kv_type& vals) { + if (keys.first != keys.second) { + osmium::builder::TagListBuilder tl_builder(m_buffer, &builder); + auto kit = keys.first; + auto vit = vals.first; + while (kit != keys.second) { + if (vit == vals.second) { + // this is against the spec, must have same number of elements + throw osmium::pbf_error("PBF format error"); + } + const auto& k = m_stringtable.at(*kit++); + const auto& v = m_stringtable.at(*vit++); + tl_builder.add_tag(k.first, k.second, v.first, v.second); + } + } + } + + int32_t convert_pbf_coordinate(int64_t c) const { + return (c * m_granularity + m_lon_offset) / resolution_convert; + } + + void decode_node(const ptr_len_type& data) { + osmium::builder::NodeBuilder builder(m_buffer); + osmium::Node& node = builder.object(); + + kv_type keys; + kv_type vals; + int64_t lon = std::numeric_limits::max(); + int64_t lat = std::numeric_limits::max(); + + ptr_len_type user = { "", 0 }; + + protozero::pbf_message pbf_node(data); + while (pbf_node.next()) { + switch (pbf_node.tag()) { + case OSMFormat::Node::required_sint64_id: + node.set_id(pbf_node.get_sint64()); + break; + case OSMFormat::Node::packed_uint32_keys: + keys = pbf_node.get_packed_uint32(); + break; + case OSMFormat::Node::packed_uint32_vals: + vals = pbf_node.get_packed_uint32(); + break; + case OSMFormat::Node::optional_Info_info: + user = decode_info(pbf_node.get_data(), builder.object()); + break; + case OSMFormat::Node::required_sint64_lat: + lat = pbf_node.get_sint64(); + break; + case OSMFormat::Node::required_sint64_lon: + lon = pbf_node.get_sint64(); + break; + default: + pbf_node.skip(); + } + } + + if (node.visible()) { + if (lon == std::numeric_limits::max() || + lat == std::numeric_limits::max()) { + throw osmium::pbf_error("illegal coordinate format"); + } + node.set_location(osmium::Location( + convert_pbf_coordinate(lon), + convert_pbf_coordinate(lat) + )); + } + + builder.add_user(user.first, user.second); + + build_tag_list(builder, keys, vals); + + m_buffer.commit(); + } + + void decode_way(const ptr_len_type& data) { + osmium::builder::WayBuilder builder(m_buffer); + + kv_type keys; + kv_type vals; + std::pair refs; + + ptr_len_type user = { "", 0 }; + + protozero::pbf_message pbf_way(data); + while (pbf_way.next()) { + switch (pbf_way.tag()) { + case OSMFormat::Way::required_int64_id: + builder.object().set_id(pbf_way.get_int64()); + break; + case OSMFormat::Way::packed_uint32_keys: + keys = pbf_way.get_packed_uint32(); + break; + case OSMFormat::Way::packed_uint32_vals: + vals = pbf_way.get_packed_uint32(); + break; + case OSMFormat::Way::optional_Info_info: + user = decode_info(pbf_way.get_data(), builder.object()); + break; + case OSMFormat::Way::packed_sint64_refs: + refs = pbf_way.get_packed_sint64(); + break; + default: + pbf_way.skip(); + } + } + + builder.add_user(user.first, user.second); + + if (refs.first != refs.second) { + osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder); + osmium::util::DeltaDecode ref; + while (refs.first != refs.second) { + wnl_builder.add_node_ref(ref.update(*refs.first++)); + } + } + + build_tag_list(builder, keys, vals); + + m_buffer.commit(); + } + + void decode_relation(const ptr_len_type& data) { + osmium::builder::RelationBuilder builder(m_buffer); + + kv_type keys; + kv_type vals; + std::pair roles; + std::pair refs; + std::pair types; + + ptr_len_type user = { "", 0 }; + + protozero::pbf_message pbf_relation(data); + while (pbf_relation.next()) { + switch (pbf_relation.tag()) { + case OSMFormat::Relation::required_int64_id: + builder.object().set_id(pbf_relation.get_int64()); + break; + case OSMFormat::Relation::packed_uint32_keys: + keys = pbf_relation.get_packed_uint32(); + break; + case OSMFormat::Relation::packed_uint32_vals: + vals = pbf_relation.get_packed_uint32(); + break; + case OSMFormat::Relation::optional_Info_info: + user = decode_info(pbf_relation.get_data(), builder.object()); + break; + case OSMFormat::Relation::packed_int32_roles_sid: + roles = pbf_relation.get_packed_int32(); + break; + case OSMFormat::Relation::packed_sint64_memids: + refs = pbf_relation.get_packed_sint64(); + break; + case OSMFormat::Relation::packed_MemberType_types: + types = pbf_relation.get_packed_enum(); + break; + default: + pbf_relation.skip(); + } + } + + builder.add_user(user.first, user.second); + + if (refs.first != refs.second) { + osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder); + osmium::util::DeltaDecode ref; + while (roles.first != roles.second && refs.first != refs.second && types.first != types.second) { + const auto& r = m_stringtable.at(*roles.first++); + int type = *types.first++; + if (type < 0 || type > 2) { + throw osmium::pbf_error("unknown relation member type"); + } + rml_builder.add_member( + osmium::item_type(type + 1), + ref.update(*refs.first++), + r.first, + r.second + ); + } + } + + build_tag_list(builder, keys, vals); + + m_buffer.commit(); + } + + void decode_dense_nodes(const ptr_len_type& data) { + bool has_info = false; + bool has_visibles = false; + + std::pair ids; + std::pair lats; + std::pair lons; + + std::pair tags; + + std::pair versions; + std::pair timestamps; + std::pair changesets; + std::pair uids; + std::pair user_sids; + std::pair visibles; + + protozero::pbf_message pbf_dense_nodes(data); + while (pbf_dense_nodes.next()) { + switch (pbf_dense_nodes.tag()) { + case OSMFormat::DenseNodes::packed_sint64_id: + ids = pbf_dense_nodes.get_packed_sint64(); + break; + case OSMFormat::DenseNodes::optional_DenseInfo_denseinfo: + { + has_info = true; + protozero::pbf_message pbf_dense_info = pbf_dense_nodes.get_message(); + while (pbf_dense_info.next()) { + switch (pbf_dense_info.tag()) { + case OSMFormat::DenseInfo::packed_int32_version: + versions = pbf_dense_info.get_packed_int32(); + break; + case OSMFormat::DenseInfo::packed_sint64_timestamp: + timestamps = pbf_dense_info.get_packed_sint64(); + break; + case OSMFormat::DenseInfo::packed_sint64_changeset: + changesets = pbf_dense_info.get_packed_sint64(); + break; + case OSMFormat::DenseInfo::packed_sint32_uid: + uids = pbf_dense_info.get_packed_sint32(); + break; + case OSMFormat::DenseInfo::packed_sint32_user_sid: + user_sids = pbf_dense_info.get_packed_sint32(); + break; + case OSMFormat::DenseInfo::packed_bool_visible: + has_visibles = true; + visibles = pbf_dense_info.get_packed_bool(); + break; + default: + pbf_dense_info.skip(); + } + } + } + break; + case OSMFormat::DenseNodes::packed_sint64_lat: + lats = pbf_dense_nodes.get_packed_sint64(); + break; + case OSMFormat::DenseNodes::packed_sint64_lon: + lons = pbf_dense_nodes.get_packed_sint64(); + break; + case OSMFormat::DenseNodes::packed_int32_keys_vals: + tags = pbf_dense_nodes.get_packed_int32(); + break; + default: + pbf_dense_nodes.skip(); + } + } + + osmium::util::DeltaDecode dense_id; + osmium::util::DeltaDecode dense_latitude; + osmium::util::DeltaDecode dense_longitude; + osmium::util::DeltaDecode dense_uid; + osmium::util::DeltaDecode dense_user_sid; + osmium::util::DeltaDecode dense_changeset; + osmium::util::DeltaDecode dense_timestamp; + + auto tag_it = tags.first; + + while (ids.first != ids.second) { + if (lons.first == lons.second || + lats.first == lats.second) { + // this is against the spec, must have same number of elements + throw osmium::pbf_error("PBF format error"); + } + + bool visible = true; + + osmium::builder::NodeBuilder builder(m_buffer); + osmium::Node& node = builder.object(); + + node.set_id(dense_id.update(*ids.first++)); + + if (has_info) { + if (versions.first == versions.second || + changesets.first == changesets.second || + timestamps.first == timestamps.second || + uids.first == uids.second || + user_sids.first == user_sids.second) { + // this is against the spec, must have same number of elements + throw osmium::pbf_error("PBF format error"); + } + + auto version = *versions.first++; + if (version < 0) { + throw osmium::pbf_error("object version must not be negative"); + } + node.set_version(static_cast(version)); + + auto changeset_id = dense_changeset.update(*changesets.first++); + if (changeset_id < 0) { + throw osmium::pbf_error("object changeset_id must not be negative"); + } + node.set_changeset(static_cast(changeset_id)); + + node.set_timestamp(dense_timestamp.update(*timestamps.first++) * m_date_factor / 1000); + node.set_uid_from_signed(static_cast(dense_uid.update(*uids.first++))); + + if (has_visibles) { + if (visibles.first == visibles.second) { + // this is against the spec, must have same number of elements + throw osmium::pbf_error("PBF format error"); + } + visible = *visibles.first++; + } + node.set_visible(visible); + + const auto& u = m_stringtable.at(dense_user_sid.update(*user_sids.first++)); + builder.add_user(u.first, u.second); + } else { + builder.add_user(""); + } + + if (visible) { + builder.object().set_location(osmium::Location( + convert_pbf_coordinate(dense_longitude.update(*lons.first++)), + convert_pbf_coordinate(dense_latitude.update(*lats.first++)) + )); + } + + if (tag_it != tags.second) { + osmium::builder::TagListBuilder tl_builder(m_buffer, &builder); + while (tag_it != tags.second && *tag_it != 0) { + const auto& k = m_stringtable.at(*tag_it++); + if (tag_it == tags.second) { + throw osmium::pbf_error("PBF format error"); // this is against the spec, keys/vals must come in pairs + } + const auto& v = m_stringtable.at(*tag_it++); + tl_builder.add_tag(k.first, k.second, v.first, v.second); + } + + if (tag_it != tags.second) { + ++tag_it; + } + } + + m_buffer.commit(); + } + + } + + public: + + explicit PBFPrimitiveBlockDecoder(const ptr_len_type& data, osmium::osm_entity_bits::type read_types) : + m_data(data), + m_read_types(read_types) { + } + + PBFPrimitiveBlockDecoder(const PBFPrimitiveBlockDecoder&) = delete; + PBFPrimitiveBlockDecoder& operator=(const PBFPrimitiveBlockDecoder&) = delete; + + PBFPrimitiveBlockDecoder(PBFPrimitiveBlockDecoder&&) = delete; + PBFPrimitiveBlockDecoder& operator=(PBFPrimitiveBlockDecoder&&) = delete; + + ~PBFPrimitiveBlockDecoder() = default; + + osmium::memory::Buffer operator()() { + try { + decode_primitive_block_metadata(); + decode_primitive_block_data(); + } catch (std::out_of_range&) { + throw osmium::pbf_error("string id out of range"); + } + + return std::move(m_buffer); + } + + }; // class PBFPrimitiveBlockDecoder + + inline ptr_len_type decode_blob(const std::string& blob_data, std::string& output) { + int32_t raw_size; + std::pair zlib_data; + + protozero::pbf_message pbf_blob(blob_data); + while (pbf_blob.next()) { + switch (pbf_blob.tag()) { + case FileFormat::Blob::optional_bytes_raw: + { + auto data_len = pbf_blob.get_data(); + if (data_len.second > max_uncompressed_blob_size) { + throw osmium::pbf_error("illegal blob size"); + } + return data_len; + } + case FileFormat::Blob::optional_int32_raw_size: + raw_size = pbf_blob.get_int32(); + if (raw_size <= 0 || uint32_t(raw_size) > max_uncompressed_blob_size) { + throw osmium::pbf_error("illegal blob size"); + } + break; + case FileFormat::Blob::optional_bytes_zlib_data: + zlib_data = pbf_blob.get_data(); + break; + case FileFormat::Blob::optional_bytes_lzma_data: + throw osmium::pbf_error("lzma blobs not implemented"); + default: + throw osmium::pbf_error("unknown compression"); + } + } + + if (zlib_data.second != 0) { + return osmium::io::detail::zlib_uncompress_string( + zlib_data.first, + static_cast(zlib_data.second), + static_cast(raw_size), + output + ); + } + + throw osmium::pbf_error("blob contains no data"); + } + + inline osmium::Box decode_header_bbox(const ptr_len_type& data) { + int64_t left = std::numeric_limits::max(); + int64_t right = std::numeric_limits::max(); + int64_t top = std::numeric_limits::max(); + int64_t bottom = std::numeric_limits::max(); + + protozero::pbf_message pbf_header_bbox(data); + while (pbf_header_bbox.next()) { + switch (pbf_header_bbox.tag()) { + case OSMFormat::HeaderBBox::required_sint64_left: + left = pbf_header_bbox.get_sint64(); + break; + case OSMFormat::HeaderBBox::required_sint64_right: + right = pbf_header_bbox.get_sint64(); + break; + case OSMFormat::HeaderBBox::required_sint64_top: + top = pbf_header_bbox.get_sint64(); + break; + case OSMFormat::HeaderBBox::required_sint64_bottom: + bottom = pbf_header_bbox.get_sint64(); + break; + default: + pbf_header_bbox.skip(); + } + } + + if (left == std::numeric_limits::max() || + right == std::numeric_limits::max() || + top == std::numeric_limits::max() || + bottom == std::numeric_limits::max()) { + throw osmium::pbf_error("invalid bbox"); + } + + osmium::Box box; + box.extend(osmium::Location(left / resolution_convert, bottom / resolution_convert)); + box.extend(osmium::Location(right / resolution_convert, top / resolution_convert)); + + return box; + } + + inline osmium::io::Header decode_header_block(const ptr_len_type& data) { + osmium::io::Header header; + int i = 0; + + protozero::pbf_message pbf_header_block(data); + while (pbf_header_block.next()) { + switch (pbf_header_block.tag()) { + case OSMFormat::HeaderBlock::optional_HeaderBBox_bbox: + header.add_box(decode_header_bbox(pbf_header_block.get_data())); + break; + case OSMFormat::HeaderBlock::repeated_string_required_features: + { + auto feature = pbf_header_block.get_data(); + if (!strncmp("OsmSchema-V0.6", feature.first, feature.second)) { + // intentionally left blank + } else if (!strncmp("DenseNodes", feature.first, feature.second)) { + header.set("pbf_dense_nodes", true); + } else if (!strncmp("HistoricalInformation", feature.first, feature.second)) { + header.set_has_multiple_object_versions(true); + } else { + std::string msg("required feature not supported: "); + msg.append(feature.first, feature.second); + throw osmium::pbf_error(msg); + } + } + break; + case OSMFormat::HeaderBlock::repeated_string_optional_features: + header.set("pbf_optional_feature_" + std::to_string(i++), pbf_header_block.get_string()); + break; + case OSMFormat::HeaderBlock::optional_string_writingprogram: + header.set("generator", pbf_header_block.get_string()); + break; + case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp: + header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.get_int64()).to_iso()); + break; + case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_sequence_number: + header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.get_int64())); + break; + case OSMFormat::HeaderBlock::optional_string_osmosis_replication_base_url: + header.set("osmosis_replication_base_url", pbf_header_block.get_string()); + break; + default: + pbf_header_block.skip(); + } + } + + return header; + } + + /** + * Decode HeaderBlock. + * + * @param header_block_data Input data + * @returns Header object + * @throws osmium::pbf_error If there was a parsing error + */ + inline osmium::io::Header decode_header(const std::string& header_block_data) { + std::string output; + + return decode_header_block(decode_blob(header_block_data, output)); + } + + class PBFDataBlobDecoder { + + std::shared_ptr m_input_buffer; + osmium::osm_entity_bits::type m_read_types; + + public: + + PBFDataBlobDecoder(std::string&& input_buffer, osmium::osm_entity_bits::type read_types) : + m_input_buffer(std::make_shared(std::move(input_buffer))), + m_read_types(read_types) { + } + + PBFDataBlobDecoder(const PBFDataBlobDecoder&) = default; + PBFDataBlobDecoder& operator=(const PBFDataBlobDecoder&) = default; + + PBFDataBlobDecoder(PBFDataBlobDecoder&&) = default; + PBFDataBlobDecoder& operator=(PBFDataBlobDecoder&&) = default; + + ~PBFDataBlobDecoder() = default; + + osmium::memory::Buffer operator()() { + std::string output; + PBFPrimitiveBlockDecoder decoder(decode_blob(*m_input_buffer, output), m_read_types); + return decoder(); + } + + }; // class PBFDataBlobDecoder + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_PBF_DECODER_HPP diff --git a/contrib/libosmium/osmium/io/detail/pbf_input_format.hpp b/contrib/libosmium/osmium/io/detail/pbf_input_format.hpp index 93ba6cae..7817d27d 100644 --- a/contrib/libosmium/osmium/io/detail/pbf_input_format.hpp +++ b/contrib/libosmium/osmium/io/detail/pbf_input_format.hpp @@ -49,10 +49,12 @@ DEALINGS IN THE SOFTWARE. #include #include +#include + #include #include // IWYU pragma: export -#include -#include +#include +#include #include #include #include @@ -77,13 +79,13 @@ namespace osmium { namespace detail { - typedef osmium::thread::Queue> queue_type; - /** * Class for parsing PBF files. */ class PBFInputFormat : public osmium::io::detail::InputFormat { + typedef osmium::thread::Queue> queue_type; + bool m_use_thread_pool; bool m_eof { false }; queue_type m_queue; @@ -116,15 +118,10 @@ namespace osmium { } /** - * Read BlobHeader by first reading the size and then the - * BlobHeader. The BlobHeader contains a type field (which is - * checked against the expected type) and a size field. - * - * @param expected_type Expected type of data ("OSMHeader" or - * "OSMData"). - * @returns Size of the data read from BlobHeader (0 on EOF). + * Read 4 bytes in network byte order from file. They contain + * the length of the following BlobHeader. */ - size_t read_blob_header(const char* expected_type) { + uint32_t read_blob_header_size_from_file() { uint32_t size_in_network_byte_order; try { @@ -134,37 +131,76 @@ namespace osmium { return 0; // EOF } - uint32_t size = ntohl(size_in_network_byte_order); - if (size > static_cast(OSMPBF::max_blob_header_size)) { + const uint32_t size = ntohl(size_in_network_byte_order); + if (size > static_cast(max_blob_header_size)) { throw osmium::pbf_error("invalid BlobHeader size (> max_blob_header_size)"); } - OSMPBF::BlobHeader blob_header; - if (!blob_header.ParseFromString(read_from_input_queue(size))) { - throw osmium::pbf_error("failed to parse BlobHeader"); + return size; + } + + /** + * Decode the BlobHeader. Make sure it contains the expected + * type. Return the size of the following Blob. + */ + size_t decode_blob_header(protozero::pbf_message&& pbf_blob_header, const char* expected_type) { + std::pair blob_header_type; + size_t blob_header_datasize = 0; + + while (pbf_blob_header.next()) { + switch (pbf_blob_header.tag()) { + case FileFormat::BlobHeader::required_string_type: + blob_header_type = pbf_blob_header.get_data(); + break; + case FileFormat::BlobHeader::required_int32_datasize: + blob_header_datasize = pbf_blob_header.get_int32(); + break; + default: + pbf_blob_header.skip(); + } } - if (blob_header.type() != expected_type) { + if (blob_header_datasize == 0) { + throw osmium::pbf_error("PBF format error: BlobHeader.datasize missing or zero."); + } + + if (strncmp(expected_type, blob_header_type.first, blob_header_type.second)) { throw osmium::pbf_error("blob does not have expected type (OSMHeader in first blob, OSMData in following blobs)"); } - return static_cast(blob_header.datasize()); + return blob_header_datasize; + } + + size_t check_type_and_get_blob_size(const char* expected_type) { + assert(expected_type); + + auto size = read_blob_header_size_from_file(); + if (size == 0) { // EOF + return 0; + } + + std::string blob_header = read_from_input_queue(size); + + return decode_blob_header(protozero::pbf_message(blob_header), expected_type); } void parse_osm_data(osmium::osm_entity_bits::type read_types) { osmium::thread::set_thread_name("_osmium_pbf_in"); - int n = 0; - while (auto size = read_blob_header("OSMData")) { + + while (auto size = check_type_and_get_blob_size("OSMData")) { + std::string input_buffer = read_from_input_queue(size); + if (input_buffer.size() > max_uncompressed_blob_size) { + throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size()))); + } if (m_use_thread_pool) { - m_queue.push(osmium::thread::Pool::instance().submit(DataBlobParser{read_from_input_queue(size), read_types})); + m_queue.push(osmium::thread::Pool::instance().submit(PBFDataBlobDecoder{ std::move(input_buffer), read_types })); } else { std::promise promise; m_queue.push(promise.get_future()); - DataBlobParser data_blob_parser{read_from_input_queue(size), read_types}; + PBFDataBlobDecoder data_blob_parser{ std::move(input_buffer), read_types }; promise.set_value(data_blob_parser()); } - ++n; if (m_quit_input_thread) { return; @@ -198,11 +234,10 @@ namespace osmium { m_quit_input_thread(false), m_input_queue(input_queue), m_input_buffer() { - GOOGLE_PROTOBUF_VERIFY_VERSION; // handle OSMHeader - auto size = read_blob_header("OSMHeader"); - m_header = parse_header_blob(read_from_input_queue(size)); + const auto size = check_type_and_get_blob_size("OSMHeader"); + m_header = decode_header(read_from_input_queue(size)); if (m_read_which_entities != osmium::osm_entity_bits::nothing) { m_reader = std::thread(&PBFInputFormat::parse_osm_data, this, m_read_which_entities); @@ -247,10 +282,15 @@ namespace osmium { namespace { +// we want the register_input_format() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_pbf_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::pbf, [](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) { return new osmium::io::detail::PBFInputFormat(file, read_which_entities, input_queue); }); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/contrib/libosmium/osmium/io/detail/pbf_output_format.hpp b/contrib/libosmium/osmium/io/detail/pbf_output_format.hpp index b3b7a6a0..8d8a079b 100644 --- a/contrib/libosmium/osmium/io/detail/pbf_output_format.hpp +++ b/contrib/libosmium/osmium/io/detail/pbf_output_format.hpp @@ -33,75 +33,13 @@ DEALINGS IN THE SOFTWARE. */ -/* - -About the .osm.pbf file format -This is an excerpt of - -The .osm.pbf format and it's derived formats (.osh.pbf and .osc.pbf) are encoded -using googles protobuf library for the low-level storage. They are constructed -by nesting data on two levels: - -On the lower level the file is constructed using BlobHeaders and Blobs. A .osm.pbf -file contains multiple sequences of - 1. a 4-byte header size, stored in network-byte-order - 2. a BlobHeader of exactly this size - 3. a Blob - -The BlobHeader tells the reader about the type and size of the following Blob. The -Blob can contain data in raw or zlib-compressed form. After uncompressing the blob -it is treated differently depending on the type specified in the BlobHeader. - -The contents of the Blob belongs to the higher level. It contains either an HeaderBlock -(type="OSMHeader") or an PrimitiveBlock (type="OSMData"). The file needs to have -at least one HeaderBlock before the first PrimitiveBlock. - -The HeaderBlock contains meta-information like the writing program or a bbox. It may -also contain multiple "required features" that describe what kinds of input a -reading program needs to handle in order to fully understand the files' contents. - -The PrimitiveBlock can store multiple types of objects (i.e. 5 nodes, 2 ways and -1 relation). It contains one or more PrimitiveGroup which in turn contain multiple -nodes, ways or relations. A PrimitiveGroup should only contain one kind of object. - -There's a special kind of "object type" called dense-nodes. It is used to store nodes -in a very dense format, avoiding message overheads and using delta-encoding for nearly -all ids. - -All Strings are stored as indexes to rows in a StringTable. The StringTable contains -one row for each used string, so strings that are used multiple times need to be -stored only once. The StringTable is sorted by usage-count, so the most often used -string is stored at index 1. - -A simple outline of a .osm.pbf file could look like this: - - 4-bytes header size - BlobHeader - Blob - HeaderBlock - 4-bytes header size - BlobHeader - Blob - PrimitiveBlock - StringTable - PrimitiveGroup - 5 nodes - PrimitiveGroup - 2 ways - PrimitiveGroup - 1 relation - -More complete outlines of real .osm.pbf files can be created using the osmpbf-outline tool: - -*/ - #include #include #include #include #include #include -#include +#include #include #include #include @@ -109,11 +47,15 @@ More complete outlines of real .osm.pbf files can be created using the osmpbf-ou #include #include +#include + +#include + #include #include #include // IWYU pragma: export -#include -#include +#include +#include #include #include #include @@ -139,71 +81,252 @@ namespace osmium { namespace detail { - namespace { + /** + * Maximum number of items in a primitive block. + * + * The uncompressed length of a Blob *should* be less + * than 16 megabytes and *must* be less than 32 megabytes. + * + * A block may contain any number of entities, as long as + * the size limits for the surrounding blob are obeyed. + * However, for simplicity, the current Osmosis (0.38) + * as well as Osmium implementation always + * uses at most 8k entities in a block. + */ + constexpr int32_t max_entities_per_block = 8000; - /** - * Serialize a protobuf message into a Blob, optionally apply compression - * and return it together with a BlobHeader ready to be written to a file. - * - * @param type Type-string used in the BlobHeader. - * @param msg Protobuf-message. - * @param use_compression Should the output be compressed using zlib? - */ - std::string serialize_blob(const std::string& type, const google::protobuf::MessageLite& msg, bool use_compression) { - OSMPBF::Blob pbf_blob; + constexpr int location_granularity = 100; - { - std::string content; - msg.SerializeToString(&content); + /** + * convert a double lat or lon value to an int, respecting the granularity + */ + inline int64_t lonlat2int(double lonlat) { + return static_cast(std::round(lonlat * lonlat_resolution / location_granularity)); + } - pbf_blob.set_raw_size(static_cast_with_assert<::google::protobuf::int32>(content.size())); + /** + * Serialize a protobuf message into a Blob, optionally apply compression + * and return it together with a BlobHeader ready to be written to a file. + * + * @param type Type-string used in the BlobHeader. + * @param msg Protobuf-message. + * @param use_compression Should the output be compressed using zlib? + */ + inline std::string serialize_blob(const std::string& type, const std::string& msg, bool use_compression) { + std::string blob_data; + protozero::pbf_builder pbf_blob(blob_data); - if (use_compression) { - pbf_blob.set_zlib_data(osmium::io::detail::zlib_compress(content)); - } else { - pbf_blob.set_raw(content); + if (use_compression) { + pbf_blob.add_int32(FileFormat::Blob::optional_int32_raw_size, msg.size()); + pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_zlib_data, osmium::io::detail::zlib_compress(msg)); + } else { + pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_raw, msg); + } + + std::string blob_header_data; + protozero::pbf_builder pbf_blob_header(blob_header_data); + + pbf_blob_header.add_string(FileFormat::BlobHeader::required_string_type, type); + pbf_blob_header.add_int32(FileFormat::BlobHeader::required_int32_datasize, blob_data.size()); + + uint32_t sz = htonl(static_cast_with_assert(blob_header_data.size())); + + // write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob + std::string output; + output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size()); + output.append(reinterpret_cast(&sz), sizeof(sz)); + output.append(blob_header_data); + output.append(blob_data); + + return output; + } + + class DenseNodes { + + StringTable& m_stringtable; + + std::vector m_ids; + + std::vector m_versions; + std::vector m_timestamps; + std::vector m_changesets; + std::vector m_uids; + std::vector m_user_sids; + std::vector m_visibles; + + std::vector m_lats; + std::vector m_lons; + std::vector m_tags; + + osmium::util::DeltaEncode m_delta_id; + + osmium::util::DeltaEncode m_delta_timestamp; + osmium::util::DeltaEncode m_delta_changeset; + osmium::util::DeltaEncode m_delta_uid; + osmium::util::DeltaEncode m_delta_user_sid; + + osmium::util::DeltaEncode m_delta_lat; + osmium::util::DeltaEncode m_delta_lon; + + bool m_add_metadata; + bool m_add_visible; + + public: + + DenseNodes(StringTable& stringtable, bool add_metadata, bool add_visible) : + m_stringtable(stringtable), + m_add_metadata(add_metadata), + m_add_visible(add_visible) { + } + + void clear() { + m_ids.clear(); + + m_versions.clear(); + m_timestamps.clear(); + m_changesets.clear(); + m_uids.clear(); + m_user_sids.clear(); + m_visibles.clear(); + + m_lats.clear(); + m_lons.clear(); + m_tags.clear(); + + m_delta_id.clear(); + + m_delta_timestamp.clear(); + m_delta_changeset.clear(); + m_delta_uid.clear(); + m_delta_user_sid.clear(); + + m_delta_lat.clear(); + m_delta_lon.clear(); + } + + size_t size() const { + return m_ids.size() * 3 * sizeof(int64_t); + } + + void add_node(const osmium::Node& node) { + m_ids.push_back(m_delta_id.update(node.id())); + + if (m_add_metadata) { + m_versions.push_back(node.version()); + m_timestamps.push_back(m_delta_timestamp.update(node.timestamp())); + m_changesets.push_back(m_delta_changeset.update(node.changeset())); + m_uids.push_back(m_delta_uid.update(node.uid())); + m_user_sids.push_back(m_delta_user_sid.update(m_stringtable.add(node.user()))); + if (m_add_visible) { + m_visibles.push_back(node.visible()); } } - std::string blob_data; - pbf_blob.SerializeToString(&blob_data); + m_lats.push_back(m_delta_lat.update(lonlat2int(node.location().lat_without_check()))); + m_lons.push_back(m_delta_lon.update(lonlat2int(node.location().lon_without_check()))); - OSMPBF::BlobHeader pbf_blob_header; - pbf_blob_header.set_type(type); - pbf_blob_header.set_datasize(static_cast_with_assert<::google::protobuf::int32>(blob_data.size())); - - std::string blob_header_data; - pbf_blob_header.SerializeToString(&blob_header_data); - - uint32_t sz = htonl(static_cast_with_assert(blob_header_data.size())); - - // write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob - std::string output; - output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size()); - output.append(reinterpret_cast(&sz), sizeof(sz)); - output.append(blob_header_data); - output.append(blob_data); - - return output; + for (const auto& tag : node.tags()) { + m_tags.push_back(m_stringtable.add(tag.key())); + m_tags.push_back(m_stringtable.add(tag.value())); + } + m_tags.push_back(0); } - } // anonymous namespace + std::string serialize() const { + std::string data; + protozero::pbf_builder pbf_dense_nodes(data); - class PBFOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler { + pbf_dense_nodes.add_packed_sint64(OSMFormat::DenseNodes::packed_sint64_id, m_ids.cbegin(), m_ids.cend()); - /** - * Maximum number of items in a primitive block. - * - * The uncompressed length of a Blob *should* be less - * than 16 megabytes and *must* be less than 32 megabytes. - * - * A block may contain any number of entities, as long as - * the size limits for the surrounding blob are obeyed. - * However, for simplicity, the current Osmosis (0.38) - * as well as Osmium implementation always - * uses at most 8k entities in a block. - */ - static constexpr uint32_t max_block_contents = 8000; + if (m_add_metadata) { + protozero::pbf_builder pbf_dense_info(pbf_dense_nodes, OSMFormat::DenseNodes::optional_DenseInfo_denseinfo); + pbf_dense_info.add_packed_int32(OSMFormat::DenseInfo::packed_int32_version, m_versions.cbegin(), m_versions.cend()); + pbf_dense_info.add_packed_sint64(OSMFormat::DenseInfo::packed_sint64_timestamp, m_timestamps.cbegin(), m_timestamps.cend()); + pbf_dense_info.add_packed_sint64(OSMFormat::DenseInfo::packed_sint64_changeset, m_changesets.cbegin(), m_changesets.cend()); + pbf_dense_info.add_packed_sint32(OSMFormat::DenseInfo::packed_sint32_uid, m_uids.cbegin(), m_uids.cend()); + pbf_dense_info.add_packed_sint32(OSMFormat::DenseInfo::packed_sint32_user_sid, m_user_sids.cbegin(), m_user_sids.cend()); + + if (m_add_visible) { + pbf_dense_info.add_packed_bool(OSMFormat::DenseInfo::packed_bool_visible, m_visibles.cbegin(), m_visibles.cend()); + } + } + + pbf_dense_nodes.add_packed_sint64(OSMFormat::DenseNodes::packed_sint64_lat, m_lats.cbegin(), m_lats.cend()); + pbf_dense_nodes.add_packed_sint64(OSMFormat::DenseNodes::packed_sint64_lon, m_lons.cbegin(), m_lons.cend()); + + pbf_dense_nodes.add_packed_int32(OSMFormat::DenseNodes::packed_int32_keys_vals, m_tags.cbegin(), m_tags.cend()); + + return data; + } + + }; // class DenseNodes + + class PrimitiveBlock { + + std::string m_pbf_primitive_group_data; + protozero::pbf_builder m_pbf_primitive_group; + StringTable m_stringtable; + DenseNodes m_dense_nodes; + OSMFormat::PrimitiveGroup m_type; + int m_count; + + public: + + PrimitiveBlock(bool add_metadata, bool add_visible) : + m_pbf_primitive_group_data(), + m_pbf_primitive_group(m_pbf_primitive_group_data), + m_stringtable(), + m_dense_nodes(m_stringtable, add_metadata, add_visible), + m_type(OSMFormat::PrimitiveGroup::unknown), + m_count(0) { + } + + const std::string& group_data() { + if (type() == OSMFormat::PrimitiveGroup::optional_DenseNodes_dense) { + m_pbf_primitive_group.add_message(OSMFormat::PrimitiveGroup::optional_DenseNodes_dense, m_dense_nodes.serialize()); + } + return m_pbf_primitive_group_data; + } + + void reset(OSMFormat::PrimitiveGroup type) { + m_pbf_primitive_group_data.clear(); + m_stringtable.clear(); + m_dense_nodes.clear(); + m_type = type; + m_count = 0; + } + + void write_stringtable(protozero::pbf_builder& pbf_string_table) { + for (const char* s : m_stringtable) { + pbf_string_table.add_bytes(OSMFormat::StringTable::repeated_bytes_s, s); + } + } + + protozero::pbf_builder& group() { + ++m_count; + return m_pbf_primitive_group; + } + + void add_dense_node(const osmium::Node& node) { + m_dense_nodes.add_node(node); + ++m_count; + } + + size_t add_string(const char* s) { + return m_stringtable.add(s); + } + + int count() const { + return m_count; + } + + OSMFormat::PrimitiveGroup type() const { + return m_type; + } + + size_t size() const { + return m_pbf_primitive_group_data.size() + m_stringtable.size() + m_dense_nodes.size(); + } /** * The output buffer (block) will be filled to about @@ -211,715 +334,240 @@ namespace osmium { * enough space for the string table (which typically * needs about 0.1 to 0.3% of the block size). */ - static constexpr int64_t buffer_fill_percent = 95; + constexpr static size_t max_used_blob_size = max_uncompressed_blob_size * 95 / 100; + + bool can_add(OSMFormat::PrimitiveGroup type) const { + if (type != m_type) { + return false; + } + if (count() >= max_entities_per_block) { + return false; + } + return size() < max_used_blob_size; + } + + }; // class PrimitiveBlock + + class PBFOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler { + + /// Should nodes be encoded in DenseNodes? + bool m_use_dense_nodes; /** - * protobuf-struct of a HeaderBlock - */ - OSMPBF::HeaderBlock pbf_header_block; - - /** - * protobuf-struct of a PrimitiveBlock - */ - OSMPBF::PrimitiveBlock pbf_primitive_block; - - /** - * pointer to PrimitiveGroups inside the current PrimitiveBlock, - * used for writing nodes, ways or relations - */ - OSMPBF::PrimitiveGroup* pbf_nodes; - OSMPBF::PrimitiveGroup* pbf_ways; - OSMPBF::PrimitiveGroup* pbf_relations; - - /** - * To flexibly handle multiple resolutions, the granularity, or - * resolution used for representing locations is adjustable in - * multiples of 1 nanodegree. The default scaling factor is 100 - * nanodegrees, corresponding to about ~1cm at the equator. - * This is the current resolution of the OSM database. - */ - int m_location_granularity; - - /** - * The granularity used for representing timestamps is also adjustable in - * multiples of 1 millisecond. The default scaling factor is 1000 - * milliseconds, which is the current resolution of the OSM database. - */ - int m_date_granularity; - - /** - * should nodes be serialized into the dense format? + * Should the PBF blobs contain zlib compressed data? * - * nodes can be encoded one of two ways, as a Node - * (m_use_dense_nodes = false) and a special dense format. - * In the dense format, all information is stored 'column wise', - * as an array of ID's, array of latitudes, and array of - * longitudes. Each column is delta-encoded. This reduces - * header overheads and allows delta-coding to work very effectively. + * The zlib compression is optional, it's possible to store the + * blobs in raw format. Disabling the compression can improve + * the writing speed a little but the output will be 2x to 3x + * bigger. */ - bool m_use_dense_nodes {true}; + bool m_use_compression; - /** - * should the PBF blobs contain zlib compressed data? - * - * the zlib compression is optional, it's possible to store the - * blobs in raw format. Disabling the compression can improve the - * writing speed a little but the output will be 2x to 3x bigger. - */ - bool m_use_compression {true}; + /// Should metadata of objects be written? + bool m_add_metadata; - /** - * Should the string tables in the data blocks be sorted? - * - * Not sorting the string tables makes writing PBF files - * slightly faster. - */ - bool m_sort_stringtables { true }; - - /** - * While the .osm.pbf-format is able to carry all meta information, it is - * also able to omit this information to reduce size. - */ - bool m_should_add_metadata {true}; - - /** - * Should the visible flag be added on objects? - */ + /// Should the visible flag be added to objects? bool m_add_visible; - /** - * counter used to quickly check the number of objects stored inside - * the current PrimitiveBlock. When the counter reaches max_block_contents - * the PrimitiveBlock is serialized into a Blob and flushed to the file. - * - * this check is performed in check_block_contents_counter() which is - * called once for each object. - */ - uint16_t primitive_block_contents; - int primitive_block_size; + PrimitiveBlock m_primitive_block; - // StringTable management - StringTable string_table; - - /** - * These variables are used to calculate the - * delta-encoding while storing dense-nodes. It holds the last seen values - * from which the difference is stored into the protobuf. - */ - osmium::util::DeltaEncode m_delta_id; - osmium::util::DeltaEncode m_delta_lat; - osmium::util::DeltaEncode m_delta_lon; - osmium::util::DeltaEncode m_delta_timestamp; - osmium::util::DeltaEncode m_delta_changeset; - osmium::util::DeltaEncode m_delta_uid; - osmium::util::DeltaEncode<::google::protobuf::int32> m_delta_user_sid; - - bool debug; - - bool has_debug_level(int) { - return false; - } - - ///// Blob writing ///// - - void delta_encode_string_ids() { - if (pbf_nodes && pbf_nodes->has_dense()) { - OSMPBF::DenseNodes* dense = pbf_nodes->mutable_dense(); - - if (dense->has_denseinfo()) { - OSMPBF::DenseInfo* denseinfo = dense->mutable_denseinfo(); - - for (int i = 0, l=denseinfo->user_sid_size(); iuser_sid(i); - denseinfo->set_user_sid(i, m_delta_user_sid.update(user_sid)); - } - } - } - } - - /** - * Before a PrimitiveBlock gets serialized, all interim StringTable-ids needs to be - * mapped to the associated real StringTable ids. This is done in this function. - * - * This function needs to know about the concrete structure of all item types to find - * all occurrences of string-ids. - */ - void map_string_ids() { - // test, if the node-block has been allocated - if (pbf_nodes) { - // iterate over all nodes, passing them to the map_common_string_ids function - for (int i = 0, l=pbf_nodes->nodes_size(); imutable_nodes(i)); - } - - // test, if the node-block has a densenodes structure - if (pbf_nodes->has_dense()) { - // get a pointer to the densenodes structure - OSMPBF::DenseNodes* dense = pbf_nodes->mutable_dense(); - - // in the densenodes structure keys and vals are encoded in an intermixed - // array, individual nodes are separated by a value of 0 (0 in the StringTable - // is always unused). String-ids of 0 are thus kept alone. - for (int i = 0, l=dense->keys_vals_size(); i 0 to real string ids - auto sid = dense->keys_vals(i); - if (sid > 0) { - dense->set_keys_vals(i, string_table.map_string_id(sid)); - } - } - - // test if the densenodes block has meta infos - if (dense->has_denseinfo()) { - // get a pointer to the denseinfo structure - OSMPBF::DenseInfo* denseinfo = dense->mutable_denseinfo(); - - // iterate over all username string-ids - for (int i = 0, l=denseinfo->user_sid_size(); i 0 to real string ids - auto user_sid = string_table.map_string_id(denseinfo->user_sid(i)); - - // delta encode the string-id - denseinfo->set_user_sid(i, m_delta_user_sid.update(user_sid)); - } - } - } - } - - // test, if the ways-block has been allocated - if (pbf_ways) { - // iterate over all ways, passing them to the map_common_string_ids function - for (int i = 0, l=pbf_ways->ways_size(); imutable_ways(i)); - } - } - - // test, if the relations-block has been allocated - if (pbf_relations) { - // iterate over all relations - for (int i = 0, l=pbf_relations->relations_size(); imutable_relations(i); - - // pass them to the map_common_string_ids function - map_common_string_ids(relation); - - // iterate over all relation members, mapping the interim string-ids - // of the role to real string ids - for (int mi = 0; mi < relation->roles_sid_size(); ++mi) { - relation->set_roles_sid(mi, string_table.map_string_id(relation->roles_sid(mi))); - } - } - } - } - - /** - * a helper function used in map_string_ids to map common interim string-ids of the - * user name and all tags to real string ids. - * - * TPBFObject is either OSMPBF::Node, OSMPBF::Way or OSMPBF::Relation. - */ - template - void map_common_string_ids(TPBFObject* in) { - // if the object has meta-info attached - if (in->has_info()) { - // map the interim-id of the user name to a real id - OSMPBF::Info* info = in->mutable_info(); - info->set_user_sid(string_table.map_string_id(info->user_sid())); - } - - // iterate over all tags and map the interim-ids of the key and the value to real ids - for (int i = 0, l=in->keys_size(); iset_keys(i, string_table.map_string_id(in->keys(i))); - in->set_vals(i, string_table.map_string_id(in->vals(i))); - } - } - - - ///// MetaData helper ///// - - /** - * convert a double lat or lon value to an int, respecting the current blocks granularity - */ - int64_t lonlat2int(double lonlat) { - return static_cast(std::round(lonlat * OSMPBF::lonlat_resolution / location_granularity())); - } - - /** - * convert a timestamp to an int, respecting the current blocks granularity - */ - int64_t timestamp2int(time_t timestamp) { - return static_cast(std::round(timestamp * (1000.0 / date_granularity()))); - } - - /** - * helper function used in the write()-calls to apply common information from an osmium-object - * onto a pbf-object. - * - * TPBFObject is either OSMPBF::Node, OSMPBF::Way or OSMPBF::Relation. - */ - template - void apply_common_info(const osmium::OSMObject& in, TPBFObject* out) { - // set the object-id - out->set_id(in.id()); - - // iterate over all tags and set the keys and vals, recording the strings in the - // interim StringTable and storing the interim ids - for (const auto& tag : in.tags()) { - out->add_keys(string_table.record_string(tag.key())); - out->add_vals(string_table.record_string(tag.value())); - } - - if (m_should_add_metadata) { - // add an info-section to the pbf object and set the meta-info on it - OSMPBF::Info* out_info = out->mutable_info(); - if (m_add_visible) { - out_info->set_visible(in.visible()); - } - out_info->set_version(static_cast<::google::protobuf::int32>(in.version())); - out_info->set_timestamp(timestamp2int(in.timestamp())); - out_info->set_changeset(in.changeset()); - out_info->set_uid(static_cast<::google::protobuf::int32>(in.uid())); - out_info->set_user_sid(string_table.record_string(in.user())); - } - } - - - ///// High-Level Block writing ///// - - /** - * store the current pbf_header_block into a Blob and clear this struct afterwards. - */ - void store_header_block() { - if (debug && has_debug_level(1)) { - std::cerr << "storing header block" << std::endl; - } - - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(serialize_blob("OSMHeader", pbf_header_block, m_use_compression)); - - pbf_header_block.Clear(); - } - - /** - * store the interim StringTable to the current pbf_primitive_block, map all interim string ids - * to real StringTable ids and then store the current pbf_primitive_block into a Blob and clear - * this struct and all related pointers and maps afterwards. - */ void store_primitive_block() { - if (debug && has_debug_level(1)) { - std::cerr << "storing primitive block with " << primitive_block_contents << " items" << std::endl; + if (m_primitive_block.count() == 0) { + return; } - // set the granularity - pbf_primitive_block.set_granularity(location_granularity()); - pbf_primitive_block.set_date_granularity(date_granularity()); + std::string primitive_block_data; + protozero::pbf_builder primitive_block(primitive_block_data); - string_table.store_stringtable(pbf_primitive_block.mutable_stringtable(), m_sort_stringtables); - - if (m_sort_stringtables) { - map_string_ids(); - } else { - delta_encode_string_ids(); + { + protozero::pbf_builder pbf_string_table(primitive_block, OSMFormat::PrimitiveBlock::required_StringTable_stringtable); + m_primitive_block.write_stringtable(pbf_string_table); } + primitive_block.add_message(OSMFormat::PrimitiveBlock::repeated_PrimitiveGroup_primitivegroup, m_primitive_block.group_data()); + std::promise promise; m_output_queue.push(promise.get_future()); - promise.set_value(serialize_blob("OSMData", pbf_primitive_block, m_use_compression)); - - // clear the PrimitiveBlock struct - pbf_primitive_block.Clear(); - - // clear the interim StringTable and its id map - string_table.clear(); - - // reset the delta variables - m_delta_id.clear(); - m_delta_lat.clear(); - m_delta_lon.clear(); - m_delta_timestamp.clear(); - m_delta_changeset.clear(); - m_delta_uid.clear(); - m_delta_user_sid.clear(); - - // reset the contents-counter to zero - primitive_block_contents = 0; - primitive_block_size = 0; - - // reset the node/way/relation pointers to nullptr - pbf_nodes = nullptr; - pbf_ways = nullptr; - pbf_relations = nullptr; + promise.set_value(serialize_blob("OSMData", primitive_block_data, m_use_compression)); } - /** - * this little function checks primitive_block_contents counter against its maximum and calls - * store_primitive_block to flush the block to the disk when it's reached. It's also responsible - * for increasing this counter. - * - * this function also checks the estimated size of the current block and calls store_primitive_block - * when the estimated size reaches buffer_fill_percent of the maximum uncompressed blob size. - */ - void check_block_contents_counter() { - if (primitive_block_contents >= max_block_contents) { - store_primitive_block(); - } else if (primitive_block_size > OSMPBF::max_uncompressed_blob_size * buffer_fill_percent / 100) { - if (debug && has_debug_level(1)) { - std::cerr << "storing primitive_block with only " << primitive_block_contents << " items, because its ByteSize (" << primitive_block_size << ") reached " << - (static_cast(primitive_block_size) / static_cast(OSMPBF::max_uncompressed_blob_size) * 100.0) << "% of the maximum blob-size" << std::endl; - } + template + void add_meta(const osmium::OSMObject& object, T& pbf_object) { + const osmium::TagList& tags = object.tags(); - store_primitive_block(); - } + auto map_tag_key = [this](const osmium::Tag& tag) -> size_t { + return m_primitive_block.add_string(tag.key()); + }; + auto map_tag_value = [this](const osmium::Tag& tag) -> size_t { + return m_primitive_block.add_string(tag.value()); + }; - ++primitive_block_contents; - } + pbf_object.add_packed_uint32(T::enum_type::packed_uint32_keys, + boost::make_transform_iterator(tags.begin(), map_tag_key), + boost::make_transform_iterator(tags.end(), map_tag_key)); + pbf_object.add_packed_uint32(T::enum_type::packed_uint32_vals, + boost::make_transform_iterator(tags.begin(), map_tag_value), + boost::make_transform_iterator(tags.end(), map_tag_value)); - ///// Block content writing ///// - - /** - * Add a node to the block. - * - * @param node The node to add. - */ - void write_node(const osmium::Node& node) { - // add a way to the group - OSMPBF::Node* pbf_node = pbf_nodes->add_nodes(); - - // copy the common meta-info from the osmium-object to the pbf-object - apply_common_info(node, pbf_node); - - // modify lat & lon to integers, respecting the block's granularity and copy - // the ints to the pbf-object - pbf_node->set_lon(lonlat2int(node.location().lon_without_check())); - pbf_node->set_lat(lonlat2int(node.location().lat_without_check())); - } - - /** - * Add a node to the block using DenseNodes. - * - * @param node The node to add. - */ - void write_dense_node(const osmium::Node& node) { - // add a DenseNodes-Section to the PrimitiveGroup - OSMPBF::DenseNodes* dense = pbf_nodes->mutable_dense(); - - // copy the id, delta encoded - dense->add_id(m_delta_id.update(node.id())); - - // copy the longitude, delta encoded - dense->add_lon(m_delta_lon.update(lonlat2int(node.location().lon_without_check()))); - - // copy the latitude, delta encoded - dense->add_lat(m_delta_lat.update(lonlat2int(node.location().lat_without_check()))); - - // in the densenodes structure keys and vals are encoded in an intermixed - // array, individual nodes are separated by a value of 0 (0 in the StringTable - // is always unused) - // so for three nodes the keys_vals array may look like this: 3 5 2 1 0 0 8 5 - // the first node has two tags (3=>5 and 2=>1), the second node does not - // have any tags and the third node has a single tag (8=>5) - for (const auto& tag : node.tags()) { - dense->add_keys_vals(string_table.record_string(tag.key())); - dense->add_keys_vals(string_table.record_string(tag.value())); - } - dense->add_keys_vals(0); - - if (m_should_add_metadata) { - // add a DenseInfo-Section to the PrimitiveGroup - OSMPBF::DenseInfo* denseinfo = dense->mutable_denseinfo(); - - denseinfo->add_version(static_cast<::google::protobuf::int32>(node.version())); + if (m_add_metadata) { + protozero::pbf_builder pbf_info(pbf_object, T::enum_type::optional_Info_info); + pbf_info.add_int32(OSMFormat::Info::optional_int32_version, object.version()); + pbf_info.add_int64(OSMFormat::Info::optional_int64_timestamp, object.timestamp()); + pbf_info.add_int64(OSMFormat::Info::optional_int64_changeset, object.changeset()); + pbf_info.add_int32(OSMFormat::Info::optional_int32_uid, object.uid()); + pbf_info.add_uint32(OSMFormat::Info::optional_uint32_user_sid, m_primitive_block.add_string(object.user())); if (m_add_visible) { - denseinfo->add_visible(node.visible()); + pbf_info.add_bool(OSMFormat::Info::optional_bool_visible, object.visible()); } - - // copy the timestamp, delta encoded - denseinfo->add_timestamp(m_delta_timestamp.update(timestamp2int(node.timestamp()))); - - // copy the changeset, delta encoded - denseinfo->add_changeset(m_delta_changeset.update(node.changeset())); - - // copy the user id, delta encoded - denseinfo->add_uid(static_cast<::google::protobuf::int32>(m_delta_uid.update(node.uid()))); - - // record the user-name to the interim stringtable and copy the - // interim string-id to the pbf-object - denseinfo->add_user_sid(string_table.record_string(node.user())); } } - /** - * Add a way to the block. - * - * @param way The way to add. - */ - void write_way(const osmium::Way& way) { - // add a way to the group - OSMPBF::Way* pbf_way = pbf_ways->add_ways(); - - // copy the common meta-info from the osmium-object to the pbf-object - apply_common_info(way, pbf_way); - - // last way-node-id used for delta-encoding - osmium::util::DeltaEncode delta_id; - - for (const auto& node_ref : way.nodes()) { - // copy the way-node-id, delta encoded - pbf_way->add_refs(delta_id.update(node_ref.ref())); - } - - // count up blob size by the size of the Way - primitive_block_size += pbf_way->ByteSize(); - } - - /** - * Add a relation to the block. - * - * @param relation The relation to add. - */ - void write_relation(const osmium::Relation& relation) { - // add a relation to the group - OSMPBF::Relation* pbf_relation = pbf_relations->add_relations(); - - // copy the common meta-info from the osmium-object to the pbf-object - apply_common_info(relation, pbf_relation); - - osmium::util::DeltaEncode delta_id; - - for (const auto& member : relation.members()) { - // record the relation-member role to the interim stringtable and copy the - // interim string-id to the pbf-object - pbf_relation->add_roles_sid(string_table.record_string(member.role())); - - // copy the relation-member-id, delta encoded - pbf_relation->add_memids(delta_id.update(member.ref())); - - // copy the relation-member-type, mapped to the OSMPBF enum - pbf_relation->add_types(item_type_to_osmpbf_membertype(member.type())); - } - - // count up blob size by the size of the Relation - primitive_block_size += pbf_relation->ByteSize(); - } - - // objects of this class can't be copied PBFOutputFormat(const PBFOutputFormat&) = delete; PBFOutputFormat& operator=(const PBFOutputFormat&) = delete; public: - /** - * Create PBFOutputFormat object from File. - */ explicit PBFOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : OutputFormat(file, output_queue), - pbf_header_block(), - pbf_primitive_block(), - pbf_nodes(nullptr), - pbf_ways(nullptr), - pbf_relations(nullptr), - m_location_granularity(pbf_primitive_block.granularity()), - m_date_granularity(pbf_primitive_block.date_granularity()), + m_use_dense_nodes(file.get("pbf_dense_nodes") != "false"), + m_use_compression(file.get("pbf_compression") != "none" && file.get("pbf_compression") != "false"), + m_add_metadata(file.get("pbf_add_metadata") != "false" && file.get("add_metadata") != "false"), m_add_visible(file.has_multiple_object_versions()), - primitive_block_contents(0), - primitive_block_size(0), - string_table(), - m_delta_id(), - m_delta_lat(), - m_delta_lon(), - m_delta_timestamp(), - m_delta_changeset(), - m_delta_uid(), - m_delta_user_sid(), - debug(true) { - GOOGLE_PROTOBUF_VERIFY_VERSION; - if (file.get("pbf_dense_nodes") == "false") { - m_use_dense_nodes = false; - } - if (file.get("pbf_compression") == "none" || file.get("pbf_compression") == "false") { - m_use_compression = false; - } - if (file.get("pbf_sort_stringtables") == "false") { - m_sort_stringtables = false; - } - if (file.get("pbf_add_metadata") == "false") { - m_should_add_metadata = false; - } + m_primitive_block(m_add_metadata, m_add_visible) { } void write_buffer(osmium::memory::Buffer&& buffer) override final { osmium::apply(buffer.cbegin(), buffer.cend(), *this); } - - /** - * getter to access the granularity - */ - int location_granularity() const { - return m_location_granularity; - } - - /** - * setter to set the granularity - */ - PBFOutputFormat& location_granularity(int g) { - m_location_granularity = g; - return *this; - } - - - /** - * getter to access the date_granularity - */ - int date_granularity() const { - return m_date_granularity; - } - - /** - * Set date granularity. - */ - PBFOutputFormat& date_granularity(int g) { - m_date_granularity = g; - return *this; - } - - - /** - * Initialize the writing process. - * - * This initializes the header-block, sets the required-features and - * the writing-program and adds the obligatory StringTable-Index 0. - */ void write_header(const osmium::io::Header& header) override final { - // add the schema version as required feature to the HeaderBlock - pbf_header_block.add_required_features("OsmSchema-V0.6"); - - // when the densenodes-feature is used, add DenseNodes as required feature - if (m_use_dense_nodes) { - pbf_header_block.add_required_features("DenseNodes"); - } - - // when the resulting file will carry history information, add - // HistoricalInformation as required feature - if (m_file.has_multiple_object_versions()) { - pbf_header_block.add_required_features("HistoricalInformation"); - } - - // set the writing program - pbf_header_block.set_writingprogram(header.get("generator")); + std::string data; + protozero::pbf_builder pbf_header_block(data); if (!header.boxes().empty()) { - OSMPBF::HeaderBBox* pbf_bbox = pbf_header_block.mutable_bbox(); + protozero::pbf_builder pbf_header_bbox(pbf_header_block, OSMFormat::HeaderBlock::optional_HeaderBBox_bbox); + osmium::Box box = header.joined_boxes(); - pbf_bbox->set_left(static_cast<::google::protobuf::int64>(box.bottom_left().lon() * OSMPBF::lonlat_resolution)); - pbf_bbox->set_bottom(static_cast<::google::protobuf::int64>(box.bottom_left().lat() * OSMPBF::lonlat_resolution)); - pbf_bbox->set_right(static_cast<::google::protobuf::int64>(box.top_right().lon() * OSMPBF::lonlat_resolution)); - pbf_bbox->set_top(static_cast<::google::protobuf::int64>(box.top_right().lat() * OSMPBF::lonlat_resolution)); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_left, box.bottom_left().lon() * lonlat_resolution); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_right, box.top_right().lon() * lonlat_resolution); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_top, box.top_right().lat() * lonlat_resolution); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_bottom, box.bottom_left().lat() * lonlat_resolution); } + pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "OsmSchema-V0.6"); + + if (m_use_dense_nodes) { + pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "DenseNodes"); + } + + if (m_file.has_multiple_object_versions()) { + pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "HistoricalInformation"); + } + + pbf_header_block.add_string(OSMFormat::HeaderBlock::optional_string_writingprogram, header.get("generator")); + std::string osmosis_replication_timestamp = header.get("osmosis_replication_timestamp"); if (!osmosis_replication_timestamp.empty()) { osmium::Timestamp ts(osmosis_replication_timestamp.c_str()); - pbf_header_block.set_osmosis_replication_timestamp(ts); + pbf_header_block.add_int64(OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp, ts); } std::string osmosis_replication_sequence_number = header.get("osmosis_replication_sequence_number"); if (!osmosis_replication_sequence_number.empty()) { - pbf_header_block.set_osmosis_replication_sequence_number(std::atoll(osmosis_replication_sequence_number.c_str())); + pbf_header_block.add_int64(OSMFormat::HeaderBlock::optional_int64_osmosis_replication_sequence_number, std::atoll(osmosis_replication_sequence_number.c_str())); } std::string osmosis_replication_base_url = header.get("osmosis_replication_base_url"); if (!osmosis_replication_base_url.empty()) { - pbf_header_block.set_osmosis_replication_base_url(osmosis_replication_base_url); + pbf_header_block.add_string(OSMFormat::HeaderBlock::optional_string_osmosis_replication_base_url, osmosis_replication_base_url); } - store_header_block(); + std::promise promise; + m_output_queue.push(promise.get_future()); + promise.set_value(serialize_blob("OSMHeader", data, m_use_compression)); + } + + void switch_primitive_block_type(OSMFormat::PrimitiveGroup type) { + if (!m_primitive_block.can_add(type)) { + store_primitive_block(); + m_primitive_block.reset(type); + } } - /** - * Add a node to the pbf. - * - * A call to this method won't write the node to the file directly but - * cache it for later bulk-writing. Calling final() ensures that everything - * gets written and every file pointer is closed. - */ void node(const osmium::Node& node) { - // first of we check the contents-counter which may flush the cached nodes to - // disk if the limit is reached. This call also increases the contents-counter - check_block_contents_counter(); - - if (debug && has_debug_level(2)) { - std::cerr << "node " << node.id() << " v" << node.version() << std::endl; - } - - // if no PrimitiveGroup for nodes has been added, add one and save the pointer - if (!pbf_nodes) { - pbf_nodes = pbf_primitive_block.add_primitivegroup(); - } - if (m_use_dense_nodes) { - write_dense_node(node); - } else { - write_node(node); + switch_primitive_block_type(OSMFormat::PrimitiveGroup::optional_DenseNodes_dense); + m_primitive_block.add_dense_node(node); + return; } + + switch_primitive_block_type(OSMFormat::PrimitiveGroup::repeated_Node_nodes); + protozero::pbf_builder pbf_node{ m_primitive_block.group(), OSMFormat::PrimitiveGroup::repeated_Node_nodes }; + + pbf_node.add_sint64(OSMFormat::Node::required_sint64_id, node.id()); + add_meta(node, pbf_node); + + pbf_node.add_sint64(OSMFormat::Node::required_sint64_lat, lonlat2int(node.location().lat_without_check())); + pbf_node.add_sint64(OSMFormat::Node::required_sint64_lon, lonlat2int(node.location().lon_without_check())); } - /** - * Add a way to the pbf. - * - * A call to this method won't write the way to the file directly but - * cache it for later bulk-writing. Calling final() ensures that everything - * gets written and every file pointer is closed. - */ void way(const osmium::Way& way) { - // first of we check the contents-counter which may flush the cached ways to - // disk if the limit is reached. This call also increases the contents-counter - check_block_contents_counter(); + switch_primitive_block_type(OSMFormat::PrimitiveGroup::repeated_Way_ways); + protozero::pbf_builder pbf_way{ m_primitive_block.group(), OSMFormat::PrimitiveGroup::repeated_Way_ways }; - // if no PrimitiveGroup for nodes has been added, add one and save the pointer - if (!pbf_ways) { - pbf_ways = pbf_primitive_block.add_primitivegroup(); - } + pbf_way.add_int64(OSMFormat::Way::required_int64_id, way.id()); + add_meta(way, pbf_way); - write_way(way); + static auto map_node_ref = [](osmium::NodeRefList::const_iterator node_ref) noexcept -> osmium::object_id_type { + return node_ref->ref(); + }; + typedef osmium::util::DeltaEncodeIterator it_type; + + const auto& nodes = way.nodes(); + it_type first { nodes.cbegin(), nodes.cend(), map_node_ref }; + it_type last { nodes.cend(), nodes.cend(), map_node_ref }; + pbf_way.add_packed_sint64(OSMFormat::Way::packed_sint64_refs, first, last); } - /** - * Add a relation to the pbf. - * - * A call to this method won't write the way to the file directly but - * cache it for later bulk-writing. Calling final() ensures that everything - * gets written and every file pointer is closed. - */ void relation(const osmium::Relation& relation) { - // first of we check the contents-counter which may flush the cached relations to - // disk if the limit is reached. This call also increases the contents-counter - check_block_contents_counter(); + switch_primitive_block_type(OSMFormat::PrimitiveGroup::repeated_Relation_relations); + protozero::pbf_builder pbf_relation { m_primitive_block.group(), OSMFormat::PrimitiveGroup::repeated_Relation_relations }; - // if no PrimitiveGroup for relations has been added, add one and save the pointer - if (!pbf_relations) { - pbf_relations = pbf_primitive_block.add_primitivegroup(); - } + pbf_relation.add_int64(OSMFormat::Relation::required_int64_id, relation.id()); + add_meta(relation, pbf_relation); - write_relation(relation); + auto map_member_role = [this](const osmium::RelationMember& member) -> size_t { + return m_primitive_block.add_string(member.role()); + }; + pbf_relation.add_packed_int32(OSMFormat::Relation::packed_int32_roles_sid, + boost::make_transform_iterator(relation.members().begin(), map_member_role), + boost::make_transform_iterator(relation.members().end(), map_member_role)); + + static auto map_member_ref = [](osmium::RelationMemberList::const_iterator member) noexcept -> osmium::object_id_type { + return member->ref(); + }; + typedef osmium::util::DeltaEncodeIterator it_type; + const auto& members = relation.members(); + it_type first { members.cbegin(), members.cend(), map_member_ref }; + it_type last { members.cend(), members.cend(), map_member_ref }; + pbf_relation.add_packed_sint64(OSMFormat::Relation::packed_sint64_memids, first, last); + + static auto map_member_type = [](const osmium::RelationMember& member) noexcept -> int { + return osmium::item_type_to_nwr_index(member.type()); + }; + pbf_relation.add_packed_int32(OSMFormat::Relation::packed_MemberType_types, + boost::make_transform_iterator(relation.members().begin(), map_member_type), + boost::make_transform_iterator(relation.members().end(), map_member_type)); } /** - * Finalize the writing process, flush any open primitive blocks to the file and - * close the file. + * Finalize the writing process, flush any open primitive + * blocks to the file and close the file. */ void close() override final { - if (debug && has_debug_level(1)) { - std::cerr << "finishing" << std::endl; - } - - // if the current block contains any elements, flush it to the protobuf - if (primitive_block_contents > 0) { - store_primitive_block(); - } + store_primitive_block(); std::promise promise; m_output_queue.push(promise.get_future()); @@ -930,10 +578,15 @@ namespace osmium { namespace { +// we want the register_output_format() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_pbf_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::pbf, [](const osmium::io::File& file, data_queue_type& output_queue) { return new osmium::io::detail::PBFOutputFormat(file, output_queue); }); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/contrib/libosmium/osmium/io/detail/pbf_parser.hpp b/contrib/libosmium/osmium/io/detail/pbf_parser.hpp deleted file mode 100644 index f626b0b6..00000000 --- a/contrib/libosmium/osmium/io/detail/pbf_parser.hpp +++ /dev/null @@ -1,456 +0,0 @@ -#ifndef OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP -#define OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP - -/* - -This file is part of Osmium (http://osmcode.org/libosmium). - -Copyright 2013-2015 Jochen Topf and others (see README). - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -*/ - -#include -#include -#include -#include - -#include - -#include -#include // IWYU pragma: export -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace osmium { - - namespace io { - - namespace detail { - - class PBFPrimitiveBlockParser { - - static constexpr size_t initial_buffer_size = 2 * 1024 * 1024; - - const std::string& m_data; - - const OSMPBF::StringTable* m_stringtable; - int64_t m_lon_offset; - int64_t m_lat_offset; - int64_t m_date_factor; - int32_t m_granularity; - - osmium::osm_entity_bits::type m_read_types; - - osmium::memory::Buffer m_buffer; - - PBFPrimitiveBlockParser(const PBFPrimitiveBlockParser&) = delete; - PBFPrimitiveBlockParser(PBFPrimitiveBlockParser&&) = delete; - - PBFPrimitiveBlockParser& operator=(const PBFPrimitiveBlockParser&) = delete; - PBFPrimitiveBlockParser& operator=(PBFPrimitiveBlockParser&&) = delete; - - public: - - explicit PBFPrimitiveBlockParser(const std::string& data, osmium::osm_entity_bits::type read_types) : - m_data(data), - m_stringtable(nullptr), - m_lon_offset(0), - m_lat_offset(0), - m_date_factor(1000), - m_granularity(100), - m_read_types(read_types), - m_buffer(initial_buffer_size) { - } - - ~PBFPrimitiveBlockParser() = default; - - osmium::memory::Buffer operator()() { - OSMPBF::PrimitiveBlock pbf_primitive_block; - if (!pbf_primitive_block.ParseFromString(m_data)) { - throw osmium::pbf_error("failed to parse PrimitiveBlock"); - } - - m_stringtable = &pbf_primitive_block.stringtable(); - m_lon_offset = pbf_primitive_block.lon_offset(); - m_lat_offset = pbf_primitive_block.lat_offset(); - m_date_factor = pbf_primitive_block.date_granularity() / 1000; - m_granularity = pbf_primitive_block.granularity(); - - for (int i = 0; i < pbf_primitive_block.primitivegroup_size(); ++i) { - const OSMPBF::PrimitiveGroup& group = pbf_primitive_block.primitivegroup(i); - - if (group.has_dense()) { - if (m_read_types & osmium::osm_entity_bits::node) parse_dense_node_group(group); - } else if (group.ways_size() != 0) { - if (m_read_types & osmium::osm_entity_bits::way) parse_way_group(group); - } else if (group.relations_size() != 0) { - if (m_read_types & osmium::osm_entity_bits::relation) parse_relation_group(group); - } else if (group.nodes_size() != 0) { - if (m_read_types & osmium::osm_entity_bits::node) parse_node_group(group); - } else { - throw osmium::pbf_error("group of unknown type"); - } - } - - return std::move(m_buffer); - } - - private: - - template - void parse_attributes(TBuilder& builder, const TPBFObject& pbf_object) { - auto& object = builder.object(); - - object.set_id(pbf_object.id()); - - if (pbf_object.has_info()) { - object.set_version(static_cast_with_assert(pbf_object.info().version())) - .set_changeset(static_cast_with_assert(pbf_object.info().changeset())) - .set_timestamp(pbf_object.info().timestamp() * m_date_factor) - .set_uid_from_signed(pbf_object.info().uid()); - if (pbf_object.info().has_visible()) { - object.set_visible(pbf_object.info().visible()); - } - builder.add_user(m_stringtable->s(static_cast_with_assert(pbf_object.info().user_sid()))); - } else { - builder.add_user("", 1); - } - } - - void parse_node_group(const OSMPBF::PrimitiveGroup& group) { - for (int i = 0; i < group.nodes_size(); ++i) { - osmium::builder::NodeBuilder builder(m_buffer); - const OSMPBF::Node& pbf_node = group.nodes(i); - parse_attributes(builder, pbf_node); - - if (builder.object().visible()) { - builder.object().set_location(osmium::Location( - (pbf_node.lon() * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision), - (pbf_node.lat() * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision))); - } - - if (pbf_node.keys_size() > 0) { - osmium::builder::TagListBuilder tl_builder(m_buffer, &builder); - for (int tag = 0; tag < pbf_node.keys_size(); ++tag) { - tl_builder.add_tag(m_stringtable->s(static_cast(pbf_node.keys(tag))), - m_stringtable->s(static_cast(pbf_node.vals(tag)))); - } - } - - m_buffer.commit(); - } - } - - void parse_way_group(const OSMPBF::PrimitiveGroup& group) { - for (int i = 0; i < group.ways_size(); ++i) { - osmium::builder::WayBuilder builder(m_buffer); - const OSMPBF::Way& pbf_way = group.ways(i); - parse_attributes(builder, pbf_way); - - if (pbf_way.refs_size() > 0) { - osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder); - int64_t ref = 0; - for (int n = 0; n < pbf_way.refs_size(); ++n) { - ref += pbf_way.refs(n); - wnl_builder.add_node_ref(ref); - } - } - - if (pbf_way.keys_size() > 0) { - osmium::builder::TagListBuilder tl_builder(m_buffer, &builder); - for (int tag = 0; tag < pbf_way.keys_size(); ++tag) { - tl_builder.add_tag(m_stringtable->s(static_cast(pbf_way.keys(tag))), - m_stringtable->s(static_cast(pbf_way.vals(tag)))); - } - } - - m_buffer.commit(); - } - } - - void parse_relation_group(const OSMPBF::PrimitiveGroup& group) { - for (int i = 0; i < group.relations_size(); ++i) { - osmium::builder::RelationBuilder builder(m_buffer); - const OSMPBF::Relation& pbf_relation = group.relations(i); - parse_attributes(builder, pbf_relation); - - if (pbf_relation.types_size() > 0) { - osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder); - int64_t ref = 0; - for (int n = 0; n < pbf_relation.types_size(); ++n) { - ref += pbf_relation.memids(n); - rml_builder.add_member(osmpbf_membertype_to_item_type(pbf_relation.types(n)), ref, m_stringtable->s(pbf_relation.roles_sid(n))); - } - } - - if (pbf_relation.keys_size() > 0) { - osmium::builder::TagListBuilder tl_builder(m_buffer, &builder); - for (int tag = 0; tag < pbf_relation.keys_size(); ++tag) { - tl_builder.add_tag(m_stringtable->s(static_cast(pbf_relation.keys(tag))), - m_stringtable->s(static_cast(pbf_relation.vals(tag)))); - } - } - - m_buffer.commit(); - } - } - - int add_tags(const OSMPBF::DenseNodes& dense, int n, osmium::builder::NodeBuilder* builder) { - if (n >= dense.keys_vals_size()) { - return n; - } - - if (dense.keys_vals(n) == 0) { - return n+1; - } - - osmium::builder::TagListBuilder tl_builder(m_buffer, builder); - - while (n < dense.keys_vals_size()) { - int tag_key_pos = dense.keys_vals(n++); - - if (tag_key_pos == 0) { - break; - } - - tl_builder.add_tag(m_stringtable->s(tag_key_pos), - m_stringtable->s(dense.keys_vals(n))); - - ++n; - } - - return n; - } - - void parse_dense_node_group(const OSMPBF::PrimitiveGroup& group) { - int64_t last_dense_id = 0; - int64_t last_dense_latitude = 0; - int64_t last_dense_longitude = 0; - int64_t last_dense_uid = 0; - int64_t last_dense_user_sid = 0; - int64_t last_dense_changeset = 0; - int64_t last_dense_timestamp = 0; - int last_dense_tag = 0; - - const OSMPBF::DenseNodes& dense = group.dense(); - - for (int i = 0; i < dense.id_size(); ++i) { - bool visible = true; - - last_dense_id += dense.id(i); - last_dense_latitude += dense.lat(i); - last_dense_longitude += dense.lon(i); - - if (dense.has_denseinfo()) { - last_dense_changeset += dense.denseinfo().changeset(i); - last_dense_timestamp += dense.denseinfo().timestamp(i); - last_dense_uid += dense.denseinfo().uid(i); - last_dense_user_sid += dense.denseinfo().user_sid(i); - if (dense.denseinfo().visible_size() > 0) { - visible = dense.denseinfo().visible(i); - } - assert(last_dense_changeset >= 0); - assert(last_dense_timestamp >= 0); - assert(last_dense_uid >= -1); - assert(last_dense_user_sid >= 0); - } - - osmium::builder::NodeBuilder builder(m_buffer); - osmium::Node& node = builder.object(); - - node.set_id(last_dense_id); - - if (dense.has_denseinfo()) { - auto v = dense.denseinfo().version(i); - assert(v > 0); - node.set_version(static_cast(v)); - node.set_changeset(static_cast(last_dense_changeset)); - node.set_timestamp(last_dense_timestamp * m_date_factor); - node.set_uid_from_signed(static_cast(last_dense_uid)); - node.set_visible(visible); - builder.add_user(m_stringtable->s(static_cast(last_dense_user_sid))); - } else { - builder.add_user("", 1); - } - - if (visible) { - builder.object().set_location(osmium::Location( - (last_dense_longitude * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision), - (last_dense_latitude * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision))); - } - - last_dense_tag = add_tags(dense, last_dense_tag, &builder); - m_buffer.commit(); - } - } - - }; // class PBFPrimitiveBlockParser - - /** - * PBF blobs can optionally be packed with the zlib algorithm. - * This function returns the raw data (if it was unpacked) or - * the unpacked data (if it was packed). - * - * @param input_data Reference to input data. - * @returns Unpacked data - * @throws osmium::pbf_error If there was a problem parsing the PBF - */ - inline std::unique_ptr unpack_blob(const std::string& input_data) { - OSMPBF::Blob pbf_blob; - if (!pbf_blob.ParseFromString(input_data)) { - throw osmium::pbf_error("failed to parse blob"); - } - - if (pbf_blob.has_raw()) { - return std::unique_ptr(pbf_blob.release_raw()); - } else if (pbf_blob.has_zlib_data()) { - auto raw_size = pbf_blob.raw_size(); - assert(raw_size >= 0); - assert(raw_size <= OSMPBF::max_uncompressed_blob_size); - return osmium::io::detail::zlib_uncompress(pbf_blob.zlib_data(), static_cast(raw_size)); - } else if (pbf_blob.has_lzma_data()) { - throw osmium::pbf_error("lzma blobs not implemented"); - } else { - throw osmium::pbf_error("blob contains no data"); - } - } - - /** - * Parse blob as a HeaderBlock. - * - * @param input_buffer Blob data - * @returns Header object - * @throws osmium::pbf_error If there was a parsing error - */ - inline osmium::io::Header parse_header_blob(const std::string& input_buffer) { - const std::unique_ptr data = unpack_blob(input_buffer); - - OSMPBF::HeaderBlock pbf_header_block; - if (!pbf_header_block.ParseFromString(*data)) { - throw osmium::pbf_error("failed to parse HeaderBlock"); - } - - osmium::io::Header header; - for (int i = 0; i < pbf_header_block.required_features_size(); ++i) { - const std::string& feature = pbf_header_block.required_features(i); - - if (feature == "OsmSchema-V0.6") continue; - if (feature == "DenseNodes") { - header.set("pbf_dense_nodes", true); - continue; - } - if (feature == "HistoricalInformation") { - header.set_has_multiple_object_versions(true); - continue; - } - - throw osmium::pbf_error(std::string("required feature not supported: ") + feature); - } - - for (int i = 0; i < pbf_header_block.optional_features_size(); ++i) { - const std::string& feature = pbf_header_block.optional_features(i); - header.set("pbf_optional_feature_" + std::to_string(i), feature); - } - - if (pbf_header_block.has_writingprogram()) { - header.set("generator", pbf_header_block.writingprogram()); - } - - if (pbf_header_block.has_bbox()) { - const OSMPBF::HeaderBBox& pbf_bbox = pbf_header_block.bbox(); - const int64_t resolution_convert = OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision; - osmium::Box box; - box.extend(osmium::Location(pbf_bbox.left() / resolution_convert, pbf_bbox.bottom() / resolution_convert)); - box.extend(osmium::Location(pbf_bbox.right() / resolution_convert, pbf_bbox.top() / resolution_convert)); - header.add_box(box); - } - - if (pbf_header_block.has_osmosis_replication_timestamp()) { - header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.osmosis_replication_timestamp()).to_iso()); - } - - if (pbf_header_block.has_osmosis_replication_sequence_number()) { - header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.osmosis_replication_sequence_number())); - } - - if (pbf_header_block.has_osmosis_replication_base_url()) { - header.set("osmosis_replication_base_url", pbf_header_block.osmosis_replication_base_url()); - } - - return header; - } - - class DataBlobParser { - - std::shared_ptr m_input_buffer; - osmium::osm_entity_bits::type m_read_types; - - public: - - DataBlobParser(std::string&& input_buffer, osmium::osm_entity_bits::type read_types) : - m_input_buffer(std::make_shared(std::move(input_buffer))), - m_read_types(read_types) { - if (input_buffer.size() > OSMPBF::max_uncompressed_blob_size) { - throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size()))); - } - } -/* - DataBlobParser(const DataBlobParser& other) : - m_input_buffer(std::move(other.m_input_buffer)), - m_read_types(other.m_read_types) { - }*/ - - DataBlobParser(const DataBlobParser&) = default; - DataBlobParser& operator=(const DataBlobParser&) = default; - - DataBlobParser(DataBlobParser&&) = default; - DataBlobParser& operator=(DataBlobParser&&) = default; - - ~DataBlobParser() = default; - - osmium::memory::Buffer operator()() { - const std::unique_ptr data = unpack_blob(*m_input_buffer); - PBFPrimitiveBlockParser parser(*data, m_read_types); - return parser(); - } - - }; // class DataBlobParser - - } // namespace detail - - } // namespace io - -} // namespace osmium - -#endif // OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP diff --git a/contrib/libosmium/osmium/io/detail/pbf_stringtable.hpp b/contrib/libosmium/osmium/io/detail/pbf_stringtable.hpp deleted file mode 100644 index 5f540f12..00000000 --- a/contrib/libosmium/osmium/io/detail/pbf_stringtable.hpp +++ /dev/null @@ -1,218 +0,0 @@ -#ifndef OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP -#define OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP - -/* - -This file is part of Osmium (http://osmcode.org/libosmium). - -Copyright 2013-2015 Jochen Topf and others (see README). - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -*/ - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace osmium { - - namespace io { - - namespace detail { - - /** - * StringTable management for PBF writer - * - * All strings are stored as indexes to rows in a StringTable. The StringTable contains - * one row for each used string, so strings that are used multiple times need to be - * stored only once. The StringTable is sorted by usage-count, so the most often used - * string is stored at index 1. - */ - class StringTable { - - public: - - /// type for string IDs (interim and final) - typedef uint16_t string_id_type; - - private: - - /** - * this is the struct used to build the StringTable. It is stored as - * the value-part in the strings-map. - * - * when a new string is added to the map, its count is set to 0 and - * the interim_id is set to the current size of the map. This interim_id - * is then stored into the pbf-objects. - * - * before the PrimitiveBlock is serialized, the map is sorted by count - * and stored into the pbf-StringTable. Afterwards the interim-ids are - * mapped to the "real" id in the StringTable. - * - * this way often used strings get lower ids in the StringTable. As the - * protobuf-serializer stores numbers in variable bit-lengths, lower - * IDs means less used space in the resulting file. - */ - struct string_info { - - /// number of occurrences of this string - uint16_t count; - - /// an intermediate-id - string_id_type interim_id; - - }; // struct string_info - - /** - * Interim StringTable, storing all strings that should be written to - * the StringTable once the block is written to disk. - */ - typedef std::map string2string_info_type; - string2string_info_type m_strings; - - /** - * This vector is used to map the interim IDs to real StringTable IDs after - * writing all strings to the StringTable. - */ - typedef std::vector interim_id2id_type; - interim_id2id_type m_id2id_map; - - size_t m_size = 0; - - public: - - StringTable() { - } - - friend bool operator<(const string_info& lhs, const string_info& rhs) { - return lhs.count > rhs.count; - } - - /** - * record a string in the interim StringTable if it's missing, otherwise just increase its counter, - * return the interim-id assigned to the string. - */ - string_id_type record_string(const std::string& string) { - string_info& info = m_strings[string]; - if (info.interim_id == 0) { - ++m_size; - info.interim_id = static_cast_with_assert(m_size); - } else { - info.count++; - } - return info.interim_id; - } - - /** - * Sort the interim StringTable and store it to the real protobuf StringTable. - * while storing to the real table, this function fills the id2id_map with - * pairs, mapping the interim-ids to final and real StringTable ids. - * - * Note that the m_strings table is a std::map and as such is sorted lexicographically. - * When the transformation into the sortedby multimap is done, it gets sorted by - * the count. The end result (at least with the glibc standard container/algorithm - * implementation) is that the string table is sorted first by reverse count (ie descending) - * and then by reverse lexicographic order. - */ - void store_stringtable(OSMPBF::StringTable* st, bool sort) { - // add empty StringTable entry at index 0 - // StringTable index 0 is reserved as delimiter in the densenodes key/value list - // this line also ensures that there's always a valid StringTable - st->add_s(""); - - if (sort) { - std::multimap sortedbycount; - - m_id2id_map.resize(m_size+1); - - std::transform(m_strings.begin(), m_strings.end(), - std::inserter(sortedbycount, sortedbycount.begin()), - [](const std::pair& p) { - return std::pair(p.second, p.first); - }); - - string_id_type n = 0; - - for (const auto& mapping : sortedbycount) { - // add the string of the current item to the pbf StringTable - st->add_s(mapping.second); - - // store the mapping from the interim-id to the real id - m_id2id_map[mapping.first.interim_id] = ++n; - } - } else { - std::vector> sortedbyid; - sortedbyid.reserve(m_strings.size()); - - for (const auto& p : m_strings) { - sortedbyid.emplace_back(p.second.interim_id, p.first.c_str()); - } - - std::sort(sortedbyid.begin(), sortedbyid.end()); - for (const auto& mapping : sortedbyid) { - st->add_s(mapping.second); - } - } - } - - /** - * Map from an interim ID to a real string ID. - */ - string_id_type map_string_id(const string_id_type interim_id) const { - return m_id2id_map[interim_id]; - } - - template - string_id_type map_string_id(const T interim_id) const { - return map_string_id(static_cast_with_assert(interim_id)); - } - - /** - * Clear the stringtable, preparing for the next block. - */ - void clear() { - m_strings.clear(); - m_id2id_map.clear(); - m_size = 0; - } - - }; // class StringTable - - } // namespace detail - - } // namespace io - -} // namespace osmium - -#endif // OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP diff --git a/contrib/libosmium/osmium/io/detail/protobuf_tags.hpp b/contrib/libosmium/osmium/io/detail/protobuf_tags.hpp new file mode 100644 index 00000000..3f230876 --- /dev/null +++ b/contrib/libosmium/osmium/io/detail/protobuf_tags.hpp @@ -0,0 +1,170 @@ +#ifndef OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP +#define OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include + +namespace osmium { + + namespace io { + + namespace detail { + + // directly translated from + // https://github.com/scrosby/OSM-binary/blob/master/src/fileformat.proto + + namespace FileFormat { + + enum class Blob : protozero::pbf_tag_type { + optional_bytes_raw = 1, + optional_int32_raw_size = 2, + optional_bytes_zlib_data = 3, + optional_bytes_lzma_data = 4 + }; + + enum class BlobHeader : protozero::pbf_tag_type { + required_string_type = 1, + optional_bytes_indexdata = 2, + required_int32_datasize = 3 + }; + + } // namespace FileFormat + + // directly translated from + // https://github.com/scrosby/OSM-binary/blob/master/src/osmformat.proto + + namespace OSMFormat { + + enum class HeaderBlock : protozero::pbf_tag_type { + optional_HeaderBBox_bbox = 1, + repeated_string_required_features = 4, + repeated_string_optional_features = 5, + optional_string_writingprogram = 16, + optional_string_source = 17, + optional_int64_osmosis_replication_timestamp = 32, + optional_int64_osmosis_replication_sequence_number = 33, + optional_string_osmosis_replication_base_url = 34 + }; + + enum class HeaderBBox : protozero::pbf_tag_type { + required_sint64_left = 1, + required_sint64_right = 2, + required_sint64_top = 3, + required_sint64_bottom = 4 + }; + + enum class PrimitiveBlock : protozero::pbf_tag_type { + required_StringTable_stringtable = 1, + repeated_PrimitiveGroup_primitivegroup = 2, + optional_int32_granularity = 17, + optional_int32_date_granularity = 18, + optional_int64_lat_offset = 19, + optional_int64_lon_offset = 20 + }; + + enum class PrimitiveGroup : protozero::pbf_tag_type { + unknown = 0, + repeated_Node_nodes = 1, + optional_DenseNodes_dense = 2, + repeated_Way_ways = 3, + repeated_Relation_relations = 4, + repeated_ChangeSet_changesets = 5 + }; + + enum class StringTable : protozero::pbf_tag_type { + repeated_bytes_s = 1 + }; + + enum class Info : protozero::pbf_tag_type { + optional_int32_version = 1, + optional_int64_timestamp = 2, + optional_int64_changeset = 3, + optional_int32_uid = 4, + optional_uint32_user_sid = 5, + optional_bool_visible = 6 + }; + + enum class DenseInfo : protozero::pbf_tag_type { + packed_int32_version = 1, + packed_sint64_timestamp = 2, + packed_sint64_changeset = 3, + packed_sint32_uid = 4, + packed_sint32_user_sid = 5, + packed_bool_visible = 6 + }; + + enum class Node : protozero::pbf_tag_type { + required_sint64_id = 1, + packed_uint32_keys = 2, + packed_uint32_vals = 3, + optional_Info_info = 4, + required_sint64_lat = 8, + required_sint64_lon = 9 + }; + + enum class DenseNodes : protozero::pbf_tag_type { + packed_sint64_id = 1, + optional_DenseInfo_denseinfo = 5, + packed_sint64_lat = 8, + packed_sint64_lon = 9, + packed_int32_keys_vals = 10 + }; + + enum class Way : protozero::pbf_tag_type { + required_int64_id = 1, + packed_uint32_keys = 2, + packed_uint32_vals = 3, + optional_Info_info = 4, + packed_sint64_refs = 8 + }; + + enum class Relation : protozero::pbf_tag_type { + required_int64_id = 1, + packed_uint32_keys = 2, + packed_uint32_vals = 3, + optional_Info_info = 4, + packed_int32_roles_sid = 8, + packed_sint64_memids = 9, + packed_MemberType_types = 10 + }; + + } // namespace OSMFormat + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP diff --git a/contrib/libosmium/osmium/io/detail/string_table.hpp b/contrib/libosmium/osmium/io/detail/string_table.hpp new file mode 100644 index 00000000..ae9d5f0c --- /dev/null +++ b/contrib/libosmium/osmium/io/detail/string_table.hpp @@ -0,0 +1,250 @@ +#ifndef OSMIUM_IO_DETAIL_STRING_TABLE_HPP +#define OSMIUM_IO_DETAIL_STRING_TABLE_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + namespace io { + + namespace detail { + + /** + * class StringStore + * + * Storage of lots of strings (const char *). Memory is allocated in chunks. + * If a string is added and there is no space in the current chunk, a new + * chunk will be allocated. Strings added to the store must not be larger + * than the chunk size. + * + * All memory is released when the destructor is called. There is no other way + * to release all or part of the memory. + * + */ + class StringStore { + + size_t m_chunk_size; + + std::list m_chunks; + + void add_chunk() { + m_chunks.push_front(std::string()); + m_chunks.front().reserve(m_chunk_size); + } + + public: + + StringStore(size_t chunk_size) : + m_chunk_size(chunk_size), + m_chunks() { + add_chunk(); + } + + void clear() noexcept { + m_chunks.erase(std::next(m_chunks.begin()), m_chunks.end()); + m_chunks.front().clear(); + } + + /** + * Add a null terminated string to the store. This will + * automatically get more memory if we are out. + * Returns a pointer to the copy of the string we have + * allocated. + */ + const char* add(const char* string) { + size_t len = std::strlen(string) + 1; + + assert(len <= m_chunk_size); + + size_t chunk_len = m_chunks.front().size(); + if (chunk_len + len > m_chunks.front().capacity()) { + add_chunk(); + chunk_len = 0; + } + + m_chunks.front().append(string); + m_chunks.front().append(1, '\0'); + + return m_chunks.front().c_str() + chunk_len; + } + + class const_iterator : public std::iterator { + + typedef std::list::const_iterator it_type; + it_type m_it; + const it_type m_last; + const char* m_pos; + + public: + + const_iterator(it_type it, it_type last) : + m_it(it), + m_last(last), + m_pos(it == last ? nullptr : m_it->c_str()) { + } + + const_iterator& operator++() { + assert(m_it != m_last); + auto last_pos = m_it->c_str() + m_it->size(); + while (m_pos != last_pos && *m_pos) ++m_pos; + if (m_pos != last_pos) ++m_pos; + if (m_pos == last_pos) { + ++m_it; + if (m_it != m_last) { + m_pos = m_it->c_str(); + } else { + m_pos = nullptr; + } + } + return *this; + } + + const_iterator operator++(int) { + const_iterator tmp(*this); + operator++(); + return tmp; + } + + bool operator==(const const_iterator& rhs) const { + return m_it == rhs.m_it && m_pos == rhs.m_pos; + } + + bool operator!=(const const_iterator& rhs) const { + return !(*this == rhs); + } + + const char* operator*() const { + assert(m_it != m_last); + assert(m_pos != nullptr); + return m_pos; + } + + }; // class const_iterator + + const_iterator begin() const { + if (m_chunks.front().empty()) { + return end(); + } + return const_iterator(m_chunks.begin(), m_chunks.end()); + } + + const_iterator end() const { + return const_iterator(m_chunks.end(), m_chunks.end()); + } + + // These functions get you some idea how much memory was + // used. + int get_chunk_size() const noexcept { + return m_chunk_size; + } + + int get_chunk_count() const noexcept { + return m_chunks.size(); + } + + int get_used_bytes_in_last_chunk() const noexcept { + return m_chunks.front().size(); + } + + }; // class StringStore + + struct StrComp { + + bool operator()(const char* lhs, const char* rhs) const { + return strcmp(lhs, rhs) < 0; + } + + }; // struct StrComp + + class StringTable { + + StringStore m_strings; + std::map m_index; + size_t m_size; + + public: + + StringTable() : + m_strings(1024 * 1024), + m_index(), + m_size(0) { + m_strings.add(""); + } + + void clear() { + m_strings.clear(); + m_index.clear(); + m_size = 0; + m_strings.add(""); + } + + size_t size() const noexcept { + return m_size + 1; + } + + size_t add(const char* s) { + auto f = m_index.find(s); + if (f != m_index.end()) { + return f->second; + } + + const char* cs = m_strings.add(s); + m_index[cs] = ++m_size; + return m_size; + } + + StringStore::const_iterator begin() const { + return m_strings.begin(); + } + + StringStore::const_iterator end() const { + return m_strings.end(); + } + + }; // class StringTable + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_STRING_TABLE_HPP diff --git a/contrib/libosmium/osmium/io/detail/xml_input_format.hpp b/contrib/libosmium/osmium/io/detail/xml_input_format.hpp index 0ae624cb..b0f3da33 100644 --- a/contrib/libosmium/osmium/io/detail/xml_input_format.hpp +++ b/contrib/libosmium/osmium/io/detail/xml_input_format.hpp @@ -192,6 +192,8 @@ namespace osmium { std::atomic& m_done; + bool m_header_is_done; + /** * A C++ wrapper for the Expat parser that makes sure no memory is leaked. */ @@ -247,16 +249,25 @@ namespace osmium { T& m_data; std::promise& m_promise; + bool m_done; public: PromiseKeeper(T& data, std::promise& promise) : m_data(data), - m_promise(promise) { + m_promise(promise), + m_done(false) { + } + + void fullfill_promise() { + if (!m_done) { + m_promise.set_value(m_data); + m_done = true; + } } ~PromiseKeeper() { - m_promise.set_value(m_data); + fullfill_promise(); } }; // class PromiseKeeper @@ -280,7 +291,8 @@ namespace osmium { m_queue(queue), m_header_promise(header_promise), m_read_types(read_types), - m_done(done) { + m_done(done), + m_header_is_done(false) { } /** @@ -306,7 +318,8 @@ namespace osmium { m_queue(other.m_queue), m_header_promise(other.m_header_promise), m_read_types(other.m_read_types), - m_done(other.m_done) { + m_done(other.m_done), + m_header_is_done(other.m_header_is_done) { } XMLParser(XMLParser&&) = default; @@ -327,6 +340,9 @@ namespace osmium { last = data.empty(); try { parser(data, last); + if (m_header_is_done) { + promise_keeper.fullfill_promise(); + } } catch (ParserIsDone&) { return true; } catch (...) { @@ -420,6 +436,7 @@ namespace osmium { } void header_is_done() { + m_header_is_done = true; if (m_read_types == osmium::osm_entity_bits::nothing) { throw ParserIsDone(); } @@ -721,10 +738,15 @@ namespace osmium { namespace { +// we want the register_input_format() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_xml_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::xml, [](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) { return new osmium::io::detail::XMLInputFormat(file, read_which_entities, input_queue); }); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/contrib/libosmium/osmium/io/detail/xml_output_format.hpp b/contrib/libosmium/osmium/io/detail/xml_output_format.hpp index 65ba171b..2a381d5a 100644 --- a/contrib/libosmium/osmium/io/detail/xml_output_format.hpp +++ b/contrib/libosmium/osmium/io/detail/xml_output_format.hpp @@ -85,6 +85,9 @@ namespace osmium { case '\'': out += "'"; break; case '<': out += "<"; break; case '>': out += ">"; break; + case '\n': out += " "; break; + case '\r': out += " "; break; + case '\t': out += " "; break; default: out += *in; break; } } @@ -126,6 +129,7 @@ namespace osmium { operation m_last_op {operation::op_none}; + const bool m_add_metadata; const bool m_write_visible_flag; const bool m_write_change_ops; @@ -146,31 +150,33 @@ namespace osmium { void write_meta(const osmium::OSMObject& object) { oprintf(*m_out, " id=\"%" PRId64 "\"", object.id()); - if (object.version()) { - oprintf(*m_out, " version=\"%d\"", object.version()); - } + if (m_add_metadata) { + if (object.version()) { + oprintf(*m_out, " version=\"%d\"", object.version()); + } - if (object.timestamp()) { - *m_out += " timestamp=\""; - *m_out += object.timestamp().to_iso(); - *m_out += "\""; - } + if (object.timestamp()) { + *m_out += " timestamp=\""; + *m_out += object.timestamp().to_iso(); + *m_out += "\""; + } - if (!object.user_is_anonymous()) { - oprintf(*m_out, " uid=\"%d\" user=\"", object.uid()); - xml_string(*m_out, object.user()); - *m_out += "\""; - } + if (!object.user_is_anonymous()) { + oprintf(*m_out, " uid=\"%d\" user=\"", object.uid()); + xml_string(*m_out, object.user()); + *m_out += "\""; + } - if (object.changeset()) { - oprintf(*m_out, " changeset=\"%d\"", object.changeset()); - } + if (object.changeset()) { + oprintf(*m_out, " changeset=\"%d\"", object.changeset()); + } - if (m_write_visible_flag) { - if (object.visible()) { - *m_out += " visible=\"true\""; - } else { - *m_out += " visible=\"false\""; + if (m_write_visible_flag) { + if (object.visible()) { + *m_out += " visible=\"true\""; + } else { + *m_out += " visible=\"false\""; + } } } } @@ -224,9 +230,10 @@ namespace osmium { public: - explicit XMLOutputBlock(osmium::memory::Buffer&& buffer, bool write_visible_flag, bool write_change_ops) : + explicit XMLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool write_visible_flag, bool write_change_ops) : m_input_buffer(std::make_shared(std::move(buffer))), m_out(std::make_shared()), + m_add_metadata(add_metadata), m_write_visible_flag(write_visible_flag && !write_change_ops), m_write_change_ops(write_change_ops) { } @@ -392,12 +399,14 @@ namespace osmium { class XMLOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler { + bool m_add_metadata; bool m_write_visible_flag; public: XMLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : OutputFormat(file, output_queue), + m_add_metadata(file.get("add_metadata") != "false"), m_write_visible_flag(file.has_multiple_object_versions() || m_file.is_true("force_visible_flag")) { } @@ -408,7 +417,7 @@ namespace osmium { } void write_buffer(osmium::memory::Buffer&& buffer) override final { - m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_write_visible_flag, m_file.is_true("xml_change_format")})); + m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_add_metadata, m_write_visible_flag, m_file.is_true("xml_change_format")})); } void write_header(const osmium::io::Header& header) override final { @@ -468,10 +477,15 @@ namespace osmium { namespace { +// we want the register_output_format() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_xml_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::xml, [](const osmium::io::File& file, data_queue_type& output_queue) { return new osmium::io::detail::XMLOutputFormat(file, output_queue); }); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/contrib/libosmium/osmium/io/detail/zlib.hpp b/contrib/libosmium/osmium/io/detail/zlib.hpp index a402bf7e..fc0baf34 100644 --- a/contrib/libosmium/osmium/io/detail/zlib.hpp +++ b/contrib/libosmium/osmium/io/detail/zlib.hpp @@ -85,23 +85,24 @@ namespace osmium { * * @param input Compressed input data. * @param raw_size Size of uncompressed data. - * @returns Uncompressed data. + * @param output Uncompressed result data. + * @returns Pointer and size to incompressed data. */ - inline std::unique_ptr zlib_uncompress(const std::string& input, unsigned long raw_size) { - auto output = std::unique_ptr(new std::string(raw_size, '\0')); + inline std::pair zlib_uncompress_string(const char* input, unsigned long input_size, unsigned long raw_size, std::string& output) { + output.resize(raw_size); auto result = ::uncompress( - reinterpret_cast(const_cast(output->data())), + reinterpret_cast(&*output.begin()), &raw_size, - reinterpret_cast(input.data()), - osmium::static_cast_with_assert(input.size()) + reinterpret_cast(input), + input_size ); if (result != Z_OK) { throw std::runtime_error(std::string("failed to uncompress data: ") + zError(result)); } - return output; + return std::make_pair(output.data(), output.size()); } } // namespace detail diff --git a/contrib/libosmium/osmium/io/file.hpp b/contrib/libosmium/osmium/io/file.hpp index 5b6c02f1..3bbfacc6 100644 --- a/contrib/libosmium/osmium/io/file.hpp +++ b/contrib/libosmium/osmium/io/file.hpp @@ -97,7 +97,9 @@ namespace osmium { * of the file will be taken from the suffix. * An empty filename or "-" means stdin or stdout. * @param format File format as string. See the description of the - * parse_format() function for details. + * parse_format() function for details. If this is + * empty the format will be deduced from the suffix + * of the filename. */ explicit File(const std::string& filename = "", const std::string& format = "") : Options(), @@ -107,20 +109,19 @@ namespace osmium { m_format_string(format) { // stdin/stdout - if (filename == "" || filename == "-") { + if (m_filename == "-") { m_filename = ""; - default_settings_for_stdinout(); } - // filename is actually a URL + // if filename is a URL, default to XML format std::string protocol = m_filename.substr(0, m_filename.find_first_of(':')); if (protocol == "http" || protocol == "https") { - default_settings_for_url(); + m_file_format = file_format::xml; } - detect_format_from_suffix(m_filename); - - if (format != "") { + if (format.empty()) { + detect_format_from_suffix(m_filename); + } else { parse_format(format); } } @@ -140,9 +141,6 @@ namespace osmium { m_buffer(buffer), m_buffer_size(size), m_format_string(format) { - - default_settings_for_stdinout(); - if (format != "") { parse_format(format); } @@ -220,6 +218,20 @@ namespace osmium { } else if (suffixes.back() == "opl") { m_file_format = file_format::opl; suffixes.pop_back(); + } else if (suffixes.back() == "json") { + m_file_format = file_format::json; + suffixes.pop_back(); + } else if (suffixes.back() == "o5m") { + m_file_format = file_format::o5m; + suffixes.pop_back(); + } else if (suffixes.back() == "o5c") { + m_file_format = file_format::o5m; + m_has_multiple_object_versions = true; + set("o5c_change_format", true); + suffixes.pop_back(); + } else if (suffixes.back() == "debug") { + m_file_format = file_format::debug; + suffixes.pop_back(); } if (suffixes.empty()) return; @@ -240,8 +252,8 @@ namespace osmium { } /** - * Check file format etc. for consistency and throw exception if there - * is a problem. + * Check file format etc. for consistency and throw exception if + * there is a problem. * * @throws std::runtime_error */ @@ -265,36 +277,6 @@ namespace osmium { } } - /** - * Set default settings for type and encoding when the filename is - * empty or "-". If you want to have a different default setting - * override this in a subclass. - */ - void default_settings_for_stdinout() { - m_file_format = file_format::unknown; - m_file_compression = file_compression::none; - } - - /** - * Set default settings for type and encoding when the filename is - * a normal file. If you want to have a different default setting - * override this in a subclass. - */ - void default_settings_for_file() { - m_file_format = file_format::unknown; - m_file_compression = file_compression::none; - } - - /** - * Set default settings for type and encoding when the filename is a URL. - * If you want to have a different default setting override this in a - * subclass. - */ - void default_settings_for_url() { - m_file_format = file_format::xml; - m_file_compression = file_compression::none; - } - file_format format() const noexcept { return m_file_format; } diff --git a/contrib/libosmium/osmium/io/file_format.hpp b/contrib/libosmium/osmium/io/file_format.hpp index 1a63a5e4..edfb1ff9 100644 --- a/contrib/libosmium/osmium/io/file_format.hpp +++ b/contrib/libosmium/osmium/io/file_format.hpp @@ -44,7 +44,9 @@ namespace osmium { xml = 1, pbf = 2, opl = 3, - json = 4 + json = 4, + o5m = 5, + debug = 6 }; // avoid g++ false positive @@ -62,6 +64,10 @@ namespace osmium { return "OPL"; case file_format::json: return "JSON"; + case file_format::o5m: + return "O5M"; + case file_format::debug: + return "DEBUG"; } } #pragma GCC diagnostic pop diff --git a/contrib/libosmium/osmium/io/gzip_compression.hpp b/contrib/libosmium/osmium/io/gzip_compression.hpp index 27239770..204bfd52 100644 --- a/contrib/libosmium/osmium/io/gzip_compression.hpp +++ b/contrib/libosmium/osmium/io/gzip_compression.hpp @@ -231,11 +231,16 @@ namespace osmium { namespace { +// we want the register_compression() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_gzip_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip, [](int fd) { return new osmium::io::GzipCompressor(fd); }, [](int fd) { return new osmium::io::GzipDecompressor(fd); }, [](const char* buffer, size_t size) { return new osmium::io::GzipBufferDecompressor(buffer, size); } ); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/contrib/libosmium/osmium/io/pbf_input.hpp b/contrib/libosmium/osmium/io/pbf_input.hpp index 766153ed..d7f3787a 100644 --- a/contrib/libosmium/osmium/io/pbf_input.hpp +++ b/contrib/libosmium/osmium/io/pbf_input.hpp @@ -39,7 +39,6 @@ DEALINGS IN THE SOFTWARE. * Include this file if you want to read OSM PBF files. * * @attention If you include this file, you'll need to link with - * `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only), * `libz`, and enable multithreading. */ diff --git a/contrib/libosmium/osmium/io/pbf_output.hpp b/contrib/libosmium/osmium/io/pbf_output.hpp index 5f46ede2..dad1013d 100644 --- a/contrib/libosmium/osmium/io/pbf_output.hpp +++ b/contrib/libosmium/osmium/io/pbf_output.hpp @@ -39,7 +39,6 @@ DEALINGS IN THE SOFTWARE. * Include this file if you want to write OSM PBF files. * * @attention If you include this file, you'll need to link with - * `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only), * `libz`, and enable multithreading. */ diff --git a/contrib/libosmium/osmium/memory/buffer.hpp b/contrib/libosmium/osmium/memory/buffer.hpp index 7df848df..d800c685 100644 --- a/contrib/libosmium/osmium/memory/buffer.hpp +++ b/contrib/libosmium/osmium/memory/buffer.hpp @@ -37,7 +37,6 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include #include #include #include diff --git a/contrib/libosmium/osmium/memory/collection.hpp b/contrib/libosmium/osmium/memory/collection.hpp index 7deb88b4..5cf3cc6e 100644 --- a/contrib/libosmium/osmium/memory/collection.hpp +++ b/contrib/libosmium/osmium/memory/collection.hpp @@ -38,7 +38,6 @@ DEALINGS IN THE SOFTWARE. #include #include -#include namespace osmium { diff --git a/contrib/libosmium/osmium/memory/item.hpp b/contrib/libosmium/osmium/memory/item.hpp index 2679ca6f..dc544049 100644 --- a/contrib/libosmium/osmium/memory/item.hpp +++ b/contrib/libosmium/osmium/memory/item.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include diff --git a/contrib/libosmium/osmium/osm/changeset.hpp b/contrib/libosmium/osmium/osm/changeset.hpp index 20def70a..07bc0dd9 100644 --- a/contrib/libosmium/osmium/osm/changeset.hpp +++ b/contrib/libosmium/osmium/osm/changeset.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include diff --git a/contrib/libosmium/osmium/osm/crc.hpp b/contrib/libosmium/osmium/osm/crc.hpp new file mode 100644 index 00000000..eefa4a13 --- /dev/null +++ b/contrib/libosmium/osmium/osm/crc.hpp @@ -0,0 +1,223 @@ +#ifndef OSMIUM_OSM_CRC_HPP +#define OSMIUM_OSM_CRC_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + template + class CRC { + + static inline uint16_t byte_swap_16(uint16_t value) noexcept { +# if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap16(value); +# else + return (value >> 8) | (value << 8); +# endif + } + + static inline uint32_t byte_swap_32(uint32_t value) noexcept { +# if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap32(value); +# else + return (value >> 24) | + ((value >> 8) & 0x0000FF00) | + ((value << 8) & 0x00FF0000) | + (value << 24); +# endif + } + + static inline uint64_t byte_swap_64(uint64_t value) noexcept { +# if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(value); +# else + uint64_t val1 = byte_swap_32(value & 0xFFFFFFFF); + uint64_t val2 = byte_swap_32(value >> 32); + return (val1 << 32) & val2; +# endif + } + + TCRC m_crc; + + public: + + TCRC& operator()() { + return m_crc; + } + + const TCRC& operator()() const { + return m_crc; + } + + void update_bool(bool value) { + m_crc.process_byte(value); + } + + void update_int8(uint8_t value) { + m_crc.process_byte(value); + } + + void update_int16(uint16_t value) { +#if __BYTE_ORDER == __LITTLE_ENDIAN + m_crc.process_bytes(&value, sizeof(uint16_t)); +#else + uint16_t v = byte_swap_16(value); + m_crc.process_bytes(&v, sizeof(uint16_t)); +#endif + } + + void update_int32(uint32_t value) { +#if __BYTE_ORDER == __LITTLE_ENDIAN + m_crc.process_bytes(&value, sizeof(uint32_t)); +#else + uint32_t v = byte_swap_32(value); + m_crc.process_bytes(&v, sizeof(uint32_t)); +#endif + } + + void update_int64(uint64_t value) { +#if __BYTE_ORDER == __LITTLE_ENDIAN + m_crc.process_bytes(&value, sizeof(uint64_t)); +#else + uint64_t v = byte_swap_64(value); + m_crc.process_bytes(&v, sizeof(uint64_t)); +#endif + } + + void update_string(const char* str) { + while (*str) { + m_crc.process_byte(*str++); + } + } + + void update(const Timestamp& timestamp) { + update_int32(uint32_t(timestamp)); + } + + void update(const osmium::Location& location) { + update_int32(location.x()); + update_int32(location.y()); + } + + void update(const osmium::Box& box) { + update(box.bottom_left()); + update(box.top_right()); + } + + void update(const NodeRef& node_ref) { + update_int64(node_ref.ref()); + } + + void update(const NodeRefList& node_refs) { + for (const NodeRef& node_ref : node_refs) { + update(node_ref); + } + } + + void update(const TagList& tags) { + m_crc.process_bytes(tags.data(), tags.byte_size()); + } + + void update(const osmium::RelationMember& member) { + update_int64(member.ref()); + update_int16(uint16_t(member.type())); + update_string(member.role()); + } + + void update(const osmium::RelationMemberList& members) { + for (const RelationMember& member : members) { + update(member); + } + } + + void update(const osmium::OSMObject& object) { + update_int64(object.id()); + update_bool(object.visible()); + update_int32(object.version()); + update(object.timestamp()); + update_int32(object.uid()); + update_string(object.user()); + update(object.tags()); + } + + void update(const osmium::Node& node) { + update(static_cast(node)); + update(node.location()); + } + + void update(const osmium::Way& way) { + update(static_cast(way)); + update(way.nodes()); + } + + void update(const osmium::Relation& relation) { + update(static_cast(relation)); + update(relation.members()); + } + + void update(const osmium::Area& area) { + update(static_cast(area)); + for (auto it = area.cbegin(); it != area.cend(); ++it) { + if (it->type() == osmium::item_type::outer_ring || + it->type() == osmium::item_type::inner_ring) { + update(static_cast(*it)); + } + } + } + + void update(const osmium::Changeset& changeset) { + update_int64(changeset.id()); + update(changeset.created_at()); + update(changeset.closed_at()); + update(changeset.bounds()); + update_int32(changeset.num_changes()); + update_int32(changeset.uid()); + update_string(changeset.user()); + } + + }; // class CRC + +} // namespace osmium + +#endif // OSMIUM_OSM_CRC diff --git a/contrib/libosmium/osmium/osm/entity.hpp b/contrib/libosmium/osmium/osm/entity.hpp index 14861a2d..ce292c8d 100644 --- a/contrib/libosmium/osmium/osm/entity.hpp +++ b/contrib/libosmium/osmium/osm/entity.hpp @@ -35,6 +35,7 @@ DEALINGS IN THE SOFTWARE. #include #include +#include namespace osmium { diff --git a/contrib/libosmium/osmium/osm/node_ref.hpp b/contrib/libosmium/osmium/osm/node_ref.hpp index 76afa75f..72359cd0 100644 --- a/contrib/libosmium/osmium/osm/node_ref.hpp +++ b/contrib/libosmium/osmium/osm/node_ref.hpp @@ -33,11 +33,11 @@ DEALINGS IN THE SOFTWARE. */ +#include #include #include #include -#include #include #include diff --git a/contrib/libosmium/osmium/osm/timestamp.hpp b/contrib/libosmium/osmium/osm/timestamp.hpp index e4c6807a..390f0e74 100644 --- a/contrib/libosmium/osmium/osm/timestamp.hpp +++ b/contrib/libosmium/osmium/osm/timestamp.hpp @@ -39,10 +39,9 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include #include -#include +#include // IWYU pragma: keep namespace osmium { diff --git a/contrib/libosmium/osmium/thread/queue.hpp b/contrib/libosmium/osmium/thread/queue.hpp index 7fa64697..76ad9a02 100644 --- a/contrib/libosmium/osmium/thread/queue.hpp +++ b/contrib/libosmium/osmium/thread/queue.hpp @@ -41,9 +41,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include - -#include +#include // IWYU pragma: keep (for std::move) namespace osmium { diff --git a/contrib/libosmium/osmium/thread/util.hpp b/contrib/libosmium/osmium/thread/util.hpp index 62bb82ab..ca4f6dd5 100644 --- a/contrib/libosmium/osmium/thread/util.hpp +++ b/contrib/libosmium/osmium/thread/util.hpp @@ -58,7 +58,7 @@ namespace osmium { /** * Wait until the given future becomes ready. Will block if the future - * is not ready. Can be called more than once unless future.get(). + * is not ready. Can be called more than once unlike future.get(). */ template inline void wait_until_done(std::future& future) { diff --git a/contrib/libosmium/osmium/util/data_file.hpp b/contrib/libosmium/osmium/util/data_file.hpp index 31949885..22bf1910 100644 --- a/contrib/libosmium/osmium/util/data_file.hpp +++ b/contrib/libosmium/osmium/util/data_file.hpp @@ -34,7 +34,9 @@ DEALINGS IN THE SOFTWARE. */ #include +#include #include +#include #include #include diff --git a/contrib/libosmium/osmium/util/delta.hpp b/contrib/libosmium/osmium/util/delta.hpp index dd733ceb..0c77e524 100644 --- a/contrib/libosmium/osmium/util/delta.hpp +++ b/contrib/libosmium/osmium/util/delta.hpp @@ -33,6 +33,8 @@ DEALINGS IN THE SOFTWARE. */ +#include +#include #include namespace osmium { @@ -49,8 +51,8 @@ namespace osmium { public: - DeltaEncode() : - m_value(0) { + DeltaEncode(T value = 0) : + m_value(value) { } void clear() { @@ -90,6 +92,54 @@ namespace osmium { }; // class DeltaDecode + template + class DeltaEncodeIterator : public std::iterator { + + typedef TValue value_type; + + TBaseIterator m_it; + TBaseIterator m_end; + value_type m_delta; + DeltaEncode m_value; + TTransform m_trans; + + public: + + DeltaEncodeIterator(TBaseIterator first, TBaseIterator last, TTransform& trans) : + m_it(first), + m_end(last), + m_delta(m_trans(m_it)), + m_value(m_delta), + m_trans(trans) { + } + + DeltaEncodeIterator& operator++() { + if (m_it != m_end) { + m_delta = m_value.update(m_trans(++m_it)); + } + return *this; + } + + DeltaEncodeIterator operator++(int) { + DeltaEncodeIterator tmp(*this); + operator++(); + return tmp; + } + + value_type operator*() { + return m_delta; + } + + bool operator==(const DeltaEncodeIterator& rhs) const { + return m_it == rhs.m_it && m_end == rhs.m_end; + } + + bool operator!=(const DeltaEncodeIterator& rhs) const { + return !(*this == rhs); + } + + }; // class DeltaEncodeIterator + } // namespace util } // namespace osmium diff --git a/contrib/libosmium/osmium/util/endian.hpp b/contrib/libosmium/osmium/util/endian.hpp new file mode 100644 index 00000000..a5d91543 --- /dev/null +++ b/contrib/libosmium/osmium/util/endian.hpp @@ -0,0 +1,45 @@ +#ifndef OSMIUM_UTIL_ENDIAN_HPP +#define OSMIUM_UTIL_ENDIAN_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +// Windows is only available for little endian architectures +// http://stackoverflow.com/questions/6449468/can-i-safely-assume-that-windows-installations-will-always-be-little-endian +#if !defined(_WIN32) && !defined(__APPLE__) +# include +#else +# define __LITTLE_ENDIAN 1234 +# define __BYTE_ORDER __LITTLE_ENDIAN +#endif + +#endif // OSMIUM_UTIL_ENDIAN_HPP diff --git a/contrib/libosmium/osmium/util/file.hpp b/contrib/libosmium/osmium/util/file.hpp index afc595b0..461f4e64 100644 --- a/contrib/libosmium/osmium/util/file.hpp +++ b/contrib/libosmium/osmium/util/file.hpp @@ -34,6 +34,7 @@ DEALINGS IN THE SOFTWARE. */ #include +#include #include #include #include diff --git a/contrib/libosmium/osmium/util/memory_mapping.hpp b/contrib/libosmium/osmium/util/memory_mapping.hpp index 3880000e..e48aff28 100644 --- a/contrib/libosmium/osmium/util/memory_mapping.hpp +++ b/contrib/libosmium/osmium/util/memory_mapping.hpp @@ -66,7 +66,7 @@ namespace osmium { * @code * int fd = ::open(...); * { - * MemoryMapping mapping(1024, true, fd, offset); + * MemoryMapping mapping(1024, MemoryMapping::mapping_mode::write_shared, fd, offset); * // use mapping * } * ::close(fd); @@ -88,6 +88,15 @@ namespace osmium { */ class MemoryMapping { +public: + enum class mapping_mode { + readonly = 0, + write_private = 1, + write_shared = 2 + }; + +private: + /// The size of the mapping size_t m_size; @@ -97,8 +106,8 @@ namespace osmium { /// File handle we got the mapping from int m_fd; - /// Is the memory writable? - bool m_writable; + /// Mapping mode + mapping_mode m_mapping_mode; #ifdef _WIN32 HANDLE m_handle; @@ -160,15 +169,20 @@ namespace osmium { * created, otherwise a mapping based on the file descriptor will * be created. * - * @pre size > 0 or writable==true + * @pre size > 0 or mode == write_shared oder write_private * * @param size Size of the mapping in bytes - * @param writable Should the mapping be writable? + * @param mode Mapping mode: readonly, or writable (shared or private) * @param fd Open file descriptor of a file we want to map * @param offset Offset into the file where the mapping should start * @throws std::system_error if the mapping fails */ - MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0); + MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0); + + /// DEPRECATED: For backwards compatibility + MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0) : + MemoryMapping(size, writable ? mapping_mode::write_shared : mapping_mode::readonly, fd, offset) { + } /// You can not copy construct a MemoryMapping. MemoryMapping(const MemoryMapping&) = delete; @@ -249,7 +263,7 @@ namespace osmium { * Was this mapping created as a writable mapping? */ bool writable() const noexcept { - return m_writable; + return m_mapping_mode != mapping_mode::readonly; } /** @@ -282,7 +296,7 @@ namespace osmium { public: AnonymousMemoryMapping(size_t size) : - MemoryMapping(size) { + MemoryMapping(size, mapping_mode::write_private) { } #ifndef __linux__ @@ -312,13 +326,13 @@ namespace osmium { public: /** - * Create anonymous memory mapping of given size. + * Create anonymous typed memory mapping of given size. * * @param size Number of objects of type T to be mapped * @throws std::system_error if the mapping fails */ TypedMemoryMapping(size_t size) : - m_mapping(sizeof(T) * size) { + m_mapping(sizeof(T) * size, MemoryMapping::mapping_mode::write_private) { } /** @@ -326,13 +340,18 @@ namespace osmium { * contain at least `sizeof(T) * size` bytes! * * @param size Number of objects of type T to be mapped - * @param writable Should the mapping be writable? + * @param mode Mapping mode: readonly, or writable (shared or private) * @param fd Open file descriptor of a file we want to map * @param offset Offset into the file where the mapping should start * @throws std::system_error if the mapping fails */ + TypedMemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset = 0) : + m_mapping(sizeof(T) * size, mode, fd, sizeof(T) * offset) { + } + + /// DEPRECATED: For backwards compatibility TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset = 0) : - m_mapping(sizeof(T) * size, writable, fd, sizeof(T) * offset) { + m_mapping(sizeof(T) * size, writable ? MemoryMapping::mapping_mode::write_shared : MemoryMapping::mapping_mode::readonly, fd, sizeof(T) * offset) { } /// You can not copy construct a TypedMemoryMapping. @@ -499,26 +518,29 @@ inline void osmium::util::MemoryMapping::make_invalid() noexcept { #endif inline int osmium::util::MemoryMapping::get_protection() const noexcept { - if (m_writable) { - return PROT_READ | PROT_WRITE; + if (m_mapping_mode == mapping_mode::readonly) { + return PROT_READ; } - return PROT_READ; + return PROT_READ | PROT_WRITE; } inline int osmium::util::MemoryMapping::get_flags() const noexcept { if (m_fd == -1) { return MAP_PRIVATE | MAP_ANONYMOUS; } - return MAP_SHARED; + if (m_mapping_mode == mapping_mode::write_shared) { + return MAP_SHARED; + } + return MAP_PRIVATE; } -inline osmium::util::MemoryMapping::MemoryMapping(size_t size, bool writable, int fd, off_t offset) : +inline osmium::util::MemoryMapping::MemoryMapping(size_t size, mapping_mode mode, int fd, off_t offset) : m_size(initial_size(size)), m_offset(offset), m_fd(resize_fd(fd)), - m_writable(writable), + m_mapping_mode(mode), m_addr(::mmap(nullptr, m_size, get_protection(), get_flags(), m_fd, m_offset)) { - assert(writable || fd != -1); + assert(!(fd == -1 && mode == mapping_mode::readonly)); if (!is_valid()) { throw std::system_error(errno, std::system_category(), "mmap failed"); } @@ -528,18 +550,18 @@ inline osmium::util::MemoryMapping::MemoryMapping(MemoryMapping&& other) : m_size(other.m_size), m_offset(other.m_offset), m_fd(other.m_fd), - m_writable(other.m_writable), + m_mapping_mode(other.m_mapping_mode), m_addr(other.m_addr) { other.make_invalid(); } inline osmium::util::MemoryMapping& osmium::util::MemoryMapping::operator=(osmium::util::MemoryMapping&& other) { unmap(); - m_size = other.m_size; - m_offset = other.m_offset; - m_fd = other.m_fd; - m_writable = other.m_writable; - m_addr = other.m_addr; + m_size = other.m_size; + m_offset = other.m_offset; + m_fd = other.m_fd; + m_mapping_mode = other.m_mapping_mode; + m_addr = other.m_addr; other.make_invalid(); return *this; } @@ -604,20 +626,25 @@ namespace osmium { } // namespace osmium inline DWORD osmium::util::MemoryMapping::get_protection() const noexcept { - if (m_writable) { - return PAGE_READWRITE; + switch (m_mapping_mode) { + case mapping_mode::readonly: + return PAGE_READONLY; + case mapping_mode::write_private: + return PAGE_WRITECOPY; + case mapping_mode::write_shared: + return PAGE_READWRITE; } - return PAGE_READONLY; } inline DWORD osmium::util::MemoryMapping::get_flags() const noexcept { - if (m_fd == -1) { - return FILE_MAP_WRITE | FILE_MAP_COPY; + switch (m_mapping_mode) { + case mapping_mode::readonly: + return FILE_MAP_READ; + case mapping_mode::write_private: + return FILE_MAP_COPY; + case mapping_mode::write_shared: + return FILE_MAP_WRITE; } - if (m_writable) { - return FILE_MAP_WRITE; - } - return FILE_MAP_READ; } inline HANDLE osmium::util::MemoryMapping::get_handle() const noexcept { @@ -643,11 +670,11 @@ inline void osmium::util::MemoryMapping::make_invalid() noexcept { m_addr = nullptr; } -inline osmium::util::MemoryMapping::MemoryMapping(size_t size, bool writable, int fd, off_t offset) : +inline osmium::util::MemoryMapping::MemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset) : m_size(initial_size(size)), m_offset(offset), m_fd(resize_fd(fd)), - m_writable(writable), + m_mapping_mode(mode), m_handle(create_file_mapping()), m_addr(nullptr) { @@ -665,7 +692,7 @@ inline osmium::util::MemoryMapping::MemoryMapping(MemoryMapping&& other) : m_size(other.m_size), m_offset(other.m_offset), m_fd(other.m_fd), - m_writable(other.m_writable), + m_mapping_mode(other.m_mapping_mode), m_handle(std::move(other.m_handle)), m_addr(other.m_addr) { other.make_invalid(); @@ -674,12 +701,12 @@ inline osmium::util::MemoryMapping::MemoryMapping(MemoryMapping&& other) : inline osmium::util::MemoryMapping& osmium::util::MemoryMapping::operator=(osmium::util::MemoryMapping&& other) { unmap(); - m_size = other.m_size; - m_offset = other.m_offset; - m_fd = other.m_fd; - m_writable = other.m_writable; - m_handle = std::move(other.m_handle); - m_addr = other.m_addr; + m_size = other.m_size; + m_offset = other.m_offset; + m_fd = other.m_fd; + m_mapping_mode = other.m_mapping_mode; + m_handle = std::move(other.m_handle); + m_addr = other.m_addr; other.make_invalid(); other.m_handle = nullptr; return *this; diff --git a/contrib/libosmium/osmium/util/minmax.hpp b/contrib/libosmium/osmium/util/minmax.hpp index 05bd39bf..2eb601a2 100644 --- a/contrib/libosmium/osmium/util/minmax.hpp +++ b/contrib/libosmium/osmium/util/minmax.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include namespace osmium { @@ -116,8 +115,6 @@ namespace osmium { }; - - } // namespace osmium #endif // OSMIUM_UTIL_MINMAX_HPP diff --git a/contrib/libosmium/protozero/byteswap.hpp b/contrib/libosmium/protozero/byteswap.hpp new file mode 100644 index 00000000..d019c28c --- /dev/null +++ b/contrib/libosmium/protozero/byteswap.hpp @@ -0,0 +1,49 @@ +#ifndef PROTOZERO_BYTESWAP_HPP +#define PROTOZERO_BYTESWAP_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#include + +namespace protozero { + +template +inline void byteswap(const char* /*data*/, char* /*result*/) { + assert(false); +} + +template <> +inline void byteswap<1>(const char* data, char* result) { + result[0] = data[0]; +} + +template <> +inline void byteswap<4>(const char* data, char* result) { + result[3] = data[0]; + result[2] = data[1]; + result[1] = data[2]; + result[0] = data[3]; +} + +template <> +inline void byteswap<8>(const char* data, char* result) { + result[7] = data[0]; + result[6] = data[1]; + result[5] = data[2]; + result[4] = data[3]; + result[3] = data[4]; + result[2] = data[5]; + result[1] = data[6]; + result[0] = data[7]; +} + +} // end namespace protozero + +#endif // PROTOZERO_BYTESWAP_HPP diff --git a/contrib/libosmium/protozero/exception.hpp b/contrib/libosmium/protozero/exception.hpp new file mode 100644 index 00000000..1229f7dc --- /dev/null +++ b/contrib/libosmium/protozero/exception.hpp @@ -0,0 +1,68 @@ +#ifndef PROTOZERO_EXCEPTION_HPP +#define PROTOZERO_EXCEPTION_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file exception.hpp + * + * @brief Contains the exceptions used in the protozero library. + */ + +#include + +/** + * @brief All parts of the protozero header-only library are in this namespace. + */ +namespace protozero { + +/** + * All exceptions explicitly thrown by the functions of the protozero library + * derive from this exception. + */ +struct exception : std::exception { + /// Returns the explanatory string. + const char *what() const noexcept { return "pbf exception"; } +}; + +/** + * This exception is thrown when parsing a varint thats larger than allowed. + * This should never happen unless the data is corrupted. + */ +struct varint_too_long_exception : exception { + /// Returns the explanatory string. + const char *what() const noexcept { return "varint too long exception"; } +}; + +/** + * This exception is thrown when the wire type of a pdf field is unknown. + * This should never happen unless the data is corrupted. + */ +struct unknown_pbf_wire_type_exception : exception { + /// Returns the explanatory string. + const char *what() const noexcept { return "unknown pbf field type exception"; } +}; + +/** + * This exception is thrown when we are trying to read a field and there + * are not enough bytes left in the buffer to read it. Almost all functions + * of the pbf_reader class can throw this exception. + * + * This should never happen unless the data is corrupted or you have + * initialized the pbf_reader object with incomplete data. + */ +struct end_of_buffer_exception : exception { + /// Returns the explanatory string. + const char *what() const noexcept { return "end of buffer exception"; } +}; + +} // end namespace protozero + +#endif // PROTOZERO_EXCEPTION_HPP diff --git a/contrib/libosmium/protozero/pbf_builder.hpp b/contrib/libosmium/protozero/pbf_builder.hpp new file mode 100644 index 00000000..d49a7ba5 --- /dev/null +++ b/contrib/libosmium/protozero/pbf_builder.hpp @@ -0,0 +1,111 @@ +#ifndef PROTOZERO_PBF_BUILDER_HPP +#define PROTOZERO_PBF_BUILDER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#include + +#include +#include + +namespace protozero { + +template +class pbf_builder : public pbf_writer { + + static_assert(std::is_same::type>::value, "T must be enum with underlying type protozero::pbf_tag_type"); + +public: + + using enum_type = T; + + pbf_builder(std::string& data) noexcept : + pbf_writer(data) { + } + + template + pbf_builder(pbf_writer& parent_writer, P tag) noexcept : + pbf_writer(parent_writer, pbf_tag_type(tag)) { + } + +#define PROTOZERO_WRITER_WRAP_ADD_SCALAR(name, type) \ + inline void add_##name(T tag, type value) { \ + pbf_writer::add_##name(pbf_tag_type(tag), value); \ + } + + PROTOZERO_WRITER_WRAP_ADD_SCALAR(bool, bool) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(enum, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(int32, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint32, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint32, uint32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(int64, int64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint64, int64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint64, uint64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed32, uint32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed32, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed64, uint64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed64, int64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(float, float) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(double, double) + + inline void add_bytes(T tag, const char* value, size_t size) { + pbf_writer::add_bytes(pbf_tag_type(tag), value, size); + } + + inline void add_bytes(T tag, const std::string& value) { + pbf_writer::add_bytes(pbf_tag_type(tag), value); + } + + inline void add_string(T tag, const char* value, size_t size) { + pbf_writer::add_string(pbf_tag_type(tag), value, size); + } + + inline void add_string(T tag, const std::string& value) { + pbf_writer::add_string(pbf_tag_type(tag), value); + } + + inline void add_string(T tag, const char* value) { + pbf_writer::add_string(pbf_tag_type(tag), value); + } + + inline void add_message(T tag, const char* value, size_t size) { + pbf_writer::add_message(pbf_tag_type(tag), value, size); + } + + inline void add_message(T tag, const std::string& value) { + pbf_writer::add_message(pbf_tag_type(tag), value); + } + +#define PROTOZERO_WRITER_WRAP_ADD_PACKED(name) \ + template \ + inline void add_packed_##name(T tag, InputIterator first, InputIterator last) { \ + pbf_writer::add_packed_##name(pbf_tag_type(tag), first, last); \ + } + + PROTOZERO_WRITER_WRAP_ADD_PACKED(bool) + PROTOZERO_WRITER_WRAP_ADD_PACKED(enum) + PROTOZERO_WRITER_WRAP_ADD_PACKED(int32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sint32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(uint32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(int64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sint64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(uint64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(float) + PROTOZERO_WRITER_WRAP_ADD_PACKED(double) + +}; + +} // end namespace protozero + +#endif // PROTOZERO_PBF_BUILDER_HPP diff --git a/contrib/libosmium/protozero/pbf_message.hpp b/contrib/libosmium/protozero/pbf_message.hpp new file mode 100644 index 00000000..af29a00f --- /dev/null +++ b/contrib/libosmium/protozero/pbf_message.hpp @@ -0,0 +1,50 @@ +#ifndef PROTOZERO_PBF_MESSAGE_HPP +#define PROTOZERO_PBF_MESSAGE_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#include + +#include +#include + +namespace protozero { + +template +class pbf_message : public pbf_reader { + + static_assert(std::is_same::type>::value, "T must be enum with underlying type protozero::pbf_tag_type"); + +public: + + using enum_type = T; + + template + pbf_message(Args&&... args) noexcept : + pbf_reader(std::forward(args)...) { + } + + inline bool next() { + return pbf_reader::next(); + } + + inline bool next(T tag) { + return pbf_reader::next(pbf_tag_type(tag)); + } + + inline T tag() const noexcept { + return T(pbf_reader::tag()); + } + +}; + +} // end namespace protozero + +#endif // PROTOZERO_PBF_MESSAGE_HPP diff --git a/contrib/libosmium/protozero/pbf_reader.hpp b/contrib/libosmium/protozero/pbf_reader.hpp new file mode 100644 index 00000000..1c5ed0d7 --- /dev/null +++ b/contrib/libosmium/protozero/pbf_reader.hpp @@ -0,0 +1,1059 @@ +#ifndef PROTOZERO_PBF_READER_HPP +#define PROTOZERO_PBF_READER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_reader.hpp + * + * @brief Contains the pbf_reader class. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#if __BYTE_ORDER != __LITTLE_ENDIAN +# include +#endif + +/// Wrapper for assert() used for testing +#ifndef protozero_assert +# define protozero_assert(x) assert(x) +#endif + +namespace protozero { + +/** + * This class represents a protobuf message. Either a top-level message or + * a nested sub-message. Top-level messages can be created from any buffer + * with a pointer and length: + * + * @code + * std::string buffer; + * // fill buffer... + * pbf_reader message(buffer.data(), buffer.size()); + * @endcode + * + * Sub-messages are created using get_message(): + * + * @code + * pbf_reader message(...); + * message.next(); + * pbf_reader submessage = message.get_message(); + * @endcode + * + * All methods of the pbf_reader class except get_bytes() and get_string() + * provide the strong exception guarantee, ie they either succeed or do not + * change the pbf_reader object they are called on. Use the get_data() method + * instead of get_bytes() or get_string(), if you need this guarantee. + */ +class pbf_reader { + + // A pointer to the next unread data. + const char *m_data = nullptr; + + // A pointer to one past the end of data. + const char *m_end = nullptr; + + // The wire type of the current field. + pbf_wire_type m_wire_type = pbf_wire_type::unknown; + + // The tag of the current field. + pbf_tag_type m_tag = 0; + + template + inline T get_fixed() { + T result; + skip_bytes(sizeof(T)); +#if __BYTE_ORDER == __LITTLE_ENDIAN + memcpy(&result, m_data - sizeof(T), sizeof(T)); +#else + byteswap(m_data - sizeof(T), reinterpret_cast(&result)); +#endif + return result; + } + +#if __BYTE_ORDER == __LITTLE_ENDIAN + template + inline std::pair packed_fixed() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + protozero_assert(len % sizeof(T) == 0); + return std::make_pair(reinterpret_cast(m_data-len), reinterpret_cast(m_data)); + } + +#else + + template + class const_fixed_iterator : public std::iterator { + + const char* m_data; + const char* m_end; + + public: + + const_fixed_iterator() noexcept : + m_data(nullptr), + m_end(nullptr) { + } + + const_fixed_iterator(const char *data, const char* end) noexcept : + m_data(data), + m_end(end) { + } + + const_fixed_iterator(const const_fixed_iterator&) noexcept = default; + const_fixed_iterator(const_fixed_iterator&&) noexcept = default; + + const_fixed_iterator& operator=(const const_fixed_iterator&) noexcept = default; + const_fixed_iterator& operator=(const_fixed_iterator&&) noexcept = default; + + ~const_fixed_iterator() noexcept = default; + + T operator*() { + T result; + byteswap(m_data, reinterpret_cast(&result)); + return result; + } + + const_fixed_iterator& operator++() { + m_data += sizeof(T); + return *this; + } + + const_fixed_iterator operator++(int) { + const const_fixed_iterator tmp(*this); + ++(*this); + return tmp; + } + + bool operator==(const const_fixed_iterator& rhs) const noexcept { + return m_data == rhs.m_data && m_end == rhs.m_end; + } + + bool operator!=(const const_fixed_iterator& rhs) const noexcept { + return !(*this == rhs); + } + + }; // class const_fixed_iterator + + template + inline std::pair, const_fixed_iterator> packed_fixed() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + protozero_assert(len % sizeof(T) == 0); + return std::make_pair(const_fixed_iterator(m_data-len, m_data), + const_fixed_iterator(m_data, m_data)); + } +#endif + + template inline T get_varint(); + template inline T get_svarint(); + + inline pbf_length_type get_length() { return get_varint(); } + + inline void skip_bytes(pbf_length_type len); + + inline pbf_length_type get_len_and_skip(); + +public: + + /** + * Construct a pbf_reader message from a data pointer and a length. The pointer + * will be stored inside the pbf_reader object, no data is copied. So you must + * make sure the buffer stays valid as long as the pbf_reader object is used. + * + * The buffer must contain a complete protobuf message. + * + * @post There is no current field. + */ + inline pbf_reader(const char *data, size_t length) noexcept; + + /** + * Construct a pbf_reader message from a data pointer and a length. The pointer + * will be stored inside the pbf_reader object, no data is copied. So you must + * make sure the buffer stays valid as long as the pbf_reader object is used. + * + * The buffer must contain a complete protobuf message. + * + * @post There is no current field. + */ + inline pbf_reader(std::pair data) noexcept; + + /** + * Construct a pbf_reader message from a std::string. A pointer to the string + * internals will be stored inside the pbf_reader object, no data is copied. + * So you must make sure the string is unchanged as long as the pbf_reader + * object is used. + * + * The string must contain a complete protobuf message. + * + * @post There is no current field. + */ + inline pbf_reader(const std::string& data) noexcept; + + /** + * pbf_reader can be default constructed and behaves like it has an empty + * buffer. + */ + inline pbf_reader() noexcept = default; + + /// pbf_reader messages can be copied trivially. + inline pbf_reader(const pbf_reader&) noexcept = default; + + /// pbf_reader messages can be moved trivially. + inline pbf_reader(pbf_reader&&) noexcept = default; + + /// pbf_reader messages can be copied trivially. + inline pbf_reader& operator=(const pbf_reader& other) noexcept = default; + + /// pbf_reader messages can be moved trivially. + inline pbf_reader& operator=(pbf_reader&& other) noexcept = default; + + inline ~pbf_reader() = default; + + /** + * In a boolean context the pbf_reader class evaluates to `true` if there are + * still fields available and to `false` if the last field has been read. + */ + inline operator bool() const noexcept; + + /** + * Return the length in bytes of the current message. If you have + * already called next() and/or any of the get_*() functions, this will + * return the remaining length. + * + * This can, for instance, be used to estimate the space needed for a + * buffer. Of course you have to know reasonably well what data to expect + * and how it is encoded for this number to have any meaning. + */ + size_t length() const noexcept { + return size_t(m_end - m_data); + } + + /** + * Set next field in the message as the current field. This is usually + * called in a while loop: + * + * @code + * pbf_reader message(...); + * while (message.next()) { + * // handle field + * } + * @endcode + * + * @returns `true` if there is a next field, `false` if not. + * @pre There must be no current field. + * @post If it returns `true` there is a current field now. + */ + inline bool next(); + + /** + * Set next field with given tag in the message as the current field. + * Fields with other tags are skipped. This is usually called in a while + * loop for repeated fields: + * + * @code + * pbf_reader message(...); + * while (message.next(17)) { + * // handle field + * } + * @endcode + * + * or you can call it just once to get the one field with this tag: + * + * @code + * pbf_reader message(...); + * if (message.next(17)) { + * // handle field + * } + * @endcode + * + * @returns `true` if there is a next field with this tag. + * @pre There must be no current field. + * @post If it returns `true` there is a current field now with the given tag. + */ + inline bool next(pbf_tag_type tag); + + /** + * The tag of the current field. The tag is the field number from the + * description in the .proto file. + * + * Call next() before calling this function to set the current field. + * + * @returns tag of the current field. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + inline pbf_tag_type tag() const noexcept; + + /** + * Get the wire type of the current field. The wire types are: + * + * * 0 - varint + * * 1 - 64 bit + * * 2 - length-delimited + * * 5 - 32 bit + * + * All other types are illegal. + * + * Call next() before calling this function to set the current field. + * + * @returns wire type of the current field. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + inline pbf_wire_type wire_type() const noexcept; + + /** + * Check the wire type of the current field. + * + * @returns `true` if the current field has the given wire type. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + inline bool has_wire_type(pbf_wire_type type) const noexcept; + + /** + * Consume the current field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @post The current field was consumed and there is no current field now. + */ + inline void skip(); + + ///@{ + /** + * @name Scalar field accessor functions + */ + + /** + * Consume and return value of current "bool" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bool". + * @post The current field was consumed and there is no current field now. + */ + inline bool get_bool(); + + /** + * Consume and return value of current "enum" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "enum". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_enum() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "int32" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "int32". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_int32() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "sint32" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sint32". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_sint32() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_svarint(); + } + + /** + * Consume and return value of current "uint32" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "uint32". + * @post The current field was consumed and there is no current field now. + */ + inline uint32_t get_uint32() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "int64" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "int64". + * @post The current field was consumed and there is no current field now. + */ + inline int64_t get_int64() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "sint64" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sint64". + * @post The current field was consumed and there is no current field now. + */ + inline int64_t get_sint64() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_svarint(); + } + + /** + * Consume and return value of current "uint64" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "uint64". + * @post The current field was consumed and there is no current field now. + */ + inline uint64_t get_uint64() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "fixed32" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "fixed32". + * @post The current field was consumed and there is no current field now. + */ + inline uint32_t get_fixed32(); + + /** + * Consume and return value of current "sfixed32" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sfixed32". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_sfixed32(); + + /** + * Consume and return value of current "fixed64" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "fixed64". + * @post The current field was consumed and there is no current field now. + */ + inline uint64_t get_fixed64(); + + /** + * Consume and return value of current "sfixed64" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sfixed64". + * @post The current field was consumed and there is no current field now. + */ + inline int64_t get_sfixed64(); + + /** + * Consume and return value of current "float" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "float". + * @post The current field was consumed and there is no current field now. + */ + inline float get_float(); + + /** + * Consume and return value of current "double" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "double". + * @post The current field was consumed and there is no current field now. + */ + inline double get_double(); + + /** + * Consume and return value of current "bytes" or "string" field. + * + * @returns A pair with a pointer to the data and the length of the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bytes" or "string". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_data(); + + /** + * Consume and return value of current "bytes" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bytes". + * @post The current field was consumed and there is no current field now. + */ + inline std::string get_bytes(); + + /** + * Consume and return value of current "string" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "string". + * @post The current field was consumed and there is no current field now. + */ + inline std::string get_string(); + + /** + * Consume and return value of current "message" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "message". + * @post The current field was consumed and there is no current field now. + */ + inline pbf_reader get_message() { + return pbf_reader(get_data()); + } + + ///@} + +private: + + template + class const_varint_iterator : public std::iterator { + + protected: + + const char* m_data; + const char* m_end; + + public: + + const_varint_iterator() noexcept : + m_data(nullptr), + m_end(nullptr) { + } + + const_varint_iterator(const char *data, const char* end) noexcept : + m_data(data), + m_end(end) { + } + + const_varint_iterator(const const_varint_iterator&) noexcept = default; + const_varint_iterator(const_varint_iterator&&) noexcept = default; + + const_varint_iterator& operator=(const const_varint_iterator&) noexcept = default; + const_varint_iterator& operator=(const_varint_iterator&&) noexcept = default; + + ~const_varint_iterator() noexcept = default; + + T operator*() { + const char* d = m_data; // will be thrown away + return static_cast(decode_varint(&d, m_end)); + } + + const_varint_iterator& operator++() { + // Ignore the result, we call decode_varint() just for the + // side-effect of updating m_data. + decode_varint(&m_data, m_end); + return *this; + } + + const_varint_iterator operator++(int) { + const const_varint_iterator tmp(*this); + ++(*this); + return tmp; + } + + bool operator==(const const_varint_iterator& rhs) const noexcept { + return m_data == rhs.m_data && m_end == rhs.m_end; + } + + bool operator!=(const const_varint_iterator& rhs) const noexcept { + return !(*this == rhs); + } + + }; // class const_varint_iterator + + template + class const_svarint_iterator : public const_varint_iterator { + + public: + + const_svarint_iterator() noexcept : + const_varint_iterator() { + } + + const_svarint_iterator(const char *data, const char* end) noexcept : + const_varint_iterator(data, end) { + } + + const_svarint_iterator(const const_svarint_iterator&) = default; + const_svarint_iterator(const_svarint_iterator&&) = default; + + const_svarint_iterator& operator=(const const_svarint_iterator&) = default; + const_svarint_iterator& operator=(const_svarint_iterator&&) = default; + + ~const_svarint_iterator() = default; + + T operator*() { + const char* d = this->m_data; // will be thrown away + return static_cast(decode_zigzag64(decode_varint(&d, this->m_end))); + } + + const_svarint_iterator& operator++() { + // Ignore the result, we call decode_varint() just for the + // side-effect of updating m_data. + decode_varint(&this->m_data, this->m_end); + return *this; + } + + const_svarint_iterator operator++(int) { + const const_svarint_iterator tmp(*this); + ++(*this); + return tmp; + } + + }; // class const_svarint_iterator + +public: + + /// Forward iterator for iterating over bool (int32 varint) values. + typedef const_varint_iterator< int32_t> const_bool_iterator; + + /// Forward iterator for iterating over enum (int32 varint) values. + typedef const_varint_iterator< int32_t> const_enum_iterator; + + /// Forward iterator for iterating over int32 (varint) values. + typedef const_varint_iterator< int32_t> const_int32_iterator; + + /// Forward iterator for iterating over sint32 (varint) values. + typedef const_svarint_iterator const_sint32_iterator; + + /// Forward iterator for iterating over uint32 (varint) values. + typedef const_varint_iterator const_uint32_iterator; + + /// Forward iterator for iterating over int64 (varint) values. + typedef const_varint_iterator< int64_t> const_int64_iterator; + + /// Forward iterator for iterating over sint64 (varint) values. + typedef const_svarint_iterator const_sint64_iterator; + + /// Forward iterator for iterating over uint64 (varint) values. + typedef const_varint_iterator const_uint64_iterator; + + ///@{ + /** + * @name Repeated packed field accessor functions + */ + + /** + * Consume current "repeated packed bool" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed bool". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_bool(); + + /** + * Consume current "repeated packed enum" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed enum". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_enum(); + + /** + * Consume current "repeated packed int32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed int32". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_int32(); + + /** + * Consume current "repeated packed sint32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sint32". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_sint32(); + + /** + * Consume current "repeated packed uint32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed uint32". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_uint32(); + + /** + * Consume current "repeated packed int64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed int64". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_int64(); + + /** + * Consume current "repeated packed sint64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sint64". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_sint64(); + + /** + * Consume current "repeated packed uint64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed uint64". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_uint64(); + + /** + * Consume current "repeated packed fixed32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed fixed32". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_fixed32() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed sfixed32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sfixed32". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_sfixed32() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed fixed64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed fixed64". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_fixed64() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed sfixed64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sfixed64". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_sfixed64() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed float" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed float". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_float() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed double" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed double". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_double() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + ///@} + +}; // class pbf_reader + +pbf_reader::pbf_reader(const char *data, size_t length) noexcept + : m_data(data), + m_end(data + length), + m_wire_type(pbf_wire_type::unknown), + m_tag(0) { +} + +pbf_reader::pbf_reader(std::pair data) noexcept + : m_data(data.first), + m_end(data.first + data.second), + m_wire_type(pbf_wire_type::unknown), + m_tag(0) { +} + +pbf_reader::pbf_reader(const std::string& data) noexcept + : m_data(data.data()), + m_end(data.data() + data.size()), + m_wire_type(pbf_wire_type::unknown), + m_tag(0) { +} + +pbf_reader::operator bool() const noexcept { + return m_data < m_end; +} + +bool pbf_reader::next() { + if (m_data == m_end) { + return false; + } + + auto value = get_varint(); + m_tag = value >> 3; + + // tags 0 and 19000 to 19999 are not allowed as per + // https://developers.google.com/protocol-buffers/docs/proto + protozero_assert(((m_tag > 0 && m_tag < 19000) || (m_tag > 19999 && m_tag <= ((1 << 29) - 1))) && "tag out of range"); + + m_wire_type = pbf_wire_type(value & 0x07); +// XXX do we want this check? or should it throw an exception? +// protozero_assert((m_wire_type <=2 || m_wire_type == 5) && "illegal wire type"); + return true; +} + +bool pbf_reader::next(pbf_tag_type requested_tag) { + while (next()) { + if (m_tag == requested_tag) { + return true; + } else { + skip(); + } + } + return false; +} + +pbf_tag_type pbf_reader::tag() const noexcept { + return m_tag; +} + +pbf_wire_type pbf_reader::wire_type() const noexcept { + return m_wire_type; +} + +bool pbf_reader::has_wire_type(pbf_wire_type type) const noexcept { + return wire_type() == type; +} + +void pbf_reader::skip_bytes(pbf_length_type len) { + if (m_data + len > m_end) { + throw end_of_buffer_exception(); + } + m_data += len; + +// In debug builds reset the tag to zero so that we can detect (some) +// wrong code. +#ifndef NDEBUG + m_tag = 0; +#endif +} + +void pbf_reader::skip() { + protozero_assert(tag() != 0 && "call next() before calling skip()"); + switch (wire_type()) { + case pbf_wire_type::varint: + (void)get_uint32(); // called for the side-effect of skipping value + break; + case pbf_wire_type::fixed64: + skip_bytes(8); + break; + case pbf_wire_type::length_delimited: + skip_bytes(get_length()); + break; + case pbf_wire_type::fixed32: + skip_bytes(4); + break; + default: + throw unknown_pbf_wire_type_exception(); + } +} + +pbf_length_type pbf_reader::get_len_and_skip() { + auto len = get_length(); + skip_bytes(len); + return len; +} + +template +T pbf_reader::get_varint() { + return static_cast(decode_varint(&m_data, m_end)); +} + +template +T pbf_reader::get_svarint() { + protozero_assert((has_wire_type(pbf_wire_type::varint) || has_wire_type(pbf_wire_type::length_delimited)) && "not a varint"); + return static_cast(decode_zigzag64(decode_varint(&m_data, m_end))); +} + +uint32_t pbf_reader::get_fixed32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed"); + return get_fixed(); +} + +int32_t pbf_reader::get_sfixed32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed"); + return get_fixed(); +} + +uint64_t pbf_reader::get_fixed64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed"); + return get_fixed(); +} + +int64_t pbf_reader::get_sfixed64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed"); + return get_fixed(); +} + +float pbf_reader::get_float() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed"); + return get_fixed(); +} + +double pbf_reader::get_double() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed"); + return get_fixed(); +} + +bool pbf_reader::get_bool() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + protozero_assert((*m_data & 0x80) == 0 && "not a 1 byte varint"); + skip_bytes(1); + return m_data[-1] != 0; // -1 okay because we incremented m_data the line before +} + +std::pair pbf_reader::get_data() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::length_delimited) && "not of type string, bytes or message"); + auto len = get_len_and_skip(); + return std::make_pair(m_data-len, len); +} + +std::string pbf_reader::get_bytes() { + auto d = get_data(); + return std::string(d.first, d.second); +} + +std::string pbf_reader::get_string() { + return get_bytes(); +} + +std::pair pbf_reader::get_packed_bool() { + return get_packed_int32(); +} + +std::pair pbf_reader::get_packed_enum() { + return get_packed_int32(); +} + +std::pair pbf_reader::get_packed_int32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_int32_iterator(m_data-len, m_data), + pbf_reader::const_int32_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_uint32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_uint32_iterator(m_data-len, m_data), + pbf_reader::const_uint32_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_sint32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_sint32_iterator(m_data-len, m_data), + pbf_reader::const_sint32_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_int64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_int64_iterator(m_data-len, m_data), + pbf_reader::const_int64_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_uint64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_uint64_iterator(m_data-len, m_data), + pbf_reader::const_uint64_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_sint64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_sint64_iterator(m_data-len, m_data), + pbf_reader::const_sint64_iterator(m_data, m_data)); +} + +} // end namespace protozero + +#endif // PROTOZERO_PBF_READER_HPP diff --git a/contrib/libosmium/protozero/pbf_types.hpp b/contrib/libosmium/protozero/pbf_types.hpp new file mode 100644 index 00000000..9f38584f --- /dev/null +++ b/contrib/libosmium/protozero/pbf_types.hpp @@ -0,0 +1,49 @@ +#ifndef PROTOZERO_PBF_TYPES_HPP +#define PROTOZERO_PBF_TYPES_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_types.hpp + * + * @brief Contains the declaration of low-level types used in the pbf format. + */ + +#include + +namespace protozero { + + /** + * The type used for field tags (field numbers). + */ + typedef uint32_t pbf_tag_type; + + /** + * The type used to encode type information. + * See the table on + * https://developers.google.com/protocol-buffers/docs/encoding + */ + enum class pbf_wire_type : uint32_t { + varint = 0, // int32/64, uint32/64, sint32/64, bool, enum + fixed64 = 1, // fixed64, sfixed64, double + length_delimited = 2, // string, bytes, embedded messages, + // packed repeated fields + fixed32 = 5, // fixed32, sfixed32, float + unknown = 99 // used for default setting in this library + }; + + /** + * The type used for length values, such as the length of a field. + */ + typedef uint32_t pbf_length_type; + +} // end namespace protozero + +#endif // PROTOZERO_PBF_TYPES_HPP diff --git a/contrib/libosmium/protozero/pbf_writer.hpp b/contrib/libosmium/protozero/pbf_writer.hpp new file mode 100644 index 00000000..53cbfdf1 --- /dev/null +++ b/contrib/libosmium/protozero/pbf_writer.hpp @@ -0,0 +1,664 @@ +#ifndef PROTOZERO_PBF_WRITER_HPP +#define PROTOZERO_PBF_WRITER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_writer.hpp + * + * @brief Contains the pbf_writer class. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#if __BYTE_ORDER != __LITTLE_ENDIAN +# include +#endif + +/// Wrapper for assert() used for testing +#ifndef protozero_assert +# define protozero_assert(x) assert(x) +#endif + +namespace protozero { + +/** + * The pbf_writer is used to write PBF formatted messages into a buffer. + * + * Almost all methods in this class can throw an std::bad_alloc exception if + * the std::string used as a buffer wants to resize. + */ +class pbf_writer { + + std::string* m_data; + pbf_writer* m_parent_writer; + size_t m_pos = 0; + + inline void add_varint(uint64_t value) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); + protozero_assert(m_data); + write_varint(std::back_inserter(*m_data), value); + } + + inline void add_field(pbf_tag_type tag, pbf_wire_type type) { + protozero_assert(((tag > 0 && tag < 19000) || (tag > 19999 && tag <= ((1 << 29) - 1))) && "tag out of range"); + uint32_t b = (tag << 3) | uint32_t(type); + add_varint(b); + } + + inline void add_tagged_varint(pbf_tag_type tag, uint64_t value) { + add_field(tag, pbf_wire_type::varint); + add_varint(value); + } + + template + inline void add_fixed(T value) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); + protozero_assert(m_data); +#if __BYTE_ORDER == __LITTLE_ENDIAN + m_data->append(reinterpret_cast(&value), sizeof(T)); +#else + auto size = m_data->size(); + m_data->resize(size + sizeof(T)); + byteswap(reinterpret_cast(&value), const_cast(m_data->data() + size)); +#endif + } + + template + inline void add_packed_fixed(pbf_tag_type tag, It first, It last, std::input_iterator_tag) { + if (first == last) { + return; + } + + pbf_writer sw(*this, tag); + + while (first != last) { + sw.add_fixed(*first++); + } + } + + template + inline void add_packed_fixed(pbf_tag_type tag, It first, It last, std::forward_iterator_tag) { + if (first == last) { + return; + } + + add_length_varint(tag, sizeof(T) * pbf_length_type(std::distance(first, last))); + + while (first != last) { + add_fixed(*first++); + } + } + + template + inline void add_packed_varint(pbf_tag_type tag, It first, It last) { + if (first == last) { + return; + } + + pbf_writer sw(*this, tag); + + while (first != last) { + sw.add_varint(uint64_t(*first++)); + } + } + + template + inline void add_packed_svarint(pbf_tag_type tag, It first, It last) { + if (first == last) { + return; + } + + pbf_writer sw(*this, tag); + + while (first != last) { + sw.add_varint(encode_zigzag64(*first++)); + } + } + + // The number of bytes to reserve for the varint holding the length of + // a length-delimited field. The length has to fit into pbf_length_type, + // and a varint needs 8 bit for every 7 bit. + static const int reserve_bytes = sizeof(pbf_length_type) * 8 / 7 + 1; + + inline void open_submessage(pbf_tag_type tag) { + protozero_assert(m_pos == 0); + protozero_assert(m_data); + add_field(tag, pbf_wire_type::length_delimited); + m_data->append(size_t(reserve_bytes), '\0'); + m_pos = m_data->size(); + } + + inline void close_submessage() { + protozero_assert(m_pos != 0); + protozero_assert(m_data); + auto length = pbf_length_type(m_data->size() - m_pos); + + protozero_assert(m_data->size() >= m_pos - reserve_bytes); + auto n = write_varint(m_data->begin() + long(m_pos) - reserve_bytes, length); + + m_data->erase(m_data->begin() + long(m_pos) - reserve_bytes + n, m_data->begin() + long(m_pos)); + m_pos = 0; + } + + inline void add_length_varint(pbf_tag_type tag, pbf_length_type length) { + add_field(tag, pbf_wire_type::length_delimited); + add_varint(length); + } + +public: + + /** + * Create a writer using the given string as a data store. The pbf_writer + * stores a reference to that string and adds all data to it. + */ + inline explicit pbf_writer(std::string& data) noexcept : + m_data(&data), + m_parent_writer(nullptr), + m_pos(0) { + } + + /** + * Create a writer without a data store. In this form the writer can not + * be used! + */ + inline pbf_writer() noexcept : + m_data(nullptr), + m_parent_writer(nullptr), + m_pos(0) { + } + + /** + * Construct a pbf_writer for a submessage from the pbf_writer of the + * parent message. + * + * @param parent_writer The pbf_writer + * @param tag Tag (field number) of the field that will be written + */ + inline pbf_writer(pbf_writer& parent_writer, pbf_tag_type tag) : + m_data(parent_writer.m_data), + m_parent_writer(&parent_writer), + m_pos(0) { + m_parent_writer->open_submessage(tag); + } + + /// A pbf_writer object can be copied + pbf_writer(const pbf_writer&) noexcept = default; + + /// A pbf_writer object can be copied + pbf_writer& operator=(const pbf_writer&) noexcept = default; + + /// A pbf_writer object can be moved + inline pbf_writer(pbf_writer&&) noexcept = default; + + /// A pbf_writer object can be moved + inline pbf_writer& operator=(pbf_writer&&) noexcept = default; + + inline ~pbf_writer() { + if (m_parent_writer) { + m_parent_writer->close_submessage(); + } + } + + ///@{ + /** + * @name Scalar field writer functions + */ + + /** + * Add "bool" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_bool(pbf_tag_type tag, bool value) { + add_field(tag, pbf_wire_type::varint); + add_fixed(value); + } + + /** + * Add "enum" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_enum(pbf_tag_type tag, int32_t value) { + add_tagged_varint(tag, uint64_t(value)); + } + + /** + * Add "int32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_int32(pbf_tag_type tag, int32_t value) { + add_tagged_varint(tag, uint64_t(value)); + } + + /** + * Add "sint32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sint32(pbf_tag_type tag, int32_t value) { + add_tagged_varint(tag, encode_zigzag32(value)); + } + + /** + * Add "uint32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_uint32(pbf_tag_type tag, uint32_t value) { + add_tagged_varint(tag, value); + } + + /** + * Add "int64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_int64(pbf_tag_type tag, int64_t value) { + add_tagged_varint(tag, uint64_t(value)); + } + + /** + * Add "sint64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sint64(pbf_tag_type tag, int64_t value) { + add_tagged_varint(tag, encode_zigzag64(value)); + } + + /** + * Add "uint64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_uint64(pbf_tag_type tag, uint64_t value) { + add_tagged_varint(tag, value); + } + + /** + * Add "fixed32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_fixed32(pbf_tag_type tag, uint32_t value) { + add_field(tag, pbf_wire_type::fixed32); + add_fixed(value); + } + + /** + * Add "sfixed32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sfixed32(pbf_tag_type tag, int32_t value) { + add_field(tag, pbf_wire_type::fixed32); + add_fixed(value); + } + + /** + * Add "fixed64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_fixed64(pbf_tag_type tag, uint64_t value) { + add_field(tag, pbf_wire_type::fixed64); + add_fixed(value); + } + + /** + * Add "sfixed64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sfixed64(pbf_tag_type tag, int64_t value) { + add_field(tag, pbf_wire_type::fixed64); + add_fixed(value); + } + + /** + * Add "float" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_float(pbf_tag_type tag, float value) { + add_field(tag, pbf_wire_type::fixed32); + add_fixed(value); + } + + /** + * Add "double" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_double(pbf_tag_type tag, double value) { + add_field(tag, pbf_wire_type::fixed64); + add_fixed(value); + } + + /** + * Add "bytes" field to data. + * + * @param tag Tag (field number) of the field + * @param value Pointer to value to be written + * @param size Number of bytes to be written + */ + inline void add_bytes(pbf_tag_type tag, const char* value, size_t size) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); + protozero_assert(m_data); + assert(size <= std::numeric_limits::max()); + add_length_varint(tag, pbf_length_type(size)); + m_data->append(value, size); + } + + /** + * Add "bytes" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_bytes(pbf_tag_type tag, const std::string& value) { + add_bytes(tag, value.data(), value.size()); + } + + /** + * Add "string" field to data. + * + * @param tag Tag (field number) of the field + * @param value Pointer to value to be written + * @param size Number of bytes to be written + */ + inline void add_string(pbf_tag_type tag, const char* value, size_t size) { + add_bytes(tag, value, size); + } + + /** + * Add "string" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_string(pbf_tag_type tag, const std::string& value) { + add_bytes(tag, value.data(), value.size()); + } + + /** + * Add "string" field to data. Bytes from the value are written until + * a null byte is encountered. The null byte is not added. + * + * @param tag Tag (field number) of the field + * @param value Pointer to value to be written + */ + inline void add_string(pbf_tag_type tag, const char* value) { + add_bytes(tag, value, std::strlen(value)); + } + + /** + * Add "message" field to data. + * + * @param tag Tag (field number) of the field + * @param value Pointer to message to be written + * @param size Length of the message + */ + inline void add_message(pbf_tag_type tag, const char* value, size_t size) { + add_bytes(tag, value, size); + } + + /** + * Add "message" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written. The value must be a complete message. + */ + inline void add_message(pbf_tag_type tag, const std::string& value) { + add_bytes(tag, value.data(), value.size()); + } + + ///@} + + ///@{ + /** + * @name Repeated packed field writer functions + */ + + /** + * Add "repeated packed bool" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to bool. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_bool(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed enum" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_enum(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed int32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_int32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed sint32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sint32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_svarint(tag, first, last); + } + + /** + * Add "repeated packed uint32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_uint32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed int64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_int64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed sint64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sint64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_svarint(tag, first, last); + } + + /** + * Add "repeated packed uint64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_uint64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed fixed32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_fixed32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed sfixed32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sfixed32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed fixed64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_fixed64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed sfixed64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sfixed64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed float" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to float. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_float(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed double" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to double. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_double(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + ///@} + +}; // class pbf_writer + +} // end namespace protozero + +#endif // PROTOZERO_PBF_WRITER_HPP diff --git a/contrib/libosmium/protozero/varint.hpp b/contrib/libosmium/protozero/varint.hpp new file mode 100644 index 00000000..bc9c3296 --- /dev/null +++ b/contrib/libosmium/protozero/varint.hpp @@ -0,0 +1,136 @@ +#ifndef PROTOZERO_VARINT_HPP +#define PROTOZERO_VARINT_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file varint.hpp + * + * @brief Contains low-level varint and zigzag encoding and decoding functions. + */ + +#if __BYTE_ORDER != __LITTLE_ENDIAN +# error "This code only works on little endian machines." +#endif + +#include + +#include + +namespace protozero { + +/** + * The maximum length of a 64bit varint. + */ +const int8_t max_varint_length = sizeof(uint64_t) * 8 / 7 + 1; + +// from https://github.com/facebook/folly/blob/master/folly/Varint.h +/** + * Decode a 64bit varint. + * + * String exception guarantee: if there is an exception the data pointer will + * not be changed. + * + * @param[in,out] data Pointer to pointer to the input data. After the function + * returns this will point to the next data to be read. + * @param[in] end Pointer one past the end of the input data. + * @returns The decoded integer + * @throws varint_too_long_exception if the varint is longer then the maximum + * length that would fit in a 64bit int. Usually this means your data + * is corrupted or you are trying to read something as a varint that + * isn't. + * @throws end_of_buffer_exception if the *end* of the buffer was reached + * before the end of the varint. + */ +inline uint64_t decode_varint(const char** data, const char* end) { + const int8_t* begin = reinterpret_cast(*data); + const int8_t* iend = reinterpret_cast(end); + const int8_t* p = begin; + uint64_t val = 0; + + if (iend - begin >= max_varint_length) { // fast path + do { + int64_t b; + b = *p++; val = uint64_t((b & 0x7f) ); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 7); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 14); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 21); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 28); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 35); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 42); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 49); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 56); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 63); if (b >= 0) break; + throw varint_too_long_exception(); + } while (false); + } else { + int shift = 0; + while (p != iend && *p < 0) { + val |= uint64_t(*p++ & 0x7f) << shift; + shift += 7; + } + if (p == iend) { + throw end_of_buffer_exception(); + } + val |= uint64_t(*p++) << shift; + } + + *data = reinterpret_cast(p); + return val; +} + +/** + * Varint-encode a 64bit integer. + */ +template +inline int write_varint(OutputIterator data, uint64_t value) { + int n=1; + + while (value >= 0x80) { + *data++ = char((value & 0x7f) | 0x80); + value >>= 7; + ++n; + } + *data++ = char(value); + + return n; +} + +/** + * ZigZag encodes a 32 bit integer. + */ +inline uint32_t encode_zigzag32(int32_t value) noexcept { + return (static_cast(value) << 1) ^ (static_cast(value >> 31)); +} + +/** + * ZigZag encodes a 64 bit integer. + */ +inline uint64_t encode_zigzag64(int64_t value) noexcept { + return (static_cast(value) << 1) ^ (static_cast(value >> 63)); +} + +/** + * Decodes a 32 bit ZigZag-encoded integer. + */ +inline int32_t decode_zigzag32(uint32_t value) noexcept { + return int32_t(value >> 1) ^ -int32_t(value & 1); +} + +/** + * Decodes a 64 bit ZigZag-encoded integer. + */ +inline int64_t decode_zigzag64(uint64_t value) noexcept { + return int64_t(value >> 1) ^ -int64_t(value & 1); +} + +} // end namespace protozero + +#endif // PROTOZERO_VARINT_HPP diff --git a/contrib/protobuf/fileformat.proto b/contrib/protobuf/fileformat.proto deleted file mode 100644 index e6fd9379..00000000 --- a/contrib/protobuf/fileformat.proto +++ /dev/null @@ -1,54 +0,0 @@ -/** Copyright (c) 2010 Scott A. Crosby. - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with this program. If not, see . - -*/ - -option optimize_for = LITE_RUNTIME; -option java_package = "crosby.binary"; -package OSMPBF; - -//protoc --java_out=../.. fileformat.proto - - -// -// STORAGE LAYER: Storing primitives. -// - -message Blob { - optional bytes raw = 1; // No compression - optional int32 raw_size = 2; // When compressed, the uncompressed size - - // Possible compressed versions of the data. - optional bytes zlib_data = 3; - - // PROPOSED feature for LZMA compressed data. SUPPORT IS NOT REQUIRED. - optional bytes lzma_data = 4; - - // Formerly used for bzip2 compressed data. Depreciated in 2010. - optional bytes OBSOLETE_bzip2_data = 5 [deprecated=true]; // Don't reuse this tag number. -} - -/* A file contains an sequence of fileblock headers, each prefixed by -their length in network byte order, followed by a data block -containing the actual data. types staring with a "_" are reserved. -*/ - -message BlobHeader { - required string type = 1; - optional bytes indexdata = 2; - required int32 datasize = 3; -} - - diff --git a/contrib/protobuf/osmformat.proto b/contrib/protobuf/osmformat.proto deleted file mode 100644 index f919bacc..00000000 --- a/contrib/protobuf/osmformat.proto +++ /dev/null @@ -1,260 +0,0 @@ -/** Copyright (c) 2010 Scott A. Crosby. - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with this program. If not, see . - -*/ - -option optimize_for = LITE_RUNTIME; -option java_package = "crosby.binary"; -package OSMPBF; - -/* OSM Binary file format - -This is the master schema file of the OSM binary file format. This -file is designed to support limited random-access and future -extendability. - -A binary OSM file consists of a sequence of FileBlocks (please see -fileformat.proto). The first fileblock contains a serialized instance -of HeaderBlock, followed by a sequence of PrimitiveBlock blocks that -contain the primitives. - -Each primitiveblock is designed to be independently parsable. It -contains a string table storing all strings in that block (keys and -values in tags, roles in relations, usernames, etc.) as well as -metadata containing the precision of coordinates or timestamps in that -block. - -A primitiveblock contains a sequence of primitive groups, each -containing primitives of the same type (nodes, densenodes, ways, -relations). Coordinates are stored in signed 64-bit integers. Lat&lon -are measured in units nanodegrees. The default of -granularity of 100 nanodegrees corresponds to about 1cm on the ground, -and a full lat or lon fits into 32 bits. - -Converting an integer to a lattitude or longitude uses the formula: -$OUT = IN * granularity / 10**9$. Many encoding schemes use delta -coding when representing nodes and relations. - -*/ - -////////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////// - -/* Contains the file header. */ - -message HeaderBlock { - optional HeaderBBox bbox = 1; - /* Additional tags to aid in parsing this dataset */ - repeated string required_features = 4; - repeated string optional_features = 5; - - optional string writingprogram = 16; - optional string source = 17; // From the bbox field. - - /* Tags that allow continuing an Osmosis replication */ - - // replication timestamp, expressed in seconds since the epoch, - // otherwise the same value as in the "timestamp=..." field - // in the state.txt file used by Osmosis - optional int64 osmosis_replication_timestamp = 32; - - // replication sequence number (sequenceNumber in state.txt) - optional int64 osmosis_replication_sequence_number = 33; - - // replication base URL (from Osmosis' configuration.txt file) - optional string osmosis_replication_base_url = 34; -} - - -/** The bounding box field in the OSM header. BBOX, as used in the OSM -header. Units are always in nanodegrees -- they do not obey -granularity rules. */ - -message HeaderBBox { - required sint64 left = 1; - required sint64 right = 2; - required sint64 top = 3; - required sint64 bottom = 4; -} - - -/////////////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////////////// - - -message PrimitiveBlock { - required StringTable stringtable = 1; - repeated PrimitiveGroup primitivegroup = 2; - - // Granularity, units of nanodegrees, used to store coordinates in this block - optional int32 granularity = 17 [default=100]; - // Offset value between the output coordinates coordinates and the granularity grid in unites of nanodegrees. - optional int64 lat_offset = 19 [default=0]; - optional int64 lon_offset = 20 [default=0]; - -// Granularity of dates, normally represented in units of milliseconds since the 1970 epoch. - optional int32 date_granularity = 18 [default=1000]; - - - // Proposed extension: - //optional BBox bbox = XX; -} - -// Group of OSMPrimitives. All primitives in a group must be the same type. -message PrimitiveGroup { - repeated Node nodes = 1; - optional DenseNodes dense = 2; - repeated Way ways = 3; - repeated Relation relations = 4; - repeated ChangeSet changesets = 5; -} - - -/** String table, contains the common strings in each block. - - Note that we reserve index '0' as a delimiter, so the entry at that - index in the table is ALWAYS blank and unused. - - */ -message StringTable { - repeated bytes s = 1; -} - -/* Optional metadata that may be included into each primitive. */ -message Info { - optional int32 version = 1 [default = -1]; - optional int64 timestamp = 2; - optional int64 changeset = 3; - optional int32 uid = 4; - optional uint32 user_sid = 5; // String IDs - - // The visible flag is used to store history information. It indicates that - // the current object version has been created by a delete operation on the - // OSM API. - // When a writer sets this flag, it MUST add a required_features tag with - // value "HistoricalInformation" to the HeaderBlock. - // If this flag is not available for some object it MUST be assumed to be - // true if the file has the required_features tag "HistoricalInformation" - // set. - optional bool visible = 6; -} - -/** Optional metadata that may be included into each primitive. Special dense format used in DenseNodes. */ -message DenseInfo { - repeated int32 version = 1 [packed = true]; - repeated sint64 timestamp = 2 [packed = true]; // DELTA coded - repeated sint64 changeset = 3 [packed = true]; // DELTA coded - repeated sint32 uid = 4 [packed = true]; // DELTA coded - repeated sint32 user_sid = 5 [packed = true]; // String IDs for usernames. DELTA coded - - // The visible flag is used to store history information. It indicates that - // the current object version has been created by a delete operation on the - // OSM API. - // When a writer sets this flag, it MUST add a required_features tag with - // value "HistoricalInformation" to the HeaderBlock. - // If this flag is not available for some object it MUST be assumed to be - // true if the file has the required_features tag "HistoricalInformation" - // set. - repeated bool visible = 6 [packed = true]; -} - - -// THIS IS STUB DESIGN FOR CHANGESETS. NOT USED RIGHT NOW. -// TODO: REMOVE THIS? -message ChangeSet { - required int64 id = 1; -// -// // Parallel arrays. -// repeated uint32 keys = 2 [packed = true]; // String IDs. -// repeated uint32 vals = 3 [packed = true]; // String IDs. -// -// optional Info info = 4; - -// optional int64 created_at = 8; -// optional int64 closetime_delta = 9; -// optional bool open = 10; -// optional HeaderBBox bbox = 11; -} - - -message Node { - required sint64 id = 1; - // Parallel arrays. - repeated uint32 keys = 2 [packed = true]; // String IDs. - repeated uint32 vals = 3 [packed = true]; // String IDs. - - optional Info info = 4; // May be omitted in omitmeta - - required sint64 lat = 8; - required sint64 lon = 9; -} - -/* Used to densly represent a sequence of nodes that do not have any tags. - -We represent these nodes columnwise as five columns: ID's, lats, and -lons, all delta coded. When metadata is not omitted, - -We encode keys & vals for all nodes as a single array of integers -containing key-stringid and val-stringid, using a stringid of 0 as a -delimiter between nodes. - - ( ( )* '0' )* - */ - -message DenseNodes { - repeated sint64 id = 1 [packed = true]; // DELTA coded - - //repeated Info info = 4; - optional DenseInfo denseinfo = 5; - - repeated sint64 lat = 8 [packed = true]; // DELTA coded - repeated sint64 lon = 9 [packed = true]; // DELTA coded - - // Special packing of keys and vals into one array. May be empty if all nodes in this block are tagless. - repeated int32 keys_vals = 10 [packed = true]; -} - - -message Way { - required int64 id = 1; - // Parallel arrays. - repeated uint32 keys = 2 [packed = true]; - repeated uint32 vals = 3 [packed = true]; - - optional Info info = 4; - - repeated sint64 refs = 8 [packed = true]; // DELTA coded -} - -message Relation { - enum MemberType { - NODE = 0; - WAY = 1; - RELATION = 2; - } - required int64 id = 1; - - // Parallel arrays. - repeated uint32 keys = 2 [packed = true]; - repeated uint32 vals = 3 [packed = true]; - - optional Info info = 4; - - // Parallel arrays - repeated int32 roles_sid = 8 [packed = true]; - repeated sint64 memids = 9 [packed = true]; // DELTA encoded - repeated MemberType types = 10 [packed = true]; -} - diff --git a/contrib/protobuf/osmpbf/osmpbf.h b/contrib/protobuf/osmpbf/osmpbf.h deleted file mode 100644 index 77194763..00000000 --- a/contrib/protobuf/osmpbf/osmpbf.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef OSMPBF_H -#define OSMPBF_H - -// this describes the low-level blob storage -#include // IWYU pragma: export - -// this describes the high-level OSM objects -#include // IWYU pragma: export - -#define OSMPBF_VERSION "1.3.3" - -namespace OSMPBF { - - // the maximum size of a blob header in bytes - const int max_blob_header_size = 64 * 1024; // 64 kB - - // the maximum size of an uncompressed blob in bytes - const int max_uncompressed_blob_size = 32 * 1024 * 1024; // 32 MB - - // resolution for longitude/latitude used for conversion - // between representation as double and as int - const int lonlat_resolution = 1000 * 1000 * 1000; - -} - -#endif // OSMPBF_H diff --git a/tests/test-hstore-match-only.cpp b/tests/test-hstore-match-only.cpp index 351a77ab..95002dd5 100644 --- a/tests/test-hstore-match-only.cpp +++ b/tests/test-hstore-match-only.cpp @@ -60,7 +60,7 @@ int main(int argc, char *argv[]) { osmdata.start(); - parser->stream_file("libxml2", "tests/hstore-match-only.osm", &osmdata); + parser->stream_file("xml", "tests/hstore-match-only.osm", &osmdata); parser.reset(nullptr); diff --git a/tests/test-options-parse.cpp b/tests/test-options-parse.cpp index 6f3f79c8..35d01912 100644 --- a/tests/test-options-parse.cpp +++ b/tests/test-options-parse.cpp @@ -257,7 +257,7 @@ void test_random_perms() add_arg_and_val_or_not("--prefix", args, options.prefix.c_str(), get_random_string(15)); - //--input-reader Input frontend. libxml2 - Parse XML using libxml2. (default) primitive - Primitive XML parsing. pbf - OSM binary format. + //--input-reader Input frontend. auto, o5m, xml, pbf if (options.tag_transform_script) { add_arg_and_val_or_not("--tag-transform-script", args, options.tag_transform_script->c_str(), get_random_string(15)); diff --git a/tests/test-output-multi-line-storage.cpp b/tests/test-output-multi-line-storage.cpp index a53f0bd8..2a39273d 100644 --- a/tests/test-output-multi-line-storage.cpp +++ b/tests/test-output-multi-line-storage.cpp @@ -58,7 +58,7 @@ int main(int argc, char *argv[]) { osmdata.start(); - parser.stream_file("libxml2", "tests/test_output_multi_line_storage.osm", &osmdata); + parser.stream_file("xml", "tests/test_output_multi_line_storage.osm", &osmdata); osmdata.stop(); diff --git a/tests/test-output-multi-poly-trivial.cpp b/tests/test-output-multi-poly-trivial.cpp index 0ee2877f..7e9f9b6e 100644 --- a/tests/test-output-multi-poly-trivial.cpp +++ b/tests/test-output-multi-poly-trivial.cpp @@ -38,7 +38,7 @@ void run_osm2pgsql(options_t &options) { osmdata.start(); - parser.stream_file("libxml2", "tests/test_output_multi_poly_trivial.osm", &osmdata); + parser.stream_file("xml", "tests/test_output_multi_poly_trivial.osm", &osmdata); osmdata.stop(); } diff --git a/tests/test-output-multi-tags.cpp b/tests/test-output-multi-tags.cpp index 3f8077ad..07709b2b 100644 --- a/tests/test-output-multi-tags.cpp +++ b/tests/test-output-multi-tags.cpp @@ -58,7 +58,7 @@ int main(int argc, char *argv[]) { osmdata.start(); - parser.stream_file("libxml2", "tests/test_output_multi_tags.osm", &osmdata); + parser.stream_file("xml", "tests/test_output_multi_tags.osm", &osmdata); osmdata.stop(); diff --git a/tests/test-output-pgsql-schema.cpp b/tests/test-output-pgsql-schema.cpp index 8dc31a5c..daffb704 100644 --- a/tests/test-output-pgsql-schema.cpp +++ b/tests/test-output-pgsql-schema.cpp @@ -83,7 +83,7 @@ void test_other_output_schema() { osmdata.start(); - parser->stream_file("libxml2", "tests/test_output_pgsql_z_order.osm", &osmdata); + parser->stream_file("xml", "tests/test_output_pgsql_z_order.osm", &osmdata); parser.reset(nullptr); diff --git a/tests/test-output-pgsql-z_order.cpp b/tests/test-output-pgsql-z_order.cpp index 68ec098c..2ae01958 100644 --- a/tests/test-output-pgsql-z_order.cpp +++ b/tests/test-output-pgsql-z_order.cpp @@ -79,7 +79,7 @@ void test_z_order() { osmdata.start(); - parser->stream_file("libxml2", "tests/test_output_pgsql_z_order.osm", &osmdata); + parser->stream_file("xml", "tests/test_output_pgsql_z_order.osm", &osmdata); parser.reset(nullptr); diff --git a/tests/test-output-pgsql.cpp b/tests/test-output-pgsql.cpp index 769fa713..ae721a5a 100644 --- a/tests/test-output-pgsql.cpp +++ b/tests/test-output-pgsql.cpp @@ -200,7 +200,7 @@ void test_area_way_simple() { osmdata.start(); - parser->stream_file("libxml2", "tests/test_output_pgsql_way_area.osm", &osmdata); + parser->stream_file("xml", "tests/test_output_pgsql_way_area.osm", &osmdata); parser.reset(nullptr); @@ -246,7 +246,7 @@ void test_route_rel() { osmdata.start(); - parser->stream_file("libxml2", "tests/test_output_pgsql_route_rel.osm", &osmdata); + parser->stream_file("xml", "tests/test_output_pgsql_route_rel.osm", &osmdata); parser.reset(nullptr);