VladPolskiy/osm2pgsql
1
0
Fork 0
mirror of https://github.com/osm2pgsql-dev/osm2pgsql.git synced 2025-08-16 15:04:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add new command line parameter --schema that sets the default schema

Browse Source 
This will be used as default for --middle-schema, --output-pgsql-schema,
and for the different ways of setting a schema in the flex output and
generalizer code.

This removes the last places where the schema was hardcoded to "public"
(except as a default for this command line parameter and in some legacy
gazetteer code).
This commit is contained in:
Jochen Topf
2023-08-14 14:07:58 +02:00
parent e340a4cb14
commit 43dd112612
14 changed files with 105 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/command-line-parser.cpp
View File

@ -83,6 +83,7 @@ struct option const long_options[] = {
{"prefix", required_argument, nullptr, 'p'},
{"proj", required_argument, nullptr, 'E'},
{"reproject-area", no_argument, nullptr, 213},
{"schema", required_argument, nullptr, 218},
{"slim", no_argument, nullptr, 's'},
{"style", required_argument, nullptr, 'S'},
{"tablespace-index", required_argument, nullptr, 'i'},
@ -140,6 +141,7 @@ Common options:\n\
information in slim mode instead of in PostgreSQL.\n\
This is a single large file (> 50GB). Only recommended\n\
for full planet imports. Default is disabled.\n\
--schema=SCHEMA Default schema (default: 'public').\n\
\n\
Database options:\n\
-d|--database=DB The name of the PostgreSQL database to connect to or\n\
@ -181,7 +183,7 @@ Middle options:\n\
--cache-strategy=STRATEGY Deprecated. Not used any more.\n\
-x|--extra-attributes Include attributes (user name, user id, changeset\n\
id, timestamp and version) for each object in the database.\n\
--middle-schema=SCHEMA Schema to use for middle tables (default: 'public').\n\
--middle-schema=SCHEMA Schema to use for middle tables (default: setting of --schema).\n\
--middle-way-node-index-id-shift=SHIFT Set ID shift for bucket index.\n\
--middle-database-format=FORMAT Set middle db format (default: legacy).\n\
--middle-with-nodes Store tagged nodes in db (new middle db format only).\n\
@ -213,7 +215,7 @@ Pgsql output options:\n\
-K|--keep-coastlines Keep coastline data rather than filtering it out.\n\
Default: discard objects tagged natural=coastline.\n\
--output-pgsql-schema=SCHEMA Schema to use for pgsql output tables\n\
(default: 'public').\n\
(default: setting of --schema).\n\
--reproject-area Compute area column using web mercator coordinates.\n\
\n\
Expiry options:\n\
@ -728,6 +730,13 @@ options_t parse_command_line(int argc, char *argv[])
options.with_forward_dependencies =
parse_with_forward_dependencies_param(optarg);
break;
case 218: // --schema
options.dbschema = optarg;
if (options.dbschema.empty()) {
throw std::runtime_error{"Schema can not be empty."};
}
check_identifier(options.dbschema, "--schema parameter");
break;
case 300: // --middle-way-node-index-id-shift
options.way_node_index_id_shift = atoi(optarg);
break;
@ -763,6 +772,14 @@ options_t parse_command_line(int argc, char *argv[])
}
} //end while
if (options.middle_dbschema.empty()) {
options.middle_dbschema = options.dbschema;
}
if (options.output_dbschema.empty()) {
options.output_dbschema = options.dbschema;
}
//they were looking for usage info
if (print_help) {
options.command = command_t::help;

2
src/expire-output.hpp
View File

@ -81,7 +81,7 @@ private:
std::string m_filename;
/// The schema for output
std::string m_schema{"public"};
std::string m_schema;
/// The table (if any) for output
std::string m_table;

9
src/flex-lua-expire-output.cpp
View File

@ -18,7 +18,7 @@
#include <lua.hpp>
static expire_output_t &
create_expire_output(lua_State *lua_state,
create_expire_output(lua_State *lua_state, std::string const &default_schema,
std::vector<expire_output_t> *expire_outputs)
{
auto &new_expire_output = expire_outputs->emplace_back();
@ -30,8 +30,8 @@ create_expire_output(lua_State *lua_state,
lua_pop(lua_state, 1); // "filename"
// optional "schema" and "table" fields
auto const *schema = luaX_get_table_string(lua_state, "schema", -1,
"The expire output", "public");
auto const *schema = luaX_get_table_string(
lua_state, "schema", -1, "The expire output", default_schema.c_str());
check_identifier(schema, "schema field");
auto const *table =
luaX_get_table_string(lua_state, "table", -2, "The expire output", "");
@ -72,6 +72,7 @@ create_expire_output(lua_State *lua_state,
}
int setup_flex_expire_output(lua_State *lua_state,
std::string const &default_schema,
std::vector<expire_output_t> *expire_outputs)
{
if (lua_type(lua_state, 1) != LUA_TTABLE) {
@ -79,7 +80,7 @@ int setup_flex_expire_output(lua_State *lua_state,
"Argument #1 to 'define_expire_output' must be a Lua table."};
}
create_expire_output(lua_state, expire_outputs);
create_expire_output(lua_state, default_schema, expire_outputs);
void *ptr = lua_newuserdata(lua_state, sizeof(std::size_t));
auto *num = new (ptr) std::size_t{};

1
src/flex-lua-expire-output.hpp
View File

@ -20,6 +20,7 @@ static char const *const osm2pgsql_expire_output_name =
"osm2pgsql.ExpireOutput";
int setup_flex_expire_output(lua_State *lua_state,
std::string const &default_schema,
std::vector<expire_output_t> *expire_outputs);
#endif // OSM2PGSQL_FLEX_LUA_EXPIRE_OUTPUT_HPP

8
src/flex-lua-table.cpp
View File

@ -29,6 +29,7 @@ static void check_tablespace(std::string const &tablespace)
}
static flex_table_t &create_flex_table(lua_State *lua_state,
std::string const &default_schema,
std::vector<flex_table_t> *tables)
{
std::string const table_name =
@ -40,7 +41,7 @@ static flex_table_t &create_flex_table(lua_State *lua_state,
throw fmt_error("Table with name '{}' already exists.", table_name);
}
auto &new_table = tables->emplace_back(table_name);
auto &new_table = tables->emplace_back(default_schema, table_name);
lua_pop(lua_state, 1); // "name"
@ -411,14 +412,15 @@ static void setup_flex_table_indexes(lua_State *lua_state, flex_table_t *table,
int setup_flex_table(lua_State *lua_state, std::vector<flex_table_t> *tables,
std::vector<expire_output_t> *expire_outputs,
bool updatable, bool append_mode)
std::string const &default_schema, bool updatable,
bool append_mode)
{
if (lua_type(lua_state, 1) != LUA_TTABLE) {
throw std::runtime_error{
"Argument #1 to 'define_table' must be a table."};
}
auto &new_table = create_flex_table(lua_state, tables);
auto &new_table = create_flex_table(lua_state, default_schema, tables);
setup_flex_table_id_columns(lua_state, &new_table);
setup_flex_table_columns(lua_state, &new_table, expire_outputs,
append_mode);

4
src/flex-lua-table.hpp
View File

@ -10,6 +10,7 @@
* For a full list of authors see the git log.
*/
#include <string>
#include <vector>
class expire_output_t;
@ -20,6 +21,7 @@ static char const *const osm2pgsql_table_name = "osm2pgsql.Table";
int setup_flex_table(lua_State *lua_state, std::vector<flex_table_t> *tables,
std::vector<expire_output_t> *expire_outputs,
bool updatable, bool append_mode);
std::string const &default_schema, bool updatable,
bool append_mode);
#endif // OSM2PGSQL_FLEX_LUA_TABLE_HPP

11
src/flex-table.hpp
View File

@ -59,7 +59,10 @@ public:
permanent
};
explicit flex_table_t(std::string name) : m_name(std::move(name)) {}
flex_table_t(std::string schema, std::string name)
: m_schema(std::move(schema)), m_name(std::move(name))
{
}
std::string const &name() const noexcept { return m_name; }
@ -195,12 +198,12 @@ public:
bool has_columns_with_expire() const noexcept;
private:
/// The schema this table is in
std::string m_schema;
/// The name of the table
std::string m_name;
/// The schema this table is in
std::string m_schema{"public"};
/// The table space used for this table (empty for default tablespace)
std::string m_data_tablespace;

2
src/gen/gen-base.cpp
View File

@ -20,7 +20,7 @@ gen_base_t::gen_base_t(pg_conn_t *connection, params_t *params)
assert(connection);
assert(params);
params->check_identifier_with_default("schema", "public");
params->check_identifier_with_default("schema", "");
auto const schema = params->get_identifier("schema");
if (params->has("src_table")) {

57
src/gen/osm2pgsql-gen.cpp
View File

@ -90,7 +90,8 @@ Main Options:
-c|--create Run in create mode (default)
-S|--style=FILE The Lua config file (same as for osm2pgsql)
-j|--jobs=NUM Number of parallel jobs (default 1)
--middle-schema=SCHEMA Database schema for middle tables
--middle-schema=SCHEMA Database schema for middle tables (default set with --schema)
--schema=SCHEMA Default database schema (default: 'public')
Help/Version Options:
-h|--help Print this help text and stop
@ -127,6 +128,7 @@ static std::array<option, 20> const long_options = {
{"style", required_argument, nullptr, 'S'},
{"log-sql", no_argument, nullptr, 201},
{"middle-schema", required_argument, nullptr, 202},
{"schema", required_argument, nullptr, 203},
{nullptr, 0, nullptr, 0}}};
struct tile_extent
@ -178,9 +180,10 @@ static tile_extent get_extent_from_db(pg_conn_t const &db_connection,
}
static tile_extent get_extent_from_db(pg_conn_t const &db_connection,
std::string const &default_schema,
params_t const &params, uint32_t zoom)
{
auto const schema = params.get_string("schema", "public");
auto const schema = params.get_string("schema", default_schema);
std::string table;
if (params.has("src_table")) {
table = params.get_string("src_table");
@ -267,8 +270,9 @@ void run_tile_gen(std::string const &conninfo, gen_base_t *master_generalizer,
class genproc_t
{
public:
genproc_t(std::string const &filename, std::string conninfo, bool append,
bool updatable, uint32_t jobs);
genproc_t(std::string const &filename, std::string conninfo,
std::string dbschema, bool append, bool updatable,
uint32_t jobs);
int app_define_table()
{
@ -281,12 +285,13 @@ public:
#endif
return setup_flex_table(m_lua_state.get(), &m_tables, &m_expire_outputs,
true, m_append);
m_dbschema, true, m_append);
}
int app_define_expire_output()
{
return setup_flex_expire_output(m_lua_state.get(), &m_expire_outputs);
return setup_flex_expire_output(m_lua_state.get(), m_dbschema,
&m_expire_outputs);
}
int app_run_gen()
@ -308,6 +313,10 @@ public:
auto params = parse_params();
if (!params.has("schema")) {
params.set("schema", m_dbschema);
}
write_to_debug_log(params, "Params (config):");
log_debug("Connecting to database...");
@ -440,7 +449,8 @@ private:
log_debug("Truncating table '{}'...", table);
db_connection.exec("TRUNCATE {}", table);
} else {
auto const extent = get_extent_from_db(db_connection, params, zoom);
auto const extent =
get_extent_from_db(db_connection, m_dbschema, params, zoom);
if (extent.valid) {
auto const num_tiles = (extent.xmax - extent.xmin + 1) *
@ -493,6 +503,7 @@ private:
std::vector<expire_output_t> m_expire_outputs;
std::string m_conninfo;
std::string m_dbschema;
std::size_t m_gen_run = 0;
uint32_t m_jobs;
bool m_append;
@ -505,9 +516,10 @@ TRAMPOLINE(app_run_gen, run_gen)
TRAMPOLINE(app_run_sql, run_sql)
genproc_t::genproc_t(std::string const &filename, std::string conninfo,
bool append, bool updatable, uint32_t jobs)
: m_conninfo(std::move(conninfo)), m_jobs(jobs), m_append(append),
m_updatable(updatable)
std::string dbschema, bool append, bool updatable,
uint32_t jobs)
: m_conninfo(std::move(conninfo)), m_dbschema(std::move(dbschema)),
m_jobs(jobs), m_append(append), m_updatable(updatable)
{
setup_lua_environment(lua_state(), filename, append);
@ -587,7 +599,8 @@ int main(int argc, char *argv[])
{
try {
database_options_t database_options;
std::string schema{"public"};
std::string dbschema{"public"};
std::string middle_dbschema{};
std::string log_level;
std::string style;
uint32_t jobs = 1;
@ -641,12 +654,20 @@ int main(int argc, char *argv[])
get_logger().enable_sql();
break;
case 202: // --middle-schema
schema = optarg;
if (schema.empty()) {
middle_dbschema = optarg;
if (middle_dbschema.empty()) {
log_error("Schema must not be empty");
return 2;
}
check_identifier(schema, "--middle-schema");
check_identifier(middle_dbschema, "--middle-schema");
break;
case 203: // --schema
dbschema = optarg;
if (dbschema.empty()) {
log_error("Schema must not be empty");
return 2;
}
check_identifier(dbschema, "--schema");
break;
default:
log_error("Unknown argument");
@ -674,6 +695,10 @@ int main(int argc, char *argv[])
return 2;
}
if (middle_dbschema.empty()) {
middle_dbschema = dbschema;
}
util::timer_t timer_overall;
log_info("osm2pgsql-gen version {}", get_osm2pgsql_version());
@ -704,7 +729,7 @@ int main(int argc, char *argv[])
init_database_capabilities(db_connection);
}
properties_t properties{conninfo, schema};
properties_t properties{conninfo, middle_dbschema};
properties.load();
if (style.empty()) {
@ -716,7 +741,7 @@ int main(int argc, char *argv[])
}
bool const updatable = properties.get_bool("updatable", false);
genproc_t gen{style, conninfo, append, updatable, jobs};
genproc_t gen{style, conninfo, dbschema, append, updatable, jobs};
gen.run();
osmium::MemoryUsage const mem;

7
src/options.hpp
View File

@ -98,11 +98,14 @@ struct options_t
/// Pg Tablespace to store slim tables (no default TABLESPACE)
std::string tblsslim_data{};
/// Default Pg schema.
std::string dbschema{"public"};
/// Pg schema to store middle tables in.
std::string middle_dbschema{"public"};
std::string middle_dbschema{};
/// Pg schema to store output tables in.
std::string output_dbschema{"public"};
std::string output_dbschema{};
std::string style{}; ///< style file to use

1
src/osm2pgsql.cpp
View File

@ -90,6 +90,7 @@ static void check_db(options_t const &options)
init_database_capabilities(db_connection);
check_schema(options.dbschema);
check_schema(options.middle_dbschema);
check_schema(options.output_dbschema);
}

4
src/output-flex.cpp
View File

@ -402,6 +402,7 @@ int output_flex_t::app_define_table()
}
return setup_flex_table(lua_state(), m_tables.get(), m_expire_outputs.get(),
get_options()->dbschema,
get_options()->slim && !get_options()->droptemp,
get_options()->append);
}
@ -414,7 +415,8 @@ int output_flex_t::app_define_expire_output()
" main Lua code, not in any of the callbacks."};
}
return setup_flex_expire_output(lua_state(), m_expire_outputs.get());
return setup_flex_expire_output(lua_state(), get_options()->dbschema,
m_expire_outputs.get());
}
// Check that the first element on the Lua stack is a "type_name"

2
tests/common-options.hpp
View File

@ -28,6 +28,8 @@ public:
m_opt.cache = 2;
m_opt.append = false;
m_opt.projection = reprojection::create_projection(PROJ_SPHERE_MERC);
m_opt.middle_dbschema = "public";
m_opt.output_dbschema = "public";
}
operator options_t() const { return m_opt; }

22
tests/test-flex-indexes.cpp
View File

@ -48,7 +48,7 @@ TEST_CASE("check index with single column", "[NoDB]")
{
test_framework const tf;
flex_table_t table{"test_table"};
flex_table_t table{"public", "test_table"};
table.add_column("geom", "geometry", "");
REQUIRE(table.indexes().empty());
@ -71,7 +71,7 @@ TEST_CASE("check index with multiple columns", "[NoDB]")
{
test_framework const tf;
flex_table_t table{"test_table"};
flex_table_t table{"public", "test_table"};
table.add_column("a", "int", "");
table.add_column("b", "int", "");
@ -93,7 +93,7 @@ TEST_CASE("check unique index", "[NoDB]")
{
test_framework const tf;
flex_table_t table{"test_table"};
flex_table_t table{"public", "test_table"};
table.add_column("col", "int", "");
REQUIRE(tf.run_lua(
@ -115,7 +115,7 @@ TEST_CASE("check index with tablespace from table", "[NoDB]")
{
test_framework const tf;
flex_table_t table{"test_table"};
flex_table_t table{"public", "test_table"};
table.set_index_tablespace("foo");
table.add_column("col", "int", "");
@ -137,7 +137,7 @@ TEST_CASE("check index with tablespace", "[NoDB]")
{
test_framework const tf;
flex_table_t table{"test_table"};
flex_table_t table{"public", "test_table"};
table.add_column("col", "int", "");
REQUIRE(tf.run_lua("return { method = 'btree', column = 'col', tablespace "
@ -159,7 +159,7 @@ TEST_CASE("check index with expression and where clause", "[NoDB]")
{
test_framework const tf;
flex_table_t table{"test_table"};
flex_table_t table{"public", "test_table"};
table.add_column("col", "text", "");
REQUIRE(tf.run_lua("return { method = 'btree', expression = 'lower(col)',"
@ -181,7 +181,7 @@ TEST_CASE("check index with include", "[NoDB]")
{
test_framework const tf;
flex_table_t table{"test_table"};
flex_table_t table{"public", "test_table"};
table.add_column("col", "int", "");
table.add_column("extra", "int", "");
@ -204,7 +204,7 @@ TEST_CASE("check index with include as array", "[NoDB]")
{
test_framework const tf;
flex_table_t table{"test_table"};
flex_table_t table{"public", "test_table"};
table.add_column("col", "int", "");
table.add_column("extra", "int", "");
@ -227,7 +227,7 @@ TEST_CASE("check index with empty include array", "[NoDB]")
{
test_framework const tf;
flex_table_t table{"test_table"};
flex_table_t table{"public", "test_table"};
table.add_column("col", "int", "");
table.add_column("extra", "int", "");
@ -250,7 +250,7 @@ TEST_CASE("check multiple indexes", "[NoDB]")
{
test_framework const tf;
flex_table_t table{"test_table"};
flex_table_t table{"public", "test_table"};
table.add_column("a", "int", "");
table.add_column("b", "int", "");
@ -273,7 +273,7 @@ TEST_CASE("check various broken index configs", "[NoDB]")
{
test_framework const tf;
flex_table_t table{"test_table"};
flex_table_t table{"public", "test_table"};
table.add_column("col", "text", "");
SECTION("empty index description") { REQUIRE(tf.run_lua("return {}")); }