diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index f32edf3bcd6..79e452f6801 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -56,9 +56,17 @@ TARGET_LINK_LIBRARIES(mariadb-check ${CLIENT_LIB}) MYSQL_ADD_EXECUTABLE(mariadb-dump mysqldump.cc ../sql-common/my_user.c connection_pool.cc) TARGET_LINK_LIBRARIES(mariadb-dump ${CLIENT_LIB}) +ADD_LIBRARY(import_util STATIC import_util.cc import_util.h) +IF (PCRE2_DEBIAN_HACK) + SET_SOURCE_FILES_PROPERTIES(import_util.cc PROPERTIES COMPILE_FLAGS "${PCRE2_DEBIAN_HACK}") +ENDIF() +TARGET_LINK_LIBRARIES(import_util PRIVATE pcre2-posix pcre2-8) +TARGET_INCLUDE_DIRECTORIES(import_util PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) + MYSQL_ADD_EXECUTABLE(mariadb-import mysqlimport.cc) -target_include_directories(mariadb-import PRIVATE ${CMAKE_SOURCE_DIR}/tpool) -target_link_libraries(mariadb-import PRIVATE tpool ${CLIENT_LIB}) +TARGET_INCLUDE_DIRECTORIES(mariadb-import PRIVATE ${CMAKE_SOURCE_DIR}/tpool) +TARGET_LINK_LIBRARIES(mariadb-import PRIVATE tpool ${CLIENT_LIB} import_util) + MYSQL_ADD_EXECUTABLE(mariadb-upgrade mysql_upgrade.c COMPONENT Server) TARGET_LINK_LIBRARIES(mariadb-upgrade ${CLIENT_LIB}) @@ -101,3 +109,6 @@ FOREACH(t mariadb mariadb-test mariadb-check mariadb-dump mariadb-import mariadb ENDFOREACH() ADD_DEFINITIONS(-DHAVE_DLOPEN) +IF(WITH_UNIT_TESTS) + ADD_SUBDIRECTORY(${PROJECT_SOURCE_DIR}/unittest/client ${PROJECT_BINARY_DIR}/unittest/client) +ENDIF() diff --git a/client/import_util.cc b/client/import_util.cc new file mode 100644 index 00000000000..baade1535c6 --- /dev/null +++ b/client/import_util.cc @@ -0,0 +1,217 @@ +/* + Copyright (c) 2024, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA +*/ + +/* + This file contains some routines to do client-side parsing of CREATE TABLE + statements. The goal is to extract the primary key, constraints, and + secondary key. his is useful for optimizing the import process, to delay + secondary index creation until after the data has been loaded. +*/ + +#include +#include +#include + +#include "import_util.h" +#include + +/** + * Extract the first CREATE TABLE statement from a script. + * + * @param script The input script containing SQL statements. + * @return std::string The first CREATE TABLE statement found, or an empty + * string if not found. + */ +std::string extract_first_create_table(const std::string &script) +{ + regex_t create_table_regex; + regmatch_t match[2]; + const char *pattern= "(CREATE\\s+TABLE\\s+[^;]+;)\\s*\\n"; + regcomp(&create_table_regex, pattern, REG_EXTENDED); + + if (regexec(&create_table_regex, script.c_str(), 2, match, 0) == 0) + { + std::string result= + script.substr(match[1].rm_so, match[1].rm_eo - match[1].rm_so); + regfree(&create_table_regex); + return result; + } + + regfree(&create_table_regex); + return ""; +} + +TableDDLInfo::TableDDLInfo(const std::string &create_table_stmt) +{ + regex_t primary_key_regex, constraint_regex, index_regex, engine_regex, + table_name_regex; + constexpr size_t MAX_MATCHES= 10; + regmatch_t match[10]; + + regcomp(&primary_key_regex, "\\n\\s*(PRIMARY\\s+KEY\\s+(.*?)),?\\n", + REG_EXTENDED); + regcomp(&constraint_regex, + "\\n\\s*(CONSTRAINT\\s+(`?(?:[^`]|``)+`?)\\s+.*?),?\\n", + REG_EXTENDED); + regcomp(&index_regex, + "\\n\\s*(((?:UNIQUE|FULLTEXT|VECTOR|SPATIAL)\\s+)?(INDEX|KEY)\\s+(`(?:[^`]|``)+`)\\s+.*?),?\\n", + REG_EXTENDED); + regcomp(&engine_regex, "\\bENGINE\\s*=\\s*(\\w+)", REG_EXTENDED); + regcomp(&table_name_regex, "CREATE\\s+TABLE\\s+(`?(?:[^`]|``)+`?)\\s*\\(", + REG_EXTENDED); + + const char *stmt= create_table_stmt.c_str(); + const char *search_start= stmt; + + // Extract primary key + if (regexec(&primary_key_regex, search_start, MAX_MATCHES, match, 0) == 0) + { + primary_key= {std::string(stmt + match[1].rm_so, match[1].rm_eo - match[1].rm_so), + "PRIMARY"}; + } + + // Extract constraints and foreign keys + search_start= stmt; + while (regexec(&constraint_regex, search_start, MAX_MATCHES, match, 0) == 0) + { + assert(match[2].rm_so != -1); + assert(match[1].rm_so != -1); + std::string name(search_start + match[2].rm_so, match[2].rm_eo - match[2].rm_so); + std::string definition(search_start + match[1].rm_so, match[1].rm_eo - match[1].rm_so); + constraints.push_back({definition, name}); + search_start+= match[0].rm_eo - 1; + } + + // Extract secondary indexes + search_start= stmt; + while (regexec(&index_regex, search_start, MAX_MATCHES, match, 0) == 0) + { + assert(match[4].rm_so != -1); + std::string name(search_start + match[4].rm_so, match[4].rm_eo - match[4].rm_so); + std::string definition(search_start + match[1].rm_so, match[1].rm_eo - match[1].rm_so); + secondary_indexes.push_back({definition, name}); + search_start+= match[0].rm_eo -1; + } + + // Extract storage engine + if (regexec(&engine_regex, stmt, MAX_MATCHES, match, 0) == 0) + { + storage_engine= std::string(stmt + match[1].rm_so, match[1].rm_eo - match[1].rm_so); + } + + // Extract table name + if (regexec(&table_name_regex, stmt, MAX_MATCHES, match, 0) == 0) + { + table_name= std::string(stmt + match[1].rm_so, match[1].rm_eo - match[1].rm_so); + } + if (primary_key.definition.empty() && storage_engine == "InnoDB") + { + for (const auto &index : secondary_indexes) + { + if (index.definition.find("UNIQUE") != std::string::npos) + { + non_pk_clustering_key_name= index.name; + break; + } + } + } + regfree(&primary_key_regex); + regfree(&constraint_regex); + regfree(&index_regex); + regfree(&engine_regex); + regfree(&table_name_regex); +} + +/** + Convert a KeyOrConstraintDefinitionType enum value to its + corresponding string representation. + + @param type The KeyOrConstraintDefinitionType enum value. + @return std::string The string representation of the + KeyOrConstraintDefinitionType. +*/ +static std::string to_string(KeyOrConstraintType type) +{ + switch (type) + { + case KeyOrConstraintType::CONSTRAINT: + return "CONSTRAINT"; + case KeyOrConstraintType::INDEX: + return "INDEX"; + default: + return "UNKNOWN"; + } +} + +std::string TableDDLInfo::generate_alter_add( + const std::vector &definitions, + KeyOrConstraintType type) const +{ + if (definitions.empty() || + (type == KeyOrConstraintType::INDEX && definitions.size() == 1 + && !non_pk_clustering_key_name.empty())) + { + return ""; + } + + std::string sql= "ALTER TABLE " + table_name + " "; + bool need_comma= false; + for (const auto &definition : definitions) + { + /* + Do not add or drop clustering secondary index + */ + if (type == KeyOrConstraintType::INDEX && + definition.name == non_pk_clustering_key_name) + continue; + + if (need_comma) + sql+= ", "; + else + need_comma= true; + sql+= "ADD " + definition.definition; + } + return sql; +} + +std::string TableDDLInfo::generate_alter_drop( + const std::vector &definitions, KeyOrConstraintType type) const +{ + if (definitions.empty() || + (type == KeyOrConstraintType::INDEX && definitions.size() == 1 && + !non_pk_clustering_key_name.empty())) + { + return ""; + } + + std::string sql= "ALTER TABLE " + table_name + " "; + bool need_comma= false; + for (const auto &definition : definitions) + { + if (type == KeyOrConstraintType::INDEX && + definition.name == non_pk_clustering_key_name) + continue; + + if (need_comma) + sql+= ", "; + else + need_comma= true; + sql+= "DROP " + to_string(type) + " " + + definition.name; + } + return sql; +} diff --git a/client/import_util.h b/client/import_util.h new file mode 100644 index 00000000000..0096dd354fb --- /dev/null +++ b/client/import_util.h @@ -0,0 +1,89 @@ +/* + Copyright (c) 2024, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA +*/ + +#pragma once +#include +#include + +/* TABLE DDL INFO - representation of parsed CREATE TABLE Statement */ + +enum class KeyOrConstraintType +{ + CONSTRAINT, + INDEX, + UNKNOWN +}; + +/** + * Struct representing a table keyor constraint definition + */ +struct KeyDefinition +{ + /** Full key or constraint definition string, + e.g UNIQUE KEY `uniq_idx` (`col`) */ + std::string definition; + /** The name of key or constraint, including escape chars */ + std::string name; +}; + +/** + Information about keys and constraints, extracted from + CREATE TABLE statement + */ +struct TableDDLInfo +{ + TableDDLInfo(const std::string &create_table_stmt); + KeyDefinition primary_key; + std::vector constraints; + std::vector secondary_indexes; + std::string storage_engine; + std::string table_name; + /* Innodb is using first UNIQUE key for clustering, if no PK is set*/ + std::string non_pk_clustering_key_name; + + /** + Generate ALTER TABLE ADD/DROP statements for keys or constraints. + The goal is to remove indexes/constraints before the data is imported + and recreate them after import. + PRIMARY key is not affected by these operations + */ + std::string generate_alter_add(const std::vector &defs, + KeyOrConstraintType type) const; + std::string generate_alter_drop(const std::vector &defs, + KeyOrConstraintType type) const; + + + std::string drop_constraints_sql() const + { + return generate_alter_drop(constraints, KeyOrConstraintType::CONSTRAINT); + } + std::string add_constraints_sql() const + { + return generate_alter_add(constraints, KeyOrConstraintType::CONSTRAINT); + } + std::string drop_secondary_indexes_sql() const + { + return generate_alter_drop(secondary_indexes, + KeyOrConstraintType::INDEX); + } + std::string add_secondary_indexes_sql() const + { + return generate_alter_add(secondary_indexes, + KeyOrConstraintType::INDEX); + } +}; +std::string extract_first_create_table(const std::string &script); diff --git a/client/mysqlimport.cc b/client/mysqlimport.cc index 8be814d21b2..4f78aad2518 100644 --- a/client/mysqlimport.cc +++ b/client/mysqlimport.cc @@ -43,6 +43,7 @@ #include #include #include +#include "import_util.h" tpool::thread_pool *thread_pool; static std::vector all_tp_connections; @@ -55,7 +56,10 @@ static char *field_escape(char *to,const char *from,uint length); static char *add_load_option(char *ptr,const char *object, const char *statement); -static my_bool verbose=0,lock_tables=0,ignore_errors=0,opt_delete=0, +static std::string parse_sql_script(const char *filepath, bool *tz_utc, + std::vector *trigger_defs); + +static my_bool verbose=0,lock_tables=0,ignore_errors=0,opt_delete=0, replace, silent, ignore, ignore_foreign_keys, opt_compress, opt_low_priority, tty_password; static my_bool debug_info_flag= 0, debug_check_flag= 0; @@ -70,6 +74,7 @@ static char * opt_mysql_unix_port=0; static char *opt_plugin_dir= 0, *opt_default_auth= 0; static longlong opt_ignore_lines= -1; static char *opt_dir; +static char opt_innodb_optimize_keys; #include @@ -82,9 +87,26 @@ struct table_load_params std::string data_file; /* name of the file to load with LOAD DATA INFILE */ std::string sql_file; /* name of the file that contains CREATE TABLE or CREATE VIEW */ - std::string tablename; /* name of the table */ std::string dbname; /* name of the database */ + bool tz_utc= false; /* true if the script sets the timezone to UTC */ + bool is_view= false; /* true if the script is for a VIEW */ + std::vector triggers; /* CREATE TRIGGER statements */ ulonglong size= 0; /* size of the data file */ + std::string sql_text; /* content of the SQL file, without triggers */ + TableDDLInfo ddl_info; /* parsed CREATE TABLE statement */ + + table_load_params(const char* dfile, const char* sqlfile, + const char* db, ulonglong data_size) + : data_file(dfile), sql_file(sqlfile), + dbname(db), triggers(), + size(data_size), + sql_text(parse_sql_script(sqlfile, &tz_utc, &triggers)), + ddl_info(sql_text) + { + is_view= ddl_info.table_name.empty(); + } + int create_table_or_view(MYSQL *); + int load_data(MYSQL *); }; std::unordered_set ignore_databases; @@ -173,6 +195,9 @@ static struct my_option my_long_options[] = "--ignore-table=database.table. Only takes effect when used together with " "--dir option", 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb-optimize-keys", 0, "Create secondary indexes after data load (Innodb only).", + &opt_innodb_optimize_keys, &opt_innodb_optimize_keys, 0, GET_BOOL, NO_ARG, + 1, 0, 0, 0, 0, 0}, {"lines-terminated-by", 0, "Lines in the input file are terminated by the given string.", &lines_terminated, &lines_terminated, 0, GET_STR, @@ -512,8 +537,7 @@ static int exec_sql(MYSQL *mysql, const std::string& s) @return content of the file as a string, excluding CREATE TRIGGER statements */ -static std::string parse_sql_script(const char *filepath, bool *tz_utc, std::vector *create_trigger_Defs, - std::string *engine) +static std::string parse_sql_script(const char *filepath, bool *tz_utc, std::vector *create_trigger_Defs) { /*Read full file to string*/ std::ifstream t(filepath); @@ -552,16 +576,6 @@ static std::string parse_sql_script(const char *filepath, bool *tz_utc, std::vec with --tz-utc option */ *tz_utc= sql_text.find("SET TIME_ZONE='+00:00'") != std::string::npos; - - *engine= ""; - auto engine_pos= sql_text.find("ENGINE="); - if (engine_pos != std::string::npos) - { - engine_pos+= 7; - auto end_pos= sql_text.find_first_of(" ", engine_pos); - if (end_pos != std::string::npos) - *engine= sql_text.substr(engine_pos, end_pos - engine_pos); - } return sql_text; } @@ -584,64 +598,88 @@ static int create_db_if_not_exists(MYSQL *mysql, const char *dbname) return 0; } -static int handle_one_table(const table_load_params *params, MYSQL *mysql) + +#include "import_util.h" + +int table_load_params::create_table_or_view(MYSQL* mysql) +{ + if (sql_file.empty()) + return 0; + + if (verbose && !sql_file.empty()) + { + fprintf(stdout, "Executing SQL script %s\n", sql_file.c_str()); + } + if (!dbname.empty()) + { + if (mysql_select_db(mysql, dbname.c_str())) + { + if (create_db_if_not_exists(mysql, dbname.c_str())) + return 1; + if (mysql_select_db(mysql, dbname.c_str())) + { + db_error(mysql); + return 1; + } + } + } + if (sql_text.empty()) + { + fprintf(stderr, "Error: CREATE TABLE statement not found in %s\n", + sql_file.c_str()); + return 1; + } + if (execute_sql_batch(mysql, sql_text.c_str(), sql_file.c_str())) + return 1; + /* + Temporarily drop from table definitions, if --innodb-optimize-keys is given. We'll add them back + later, after the data is loaded. +*/ + auto drop_constraints_sql= ddl_info.drop_constraints_sql(); + if (!drop_constraints_sql.empty()) + { + if (exec_sql(mysql, drop_constraints_sql)) + return 1; + } + return 0; +} + +int table_load_params::load_data(MYSQL *mysql) { char tablename[FN_REFLEN], hard_path[FN_REFLEN], escaped_name[FN_REFLEN * 2 + 1], sql_statement[FN_REFLEN*16+256], *end; - DBUG_ENTER("handle_one_table"); - DBUG_PRINT("enter",("datafile: %s",params->data_file.c_str())); + DBUG_ENTER("table_load_params::load"); + DBUG_PRINT("enter",("datafile: %s",data_file.c_str())); + DBUG_ASSERT(!dbname.empty()); + + if (data_file.empty()) + DBUG_RETURN(0); if (aborting) - return 1; - if (verbose && !params->sql_file.empty()) + DBUG_RETURN(0); + + if (!dbname.empty() && mysql_select_db(mysql, dbname.c_str())) { - fprintf(stdout, "Executing SQL script %s\n", params->sql_file.c_str()); + db_error(mysql); + DBUG_RETURN(1); } - if (!params->dbname.empty()) - { - if (mysql_select_db(mysql, params->dbname.c_str())) - { - if (create_db_if_not_exists(mysql, params->dbname.c_str())) - DBUG_RETURN(1); - if (mysql_select_db(mysql, params->dbname.c_str())) - db_error(mysql); - } - } - - const char *filename= params->data_file.c_str(); - if (!filename[0]) - filename= params->sql_file.c_str(); + const char *filename= data_file.c_str(); fn_format(tablename, filename, "", "", 1 | 2); /* removes path & ext. */ - const char *db= current_db ? current_db : params->dbname.c_str(); + const char *db= current_db ? current_db : dbname.c_str(); std::string full_tablename= quote_identifier(db); full_tablename+= "."; full_tablename+= quote_identifier(tablename); - bool tz_utc= false; - std::string engine; - std::vector triggers; - if (!params->sql_file.empty()) - { - std::string sql_text= parse_sql_script(params->sql_file.c_str(), &tz_utc, &triggers,&engine); - if (execute_sql_batch(mysql, sql_text.c_str(),params->sql_file.c_str())) - DBUG_RETURN(1); - if (params->data_file.empty()) - { - /* - We only use .sql extension for VIEWs, so we're done - with this file, there is no data to load. - */ - DBUG_RETURN(0); - } - if (tz_utc && exec_sql(mysql, "SET TIME_ZONE='+00:00';")) - DBUG_RETURN(1); - if (exec_sql(mysql, std::string("ALTER TABLE ") + full_tablename + " DISABLE KEYS")) - DBUG_RETURN(1); - } + if (tz_utc && exec_sql(mysql, "SET TIME_ZONE='+00:00';")) + DBUG_RETURN(1); + if (exec_sql(mysql, + std::string("ALTER TABLE ") + full_tablename + " DISABLE KEYS")) + DBUG_RETURN(1); + if (!opt_local_file) strmov(hard_path,filename); else @@ -656,9 +694,21 @@ static int handle_one_table(const table_load_params *params, MYSQL *mysql) if (exec_sql(mysql, sql_statement)) DBUG_RETURN(1); } + + + bool recreate_secondary_keys= false; + if (opt_innodb_optimize_keys && ddl_info.storage_engine == "InnoDB") + { + auto drop_secondary_keys_sql= ddl_info.drop_secondary_indexes_sql(); + if (!drop_secondary_keys_sql.empty()) + { + recreate_secondary_keys= true; + if (exec_sql(mysql, drop_secondary_keys_sql)) + DBUG_RETURN(1); + } + } if (exec_sql(mysql, "SET collation_database=binary")) DBUG_RETURN(1); - to_unix_path(hard_path); if (verbose) { @@ -707,34 +757,50 @@ static int handle_one_table(const table_load_params *params, MYSQL *mysql) fprintf(stdout, "%s.%s: %s\n", db, tablename, info); } - /* Create triggers after loading data */ - for (const auto &trigger: triggers) + + if (exec_sql(mysql, std::string("ALTER TABLE ") + full_tablename + " ENABLE KEYS;")) + DBUG_RETURN(1); + + if (ddl_info.storage_engine == "MyISAM" || ddl_info.storage_engine == "Aria") { - if (mysql_query(mysql,trigger.c_str())) - { - db_error_with_table(mysql, tablename); + /* Avoid "table was not properly closed" warnings */ + if (exec_sql(mysql, std::string("FLUSH TABLE ").append(full_tablename).c_str())) DBUG_RETURN(1); - } } - - if (!params->sql_file.empty()) + if (recreate_secondary_keys) { - if (exec_sql(mysql, std::string("ALTER TABLE ") + full_tablename + " ENABLE KEYS;")) - DBUG_RETURN(1); - - if (engine == "MyISAM" || engine == "Aria") + auto create_secondary_keys_sql= ddl_info.add_secondary_indexes_sql(); + if (!create_secondary_keys_sql.empty()) { - /* Avoid "table was not properly closed" warnings */ - if (exec_sql(mysql, std::string("FLUSH TABLE ").append(full_tablename).c_str())) + if (verbose) + { + fprintf(stdout, "Adding secondary indexes to table %s\n", ddl_info.table_name.c_str()); + } + if (exec_sql(mysql, create_secondary_keys_sql)) DBUG_RETURN(1); } } - if (tz_utc) { if (exec_sql(mysql, "SET TIME_ZONE=@save_tz;")) DBUG_RETURN(1); } + /* Restore constrains and triggers */ + for (const auto &create_trigger_def : triggers) + { + if (exec_sql(mysql, create_trigger_def)) + return 1; + } + + if (opt_innodb_optimize_keys && ddl_info.storage_engine == "InnoDB") + { + std::string constraints= ddl_info.add_constraints_sql(); + if (!constraints.empty()) + { + if (exec_sql(mysql, constraints)) + return 1; + } + } DBUG_RETURN(0); } @@ -921,8 +987,10 @@ static void db_error(MYSQL *mysql) const char *info= mysql_info(mysql); auto err= mysql_errno(mysql); auto err_text = mysql_error(mysql); - - my_printf_error(0,"Error: %d %s %s", MYF(0), err, err_text, info); + if (info) + my_printf_error(0,"Error: %d %s %s", MYF(0), err, err_text, info); + else + my_printf_error(0, "Error %d %s", MYF(0), err, err_text); safe_exit(1, mysql); } @@ -986,10 +1054,11 @@ void set_exitcode(int code) static thread_local MYSQL *thread_local_mysql; -void load_single_table(void *arg) +static void load_single_table(void *arg) { int error; - if((error= handle_one_table((const table_load_params *) arg, thread_local_mysql))) + table_load_params *params= (table_load_params *) arg; + if ((error= params->load_data(thread_local_mysql))) set_exitcode(error); } @@ -1050,8 +1119,7 @@ static void tpool_thread_exit(void) @note files are sorted by size, descending */ static void scan_backup_dir(const char *dir, - std::vector &files, - std::vector &views) + std::vector &files) { MY_DIR *dir_info; std::vector subdirs; @@ -1095,8 +1163,6 @@ static void scan_backup_dir(const char *dir, } for (size_t j= 0; j < dir_info2->number_of_files; j++) { - table_load_params par; - par.dbname= dbname; fi= &dir_info2->dir_entry[j]; if (has_extension(fi->name, ".sql") || has_extension(fi->name, ".txt")) { @@ -1123,13 +1189,14 @@ static void scan_backup_dir(const char *dir, /* test file*/ if (has_extension(fi->name, ".txt")) { - par.data_file= file; - par.size= fi->mystat->st_size; - par.sql_file= file.substr(0, file.size() - 4) + ".sql"; - if (access(par.sql_file.c_str(), F_OK)) + auto sql_file= file.substr(0, file.size() - 4) + ".sql"; + if (access(sql_file.c_str(), F_OK)) { - fatal_error("Expected file '%s' is missing",par.sql_file.c_str()); + fatal_error("Expected file '%s' is missing",sql_file.c_str()); } + table_load_params par(file.c_str(), sql_file.c_str(), dbname, + fi->mystat->st_size); + files.push_back(par); } else if (has_extension(fi->name, ".sql")) @@ -1141,9 +1208,9 @@ static void scan_backup_dir(const char *dir, std::string txt_file= file.substr(0, file.size() - 4) + ".txt"; if (access(txt_file.c_str(), F_OK)) { - par.sql_file= file; - par.size= fi->mystat->st_size; - views.push_back(par); + table_load_params par("", file.c_str(), dbname, + fi->mystat->st_size); + files.push_back(par); } } else @@ -1160,16 +1227,17 @@ static void scan_backup_dir(const char *dir, std::sort(files.begin(), files.end(), [](const table_load_params &a, const table_load_params &b) -> bool { + /* Sort views after base tables */ + if (a.is_view && !b.is_view) + return false; + if (!a.is_view && b.is_view) + return true; + /* Sort by size descending */ if (a.size > b.size) return true; if (a.size < b.size) return false; - return a.sql_file < b.sql_file; - }); - - std::sort(views.begin(), views.end(), - [](const table_load_params &a, const table_load_params &b) -> bool - { + /* If sizes are equal, sort by name */ return a.sql_file < b.sql_file; }); } @@ -1193,79 +1261,79 @@ int main(int argc, char **argv) free_defaults(argv_to_free); return(1); } - + if (opt_use_threads > MAX_THREADS) + { + fatal_error("Too many connections, max value for --parallel is %d\n", + MAX_THREADS); + } sf_leaking_memory=0; /* from now on we cleanup properly */ - std::vector files_to_load, views_to_load; + std::vector files_to_load; if (opt_dir) { ignore_foreign_keys= 1; if (argc) fatal_error("Invalid arguments for --dir option"); - scan_backup_dir(opt_dir, files_to_load, views_to_load); + scan_backup_dir(opt_dir, files_to_load); } else { for (; *argv != NULL; argv++) { - table_load_params p{}; - p.data_file= *argv; + table_load_params p(*argv, "", current_db, 0); files_to_load.push_back(p); } } + if (files_to_load.empty()) + { + fatal_error("No files to load"); + return 1; + } + MYSQL *mysql= + db_connect(current_host, current_db, current_user, opt_password); + if (!mysql) + { + free_defaults(argv_to_free); + return 1; + } + for (auto &f : files_to_load) + { + if (f.create_table_or_view(mysql)) + set_exitcode(1); + } if (opt_use_threads && !lock_tables) { - if (opt_use_threads > MAX_THREADS) - { - fatal_error("Too many connections, max value for --parallel is %d\n", - MAX_THREADS); - } init_tp_connections(opt_use_threads); thread_pool= tpool::create_thread_pool_generic(opt_use_threads,opt_use_threads); thread_pool->set_thread_callbacks(tpool_thread_init,tpool_thread_exit); - std::vector all_tasks; - for (const auto &f: files_to_load) - all_tasks.push_back(tpool::task(load_single_table, (void *)&f)); + std::vector load_tasks; + for (const auto &f : files_to_load) + { + load_tasks.push_back(tpool::task(load_single_table, (void *) &f)); + } - for (auto &t: all_tasks) + for (auto &t: load_tasks) thread_pool->submit_task(&t); delete thread_pool; + close_tp_connections(); thread_pool= nullptr; - files_to_load.clear(); } - /* - The following block handles single-threaded load. - Also views that must be created after the base tables, are created here. - - BUG: funny case would be views that select from other views, won't generally work. - It won't work in mysqldump either, but it's not a common case. - */ - if (!files_to_load.empty() || !views_to_load.empty()) + else { - MYSQL *mysql= db_connect(current_host,current_db,current_user,opt_password); - if (!mysql) - { - free_defaults(argv_to_free); - return(1); /* purecov: dead code */ - } - if (lock_tables) lock_table(mysql, argc, argv); - for (const auto &f : files_to_load) - if ((error= handle_one_table(&f, mysql))) - set_exitcode(error); - - for (const auto &v : views_to_load) - if ((error= handle_one_table(&v, mysql))) + for (auto &f : files_to_load) + { + if ((error= f.load_data(mysql))) set_exitcode(error); - - db_disconnect(current_host, mysql); + } } + mysql_close(mysql); safe_exit(0, 0); return(exitcode); } diff --git a/mysql-test/main/mariadb-import.result b/mysql-test/main/mariadb-import.result index aa683070eac..8fe1c57fd63 100644 --- a/mysql-test/main/mariadb-import.result +++ b/mysql-test/main/mariadb-import.result @@ -77,89 +77,88 @@ test_suppressions.sql test_suppressions.txt Connecting to localhost Executing SQL script vardir/tmp/dump/mysql/help_topic.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/help_topic.txt into help_topic -mysql.help_topic: Records: 839 Deleted: 0 Skipped: 0 Warnings: 0 Executing SQL script vardir/tmp/dump/mysql/time_zone_transition.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_transition.txt into time_zone_transition -mysql.time_zone_transition: Records: 394 Deleted: 0 Skipped: 0 Warnings: 0 Executing SQL script vardir/tmp/dump/mtr/global_suppressions.sql -Loading data from LOCAL file: vardir/tmp/dump/mtr/global_suppressions.txt into global_suppressions -mtr.global_suppressions: Records: 99 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/help_keyword.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/help_keyword.txt into help_keyword -mysql.help_keyword: Records: 106 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/help_relation.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/help_relation.txt into help_relation -mysql.help_relation: Records: 202 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/help_category.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/help_category.txt into help_category -mysql.help_category: Records: 50 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/time_zone_transition_type.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_transition_type.txt into time_zone_transition_type -mysql.time_zone_transition_type: Records: 32 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/global_priv.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/global_priv.txt into global_priv -mysql.global_priv: Records: 5 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/time_zone_leap_second.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_leap_second.txt into time_zone_leap_second -mysql.time_zone_leap_second: Records: 23 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/proxies_priv.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/proxies_priv.txt into proxies_priv -mysql.proxies_priv: Records: 4 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/tables_priv.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/tables_priv.txt into tables_priv -mysql.tables_priv: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/time_zone_name.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_name.txt into time_zone_name -mysql.time_zone_name: Records: 7 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/time_zone.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone.txt into time_zone -mysql.time_zone: Records: 6 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/test/t1.sql -Loading data from LOCAL file: vardir/tmp/dump/test/t1.txt into t1 -test.t1: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/column_stats.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/column_stats.txt into column_stats -mysql.column_stats: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/columns_priv.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/columns_priv.txt into columns_priv -mysql.columns_priv: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/db.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/db.txt into db -mysql.db: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/func.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/func.txt into func -mysql.func: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/gtid_slave_pos.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/gtid_slave_pos.txt into gtid_slave_pos -mysql.gtid_slave_pos: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/index_stats.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/index_stats.txt into index_stats -mysql.index_stats: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/plugin.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/plugin.txt into plugin -mysql.plugin: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/procs_priv.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/procs_priv.txt into procs_priv -mysql.procs_priv: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/roles_mapping.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/roles_mapping.txt into roles_mapping -mysql.roles_mapping: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/servers.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/servers.txt into servers -mysql.servers: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/mysql/table_stats.sql -Loading data from LOCAL file: vardir/tmp/dump/mysql/table_stats.txt into table_stats -mysql.table_stats: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 Executing SQL script vardir/tmp/dump/mysql/event.sql -Executing SQL script vardir/tmp/dump/mysql/general_log.sql Executing SQL script vardir/tmp/dump/mysql/innodb_index_stats.sql Executing SQL script vardir/tmp/dump/mysql/innodb_table_stats.sql -Executing SQL script vardir/tmp/dump/mysql/slow_log.sql -Executing SQL script vardir/tmp/dump/mysql/transaction_registry.sql +Executing SQL script vardir/tmp/dump/mysql/help_keyword.sql +Executing SQL script vardir/tmp/dump/mysql/help_relation.sql +Executing SQL script vardir/tmp/dump/mysql/help_category.sql +Executing SQL script vardir/tmp/dump/mysql/time_zone_transition_type.sql +Executing SQL script vardir/tmp/dump/mysql/global_priv.sql +Executing SQL script vardir/tmp/dump/mysql/time_zone_leap_second.sql +Executing SQL script vardir/tmp/dump/mysql/proxies_priv.sql +Executing SQL script vardir/tmp/dump/mysql/tables_priv.sql +Executing SQL script vardir/tmp/dump/mysql/time_zone_name.sql +Executing SQL script vardir/tmp/dump/mysql/time_zone.sql +Executing SQL script vardir/tmp/dump/test/t1.sql +Executing SQL script vardir/tmp/dump/mysql/column_stats.sql +Executing SQL script vardir/tmp/dump/mysql/columns_priv.sql +Executing SQL script vardir/tmp/dump/mysql/db.sql +Executing SQL script vardir/tmp/dump/mysql/func.sql +Executing SQL script vardir/tmp/dump/mysql/gtid_slave_pos.sql +Executing SQL script vardir/tmp/dump/mysql/index_stats.sql +Executing SQL script vardir/tmp/dump/mysql/plugin.sql +Executing SQL script vardir/tmp/dump/mysql/procs_priv.sql +Executing SQL script vardir/tmp/dump/mysql/roles_mapping.sql +Executing SQL script vardir/tmp/dump/mysql/servers.sql +Executing SQL script vardir/tmp/dump/mysql/table_stats.sql Executing SQL script vardir/tmp/dump/mysql/user.sql +Executing SQL script vardir/tmp/dump/mysql/transaction_registry.sql +Executing SQL script vardir/tmp/dump/mysql/slow_log.sql Executing SQL script vardir/tmp/dump/test/v1.sql -Disconnecting from localhost +Executing SQL script vardir/tmp/dump/mysql/general_log.sql +Loading data from LOCAL file: vardir/tmp/dump/mysql/help_topic.txt into help_topic +mysql.help_topic: Records: 839 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_transition.txt into time_zone_transition +mysql.time_zone_transition: Records: 394 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mtr/global_suppressions.txt into global_suppressions +mtr.global_suppressions: Records: 99 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/help_keyword.txt into help_keyword +mysql.help_keyword: Records: 106 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/help_relation.txt into help_relation +mysql.help_relation: Records: 202 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/help_category.txt into help_category +mysql.help_category: Records: 50 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_transition_type.txt into time_zone_transition_type +mysql.time_zone_transition_type: Records: 32 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/global_priv.txt into global_priv +mysql.global_priv: Records: 5 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_leap_second.txt into time_zone_leap_second +mysql.time_zone_leap_second: Records: 23 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/proxies_priv.txt into proxies_priv +mysql.proxies_priv: Records: 4 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/tables_priv.txt into tables_priv +mysql.tables_priv: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone_name.txt into time_zone_name +mysql.time_zone_name: Records: 7 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/time_zone.txt into time_zone +mysql.time_zone: Records: 6 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/test/t1.txt into t1 +test.t1: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/column_stats.txt into column_stats +mysql.column_stats: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/columns_priv.txt into columns_priv +mysql.columns_priv: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/db.txt into db +mysql.db: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/func.txt into func +mysql.func: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/gtid_slave_pos.txt into gtid_slave_pos +mysql.gtid_slave_pos: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/index_stats.txt into index_stats +mysql.index_stats: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/plugin.txt into plugin +mysql.plugin: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/procs_priv.txt into procs_priv +mysql.procs_priv: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/roles_mapping.txt into roles_mapping +mysql.roles_mapping: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/servers.txt into servers +mysql.servers: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from LOCAL file: vardir/tmp/dump/mysql/table_stats.txt into table_stats +mysql.table_stats: Records: 0 Deleted: 0 Skipped: 0 Warnings: 0 drop table t1; drop view v1; create database db2; @@ -171,13 +170,35 @@ PRIMARY KEY (id) CREATE TABLE child ( id INT, parent_id INT, +c CHAR(4), INDEX par_ind (parent_id), +UNIQUE INDEX(c), FOREIGN KEY (parent_id) REFERENCES parent(id) -ON DELETE CASCADE +ON DELETE CASCADE, +CHECK (c >= 'a') ) ENGINE=INNODB; insert into parent values(1),(2); -insert into child values (1,1),(1,2),(2,1),(2,2); +insert into child values (1,1,'a'),(1,2,'b'),(2,1,'c'),(2,2,'d'); +CREATE TABLE offices ( +id int NOT NULL AUTO_INCREMENT, +PRIMARY KEY (id) +) ENGINE=InnoDB; +CREATE TABLE users ( +id int NOT NULL AUTO_INCREMENT, +office_id int DEFAULT NULL, +slogan text GENERATED ALWAYS AS (concat('Hello world #',office_id)) STORED, +PRIMARY KEY (id), +KEY office_id (office_id), +CONSTRAINT users_ibfk_1 FOREIGN KEY (office_id) REFERENCES offices (id) +) ENGINE=InnoDB; +insert into offices values(); +insert into offices values(); +insert into offices values(); +insert into offices values(); +insert into users (office_id) values (1); +insert into users (office_id) values (2); +insert into users (office_id) values (3); drop database db2; use db2; select * from parent; @@ -185,13 +206,116 @@ id 1 2 select * from child; -id parent_id -1 1 -1 2 -2 1 -2 2 -drop table child; -drop table parent; +id parent_id c +1 1 a +1 2 b +2 1 c +2 2 d +show create table parent; +Table Create Table +parent CREATE TABLE `parent` ( + `id` int(11) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +show create table child; +Table Create Table +child CREATE TABLE `child` ( + `id` int(11) DEFAULT NULL, + `parent_id` int(11) DEFAULT NULL, + `c` char(4) DEFAULT NULL, + UNIQUE KEY `c` (`c`), + KEY `par_ind` (`parent_id`), + CONSTRAINT `child_ibfk_1` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) ON DELETE CASCADE, + CONSTRAINT `CONSTRAINT_1` CHECK (`c` >= 'a') +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci +drop database db2; +# Repeat import with --verbose to see "Adding secondary keys" in the output +Connecting to localhost +Executing SQL script vardir/tmp/dump/db2/users.sql +Executing SQL script vardir/tmp/dump/db2/child.sql +Executing SQL script vardir/tmp/dump/db2/offices.sql +Executing SQL script vardir/tmp/dump/db2/parent.sql +Loading data from SERVER file: vardir/tmp/dump/db2/users.txt into users +db2.users: Records: 3 Deleted: 0 Skipped: 0 Warnings: 0 +Adding secondary indexes to table `users` +Loading data from SERVER file: vardir/tmp/dump/db2/child.txt into child +db2.child: Records: 4 Deleted: 0 Skipped: 0 Warnings: 0 +Adding secondary indexes to table `child` +Loading data from SERVER file: vardir/tmp/dump/db2/offices.txt into offices +db2.offices: Records: 4 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from SERVER file: vardir/tmp/dump/db2/parent.txt into parent +db2.parent: Records: 2 Deleted: 0 Skipped: 0 Warnings: 0 +# Repeat import with --verbose and --innodb-optimize-indexes=0, to "not" see "Adding secondary indexes" +Connecting to localhost +Executing SQL script vardir/tmp/dump/db2/users.sql +Executing SQL script vardir/tmp/dump/db2/child.sql +Executing SQL script vardir/tmp/dump/db2/offices.sql +Executing SQL script vardir/tmp/dump/db2/parent.sql +Loading data from SERVER file: vardir/tmp/dump/db2/users.txt into users +db2.users: Records: 3 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from SERVER file: vardir/tmp/dump/db2/child.txt into child +db2.child: Records: 4 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from SERVER file: vardir/tmp/dump/db2/offices.txt into offices +db2.offices: Records: 4 Deleted: 0 Skipped: 0 Warnings: 0 +Loading data from SERVER file: vardir/tmp/dump/db2/parent.txt into parent +db2.parent: Records: 2 Deleted: 0 Skipped: 0 Warnings: 0 +drop database db2; +create database db2; +use db2; +create table vec (id int auto_increment primary key, v vector(5) not null, +vector index (v)) ENGINE=InnoDB; +insert vec(v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'), +(x'f511303f72224a3fdd05fe3eb22a133ffae86a3f'), +(x'f09baa3ea172763f123def3e0c7fe53e288bf33e'), +(x'b97a523f2a193e3eb4f62e3f2d23583e9dd60d3f'), +(x'f7c5df3e984b2b3e65e59d3d7376db3eac63773e'), +(x'de01453ffa486d3f10aa4d3fdd66813c71cb163f'), +(x'76edfc3e4b57243f10f8423fb158713f020bda3e'), +(x'56926c3fdf098d3e2c8c5e3d1ad4953daa9d0b3e'), +(x'7b713f3e5258323f80d1113d673b2b3f66e3583f'), +(x'6ca1d43e9df91b3fe580da3e1c247d3f147cf33e'); +create table ft(v text, fulltext(v)) ENGINE=InnoDB; +insert into ft(v) values ('Once upon a time'), +('There was a wicked witch'), ('Who ate everybody up'); +create table locations (id int auto_increment primary key, geom geometry NOT NULL) ENGINE=InnoDB; +create spatial index idx_geom on locations (geom); +insert into locations (geom) values (ST_GeomFromText('POINT(40.785091 -73.968285)')); +# use --verbose to see "Adding secondary indexes" in the output +Connecting to localhost +Executing SQL script vardir/tmp/dump/db2/vec.sql +Executing SQL script vardir/tmp/dump/db2/ft.sql +Executing SQL script vardir/tmp/dump/db2/locations.sql +Loading data from SERVER file: vardir/tmp/dump/db2/vec.txt into vec +db2.vec: Records: 10 Deleted: 0 Skipped: 0 Warnings: 0 +Adding secondary indexes to table `vec` +Loading data from SERVER file: vardir/tmp/dump/db2/ft.txt into ft +db2.ft: Records: 3 Deleted: 0 Skipped: 0 Warnings: 0 +Adding secondary indexes to table `ft` +Loading data from SERVER file: vardir/tmp/dump/db2/locations.txt into locations +db2.locations: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0 +Adding secondary indexes to table `locations` +show index from vec; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored +vec 0 PRIMARY 1 id A 10 NULL NULL BTREE NO +vec 1 v 1 v A NULL NULL NULL VECTOR NO +show index from locations; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored +locations 0 PRIMARY 1 id A 0 NULL NULL BTREE NO +locations 1 idx_geom 1 geom A NULL 32 NULL SPATIAL NO +show index from ft; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored +ft 1 v 1 v NULL NULL NULL NULL YES FULLTEXT NO +select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from vec order by d limit 3; +id d +9 0.47199 +10 0.50690 +3 0.58656 +select * from ft where match(v) against('wicked'); +v +There was a wicked witch +drop database db2; +create database db2; +use db2; CREATE TABLE animals (id mediumint(9) NOT NULL AUTO_INCREMENT, name char(30) NOT NULL, @@ -216,12 +340,11 @@ use test; drop database db2; Connecting to localhost Executing SQL script vardir/tmp/dump/db2/animals.sql +Executing SQL script vardir/tmp/dump/db2/animal_count.sql Loading data from LOCAL file: vardir/tmp/dump/db2/animals.txt into animals db2.animals: Records: 2 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/db2/animal_count.sql Loading data from LOCAL file: vardir/tmp/dump/db2/animal_count.txt into animal_count db2.animal_count: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0 -Disconnecting from localhost use db2; # Content of tables after import select * from animals; @@ -239,10 +362,9 @@ use test; drop database db2; Connecting to localhost Executing SQL script vardir/tmp/dump/db2/t1.sql +Executing SQL script vardir/tmp/dump/db2/a1.sql Loading data from LOCAL file: vardir/tmp/dump/db2/t1.txt into t1 db2.t1: Records: 1 Deleted: 0 Skipped: 0 Warnings: 0 -Executing SQL script vardir/tmp/dump/db2/a1.sql -Disconnecting from localhost use db2; select * from t1; val diff --git a/mysql-test/main/mariadb-import.test b/mysql-test/main/mariadb-import.test index 5aee5304e13..4293c5b4496 100644 --- a/mysql-test/main/mariadb-import.test +++ b/mysql-test/main/mariadb-import.test @@ -28,7 +28,7 @@ select * from t1; # Test --dir --replace_result $MYSQLTEST_VARDIR vardir -# Ignore mtr.test_suppressions (may have suppressions or now), mysql.proc is smaller without perfschema/sys schema +# Ignore mtr.test_suppressions (may have suppressions or not), mysql.proc is smaller without perfschema/sys schema --exec $MYSQL_IMPORT --local --verbose --dir $MYSQLTEST_VARDIR/tmp/dump --ignore-table=mtr.test_suppressions --ignore-table=mysql.proc drop table t1; @@ -45,28 +45,108 @@ CREATE TABLE parent ( CREATE TABLE child ( id INT, parent_id INT, + c CHAR(4), INDEX par_ind (parent_id), + UNIQUE INDEX(c), FOREIGN KEY (parent_id) REFERENCES parent(id) - ON DELETE CASCADE + ON DELETE CASCADE, + CHECK (c >= 'a') ) ENGINE=INNODB; insert into parent values(1),(2); -insert into child values (1,1),(1,2),(2,1),(2,2); +insert into child values (1,1,'a'),(1,2,'b'),(2,1,'c'),(2,2,'d'); + +# Example from https://github.com/mydumper/mydumper/issues/395 (can't repeat now) +CREATE TABLE offices ( + id int NOT NULL AUTO_INCREMENT, + PRIMARY KEY (id) +) ENGINE=InnoDB; + +CREATE TABLE users ( + id int NOT NULL AUTO_INCREMENT, + office_id int DEFAULT NULL, + slogan text GENERATED ALWAYS AS (concat('Hello world #',office_id)) STORED, + PRIMARY KEY (id), + KEY office_id (office_id), + CONSTRAINT users_ibfk_1 FOREIGN KEY (office_id) REFERENCES offices (id) +) ENGINE=InnoDB; + +insert into offices values(); +insert into offices values(); +insert into offices values(); +insert into offices values(); + +insert into users (office_id) values (1); +insert into users (office_id) values (2); +insert into users (office_id) values (3); --mkdir $MYSQLTEST_VARDIR/tmp/dump --exec $MYSQL_DUMP --dir=$MYSQLTEST_VARDIR/tmp/dump --all-databases + drop database db2; --replace_result $MYSQLTEST_VARDIR vardir --exec $MYSQL_IMPORT --local --silent --dir $MYSQLTEST_VARDIR/tmp/dump --database=db2 --parallel=2 use db2; select * from parent; select * from child; -drop table child; -drop table parent; +show create table parent; +show create table child; +drop database db2; + +--echo # Repeat import with --verbose to see "Adding secondary keys" in the output +--replace_result $MYSQLTEST_VARDIR vardir +--exec $MYSQL_IMPORT --verbose --dir $MYSQLTEST_VARDIR/tmp/dump --database=db2 + +--echo # Repeat import with --verbose and --innodb-optimize-indexes=0, to "not" see "Adding secondary indexes" +--replace_result $MYSQLTEST_VARDIR vardir +--exec $MYSQL_IMPORT --verbose --dir $MYSQLTEST_VARDIR/tmp/dump --database=db2 --innodb-optimize-keys=0 --rmdir $MYSQLTEST_VARDIR/tmp/dump -# Test with triggers (using https://mariadb.com/kb/en/trigger-overview/ example) +drop database db2; +create database db2; +use db2; +# Test with vector/fulltext/spatial indexes +create table vec (id int auto_increment primary key, v vector(5) not null, + vector index (v)) ENGINE=InnoDB; +insert vec(v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'), + (x'f511303f72224a3fdd05fe3eb22a133ffae86a3f'), + (x'f09baa3ea172763f123def3e0c7fe53e288bf33e'), + (x'b97a523f2a193e3eb4f62e3f2d23583e9dd60d3f'), + (x'f7c5df3e984b2b3e65e59d3d7376db3eac63773e'), + (x'de01453ffa486d3f10aa4d3fdd66813c71cb163f'), + (x'76edfc3e4b57243f10f8423fb158713f020bda3e'), + (x'56926c3fdf098d3e2c8c5e3d1ad4953daa9d0b3e'), + (x'7b713f3e5258323f80d1113d673b2b3f66e3583f'), + (x'6ca1d43e9df91b3fe580da3e1c247d3f147cf33e'); +create table ft(v text, fulltext(v)) ENGINE=InnoDB; +insert into ft(v) values ('Once upon a time'), + ('There was a wicked witch'), ('Who ate everybody up'); +create table locations (id int auto_increment primary key, geom geometry NOT NULL) ENGINE=InnoDB; +create spatial index idx_geom on locations (geom); +insert into locations (geom) values (ST_GeomFromText('POINT(40.785091 -73.968285)')); +--mkdir $MYSQLTEST_VARDIR/tmp/dump +--exec $MYSQL_DUMP --dir=$MYSQLTEST_VARDIR/tmp/dump db2 +--echo # use --verbose to see "Adding secondary indexes" in the output +--replace_result $MYSQLTEST_VARDIR vardir +--exec $MYSQL_IMPORT --verbose --dir $MYSQLTEST_VARDIR/tmp/dump --database=db2 + +# smoke-test restored tables +show index from vec; +show index from locations; +show index from ft; + +--replace_regex /(\.\d{5})\d+/\1/ +select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from vec order by d limit 3; +select * from ft where match(v) against('wicked'); + +--rmdir $MYSQLTEST_VARDIR/tmp/dump + +drop database db2; +create database db2; +use db2; + +# Test with triggers (using https://mariadb.com/kb/en/trigger-overview/ example) CREATE TABLE animals (id mediumint(9) NOT NULL AUTO_INCREMENT, name char(30) NOT NULL, @@ -145,4 +225,3 @@ use test; --exec $MYSQL_IMPORT --dir $MYSQLTEST_VARDIR/tmp/dump --parallel=300 2>&1 --rmdir $MYSQLTEST_VARDIR/tmp/dump - diff --git a/mysql-test/main/mysqldump.result b/mysql-test/main/mysqldump.result index 47928866f2e..f7d10b0f7c4 100644 --- a/mysql-test/main/mysqldump.result +++ b/mysql-test/main/mysqldump.result @@ -4617,7 +4617,8 @@ Abernathy aberrant aberration drop table words; -mariadb-import: Error: 1146, Table 'test.words' doesn't exist, when using table: words +Error: 1146, Table 'test.words' doesn't exist, when using statement: ALTER TABLE `test`.`words` DISABLE KEYS +mariadb-import: Error 1146 Table 'test.words' doesn't exist drop table t1; drop table t2; drop table words2; diff --git a/unittest/client/CMakeLists.txt b/unittest/client/CMakeLists.txt new file mode 100644 index 00000000000..9e40329e5e4 --- /dev/null +++ b/unittest/client/CMakeLists.txt @@ -0,0 +1,3 @@ +ADD_EXECUTABLE(import_util-t import_util-t.cc) +TARGET_LINK_LIBRARIES(import_util-t PRIVATE import_util mytap) +ADD_TEST(import_util import_util-t) diff --git a/unittest/client/import_util-t.cc b/unittest/client/import_util-t.cc new file mode 100644 index 00000000000..2600cff10b2 --- /dev/null +++ b/unittest/client/import_util-t.cc @@ -0,0 +1,149 @@ +/* Copyright (c) 2024, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA + */ + +#include "my_config.h" +#include "import_util.h" + +#include +#include + +inline bool operator==(const KeyDefinition &lhs, const KeyDefinition &rhs) +{ + return lhs.definition == rhs.definition && lhs.name == rhs.name; +} + +/* + Test parsing of CREATE TABLE in mariadb-import utility +*/ +static void test_ddl_parser() +{ + std::string script= R"( + -- Some SQL script + CREATE TABLE `book` ( + `id` mediumint(8) unsigned NOT NULL AUTO_INCREMENT, + `title` varchar(200) NOT NULL, + `author_id` smallint(5) unsigned NOT NULL, + `publisher_id` smallint(5) unsigned NOT NULL, + `excerpt` text, + PRIMARY KEY (`id`), + KEY `fk_book_author` (`author_id`), + KEY `fk_book_publisher` (`publisher_id`), + UNIQUE KEY `title_author` (`title`,`author`), + FULLTEXT KEY `excerpt` (`excerpt`), + CONSTRAINT `fk_book_author` FOREIGN KEY (`author_id`) REFERENCES `author` (`id`) ON DELETE CASCADE + CONSTRAINT `fk_book_publisher` FOREIGN KEY (`publisher_id`) REFERENCES `publisher` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci; +)"; + + auto create_table_stmt= extract_first_create_table(script); + ok(!create_table_stmt.empty(), "CREATE TABLE statement found"); + + TableDDLInfo ddl_info(create_table_stmt); + + const std::string& table_name= ddl_info.table_name; + const std::string& storage_engine= ddl_info.storage_engine; + + ok(table_name == "`book`", "Table name is OK"); + ok(storage_engine == "InnoDB", "Storage engine is OK"); + ok(ddl_info.primary_key == KeyDefinition{"PRIMARY KEY (`id`)", "PRIMARY"}, + "Primary key def is OK"); + + ok(ddl_info.secondary_indexes.size() == 4, "Secondary index size is OK"); + const auto &sec_indexes= ddl_info.secondary_indexes; + ok(sec_indexes[0] == KeyDefinition{"KEY `fk_book_author` (`author_id`)","`fk_book_author`"}, + "First key is OK"); + ok(sec_indexes[1] == + KeyDefinition{"KEY `fk_book_publisher` (`publisher_id`)", + "`fk_book_publisher`"}, + "Second key is OK"); + + ok(ddl_info.constraints.size() == 2, "Constraints size correct"); + ok(ddl_info.constraints[0] == + KeyDefinition{"CONSTRAINT `fk_book_author` FOREIGN KEY (`author_id`) REFERENCES " + "`author` (`id`) ON DELETE CASCADE","`fk_book_author`"}, + "First constraint OK"); + + std::string drop_constraints= ddl_info.drop_constraints_sql(); + ok(drop_constraints == + "ALTER TABLE `book` DROP CONSTRAINT `fk_book_author`, DROP CONSTRAINT `fk_book_publisher`", + "Drop constraints SQL is \"%s\"", drop_constraints.c_str()); + std::string add_constraints= ddl_info.add_constraints_sql(); + ok(add_constraints == + "ALTER TABLE `book` ADD CONSTRAINT `fk_book_author` FOREIGN KEY (`author_id`) " + "REFERENCES `author` (`id`) ON DELETE CASCADE, " + "ADD CONSTRAINT `fk_book_publisher` FOREIGN KEY (`publisher_id`) " + "REFERENCES `publisher` (`id`) ON DELETE CASCADE", + "Add constraints SQL is \"%s\"",add_constraints.c_str()); + + std::string drop_secondary_indexes= + ddl_info.drop_secondary_indexes_sql(); + ok(drop_secondary_indexes == + "ALTER TABLE `book` " + "DROP INDEX `fk_book_author`, " + "DROP INDEX `fk_book_publisher`, " + "DROP INDEX `title_author`, " + "DROP INDEX `excerpt`", + "Drop secondary indexes SQL is \"%s\"", drop_secondary_indexes.c_str()); + std::string add_secondary_indexes= + ddl_info.add_secondary_indexes_sql(); + ok(add_secondary_indexes == + "ALTER TABLE `book` ADD KEY `fk_book_author` (`author_id`), " + "ADD KEY `fk_book_publisher` (`publisher_id`), " + "ADD UNIQUE KEY `title_author` (`title`,`author`), " + "ADD FULLTEXT KEY `excerpt` (`excerpt`)", + "Add secondary indexes SQL is \"%s\"", add_secondary_indexes.c_str()); +} + +/* + For Innodb table without PK, and but with Unique key + (which is used for clustering, instead of PK) + this key will not be added and dropped by + the import utility +*/ +static void innodb_non_pk_clustering_key() +{ + auto create_table_stmt= R"( + CREATE TABLE `book` ( + `id` mediumint(8), + `uniq` varchar(200), + UNIQUE KEY `id` (`id`), + UNIQUE KEY `uniq` (`uniq`), + KEY `id_uniq` (`id`,`uniq`) + ) ENGINE=InnoDB; + )"; + TableDDLInfo ddl_info(create_table_stmt); + ok(ddl_info.non_pk_clustering_key_name == "`id`", + "Non-PK clustering key is %s", + ddl_info.non_pk_clustering_key_name.c_str()); + ok(ddl_info.primary_key.definition.empty(), + "Primary key is %s", ddl_info.primary_key.definition.c_str()); + ok(ddl_info.secondary_indexes.size() == 3, + "Secondary indexes size is %zu", + ddl_info.secondary_indexes.size()); + ok(!ddl_info.add_secondary_indexes_sql().empty(), + "Some secondary indexes to add"); + ok(!ddl_info.drop_secondary_indexes_sql().empty(), + "Some secondary indexes to drop"); +} +int main() +{ + plan(18); + diag("Testing DDL parser"); + + test_ddl_parser(); + innodb_non_pk_clustering_key(); + return exit_status(); +} diff --git a/unittest/mytap/CMakeLists.txt b/unittest/mytap/CMakeLists.txt index 7bdb5b95afc..75906806bb5 100644 --- a/unittest/mytap/CMakeLists.txt +++ b/unittest/mytap/CMakeLists.txt @@ -15,3 +15,4 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include) ADD_LIBRARY(mytap tap.c) +TARGET_INCLUDE_DIRECTORIES(mytap PUBLIC ${CMAKE_SOURCE_DIR}/unittest/mytap)