mirror of
https://github.com/MariaDB/server.git
synced 2025-07-25 15:08:40 +00:00

This patch adds the ability to store extra field metadata in the table map event. This data can include pack_length() or field_lenght() for fields such as CHAR or VARCHAR enabling developers to add code that can check for compatibilty between master and slave columns. More importantly, the extra field metadata can be used to store data from the master correctly should a VARCHAR field on the master be <= 255 bytes while the same field on the slave is > 255 bytes. The patch also includes the needed changes to unpack to ensure that data which is smaller on the master can be unpacked correctly on the slave. WL#3915 : (NDB) master's cols > slave Slave starts accepting and handling rows of master's tables which have more columns. The most important part of implementation is how to caclulate the amount of bytes to skip for unknown by slave column. mysql-test/suite/binlog/t/binlog_row_mix_innodb_myisam.test: WL#3228 : RBR using different table defs on slave/master This patch changes the test to coincide with changes to binlog size of table map event. mysql-test/suite/rpl/r/rpl_row_basic_11bugs.result: WL#3228 : RBR using different table defs on slave/master This patch contains a new result file as a result of the change to the size of the tablemap log event. mysql-test/suite/rpl/r/rpl_row_create_table.result: WL#3228 : RBR using different table defs on slave/master This patch contains a new result file as a result of the change to the size of the tablemap log event. mysql-test/suite/rpl/r/rpl_row_flsh_tbls.result: WL#3228 : RBR using different table defs on slave/master This patch contains a new result file as a result of the change to the size of the tablemap log event. mysql-test/suite/rpl/r/rpl_row_inexist_tbl.result: WL#3228 : RBR using different table defs on slave/master This patch contains a new result file as a result of the change to the size of the tablemap log event. mysql-test/suite/rpl/r/rpl_row_log.result: WL#3228 : RBR using different table defs on slave/master This patch contains a new result file as a result of the change to the size of the tablemap log event. mysql-test/suite/rpl/r/rpl_row_log_innodb.result: WL#3228 : RBR using different table defs on slave/master This patch contains a new result file as a result of the change to the size of the tablemap log event. mysql-test/suite/rpl/r/rpl_row_max_relay_size.result: WL#3228 : RBR using different table defs on slave/master This patch contains a new result file as a result of the change to the size of the tablemap log event. mysql-test/suite/rpl/r/rpl_row_tabledefs_2myisam.result: WL#3228 : RBR using different table defs on slave/master This patch contains a new result file as a result of the change to the size of the tablemap log event. mysql-test/suite/rpl/r/rpl_row_tabledefs_3innodb.result: WL#3228 : RBR using different table defs on slave/master This patch contains a new result file as a result of the change to the size of the tablemap log event. mysql-test/suite/rpl/r/rpl_row_until.result: WL#3228 : RBR using different table defs on slave/master This patch contains a new result file as a result of the change to the size of the tablemap log event. mysql-test/suite/rpl/r/rpl_skip_error.result: WL#3228 : RBR using different table defs on slave/master This patch contains a new result file as a result of the change to the size of the tablemap log event. mysql-test/suite/rpl/t/disabled.def: WL#3915 master's cols > slave Disabled the rpl_stm_extraColmaster_ndb test because statement-based replication is not supported in NDB at this time. It can be enabled when statement-based replication for NDB is released. mysql-test/suite/rpl/t/rpl_row_create_table.test: WL#3228 : RBR using different table defs on slave/master This patch corrects binlog positions a result of the change to the size of the tablemap log event. mysql-test/suite/rpl/t/rpl_row_flsh_tbls.test: WL#3228 : RBR using different table defs on slave/master This patch corrects binlog positions a result of the change to the size of the tablemap log event. mysql-test/suite/rpl_ndb/r/rpl_ndb_log.result: WL#3228 : RBR using different table defs on slave/master This patch contains a new result file as a result of the change to the size of the tablemap log event. mysql-test/suite/rpl_ndb/r/rpl_truncate_7ndb.result: WL#3228 : RBR using different table defs on slave/master This patch contains a new result file as a result of the change to the size of the tablemap log event. sql/field.cc: WL#3228 : RBR using different table defs on slave/master This patch includes updates to the unpack() methods for the variable length fields. A new parameter was added (from_length) that is the value stored in the field_metadata of the table map from the table_def class. If the value is non-zero and less than what the field on the slave is then use the from_length else use the original value from the field on the slave. sql/field.h: L#3228 : RBR using different table defs on slave/master This patch includes updates to the unpack() methods for the variable length fields. A new parameter was added (from_length) that is the value stored in the field_metadata of the table map from the table_def class. sql/log_event.cc: WL#3228 : RBR using different table defs on slave/master This patch adds methods to calculate the field metadata size, prepare the field metadata for writing to the binlog, and additions to the Table_map_log_event::write_body method to include the field metadata in the table map that is written to the binlog. WL#3915 master's cols > slave copying extra (slave's) fields returns early if master's table version is wider; removing assert in the way of master > slave cols. sql/log_event.h: WL#3228 : RBR using different table defs on slave/master This patch adds method declarations and variables needed to support storing field metadata in the table map that is written to the binlog. sql/rpl_record.cc: WL#3228 : RBR using different table defs on slave/master This patch modifies the unpack_row() method to unpack fields passing in the value from the table_def class. This value is the extra field metadata stored there from the master. WL#3915 master's cols > slave adding a snippet that shift exectution curson donw the row skipping unknown by slave fields' data. sql/rpl_rli.h: WL#3228 : RBR using different table defs on slave/master This patch adds a helper function to retrieve the table_def for a given table in the RPL_TABLE_LIST structure. sql/rpl_utility.cc: WL#3228 : RBR using different table defs on slave/master This patch adds a helper method that retrieves the correct size parameter for the field. This method is used to compare the size as sent by the master with that on the slave for all types of fields that can vary in size and storage requirements. WL#3915 master's cols > slave Remove warning message for master's cols > slave. sql/rpl_utility.h: WL#3228 : RBR using different table defs on slave/master This patch changes the table_def class constructor to pass in the raw data read from the table map and extract it into an array of dimension size (number of fields). It also adds a method to return the field metadata for any field. The method returns the data stored in the table map or 0 if no data was stored for that field. Lastly, a method to return the results of field->maybe_null() is included so that the slave can determine if a field that is not on the slave is null. mysql-test/suite/rpl/t/rpl_colSize.test: WL#3228 : RBR using different table defs on slave/master This patch contains a new test designed to test the feature of having columns on the master that are smaller than what is on the slave. mysql-test/suite/rpl/t/rpl_extraColmaster_innodb-master.opt: WL#3915 master's cols > slave option for innodb mysql-test/suite/rpl/t/rpl_extraColmaster_innodb-slave.opt: WL#3915 master's cols > slave option for innodb mysql-test/suite/rpl/t/rpl_extraColmaster_innodb.test: WL#3915 master's cols > slave Test of innodb. Test runs in both statement- and row-based replication. mysql-test/suite/rpl/t/rpl_extraColmaster_myisam.test: WL#3915 master's cols > slave Test of myisam. Test runs in both statement- and row-based replication. mysql-test/suite/rpl/r/rpl_colSize.result: WL#3228 : RBR using different table defs on slave/master This patch contains a result file for the new test designed to test the feature of having columns on the master that are smaller than what is on the slave. mysql-test/suite/rpl/t/rpl_row_extraColmaster_ndb.test: WL#3915 master's cols > slave Test of ndb. Test runs in row-based replication. mysql-test/suite/rpl/t/rpl_stm_extraColmaster_ndb.test: WL#3915 master's cols > slave Test of ndb. Test runs in statement-based replication. mysql-test/suite/rpl/r/rpl_extraColmaster_innodb.result: WL#3915 master's cols > slave new results mysql-test/suite/rpl/r/rpl_extraColmaster_myisam.result: WL#3915 master's cols > slave new results mysql-test/extra/rpl_tests/rpl_extraMaster_Col.test: WL#3915 master's cols > slave basic tests checking altering and skipping extra fields by slave. The fields can be of any possible types. mysql-test/suite/rpl/r/rpl_row_extraColmaster_ndb.result: WL#3915 master's cols > slave new results
258 lines
7.7 KiB
C++
258 lines
7.7 KiB
C++
/* Copyright (C) 2006 MySQL AB
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation; version 2 of the License.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program; if not, write to the Free Software
|
|
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
|
|
|
|
#ifndef RPL_UTILITY_H
|
|
#define RPL_UTILITY_H
|
|
|
|
#ifndef __cplusplus
|
|
#error "Don't include this C++ header file from a non-C++ file!"
|
|
#endif
|
|
|
|
#include "mysql_priv.h"
|
|
|
|
struct st_relay_log_info;
|
|
typedef st_relay_log_info RELAY_LOG_INFO;
|
|
|
|
|
|
/**
|
|
A table definition from the master.
|
|
|
|
The responsibilities of this class is:
|
|
- Extract and decode table definition data from the table map event
|
|
- Check if table definition in table map is compatible with table
|
|
definition on slave
|
|
|
|
Currently, the only field type data available is an array of the
|
|
type operators that are present in the table map event.
|
|
|
|
@todo Add type operands to this structure to allow detection of
|
|
difference between, e.g., BIT(5) and BIT(10).
|
|
*/
|
|
|
|
class table_def
|
|
{
|
|
public:
|
|
/**
|
|
Convenience declaration of the type of the field type data in a
|
|
table map event.
|
|
*/
|
|
typedef unsigned char field_type;
|
|
|
|
/**
|
|
Constructor.
|
|
|
|
@param types Array of types
|
|
@param size Number of elements in array 'types'
|
|
@param field_metadata Array of extra information about fields
|
|
@param metadata_size Size of the field_metadata array
|
|
@param null_bitmap The bitmap of fields that can be null
|
|
*/
|
|
table_def(field_type *types, ulong size, uchar *field_metadata,
|
|
int metadata_size, uchar *null_bitmap)
|
|
: m_size(size), m_type(0),
|
|
m_field_metadata(0), m_null_bits(0), m_memory(NULL)
|
|
{
|
|
m_memory= (uchar *)my_multi_malloc(MYF(MY_WME),
|
|
&m_type, size,
|
|
&m_field_metadata, size * sizeof(short),
|
|
&m_null_bits, (m_size + 7) / 8,
|
|
NULL);
|
|
if (m_type)
|
|
memcpy(m_type, types, size);
|
|
else
|
|
m_size= 0;
|
|
/*
|
|
Extract the data from the table map into the field metadata array
|
|
iff there is field metadata. The variable metadata_size will be
|
|
0 if we are replicating from an older version server since no field
|
|
metadata was written to the table map. This can also happen if
|
|
there were no fields in the master that needed extra metadata.
|
|
*/
|
|
if (m_size && metadata_size)
|
|
{
|
|
int index= 0;
|
|
for (unsigned int i= 0; i < m_size; i++)
|
|
{
|
|
switch (m_type[i]) {
|
|
case MYSQL_TYPE_TINY_BLOB:
|
|
case MYSQL_TYPE_BLOB:
|
|
case MYSQL_TYPE_MEDIUM_BLOB:
|
|
case MYSQL_TYPE_LONG_BLOB:
|
|
case MYSQL_TYPE_DOUBLE:
|
|
case MYSQL_TYPE_FLOAT:
|
|
{
|
|
/*
|
|
These types store a single byte.
|
|
*/
|
|
m_field_metadata[i]= (uchar)field_metadata[index];
|
|
index++;
|
|
break;
|
|
}
|
|
case MYSQL_TYPE_SET:
|
|
case MYSQL_TYPE_ENUM:
|
|
case MYSQL_TYPE_STRING:
|
|
{
|
|
short int x= field_metadata[index++] << 8U; // real_type
|
|
x = x + field_metadata[index++]; // pack or field length
|
|
m_field_metadata[i]= x;
|
|
break;
|
|
}
|
|
case MYSQL_TYPE_BIT:
|
|
{
|
|
short int x= field_metadata[index++];
|
|
x = x + (field_metadata[index++] << 8U);
|
|
m_field_metadata[i]= x;
|
|
break;
|
|
}
|
|
case MYSQL_TYPE_VARCHAR:
|
|
{
|
|
/*
|
|
These types store two bytes.
|
|
*/
|
|
char *ptr= (char *)&field_metadata[index];
|
|
m_field_metadata[i]= sint2korr(ptr);
|
|
index= index + sizeof(short int);
|
|
break;
|
|
}
|
|
case MYSQL_TYPE_NEWDECIMAL:
|
|
{
|
|
short int x= field_metadata[index++] << 8U; // precision
|
|
x = x + field_metadata[index++]; // decimals
|
|
m_field_metadata[i]= x;
|
|
break;
|
|
}
|
|
default:
|
|
m_field_metadata[i]= 0;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if (m_size && null_bitmap)
|
|
memcpy(m_null_bits, null_bitmap, (m_size + 7) / 8);
|
|
}
|
|
|
|
~table_def() {
|
|
my_free(m_memory, MYF(0));
|
|
#ifndef DBUG_OFF
|
|
m_type= 0;
|
|
m_size= 0;
|
|
#endif
|
|
}
|
|
|
|
/**
|
|
Return the number of fields there is type data for.
|
|
|
|
@return The number of fields that there is type data for.
|
|
*/
|
|
ulong size() const { return m_size; }
|
|
|
|
|
|
/*
|
|
Return a representation of the type data for one field.
|
|
|
|
@param index Field index to return data for
|
|
|
|
@return Will return a representation of the type data for field
|
|
<code>index</code>. Currently, only the type identifier is
|
|
returned.
|
|
*/
|
|
field_type type(ulong index) const
|
|
{
|
|
DBUG_ASSERT(index < m_size);
|
|
return m_type[index];
|
|
}
|
|
|
|
|
|
/*
|
|
This function allows callers to get the extra field data from the
|
|
table map for a given field. If there is no metadata for that field
|
|
or there is no extra metadata at all, the function returns 0.
|
|
|
|
The function returns the value for the field metadata for column at
|
|
position indicated by index. As mentioned, if the field was a type
|
|
that stores field metadata, that value is returned else zero (0) is
|
|
returned. This method is used in the unpack() methods of the
|
|
corresponding fields to properly extract the data from the binary log
|
|
in the event that the master's field is smaller than the slave.
|
|
*/
|
|
uint16 field_metadata(uint index) const
|
|
{
|
|
DBUG_ASSERT(index < m_size);
|
|
if (m_field_metadata)
|
|
return m_field_metadata[index];
|
|
else
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
This function returns whether the field on the master can be null.
|
|
This value is derived from field->maybe_null().
|
|
*/
|
|
my_bool maybe_null(uint index) const
|
|
{
|
|
DBUG_ASSERT(index < m_size);
|
|
return ((m_null_bits[(index / 8)] &
|
|
(1 << (index % 8))) == (1 << (index %8)));
|
|
}
|
|
|
|
/*
|
|
This function returns the field size in raw bytes based on the type
|
|
and the encoded field data from the master's raw data. This method can
|
|
be used for situations where the slave needs to skip a column (e.g.,
|
|
WL#3915) or needs to advance the pointer for the fields in the raw
|
|
data from the master to a specific column.
|
|
*/
|
|
uint32 calc_field_size(uint col, uchar *master_data);
|
|
|
|
/**
|
|
Decide if the table definition is compatible with a table.
|
|
|
|
Compare the definition with a table to see if it is compatible
|
|
with it.
|
|
|
|
A table definition is compatible with a table if:
|
|
- the columns types of the table definition is a (not
|
|
necessarily proper) prefix of the column type of the table, or
|
|
- the other way around
|
|
|
|
@param rli Pointer to relay log info
|
|
@param table Pointer to table to compare with.
|
|
|
|
@retval 1 if the table definition is not compatible with @c table
|
|
@retval 0 if the table definition is compatible with @c table
|
|
*/
|
|
int compatible_with(RELAY_LOG_INFO const *rli, TABLE *table) const;
|
|
|
|
private:
|
|
ulong m_size; // Number of elements in the types array
|
|
field_type *m_type; // Array of type descriptors
|
|
short int *m_field_metadata;
|
|
uchar *m_null_bits;
|
|
uchar *m_memory;
|
|
};
|
|
|
|
/**
|
|
Extend the normal table list with a few new fields needed by the
|
|
slave thread, but nowhere else.
|
|
*/
|
|
struct RPL_TABLE_LIST
|
|
: public TABLE_LIST
|
|
{
|
|
bool m_tabledef_valid;
|
|
table_def m_tabledef;
|
|
};
|
|
|
|
#endif /* RPL_UTILITY_H */
|