From ce228b68dadc0287a56b7457fce06a1305af831d Mon Sep 17 00:00:00 2001 From: Georg Richter Date: Mon, 10 Oct 2022 09:01:19 +0200 Subject: [PATCH] CONPY-227: Replace collections.named_tuple Replaced collections.named_tuple by C-Pythons PyStruct_Sequence. All conversion are done now before fetching a row: converting to other result types than tuple (named tuple or dict) now have less overhead and are significantly faster. --- include/mariadb_python.h | 1 + mariadb/cursors.py | 66 +++--------------------- mariadb/mariadb_codecs.c | 51 +++++++++++++++++- mariadb/mariadb_connection.c | 5 ++ mariadb/mariadb_cursor.c | 5 ++ testing/test/integration/test_dbapi20.py | 7 ++- 6 files changed, 74 insertions(+), 61 deletions(-) diff --git a/include/mariadb_python.h b/include/mariadb_python.h index cef286f..705e7fc 100755 --- a/include/mariadb_python.h +++ b/include/mariadb_python.h @@ -208,6 +208,7 @@ typedef struct { PyObject *status_callback; #endif PyObject *last_executed_stmt; + PyObject *converter; } MrdbConnection; typedef struct { diff --git a/mariadb/cursors.py b/mariadb/cursors.py index 5114197..30fdb05 100644 --- a/mariadb/cursors.py +++ b/mariadb/cursors.py @@ -18,7 +18,6 @@ # import mariadb -import collections import datetime from numbers import Number from mariadb.constants import CURSOR, STATUS, CAPABILITY, INDICATOR @@ -28,6 +27,8 @@ PARAMSTYLE_QMARK = 1 PARAMSTYLE_FORMAT = 2 PARAMSTYLE_PYFORMAT = 3 +ROWS_ALL = -1 + RESULT_TUPLE = 0 RESULT_NAMEDTUPLE = 1 RESULT_DICTIONARY = 2 @@ -364,7 +365,7 @@ class Cursor(mariadb._mariadb.cursor): """ Internal use only - fetches row and converts values, if connections has a converter. + fetches row and converts values, if connection has a converter. """ self.check_closed() @@ -372,20 +373,7 @@ class Cursor(mariadb._mariadb.cursor): # exception if not self.field_count: raise mariadb.ProgrammingError("Cursor doesn't have a result set") - row = super().fetchone() - if self._connection._converter and row: - tmp_l = list(row) - if not self._description: - self._description = super().description - for i, v in enumerate(row): - type = self.description[i][1] - if type in self._connection._converter: - func = self._connection._converter[type] - tmp_l[i] = func(v) - else: - tmp_l[i] = v - row = tuple(tmp_l) - return row + return super().fetchone() def close(self): """ @@ -412,17 +400,7 @@ class Cursor(mariadb._mariadb.cursor): self.check_closed() row = self._fetch_row() - if not row: - return row - if self._resulttype == RESULT_DICTIONARY: - ret = dict(zip(list(d[0] for d in self.description), row)) - elif self._resulttype == RESULT_NAMEDTUPLE: - ret = collections.namedtuple('Row1', list(d[0] - for d in self.description)) - ret = ret._make(row) - else: - ret = row - return ret + return row def fetchmany(self, size: int = 0): """ @@ -445,9 +423,7 @@ class Cursor(mariadb._mariadb.cursor): if size == 0: size = self.arraysize - rows = super().fetchrows(size) - - return rows + return super().fetchrows(size) def fetchall(self): """ @@ -458,35 +434,7 @@ class Cursor(mariadb._mariadb.cursor): produce a result set or execute() wasn't called before. """ self.check_closed() - rows = super().fetchrows(ROWS_EOF) - - if self._connection._converter: - for idx, row in enumerate(rows): - tmp_l = list(row) - if not self._description: - self._description = super().description - for i, v in enumerate(row): - type = self.description[i][1] - if type in self._connection._converter: - func = self._connection._converter[type] - tmp_l[i] = func(v) - else: - tmp_l[i] = v - rows[idx] = tuple(tmp_l) - - if self._resulttype == RESULT_TUPLE: - return rows - - ret = [] - for row in rows: - if self._resulttype == RESULT_DICTIONARY: - new_row = dict(zip(list(d[0] for d in self.description), row)) - elif self._resulttype == RESULT_NAMEDTUPLE: - new_row = collections.namedtuple('Row1', list(d[0] - for d in self.description)) - new_row = new_row._make(row) - ret.append(new_row) - return ret + return super().fetchrows(ROWS_EOF) def __iter__(self): return iter(self.fetchone, None) diff --git a/mariadb/mariadb_codecs.c b/mariadb/mariadb_codecs.c index feae129..a508b74 100644 --- a/mariadb/mariadb_codecs.c +++ b/mariadb/mariadb_codecs.c @@ -57,7 +57,7 @@ enum enum_extended_field_type mariadb_extended_field_type(const MYSQL_FIELD *fie /* converts a Python date/time/datetime object to MYSQL_TIME - */ +*/ static void mariadb_pydate_to_tm(enum enum_field_types type, PyObject *obj, @@ -403,6 +403,27 @@ static PyObject *Mrdb_GetTimeDelta(MYSQL_TIME *tm) return PyDelta_FromDSU(days, second, second_part); } + +static PyObject *ma_convert_value(MrdbCursor *self, + enum enum_field_types type, + PyObject *value) +{ + PyObject *key= PyLong_FromLongLong(type); + PyObject *func; + PyObject *new_value= NULL; + + if (!self->connection->converter || value == Py_None) + return NULL; + + if ((func= PyDict_GetItem(self->connection->converter, key)) && + PyCallable_Check(func)) + { + PyObject *arglist= PyTuple_New(1); + PyTuple_SetItem(arglist, 0, value); + new_value= PyObject_CallObject(func, arglist); + } + return new_value; +} void field_fetch_fromtext(MrdbCursor *self, char *data, unsigned int column) @@ -541,6 +562,20 @@ field_fetch_fromtext(MrdbCursor *self, char *data, unsigned int column) default: break; } + /* check if values need to be converted */ + if (self->connection->converter) + { + PyObject *val; + enum enum_field_types type; + + if (ext_type == EXT_TYPE_JSON) + type= MYSQL_TYPE_JSON; + else + type= self->fields[column].type; + + if ((val= ma_convert_value(self, type, self->values[column]))) + self->values[column]= val; + } } /* field_fetch_callback @@ -772,6 +807,20 @@ field_fetch_callback(void *data, unsigned int column, unsigned char **row) default: break; } + /* check if values need to be converted */ + if (self->connection->converter) + { + PyObject *val; + enum enum_field_types type; + + if (ext_type == EXT_TYPE_JSON) + type= MYSQL_TYPE_JSON; + else + type= self->fields[column].type; + + if ((val= ma_convert_value(self, type, self->values[column]))) + self->values[column]= val; + } } /* mariadb_get_column_info diff --git a/mariadb/mariadb_connection.c b/mariadb/mariadb_connection.c index 2d47be9..8bc62e5 100644 --- a/mariadb/mariadb_connection.c +++ b/mariadb/mariadb_connection.c @@ -181,6 +181,11 @@ PyMemberDef MrdbConnection_Members[] = offsetof(MrdbConnection, closed), READONLY, "Indicates if connection was closed"}, + {"_converter", + T_OBJECT, + offsetof(MrdbConnection, converter), + 0, + "Conversion dictionary"}, {NULL} /* always last */ }; #if MARIADB_PACKAGE_VERSION_ID > 30301 diff --git a/mariadb/mariadb_cursor.c b/mariadb/mariadb_cursor.c index 4723bd8..6c66b74 100644 --- a/mariadb/mariadb_cursor.c +++ b/mariadb/mariadb_cursor.c @@ -203,6 +203,11 @@ static struct PyMemberDef MrdbCursor_Members[] = offsetof(MrdbCursor, parseinfo.paramlist), READONLY, MISSING_DOC}, + {"_resulttype", + T_UINT, + offsetof(MrdbCursor, result_format), + 0, + MISSING_DOC}, {"_keys", T_OBJECT, offsetof(MrdbCursor, parseinfo.keys), diff --git a/testing/test/integration/test_dbapi20.py b/testing/test/integration/test_dbapi20.py index be89b88..ea88e60 100644 --- a/testing/test/integration/test_dbapi20.py +++ b/testing/test/integration/test_dbapi20.py @@ -142,6 +142,7 @@ class DatabaseAPI20Test(unittest.TestCase): # execute is busted. pass finally: + cur.close() con.close() def _connect(self): @@ -252,8 +253,8 @@ class DatabaseAPI20Test(unittest.TestCase): try: cur = con.cursor() finally: + cur.close() con.close() - del cur def test_cursor_isolation(self): con = self._connect() @@ -272,6 +273,8 @@ class DatabaseAPI20Test(unittest.TestCase): self.assertEqual(len(booze[0]), 1) self.assertEqual(booze[0][0], 'Victoria Bitter') finally: + cur1.close() + cur2.close() con.close() def test_description(self): @@ -326,6 +329,7 @@ class DatabaseAPI20Test(unittest.TestCase): 'executing no-result statements (eg. DDL)' ) finally: + cur.close() con.close() def test_rowcount(self): @@ -349,6 +353,7 @@ class DatabaseAPI20Test(unittest.TestCase): 'executing no-result statements' ) finally: + cur.close() con.close() lower_func = 'lower'