diff --git a/mysql-test/suite/innodb_fts/r/index_table.result b/mysql-test/suite/innodb_fts/r/index_table.result index 909a889db4230..78832669cdbca 100644 --- a/mysql-test/suite/innodb_fts/r/index_table.result +++ b/mysql-test/suite/innodb_fts/r/index_table.result @@ -5,7 +5,7 @@ id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, title VARCHAR(200), content TEXT ) ENGINE= InnoDB; -SET STATEMENT debug_dbug='+d,innodb_report_deadlock' FOR +SET STATEMENT debug_dbug='+d,fts_load_stopword_fail' FOR CREATE FULLTEXT INDEX idx ON articles (title, content); ERROR HY000: Got error 11 "Resource temporarily unavailable" from storage engine InnoDB CREATE FULLTEXT INDEX idx ON articles (title, content); diff --git a/mysql-test/suite/innodb_fts/t/index_table.test b/mysql-test/suite/innodb_fts/t/index_table.test index 89c0905323083..cfe27b4226848 100644 --- a/mysql-test/suite/innodb_fts/t/index_table.test +++ b/mysql-test/suite/innodb_fts/t/index_table.test @@ -18,7 +18,7 @@ CREATE TABLE articles ( ) ENGINE= InnoDB; --error ER_GET_ERRNO -SET STATEMENT debug_dbug='+d,innodb_report_deadlock' FOR +SET STATEMENT debug_dbug='+d,fts_load_stopword_fail' FOR CREATE FULLTEXT INDEX idx ON articles (title, content); CREATE FULLTEXT INDEX idx ON articles (title, content); diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 9e3a23b34ab46..c1b4a5c9559fc 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -172,10 +172,10 @@ SET(INNOBASE_SOURCES fts/fts0ast.cc fts/fts0blex.cc fts/fts0config.cc + fts/fts0exec.cc fts/fts0opt.cc fts/fts0pars.cc fts/fts0que.cc - fts/fts0sql.cc fts/fts0tlex.cc gis/gis0geo.cc gis/gis0rtree.cc @@ -242,6 +242,7 @@ SET(INNOBASE_SOURCES include/fsp_binlog.h include/fts0ast.h include/fts0blex.h + include/fts0exec.h include/fts0fts.h include/fts0opt.h include/fts0pars.h @@ -315,6 +316,7 @@ SET(INNOBASE_SOURCES include/row0mysql.h include/row0purge.h include/row0quiesce.h + include/row0query.h include/row0row.h include/row0row.inl include/row0sel.h @@ -403,6 +405,7 @@ SET(INNOBASE_SOURCES row/row0undo.cc row/row0upd.cc row/row0quiesce.cc + row/row0query.cc row/row0vers.cc srv/srv0mon.cc srv/srv0srv.cc diff --git a/storage/innobase/fts/fts0config.cc b/storage/innobase/fts/fts0config.cc index 4ff14edf5d016..8fa6d87ecc281 100644 --- a/storage/innobase/fts/fts0config.cc +++ b/storage/innobase/fts/fts0config.cc @@ -27,100 +27,33 @@ Created 2007/5/9 Sunny Bains #include "trx0roll.h" #include "row0sel.h" +#include "fts0exec.h" #include "fts0priv.h" +#include "log.h" -/******************************************************************//** -Callback function for fetching the config value. -@return always returns TRUE */ -static -ibool -fts_config_fetch_value( -/*===================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to - ib_vector_t */ -{ - sel_node_t* node = static_cast(row); - fts_string_t* value = static_cast(user_arg); - - dfield_t* dfield = que_node_get_val(node->select_list); - dtype_t* type = dfield_get_type(dfield); - ulint len = dfield_get_len(dfield); - void* data = dfield_get_data(dfield); - - ut_a(dtype_get_mtype(type) == DATA_VARCHAR); - - if (len != UNIV_SQL_NULL) { - ulint max_len = ut_min(value->f_len - 1, len); - - memcpy(value->f_str, data, max_len); - value->f_len = max_len; - value->f_str[value->f_len] = '\0'; - } - - return(TRUE); -} - -/******************************************************************//** -Get value from the config table. The caller must ensure that enough +/** Get value from the config table. The caller must ensure that enough space is allocated for value to hold the column contents. +@param trx transaction +@param table Indexed fts table +@param name name of the key +@param value value of the key @return DB_SUCCESS or error code */ -dberr_t -fts_config_get_value( -/*=================*/ - trx_t* trx, /*!< transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: get config value for - this parameter name */ - fts_string_t* value) /*!< out: value read from - config table */ +dberr_t fts_config_get_value(FTSQueryExecutor *executor, const dict_table_t *table, + const char *name, fts_string_t *value) { - pars_info_t* info; - que_t* graph; - dberr_t error; - ulint name_len = strlen(name); - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - *value->f_str = '\0'; - ut_a(value->f_len > 0); - - pars_info_bind_function(info, "my_func", fts_config_fetch_value, - value); - - /* The len field of value must be set to the max bytes that - it can hold. On a successful read, the len field will be set - to the actual number of bytes copied to value. */ - pars_info_bind_varchar_literal(info, "name", (byte*) name, name_len); - - fts_table->suffix = "CONFIG"; - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS SELECT value FROM $table_name" - " WHERE key = :name;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - trx->op_info = "getting FTS config value"; - - error = fts_eval_sql(trx, graph); - que_graph_free(graph); - return(error); + executor->trx()->op_info = "getting FTS config value"; + ConfigReader reader; + dberr_t err= executor->read_config_with_lock(name, reader); + if (err == DB_SUCCESS) + { + ulint max_len= ut_min(value->f_len - 1, reader.value_span.size()); + memcpy(value->f_str, reader.value_span.data(), max_len); + value->f_len= max_len; + value->f_str[value->f_len]= '\0'; + executor->release_lock(); + } + else value->f_str[0]= '\0'; + return err; } /*********************************************************************//** @@ -157,105 +90,38 @@ column contents. dberr_t fts_config_get_index_value( /*=======================*/ - trx_t* trx, /*!< transaction */ + FTSQueryExecutor* executor, /*!< in: query executor */ dict_index_t* index, /*!< in: index */ const char* param, /*!< in: get config value for this parameter name */ fts_string_t* value) /*!< out: value read from config table */ { - char* name; - dberr_t error; - fts_table_t fts_table; - - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, - index->table); - /* We are responsible for free'ing name. */ - name = fts_config_create_index_param_name(param, index); + char *name = fts_config_create_index_param_name(param, index); - error = fts_config_get_value(trx, &fts_table, name, value); + dberr_t error = fts_config_get_value(executor, index->table, name, value); ut_free(name); return(error); } -/******************************************************************//** -Set the value in the config table for name. +/** Set the value in the config table for name. +@param executor query executor +@param table indexed fulltext table +@param name key for the config +@param value value of the key @return DB_SUCCESS or error code */ dberr_t -fts_config_set_value( -/*=================*/ - trx_t* trx, /*!< transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: get config value for - this parameter name */ - const fts_string_t* - value) /*!< in: value to update */ +fts_config_set_value(FTSQueryExecutor *executor, const dict_table_t *table, + const char *name, const fts_string_t *value) { - pars_info_t* info; - que_t* graph; - dberr_t error; - undo_no_t undo_no; - undo_no_t n_rows_updated; - ulint name_len = strlen(name); - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - pars_info_bind_varchar_literal(info, "name", (byte*) name, name_len); - pars_info_bind_varchar_literal(info, "value", - value->f_str, value->f_len); - - const bool dict_locked = fts_table->table->fts->dict_locked; - - fts_table->suffix = "CONFIG"; - fts_get_table_name(fts_table, table_name, dict_locked); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, info, - "BEGIN UPDATE $table_name SET value = :value" - " WHERE key = :name;"); - - trx->op_info = "setting FTS config value"; - - undo_no = trx->undo_no; - - error = fts_eval_sql(trx, graph); - - que_graph_free(graph); - - n_rows_updated = trx->undo_no - undo_no; - - /* Check if we need to do an insert. */ - if (error == DB_SUCCESS && n_rows_updated == 0) { - info = pars_info_create(); - - pars_info_bind_varchar_literal( - info, "name", (byte*) name, name_len); - - pars_info_bind_varchar_literal( - info, "value", value->f_str, value->f_len); - - fts_get_table_name(fts_table, table_name, dict_locked); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, info, - "BEGIN\n" - "INSERT INTO $table_name VALUES(:name, :value);"); - - trx->op_info = "inserting FTS config value"; - - error = fts_eval_sql(trx, graph); - - que_graph_free(graph); - } - - return(error); + executor->trx()->op_info= "setting FTS config value"; + char value_str[FTS_MAX_CONFIG_VALUE_LEN + 1]; + memcpy(value_str, value->f_str, value->f_len); + value_str[value->f_len]= '\0'; + return executor->update_config_record(name, value_str); } /******************************************************************//** @@ -264,95 +130,67 @@ Set the value specific to an FTS index in the config table. dberr_t fts_config_set_index_value( /*=======================*/ - trx_t* trx, /*!< transaction */ + FTSQueryExecutor* executor, /*!< in: query executor */ dict_index_t* index, /*!< in: index */ const char* param, /*!< in: get config value for this parameter name */ fts_string_t* value) /*!< out: value read from config table */ { - char* name; - dberr_t error; - fts_table_t fts_table; - - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, - index->table); - /* We are responsible for free'ing name. */ - name = fts_config_create_index_param_name(param, index); + char *name = fts_config_create_index_param_name(param, index); - error = fts_config_set_value(trx, &fts_table, name, value); + dberr_t error = fts_config_set_value(executor, index->table, name, value); ut_free(name); return(error); } -/******************************************************************//** -Get an ulint value from the config table. +/** Get an ulint value from the config table. +@param executor Fulltext executor +@param table user table +@param name key value +@param int_value value of the key @return DB_SUCCESS if all OK else error code */ dberr_t -fts_config_get_ulint( -/*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: param name */ - ulint* int_value) /*!< out: value */ +fts_config_get_ulint(FTSQueryExecutor *executor, const dict_table_t *table, + const char *name, ulint *int_value) { - dberr_t error; - fts_string_t value; - - /* We set the length of value to the max bytes it can hold. This - information is used by the callback that reads the value.*/ - value.f_len = FTS_MAX_CONFIG_VALUE_LEN; - value.f_str = static_cast(ut_malloc_nokey(value.f_len + 1)); - - error = fts_config_get_value(trx, fts_table, name, &value); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") reading `" << name << "'"; - } else { - *int_value = strtoul((char*) value.f_str, NULL, 10); - } - - ut_free(value.f_str); - - return(error); + fts_string_t value; + /* We set the length of value to the max bytes it can hold. This + information is used by the callback that reads the value.*/ + value.f_len= FTS_MAX_CONFIG_VALUE_LEN; + value.f_str= static_cast(ut_malloc_nokey(value.f_len + 1)); + dberr_t error= fts_config_get_value(executor, table, name, &value); + if (UNIV_UNLIKELY(error != DB_SUCCESS)) + sql_print_error("InnoDB: (%s) reading `%s'", ut_strerr(error), name); + else *int_value = strtoul((char*) value.f_str, NULL, 10); + ut_free(value.f_str); + return error; } -/******************************************************************//** -Set an ulint value in the config table. +/** Set an ulint value in the config table. +@param trx transaction +@param table user table +@param name name of the key +@param int_value value of the key to be set @return DB_SUCCESS if all OK else error code */ dberr_t -fts_config_set_ulint( -/*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: param name */ - ulint int_value) /*!< in: value */ +fts_config_set_ulint(FTSQueryExecutor *executor, const dict_table_t *table, + const char *name, ulint int_value) { - dberr_t error; - fts_string_t value; - - /* We set the length of value to the max bytes it can hold. This - information is used by the callback that reads the value.*/ - value.f_len = FTS_MAX_CONFIG_VALUE_LEN; - value.f_str = static_cast(ut_malloc_nokey(value.f_len + 1)); - - ut_a(FTS_MAX_INT_LEN < FTS_MAX_CONFIG_VALUE_LEN); - - value.f_len = (ulint) snprintf( - (char*) value.f_str, FTS_MAX_INT_LEN, ULINTPF, int_value); - - error = fts_config_set_value(trx, fts_table, name, &value); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") writing `" << name << "'"; - } - - ut_free(value.f_str); - - return(error); + fts_string_t value; + /* We set the length of value to the max bytes it can hold. This + information is used by the callback that reads the value.*/ + value.f_len= FTS_MAX_CONFIG_VALUE_LEN; + value.f_str= static_cast(ut_malloc_nokey(value.f_len + 1)); + ut_a(FTS_MAX_INT_LEN < FTS_MAX_CONFIG_VALUE_LEN); + value.f_len= (ulint) snprintf((char*) value.f_str, FTS_MAX_INT_LEN, + ULINTPF, int_value); + dberr_t error= fts_config_set_value(executor, table, name, &value); + if (UNIV_UNLIKELY(error != DB_SUCCESS)) + sql_print_error("InnoDB: (%s) writing `%s'", ut_strerr(error), name); + ut_free(value.f_str); + return error; } diff --git a/storage/innobase/fts/fts0exec.cc b/storage/innobase/fts/fts0exec.cc new file mode 100644 index 0000000000000..ee0675be648fa --- /dev/null +++ b/storage/innobase/fts/fts0exec.cc @@ -0,0 +1,1026 @@ +/***************************************************************************** +Copyright (c) 2025, MariaDB PLC. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*****************************************************************************/ + +/**************************************************//** +@file fts/fts0exec.cc + +Created 2025/11/05 +*******************************************************/ + +#include "fts0exec.h" +#include "row0query.h" +#include "fts0fts.h" +#include "fts0types.h" +#include "fts0vlc.h" +#include "fts0priv.h" +#include "btr0btr.h" +#include "btr0cur.h" +#include "dict0dict.h" +#include "row0ins.h" +#include "row0upd.h" +#include "row0sel.h" +#include "eval0eval.h" +#include "que0que.h" +#include "trx0trx.h" +#include "lock0lock.h" +#include "rem0cmp.h" +#include "page0cur.h" +#include "ha_prototypes.h" + + +FTSQueryExecutor::FTSQueryExecutor( + trx_t *trx, const dict_table_t *fts_table) + : m_executor(trx), m_table(fts_table) +{} + +FTSQueryExecutor::~FTSQueryExecutor() +{ + for (uint8_t i= 0; i < FTS_NUM_AUX_INDEX; i++) + if (m_aux_tables[i]) m_aux_tables[i]->release(); + + for (uint8_t i= 0; i < NUM_DELETION_TABLES; i++) + if (m_common_tables[i]) m_common_tables[i]->release(); + + if (m_config_table) m_config_table->release(); +} + +dberr_t FTSQueryExecutor::open_aux_table(uint8_t aux_index) noexcept +{ + if (m_aux_tables[aux_index]) return DB_SUCCESS; + + char table_name[MAX_FULL_NAME_LEN]; + construct_table_name(table_name, fts_get_suffix(aux_index), false); + + m_aux_tables[aux_index]= dict_table_open_on_name( + table_name, false, DICT_ERR_IGNORE_TABLESPACE); + return m_aux_tables[aux_index] ? DB_SUCCESS : DB_TABLE_NOT_FOUND; +} + +dberr_t FTSQueryExecutor::open_all_aux_tables(dict_index_t *fts_index) noexcept +{ + for (uint8_t idx= 0; idx < FTS_NUM_AUX_INDEX; idx++) + { + dict_table_t *table= m_aux_tables[idx]; + if (table) + { + table->release(); + m_aux_tables[idx]= nullptr; + } + } + m_index= fts_index; + for (uint8_t idx= 0; idx < FTS_NUM_AUX_INDEX; idx++) + { + dberr_t err= open_aux_table(idx); + if (err) return err; + } + return DB_SUCCESS; +} + +const char* FTSQueryExecutor::get_deletion_table_name(FTSDeletionTable table_type) noexcept +{ + switch (table_type) + { + case FTSDeletionTable::DELETED: return "DELETED"; + case FTSDeletionTable::DELETED_CACHE: return "DELETED_CACHE"; + case FTSDeletionTable::BEING_DELETED: return "BEING_DELETED"; + case FTSDeletionTable::BEING_DELETED_CACHE: return "BEING_DELETED_CACHE"; + default: return nullptr; + } +} + +/** Helper to convert table name to deletion table enum */ +static FTSDeletionTable get_deletion_table_type(const char* tbl_name) +{ + if (!strcmp(tbl_name, "DELETED")) return FTSDeletionTable::DELETED; + if (!strcmp(tbl_name, "DELETED_CACHE")) return FTSDeletionTable::DELETED_CACHE; + if (!strcmp(tbl_name, "BEING_DELETED")) return FTSDeletionTable::BEING_DELETED; + if (!strcmp(tbl_name, "BEING_DELETED_CACHE")) return FTSDeletionTable::BEING_DELETED_CACHE; + return FTSDeletionTable::MAX_DELETION_TABLES; +} + +dberr_t FTSQueryExecutor::open_deletion_table(FTSDeletionTable table_type) noexcept +{ + uint8_t index= to_index(table_type); + if (index >= NUM_DELETION_TABLES) + return DB_ERROR; + + if (m_common_tables[index]) return DB_SUCCESS; + + const char* suffix_name= get_deletion_table_name(table_type); + if (!suffix_name) return DB_ERROR; + + char table_name[MAX_FULL_NAME_LEN]; + construct_table_name(table_name, suffix_name, true); + + m_common_tables[index]= dict_table_open_on_name( + table_name, false, DICT_ERR_IGNORE_TABLESPACE); + return m_common_tables[index] ? DB_SUCCESS : DB_TABLE_NOT_FOUND; +} + +dberr_t FTSQueryExecutor::open_config_table() noexcept +{ + if (m_config_table) return DB_SUCCESS; + char table_name[MAX_FULL_NAME_LEN]; + construct_table_name(table_name, "CONFIG", true); + + m_config_table= dict_table_open_on_name( + table_name, false, DICT_ERR_IGNORE_TABLESPACE); + return m_config_table ? DB_SUCCESS : DB_TABLE_NOT_FOUND; +} + +dberr_t FTSQueryExecutor::open_all_deletion_tables() noexcept +{ + for (uint8_t i= 0; i < NUM_DELETION_TABLES; i++) + { + FTSDeletionTable table_type= static_cast(i); + dberr_t err= open_deletion_table(table_type); + if (err) return err; + } + return DB_SUCCESS; +} + +dberr_t FTSQueryExecutor::lock_aux_tables(uint8_t aux_index, + lock_mode mode) noexcept +{ + dict_table_t *table= m_aux_tables[aux_index]; + if (table == nullptr) return DB_TABLE_NOT_FOUND; + dberr_t err= m_executor.lock_table(table, mode); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + return err; +} + +dberr_t FTSQueryExecutor::lock_all_aux(lock_mode mode) noexcept +{ + for (uint8_t aux_index= 0; aux_index < FTS_NUM_AUX_INDEX; + aux_index++) + { + dict_table_t *table= m_aux_tables[aux_index]; + if (table == nullptr) return DB_TABLE_NOT_FOUND; + dberr_t err= m_executor.lock_table(table, mode); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + if (err) return err; + } + return DB_SUCCESS; +} + +dberr_t FTSQueryExecutor::lock_common_tables(uint8_t index, + lock_mode mode) noexcept +{ + dict_table_t *table= m_common_tables[index]; + if (table == nullptr) return DB_TABLE_NOT_FOUND; + dberr_t err= m_executor.lock_table(table, mode); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + return err; +} + +dberr_t FTSQueryExecutor::lock_all_common(lock_mode mode) noexcept +{ + for (uint8_t i= 0; i < NUM_DELETION_TABLES; i++) + { + dict_table_t *table= m_common_tables[i]; + if (table == nullptr) return DB_TABLE_NOT_FOUND; + dberr_t err= m_executor.lock_table(table, mode); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + if (err) return err; + } + return DB_SUCCESS; +} + +dberr_t FTSQueryExecutor::insert_aux_record( + uint8_t aux_index, const fts_aux_data_t* aux_data) noexcept +{ + ut_ad(!dict_sys.locked()); + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + err= lock_aux_tables(aux_index, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_aux_tables[aux_index]; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 7 || index->n_uniq != 2) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t fields[7]; + doc_id_t first_doc_id, last_doc_id; + + dtuple_t tuple{0, 7, 2, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 7); + /* Field 0: word (VARCHAR) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, aux_data->word, aux_data->word_len); + + /* Field 1: first_doc_id (INT) */ + field= dtuple_get_nth_field(&tuple, 1); + fts_write_doc_id(&first_doc_id, aux_data->first_doc_id); + dfield_set_data(field, &first_doc_id, sizeof(doc_id_t)); + + /* Field 2: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&tuple, 2); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 3: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&tuple, 3); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + /* Field 4: last_doc_id (UNSIGNED INT) */ + field= dtuple_get_nth_field(&tuple, 4); + fts_write_doc_id(&last_doc_id, aux_data->last_doc_id); + dfield_set_data(field, &last_doc_id, sizeof(doc_id_t)); + + /* Field 5: doc_count (UINT32_T) */ + byte doc_count[4]; + mach_write_to_4(doc_count, aux_data->doc_count); + field= dtuple_get_nth_field(&tuple, 5); + dfield_set_data(field, doc_count, sizeof(doc_count)); + + /* Field 6: ilist (VARBINARY) */ + field= dtuple_get_nth_field(&tuple, 6); + dfield_set_data(field, aux_data->ilist, aux_data->ilist_len); + + return m_executor.insert_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::insert_common_record( + const char *tbl_name, doc_id_t doc_id) noexcept +{ + ut_ad(!dict_sys.locked()); + FTSDeletionTable table_type= get_deletion_table_type(tbl_name); + if (table_type == FTSDeletionTable::MAX_DELETION_TABLES) return DB_ERROR; + + dberr_t err= open_deletion_table(table_type); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= to_index(table_type); + err= lock_common_tables(index_no, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 3 || index->n_uniq != 1) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t fields[3]; + + dtuple_t tuple{0, 3, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 3); + /* Field 0: doc_id (INT) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id(&write_doc_id, doc_id); + dfield_set_data(field, &write_doc_id, sizeof(doc_id_t)); + + /* Field 1: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&tuple, 1); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 2: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&tuple, 2); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + return m_executor.insert_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::insert_config_record( + const char *key, const char *value) noexcept +{ + dberr_t err= open_config_table(); + if (err != DB_SUCCESS) return err; + + err= m_executor.lock_table(m_config_table, LOCK_IX); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_config_table; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 4 || index->n_uniq != 1) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t fields[4]; + + dtuple_t tuple{0, 4, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 4); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + /* Field 1: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&tuple, 1); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 2: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&tuple, 2); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + /* Field 3: value (CHAR(200)) */ + field= dtuple_get_nth_field(&tuple, 3); + dfield_set_data(field, value, strlen(value)); + + return m_executor.insert_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::update_config_record( + const char *key, const char *value) noexcept +{ + dberr_t err= open_config_table(); + if (err != DB_SUCCESS) return err; + + err= m_executor.lock_table(m_config_table, LOCK_IX); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_config_table; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 4 || index->n_uniq != 1) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t search_fields[1]; + dfield_t insert_fields[4]; + + dtuple_t search_tuple{0, 1, 1, 0, search_fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, index, 1); + dfield_t *field= dtuple_get_nth_field(&search_tuple, 0); + dfield_set_data(field, key, strlen(key)); + + dtuple_t insert_tuple{0, 4, 1, 0, insert_fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&insert_tuple, index, 4); + + /* Field 0: key (CHAR(50)) */ + field= dtuple_get_nth_field(&insert_tuple, 0); + dfield_set_data(field, key, strlen(key)); + + /* Field 1: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&insert_tuple, 1); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 2: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&insert_tuple, 2); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + /* Field 3: value (CHAR(200)) */ + field= dtuple_get_nth_field(&insert_tuple, 3); + dfield_set_data(field, value, strlen(value)); + + upd_field_t upd_field; + upd_field.field_no= 3; + upd_field.orig_len= 0; + upd_field.exp= nullptr; + dfield_set_data(&upd_field.new_val, value, strlen(value)); + dict_col_copy_type(dict_index_get_nth_col(index, 3), + dfield_get_type(&upd_field.new_val)); + + upd_t update; + update.heap= nullptr; + update.info_bits= 0; + update.old_vrow= nullptr; + update.n_fields= 1; + update.fields= &upd_field; + + return m_executor.replace_record(table, &search_tuple, &update, + &insert_tuple); +} + +dberr_t FTSQueryExecutor::delete_aux_record( + uint8_t aux_index, const fts_aux_data_t* aux_data) noexcept +{ + ut_ad(!dict_sys.locked()); + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + err= lock_aux_tables(aux_index, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_aux_tables[aux_index]; + dict_index_t* index= dict_table_get_first_index(table); + + if (dict_table_get_next_index(index) != nullptr) + return DB_ERROR; + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: word (VARCHAR) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, aux_data->word, aux_data->word_len); + + return m_executor.delete_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::delete_common_record( + const char *table_name, doc_id_t doc_id) noexcept +{ + ut_ad(!dict_sys.locked()); + FTSDeletionTable table_type= get_deletion_table_type(table_name); + if (table_type == FTSDeletionTable::MAX_DELETION_TABLES) + return DB_ERROR; + + dberr_t err= open_deletion_table(table_type); + if (err != DB_SUCCESS) return err; + + uint8_t cached_index= to_index(table_type); + err= lock_common_tables(cached_index, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[cached_index]; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: doc_id */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id(&write_doc_id, doc_id); + dfield_set_data(field, &write_doc_id, sizeof(doc_id_t)); + + return m_executor.delete_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::delete_all_common_records( + const char *table_name) noexcept +{ + ut_ad(!dict_sys.locked()); + FTSDeletionTable table_type= get_deletion_table_type(table_name); + if (table_type == FTSDeletionTable::MAX_DELETION_TABLES) return DB_ERROR; + + dberr_t err= open_deletion_table(table_type); + if (err != DB_SUCCESS) return err; + + uint8_t cached_index= to_index(table_type); + err= lock_common_tables(cached_index, LOCK_X); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[cached_index]; + return m_executor.delete_all(table); +} + +dberr_t FTSQueryExecutor::delete_config_record( + const char *key) noexcept +{ + ut_ad(!dict_sys.locked()); + dberr_t err= open_config_table(); + if (err != DB_SUCCESS) return err; + + err= m_executor.lock_table(m_config_table, LOCK_IX); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_config_table; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + return m_executor.delete_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::read_config_with_lock(const char *key, + RecordCallback& callback) noexcept +{ + ut_ad(!dict_sys.locked()); + dberr_t err= open_config_table(); + if (err != DB_SUCCESS) return err; + + err= m_executor.lock_table(m_config_table, LOCK_IX); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_config_table; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + err= m_executor.select_for_update(table, &tuple, &callback); + return (err == DB_SUCCESS_LOCKED_REC) ? DB_SUCCESS : err; +} + +dberr_t FTSQueryExecutor::read_aux(uint8_t aux_index, + const char *word, + page_cur_mode_t mode, + RecordCallback& callback) noexcept +{ + ut_ad(!dict_sys.locked()); + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + + err= lock_aux_tables(aux_index, LOCK_IS); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_aux_tables[aux_index]; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: word (VARCHAR) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, word, strlen(word)); + + err= m_executor.read(table, &tuple, mode, callback); + return (err == DB_SUCCESS_LOCKED_REC) ? DB_SUCCESS : err; +} + +dberr_t FTSQueryExecutor::read_aux_all(uint8_t aux_index, RecordCallback& callback) noexcept +{ + ut_ad(!dict_sys.locked()); + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + + err= lock_aux_tables(aux_index, LOCK_IS); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_aux_tables[aux_index]; + err= m_executor.read(table, nullptr, PAGE_CUR_GE, callback); + return (err == DB_SUCCESS_LOCKED_REC) ? DB_SUCCESS : err; +} + +dberr_t FTSQueryExecutor::read_all_common(const char *tbl_name, + RecordCallback& callback) noexcept +{ + ut_ad(!dict_sys.locked()); + FTSDeletionTable table_type= get_deletion_table_type(tbl_name); + if (table_type == FTSDeletionTable::MAX_DELETION_TABLES) return DB_ERROR; + + dberr_t err= open_deletion_table(table_type); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= to_index(table_type); + err= lock_common_tables(index_no, LOCK_IS); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + err= m_executor.read(table, nullptr, PAGE_CUR_GE, callback); + return (err == DB_SUCCESS_LOCKED_REC) ? DB_SUCCESS : err; +} + +CommonTableReader::CommonTableReader() : RecordCallback( + [this](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + doc_id_t doc_id; + if (extract_common_fields(rec, index, &doc_id)) + doc_ids.push_back(doc_id); + return DB_SUCCESS; + }, + [](const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index) -> RecordCompareAction + { return RecordCompareAction::PROCESS; }) {} + + +ConfigReader::ConfigReader() : RecordCallback( + [this](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + const byte *value_data, *key_data; + ulint value_len, key_len; + if (extract_config_fields(rec, index, &key_data, &key_len, + &value_data, &value_len)) + { + if (value_data && value_len != UNIV_SQL_NULL && value_len > 0) + value_span= span( + reinterpret_cast(value_data), value_len); + } + return DB_SUCCESS; + }, + [](const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index) -> RecordCompareAction + { + return compare_config_key(search_tuple, rec, index); + }) {} + +/** Initial size of nodes in fts_word_t. */ +static const ulint FTS_WORD_NODES_INIT_SIZE= 64; + +/** Initialize fts_word_t structure */ +static void init_fts_word(fts_word_t* word, const byte* utf8, ulint len) +{ + mem_heap_t* heap= mem_heap_create(sizeof(fts_node_t)); + memset(word, 0, sizeof(*word)); + word->text.f_len= len; + word->text.f_str= static_cast(mem_heap_alloc(heap, len + 1)); + memcpy(word->text.f_str, utf8, len); + word->text.f_str[len]= 0; + word->heap_alloc= ib_heap_allocator_create(heap); + word->nodes= ib_vector_create(word->heap_alloc, sizeof(fts_node_t), + FTS_WORD_NODES_INIT_SIZE); +} + +bool AuxRecordReader::extract_aux_fields( + const rec_t* rec, const dict_index_t* index, + AuxRecordFields& fields, bool word_only) +{ + const byte *ilist_data= nullptr; + bool ilist_external= false; + if (dict_table_is_comp(index->table)) + { + const byte* lens= + rec - REC_N_NEW_EXTRA_BYTES - 1 - index->n_core_null_bytes; + ulint wlen= *lens--; + if (UNIV_UNLIKELY(wlen & 0x80) && DATA_BIG_COL(index->fields[0].col)) + { + wlen<<= 8; + wlen|= *lens--; + fields.word_len= (wlen & 0x3fff); + } + else + fields.word_len= wlen; + fields.word_data= rec; + + if (fields.word_len == UNIV_SQL_NULL || fields.word_len > FTS_MAX_WORD_LEN) + return false; + + /* If only word field is needed, return early */ + if (word_only) + return true; + + const byte* fixed_fields_start= rec + fields.word_len; + fields.first_doc_id= mach_read_from_8(fixed_fields_start); + /* DOC_ID (8) + DATA_TRX_ID(6) + DATA_ROLL_PTR(7) */ + fields.last_doc_id= mach_read_from_8(fixed_fields_start + 21); + fields.doc_count= mach_read_from_4(fixed_fields_start + 29); + + /* Extract ilist length from current lens position */ + ulint ilen= *lens--; + if (UNIV_UNLIKELY(ilen & 0x80) && DATA_BIG_COL(index->fields[6].col)) + { + ilen <<= 8; + ilen |= *lens--; + ilist_external= (ilen & REC_OFFS_EXTERNAL); + fields.ilist_len= (ilen & 0x3fff); + } + else + { + fields.ilist_len= ilen; + ilist_external= false; + } + + ilist_data= fixed_fields_start + 33; + } + else + { + fields.word_data= rec_get_nth_field_old(rec, 0, &fields.word_len); + if (!fields.word_data || fields.word_len == UNIV_SQL_NULL || + fields.word_len > FTS_MAX_WORD_LEN) + return false; + + if (word_only) + return true; + + ulint len; + const byte* data= rec_get_nth_field_old(rec, 1, &len); + fields.first_doc_id= fts_read_doc_id(data); + data= rec_get_nth_field_old(rec, 4, &len); + fields.last_doc_id= fts_read_doc_id(data); + data= rec_get_nth_field_old(rec, 5, &len); + fields.doc_count= mach_read_from_4(data); + + ilist_data= data + 4; + ulint offs= rec_get_nth_field_offs_old(rec, 6, &fields.ilist_len); + ilist_external= (offs & REC_OFFS_EXTERNAL); + } + + if (ilist_external) + { + fields.ilist_heap= mem_heap_create(fields.ilist_len + 1000); + ulint external_len; + byte* external_data= btr_copy_externally_stored_field( + &external_len, ilist_data, index->table->space->zip_size(), + fields.ilist_len, fields.ilist_heap); + if (external_data) + { + fields.ilist_data= external_data; + fields.ilist_len= external_len; + } + else return false; + } + else fields.ilist_data= const_cast(ilist_data); + return true; +} + +/** AuxRecordReader default word processor implementation */ +dberr_t AuxRecordReader::default_word_processor( + const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets, void* user_arg) +{ + ib_vector_t *words= static_cast(user_arg); + AuxRecordFields fields; + /* Use optimized field extraction with external BLOB handling */ + if (!extract_aux_fields(rec, index, fields)) + return DB_SUCCESS; + fts_word_t *word; + bool is_word_init = false; + + ut_ad(fields.word_len <= FTS_MAX_WORD_LEN); + + if (ib_vector_size(words) == 0) + { + /* First word - push and initialize */ + word = static_cast(ib_vector_push(words, nullptr)); + init_fts_word(word, fields.word_data, fields.word_len); + is_word_init = true; + } + else + { + /* Check if this word is different from the last word */ + word = static_cast(ib_vector_last(words)); + if (fields.word_len != word->text.f_len || + memcmp(word->text.f_str, fields.word_data, fields.word_len)) + { + /* Different word - push new word and initialize */ + word = static_cast(ib_vector_push(words, nullptr)); + init_fts_word(word, fields.word_data, fields.word_len); + is_word_init = true; + } + } + fts_node_t *node= static_cast( + ib_vector_push(word->nodes, nullptr)); + + /* Use extracted field values */ + node->first_doc_id= fields.first_doc_id; + node->last_doc_id= fields.last_doc_id; + node->doc_count= fields.doc_count; + + node->ilist_size_alloc= node->ilist_size= 0; + node->ilist= nullptr; + + if (fields.ilist_data && fields.ilist_len != UNIV_SQL_NULL && fields.ilist_len > 0) + { + node->ilist_size_alloc= node->ilist_size= fields.ilist_len; + if (fields.ilist_len) + { + node->ilist= static_cast(ut_malloc_nokey(fields.ilist_len)); + memcpy(node->ilist, fields.ilist_data, fields.ilist_len); + } + if (fields.ilist_len == 0) return DB_SUCCESS_LOCKED_REC; + } + + if (this->total_memory) + { + if (is_word_init) + { + *this->total_memory+= + sizeof(fts_word_t) + sizeof(ib_alloc_t) + + sizeof(ib_vector_t) + fields.word_len + + sizeof(fts_node_t) * FTS_WORD_NODES_INIT_SIZE; + } + *this->total_memory += node->ilist_size; + if (*this->total_memory >= fts_result_cache_limit) + return DB_FTS_EXCEED_RESULT_CACHE_LIMIT; + } + return DB_SUCCESS; +} + +/** AuxRecordReader comparison logic implementation */ +RecordCompareAction AuxRecordReader::compare_record( + const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index) noexcept +{ + if (!search_tuple) return RecordCompareAction::PROCESS; + int cmp_result; + switch (compare_mode) + { + case AuxCompareMode::GREATER_EQUAL: + case AuxCompareMode::GREATER: + { + int match= 0; + cmp_result= cmp_dtuple_rec_bytes(rec, *index, *search_tuple, &match, + index->table->not_redundant()); + if (compare_mode == AuxCompareMode::GREATER_EQUAL) + return (cmp_result <= 0) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; + else + return (cmp_result < 0) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; + } + case AuxCompareMode::LIKE: + case AuxCompareMode::EQUAL: + { + AuxRecordFields fields; + if (!extract_aux_fields(rec, index, fields, true)) + return RecordCompareAction::SKIP; + + const dfield_t* search_field= dtuple_get_nth_field(search_tuple, 0); + const void* search_data= dfield_get_data(search_field); + ulint search_len= dfield_get_len(search_field); + if (!search_data || search_len == UNIV_SQL_NULL) + return RecordCompareAction::PROCESS; + if (!fields.word_data || fields.word_len == UNIV_SQL_NULL) + return RecordCompareAction::SKIP; + + const dtype_t* type= dfield_get_type(search_field); + cmp_result= cmp_data(type->mtype, type->prtype, false, + static_cast(search_data), + search_len, fields.word_data, fields.word_len); + + if (compare_mode == AuxCompareMode::EQUAL) + return cmp_result == 0 + ? RecordCompareAction::PROCESS + : RecordCompareAction::STOP; + else /* AuxCompareMode::LIKE */ + { + int prefix_cmp= cmp_data(type->mtype, type->prtype, false, + static_cast(search_data), + search_len, fields.word_data, + search_len <= fields.word_len ? search_len : fields.word_len); + + if (prefix_cmp != 0) return RecordCompareAction::STOP; + return (search_len <= fields.word_len) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; + } + } + } + return RecordCompareAction::PROCESS; +} + +bool ConfigReader::extract_config_fields( + const rec_t* rec, const dict_index_t* index, + const byte** key_data, ulint* key_len, + const byte** value_data, ulint* value_len) +{ + bool comp= dict_table_is_comp(index->table); + if (comp) + { + const byte* lens= + rec - REC_N_NEW_EXTRA_BYTES - 1 - index->n_core_null_bytes; + if (key_data && key_len) + { + *key_len = *lens; + if (*key_len & 0x80) + *key_len = ((*key_len & 0x3f) << 8) | *(lens - 1); + *key_data = rec; + } + if (value_data && value_len) + { + ulint key_field_len = *lens; + if (key_field_len & 0x80) + key_field_len = ((key_field_len & 0x3f) << 8) | *(lens - 1); + + const byte* value_start = rec + key_field_len + DATA_TRX_ID_LEN + + DATA_ROLL_PTR_LEN; + const byte* value_lens = lens; + if (key_field_len & 0x80) value_lens--; /* Skip extra key length byte */ + value_lens--; /* Skip to field 3 length */ + *value_len = *value_lens; + if (*value_len & 0x80) + *value_len = ((*value_len & 0x3f) << 8) | *(value_lens - 1); + *value_data = value_start; + } + return true; + } + else + { + if (key_data && key_len) + { + *key_data= rec_get_nth_field_old(rec, 0, key_len); + if (!*key_data || *key_len == UNIV_SQL_NULL) return false; + } + if (value_data && value_len) + { + *value_data= rec_get_nth_field_old(rec, 3, value_len); + if (!*value_data || *value_len == UNIV_SQL_NULL) return false; + } + return true; + } +} + +bool CommonTableReader::extract_common_fields( + const rec_t* rec, const dict_index_t* index, + doc_id_t* doc_id) +{ + + if (!dict_table_is_comp(index->table)) + { + ulint doc_id_len; + ulint offset= rec_get_nth_field_offs_old(rec, 0, &doc_id_len); + if (offset != 0 || doc_id_len == UNIV_SQL_NULL || doc_id_len != 8) + return false; + } + + *doc_id= mach_read_from_8(rec); + return true; +} + +/** Direct config key comparison implementation */ +RecordCompareAction ConfigReader::compare_config_key( + const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index) +{ + if (!search_tuple) return RecordCompareAction::PROCESS; + const dfield_t *search_field = dtuple_get_nth_field(search_tuple, 0); + const void *search_data = dfield_get_data(search_field); + ulint search_len = dfield_get_len(search_field); + if (!search_data || search_len == UNIV_SQL_NULL) + return RecordCompareAction::PROCESS; + + const byte *rec_key_data; + ulint rec_key_len; + if (!extract_config_fields(rec, index, &rec_key_data, &rec_key_len)) + return RecordCompareAction::SKIP; + + const dtype_t *type = dfield_get_type(search_field); + int cmp_result = cmp_data(type->mtype, type->prtype, false, + static_cast(search_data), + search_len, rec_key_data, rec_key_len); + return (cmp_result == 0) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; +} + +void FTSQueryExecutor::construct_table_name( + char *table_name, const char *suffix, bool common_table) noexcept +{ + ut_ad(m_table); + ut_ad(common_table || m_index); + const size_t dbname_len= m_table->name.dblen() + 1; + ut_ad(dbname_len > 1); + memcpy(table_name, m_table->name.m_name, dbname_len); + memcpy(table_name += dbname_len, "FTS_", 4); + table_name+= 4; + + int len= fts_write_object_id(m_table->id, table_name); + if (!common_table) + { + table_name[len]= '_'; + ++len; + len+= fts_write_object_id(m_index->id, table_name + len); + } + ut_a(len >= 16); + ut_a(len < FTS_AUX_MIN_TABLE_ID_LENGTH); + table_name+= len; + *table_name++= '_'; + strcpy(table_name, suffix); +} diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 1e4a1040f6ee4..1b81e376f1114 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -30,6 +30,7 @@ Full Text Search interface #include "dict0stats_bg.h" #include "row0sel.h" #include "fts0fts.h" +#include "fts0exec.h" #include "fts0priv.h" #include "fts0types.h" #include "fts0types.inl" @@ -44,6 +45,29 @@ static const ulint FTS_MAX_ID_LEN = 32; /** Column name from the FTS config table */ #define FTS_MAX_CACHE_SIZE_IN_MB "cache_size_in_mb" +/** Compare function to check if record's doc_id > search tuple's doc_id +@param[in] search_tuple Search tuple containing target doc_id +@param[in] rec Record to check +@param[in] index Index containing the record +@return true if record's doc_id > search tuple's doc_id */ +static +RecordCompareAction doc_id_comparator( + const dtuple_t* search_tuple, + const rec_t* rec, + const dict_index_t* index) +{ + /* Get target doc_id from search tuple */ + const dfield_t* search_field= dtuple_get_nth_field(search_tuple, 0); + const byte* search_data= static_cast(dfield_get_data(search_field)); + doc_id_t target_doc_id= fts_read_doc_id(search_data); + + /* Get doc_id from record using direct field extraction */ + const byte* rec_data= rec; + doc_id_t rec_doc_id= fts_read_doc_id(rec_data); + return (rec_doc_id > target_doc_id) + ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; +} /** Verify if a aux table name is a obsolete table by looking up the key word in the obsolete table names */ #define FTS_IS_OBSOLETE_AUX_TABLE(table_name) \ @@ -148,27 +172,6 @@ const fts_index_selector_t fts_index_selector[] = { { 0 , NULL } }; -/** Default config values for FTS indexes on a table. */ -static const char* fts_config_table_insert_values_sql = - "PROCEDURE P() IS\n" - "BEGIN\n" - "\n" - "INSERT INTO $config_table VALUES('" - FTS_MAX_CACHE_SIZE_IN_MB "', '256');\n" - "" - "INSERT INTO $config_table VALUES('" - FTS_OPTIMIZE_LIMIT_IN_SECS "', '180');\n" - "" - "INSERT INTO $config_table VALUES ('" - FTS_SYNCED_DOC_ID "', '0');\n" - "" - "INSERT INTO $config_table VALUES ('" - FTS_TOTAL_DELETED_COUNT "', '0');\n" - "" /* Note: 0 == FTS_TABLE_STATE_RUNNING */ - "INSERT INTO $config_table VALUES ('" - FTS_TABLE_STATE "', '0');\n" - "END;\n"; - /** FTS tokenize parameter for plugin parser */ struct fts_tokenize_param_t { fts_doc_t* result_doc; /*!< Result doc for tokens */ @@ -208,30 +211,6 @@ fts_add_doc_by_id( fts_trx_table_t*ftt, /*!< in: FTS trx table */ doc_id_t doc_id); /*!< in: doc id */ -/** Tokenize a document. -@param[in,out] doc document to tokenize -@param[out] result tokenization result -@param[in] parser pluggable parser */ -static -void -fts_tokenize_document( - fts_doc_t* doc, - fts_doc_t* result, - st_mysql_ftparser* parser); - -/** Continue to tokenize a document. -@param[in,out] doc document to tokenize -@param[in] add_pos add this position to all tokens from this tokenization -@param[out] result tokenization result -@param[in] parser pluggable parser */ -static -void -fts_tokenize_document_next( - fts_doc_t* doc, - ulint add_pos, - fts_doc_t* result, - st_mysql_ftparser* parser); - /** Create the vector of fts_get_doc_t instances. @param[in,out] cache fts cache @return vector of fts_get_doc_t instances */ @@ -266,7 +245,6 @@ fts_cache_destroy(fts_cache_t* cache) /** Get a character set based on precise type. @param prtype precise type @return the corresponding character set */ -UNIV_INLINE CHARSET_INFO* fts_get_charset(ulint prtype) { @@ -298,7 +276,6 @@ fts_get_charset(ulint prtype) /****************************************************************//** This function loads the default InnoDB stopword list */ -static void fts_load_default_stopword( /*======================*/ @@ -343,187 +320,157 @@ fts_load_default_stopword( stopword_info->status = STOPWORD_FROM_DEFAULT; } -/****************************************************************//** -Callback function to read a single stopword value. -@return Always return TRUE */ -static -ibool -fts_read_stopword( -/*==============*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - ib_alloc_t* allocator; - fts_stopword_t* stopword_info; - sel_node_t* sel_node; - que_node_t* exp; - ib_rbt_t* stop_words; - dfield_t* dfield; - fts_string_t str; - mem_heap_t* heap; - ib_rbt_bound_t parent; - dict_table_t* table; - - sel_node = static_cast(row); - table = sel_node->table_list->table; - stopword_info = static_cast(user_arg); - - stop_words = stopword_info->cached_stopword; - allocator = static_cast(stopword_info->heap); - heap = static_cast(allocator->arg); - - exp = sel_node->select_list; - - /* We only need to read the first column */ - dfield = que_node_get_val(exp); - - str.f_n_char = 0; - str.f_str = static_cast(dfield_get_data(dfield)); - str.f_len = dfield_get_len(dfield); - exp = que_node_get_next(exp); - ut_ad(exp); - - if (table->versioned()) { - dfield = que_node_get_val(exp); - ut_ad(dfield_get_type(dfield)->vers_sys_end()); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - if (table->versioned_by_id()) { - ut_ad(len == sizeof trx_id_max_bytes); - if (0 != memcmp(data, trx_id_max_bytes, len)) { - return true; - } - } else { - ut_ad(len == sizeof timestamp_max_bytes); - if (!IS_MAX_TIMESTAMP(data)) { - return true; - } - } - } - ut_ad(!que_node_get_next(exp)); - - /* Only create new node if it is a value not already existed */ - if (str.f_len != UNIV_SQL_NULL - && rbt_search(stop_words, &parent, &str) != 0) { - - fts_tokenizer_word_t new_word; - - new_word.nodes = ib_vector_create( - allocator, sizeof(fts_node_t), 4); - - new_word.text.f_str = static_cast( - mem_heap_alloc(heap, str.f_len + 1)); - - memcpy(new_word.text.f_str, str.f_str, str.f_len); - - new_word.text.f_n_char = 0; - new_word.text.f_len = str.f_len; - new_word.text.f_str[str.f_len] = 0; - - rbt_insert(stop_words, &new_word, &new_word); - } - - return(TRUE); -} - -/******************************************************************//** -Load user defined stopword from designated user table +/** Load user defined stopword from designated user table +@param fts fulltext structure +@param stopword_table stopword table +@param stopword_info stopword information @return whether the operation is successful */ -static -bool -fts_load_user_stopword( -/*===================*/ - fts_t* fts, /*!< in: FTS struct */ - const char* stopword_table_name, /*!< in: Stopword table - name */ - fts_stopword_t* stopword_info) /*!< in: Stopword info */ +bool fts_load_user_stopword(FTSQueryExecutor *executor, fts_t *fts, + const char *stopword_table, + fts_stopword_t *stopword_info) { - if (!fts->dict_locked) { - dict_sys.lock(SRW_LOCK_CALL); - } - - /* Validate the user table existence in the right format */ - bool ret= false; - const char* row_end; - stopword_info->charset = fts_valid_stopword_table(stopword_table_name, - &row_end); - if (!stopword_info->charset) { + trx_t* trx = executor->trx(); + if (!fts->dict_locked) dict_sys.lock(SRW_LOCK_CALL); + /* Validate the user table existence in the right format */ + bool ret= false; + const char* row_end; + stopword_info->charset= fts_valid_stopword_table( + stopword_table, &row_end); + if (!stopword_info->charset) + { cleanup: - if (!fts->dict_locked) { - dict_sys.unlock(); - } + if (!fts->dict_locked) dict_sys.unlock(); + return ret; + } - return ret; - } + if (!stopword_info->cached_stopword) + { + /* Create the stopword RB tree with the stopword column + charset. All comparison will use this charset */ + stopword_info->cached_stopword= rbt_create_arg_cmp( + sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, + (void*)stopword_info->charset); + } - trx_t* trx = trx_create(); - trx->op_info = "Load user stopword table into FTS cache"; + /* Load the stopword table */ + dict_table_t* table= + dict_sys.load_table({stopword_table, strlen(stopword_table)}); + if (!table) + goto cleanup; + + /* Use the passed executor's transaction */ + trx->op_info= "Load user stopword table into FTS cache"; + ib_rbt_t* stop_words= stopword_info->cached_stopword; + ib_alloc_t* allocator= static_cast(stopword_info->heap); + mem_heap_t* heap= static_cast(allocator->arg); + + /* Find the field number for 'value' column */ + dict_index_t* clust_index= dict_table_get_first_index(table); + ulint value_field_no= ULINT_UNDEFINED; + for (ulint i= 0; i < dict_index_get_n_fields(clust_index); i++) + { + const dict_field_t* field= dict_index_get_nth_field(clust_index, i); + if (strcmp(field->name, "value") == 0) + { + value_field_no= i; + break; + } + } + if (value_field_no == ULINT_UNDEFINED) + { + sql_print_error("InnoDB: Could not find 'value' column in " + "stopword table %s", stopword_table); + goto cleanup; + } - if (!stopword_info->cached_stopword) { - /* Create the stopword RB tree with the stopword column - charset. All comparison will use this charset */ - stopword_info->cached_stopword = rbt_create_arg_cmp( - sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, - (void*)stopword_info->charset); + auto process_stopword= [&](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + mem_heap_t *offsets_heap= nullptr; + if (offsets == nullptr) + offsets= rec_get_offsets(rec, index, nullptr, index->n_core_fields, + ULINT_UNDEFINED, &offsets_heap); - } + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, value_field_no, + &field_len); - pars_info_t* info = pars_info_create(); - - pars_info_bind_id(info, "table_stopword", stopword_table_name); - pars_info_bind_id(info, "row_end", row_end); - - pars_info_bind_function(info, "my_func", fts_read_stopword, - stopword_info); - - que_t* graph = pars_sql( - info, - "PROCEDURE P() IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT value, $row_end" - " FROM $table_stopword;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;" - "END;\n"); - - for (;;) { - dberr_t error = fts_eval_sql(trx, graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - stopword_info->status = STOPWORD_USER_TABLE; - break; - } else { - fts_sql_rollback(trx); + if (field_len == UNIV_SQL_NULL) return DB_SUCCESS; - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout reading user" - " stopword table. Retrying!"; + ib_rbt_bound_t parent; + fts_string_t str; + str.f_n_char= 0; + str.f_str= const_cast(field_data); + str.f_len= field_len; - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "Error '" << error - << "' while reading user stopword" - " table."; - ret = FALSE; - break; - } - } - } + /* Handle system versioning - check row_end column if versioned */ + if (table->versioned()) + { + ulint end_len; + const byte* end_data= rec_get_nth_field( + rec, offsets, table->vers_end + index->n_uniq + 2, &end_len); - que_graph_free(graph); - trx->free(); - ret = true; - goto cleanup; + if (table->versioned_by_id()) + { + ut_ad(end_len == sizeof trx_id_max_bytes); + if (0 != memcmp(end_data, trx_id_max_bytes, end_len)) + goto func_exit; + } + else + { + ut_ad(end_len == sizeof timestamp_max_bytes); + if (!IS_MAX_TIMESTAMP(end_data)) + goto func_exit; + } + } + + if (str.f_len != UNIV_SQL_NULL && + rbt_search(stop_words, &parent, &str) != 0) + { + fts_tokenizer_word_t new_word; + new_word.nodes= ib_vector_create(allocator, sizeof(fts_node_t), 4); + new_word.text.f_str= static_cast( + mem_heap_alloc(heap, str.f_len + 1)); + memcpy(new_word.text.f_str, str.f_str, str.f_len); + new_word.text.f_n_char= 0; + new_word.text.f_len= str.f_len; + new_word.text.f_str[str.f_len]= 0; + rbt_insert(stop_words, &new_word, &new_word); + } +func_exit: + if (offsets_heap) mem_heap_free(offsets_heap); + return DB_SUCCESS; /* Continue processing */ + }; + + RecordCallback callback(process_stopword); + /* Read all records from the stopword table */ + for (;;) + { + dberr_t error= executor->read(table, nullptr, PAGE_CUR_G, callback); + if (UNIV_LIKELY(error == DB_SUCCESS)) + { + stopword_info->status= STOPWORD_USER_TABLE; + ret= true; + break; + } + else + { + if (error == DB_LOCK_WAIT_TIMEOUT) + { + sql_print_warning("InnoDB: Lock wait timeout reading " + "user stopword table. Retrying!"); + trx->error_state= DB_SUCCESS; + } + else + { + sql_print_error("InnoDB: Error '%s' while reading user " + "stopword table.", ut_strerr(error)); + ret= false; + break; + } + } + } + goto cleanup; } /******************************************************************//** @@ -587,6 +534,46 @@ fts_cache_init( } } +/** Construct the name of an internal FTS table for the given table. +@param[in] fts_table metadata on fulltext-indexed table +@param[out] table_name a name up to MAX_FULL_NAME_LEN +@param[in] dict_locked whether dict_sys.latch is being held */ +void fts_get_table_name(const fts_table_t* fts_table, char* table_name, + bool dict_locked) +{ + if (!dict_locked) dict_sys.freeze(SRW_LOCK_CALL); + ut_ad(dict_sys.frozen()); + /* Include the separator as well. */ + const size_t dbname_len= fts_table->table->name.dblen() + 1; + ut_ad(dbname_len > 1); + memcpy(table_name, fts_table->table->name.m_name, dbname_len); + if (!dict_locked) dict_sys.unfreeze(); + + memcpy(table_name += dbname_len, "FTS_", 4); + table_name += 4; + int len; + switch (fts_table->type) + { + case FTS_COMMON_TABLE: + len= fts_write_object_id(fts_table->table_id, table_name); + break; + + case FTS_INDEX_TABLE: + len= fts_write_object_id(fts_table->table_id, table_name); + table_name[len]= '_'; + ++len; + len+= fts_write_object_id(fts_table->index_id, table_name + len); + break; + + default: ut_error; + } + ut_a(len >= 16); + ut_a(len < FTS_AUX_MIN_TABLE_ID_LENGTH); + table_name+= len; + *table_name++= '_'; + strcpy(table_name, fts_table->suffix); +} + /****************************************************************//** Create a FTS cache. */ fts_cache_t* @@ -1815,88 +1802,78 @@ CREATE TABLE $FTS_PREFIX_CONFIG @param[in] skip_doc_id_index Skip index on doc id @return DB_SUCCESS if succeed */ dberr_t -fts_create_common_tables( - trx_t* trx, - dict_table_t* table, - bool skip_doc_id_index) +fts_create_common_tables(trx_t *trx, dict_table_t *table, + bool skip_doc_id_index) { - dberr_t error; - que_t* graph; - fts_table_t fts_table; - mem_heap_t* heap = mem_heap_create(1024); - pars_info_t* info; - char fts_name[MAX_FULL_NAME_LEN]; - char full_name[sizeof(fts_common_tables) / sizeof(char*)] - [MAX_FULL_NAME_LEN]; - - dict_index_t* index = NULL; - - FTS_INIT_FTS_TABLE(&fts_table, NULL, FTS_COMMON_TABLE, table); - - error = fts_drop_common_tables(trx, &fts_table, true); - - if (error != DB_SUCCESS) { - - goto func_exit; - } - - /* Create the FTS tables that are common to an FTS index. */ - for (ulint i = 0; fts_common_tables[i] != NULL; ++i) { - - fts_table.suffix = fts_common_tables[i]; - fts_get_table_name(&fts_table, full_name[i], true); - dict_table_t* common_table = fts_create_one_common_table( - trx, table, full_name[i], fts_table.suffix, heap); - - if (!common_table) { - trx->error_state = DB_SUCCESS; - error = DB_ERROR; - goto func_exit; - } - - mem_heap_empty(heap); - } - - /* Write the default settings to the config table. */ - info = pars_info_create(); + dberr_t error= DB_SUCCESS; + char full_name[sizeof(fts_common_tables) / sizeof(char*)][MAX_FULL_NAME_LEN]; + dict_index_t *index= nullptr; + fts_table_t fts_table; + FTS_INIT_FTS_TABLE(&fts_table, NULL, FTS_COMMON_TABLE, table); + error = fts_drop_common_tables(trx, &fts_table, true); + if (error != DB_SUCCESS) return error; + mem_heap_t *heap= mem_heap_create(1024); + + FTSQueryExecutor executor(trx, table); + /* Create the FTS tables that are common to an FTS index. */ + for (ulint i = 0; fts_common_tables[i] != NULL; ++i) + { + fts_table.suffix = fts_common_tables[i]; + fts_get_table_name(&fts_table, full_name[i], true); + dict_table_t *common_table= fts_create_one_common_table( + trx, table, full_name[i], fts_table.suffix, heap); + if (!common_table) + { + trx->error_state = DB_SUCCESS; + error = DB_ERROR; + mem_heap_free(heap); + return error; + } - fts_table.suffix = "CONFIG"; - fts_get_table_name(&fts_table, fts_name, true); - pars_info_bind_id(info, "config_table", fts_name); + if (i == 2) + executor.set_config_table(common_table); - graph = pars_sql( - info, fts_config_table_insert_values_sql); + mem_heap_empty(heap); + } - error = fts_eval_sql(trx, graph); + /** Does the following insert operation: + INSERT INTO $config_table VALUES('"FTS_MAX_CACHE_SIZE_IN_MB"', '256');" + INSERT INTO $config_table VALUES('"FTS_OPTIMIZE_LIMIT_IN_SECS"', '180');" + INSERT INTO $config_table VALUES ('"FTS_SYNCED_DOC_ID "', '0');" + INSERT INTO $config_table VALUES ('"FTS_TOTAL_DELETED_COUNT "', '0');" + INSERT INTO $config_table VALUES ('"FTS_TABLE_STATE "', '0');" */ + error= executor.insert_config_record(FTS_MAX_CACHE_SIZE_IN_MB, "256"); + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_OPTIMIZE_LIMIT_IN_SECS, "180"); - que_graph_free(graph); + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_SYNCED_DOC_ID, "0"); - if (error != DB_SUCCESS || skip_doc_id_index) { + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_TOTAL_DELETED_COUNT, "0"); - goto func_exit; - } + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_TABLE_STATE, "0"); - if (table->versioned()) { - index = dict_mem_index_create(table, - FTS_DOC_ID_INDEX.str, - DICT_UNIQUE, 2); - dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); - dict_mem_index_add_field(index, table->cols[table->vers_end].name(*table).str, 0); - } else { - index = dict_mem_index_create(table, - FTS_DOC_ID_INDEX.str, - DICT_UNIQUE, 1); - dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); - } + if (error != DB_SUCCESS || skip_doc_id_index) goto func_exit; - error = row_create_index_for_mysql(index, trx, NULL, - FIL_ENCRYPTION_DEFAULT, - FIL_DEFAULT_ENCRYPTION_KEY); + if (table->versioned()) + { + index= dict_mem_index_create(table, FTS_DOC_ID_INDEX.str, DICT_UNIQUE, 2); + dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); + dict_mem_index_add_field(index, table->cols[table->vers_end].name(*table).str, 0); + } + else + { + index= dict_mem_index_create(table, FTS_DOC_ID_INDEX.str, DICT_UNIQUE, 1); + dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); + } + error= row_create_index_for_mysql(index, trx, NULL, FIL_ENCRYPTION_DEFAULT, + FIL_DEFAULT_ENCRYPTION_KEY); func_exit: - mem_heap_free(heap); - - return(error); + mem_heap_free(heap); + return error; } /** Create one FTS auxiliary index table for an FTS index. @@ -2361,39 +2338,6 @@ fts_trx_add_op( fts_trx_table_add_op(stmt_ftt, doc_id, state, fts_indexes); } -/******************************************************************//** -Fetch callback that converts a textual document id to a binary value and -stores it in the given place. -@return always returns NULL */ -static -ibool -fts_fetch_store_doc_id( -/*===================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: doc_id_t* to store - doc_id in */ -{ - int n_parsed; - sel_node_t* node = static_cast(row); - doc_id_t* doc_id = static_cast(user_arg); - dfield_t* dfield = que_node_get_val(node->select_list); - dtype_t* type = dfield_get_type(dfield); - ulint len = dfield_get_len(dfield); - - char buf[32]; - - ut_a(dtype_get_mtype(type) == DATA_VARCHAR); - ut_a(len > 0 && len < sizeof(buf)); - - memcpy(buf, dfield_get_data(dfield), len); - buf[len] = '\0'; - - n_parsed = sscanf(buf, FTS_DOC_ID_FORMAT, doc_id); - ut_a(n_parsed == 1); - - return(FALSE); -} - /*********************************************************************//** Get the next available document id. @return DB_SUCCESS if OK */ @@ -2426,70 +2370,48 @@ fts_get_next_doc_id( } /** Read the synced document id from the fts configuration table -@param table fts table -@param doc_id document id to be read -@param trx transaction to read from config table +@param executor query executor +@param table fts table +@param doc_id document id to be read @return DB_SUCCESS in case of success */ static -dberr_t fts_read_synced_doc_id(const dict_table_t *table, - doc_id_t *doc_id, - trx_t *trx) +dberr_t fts_read_synced_doc_id(FTSQueryExecutor *executor, + const dict_table_t *table, + doc_id_t *doc_id) { - dberr_t error; - char table_name[MAX_FULL_NAME_LEN]; - - fts_table_t fts_table; - fts_table.suffix= "CONFIG"; - fts_table.table_id= table->id; - fts_table.type= FTS_COMMON_TABLE; - fts_table.table= table; ut_a(table->fts->doc_col != ULINT_UNDEFINED); - - trx->op_info = "update the next FTS document id"; - pars_info_t *info= pars_info_create(); - pars_info_bind_function(info, "my_func", fts_fetch_store_doc_id, - doc_id); - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "config_table", table_name); - - que_t *graph= fts_parse_sql( - &fts_table, info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS SELECT value FROM $config_table" - " WHERE key = 'synced_doc_id' FOR UPDATE;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - + executor->trx()->op_info= "reading synced FTS document id"; + ConfigReader reader; *doc_id= 0; - error = fts_eval_sql(trx, graph); - que_graph_free(graph); + dberr_t error= executor->read_config_with_lock("synced_doc_id", reader); + if (error == DB_SUCCESS) + { + char value_buf[32]; + size_t copy_len= std::min(reader.value_span.size(), sizeof(value_buf) - 1); + memcpy(value_buf, reader.value_span.data(), copy_len); + value_buf[copy_len]= '\0'; + int n_parsed= sscanf(value_buf, FTS_DOC_ID_FORMAT, doc_id); + if (n_parsed != 1) error= DB_ERROR; + executor->release_lock(); + } return error; } /** This function fetch the Doc ID from CONFIG table, and compare with the Doc ID supplied. And store the larger one to the CONFIG table. -@param table fts table -@param cmp_doc_id Doc ID to compare -@param doc_id larger document id after comparing "cmp_doc_id" to - the one stored in CONFIG table -@param trx transaction +@param executor query executor +@param table fts table +@param cmp_doc_id Doc ID to compare +@param doc_id larger document id after comparing "cmp_doc_id" to + the one stored in CONFIG table @return DB_SUCCESS if OK */ static dberr_t fts_cmp_set_sync_doc_id( + FTSQueryExecutor *executor, const dict_table_t *table, doc_id_t cmp_doc_id, - doc_id_t *doc_id, - trx_t *trx=nullptr) + doc_id_t *doc_id) { if (srv_read_only_mode) { return DB_READ_ONLY; @@ -2497,14 +2419,8 @@ fts_cmp_set_sync_doc_id( fts_cache_t* cache= table->fts->cache; dberr_t error = DB_SUCCESS; - const trx_t* const caller_trx = trx; - - if (trx == nullptr) { - trx = trx_create(); - trx_start_internal_read_only(trx); - } retry: - error = fts_read_synced_doc_id(table, doc_id, trx); + error = fts_read_synced_doc_id(executor, table, doc_id); if (error != DB_SUCCESS) goto func_exit; @@ -2524,27 +2440,19 @@ fts_cmp_set_sync_doc_id( if (cmp_doc_id && cmp_doc_id >= *doc_id) { error = fts_update_sync_doc_id( - table, cache->synced_doc_id, trx); + executor, table, cache->synced_doc_id); } *doc_id = cache->next_doc_id; func_exit: - if (caller_trx) { - return error; - } - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - } else { + if (error != DB_SUCCESS) { *doc_id = 0; ib::error() << "(" << error << ") while getting next doc id " "for table " << table->name; - fts_sql_rollback(trx); - if (error == DB_DEADLOCK || error == DB_LOCK_WAIT_TIMEOUT) { DEBUG_SYNC_C("fts_cmp_set_sync_doc_id_retry"); std::this_thread::sleep_for(FTS_DEADLOCK_RETRY_WAIT); @@ -2552,86 +2460,24 @@ fts_cmp_set_sync_doc_id( } } - trx->clear_and_free(); - return(error); } /** Update the last document id. This function could create a new transaction to update the last document id. -@param table table to be updated -@param doc_id last document id -@param trx update trx or null +@param executor query executor +@param table table to be updated +@param doc_id last document id @retval DB_SUCCESS if OK */ dberr_t -fts_update_sync_doc_id( - const dict_table_t* table, - doc_id_t doc_id, - trx_t* trx) +fts_update_sync_doc_id(FTSQueryExecutor *executor, + const dict_table_t *table, + doc_id_t doc_id) { - byte id[FTS_MAX_ID_LEN]; - pars_info_t* info; - fts_table_t fts_table; - ulint id_len; - que_t* graph = NULL; - dberr_t error; - ibool local_trx = FALSE; - fts_cache_t* cache = table->fts->cache; - char fts_name[MAX_FULL_NAME_LEN]; - - if (srv_read_only_mode) { - return DB_READ_ONLY; - } - - fts_table.suffix = "CONFIG"; - fts_table.table_id = table->id; - fts_table.type = FTS_COMMON_TABLE; - fts_table.table = table; - - if (!trx) { - trx = trx_create(); - trx_start_internal(trx); - - trx->op_info = "setting last FTS document id"; - local_trx = TRUE; - } - - info = pars_info_create(); - - id_len = (ulint) snprintf( - (char*) id, sizeof(id), FTS_DOC_ID_FORMAT, doc_id + 1); - - pars_info_bind_varchar_literal(info, "doc_id", id, id_len); - - fts_get_table_name(&fts_table, fts_name, - table->fts->dict_locked); - pars_info_bind_id(info, "table_name", fts_name); - - graph = fts_parse_sql( - &fts_table, info, - "BEGIN" - " UPDATE $table_name SET value = :doc_id" - " WHERE key = 'synced_doc_id';"); - - error = fts_eval_sql(trx, graph); - - que_graph_free(graph); - - if (local_trx) { - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - cache->synced_doc_id = doc_id; - } else { - ib::error() << "(" << error << ") while" - " updating last doc id for table" - << table->name; - - fts_sql_rollback(trx); - } - trx->clear_and_free(); - } - - return(error); + if (srv_read_only_mode) return DB_READ_ONLY; + char id[FTS_MAX_ID_LEN]; + snprintf(id, sizeof(id), FTS_DOC_ID_FORMAT, doc_id + 1); + return executor->update_config_record("synced_doc_id", id); } /*********************************************************************//** @@ -2690,13 +2536,9 @@ fts_delete( fts_trx_table_t*ftt, /*!< in: FTS trx table */ fts_trx_row_t* row) /*!< in: row */ { - que_t* graph; - fts_table_t fts_table; - doc_id_t write_doc_id; dict_table_t* table = ftt->table; doc_id_t doc_id = row->doc_id; trx_t* trx = ftt->fts_trx->trx; - pars_info_t* info = pars_info_create(); fts_cache_t* cache = table->fts->cache; /* we do not index Documents whose Doc ID value is 0 */ @@ -2707,12 +2549,6 @@ fts_delete( ut_a(row->state == FTS_DELETE || row->state == FTS_MODIFY); - FTS_INIT_FTS_TABLE(&fts_table, "DELETED", FTS_COMMON_TABLE, table); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, doc_id); - fts_bind_doc_id(info, "doc_id", &write_doc_id); - /* It is possible we update a record that has not yet been sync-ed into cache from last crash (delete Doc will not initialize the sync). Avoid any added counter accounting until the FTS cache @@ -2737,20 +2573,9 @@ fts_delete( } /* Note the deleted document for OPTIMIZE to purge. */ - char table_name[MAX_FULL_NAME_LEN]; - trx->op_info = "adding doc id to FTS DELETED"; - - fts_table.suffix = "DELETED"; - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "deleted", table_name); - - graph = fts_parse_sql(&fts_table, info, - "BEGIN INSERT INTO $deleted VALUES (:doc_id);"); - - dberr_t error = fts_eval_sql(trx, graph); - que_graph_free(graph); + FTSQueryExecutor executor(trx, table); + dberr_t error= executor.insert_common_record("DELETED", doc_id); /* Increment the total deleted count, this is used to calculate the number of documents indexed. */ @@ -2918,105 +2743,18 @@ fts_doc_free( } /*********************************************************************//** -Callback function for fetch that stores the text of an FTS document, -converting each column to UTF-16. -@return always FALSE */ -ibool -fts_query_expansion_fetch_doc( -/*==========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ -{ - que_node_t* exp; - sel_node_t* node = static_cast(row); - fts_doc_t* result_doc = static_cast(user_arg); - dfield_t* dfield; - ulint len; - ulint doc_len; - fts_doc_t doc; - CHARSET_INFO* doc_charset = NULL; - ulint field_no = 0; - - len = 0; - - fts_doc_init(&doc); - doc.found = TRUE; - - exp = node->select_list; - doc_len = 0; - - doc_charset = result_doc->charset; - - /* Copy each indexed column content into doc->text.f_str */ - while (exp) { - dfield = que_node_get_val(exp); - len = dfield_get_len(dfield); - - /* NULL column */ - if (len == UNIV_SQL_NULL) { - exp = que_node_get_next(exp); - continue; - } - - if (!doc_charset) { - doc_charset = fts_get_charset(dfield->type.prtype); - } - - doc.charset = doc_charset; - - if (dfield_is_ext(dfield)) { - /* We ignore columns that are stored externally, this - could result in too many words to search */ - exp = que_node_get_next(exp); - continue; - } else { - doc.text.f_n_char = 0; - - doc.text.f_str = static_cast( - dfield_get_data(dfield)); - - doc.text.f_len = len; - } - - if (field_no == 0) { - fts_tokenize_document(&doc, result_doc, - result_doc->parser); - } else { - fts_tokenize_document_next(&doc, doc_len, result_doc, - result_doc->parser); - } - - exp = que_node_get_next(exp); - - doc_len += (exp) ? len + 1 : len; - - field_no++; - } - - ut_ad(doc_charset); - - if (!result_doc->charset) { - result_doc->charset = doc_charset; - } - - fts_doc_free(&doc); - - return(FALSE); -} - -/*********************************************************************//** -fetch and tokenize the document. */ -static -void -fts_fetch_doc_from_rec( -/*===================*/ - fts_get_doc_t* get_doc, /*!< in: FTS index's get_doc struct */ - dict_index_t* clust_index, /*!< in: cluster index */ - btr_pcur_t* pcur, /*!< in: cursor whose position - has been stored */ - rec_offs* offsets, /*!< in: offsets */ - fts_doc_t* doc) /*!< out: fts doc to hold parsed - documents */ +fetch and tokenize the document. */ +static +void +fts_fetch_doc_from_rec( +/*===================*/ + fts_get_doc_t* get_doc, /*!< in: FTS index's get_doc struct */ + dict_index_t* clust_index, /*!< in: cluster index */ + btr_pcur_t* pcur, /*!< in: cursor whose position + has been stored */ + rec_offs* offsets, /*!< in: offsets */ + fts_doc_t* doc) /*!< out: fts doc to hold parsed + documents */ { dict_index_t* index; const rec_t* clust_rec; @@ -3167,6 +2905,7 @@ fts_add_doc_from_tuple( mtr_start(&mtr); + FTSQueryExecutor executor(ftt->fts_trx->trx, ftt->table); ulint num_idx = ib_vector_size(cache->get_docs); for (ulint i = 0; i < num_idx; ++i) { @@ -3188,8 +2927,7 @@ fts_add_doc_from_tuple( if (table->fts->cache->stopword_info.status & STOPWORD_NOT_INIT) { - fts_load_stopword(table, NULL, NULL, - true, true); + fts_load_stopword(&executor, table); } fts_cache_add_doc( @@ -3291,6 +3029,7 @@ fts_add_doc_by_id( } } + FTSQueryExecutor executor(ftt->fts_trx->trx, table); /* If we have a match, add the data to doc structure */ if (btr_pcur_open_with_no_init(tuple, PAGE_CUR_LE, BTR_SEARCH_LEAF, &pcur, &mtr) @@ -3366,8 +3105,7 @@ fts_add_doc_by_id( if (table->fts->cache->stopword_info.status & STOPWORD_NOT_INIT) { - fts_load_stopword(table, NULL, - NULL, true, true); + fts_load_stopword(&executor, table); } fts_cache_add_doc( @@ -3440,28 +3178,6 @@ fts_add_doc_by_id( mem_heap_free(heap); } - -/*********************************************************************//** -Callback function to read a single ulint column. -return always returns TRUE */ -static -ibool -fts_read_ulint( -/*===========*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ulint */ -{ - sel_node_t* sel_node = static_cast(row); - ulint* value = static_cast(user_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - - *value = mach_read_from_4(static_cast(data)); - - return(TRUE); -} - /*********************************************************************//** Get maximum Doc ID in a table if index "FTS_DOC_ID_INDEX" exists @return max Doc ID or 0 if index "FTS_DOC_ID_INDEX" does not exist */ @@ -3533,200 +3249,19 @@ fts_get_max_doc_id( return(doc_id); } -/*********************************************************************//** -Fetch document with the given document id. -@return DB_SUCCESS if OK else error */ -dberr_t -fts_doc_fetch_by_doc_id( -/*====================*/ - fts_get_doc_t* get_doc, /*!< in: state */ - doc_id_t doc_id, /*!< in: id of document to - fetch */ - dict_index_t* index_to_use, /*!< in: caller supplied FTS index, - or NULL */ - ulint option, /*!< in: search option, if it is - greater than doc_id or equal */ - fts_sql_callback - callback, /*!< in: callback to read */ - void* arg) /*!< in: callback arg */ -{ - pars_info_t* info; - dberr_t error; - const char* select_str; - doc_id_t write_doc_id; - dict_index_t* index; - trx_t* trx = trx_create(); - que_t* graph; - - trx->op_info = "fetching indexed FTS document"; - - /* The FTS index can be supplied by caller directly with - "index_to_use", otherwise, get it from "get_doc" */ - index = (index_to_use) ? index_to_use : get_doc->index_cache->index; - - if (get_doc && get_doc->get_document_graph) { - info = get_doc->get_document_graph->info; - } else { - info = pars_info_create(); - } - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, doc_id); - fts_bind_doc_id(info, "doc_id", &write_doc_id); - pars_info_bind_function(info, "my_func", callback, arg); - - select_str = fts_get_select_columns_str(index, info, info->heap); - pars_info_bind_id(info, "table_name", index->table->name.m_name); - - if (!get_doc || !get_doc->get_document_graph) { - if (option == FTS_FETCH_DOC_BY_ID_EQUAL) { - graph = fts_parse_sql( - NULL, - info, - mem_heap_printf(info->heap, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT %s FROM $table_name" - " WHERE %s = :doc_id;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c %% NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;", - select_str, - FTS_DOC_ID.str)); - } else { - ut_ad(option == FTS_FETCH_DOC_BY_ID_LARGE); - - /* This is used for crash recovery of table with - hidden DOC ID or FTS indexes. We will scan the table - to re-processing user table rows whose DOC ID or - FTS indexed documents have not been sync-ed to disc - during recent crash. - In the case that all fulltext indexes are dropped - for a table, we will keep the "hidden" FTS_DOC_ID - column, and this scan is to retreive the largest - DOC ID being used in the table to determine the - appropriate next DOC ID. - In the case of there exists fulltext index(es), this - operation will re-tokenize any docs that have not - been sync-ed to the disk, and re-prime the FTS - cached */ - graph = fts_parse_sql( - NULL, - info, - mem_heap_printf(info->heap, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT %s, %s FROM $table_name" - " WHERE %s > :doc_id;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c %% NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;", - FTS_DOC_ID.str, - select_str, - FTS_DOC_ID.str)); - } - if (get_doc) { - get_doc->get_document_graph = graph; - } - } else { - graph = get_doc->get_document_graph; - } - - error = fts_eval_sql(trx, graph); - fts_sql_commit(trx); - trx->free(); - - if (!get_doc) { - que_graph_free(graph); - } - - return(error); -} - -/*********************************************************************//** -Write out a single word's data as new entry/entries in the INDEX table. -@return DB_SUCCESS if all OK. */ -dberr_t -fts_write_node( -/*===========*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: query graph */ - fts_table_t* fts_table, /*!< in: aux table */ - fts_string_t* word, /*!< in: word in UTF-8 */ - fts_node_t* node) /*!< in: node columns */ +/** Write out a single word's data as new entry/entries in the INDEX table. +@param executor FTS Query Executor +@param selected auxiliary index number +@param aux_data auxiliary table data +@return DB_SUCCESS if all OK or error code */ +dberr_t fts_write_node(FTSQueryExecutor *executor, uint8_t selected, + const fts_aux_data_t *aux_data) { - pars_info_t* info; - dberr_t error; - ib_uint32_t doc_count; - time_t start_time; - doc_id_t last_doc_id; - doc_id_t first_doc_id; - char table_name[MAX_FULL_NAME_LEN]; - - ut_a(node->ilist != NULL); - - if (*graph) { - info = (*graph)->info; - } else { - info = pars_info_create(); - - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "index_table_name", table_name); - } - - pars_info_bind_varchar_literal(info, "token", word->f_str, word->f_len); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &first_doc_id, node->first_doc_id); - fts_bind_doc_id(info, "first_doc_id", &first_doc_id); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &last_doc_id, node->last_doc_id); - fts_bind_doc_id(info, "last_doc_id", &last_doc_id); - - ut_a(node->last_doc_id >= node->first_doc_id); - - /* Convert to "storage" byte order. */ - mach_write_to_4((byte*) &doc_count, node->doc_count); - pars_info_bind_int4_literal( - info, "doc_count", (const ib_uint32_t*) &doc_count); - - /* Set copy_name to FALSE since it's a static. */ - pars_info_bind_literal( - info, "ilist", node->ilist, node->ilist_size, - DATA_BLOB, DATA_BINARY_TYPE); - - if (!*graph) { - - *graph = fts_parse_sql( - fts_table, - info, - "BEGIN\n" - "INSERT INTO $index_table_name VALUES" - " (:token, :first_doc_id," - " :last_doc_id, :doc_count, :ilist);"); - } - - start_time = time(NULL); - error = fts_eval_sql(trx, *graph); - elapsed_time += time(NULL) - start_time; - ++n_nodes; - - return(error); + time_t start_time = time(NULL); + dberr_t error= executor->insert_aux_record(selected, aux_data); + elapsed_time += time(NULL) - start_time; + ++n_nodes; + return error; } /** Sort an array of doc_id */ @@ -3736,157 +3271,85 @@ void fts_doc_ids_sort(ib_vector_t *doc_ids) std::sort(data, data + doc_ids->used); } -/*********************************************************************//** -Add rows to the DELETED_CACHE table. +/** Add rows to the DELETED_CACHE table. @return DB_SUCCESS if all went well else error code*/ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t -fts_sync_add_deleted_cache( -/*=======================*/ - fts_sync_t* sync, /*!< in: sync state */ - ib_vector_t* doc_ids) /*!< in: doc ids to add */ +fts_sync_add_deleted_cache(FTSQueryExecutor *executor, fts_sync_t *sync, + ib_vector_t *doc_ids) { - ulint i; - pars_info_t* info; - que_t* graph; - fts_table_t fts_table; - char table_name[MAX_FULL_NAME_LEN]; - doc_id_t dummy = 0; - dberr_t error = DB_SUCCESS; - ulint n_elems = ib_vector_size(doc_ids); - - ut_a(ib_vector_size(doc_ids) > 0); - - fts_doc_ids_sort(doc_ids); - - info = pars_info_create(); - - fts_bind_doc_id(info, "doc_id", &dummy); - - FTS_INIT_FTS_TABLE( - &fts_table, "DELETED_CACHE", FTS_COMMON_TABLE, sync->table); - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - &fts_table, - info, - "BEGIN INSERT INTO $table_name VALUES (:doc_id);"); - - for (i = 0; i < n_elems && error == DB_SUCCESS; ++i) { - doc_id_t* update; - doc_id_t write_doc_id; - - update = static_cast(ib_vector_get(doc_ids, i)); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, *update); - fts_bind_doc_id(info, "doc_id", &write_doc_id); - - error = fts_eval_sql(sync->trx, graph); - } - - que_graph_free(graph); - - return(error); + ulint n_elems= ib_vector_size(doc_ids); + ut_a(n_elems > 0); + fts_doc_ids_sort(doc_ids); + dberr_t error= DB_SUCCESS; + for (uint32_t i= 0; i < n_elems && error == DB_SUCCESS; ++i) + { + doc_id_t *update= + static_cast(ib_vector_get(doc_ids, i)); + error= executor->insert_common_record("DELETED_CACHE", *update); + } + return error; } -/** Write the words and ilist to disk. -@param[in,out] trx transaction -@param[in] index_cache index cache -@param[in] unlock_cache whether unlock cache when write node +/** Write the words and ilist to disk +@param[in,out] executor query executor +@param[in] index_cache index cache +@param[in] unlock_cache whether unlock cache when write node @return DB_SUCCESS if all went well else error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_sync_write_words( - trx_t* trx, - fts_index_cache_t* index_cache, - bool unlock_cache) +dberr_t fts_sync_write_words(FTSQueryExecutor *executor, + fts_index_cache_t *index_cache, + bool unlock_cache) { - fts_table_t fts_table; - const ib_rbt_node_t* rbt_node; - dberr_t error = DB_SUCCESS; - ibool print_error = FALSE; - dict_table_t* table = index_cache->index->table; - - FTS_INIT_INDEX_TABLE( - &fts_table, NULL, FTS_INDEX_TABLE, index_cache->index); - - /* We iterate over the entire tree, even if there is an error, - since we want to free the memory used during caching. */ - for (rbt_node = rbt_first(index_cache->words); - rbt_node; - rbt_node = rbt_next(index_cache->words, rbt_node)) { - - ulint i; - ulint selected; - fts_tokenizer_word_t* word; - - word = rbt_value(fts_tokenizer_word_t, rbt_node); - - DBUG_EXECUTE_IF( - "fts_instrument_write_words_before_select_index", - std::this_thread::sleep_for( - std::chrono::milliseconds(300));); - - selected = fts_select_index( - index_cache->charset, word->text.f_str, - word->text.f_len); - - fts_table.suffix = fts_get_suffix(selected); - - /* We iterate over all the nodes even if there was an error */ - for (i = 0; i < ib_vector_size(word->nodes); ++i) { - - fts_node_t* fts_node = static_cast( - ib_vector_get(word->nodes, i)); - - if (fts_node->synced) { - continue; - } else { - fts_node->synced = true; - } - - /*FIXME: we need to handle the error properly. */ - if (error == DB_SUCCESS) { - if (unlock_cache) { - mysql_mutex_unlock( - &table->fts->cache->lock); - } - - error = fts_write_node( - trx, - &index_cache->ins_graph[selected], - &fts_table, &word->text, fts_node); - - DEBUG_SYNC_C("fts_write_node"); - DBUG_EXECUTE_IF("fts_write_node_crash", - DBUG_SUICIDE();); - - DBUG_EXECUTE_IF( - "fts_instrument_sync_sleep", - std::this_thread::sleep_for( - std::chrono::seconds(1));); - - if (unlock_cache) { - mysql_mutex_lock( - &table->fts->cache->lock); - } - } - } + dict_table_t *table= index_cache->index->table; + bool print_error= false; + dberr_t error= DB_SUCCESS; + for (const ib_rbt_node_t *rbt_node= rbt_first(index_cache->words); + rbt_node; rbt_node = rbt_next(index_cache->words, rbt_node)) + { + fts_tokenizer_word_t *word= rbt_value(fts_tokenizer_word_t, rbt_node); + DBUG_EXECUTE_IF("fts_instrument_write_words_before_select_index", + std::this_thread::sleep_for( + std::chrono::milliseconds(300));); + uint8_t selected= fts_select_index( + index_cache->charset, word->text.f_str, word->text.f_len); + + for (ulint i = 0; i < ib_vector_size(word->nodes); ++i) + { + fts_node_t* fts_node= + static_cast(ib_vector_get(word->nodes, i)); + if (fts_node->synced) continue; + else fts_node->synced= true; + /* FIXME: we need to handle the error properly. */ + if (error == DB_SUCCESS) + { + if (unlock_cache) mysql_mutex_unlock(&table->fts->cache->lock); + fts_aux_data_t aux_data((const char*)word->text.f_str, word->text.f_len, + fts_node->first_doc_id, fts_node->last_doc_id, + static_cast(fts_node->doc_count), fts_node->ilist, + fts_node->ilist_size); + error= fts_write_node(executor, selected, &aux_data); + DEBUG_SYNC_C("fts_write_node"); + DBUG_EXECUTE_IF("fts_write_node_crash", DBUG_SUICIDE();); + DBUG_EXECUTE_IF("fts_instrument_sync_sleep", + std::this_thread::sleep_for(std::chrono::seconds(1));); + + if (unlock_cache) mysql_mutex_lock(&table->fts->cache->lock); + } - n_nodes += ib_vector_size(word->nodes); + n_nodes += ib_vector_size(word->nodes); - if (UNIV_UNLIKELY(error != DB_SUCCESS) && !print_error) { - ib::error() << "(" << error << ") writing" - " word node to FTS auxiliary index table " - << table->name; - print_error = TRUE; - } - } + if (UNIV_UNLIKELY(error != DB_SUCCESS) && !print_error) + { + sql_print_error("InnoDB: ( %s ) writing word node to FTS auxiliary " + "index table %s", ut_strerr(error), + table->name.m_name); + print_error= true; + } + } + } - return(error); + return error; } /*********************************************************************//** @@ -3901,24 +3364,22 @@ fts_sync_begin( trx_start_internal(sync->trx); } -/*********************************************************************//** -Run SYNC on the table, i.e., write out data from the index specific -cache to the FTS aux INDEX table and FTS aux doc id stats table. +/** Run SYNC on the table, i.e., write out data from the index +specific cache to the FTS aux INDEX table and FTS aux doc id +stats table. +@param executor query executor +@param sync sync state +@param index_cache index cache @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t -fts_sync_index( -/*===========*/ - fts_sync_t* sync, /*!< in: sync state */ - fts_index_cache_t* index_cache) /*!< in: index cache */ +fts_sync_index(FTSQueryExecutor *executor, fts_sync_t *sync, + fts_index_cache_t *index_cache) { - trx_t* trx = sync->trx; - - trx->op_info = "doing SYNC index"; - - ut_ad(rbt_validate(index_cache->words)); - - return(fts_sync_write_words(trx, index_cache, sync->unlock_cache)); + trx_t *trx= sync->trx; + trx->op_info= "doing SYNC index"; + ut_ad(rbt_validate(index_cache->words)); + return fts_sync_write_words(executor, index_cache, sync->unlock_cache); } /** Check if index cache has been synced completely @@ -3972,13 +3433,10 @@ fts_sync_index_reset( } } -/** Commit the SYNC, change state of processed doc ids etc. -@param[in,out] sync sync state -@return DB_SUCCESS if all OK */ +/** Commit the SYNC, change state of processed doc ids etc.\n@param[in,out]\texecutor\t\tquery executor\n@param[in,out]\tsync\t\t\tsync state\n@return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t -fts_sync_commit( - fts_sync_t* sync) +fts_sync_commit(FTSQueryExecutor *executor, fts_sync_t *sync) { dberr_t error; trx_t* trx = sync->trx; @@ -3989,8 +3447,8 @@ fts_sync_commit( /* After each Sync, update the CONFIG table about the max doc id we just sync-ed to index table */ - error = fts_cmp_set_sync_doc_id(sync->table, sync->max_doc_id, - &last_doc_id, trx); + error = fts_cmp_set_sync_doc_id(executor, sync->table, sync->max_doc_id, + &last_doc_id); /* Get the list of deleted documents that are either in the cache or were headed there but were deleted before the add @@ -3999,7 +3457,7 @@ fts_sync_commit( if (error == DB_SUCCESS && ib_vector_size(cache->deleted_doc_ids) > 0) { error = fts_sync_add_deleted_cache( - sync, cache->deleted_doc_ids); + executor, sync, cache->deleted_doc_ids); } /* We need to do this within the deleted lock since fts_delete() can @@ -4117,11 +3575,13 @@ fts_sync( sync->unlock_cache = unlock_cache; sync->in_progress = true; + const size_t fts_cache_size= fts_max_cache_size; DEBUG_SYNC_C("fts_sync_begin"); fts_sync_begin(sync); - + FTSQueryExecutor executor(sync->trx, sync->table); + error = executor.open_config_table(); + if (error) goto end_sync; begin_sync: - const size_t fts_cache_size= fts_max_cache_size; if (cache->total_size > fts_cache_size) { /* Avoid the case: sync never finish when insert/update keeps comming. */ @@ -4135,19 +3595,21 @@ fts_sync( } for (i = 0; i < ib_vector_size(cache->indexes); ++i) { - fts_index_cache_t* index_cache; - - index_cache = static_cast( + fts_index_cache_t *index_cache= + static_cast( ib_vector_get(cache->indexes, i)); if (index_cache->index->to_be_dropped) { continue; } + error= executor.open_all_aux_tables(index_cache->index); + + if (error) goto end_sync; DBUG_EXECUTE_IF("fts_instrument_sync_before_syncing", std::this_thread::sleep_for( std::chrono::milliseconds(300));); - error = fts_sync_index(sync, index_cache); + error = fts_sync_index(&executor, sync, index_cache); if (error != DB_SUCCESS) { goto end_sync; @@ -4184,7 +3646,7 @@ fts_sync( end_sync: if (error == DB_SUCCESS && !sync->interrupted) { - error = fts_sync_commit(sync); + error = fts_sync_commit(&executor, sync); } else { fts_sync_rollback(sync); } @@ -4506,7 +3968,6 @@ fts_tokenize_by_parser( @param[in,out] doc document to tokenize @param[out] result tokenization result @param[in] parser pluggable parser */ -static void fts_tokenize_document( fts_doc_t* doc, @@ -4536,12 +3997,6 @@ fts_tokenize_document( } } -/** Continue to tokenize a document. -@param[in,out] doc document to tokenize -@param[in] add_pos add this position to all tokens from this tokenization -@param[out] result tokenization result -@param[in] parser pluggable parser */ -static void fts_tokenize_document_next( fts_doc_t* doc, @@ -4656,7 +4111,12 @@ fts_init_doc_id( /* Then compare this value with the ID value stored in the CONFIG table. The larger one will be our new initial Doc ID */ - fts_cmp_set_sync_doc_id(table, 0, &max_doc_id); + trx_t* trx = trx_create(); + trx_start_internal_read_only(trx); + FTSQueryExecutor executor(trx, table); + fts_cmp_set_sync_doc_id(&executor, table, 0, &max_doc_id); + fts_sql_commit(trx); + trx->free(); /* If DICT_TF2_FTS_ADD_DOC_ID is set, we are in the process of creating index (and add doc id column. No need to recovery @@ -4676,81 +4136,6 @@ fts_init_doc_id( return(max_doc_id); } -/*********************************************************************//** -Fetch COUNT(*) from specified table. -@return the number of rows in the table */ -ulint -fts_get_rows_count( -/*===============*/ - fts_table_t* fts_table) /*!< in: fts table to read */ -{ - trx_t* trx; - pars_info_t* info; - que_t* graph; - dberr_t error; - ulint count = 0; - char table_name[MAX_FULL_NAME_LEN]; - - trx = trx_create(); - trx->op_info = "fetching FT table rows count"; - - info = pars_info_create(); - - pars_info_bind_function(info, "my_func", fts_read_ulint, &count); - - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT COUNT(*)" - " FROM $table_name;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - - break; /* Exit the loop. */ - } else { - fts_sql_rollback(trx); - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading" - " FTS table. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "(" << error - << ") while reading FTS table " - << table_name; - - break; /* Exit the loop. */ - } - } - } - - que_graph_free(graph); - - trx->free(); - - return(count); -} - /*********************************************************************//** Free the modified rows of a table. */ UNIV_INLINE @@ -5576,287 +4961,264 @@ FTS. bool fts_load_stopword( /*==============*/ - const dict_table_t* - table, /*!< in: Table with FTS */ - trx_t* trx, /*!< in: Transactions */ - const char* session_stopword_table, /*!< in: Session stopword table - name */ - bool stopword_is_on, /*!< in: Whether stopword - option is turned on/off */ - bool reload) /*!< in: Whether it is - for reloading FTS table */ + FTSQueryExecutor* executor, /*!< in: FTSQueryExecutor */ + const dict_table_t* table) /*!< in: Table with FTS */ { - fts_table_t fts_table; + fts_cache_t* cache = table->fts->cache; fts_string_t str; dberr_t error = DB_SUCCESS; - ulint use_stopword; - fts_cache_t* cache; const char* stopword_to_use = NULL; - ibool new_trx = FALSE; byte str_buffer[MAX_FULL_NAME_LEN + 1]; - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, table); - - cache = table->fts->cache; - - if (!reload && !(cache->stopword_info.status & STOPWORD_NOT_INIT)) { + /* If stopword is already initialized, return success */ + if (!(cache->stopword_info.status & STOPWORD_NOT_INIT)) { return true; } - if (!trx) { - trx = trx_create(); -#ifdef UNIV_DEBUG - trx->start_line = __LINE__; - trx->start_file = __FILE__; -#endif - trx_start_internal_low(trx, !high_level_read_only); - trx->op_info = "upload FTS stopword"; - new_trx = TRUE; - } - - /* First check whether stopword filtering is turned off */ - if (reload) { - error = fts_config_get_ulint( - trx, &fts_table, FTS_USE_STOPWORD, &use_stopword); - } else { - use_stopword = (ulint) stopword_is_on; - - error = fts_config_set_ulint( - trx, &fts_table, FTS_USE_STOPWORD, use_stopword); - } - + /* First check if stopwords are enabled */ + ulint use_stopword = 0; + error = fts_config_get_ulint(executor, table, + FTS_USE_STOPWORD, &use_stopword); + if (error != DB_SUCCESS) { - goto cleanup; + /* Failed to read config */ + return false; } - - /* If stopword is turned off, no need to continue to load the - stopword into cache, but still need to do initialization */ + if (!use_stopword) { + /* Stopwords are disabled, mark as initialized but off */ cache->stopword_info.status = STOPWORD_OFF; - goto cleanup; + return true; } - if (reload) { - /* Fetch the stopword table name from FTS config - table */ - str.f_n_char = 0; - str.f_str = str_buffer; - str.f_len = sizeof(str_buffer) - 1; - - error = fts_config_get_value( - trx, &fts_table, FTS_STOPWORD_TABLE_NAME, &str); + /* Read FTS_STOPWORD_TABLE_NAME from CONFIG table */ + str.f_n_char = 0; + str.f_str = str_buffer; + str.f_len = sizeof(str_buffer) - 1; - if (error != DB_SUCCESS) { - goto cleanup; - } + error = fts_config_get_value(executor, table, FTS_STOPWORD_TABLE_NAME, &str); - if (*str.f_str) { - stopword_to_use = (const char*) str.f_str; - } - } else { - stopword_to_use = session_stopword_table; + if (error == DB_SUCCESS && *str.f_str) { + stopword_to_use = (const char*) str.f_str; } + /* Load user stopword table if specified, otherwise load default */ if (stopword_to_use - && fts_load_user_stopword(table->fts, stopword_to_use, - &cache->stopword_info)) { - /* Save the stopword table name to the configure - table */ - if (!reload) { - str.f_n_char = 0; - str.f_str = (byte*) stopword_to_use; - str.f_len = strlen(stopword_to_use); - - error = fts_config_set_value( - trx, &fts_table, FTS_STOPWORD_TABLE_NAME, &str); - } + && fts_load_user_stopword(executor, table->fts, stopword_to_use, + &cache->stopword_info)) { + /* Successfully loaded user stopword table */ } else { /* Load system default stopword list */ fts_load_default_stopword(&cache->stopword_info); } -cleanup: - if (new_trx) { - if (error == DB_SUCCESS) { - fts_sql_commit(trx); - } else { - fts_sql_rollback(trx); - } - - trx->clear_and_free(); - } - + /* Initialize cached_stopword RB-tree if not already created */ if (!cache->stopword_info.cached_stopword) { cache->stopword_info.cached_stopword = rbt_create_arg_cmp( sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, &my_charset_latin1); } - return error == DB_SUCCESS; + return true; } -/**********************************************************************//** -Callback function when we initialize the FTS at the start up -time. It recovers the maximum Doc IDs presented in the current table. -Tested by innodb_fts.crash_recovery -@return: always returns TRUE */ -static -ibool -fts_init_get_doc_id( -/*================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: table with fts */ -{ - doc_id_t doc_id = FTS_NULL_DOC_ID; - sel_node_t* node = static_cast(row); - que_node_t* exp = node->select_list; - dict_table_t* table = static_cast(user_arg); - fts_cache_t* cache = table->fts->cache; - - ut_ad(ib_vector_is_empty(cache->get_docs)); - - /* Copy each indexed column content into doc->text.f_str */ - if (exp) { - dfield_t* dfield = que_node_get_val(exp); - dtype_t* type = dfield_get_type(dfield); - void* data = dfield_get_data(dfield); - - ut_a(dtype_get_mtype(type) == DATA_INT); - - doc_id = static_cast(mach_read_from_8( - static_cast(data))); - - exp = que_node_get_next(que_node_get_next(exp)); - if (exp) { - ut_ad(table->versioned()); - dfield = que_node_get_val(exp); - type = dfield_get_type(dfield); - ut_ad(type->vers_sys_end()); - data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - if (table->versioned_by_id()) { - ut_ad(len == sizeof trx_id_max_bytes); - if (0 != memcmp(data, trx_id_max_bytes, len)) { - return true; - } - } else { - ut_ad(len == sizeof timestamp_max_bytes); - if (!IS_MAX_TIMESTAMP(data)) { - return true; - } - } - ut_ad(!(exp = que_node_get_next(exp))); - } - ut_ad(!exp); - - if (doc_id >= cache->next_doc_id) { - cache->next_doc_id = doc_id + 1; - } - } - - return(TRUE); -} - -/**********************************************************************//** -Callback function when we initialize the FTS at the start up +/** Callback function when we initialize the FTS at the start up time. It recovers Doc IDs that have not sync-ed to the auxiliary table, and require to bring them back into FTS index. +@param executor query executor +@param get_doc Document +@param doc_id document id to be fetched @return: always returns TRUE */ -static -ibool -fts_init_recover_doc( -/*=================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts cache */ +static void fts_init_recover_all_docs(FTSQueryExecutor *executor, + fts_get_doc_t *get_doc, + doc_id_t doc_id) { + executor->trx()->op_info= "fetching indexed FTS document"; + dict_index_t *fts_index= get_doc->index_cache->index; + dict_table_t *user_table= fts_index->table; + dict_index_t *fts_doc_id_index= user_table->fts_doc_id_index; + dict_index_t *clust_index= dict_table_get_first_index(user_table); + fts_cache_t *cache= get_doc->cache; + ut_a(user_table->fts->doc_col != ULINT_UNDEFINED); + ut_a(fts_doc_id_index); + /* Map FTS index columns to clustered index field positions */ + ulint *clust_field_nos= static_cast( + mem_heap_alloc(executor->get_heap(), + fts_index->n_user_defined_cols * sizeof(ulint))); + + for (unsigned i= 0; i < fts_index->n_user_defined_cols; i++) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + clust_field_nos[i]= dict_col_get_index_pos(fts_field->col, clust_index); + } - fts_doc_t doc; - ulint doc_len = 0; - ulint field_no = 0; - fts_get_doc_t* get_doc = static_cast(user_arg); - doc_id_t doc_id = FTS_NULL_DOC_ID; - sel_node_t* node = static_cast(row); - que_node_t* exp = node->select_list; - fts_cache_t* cache = get_doc->cache; - st_mysql_ftparser* parser = get_doc->index_cache->index->parser; - - fts_doc_init(&doc); - doc.found = TRUE; - - ut_ad(cache); - - /* Copy each indexed column content into doc->text.f_str */ - while (exp) { - dfield_t* dfield = que_node_get_val(exp); - ulint len = dfield_get_len(dfield); - - if (field_no == 0) { - dtype_t* type = dfield_get_type(dfield); - void* data = dfield_get_data(dfield); - - ut_a(dtype_get_mtype(type) == DATA_INT); - - doc_id = static_cast(mach_read_from_8( - static_cast(data))); + dfield_t fields[1]; + dtuple_t search_tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, fts_doc_id_index, 1); + dfield_t* dfield= dtuple_get_nth_field(&search_tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id((byte*) &write_doc_id, doc_id); + dfield_set_data(dfield, &write_doc_id, sizeof(write_doc_id)); + + auto process_doc_recovery= [get_doc, cache, user_table, fts_index, + clust_field_nos](const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + fts_doc_t doc; + ulint doc_len= 0; + doc_id_t doc_id= FTS_NULL_DOC_ID; + st_mysql_ftparser* parser= fts_index->parser; - field_no++; - exp = que_node_get_next(exp); - continue; - } + fts_doc_init(&doc); + doc.found= TRUE; - if (len == UNIV_SQL_NULL) { - exp = que_node_get_next(exp); - continue; - } + /* Extract doc_id from the clustered index record */ + ulint doc_col_pos= dict_col_get_index_pos( + &user_table->cols[user_table->fts->doc_col], index); - ut_ad(get_doc); + ulint processed_field= 0; + ulint len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, doc_col_pos, &len); - if (!get_doc->index_cache->charset) { - get_doc->index_cache->charset = fts_get_charset( - dfield->type.prtype); - } + if (len == sizeof(doc_id_t)) + { + doc_id= fts_read_doc_id(doc_id_data); - doc.charset = get_doc->index_cache->charset; + /* Process each indexed column content */ + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, col_pos, + &field_len); + if (field_len == UNIV_SQL_NULL) + continue; + if (!get_doc->index_cache->charset) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + get_doc->index_cache->charset= fts_get_charset( + fts_field->col->prtype); + } + doc.charset= get_doc->index_cache->charset; + + /* Handle externally stored fields */ + if (rec_offs_nth_extern(offsets, col_pos)) + doc.text.f_str= btr_copy_externally_stored_field( + &doc.text.f_len, const_cast(field_data), + user_table->space->zip_size(), field_len, + static_cast(doc.self_heap->arg)); + else + { + doc.text.f_str= const_cast(field_data); + doc.text.f_len= field_len; + } - if (dfield_is_ext(dfield)) { - dict_table_t* table = cache->sync->table; + if (processed_field == 0) + fts_tokenize_document(&doc, NULL, parser); + else + fts_tokenize_document_next(&doc, doc_len, NULL, parser); - doc.text.f_str = btr_copy_externally_stored_field( - &doc.text.f_len, - static_cast(dfield_get_data(dfield)), - table->space->zip_size(), len, - static_cast(doc.self_heap->arg)); - } else { - doc.text.f_str = static_cast( - dfield_get_data(dfield)); + processed_field++; + doc_len+= + (i < (unsigned) get_doc->index_cache->index->n_user_defined_cols - 1) + ? field_len + 1 + : field_len; + } - doc.text.f_len = len; - } + fts_cache_add_doc(cache, get_doc->index_cache, doc_id, doc.tokens); + fts_doc_free(&doc); + cache->added++; - if (field_no == 1) { - fts_tokenize_document(&doc, NULL, parser); - } else { - fts_tokenize_document_next(&doc, doc_len, NULL, parser); - } - - exp = que_node_get_next(exp); + if (doc_id >= cache->next_doc_id) + cache->next_doc_id= doc_id + 1; + } - doc_len += (exp) ? len + 1 : len; + return DB_SUCCESS; + }; + RecordCallback reader(process_doc_recovery, doc_id_comparator); + if (fts_doc_id_index == clust_index) + executor->read(user_table, &search_tuple, PAGE_CUR_G, reader); + else + executor->read_by_index(user_table, fts_doc_id_index, + &search_tuple, PAGE_CUR_G, reader); +} - field_no++; - } +/** Get the next large document id and update it in fulltext cache +@param executor query executor +@param doc_id document id to be updated +@param index fulltext index */ +static void fts_init_get_doc_id(FTSQueryExecutor *executor, doc_id_t doc_id, dict_index_t *index) +{ + executor->trx()->op_info= "fetching indexed FTS document"; + dict_table_t* user_table= index->table; + fts_cache_t* cache= user_table->fts->cache; + ut_a(user_table->fts->doc_col != ULINT_UNDEFINED); - fts_cache_add_doc(cache, get_doc->index_cache, doc_id, doc.tokens); + dfield_t fields[1]; + dtuple_t search_tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, index, 1); + dfield_t* dfield= dtuple_get_nth_field(&search_tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id((byte*) &write_doc_id, doc_id); + dfield_set_data(dfield, &write_doc_id, sizeof(write_doc_id)); + + auto process_doc_id= [cache, user_table](const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + ulint doc_col_pos= dict_col_get_index_pos( + &user_table->cols[user_table->fts->doc_col], index); - fts_doc_free(&doc); + ulint len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, doc_col_pos, &len); - cache->added++; + if (len == sizeof(doc_id_t)) + { + doc_id_t found_doc_id= mach_read_from_8(doc_id_data); + if (user_table->versioned()) + { + ulint vers_end_pos= dict_col_get_index_pos( + &user_table->cols[user_table->vers_end], index); + ulint vers_len; + const byte* vers_data= rec_get_nth_field(rec, offsets, + vers_end_pos, &vers_len); - if (doc_id >= cache->next_doc_id) { - cache->next_doc_id = doc_id + 1; - } + if (user_table->versioned_by_id()) + { + if (vers_len == sizeof(trx_id_max_bytes) && + memcmp(vers_data, trx_id_max_bytes, vers_len) != 0) + return DB_SUCCESS; + } + else + { + if (vers_len == sizeof(timestamp_max_bytes) && + !IS_MAX_TIMESTAMP(vers_data)) + return DB_SUCCESS; + } + } - return(TRUE); + /* Update cache->next_doc_id if this doc_id is larger */ + if (found_doc_id >= cache->next_doc_id) + cache->next_doc_id= found_doc_id + 1; + } + return DB_SUCCESS; + }; + + RecordCallback reader(process_doc_id, doc_id_comparator); + if (dict_index_is_clust(index)) + executor->read(user_table, &search_tuple, PAGE_CUR_G, reader); + else + executor->read_by_index(user_table, index, &search_tuple, + PAGE_CUR_G, reader); } /**********************************************************************//** @@ -5876,6 +5238,9 @@ fts_init_index( fts_get_doc_t* get_doc = NULL; fts_cache_t* cache = table->fts->cache; bool need_init = false; + /* Declare variables before any goto to avoid initialization bypass */ + trx_t *trx = nullptr; + FTSQueryExecutor *executor= nullptr; /* First check cache->get_docs is initialized */ if (!has_cache_lock) { @@ -5896,13 +5261,19 @@ fts_init_index( start_doc = cache->synced_doc_id; + /* Create single FTSQueryExecutor for all operations */ + trx = trx_create(); + trx_start_internal_read_only(trx); + executor= new FTSQueryExecutor(trx, table); + if (!start_doc) { - trx_t *trx = trx_create(); - trx_start_internal_read_only(trx); - dberr_t err= fts_read_synced_doc_id(table, &start_doc, trx); - fts_sql_commit(trx); - trx->free(); + dberr_t err= fts_read_synced_doc_id( + executor, table, &start_doc); if (err != DB_SUCCESS) { + fts_sql_commit(trx); + trx->free(); + delete executor; + executor = nullptr; goto func_exit; } if (start_doc) { @@ -5919,13 +5290,11 @@ fts_init_index( ut_a(index); - fts_doc_fetch_by_doc_id(NULL, start_doc, index, - FTS_FETCH_DOC_BY_ID_LARGE, - fts_init_get_doc_id, table); + fts_init_get_doc_id(executor, start_doc, index); } else { if (table->fts->cache->stopword_info.status & STOPWORD_NOT_INIT) { - fts_load_stopword(table, NULL, NULL, true, true); + fts_load_stopword(executor, table); } for (ulint i = 0; i < ib_vector_size(cache->get_docs); ++i) { @@ -5934,9 +5303,7 @@ fts_init_index( index = get_doc->index_cache->index; - fts_doc_fetch_by_doc_id(NULL, start_doc, index, - FTS_FETCH_DOC_BY_ID_LARGE, - fts_init_recover_doc, get_doc); + fts_init_recover_all_docs(executor, get_doc, start_doc); } } @@ -5944,6 +5311,15 @@ fts_init_index( fts_get_docs_clear(cache->get_docs); + /* Commit transaction and cleanup */ + if (trx) { + fts_sql_commit(trx); + trx->free(); + } + if (executor) { + delete executor; + } + func_exit: if (!has_cache_lock) { mysql_mutex_unlock(&cache->lock); diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 76380b58d3b39..fb469419709dd 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -27,6 +27,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang ***********************************************************************/ #include "fts0fts.h" +#include "fts0exec.h" #include "row0sel.h" #include "que0types.h" #include "fts0priv.h" @@ -70,9 +71,6 @@ static bool fts_opt_start_shutdown = false; Protected by fts_optimize_wq->mutex. */ static pthread_cond_t fts_opt_shutdown_cond; -/** Initial size of nodes in fts_word_t. */ -static const ulint FTS_WORD_NODES_INIT_SIZE = 64; - /** Last time we did check whether system need a sync */ static time_t last_check_sync_time; @@ -140,11 +138,6 @@ struct fts_optimize_t { char* name_prefix; /*!< FTS table name prefix */ - fts_table_t fts_index_table;/*!< Common table definition */ - - /*!< Common table definition */ - fts_table_t fts_common_table; - dict_table_t* table; /*!< Table that has to be queried */ dict_index_t* index; /*!< The FTS index to be optimized */ @@ -235,28 +228,6 @@ static ulint fts_optimize_time_limit; /** It's defined in fts0fts.cc */ extern const char* fts_common_tables[]; -/** SQL Statement for changing state of rows to be deleted from FTS Index. */ -static const char* fts_init_delete_sql = - "BEGIN\n" - "\n" - "INSERT INTO $BEING_DELETED\n" - "SELECT doc_id FROM $DELETED;\n" - "\n" - "INSERT INTO $BEING_DELETED_CACHE\n" - "SELECT doc_id FROM $DELETED_CACHE;\n"; - -static const char* fts_delete_doc_ids_sql = - "BEGIN\n" - "\n" - "DELETE FROM $DELETED WHERE doc_id = :doc_id1;\n" - "DELETE FROM $DELETED_CACHE WHERE doc_id = :doc_id2;\n"; - -static const char* fts_end_delete_sql = - "BEGIN\n" - "\n" - "DELETE FROM $BEING_DELETED;\n" - "DELETE FROM $BEING_DELETED_CACHE;\n"; - /**********************************************************************//** Initialize fts_zip_t. */ static @@ -327,238 +298,52 @@ fts_zip_init( *zip->word.f_str = '\0'; } -/**********************************************************************//** -Create a fts_optimizer_word_t instance. -@return new instance */ -static -fts_word_t* -fts_word_init( -/*==========*/ - fts_word_t* word, /*!< in: word to initialize */ - byte* utf8, /*!< in: UTF-8 string */ - ulint len) /*!< in: length of string in bytes */ -{ - mem_heap_t* heap = mem_heap_create(sizeof(fts_node_t)); - - memset(word, 0, sizeof(*word)); - - word->text.f_len = len; - word->text.f_str = static_cast(mem_heap_alloc(heap, len + 1)); - - /* Need to copy the NUL character too. */ - memcpy(word->text.f_str, utf8, word->text.f_len); - word->text.f_str[word->text.f_len] = 0; - - word->heap_alloc = ib_heap_allocator_create(heap); - - word->nodes = ib_vector_create( - word->heap_alloc, sizeof(fts_node_t), FTS_WORD_NODES_INIT_SIZE); - - return(word); -} - -/**********************************************************************//** -Read the FTS INDEX row. -@return fts_node_t instance */ -static -fts_node_t* -fts_optimize_read_node( -/*===================*/ - fts_word_t* word, /*!< in: */ - que_node_t* exp) /*!< in: */ -{ - int i; - fts_node_t* node = static_cast( - ib_vector_push(word->nodes, NULL)); - - /* Start from 1 since the first node has been read by the caller */ - for (i = 1; exp; exp = que_node_get_next(exp), ++i) { - - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - ulint len = dfield_get_len(dfield); - - ut_a(len != UNIV_SQL_NULL); - - /* Note: The column numbers below must match the SELECT */ - switch (i) { - case 1: /* DOC_COUNT */ - node->doc_count = mach_read_from_4(data); - break; - - case 2: /* FIRST_DOC_ID */ - node->first_doc_id = fts_read_doc_id(data); - break; - - case 3: /* LAST_DOC_ID */ - node->last_doc_id = fts_read_doc_id(data); - break; - - case 4: /* ILIST */ - node->ilist_size_alloc = node->ilist_size = len; - node->ilist = static_cast(ut_malloc_nokey(len)); - memcpy(node->ilist, data, len); - break; - - default: - ut_error; - } - } - - /* Make sure all columns were read. */ - ut_a(i == 5); - - return(node); -} - -/**********************************************************************//** -Callback function to fetch the rows in an FTS INDEX record. -@return always returns non-NULL */ -ibool -fts_optimize_index_fetch_node( -/*==========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - fts_word_t* word; - sel_node_t* sel_node = static_cast(row); - fts_fetch_t* fetch = static_cast(user_arg); - ib_vector_t* words = static_cast(fetch->read_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint dfield_len = dfield_get_len(dfield); - fts_node_t* node; - bool is_word_init = false; - - ut_a(dfield_len <= FTS_MAX_WORD_LEN); - - if (ib_vector_size(words) == 0) { - - word = static_cast(ib_vector_push(words, NULL)); - fts_word_init(word, (byte*) data, dfield_len); - is_word_init = true; - } - - word = static_cast(ib_vector_last(words)); - - if (dfield_len != word->text.f_len - || memcmp(word->text.f_str, data, dfield_len)) { - - word = static_cast(ib_vector_push(words, NULL)); - fts_word_init(word, (byte*) data, dfield_len); - is_word_init = true; - } - - node = fts_optimize_read_node(word, que_node_get_next(exp)); - - fetch->total_memory += node->ilist_size; - if (is_word_init) { - fetch->total_memory += sizeof(fts_word_t) - + sizeof(ib_alloc_t) + sizeof(ib_vector_t) + dfield_len - + sizeof(fts_node_t) * FTS_WORD_NODES_INIT_SIZE; - } else if (ib_vector_size(words) > FTS_WORD_NODES_INIT_SIZE) { - fetch->total_memory += sizeof(fts_node_t); - } - - if (fetch->total_memory >= fts_result_cache_limit) { - return(FALSE); - } - - return(TRUE); -} - -/**********************************************************************//** -Read the rows from the FTS inde. -@return DB_SUCCESS or error code */ -dberr_t -fts_index_fetch_nodes( -/*==================*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: prepared statement */ - fts_table_t* fts_table, /*!< in: table of the FTS INDEX */ - const fts_string_t* - word, /*!< in: the word to fetch */ - fts_fetch_t* fetch) /*!< in: fetch callback.*/ +dberr_t fts_index_fetch_nodes(FTSQueryExecutor *executor, dict_index_t *index, + const fts_string_t *word, void *user_arg, + FTSRecordProcessor processor, + AuxCompareMode compare_mode) { - pars_info_t* info; - dberr_t error; - char table_name[MAX_FULL_NAME_LEN]; - - trx->op_info = "fetching FTS index nodes"; - - if (*graph) { - info = (*graph)->info; - } else { - ulint selected; - - info = pars_info_create(); - - ut_a(fts_table->type == FTS_INDEX_TABLE); - - selected = fts_select_index(fts_table->charset, - word->f_str, word->f_len); - - fts_table->suffix = fts_get_suffix(selected); - - fts_get_table_name(fts_table, table_name); - - pars_info_bind_id(info, "table_name", table_name); - } - - pars_info_bind_function(info, "my_func", fetch->read_record, fetch); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - if (!*graph) { - - *graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT word, doc_count, first_doc_id, last_doc_id," - " ilist\n" - " FROM $table_name\n" - " WHERE word LIKE :word\n" - " ORDER BY first_doc_id;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - } - - for (;;) { - error = fts_eval_sql(trx, *graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - - break; /* Exit the loop. */ - } else { - fts_sql_rollback(trx); - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading" - " FTS index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "(" << error - << ") while reading FTS index."; - - break; /* Exit the loop. */ - } - } - } - - return(error); + dberr_t error= DB_SUCCESS; + trx_t *trx= executor->trx(); + trx->op_info= "fetching FTS index nodes"; + CHARSET_INFO *cs= fts_index_get_charset(index); + uint8_t selected= fts_select_index(cs, word->f_str, word->f_len); + ulint total_memory= 0; + for (;;) + { + AuxRecordReader reader= processor + ? AuxRecordReader(user_arg, processor, compare_mode) + : AuxRecordReader(user_arg, &total_memory, compare_mode); + if (word->f_len == 0) + error= executor->read_aux_all(selected, reader); + else + error= executor->read_aux( + selected, reinterpret_cast(word->f_str), + PAGE_CUR_GE, reader); + if (UNIV_LIKELY(error == DB_SUCCESS || error == DB_RECORD_NOT_FOUND)) + { + if (error == DB_RECORD_NOT_FOUND) error = DB_SUCCESS; + fts_sql_commit(trx); + break; + } + else + { + fts_sql_rollback(trx); + if (error == DB_LOCK_WAIT_TIMEOUT) + { + sql_print_warning("InnoDB: Lock wait timeout reading FTS index." + "Retrying!"); + trx->error_state= DB_SUCCESS; + } + else + { + sql_print_error("InnoDB: (%s) while reading FTS index.", + ut_strerr(error)); + break; + } + } + } + return error; } /**********************************************************************//** @@ -662,88 +447,6 @@ fts_zip_read_word( return(zip->status == Z_OK || zip->status == Z_STREAM_END ? ptr : NULL); } -/**********************************************************************//** -Callback function to fetch and compress the word in an FTS -INDEX record. -@return FALSE on EOF */ -static -ibool -fts_fetch_index_words( -/*==================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - sel_node_t* sel_node = static_cast(row); - fts_zip_t* zip = static_cast(user_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - - ut_a(dfield_get_len(dfield) <= FTS_MAX_WORD_LEN); - - uint16 len = uint16(dfield_get_len(dfield)); - void* data = dfield_get_data(dfield); - - /* Skip the duplicate words. */ - if (zip->word.f_len == len && !memcmp(zip->word.f_str, data, len)) { - return(TRUE); - } - - memcpy(zip->word.f_str, data, len); - zip->word.f_len = len; - - ut_a(zip->zp->avail_in == 0); - ut_a(zip->zp->next_in == NULL); - - /* The string is prefixed by len. */ - /* FIXME: This is not byte order agnostic (InnoDB data files - with FULLTEXT INDEX are not portable between little-endian and - big-endian systems!) */ - zip->zp->next_in = reinterpret_cast(&len); - zip->zp->avail_in = sizeof(len); - - /* Compress the word, create output blocks as necessary. */ - while (zip->zp->avail_in > 0) { - - /* No space left in output buffer, create a new one. */ - if (zip->zp->avail_out == 0) { - byte* block; - - block = static_cast( - ut_malloc_nokey(zip->block_sz)); - - ib_vector_push(zip->blocks, &block); - - zip->zp->next_out = block; - zip->zp->avail_out = static_cast(zip->block_sz); - } - - switch (zip->status = deflate(zip->zp, Z_NO_FLUSH)) { - case Z_OK: - if (zip->zp->avail_in == 0) { - zip->zp->next_in = static_cast(data); - zip->zp->avail_in = uInt(len); - ut_a(len <= FTS_MAX_WORD_LEN); - len = 0; - } - continue; - - case Z_STREAM_END: - case Z_BUF_ERROR: - case Z_STREAM_ERROR: - default: - ut_error; - } - } - - /* All data should have been compressed. */ - ut_a(zip->zp->avail_in == 0); - zip->zp->next_in = NULL; - - ++zip->n_words; - - return(zip->n_words >= zip->max_words ? FALSE : TRUE); -} - /**********************************************************************//** Finish Zip deflate. */ static @@ -786,241 +489,190 @@ fts_zip_deflate_end( memset(zip->zp, 0, sizeof(*zip->zp)); } -/**********************************************************************//** -Read the words from the FTS INDEX. +/** Read the words from the FTS INDEX. +@param executor query executor +@param optim optimize scratch pad +@param word get words gerater than this +@param n_words max words to read @return DB_SUCCESS if all OK, DB_TABLE_NOT_FOUND if no more indexes to search else error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_index_fetch_words( -/*==================*/ - fts_optimize_t* optim, /*!< in: optimize scratch pad */ - const fts_string_t* word, /*!< in: get words greater than this - word */ - ulint n_words)/*!< in: max words to read */ +dberr_t fts_index_fetch_words(FTSQueryExecutor *executor, + fts_optimize_t *optim, + const fts_string_t *word, + ulint n_words) { - pars_info_t* info; - que_t* graph; - ulint selected; - fts_zip_t* zip = NULL; - dberr_t error = DB_SUCCESS; - mem_heap_t* heap = static_cast(optim->self_heap->arg); - ibool inited = FALSE; + dberr_t error= DB_SUCCESS; + mem_heap_t *heap= static_cast(optim->self_heap->arg); + optim->trx->op_info= "fetching FTS index words"; - optim->trx->op_info = "fetching FTS index words"; + if (optim->zip == NULL) + optim->zip = fts_zip_create(heap, FTS_ZIP_BLOCK_SIZE, n_words); + else fts_zip_initialize(optim->zip); - if (optim->zip == NULL) { - optim->zip = fts_zip_create(heap, FTS_ZIP_BLOCK_SIZE, n_words); - } else { - fts_zip_initialize(optim->zip); - } + CHARSET_INFO *cs= fts_index_get_charset(optim->index); - for (selected = fts_select_index( - optim->fts_index_table.charset, word->f_str, word->f_len); - selected < FTS_NUM_AUX_INDEX; - selected++) { - - char table_name[MAX_FULL_NAME_LEN]; - - optim->fts_index_table.suffix = fts_get_suffix(selected); - - info = pars_info_create(); - - pars_info_bind_function( - info, "my_func", fts_fetch_index_words, optim->zip); - - pars_info_bind_varchar_literal( - info, "word", word->f_str, word->f_len); - - fts_get_table_name(&optim->fts_index_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - &optim->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT word\n" - " FROM $table_name\n" - " WHERE word > :word\n" - " ORDER BY word;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - zip = optim->zip; - - for (;;) { - int err; - - if (!inited && ((err = deflateInit(zip->zp, 9)) - != Z_OK)) { - ib::error() << "ZLib deflateInit() failed: " - << err; - - error = DB_ERROR; - break; - } else { - inited = TRUE; - error = fts_eval_sql(optim->trx, graph); - } + /* Create compression processor with state */ + bool compress_inited = false; + auto compress_processor= [&compress_inited]( + const rec_t *rec, const dict_index_t *index, + const rec_offs *offsets, void *user_arg) -> dberr_t + { + fts_zip_t* zip= static_cast(user_arg); + AuxRecordReader::AuxRecordFields fields; - if (UNIV_LIKELY(error == DB_SUCCESS)) { - //FIXME fts_sql_commit(optim->trx); - break; - } else { - //FIXME fts_sql_rollback(optim->trx); + /* Use optimized field extraction (only need word field) */ + if (!AuxRecordReader::extract_aux_fields(rec, index, fields, true)) + return DB_SUCCESS; - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout" - " reading document. Retrying!"; + if (!fields.word_data || fields.word_len == UNIV_SQL_NULL || + fields.word_len > FTS_MAX_WORD_LEN) + return DB_SUCCESS; - /* We need to reset the ZLib state. */ - inited = FALSE; - deflateEnd(zip->zp); - fts_zip_init(zip); + /* Skip duplicate words */ + if (zip->word.f_len == fields.word_len && + !memcmp(zip->word.f_str, fields.word_data, fields.word_len)) + return DB_SUCCESS; - optim->trx->error_state = DB_SUCCESS; - } else { - ib::error() << "(" << error - << ") while reading document."; + /* Initialize deflate if not done yet */ + if (!compress_inited) + { + int err = deflateInit(zip->zp, 9); + if (err != Z_OK) + { + sql_print_error("InnoDB: ZLib deflateInit() failed: %d", err); + return DB_ERROR; + } + compress_inited = true; + } - break; /* Exit the loop. */ - } - } - } + /* Update current word */ + memcpy(zip->word.f_str, fields.word_data, fields.word_len); + zip->word.f_len = fields.word_len; + ut_a(zip->zp->avail_in == 0); + ut_a(zip->zp->next_in == NULL); - que_graph_free(graph); + /* Compress the word with length prefix */ + uint16_t len = static_cast(fields.word_len); + zip->zp->next_in = reinterpret_cast(&len); + zip->zp->avail_in = sizeof(len); - /* Check if max word to fetch is exceeded */ - if (optim->zip->n_words >= n_words) { - break; - } - } + /* Compress the word, create output blocks as necessary */ + while (zip->zp->avail_in > 0) + { + /* No space left in output buffer, create a new one */ + if (zip->zp->avail_out == 0) + { + byte* block= static_cast(ut_malloc_nokey(zip->block_sz)); + ib_vector_push(zip->blocks, &block); + zip->zp->next_out= block; + zip->zp->avail_out= static_cast(zip->block_sz); + } + + switch (zip->status = deflate(zip->zp, Z_NO_FLUSH)) + { + case Z_OK: + if (zip->zp->avail_in == 0) + { + zip->zp->next_in= static_cast( + const_cast(fields.word_data)); + zip->zp->avail_in = static_cast(len); + ut_a(len <= FTS_MAX_WORD_LEN); + len = 0; + } + continue; + case Z_STREAM_END: + case Z_BUF_ERROR: + case Z_STREAM_ERROR: + default: + ut_error; + } + } - if (error == DB_SUCCESS && zip->status == Z_OK && zip->n_words > 0) { + /* All data should have been compressed */ + ut_a(zip->zp->avail_in == 0); + zip->zp->next_in = NULL; - /* All data should have been read. */ - ut_a(zip->zp->avail_in == 0); + ++zip->n_words; - fts_zip_deflate_end(zip); - } else { - deflateEnd(zip->zp); - } + /* Continue until we reach max words */ + return zip->n_words < zip->max_words ? DB_SUCCESS : DB_SUCCESS_LOCKED_REC; + }; - return(error); -} + for (uint8_t selected= fts_select_index(cs, word->f_str, word->f_len); + selected < FTS_NUM_AUX_INDEX; selected++) + { + for (;;) + { + AuxRecordReader aux_reader(optim->zip, compress_processor, + AuxCompareMode::GREATER); + + if (word->f_len == 0) + error= executor->read_aux_all(selected, aux_reader); + else error= executor->read_aux( + selected, + reinterpret_cast(word->f_str), + PAGE_CUR_G, aux_reader); + + if (UNIV_LIKELY(error == DB_SUCCESS || error == DB_RECORD_NOT_FOUND)) { + if (error == DB_RECORD_NOT_FOUND) error = DB_SUCCESS; + break; + } + else + { + if (error == DB_LOCK_WAIT_TIMEOUT) + { + sql_print_warning("InnoDB: Lock wait timeout reading " + "words. Retrying!"); + if (compress_inited) + { + deflateEnd(optim->zip->zp); + fts_zip_init(optim->zip); + compress_inited= false; + } + optim->trx->error_state = DB_SUCCESS; + } + else + { + sql_print_error("InnoDB: (%s) while reading words.", + ut_strerr(error)); + break; + } + } + } -/**********************************************************************//** -Callback function to fetch the doc id from the record. -@return always returns TRUE */ -static -ibool -fts_fetch_doc_ids( -/*==============*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - que_node_t* exp; - int i = 0; - sel_node_t* sel_node = static_cast(row); - fts_doc_ids_t* fts_doc_ids = static_cast(user_arg); - doc_id_t* update = static_cast( - ib_vector_push(fts_doc_ids->doc_ids, NULL)); - - for (exp = sel_node->select_list; - exp; - exp = que_node_get_next(exp), ++i) { - - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - - ut_a(len != UNIV_SQL_NULL); - - /* Note: The column numbers below must match the SELECT. */ - switch (i) { - case 0: /* DOC_ID */ - *update = fts_read_doc_id( - static_cast(data)); - break; + if (optim->zip->n_words >= n_words) break; + } - default: - ut_error; - } - } + fts_zip_t *zip = optim->zip; + if (error == DB_SUCCESS && zip->status == Z_OK && zip->n_words > 0) { + /* All data should have been read */ + ut_a(zip->zp->avail_in == 0); + fts_zip_deflate_end(zip); + } + else deflateEnd(zip->zp); - return(TRUE); + return error; } -/**********************************************************************//** -Read the rows from a FTS common auxiliary table. -@return DB_SUCCESS or error code */ -dberr_t -fts_table_fetch_doc_ids( -/*====================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: table */ - fts_doc_ids_t* doc_ids) /*!< in: For collecting doc ids */ +dberr_t fts_table_fetch_doc_ids(FTSQueryExecutor *executor, + dict_table_t *table, const char *tbl_name, + fts_doc_ids_t *doc_ids) noexcept { - dberr_t error; - que_t* graph; - pars_info_t* info = pars_info_create(); - ibool alloc_bk_trx = FALSE; - char table_name[MAX_FULL_NAME_LEN]; - - ut_a(fts_table->suffix != NULL); - ut_a(fts_table->type == FTS_COMMON_TABLE); - - if (!trx) { - trx = trx_create(); - alloc_bk_trx = TRUE; - } - - trx->op_info = "fetching FTS doc ids"; - - pars_info_bind_function(info, "my_func", fts_fetch_doc_ids, doc_ids); - - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT doc_id FROM $table_name;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - error = fts_eval_sql(trx, graph); - fts_sql_commit(trx); - que_graph_free(graph); - - if (error == DB_SUCCESS) { - fts_doc_ids_sort(doc_ids->doc_ids); - } - - if (alloc_bk_trx) { - trx->free(); - } + ut_ad(executor != nullptr); + executor->trx()->op_info = "fetching FTS doc ids"; + CommonTableReader reader; + dberr_t err= executor->read_all_common(tbl_name, reader); - return(error); + if (err == DB_SUCCESS) + { + const auto& doc_id_vector= reader.get_doc_ids(); + for (doc_id_t doc_id : doc_id_vector) + ib_vector_push(doc_ids->doc_ids, &doc_id); + fts_doc_ids_sort(doc_ids->doc_ids); + } + + return err; } /**********************************************************************//** @@ -1419,87 +1071,53 @@ fts_optimize_word( return(nodes); } -/**********************************************************************//** -Update the FTS index table. This is a delete followed by an insert. +/** Write the words and ilist to disk. +@param executor query executor +@param index fulltext index +@param word word to update +@param nodes nodes to update @return DB_SUCCESS or error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_write_word( -/*====================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: table of FTS index */ - fts_string_t* word, /*!< in: word data to write */ - ib_vector_t* nodes) /*!< in: the nodes to write */ +dberr_t fts_optimize_write_word(FTSQueryExecutor *executor, dict_index_t *index, + fts_string_t *word, ib_vector_t *nodes) { - ulint i; - pars_info_t* info; - que_t* graph; - ulint selected; - dberr_t error = DB_SUCCESS; - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - ut_ad(fts_table->charset); - - pars_info_bind_varchar_literal( - info, "word", word->f_str, word->f_len); - - selected = fts_select_index(fts_table->charset, - word->f_str, word->f_len); - - fts_table->suffix = fts_get_suffix(selected); - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "BEGIN DELETE FROM $table_name WHERE word = :word;"); - - error = fts_eval_sql(trx, graph); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") during optimize," - " when deleting a word from the FTS index."; - } - - que_graph_free(graph); - graph = NULL; - - /* Even if the operation needs to be rolled back and redone, - we iterate over the nodes in order to free the ilist. */ - for (i = 0; i < ib_vector_size(nodes); ++i) { - - fts_node_t* node = (fts_node_t*) ib_vector_get(nodes, i); - - if (error == DB_SUCCESS) { - /* Skip empty node. */ - if (node->ilist == NULL) { - ut_ad(node->ilist_size == 0); - continue; - } - - error = fts_write_node( - trx, &graph, fts_table, word, node); + CHARSET_INFO *cs= fts_index_get_charset(index); + uint8_t selected= fts_select_index(cs, word->f_str, word->f_len); + fts_aux_data_t aux_data((const char*)word->f_str, word->f_len); + dberr_t err= executor->delete_aux_record(selected, &aux_data); + if (err != DB_SUCCESS) + { + sql_print_error("InnoDB: (%s) during optimize, when " + "deleting a word from the FTS index.", + ut_strerr(err)); + return err; + } - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ")" - " during optimize, while adding a" - " word to the FTS index."; - } - } + for (ulint i = 0; i < ib_vector_size(nodes); ++i) + { + fts_node_t* node = (fts_node_t*) ib_vector_get(nodes, i); + if (!node->ilist || node->ilist_size == 0) continue; - ut_free(node->ilist); - node->ilist = NULL; - node->ilist_size = node->ilist_size_alloc = 0; - } + fts_aux_data_t insert_data( + (const char*)word->f_str, word->f_len, + node->first_doc_id, node->last_doc_id, + static_cast(node->doc_count), node->ilist, + node->ilist_size); - if (graph != NULL) { - que_graph_free(graph); - } + err = executor->insert_aux_record(selected, &insert_data); + if (err != DB_SUCCESS) + { + sql_print_error("InnoDB: (%s) during optimize, when " + "inserting a word to the FTS index.", + ut_strerr(err)); + return err; + } + ut_free(node->ilist); + node->ilist= nullptr; + node->ilist_size= node->ilist_size_alloc= 0; + } - return(error); + return DB_SUCCESS; } /**********************************************************************//** @@ -1519,12 +1137,14 @@ fts_word_free( } /**********************************************************************//** -Optimize the word ilist and rewrite data to the FTS index. +Compact the nodes for a given word, the nodes passed in are +already optimized. @return status one of RESTART, EXIT, ERROR */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_compact( /*=================*/ + FTSQueryExecutor* executor, /*!< in: query executor */ fts_optimize_t* optim, /*!< in: optimize state data */ dict_index_t* index, /*!< in: current FTS being optimized */ time_t start_time) /*!< in: optimize start time */ @@ -1536,7 +1156,6 @@ fts_optimize_compact( for (i = 0; i < size && error == DB_SUCCESS && !optim->done; ++i) { fts_word_t* word; ib_vector_t* nodes; - trx_t* trx = optim->trx; word = (fts_word_t*) ib_vector_get(optim->words, i); @@ -1547,13 +1166,13 @@ fts_optimize_compact( /* Update the data on disk. */ error = fts_optimize_write_word( - trx, &optim->fts_index_table, &word->text, nodes); + executor, index, &word->text, nodes); if (error == DB_SUCCESS) { /* Write the last word optimized to the config table, we use this value for restarting optimize. */ error = fts_config_set_index_value( - optim->trx, index, + executor, index, FTS_LAST_OPTIMIZED_WORD, &word->text); } @@ -1599,18 +1218,10 @@ fts_optimize_create( optim->trx = trx_create(); trx_start_internal(optim->trx); - optim->fts_common_table.table_id = table->id; - optim->fts_common_table.type = FTS_COMMON_TABLE; - optim->fts_common_table.table = table; - - optim->fts_index_table.table_id = table->id; - optim->fts_index_table.type = FTS_INDEX_TABLE; - optim->fts_index_table.table = table; - /* The common prefix for all this parent table's aux tables. */ char table_id[FTS_AUX_MIN_TABLE_ID_LENGTH]; const size_t table_id_len = 1 - + size_t(fts_get_table_id(&optim->fts_common_table, table_id)); + + size_t(fts_write_object_id(table->id, table_id)); dict_sys.freeze(SRW_LOCK_CALL); /* Include the separator as well. */ const size_t dbname_len = table->name.dblen() + 1; @@ -1680,175 +1291,128 @@ fts_optimize_free( mem_heap_free(heap); } -/**********************************************************************//** -Get the max time optimize should run in millisecs. +/** Get the max time optimize should run in millisecs. +@param executor query executor +@param table user table to be optimized @return max optimize time limit in millisecs. */ static -ulint -fts_optimize_get_time_limit( -/*========================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table) /*!< in: aux table */ +ulint fts_optimize_get_time_limit(FTSQueryExecutor *executor, const dict_table_t *table) { - ulint time_limit = 0; - - fts_config_get_ulint( - trx, fts_table, - FTS_OPTIMIZE_LIMIT_IN_SECS, &time_limit); - - /* FIXME: This is returning milliseconds, while the variable - is being stored and interpreted as seconds! */ - return(time_limit * 1000); + ulint time_limit= 0; + fts_config_get_ulint(executor, table, FTS_OPTIMIZE_LIMIT_IN_SECS, &time_limit); + /* FIXME: This is returning milliseconds, while the variable + is being stored and interpreted as seconds! */ + return(time_limit * 1000); } -/**********************************************************************//** -Run OPTIMIZE on the given table. Note: this can take a very long time -(hours). */ +/** Run OPTIMIZE on the given table. Note: this can take a very +long time (hours). +@param executor query executor +@param optim optimize instance +@param index current fts being optimized +@param word starting word to optimize */ static -void -fts_optimize_words( -/*===============*/ - fts_optimize_t* optim, /*!< in: optimize instance */ - dict_index_t* index, /*!< in: current FTS being optimized */ - fts_string_t* word) /*!< in: the starting word to optimize */ +void fts_optimize_words(FTSQueryExecutor *executor, fts_optimize_t *optim, + dict_index_t *index, fts_string_t *word) { - fts_fetch_t fetch; - que_t* graph = NULL; - CHARSET_INFO* charset = optim->fts_index_table.charset; - - ut_a(!optim->done); - - /* Get the time limit from the config table. */ - fts_optimize_time_limit = fts_optimize_get_time_limit( - optim->trx, &optim->fts_common_table); - - const time_t start_time = time(NULL); - - /* Setup the callback to use for fetching the word ilist etc. */ - fetch.read_arg = optim->words; - fetch.read_record = fts_optimize_index_fetch_node; - - while (!optim->done) { - dberr_t error; - trx_t* trx = optim->trx; - ulint selected; - - ut_a(ib_vector_size(optim->words) == 0); - - selected = fts_select_index(charset, word->f_str, word->f_len); - - /* Read the index records to optimize. */ - fetch.total_memory = 0; - error = fts_index_fetch_nodes( - trx, &graph, &optim->fts_index_table, word, - &fetch); - ut_ad(fetch.total_memory < fts_result_cache_limit); - - if (error == DB_SUCCESS) { - /* There must be some nodes to read. */ - ut_a(ib_vector_size(optim->words) > 0); - - /* Optimize the nodes that were read and write - back to DB. */ - error = fts_optimize_compact(optim, index, start_time); - - if (error == DB_SUCCESS) { - fts_sql_commit(optim->trx); - } else { - fts_sql_rollback(optim->trx); - } - } - - ib_vector_reset(optim->words); - - if (error == DB_SUCCESS) { - if (!optim->done) { - if (!fts_zip_read_word(optim->zip, word)) { - optim->done = TRUE; - } else if (selected - != fts_select_index( - charset, word->f_str, - word->f_len) - && graph) { - que_graph_free(graph); - graph = NULL; - } - } - } else if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout during optimize." - " Retrying!"; + ut_a(!optim->done); + /* Get the time limit from the config table. */ + fts_optimize_time_limit= + fts_optimize_get_time_limit(executor, index->table); + const time_t start_time= time(NULL); - trx->error_state = DB_SUCCESS; - } else if (error == DB_DEADLOCK) { - ib::warn() << "Deadlock during optimize. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - optim->done = TRUE; /* Exit the loop. */ - } - } + while (!optim->done) + { + trx_t *trx= optim->trx; + ut_a(ib_vector_size(optim->words) == 0); + /* Read the index records to optimize. */ + dberr_t error= fts_index_fetch_nodes( + executor, index, word, optim->words, nullptr, AuxCompareMode::EQUAL); + if (error == DB_SUCCESS) + { + /* There must be some nodes to read. */ + ut_a(ib_vector_size(optim->words) > 0); + /* Optimize the nodes that were read and write back to DB. */ + error = fts_optimize_compact(executor, optim, index, start_time); + if (error == DB_SUCCESS) fts_sql_commit(optim->trx); + else fts_sql_rollback(optim->trx); + } + ib_vector_reset(optim->words); - if (graph != NULL) { - que_graph_free(graph); - } + if (error == DB_SUCCESS) + { + if (!optim->done && !fts_zip_read_word(optim->zip, word)) + optim->done= TRUE; + } + else if (error == DB_LOCK_WAIT_TIMEOUT) + { + sql_print_warning("InnoDB: Lock wait timeout during optimize. " + "Retrying!"); + trx->error_state= DB_SUCCESS; + } + else if (error == DB_DEADLOCK) + { + sql_print_warning("InnoDB: Deadlock during optimize. Retrying!"); + trx->error_state = DB_SUCCESS; + } + else optim->done = TRUE; + } } -/**********************************************************************//** -Optimize is complete. Set the completion time, and reset the optimize -start string for this FTS index to "". +/** Optimize is complete. Set the completion time, and reset the +optimize start string for this FTS index to "". +@param executor query executor +@param optim optimize instance +@param index table with one FTS index @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t -fts_optimize_index_completed( -/*=========================*/ - fts_optimize_t* optim, /*!< in: optimize instance */ - dict_index_t* index) /*!< in: table with one FTS index */ +fts_optimize_index_completed(FTSQueryExecutor *executor, + fts_optimize_t *optim, + dict_index_t *index) { - fts_string_t word; - dberr_t error; - byte buf[sizeof(ulint)]; - - /* If we've reached the end of the index then set the start - word to the empty string. */ - - word.f_len = 0; - word.f_str = buf; - *word.f_str = '\0'; - - error = fts_config_set_index_value( - optim->trx, index, FTS_LAST_OPTIMIZED_WORD, &word); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") while updating" - " last optimized word!"; - } - - return(error); + fts_string_t word; + dberr_t error= DB_SUCCESS; + byte buf[sizeof(ulint)]; + /* If we've reached the end of the index then set the start + word to the empty string. */ + word.f_len= 0; + word.f_str= buf; + *word.f_str= '\0'; + + error= fts_config_set_index_value( + executor, index, FTS_LAST_OPTIMIZED_WORD, &word); + + if (UNIV_UNLIKELY(error != DB_SUCCESS)) + sql_print_error("InnoDB: (%s) while updating last optimized word!", + ut_strerr(error)); + return error; } -/**********************************************************************//** -Read the list of words from the FTS auxiliary index that will be -optimized in this pass. +/** Read the words that will be optimized in this pass. +@param executor query executor +@param optim optimize instance +@param index table with one FTS index +@param word buffer to use @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_index_read_words( -/*==========================*/ - fts_optimize_t* optim, /*!< in: optimize instance */ - dict_index_t* index, /*!< in: table with one FTS index */ - fts_string_t* word) /*!< in: buffer to use */ + FTSQueryExecutor* executor, + fts_optimize_t* optim, + dict_index_t* index, + fts_string_t* word) { dberr_t error = DB_SUCCESS; if (optim->del_list_regenerated) { word->f_len = 0; } else { - /* Get the last word that was optimized from the config table. */ error = fts_config_get_index_value( - optim->trx, index, FTS_LAST_OPTIMIZED_WORD, word); + executor, index, FTS_LAST_OPTIMIZED_WORD, word); } /* If record not found then we start from the top. */ @@ -1857,10 +1421,11 @@ fts_optimize_index_read_words( error = DB_SUCCESS; } + optim->index = index; while (error == DB_SUCCESS) { error = fts_index_fetch_words( - optim, word, fts_num_word_optimize); + executor, optim, word, fts_num_word_optimize); if (error == DB_SUCCESS) { /* Reset the last optimized word to '' if no @@ -1885,17 +1450,14 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_index( /*===============*/ - fts_optimize_t* optim, /*!< in: optimize instance */ - dict_index_t* index) /*!< in: table with one FTS index */ + FTSQueryExecutor* executor, /*!< in: query executor */ + fts_optimize_t* optim, /*!< in: optimize instance */ + dict_index_t* index) /*!< in: table with one FTS index */ { fts_string_t word; dberr_t error; byte str[FTS_MAX_WORD_LEN + 1]; - /* Set the current index that we have to optimize. */ - optim->fts_index_table.index_id = index->id; - optim->fts_index_table.charset = fts_index_get_charset(index); - optim->done = FALSE; /* Optimize until !done */ /* We need to read the last word optimized so that we start from @@ -1909,7 +1471,7 @@ fts_optimize_index( memset(word.f_str, 0x0, word.f_len); /* Read the words that will be optimized in this pass. */ - error = fts_optimize_index_read_words(optim, index, &word); + error = fts_optimize_index_read_words(executor, optim, index, &word); if (error == DB_SUCCESS) { int zip_error; @@ -1929,7 +1491,7 @@ fts_optimize_index( optim->done = TRUE; } else { - fts_optimize_words(optim, index, &word); + fts_optimize_words(executor, optim, index, &word); } /* If we couldn't read any records then optimize is @@ -1938,7 +1500,8 @@ fts_optimize_index( completed. */ if (error == DB_SUCCESS && optim->zip->n_words == 0) { - error = fts_optimize_index_completed(optim, index); + error = fts_optimize_index_completed( + executor, optim, index); if (error == DB_SUCCESS) { ++optim->n_completed; @@ -1949,190 +1512,99 @@ fts_optimize_index( return(error); } -/**********************************************************************//** -Delete the document ids in the delete, and delete cache tables. +/** Purge the doc ids that are in the snapshot from +the master deleted table. +@param executor query executor +@param optim optimize instance @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_purge_deleted_doc_ids( -/*===============================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_purge_deleted_doc_ids(FTSQueryExecutor *executor, + fts_optimize_t *optim) { - ulint i; - pars_info_t* info; - que_t* graph; - doc_id_t* update; - doc_id_t write_doc_id; - dberr_t error = DB_SUCCESS; - char deleted[MAX_FULL_NAME_LEN]; - char deleted_cache[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - ut_a(ib_vector_size(optim->to_delete->doc_ids) > 0); - - update = static_cast( - ib_vector_get(optim->to_delete->doc_ids, 0)); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, *update); - - /* This is required for the SQL parser to work. It must be able - to find the following variables. So we do it twice. */ - fts_bind_doc_id(info, "doc_id1", &write_doc_id); - fts_bind_doc_id(info, "doc_id2", &write_doc_id); - - /* Make sure the following two names are consistent with the name - used in the fts_delete_doc_ids_sql */ - optim->fts_common_table.suffix = fts_common_tables[3]; - fts_get_table_name(&optim->fts_common_table, deleted); - pars_info_bind_id(info, fts_common_tables[3], deleted); - - optim->fts_common_table.suffix = fts_common_tables[4]; - fts_get_table_name(&optim->fts_common_table, deleted_cache); - pars_info_bind_id(info, fts_common_tables[4], deleted_cache); - - graph = fts_parse_sql(NULL, info, fts_delete_doc_ids_sql); - - /* Delete the doc ids that were copied at the start. */ - for (i = 0; i < ib_vector_size(optim->to_delete->doc_ids); ++i) { - - update = static_cast(ib_vector_get( - optim->to_delete->doc_ids, i)); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, *update); - - fts_bind_doc_id(info, "doc_id1", &write_doc_id); - - fts_bind_doc_id(info, "doc_id2", &write_doc_id); - - error = fts_eval_sql(optim->trx, graph); - - // FIXME: Check whether delete actually succeeded! - if (error != DB_SUCCESS) { - - fts_sql_rollback(optim->trx); - break; - } - } - - que_graph_free(graph); + dberr_t error= DB_SUCCESS; + ut_a(ib_vector_size(optim->to_delete->doc_ids) > 0); + for (ulint i= 0; + i < ib_vector_size(optim->to_delete->doc_ids) && error != DB_SUCCESS; + ++i) + { + doc_id_t *update= + static_cast(ib_vector_get(optim->to_delete->doc_ids, i)); + error= executor->delete_common_record("DELETED", *update); + if (error == DB_SUCCESS) + error= executor->delete_common_record("DELETED_CACHE", *update); + } - return(error); + if (error != DB_SUCCESS) + fts_sql_rollback(optim->trx); + return error; } -/**********************************************************************//** -Delete the document ids in the pending delete, and delete tables. +/** Delete the document ids in the pending delete, and delete tables. +@param executor query executor +@param optim optimize instance @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_purge_deleted_doc_id_snapshot( -/*=======================================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_purge_deleted_doc_id_snapshot(FTSQueryExecutor *executor, + fts_optimize_t *optim) { - dberr_t error; - que_t* graph; - pars_info_t* info; - char being_deleted[MAX_FULL_NAME_LEN]; - char being_deleted_cache[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - /* Make sure the following two names are consistent with the name - used in the fts_end_delete_sql */ - optim->fts_common_table.suffix = fts_common_tables[0]; - fts_get_table_name(&optim->fts_common_table, being_deleted); - pars_info_bind_id(info, fts_common_tables[0], being_deleted); - - optim->fts_common_table.suffix = fts_common_tables[1]; - fts_get_table_name(&optim->fts_common_table, being_deleted_cache); - pars_info_bind_id(info, fts_common_tables[1], being_deleted_cache); - - /* Delete the doc ids that were copied to delete pending state at - the start of optimize. */ - graph = fts_parse_sql(NULL, info, fts_end_delete_sql); - - error = fts_eval_sql(optim->trx, graph); - que_graph_free(graph); - - return(error); + dberr_t error= executor->delete_all_common_records("BEING_DELETED"); + if (error == DB_SUCCESS) + error= executor->delete_all_common_records("BEING_DELETED_CACHE"); + return error; } -/**********************************************************************//** -Copy the deleted doc ids that will be purged during this optimize run -to the being deleted FTS auxiliary tables. The transaction is committed -upon successfull copy and rolled back on DB_DUPLICATE_KEY error. +/** Check if there are records in BEING_DELETED table +@param executor query executor +@param optim optimize fts instance +@param n_rows number of rows exist in being_deleted table @return DB_SUCCESS if all OK */ static -ulint -fts_optimize_being_deleted_count( -/*=============================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_being_deleted_count(FTSQueryExecutor *executor, + fts_optimize_t *optim, + ulint *n_rows) { - fts_table_t fts_table; - - FTS_INIT_FTS_TABLE(&fts_table, "BEING_DELETED", FTS_COMMON_TABLE, - optim->table); - - return(fts_get_rows_count(&fts_table)); + CommonTableReader reader; + dberr_t err= executor->read_all_common("BEING_DELETED", reader); + if (err == DB_SUCCESS) *n_rows= reader.get_doc_ids().size(); + return err; } -/*********************************************************************//** -Copy the deleted doc ids that will be purged during this optimize run -to the being deleted FTS auxiliary tables. The transaction is committed -upon successfull copy and rolled back on DB_DUPLICATE_KEY error. -@return DB_SUCCESS if all OK */ +/** Create a snapshot of deleted document IDs by moving them from +DELETED to BEING_DELETED and from DELETED_CACHE to +BEING_DELETED_CACHE. +@param executor query executor +@param optim optimize fts instance +@return DB_SUCCESS or error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_create_deleted_doc_id_snapshot( -/*========================================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_create_deleted_doc_id_snapshot(FTSQueryExecutor *executor, + fts_optimize_t *optim) { - dberr_t error; - que_t* graph; - pars_info_t* info; - char being_deleted[MAX_FULL_NAME_LEN]; - char deleted[MAX_FULL_NAME_LEN]; - char being_deleted_cache[MAX_FULL_NAME_LEN]; - char deleted_cache[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); + dberr_t err= DB_SUCCESS; + CommonTableReader reader; - /* Make sure the following four names are consistent with the name - used in the fts_init_delete_sql */ - optim->fts_common_table.suffix = fts_common_tables[0]; - fts_get_table_name(&optim->fts_common_table, being_deleted); - pars_info_bind_id(info, fts_common_tables[0], being_deleted); + err= executor->read_all_common("DELETED", reader); + if (err != DB_SUCCESS && err != DB_RECORD_NOT_FOUND) return err; - optim->fts_common_table.suffix = fts_common_tables[3]; - fts_get_table_name(&optim->fts_common_table, deleted); - pars_info_bind_id(info, fts_common_tables[3], deleted); - - optim->fts_common_table.suffix = fts_common_tables[1]; - fts_get_table_name(&optim->fts_common_table, being_deleted_cache); - pars_info_bind_id(info, fts_common_tables[1], being_deleted_cache); - - optim->fts_common_table.suffix = fts_common_tables[4]; - fts_get_table_name(&optim->fts_common_table, deleted_cache); - pars_info_bind_id(info, fts_common_tables[4], deleted_cache); - - /* Move doc_ids that are to be deleted to state being deleted. */ - graph = fts_parse_sql(NULL, info, fts_init_delete_sql); - - error = fts_eval_sql(optim->trx, graph); - - que_graph_free(graph); + const auto& deleted_doc_ids = reader.get_doc_ids(); + for (doc_id_t doc_id : deleted_doc_ids) + { + err= executor->insert_common_record("BEING_DELETED", doc_id); + if (err != DB_SUCCESS) return err; + } - if (error != DB_SUCCESS) { - fts_sql_rollback(optim->trx); - } else { - fts_sql_commit(optim->trx); - } + reader.clear(); + err= executor->read_all_common("DELETED_CACHE", reader); + if (err != DB_SUCCESS && err != DB_RECORD_NOT_FOUND) return err; - optim->del_list_regenerated = TRUE; + const auto& deleted_cache_doc_ids= reader.get_doc_ids(); + for (doc_id_t doc_id : deleted_cache_doc_ids) + { + err= executor->insert_common_record("BEING_DELETED_CACHE", doc_id); + if (err != DB_SUCCESS) return err; + } - return(error); + optim->del_list_regenerated= TRUE; + return err; } /*********************************************************************//** @@ -2143,44 +1615,38 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_read_deleted_doc_id_snapshot( /*======================================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ + FTSQueryExecutor* executor, /*!< in: FTS query executor */ + fts_optimize_t* optim) /*!< in: optimize instance */ { - dberr_t error; - - optim->fts_common_table.suffix = "BEING_DELETED"; - /* Read the doc_ids to delete. */ - error = fts_table_fetch_doc_ids( - optim->trx, &optim->fts_common_table, optim->to_delete); + dberr_t error = fts_table_fetch_doc_ids( + executor, optim->table, "BEING_DELETED", + optim->to_delete); if (error == DB_SUCCESS) { - optim->fts_common_table.suffix = "BEING_DELETED_CACHE"; - /* Read additional doc_ids to delete. */ error = fts_table_fetch_doc_ids( - optim->trx, &optim->fts_common_table, optim->to_delete); + executor, optim->table, "BEING_DELETED_CACHE", + optim->to_delete); } if (error != DB_SUCCESS) { - fts_doc_ids_free(optim->to_delete); optim->to_delete = NULL; } - return(error); } /*********************************************************************//** -Optimize all the FTS indexes, skipping those that have already been -optimized, since the FTS auxiliary indexes are not guaranteed to be -of the same cardinality. +Optimize the FTS indexes of a table. @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_indexes( /*=================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ + FTSQueryExecutor* executor, /*!< in: query executor */ + fts_optimize_t* optim) /*!< in: optimize instance */ { ulint i; dberr_t error = DB_SUCCESS; @@ -2192,7 +1658,17 @@ fts_optimize_indexes( index = static_cast( ib_vector_getp(fts->indexes, i)); - error = fts_optimize_index(optim, index); + + /* Open auxiliary tables for this index */ + error = executor->open_all_aux_tables(index); + if (error != DB_SUCCESS) { + break; + } + + error = fts_optimize_index(executor, optim, index); + if (error != DB_SUCCESS) { + break; + } } if (error == DB_SUCCESS) { @@ -2211,17 +1687,19 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_purge_snapshot( /*========================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ + FTSQueryExecutor* executor, /*!< in: query executor */ + fts_optimize_t* optim) /*!< in: optimize instance */ { dberr_t error; /* Delete the doc ids from the master deleted tables, that were in the snapshot that was taken at the start of optimize. */ - error = fts_optimize_purge_deleted_doc_ids(optim); + error = fts_optimize_purge_deleted_doc_ids(executor, optim); if (error == DB_SUCCESS) { /* Destroy the deleted doc id snapshot. */ - error = fts_optimize_purge_deleted_doc_id_snapshot(optim); + error = fts_optimize_purge_deleted_doc_id_snapshot( + executor, optim); } if (error == DB_SUCCESS) { @@ -2233,6 +1711,7 @@ fts_optimize_purge_snapshot( return(error); } + /*********************************************************************//** Run OPTIMIZE on the given table by a background thread. @return DB_SUCCESS if all OK */ @@ -2259,7 +1738,7 @@ fts_optimize_table_bk( if (table->is_accessible() && table->fts && table->fts->cache && table->fts->cache->deleted >= FTS_OPTIMIZE_THRESHOLD) { - error = fts_optimize_table(table); + error = fts_optimize_table(table, fts_opt_thd); slot->last_run = time(NULL); @@ -2275,13 +1754,12 @@ fts_optimize_table_bk( return(error); } -/*********************************************************************//** -Run OPTIMIZE on the given table. +/** Run OPTIMIZE on the given table. +@param table table to be optimized +@param thd thread which executes optimize table @return DB_SUCCESS if all OK */ dberr_t -fts_optimize_table( -/*===============*/ - dict_table_t* table) /*!< in: table to optimiza */ +fts_optimize_table(dict_table_t *table, THD *thd) { if (srv_read_only_mode) { return DB_READ_ONLY; @@ -2293,14 +1771,32 @@ fts_optimize_table( optim = fts_optimize_create(table); + optim->trx->mysql_thd = thd; + + /* Create FTSQueryExecutor and open common tables */ + FTSQueryExecutor executor(optim->trx, table); + error = executor.open_all_deletion_tables(); + if (error != DB_SUCCESS) { +err_exit: + fts_optimize_free(optim); + return error; + } + + error = executor.open_config_table(); + if (error) { goto err_exit; } + // FIXME: Call this only at the start of optimize, currently we // rely on DB_DUPLICATE_KEY to handle corrupting the snapshot. /* Check whether there are still records in BEING_DELETED table */ - if (fts_optimize_being_deleted_count(optim) == 0) { + ulint n_rows = 0; + error= fts_optimize_being_deleted_count(&executor, optim, &n_rows); + + if (error == DB_SUCCESS && n_rows == 0) { /* Take a snapshot of the deleted document ids, they are copied to the BEING_ tables. */ - error = fts_optimize_create_deleted_doc_id_snapshot(optim); + error = fts_optimize_create_deleted_doc_id_snapshot( + &executor, optim); } /* A duplicate error is OK, since we don't erase the @@ -2315,7 +1811,7 @@ fts_optimize_table( /* These document ids will be filtered out during the index optimization phase. They are in the snapshot that we took above, at the start of the optimize. */ - error = fts_optimize_read_deleted_doc_id_snapshot(optim); + error = fts_optimize_read_deleted_doc_id_snapshot(&executor, optim); if (error == DB_SUCCESS) { @@ -2326,7 +1822,7 @@ fts_optimize_table( /* We would do optimization only if there are deleted records to be cleaned up */ if (ib_vector_size(optim->to_delete->doc_ids) > 0) { - error = fts_optimize_indexes(optim); + error = fts_optimize_indexes(&executor, optim); } } else { @@ -2344,8 +1840,10 @@ fts_optimize_table( /* Purge the doc ids that were in the snapshot from the snapshot tables and the master deleted table. */ - error = fts_optimize_purge_snapshot(optim); + error = fts_optimize_purge_snapshot( + &executor, optim); } + } } @@ -2430,8 +1928,8 @@ fts_optimize_remove_table( if (fts_opt_start_shutdown) { - ib::info() << "Try to remove table " << table->name - << " after FTS optimize thread exiting."; + sql_print_information("InnoDB: Try to remove table %s after FTS optimize " + "thread exiting.", table->name.m_name); while (fts_optimize_wq) std::this_thread::sleep_for(std::chrono::milliseconds(10)); return; @@ -2470,8 +1968,9 @@ fts_optimize_request_sync_table( /* FTS optimizer thread is already exited */ if (fts_opt_start_shutdown) { - ib::info() << "Try to sync table " << table->name - << " after FTS optimize thread exiting."; + sql_print_information("InnoDB: Try to sync table %s " + "after FTS optimize thread exiting.", + table->name.m_name); } else if (table->fts->sync_message) { /* If the table already has SYNC message in fts_optimize_wq queue then ignore it */ @@ -2767,7 +2266,7 @@ static void fts_optimize_callback(void *) pthread_cond_broadcast(&fts_opt_shutdown_cond); mysql_mutex_unlock(&fts_optimize_wq->mutex); - ib::info() << "FTS optimize thread exiting."; + sql_print_information("InnoDB: FTS optimize thread exiting."); } /**********************************************************************//** diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc index 0faf23aa6e97b..912b7c85c284a 100644 --- a/storage/innobase/fts/fts0que.cc +++ b/storage/innobase/fts/fts0que.cc @@ -35,6 +35,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang #include "fts0types.h" #include "fts0plugin.h" #include "fts0vlc.h" +#include "fts0exec.h" #include #include @@ -67,9 +68,6 @@ struct fts_query_t { trx_t* trx; /*!< The query transaction */ dict_index_t* index; /*!< The FTS index to search */ - /*!< FTS auxiliary common table def */ - - fts_table_t fts_common_table; fts_table_t fts_index_table;/*!< FTS auxiliary index table def */ @@ -149,6 +147,8 @@ struct fts_query_t { fts_ast_visit_sub_exp() */ st_mysql_ftparser* parser; /*!< fts plugin parser */ + + FTSQueryExecutor* executor; /*!< shared FTS query executor */ }; /** For phrase matching, first we collect the documents and the positions @@ -270,16 +270,6 @@ struct fts_word_freq_t { double idf; /*!< Inverse document frequency */ }; -/******************************************************************** -Callback function to fetch the rows in an FTS INDEX record. -@return always TRUE */ -static -ibool -fts_query_index_fetch_nodes( -/*========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg); /*!< in: pointer to ib_vector_t */ - /******************************************************************** Read and filter nodes. @return fts_node_t instance */ @@ -310,19 +300,103 @@ fts_ast_visit_sub_exp( fts_ast_callback visitor, void* arg); -#if 0 -/*****************************************************************//*** -Find a doc_id in a word's ilist. -@return TRUE if found. */ -static -ibool -fts_query_find_doc_id( -/*==================*/ - fts_select_t* select, /*!< in/out: search the doc id selected, - update the frequency if found. */ - void* data, /*!< in: doc id ilist */ - ulint len); /*!< in: doc id ilist size */ -#endif +/** Process query records for FTS queries. +@param rec record +@param index index +@param offsets record offsets +@param user_arg user argument +@return DB_SUCCESS to continue processing, DB_SUCCESS_LOCKED_REC to stop, or error code */ +static dberr_t node_query_processor( + const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets, void* user_arg) +{ + fts_query_t* query= static_cast(user_arg); + AuxRecordReader::AuxRecordFields fields; + /* Use optimized field extraction with external BLOB handling */ + if (!AuxRecordReader::extract_aux_fields(rec, index, fields)) + return DB_SUCCESS; + + if (!fields.word_data || fields.word_len == UNIV_SQL_NULL || + fields.word_len > FTS_MAX_WORD_LEN) + return DB_SUCCESS; + + ut_a(query->cur_node->type == FTS_AST_TERM + || query->cur_node->type == FTS_AST_TEXT + || query->cur_node->type == FTS_AST_PARSER_PHRASE_LIST); + + fts_node_t node; + memset(&node, 0, sizeof(node)); + + fts_string_t term; + byte buf[FTS_MAX_WORD_LEN + 1]; + term.f_str= buf; + + /* Need to consider the wildcard search case, the word frequency + is created on the search string not the actual word. So we need + to assign the frequency on search string behalf. */ + if (query->cur_node->type == FTS_AST_TERM && query->cur_node->term.wildcard) + { + term.f_len = query->cur_node->term.ptr->len; + ut_ad(FTS_MAX_WORD_LEN >= term.f_len); + memcpy(term.f_str, query->cur_node->term.ptr->str, term.f_len); + } + else + { + term.f_len = fields.word_len; + ut_ad(FTS_MAX_WORD_LEN >= fields.word_len); + memcpy(term.f_str, fields.word_data, fields.word_len); + } + + /* Lookup the word in our rb tree, it must exist. */ + ib_rbt_bound_t parent; + int ret= rbt_search(query->word_freqs, &parent, &term); + + ut_a(ret == 0); + fts_word_freq_t* word_freq= rbt_value(fts_word_freq_t, parent.last); + bool skip = false; + + /* Use extracted field values */ + node.first_doc_id = fields.first_doc_id; + skip= (query->oper == FTS_EXIST && query->upper_doc_id > 0 && + node.first_doc_id > query->upper_doc_id); + + node.last_doc_id = fields.last_doc_id; + skip= (query->oper == FTS_EXIST && query->lower_doc_id > 0 && + node.last_doc_id < query->lower_doc_id); + + word_freq->doc_count += fields.doc_count; + + if (!skip) + { + if (fields.ilist_data && fields.ilist_len != UNIV_SQL_NULL && + fields.ilist_len > 0) + { + /* Process the ilist data (either inline or external) */ + query->error= fts_query_filter_doc_ids( + query, &word_freq->word, word_freq, &node, + fields.ilist_data, fields.ilist_len, FALSE); + + if (query->error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT) + return DB_SUCCESS; + return query->error; + } + } + return DB_SUCCESS; +} + +/* Comparator that signals how to treat the current record */ +RecordCompareAction doc_id_exact_match_comparator( + const dtuple_t* search_tuple, const rec_t* rec, const dict_index_t* index) +{ + const dfield_t* search_field= dtuple_get_nth_field(search_tuple, 0); + const byte* search_data= + static_cast(dfield_get_data(search_field)); + doc_id_t target_doc_id= fts_read_doc_id(search_data); + doc_id_t rec_doc_id= fts_read_doc_id(rec); + return rec_doc_id == target_doc_id + ? RecordCompareAction::PROCESS + : RecordCompareAction::STOP; +} /*************************************************************//** This function implements a simple "blind" query expansion search: @@ -371,107 +445,6 @@ fts_proximity_get_positions( fts_proximity_t* qualified_pos); /*!< out: the position info records ranges containing all matching words. */ -#if 0 -/******************************************************************** -Get the total number of words in a documents. */ -static -ulint -fts_query_terms_in_document( -/*========================*/ - /*!< out: DB_SUCCESS if all go well - else error code */ - fts_query_t* query, /*!< in: FTS query state */ - doc_id_t doc_id, /*!< in: the word to check */ - ulint* total); /*!< out: total words in document */ -#endif - -#if 0 -/*******************************************************************//** -Print the table used for calculating LCS. */ -static -void -fts_print_lcs_table( -/*================*/ - const ulint* table, /*!< in: array to print */ - ulint n_rows, /*!< in: total no. of rows */ - ulint n_cols) /*!< in: total no. of cols */ -{ - ulint i; - - for (i = 0; i < n_rows; ++i) { - ulint j; - - printf("\n"); - - for (j = 0; j < n_cols; ++j) { - - printf("%2lu ", FTS_ELEM(table, n_cols, i, j)); - } - } -} - -/******************************************************************** -Find the longest common subsequence between the query string and -the document. */ -static -ulint -fts_query_lcs( -/*==========*/ - /*!< out: LCS (length) between - two ilists */ - const ulint* p1, /*!< in: word positions of query */ - ulint len_p1, /*!< in: no. of elements in p1 */ - const ulint* p2, /*!< in: word positions within document */ - ulint len_p2) /*!< in: no. of elements in p2 */ -{ - int i; - ulint len = 0; - ulint r = len_p1; - ulint c = len_p2; - ulint size = (r + 1) * (c + 1) * sizeof(ulint); - ulint* table = (ulint*) ut_malloc_nokey(size); - - /* Traverse the table backwards, from the last row to the first and - also from the last column to the first. We compute the smaller - common subsequences first, then use the calculated values to determine - the longest common subsequence. The result will be in TABLE[0][0]. */ - for (i = r; i >= 0; --i) { - int j; - - for (j = c; j >= 0; --j) { - - if (p1[i] == (ulint) -1 || p2[j] == (ulint) -1) { - - FTS_ELEM(table, c, i, j) = 0; - - } else if (p1[i] == p2[j]) { - - FTS_ELEM(table, c, i, j) = FTS_ELEM( - table, c, i + 1, j + 1) + 1; - - } else { - - ulint value; - - value = ut_max( - FTS_ELEM(table, c, i + 1, j), - FTS_ELEM(table, c, i, j + 1)); - - FTS_ELEM(table, c, i, j) = value; - } - } - } - - len = FTS_ELEM(table, c, 0, 0); - - fts_print_lcs_table(table, r, c); - printf("\nLen=" ULINTPF "\n", len); - - ut_free(table); - - return(len); -} -#endif /*******************************************************************//** Compare two fts_ranking_t instance on their rank value and doc ids in @@ -1100,7 +1073,6 @@ fts_query_difference( const fts_string_t* token) /*!< in: token to search */ { ulint n_doc_ids= 0; - trx_t* trx = query->trx; dict_table_t* table = query->index->table; ut_a(query->oper == FTS_IGNORE); @@ -1112,10 +1084,8 @@ fts_query_difference( /* There is nothing we can substract from an empty set. */ if (query->doc_ids && !rbt_empty(query->doc_ids)) { ulint i; - fts_fetch_t fetch; const ib_vector_t* nodes; const fts_index_cache_t*index_cache; - que_t* graph = NULL; fts_cache_t* cache = table->fts->cache; dberr_t error; @@ -1153,21 +1123,21 @@ fts_query_difference( return(query->error); } - /* Setup the callback args for filtering and - consolidating the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, token, &fetch); + query->executor, query->index, token, query, + node_query_processor, compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); if (error != DB_SUCCESS) { query->error = error; } - - que_graph_free(graph); } /* The size can't increase. */ @@ -1194,7 +1164,6 @@ fts_query_intersect( fts_query_t* query, /*!< in: query instance */ const fts_string_t* token) /*!< in: the token to search */ { - trx_t* trx = query->trx; dict_table_t* table = query->index->table; ut_a(query->oper == FTS_EXIST); @@ -1204,10 +1173,8 @@ fts_query_intersect( if (!(rbt_empty(query->doc_ids) && query->multi_exist)) { ulint n_doc_ids = 0; ulint i; - fts_fetch_t fetch; const ib_vector_t* nodes; const fts_index_cache_t*index_cache; - que_t* graph = NULL; fts_cache_t* cache = table->fts->cache; dberr_t error; @@ -1278,13 +1245,15 @@ fts_query_intersect( return(query->error); } - /* Setup the callback args for filtering and - consolidating the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, token, &fetch); + query->executor, query->index, token, query, + node_query_processor, compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); @@ -1292,8 +1261,6 @@ fts_query_intersect( query->error = error; } - que_graph_free(graph); - if (query->error == DB_SUCCESS) { /* Make the intesection (rb tree) the current doc id set and free the old set. */ @@ -1371,10 +1338,7 @@ fts_query_union( fts_query_t* query, /*!< in: query instance */ fts_string_t* token) /*!< in: token to search */ { - fts_fetch_t fetch; ulint n_doc_ids = 0; - trx_t* trx = query->trx; - que_t* graph = NULL; dberr_t error; ut_a(query->oper == FTS_NONE || query->oper == FTS_DECR_RATING || @@ -1390,14 +1354,16 @@ fts_query_union( fts_query_cache(query, token); - /* Setup the callback args for filtering and - consolidating the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } /* Read the nodes from disk. */ error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, token, &fetch); + query->executor, query->index, token, query, node_query_processor, + compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); @@ -1405,8 +1371,6 @@ fts_query_union( query->error = error; } - que_graph_free(graph); - if (query->error == DB_SUCCESS) { /* The size can't decrease. */ @@ -1923,476 +1887,232 @@ fts_query_match_phrase( return(phrase->found); } -/*****************************************************************//** -Callback function to fetch and search the document. +/** Callback function to fetch and search the document. +@param fts_index fulltext index +@param doc_id document id +@param arg user argument +@param expansion Expansion document @return whether the phrase is found */ static -ibool -fts_query_fetch_document( -/*=====================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ +dberr_t fts_query_fetch_document(dict_index_t *fts_index, + doc_id_t doc_id, + void *arg, bool expansion= false) { - - que_node_t* exp; - sel_node_t* node = static_cast(row); - fts_phrase_t* phrase = static_cast(user_arg); - ulint prev_len = 0; - ulint total_len = 0; - byte* document_text = NULL; - - exp = node->select_list; - - phrase->found = FALSE; - - /* For proximity search, we will need to get the whole document - from all fields, so first count the total length of the document - from all the fields */ - if (phrase->proximity_pos) { - while (exp) { - ulint field_len; - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - - if (dfield_is_ext(dfield)) { - ulint local_len = dfield_get_len(dfield); - - local_len -= BTR_EXTERN_FIELD_REF_SIZE; - - field_len = mach_read_from_4( - data + local_len + BTR_EXTERN_LEN + 4); - } else { - field_len = dfield_get_len(dfield); - } - - if (field_len != UNIV_SQL_NULL) { - total_len += field_len + 1; - } - - exp = que_node_get_next(exp); - } - - document_text = static_cast(mem_heap_zalloc( - phrase->heap, total_len)); - - if (!document_text) { - return(FALSE); - } - } - - exp = node->select_list; - - while (exp) { - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - ulint cur_len; - - if (dfield_is_ext(dfield)) { - data = btr_copy_externally_stored_field( - &cur_len, data, phrase->zip_size, - dfield_get_len(dfield), phrase->heap); - } else { - cur_len = dfield_get_len(dfield); - } - - if (cur_len != UNIV_SQL_NULL && cur_len != 0) { - if (phrase->proximity_pos) { - ut_ad(prev_len + cur_len <= total_len); - memcpy(document_text + prev_len, data, cur_len); - } else { - /* For phrase search */ - phrase->found = - fts_query_match_phrase( - phrase, - static_cast(data), - cur_len, prev_len, - phrase->heap); - } - - /* Document positions are calculated from the beginning - of the first field, need to save the length for each - searched field to adjust the doc position when search - phrases. */ - prev_len += cur_len + 1; - } - - if (phrase->found) { - break; - } - - exp = que_node_get_next(exp); - } - - if (phrase->proximity_pos) { - ut_ad(prev_len <= total_len); - - phrase->found = fts_proximity_is_word_in_range( - phrase, document_text, total_len); - } - - return(phrase->found); -} - -#if 0 -/******************************************************************** -Callback function to check whether a record was found or not. */ -static -ibool -fts_query_select( -/*=============*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ -{ - int i; - que_node_t* exp; - sel_node_t* node = row; - fts_select_t* select = user_arg; - - ut_a(select->word_freq); - ut_a(select->word_freq->doc_freqs); - - exp = node->select_list; - - for (i = 0; exp && !select->found; ++i) { - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - - switch (i) { - case 0: /* DOC_COUNT */ - if (len != UNIV_SQL_NULL && len != 0) { - - select->word_freq->doc_count += - mach_read_from_4(data); - } - break; - - case 1: /* ILIST */ - if (len != UNIV_SQL_NULL && len != 0) { - - fts_query_find_doc_id(select, data, len); - } - break; - - default: - ut_error; - } - - exp = que_node_get_next(exp); - } - - return(FALSE); -} - -/******************************************************************** -Read the rows from the FTS index, that match word and where the -doc id is between first and last doc id. -@return DB_SUCCESS if all go well else error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_query_find_term( -/*================*/ - fts_query_t* query, /*!< in: FTS query state */ - que_t** graph, /*!< in: prepared statement */ - const fts_string_t* word, /*!< in: the word to fetch */ - doc_id_t doc_id, /*!< in: doc id to match */ - ulint* min_pos,/*!< in/out: pos found must be - greater than this minimum value. */ - ibool* found) /*!< out: TRUE if found else FALSE */ -{ - pars_info_t* info; - dberr_t error; - fts_select_t select; - doc_id_t match_doc_id; - trx_t* trx = query->trx; - char table_name[MAX_FULL_NAME_LEN]; - - trx->op_info = "fetching FTS index matching nodes"; - - if (*graph) { - info = (*graph)->info; - } else { - ulint selected; - - info = pars_info_create(); - - selected = fts_select_index(*word->f_str); - query->fts_index_table.suffix = fts_get_suffix(selected); - - fts_get_table_name(&query->fts_index_table, table_name); - pars_info_bind_id(info, "index_table_name", table_name); - } - - select.found = FALSE; - select.doc_id = doc_id; - select.min_pos = *min_pos; - select.word_freq = fts_query_add_word_freq(query, word->f_str); - - pars_info_bind_function(info, "my_func", fts_query_select, &select); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &match_doc_id, doc_id); - - fts_bind_doc_id(info, "min_doc_id", &match_doc_id); - - fts_bind_doc_id(info, "max_doc_id", &match_doc_id); - - if (!*graph) { - - *graph = fts_parse_sql( - &query->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT doc_count, ilist\n" - " FROM $index_table_name\n" - " WHERE word LIKE :word AND" - " first_doc_id <= :min_doc_id AND" - " last_doc_id >= :max_doc_id\n" - " ORDER BY first_doc_id;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - } - - for (;;) { - error = fts_eval_sql(trx, *graph); - - if (error == DB_SUCCESS) { - - break; /* Exit the loop. */ - } else { - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading FTS" - " index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << error - << " while reading FTS index."; - - break; /* Exit the loop. */ - } - } - } - - /* Value to return */ - *found = select.found; - - if (*found) { - *min_pos = select.min_pos; - } - - return(error); -} - -/******************************************************************** -Callback aggregator for int columns. */ -static -ibool -fts_query_sum( -/*==========*/ - /*!< out: always returns TRUE */ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: ulint* */ -{ - - que_node_t* exp; - sel_node_t* node = row; - ulint* total = user_arg; - - exp = node->select_list; - - while (exp) { - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - - if (len != UNIV_SQL_NULL && len != 0) { - *total += mach_read_from_4(data); - } - - exp = que_node_get_next(exp); - } - - return(TRUE); -} - -/******************************************************************** -Calculate the total documents that contain a particular word (term). -@return DB_SUCCESS if all go well else error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_query_total_docs_containing_term( -/*=================================*/ - fts_query_t* query, /*!< in: FTS query state */ - const fts_string_t* word, /*!< in: the word to check */ - ulint* total) /*!< out: documents containing word */ -{ - pars_info_t* info; - dberr_t error; - que_t* graph; - ulint selected; - trx_t* trx = query->trx; - char table_name[MAX_FULL_NAME_LEN] - - trx->op_info = "fetching FTS index document count"; - - *total = 0; - - info = pars_info_create(); - - pars_info_bind_function(info, "my_func", fts_query_sum, total); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - selected = fts_select_index(*word->f_str); - - query->fts_index_table.suffix = fts_get_suffix(selected); - - fts_get_table_name(&query->fts_index_table, table_name); - - pars_info_bind_id(info, "index_table_name", table_name); - - graph = fts_parse_sql( - &query->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT doc_count\n" - " FROM $index_table_name\n" - " WHERE word = :word" - " ORDER BY first_doc_id;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (error == DB_SUCCESS) { - - break; /* Exit the loop. */ - } else { - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading FTS" - " index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << error - << " while reading FTS index."; - - break; /* Exit the loop. */ - } - } - } - - que_graph_free(graph); - - return(error); -} - -/******************************************************************** -Get the total number of words in a documents. -@return DB_SUCCESS if all go well else error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_query_terms_in_document( -/*========================*/ - fts_query_t* query, /*!< in: FTS query state */ - doc_id_t doc_id, /*!< in: the word to check */ - ulint* total) /*!< out: total words in document */ -{ - pars_info_t* info; - dberr_t error; - que_t* graph; - doc_id_t read_doc_id; - trx_t* trx = query->trx; - char table_name[MAX_FULL_NAME_LEN]; - - trx->op_info = "fetching FTS document term count"; - - *total = 0; - - info = pars_info_create(); - - pars_info_bind_function(info, "my_func", fts_query_sum, total); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &read_doc_id, doc_id); - fts_bind_doc_id(info, "doc_id", &read_doc_id); - - query->fts_index_table.suffix = "DOC_ID"; - - fts_get_table_name(&query->fts_index_table, table_name); - - pars_info_bind_id(info, "index_table_name", table_name); - - graph = fts_parse_sql( - &query->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT count\n" - " FROM $index_table_name\n" - " WHERE doc_id = :doc_id" - " BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (error == DB_SUCCESS) { - - break; /* Exit the loop. */ - } else { - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading FTS" - " doc id table. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << error << " while reading FTS" - " doc id table."; - - break; /* Exit the loop. */ - } - } - } - - que_graph_free(graph); - - return(error); -} + trx_t *trx= trx_create(); + trx->op_info= "fetching FTS document for query"; + dict_table_t *user_table= fts_index->table; + dict_index_t *fts_doc_id_index= user_table->fts_doc_id_index; + dict_index_t *clust_index= dict_table_get_first_index(user_table); + ut_a(user_table->fts->doc_col != ULINT_UNDEFINED); + ut_a(fts_doc_id_index); + + QueryExecutor executor(trx); + + /* Map FTS index columns to clustered index field positions */ + ulint *clust_field_nos= static_cast( + mem_heap_alloc(executor.get_heap(), + fts_index->n_user_defined_cols * sizeof(ulint))); + + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + clust_field_nos[i]= dict_col_get_index_pos(fts_field->col, clust_index); + } + dfield_t fields[1]; + dtuple_t search_tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N #endif + }; + dict_index_copy_types(&search_tuple, fts_doc_id_index, 1); + dfield_t* dfield= dtuple_get_nth_field(&search_tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id((byte*) &write_doc_id, doc_id); + dfield_set_data(dfield, &write_doc_id, sizeof(write_doc_id)); + + auto process_expansion_doc= [arg, fts_index, + clust_field_nos](const rec_t* rec, + const dict_index_t *index, + const rec_offs *offsets)-> dberr_t + { + fts_doc_t *result_doc= static_cast(arg); + fts_doc_t doc; + CHARSET_INFO *doc_charset= result_doc->charset; + fts_doc_init(&doc); + doc.found= TRUE; + + ulint doc_len= 0; + ulint field_no= 0; + + /* Process each indexed column content */ + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, + col_pos, &field_len); + + /* NULL column */ + if (field_len == UNIV_SQL_NULL) { + continue; + } + + /* Determine document charset from column if not provided */ + if (!doc_charset) + { + const dict_field_t* ifield= dict_index_get_nth_field(fts_index, i); + doc_charset= fts_get_charset(ifield->col->prtype); + } + + doc.charset= doc_charset; + /* Skip columns stored externally, as in fts_query_expansion_fetch_doc */ + if (rec_offs_nth_extern(offsets, col_pos)) { + continue; + } + + /* Use inline field data */ + doc.text.f_n_char= 0; + doc.text.f_str= const_cast(field_data); + doc.text.f_len= field_len; + + if (field_no == 0) + fts_tokenize_document(&doc, result_doc, result_doc->parser); + else + fts_tokenize_document_next(&doc, doc_len, result_doc, + result_doc->parser); + + /* Next field offset: add 1 for separator if more fields follow */ + doc_len+= ((i + 1) < fts_index->n_user_defined_cols) + ? field_len + 1 + : field_len; + + field_no++; + } + + ut_ad(doc_charset); + if (!result_doc->charset) { + result_doc->charset= doc_charset; + } + + fts_doc_free(&doc); + + return DB_SUCCESS; /* continue */ + }; + + auto process_doc_query= [arg, user_table, fts_index, + clust_field_nos](const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + ulint prev_len= 0; + ulint total_len= 0; + byte *document_text= nullptr; + + fts_phrase_t *phrase= static_cast(arg); + phrase->found= FALSE; + + /* Extract doc_id from the clustered index record */ + ulint doc_col_pos= dict_col_get_index_pos( + &user_table->cols[user_table->fts->doc_col], index); + + ulint len; + rec_get_nth_field_offs(offsets, doc_col_pos, &len); + if (len != sizeof(doc_id_t)) + return DB_ERROR; + + /* For proximity search, first count total document length */ + if (phrase->proximity_pos) + { + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, + col_pos, &field_len); + if (rec_offs_nth_extern(offsets, col_pos)) + { + ulint local_len= field_len; + local_len-= BTR_EXTERN_FIELD_REF_SIZE; + field_len= mach_read_from_4( + field_data + local_len + BTR_EXTERN_LEN + 4); + } + if (field_len != UNIV_SQL_NULL) + total_len+= field_len + 1; + } + + document_text= + static_cast(mem_heap_zalloc(phrase->heap, total_len)); + if (!document_text) + return DB_ERROR; + } + + /* Process each indexed column content */ + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, + col_pos, &field_len); + byte* data= const_cast(field_data); + ulint cur_len; + + if (rec_offs_nth_extern(offsets, col_pos)) + { + data= btr_copy_externally_stored_field( + &cur_len, const_cast(field_data), phrase->zip_size, + field_len, phrase->heap); + } + else cur_len= field_len; + if (cur_len != UNIV_SQL_NULL && cur_len != 0) + { + if (phrase->proximity_pos) + { + ut_ad(prev_len + cur_len <= total_len); + memcpy(document_text + prev_len, data, cur_len); + } + else + { + /* For phrase search */ + phrase->found= fts_query_match_phrase( + phrase, data, cur_len, prev_len, phrase->heap); + } + + /* Document positions are calculated from the beginning + of the first field, need to save the length for each + searched field to adjust the doc position when search + phrases. */ + prev_len+= cur_len + 1; + } + + if (phrase->found) + break; + } + + if (phrase->proximity_pos) + { + ut_ad(prev_len <= total_len); + phrase->found= fts_proximity_is_word_in_range( + phrase, document_text, total_len); + } + + return phrase->found ? DB_SUCCESS_LOCKED_REC : DB_SUCCESS; /* Stop if found, continue if not found */ + }; + + RecordProcessor proc= expansion + ? RecordProcessor(process_expansion_doc) + : RecordProcessor(process_doc_query); + RecordCallback reader(proc, doc_id_exact_match_comparator); + dberr_t err= DB_SUCCESS; + if (fts_doc_id_index == clust_index) + err= executor.read(user_table, &search_tuple, PAGE_CUR_GE, reader); + else + err= executor.read_by_index(user_table, fts_doc_id_index, + &search_tuple, PAGE_CUR_GE, reader); + trx_commit_for_mysql(trx); + trx->free(); + if (err == DB_RECORD_NOT_FOUND) err= DB_SUCCESS; + return err; +} /*****************************************************************//** Retrieve the document and match the phrase tokens. @@ -2421,9 +2141,8 @@ fts_query_match_document( *found = phrase.found = FALSE; - error = fts_doc_fetch_by_doc_id( - get_doc, match->doc_id, NULL, FTS_FETCH_DOC_BY_ID_EQUAL, - fts_query_fetch_document, &phrase); + error = fts_query_fetch_document( + get_doc->index_cache->index, match->doc_id, &phrase); if (UNIV_UNLIKELY(error != DB_SUCCESS)) { ib::error() << "(" << error << ") matching document."; @@ -2468,21 +2187,14 @@ fts_query_is_in_proximity_range( phrase.proximity_pos = qualified_pos; phrase.found = FALSE; - err = fts_doc_fetch_by_doc_id( - &get_doc, match[0]->doc_id, NULL, FTS_FETCH_DOC_BY_ID_EQUAL, - fts_query_fetch_document, &phrase); + err = fts_query_fetch_document( + get_doc.index_cache->index, match[0]->doc_id, &phrase); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { ib::error() << "(" << err << ") in verification" " phase of proximity search"; } - /* Free the prepared statement. */ - if (get_doc.get_document_graph) { - que_graph_free(get_doc.get_document_graph); - get_doc.get_document_graph = NULL; - } - mem_heap_free(phrase.heap); return(err == DB_SUCCESS && phrase.found); @@ -2591,6 +2303,7 @@ fts_query_phrase_split( ulint len = 0; ulint cur_pos = 0; fts_ast_node_t* term_node = NULL; + CHARSET_INFO* cs = fts_index_get_charset(query->index); if (node->type == FTS_AST_TEXT) { phrase.f_str = node->text.ptr->str; @@ -2614,7 +2327,7 @@ fts_query_phrase_split( } cur_len = innobase_mysql_fts_get_token( - query->fts_index_table.charset, + cs, reinterpret_cast(phrase.f_str) + cur_pos, reinterpret_cast(phrase.f_str) @@ -2637,7 +2350,7 @@ fts_query_phrase_split( result_str.f_str = term_node->term.ptr->str; result_str.f_len = term_node->term.ptr->len; result_str.f_n_char = fts_get_token_size( - query->fts_index_table.charset, + cs, reinterpret_cast(result_str.f_str), result_str.f_len); @@ -2654,8 +2367,7 @@ fts_query_phrase_split( if (fts_check_token( &result_str, - cache->stopword_info.cached_stopword, - query->fts_index_table.charset)) { + cache->stopword_info.cached_stopword, cs)) { /* Add the word to the RB tree so that we can calculate its frequency within a document. */ fts_query_add_word_freq(query, token); @@ -2717,10 +2429,7 @@ fts_query_phrase_search( /* Ignore empty strings. */ if (num_token > 0) { fts_string_t* token = NULL; - fts_fetch_t fetch; - trx_t* trx = query->trx; fts_ast_oper_t oper = query->oper; - que_t* graph = NULL; ulint i; dberr_t error; @@ -2755,11 +2464,6 @@ fts_query_phrase_search( } } - /* Setup the callback args for filtering and consolidating - the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; - for (i = 0; i < num_token; i++) { /* Search for the first word from the phrase. */ token = static_cast( @@ -2770,9 +2474,15 @@ fts_query_phrase_search( query->matched = query->match_array[i]; } + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } + error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, - token, &fetch); + query->executor, query->index, token, query, + node_query_processor, compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); @@ -2780,9 +2490,6 @@ fts_query_phrase_search( query->error = error; } - que_graph_free(graph); - graph = NULL; - fts_query_cache(query, token); if (!(query->flags & FTS_PHRASE) @@ -2923,12 +2630,11 @@ fts_query_get_token( if (node->term.wildcard) { - token->f_str = static_cast(ut_malloc_nokey(str_len + 2)); - token->f_len = str_len + 1; + token->f_str = static_cast(ut_malloc_nokey(str_len + 1)); + token->f_len = str_len; memcpy(token->f_str, node->term.ptr->str, str_len); - token->f_str[str_len] = '%'; token->f_str[token->f_len] = 0; new_ptr = token->f_str; @@ -3109,78 +2815,6 @@ fts_ast_visit_sub_exp( DBUG_RETURN(error); } -#if 0 -/*****************************************************************//*** -Check if the doc id exists in the ilist. -@return TRUE if doc id found */ -static -ulint -fts_query_find_doc_id( -/*==================*/ - fts_select_t* select, /*!< in/out: contains the doc id to - find, we update the word freq if - document found */ - void* data, /*!< in: doc id ilist */ - ulint len) /*!< in: doc id ilist size */ -{ - byte* ptr = data; - doc_id_t doc_id = 0; - ulint decoded = 0; - - /* Decode the ilist and search for selected doc_id. We also - calculate the frequency of the word in the document if found. */ - while (decoded < len && !select->found) { - ulint freq = 0; - ulint min_pos = 0; - ulint last_pos = 0; - ulint pos = fts_decode_vlc(&ptr); - - /* Add the delta. */ - doc_id += pos; - - while (*ptr) { - ++freq; - last_pos += fts_decode_vlc(&ptr); - - /* Only if min_pos is not set and the current - term exists in a position greater than the - min_pos of the previous term. */ - if (min_pos == 0 && last_pos > select->min_pos) { - min_pos = last_pos; - } - } - - /* Skip the end of word position marker. */ - ++ptr; - - /* Bytes decoded so far. */ - decoded = ptr - (byte*) data; - - /* A word may exist in the document but we only consider a - match if it exists in a position that is greater than the - position of the previous term. */ - if (doc_id == select->doc_id && min_pos > 0) { - fts_doc_freq_t* doc_freq; - - /* Add the doc id to the doc freq rb tree, if - the doc id doesn't exist it will be created. */ - doc_freq = fts_query_add_doc_freq( - select->word_freq->doc_freqs, doc_id); - - /* Avoid duplicating the frequency tally */ - if (doc_freq->freq == 0) { - doc_freq->freq = freq; - } - - select->found = TRUE; - select->min_pos = min_pos; - } - } - - return(select->found); -} -#endif - /*****************************************************************//** Read and filter nodes. @return DB_SUCCESS if all go well, @@ -3301,156 +2935,6 @@ fts_query_filter_doc_ids( } } -/*****************************************************************//** -Read the FTS INDEX row. -@return DB_SUCCESS if all go well. */ -static -dberr_t -fts_query_read_node( -/*================*/ - fts_query_t* query, /*!< in: query instance */ - const fts_string_t* word, /*!< in: current word */ - que_node_t* exp) /*!< in: query graph node */ -{ - int i; - int ret; - fts_node_t node; - ib_rbt_bound_t parent; - fts_word_freq_t* word_freq; - ibool skip = FALSE; - fts_string_t term; - byte buf[FTS_MAX_WORD_LEN + 1]; - dberr_t error = DB_SUCCESS; - - ut_a(query->cur_node->type == FTS_AST_TERM - || query->cur_node->type == FTS_AST_TEXT - || query->cur_node->type == FTS_AST_PARSER_PHRASE_LIST); - - memset(&node, 0, sizeof(node)); - term.f_str = buf; - - /* Need to consider the wildcard search case, the word frequency - is created on the search string not the actual word. So we need - to assign the frequency on search string behalf. */ - if (query->cur_node->type == FTS_AST_TERM - && query->cur_node->term.wildcard) { - - term.f_len = query->cur_node->term.ptr->len; - ut_ad(FTS_MAX_WORD_LEN >= term.f_len); - memcpy(term.f_str, query->cur_node->term.ptr->str, term.f_len); - } else { - term.f_len = word->f_len; - ut_ad(FTS_MAX_WORD_LEN >= word->f_len); - memcpy(term.f_str, word->f_str, word->f_len); - } - - /* Lookup the word in our rb tree, it must exist. */ - ret = rbt_search(query->word_freqs, &parent, &term); - - ut_a(ret == 0); - - word_freq = rbt_value(fts_word_freq_t, parent.last); - - /* Start from 1 since the first column has been read by the caller. - Also, we rely on the order of the columns projected, to filter - out ilists that are out of range and we always want to read - the doc_count irrespective of the suitability of the row. */ - - for (i = 1; exp && !skip; exp = que_node_get_next(exp), ++i) { - - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - ulint len = dfield_get_len(dfield); - - ut_a(len != UNIV_SQL_NULL); - - /* Note: The column numbers below must match the SELECT. */ - - switch (i) { - case 1: /* DOC_COUNT */ - word_freq->doc_count += mach_read_from_4(data); - break; - - case 2: /* FIRST_DOC_ID */ - node.first_doc_id = fts_read_doc_id(data); - - /* Skip nodes whose doc ids are out range. */ - if (query->oper == FTS_EXIST - && query->upper_doc_id > 0 - && node.first_doc_id > query->upper_doc_id) { - skip = TRUE; - } - break; - - case 3: /* LAST_DOC_ID */ - node.last_doc_id = fts_read_doc_id(data); - - /* Skip nodes whose doc ids are out range. */ - if (query->oper == FTS_EXIST - && query->lower_doc_id > 0 - && node.last_doc_id < query->lower_doc_id) { - skip = TRUE; - } - break; - - case 4: /* ILIST */ - - error = fts_query_filter_doc_ids( - query, &word_freq->word, word_freq, - &node, data, len, FALSE); - - break; - - default: - ut_error; - } - } - - if (!skip) { - /* Make sure all columns were read. */ - - ut_a(i == 5); - } - - return error; -} - -/*****************************************************************//** -Callback function to fetch the rows in an FTS INDEX record. -@return always returns TRUE */ -static -ibool -fts_query_index_fetch_nodes( -/*========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to fts_fetch_t */ -{ - fts_string_t key; - sel_node_t* sel_node = static_cast(row); - fts_fetch_t* fetch = static_cast(user_arg); - fts_query_t* query = static_cast(fetch->read_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint dfield_len = dfield_get_len(dfield); - - key.f_str = static_cast(data); - key.f_len = dfield_len; - - ut_a(dfield_len <= FTS_MAX_WORD_LEN); - - /* Note: we pass error out by 'query->error' */ - query->error = fts_query_read_node(query, &key, que_node_get_next(exp)); - - if (query->error != DB_SUCCESS) { - ut_ad(query->error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT); - return(FALSE); - } else { - return(TRUE); - } -} - /*****************************************************************//** Calculate the inverse document frequency (IDF) for all the terms. */ static @@ -3837,7 +3321,7 @@ fts_query_parse( memset(&state, 0x0, sizeof(state)); - state.charset = query->fts_index_table.charset; + state.charset = fts_index_get_charset(query->index); DBUG_EXECUTE_IF("fts_instrument_query_disable_parser", query->parser = NULL;); @@ -3850,7 +3334,7 @@ fts_query_parse( } else { /* Setup the scanner to use, this depends on the mode flag. */ state.lexer = fts_lexer_create(mode, query_str, query_len); - state.charset = query->fts_index_table.charset; + state.charset = fts_index_get_charset(query->index); error = fts_parse(&state); fts_lexer_free(state.lexer); state.lexer = NULL; @@ -3931,10 +3415,6 @@ fts_query( query.deleted = fts_doc_ids_create(); query.cur_node = NULL; - query.fts_common_table.type = FTS_COMMON_TABLE; - query.fts_common_table.table_id = index->table->id; - query.fts_common_table.table = index->table; - charset = fts_index_get_charset(index); query.fts_index_table.type = FTS_INDEX_TABLE; @@ -3963,25 +3443,35 @@ fts_query( query.total_docs = dict_table_get_n_rows(index->table); - query.fts_common_table.suffix = "DELETED"; - - /* Read the deleted doc_ids, we need these for filtering. */ - error = fts_table_fetch_doc_ids( - NULL, &query.fts_common_table, query.deleted); + /* Create single FTSQueryExecutor for entire query lifecycle */ + query.executor = new FTSQueryExecutor(query_trx, index->table); - if (error != DB_SUCCESS) { - goto func_exit; + /* Prefetch all auxiliary and common tables to avoid repeated dict_sys.latch acquisitions */ + error = query.executor->open_all_aux_tables(index); + if (error == DB_SUCCESS) { + error = query.executor->open_all_deletion_tables(); } - query.fts_common_table.suffix = "DELETED_CACHE"; + if (error == DB_SUCCESS) { + /* Read the deleted doc_ids, we need these for filtering. */ + error = fts_table_fetch_doc_ids( + query.executor, index->table, "DELETED", + query.deleted); + } - error = fts_table_fetch_doc_ids( - NULL, &query.fts_common_table, query.deleted); + if (error == DB_SUCCESS) { + error = fts_table_fetch_doc_ids( + query.executor, index->table, "DELETED_CACHE", + query.deleted); + } if (error != DB_SUCCESS) { + query_trx->rollback(); goto func_exit; } + trx_commit_for_mysql(query_trx); + /* Get the deleted doc ids that are in the cache. */ fts_cache_append_deleted_doc_ids( index->table->fts->cache, query.deleted->doc_ids); @@ -4080,6 +3570,12 @@ fts_query( ut_free(lc_query_str); func_exit: + /* Clean up the dynamically allocated executor */ + if (query.executor) { + delete query.executor; + query.executor = nullptr; + } + fts_query_free(&query); query_trx->free(); @@ -4206,10 +3702,8 @@ fts_expand_query( fetch the original document and parse them. Future optimization could be done here if we support some forms of document-to-word mapping */ - fts_doc_fetch_by_doc_id(NULL, ranking->doc_id, index, - FTS_FETCH_DOC_BY_ID_EQUAL, - fts_query_expansion_fetch_doc, - &result_doc); + fts_query_fetch_document(index, ranking->doc_id, + &result_doc, true); /* Estimate memory used, see fts_process_token and fts_token_t. We ignore token size here. */ diff --git a/storage/innobase/fts/fts0sql.cc b/storage/innobase/fts/fts0sql.cc deleted file mode 100644 index 781d15f2befb0..0000000000000 --- a/storage/innobase/fts/fts0sql.cc +++ /dev/null @@ -1,208 +0,0 @@ -/***************************************************************************** - -Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2019, 2021, MariaDB Corporation. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file fts/fts0sql.cc -Full Text Search functionality. - -Created 2007-03-27 Sunny Bains -*******************************************************/ - -#include "que0que.h" -#include "trx0roll.h" -#include "pars0pars.h" -#include "dict0dict.h" -#include "fts0types.h" -#include "fts0priv.h" - -/** SQL statements for creating the ancillary FTS tables. */ - -/** Preamble to all SQL statements. */ -static const char* fts_sql_begin= - "PROCEDURE P() IS\n"; - -/** Postamble to non-committing SQL statements. */ -static const char* fts_sql_end= - "\n" - "END;\n"; - -/******************************************************************//** -Get the table id. -@return number of bytes written */ -int -fts_get_table_id( -/*=============*/ - const fts_table_t* - fts_table, /*!< in: FTS Auxiliary table */ - char* table_id) /*!< out: table id, must be at least - FTS_AUX_MIN_TABLE_ID_LENGTH bytes - long */ -{ - int len; - - ut_a(fts_table->table != NULL); - - switch (fts_table->type) { - case FTS_COMMON_TABLE: - len = fts_write_object_id(fts_table->table_id, table_id); - break; - - case FTS_INDEX_TABLE: - - len = fts_write_object_id(fts_table->table_id, table_id); - - table_id[len] = '_'; - ++len; - table_id += len; - - len += fts_write_object_id(fts_table->index_id, table_id); - break; - - default: - ut_error; - } - - ut_a(len >= 16); - ut_a(len < FTS_AUX_MIN_TABLE_ID_LENGTH); - - return(len); -} - -/** Construct the name of an internal FTS table for the given table. -@param[in] fts_table metadata on fulltext-indexed table -@param[out] table_name a name up to MAX_FULL_NAME_LEN -@param[in] dict_locked whether dict_sys.latch is being held */ -void fts_get_table_name(const fts_table_t* fts_table, char* table_name, - bool dict_locked) -{ - if (!dict_locked) { - dict_sys.freeze(SRW_LOCK_CALL); - } - ut_ad(dict_sys.frozen()); - /* Include the separator as well. */ - const size_t dbname_len = fts_table->table->name.dblen() + 1; - ut_ad(dbname_len > 1); - memcpy(table_name, fts_table->table->name.m_name, dbname_len); - if (!dict_locked) { - dict_sys.unfreeze(); - } - memcpy(table_name += dbname_len, "FTS_", 4); - table_name += 4; - table_name += fts_get_table_id(fts_table, table_name); - *table_name++ = '_'; - strcpy(table_name, fts_table->suffix); -} - -/******************************************************************//** -Parse an SQL string. -@return query graph */ -que_t* -fts_parse_sql( -/*==========*/ - fts_table_t* fts_table, /*!< in: FTS auxiliary table info */ - pars_info_t* info, /*!< in: info struct, or NULL */ - const char* sql) /*!< in: SQL string to evaluate */ -{ - char* str; - que_t* graph; - ibool dict_locked; - - str = ut_str3cat(fts_sql_begin, sql, fts_sql_end); - - dict_locked = (fts_table && fts_table->table->fts - && fts_table->table->fts->dict_locked); - - if (!dict_locked) { - /* The InnoDB SQL parser is not re-entrant. */ - dict_sys.lock(SRW_LOCK_CALL); - } - - graph = pars_sql(info, str); - ut_a(graph); - - if (!dict_locked) { - dict_sys.unlock(); - } - - ut_free(str); - - return(graph); -} - -/******************************************************************//** -Evaluate an SQL query graph. -@return DB_SUCCESS or error code */ -dberr_t -fts_eval_sql( -/*=========*/ - trx_t* trx, /*!< in: transaction */ - que_t* graph) /*!< in: Query graph to evaluate */ -{ - que_thr_t* thr; - - graph->trx = trx; - - ut_a(thr = que_fork_start_command(graph)); - - que_run_threads(thr); - - return(trx->error_state); -} - -/******************************************************************//** -Construct the column specification part of the SQL string for selecting the -indexed FTS columns for the given table. Adds the necessary bound -ids to the given 'info' and returns the SQL string. Examples: - -One indexed column named "text": - - "$sel0", - info/ids: sel0 -> "text" - -Two indexed columns named "subject" and "content": - - "$sel0, $sel1", - info/ids: sel0 -> "subject", sel1 -> "content", -@return heap-allocated WHERE string */ -const char* -fts_get_select_columns_str( -/*=======================*/ - dict_index_t* index, /*!< in: index */ - pars_info_t* info, /*!< in/out: parser info */ - mem_heap_t* heap) /*!< in: memory heap */ -{ - ulint i; - const char* str = ""; - - for (i = 0; i < index->n_user_defined_cols; i++) { - char* sel_str; - - dict_field_t* field = dict_index_get_nth_field(index, i); - - sel_str = mem_heap_printf(heap, "sel%lu", (ulong) i); - - /* Set copy_name to TRUE since it's dynamic. */ - pars_info_bind_id(info, sel_str, field->name); - - str = mem_heap_printf( - heap, "%s%s$%s", str, (*str) ? ", " : "", sel_str); - } - - return(str); -} diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 4f8fa2c5a6ee2..d7116ab868b75 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -11391,6 +11391,81 @@ ha_innobase::update_create_info( } } +/** Update FTS stopword configuration when reload is enabled +@param table FTS table +@param trx transaction +@param thd current thread +@param new_stopword_table stopword table name +@param new_stopword_is_on stopword enable setting +@return TRUE if success */ +static +bool innobase_fts_update_stopword_config(dict_table_t *table, trx_t *trx, + THD *thd, + const char *new_stopword_table, + bool new_stopword_is_on) +{ + ut_ad(dict_sys.locked()); + /* Create FTSQueryExecutor for config operations */ + FTSQueryExecutor executor(trx, table); + dberr_t error = DB_SUCCESS; + fts_string_t str; + + /* Load CONFIG table directly since we already hold dict_sys.lock() */ + char config_table_name[MAX_FULL_NAME_LEN]; + fts_table_t fts_table; + FTS_INIT_FTS_TABLE(&fts_table, nullptr, FTS_COMMON_TABLE, table); + fts_table.suffix = "CONFIG"; + fts_get_table_name(&fts_table, config_table_name, true); + const span config_name{ + config_table_name, strlen(config_table_name)}; + dict_table_t *config_table = dict_sys.load_table(config_name); + + if (config_table == nullptr) + return false; + + /* Assign CONFIG table directly to executor for optimized access */ + executor.set_config_table(config_table); + + /* Update FTS_USE_STOPWORD setting in CONFIG table */ + ulint use_stopword = (ulint) new_stopword_is_on; + fts_cache_t *cache= table->fts->cache; + error= fts_config_set_ulint( + &executor, table, FTS_USE_STOPWORD, use_stopword); + + if (error != DB_SUCCESS) + return false; + + if (!use_stopword) + { + cache->stopword_info.status = STOPWORD_OFF; + goto cleanup; + } + + /* Validate and update FTS_STOPWORD_TABLE_NAME in CONFIG + table if provided */ + if (new_stopword_table && + fts_load_user_stopword( + &executor, table->fts, new_stopword_table, + &table->fts->cache->stopword_info)) + { + /* Stopword table is valid, update CONFIG table */ + str.f_n_char = 0; + str.f_str = (byte*) new_stopword_table; + str.f_len = strlen(new_stopword_table); + + error = fts_config_set_value(&executor, table, FTS_STOPWORD_TABLE_NAME, &str); + } + else fts_load_default_stopword(&cache->stopword_info); +cleanup: + if (!cache->stopword_info.cached_stopword) + cache->stopword_info.cached_stopword= + rbt_create_arg_cmp( + sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, + &my_charset_latin1); + + return true; +} + /*****************************************************************//** Initialize the table FTS stopword list @return TRUE if success */ @@ -11413,9 +11488,24 @@ innobase_fts_load_stopword( } table->fts->dict_locked= true; - bool success= fts_load_stopword(table, trx, stopword_table, - THDVAR(thd, ft_enable_stopword), false); + bool own_trx= (trx == nullptr); + if (own_trx) + { + trx= trx_create(); + trx_start_internal(trx); + } + bool success= innobase_fts_update_stopword_config( + table, trx, thd, stopword_table, + THDVAR(thd, ft_enable_stopword)); + if (own_trx) + { + if (success) + trx_commit_for_mysql(trx); + else trx->rollback(); + trx->free(); + } table->fts->dict_locked= false; + DBUG_EXECUTE_IF("fts_load_stopword_fail", success= false;); return success; } @@ -15158,7 +15248,7 @@ ha_innobase::optimize( if (m_prebuilt->table->fts && m_prebuilt->table->fts->cache && m_prebuilt->table->space) { fts_sync_table(m_prebuilt->table); - fts_optimize_table(m_prebuilt->table); + fts_optimize_table(m_prebuilt->table, thd); } try_alter = false; } diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 5d260c7f0b072..624c83d33cf1a 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -49,6 +49,7 @@ Created July 18, 2007 Vasil Dimov #include "fts0types.h" #include "fts0opt.h" #include "fts0priv.h" +#include "fts0exec.h" #include "btr0btr.h" #include "page0zip.h" #include "fil0fil.h" @@ -2204,7 +2205,6 @@ i_s_fts_deleted_generic_fill( Field** fields; TABLE* table = (TABLE*) tables->table; trx_t* trx; - fts_table_t fts_table; fts_doc_ids_t* deleted; dict_table_t* user_table; @@ -2235,12 +2235,12 @@ i_s_fts_deleted_generic_fill( trx = trx_create(); trx->op_info = "Select for FTS DELETE TABLE"; - FTS_INIT_FTS_TABLE(&fts_table, - (being_deleted) ? "BEING_DELETED" : "DELETED", - FTS_COMMON_TABLE, user_table); - - fts_table_fetch_doc_ids(trx, &fts_table, deleted); + FTSQueryExecutor executor(trx, user_table); + fts_table_fetch_doc_ids( + &executor, user_table, + being_deleted ? "BEING_DELETED" : "DELETED", deleted); + trx_commit_for_mysql(trx); dict_table_close(user_table, thd, mdl_ticket); trx->free(); @@ -2660,100 +2660,36 @@ struct st_maria_plugin i_s_innodb_ft_index_cache = MariaDB_PLUGIN_MATURITY_STABLE }; -/*******************************************************************//** -Go through a FTS index auxiliary table, fetch its rows and fill +/** Go through a FTS index auxiliary table, fetch its rows and fill FTS word cache structure. +@param executor FTS query executor +@param reader record reader for processing auxiliary table records +@param selected auxiliary index +@param word word to select @return DB_SUCCESS on success, otherwise error code */ static -dberr_t -i_s_fts_index_table_fill_selected( -/*==============================*/ - dict_index_t* index, /*!< in: FTS index */ - ib_vector_t* words, /*!< in/out: vector to hold - fetched words */ - ulint selected, /*!< in: selected FTS index */ - fts_string_t* word) /*!< in: word to select */ -{ - pars_info_t* info; - fts_table_t fts_table; - trx_t* trx; - que_t* graph; - dberr_t error; - fts_fetch_t fetch; - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - fetch.read_arg = words; - fetch.read_record = fts_optimize_index_fetch_node; - fetch.total_memory = 0; - - DBUG_EXECUTE_IF("fts_instrument_result_cache_limit", - fts_result_cache_limit = 8192; - ); - - trx = trx_create(); - - trx->op_info = "fetching FTS index nodes"; - - pars_info_bind_function(info, "my_func", fetch.read_record, &fetch); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - FTS_INIT_INDEX_TABLE(&fts_table, fts_get_suffix(selected), - FTS_INDEX_TABLE, index); - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - &fts_table, info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT word, doc_count, first_doc_id, last_doc_id," - " ilist\n" - " FROM $table_name WHERE word >= :word;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - - break; - } else { - fts_sql_rollback(trx); - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout reading" - " FTS index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "Error occurred while reading" - " FTS index: " << error; - break; - } - } - } - - que_graph_free(graph); - - trx->free(); +dberr_t i_s_fts_index_table_fill_selected( + FTSQueryExecutor *executor, AuxRecordReader& reader, + uint8_t selected, fts_string_t *word) +{ + dberr_t error= DB_SUCCESS; + DBUG_EXECUTE_IF("fts_instrument_result_cache_limit", + fts_result_cache_limit = 8192;); + /* Fetch words from auxiliary index. If no starting word is specified, + fetch all words. Otherwise, fetch words starting from the given word. + This supports pagination when memory limits are exceeded. */ + if (word->f_str == nullptr) + error= executor->read_aux_all(selected, reader); + else + error= executor->read_aux( + selected, + reinterpret_cast(word->f_str), + PAGE_CUR_GE, reader); - if (fetch.total_memory >= fts_result_cache_limit) { - error = DB_FTS_EXCEED_RESULT_CACHE_LIMIT; - } + if (error == DB_RECORD_NOT_FOUND) + error= DB_SUCCESS; - return(error); + return error; } /*******************************************************************//** @@ -2890,6 +2826,91 @@ i_s_fts_index_table_fill_one_fetch( DBUG_RETURN(ret); } +/** Read words from a single auxiliary index with retry logic +@param executor FTS query executor +@param reader record reader +@param selected auxiliary index number +@param thd thread +@param tables tables to fill +@param conv_str conversion buffer +@param heap memory heap +@param words words vector +@return 0 on success, 1 on failure */ +static +int i_s_fts_read_aux_index_words( + FTSQueryExecutor *executor, AuxRecordReader &reader, + uint8_t selected, THD *thd, TABLE_LIST *tables, + fts_string_t *conv_str, mem_heap_t *heap, + ib_vector_t *words) +{ + fts_string_t word; + bool has_more = false; + int ret = 0; + + word.f_str = NULL; + word.f_len = 0; + word.f_n_char = 0; + + do + { + /* Fetch from index with retry logic for lock timeouts */ + for (;;) + { + trx_t *trx= executor->trx(); + dberr_t error = i_s_fts_index_table_fill_selected( + executor, reader, selected, &word); + + if (UNIV_LIKELY(error == DB_SUCCESS || + error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT)) + { + fts_sql_commit(trx); + has_more = error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT; + break; + } + else + { + fts_sql_rollback(trx); + if (error == DB_LOCK_WAIT_TIMEOUT) + { + sql_print_warning("InnoDB: Lock wait timeout" + " while reading FTS index. Retrying!"); + trx->error_state = DB_SUCCESS; + /* Clear words and retry */ + i_s_fts_index_table_free_one_fetch(words); + reader.reset_total_memory(); /* Reset memory counter on retry */ + } + else + { + sql_print_error("InnoDB: Error occurred while reading" + " FTS index: %s", ut_strerr(error)); + i_s_fts_index_table_free_one_fetch(words); + return 1; + } + } + } + + if (has_more) + { + fts_word_t *last_word; + /* Prepare start point for next fetch */ + last_word = static_cast(ib_vector_last(words)); + ut_ad(last_word != NULL); + fts_string_dup(&word, &last_word->text, heap); + reader.reset_total_memory(); /* Reset memory counter for next fetch */ + } + + /* Fill into tables */ + ret= i_s_fts_index_table_fill_one_fetch( + fts_index_get_charset(const_cast(executor->index())), + thd, tables, words,conv_str, has_more); + i_s_fts_index_table_free_one_fetch(words); + if (ret != 0) + return ret; + } while (has_more); + + return 0; +} + /*******************************************************************//** Go through a FTS index and its auxiliary tables, fetch rows in each table and fill INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE. @@ -2905,8 +2926,6 @@ i_s_fts_index_table_fill_one_index( { ib_vector_t* words; mem_heap_t* heap; - CHARSET_INFO* index_charset; - dberr_t error; int ret = 0; DBUG_ENTER("i_s_fts_index_table_fill_one_index"); @@ -2917,59 +2936,34 @@ i_s_fts_index_table_fill_one_index( words = ib_vector_create(ib_heap_allocator_create(heap), sizeof(fts_word_t), 256); - index_charset = fts_index_get_charset(index); + trx_t* trx= trx_create(); + trx->op_info= "fetching FTS index nodes"; + FTSQueryExecutor executor(trx, index->table); + dberr_t error= executor.open_all_aux_tables(index); - /* Iterate through each auxiliary table as described in - fts_index_selector */ - for (ulint selected = 0; selected < FTS_NUM_AUX_INDEX; selected++) { - fts_string_t word; - bool has_more = false; + if (error) return 1; + ulint total_memory= 0; + AuxRecordReader reader(words, &total_memory); - word.f_str = NULL; - word.f_len = 0; - word.f_n_char = 0; + for (uint8_t selected = 0; selected < FTS_NUM_AUX_INDEX; selected++) { + reader.reset_total_memory(); + ret = i_s_fts_read_aux_index_words( + &executor, reader, selected, + thd, tables, conv_str, heap, words); - do { - /* Fetch from index */ - error = i_s_fts_index_table_fill_selected( - index, words, selected, &word); - - if (error == DB_SUCCESS) { - has_more = false; - } else if (error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT) { - has_more = true; - } else { - i_s_fts_index_table_free_one_fetch(words); - ret = 1; - goto func_exit; - } - - if (has_more) { - fts_word_t* last_word; - - /* Prepare start point for next fetch */ - last_word = static_cast(ib_vector_last(words)); - ut_ad(last_word != NULL); - fts_string_dup(&word, &last_word->text, heap); - } - - /* Fill into tables */ - ret = i_s_fts_index_table_fill_one_fetch( - index_charset, thd, tables, words, conv_str, - has_more); - i_s_fts_index_table_free_one_fetch(words); - - if (ret != 0) { - goto func_exit; - } - } while (has_more); + if (ret != 0) { + goto func_exit; + } } func_exit: + trx->free(); mem_heap_free(heap); DBUG_RETURN(ret); } + + /*******************************************************************//** Fill the dynamic table INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE @return 0 on success, 1 on failure */ @@ -3116,7 +3110,6 @@ i_s_fts_config_fill( Field** fields; TABLE* table = (TABLE*) tables->table; trx_t* trx; - fts_table_t fts_table; dict_table_t* user_table; ulint i = 0; dict_index_t* index = NULL; @@ -3149,9 +3142,7 @@ i_s_fts_config_fill( trx = trx_create(); trx->op_info = "Select for FTS CONFIG TABLE"; - - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, user_table); - + FTSQueryExecutor executor(trx, user_table); if (!ib_vector_is_empty(user_table->fts->indexes)) { index = (dict_index_t*) ib_vector_getp_const( user_table->fts->indexes, 0); @@ -3178,7 +3169,7 @@ i_s_fts_config_fill( key_name = (char*) fts_config_key[i]; } - fts_config_get_value(trx, &fts_table, key_name, &value); + fts_config_get_value(&executor, user_table, key_name, &value); if (allocated) { ut_free(key_name); diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 53f88cc8ca1f5..9387adb234f24 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -346,7 +346,7 @@ btr_cur_del_mark_set_clust_rec( que_thr_t* thr, /*!< in: query thread */ const dtuple_t* entry, /*!< in: dtuple for the deleting record */ mtr_t* mtr) /*!< in/out: mini-transaction */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + MY_ATTRIBUTE((nonnull(1,2,3,4,5,7), warn_unused_result)); /*************************************************************//** Tries to compress a page of the tree if it seems useful. It is assumed that mtr holds an x-latch on the tree and on the cursor page. To avoid diff --git a/storage/innobase/include/fts0exec.h b/storage/innobase/include/fts0exec.h new file mode 100644 index 0000000000000..c828e33fdb2c1 --- /dev/null +++ b/storage/innobase/include/fts0exec.h @@ -0,0 +1,456 @@ +/***************************************************************************** + +Copyright (c) 2025, MariaDB PLC. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/fts0exec.h +FTS Query Builder - Abstraction layer for FTS operations + +Created 2025/10/30 +*******************************************************/ + +#pragma once + +#include "row0query.h" +#include "fts0fts.h" +#include "fts0types.h" +#include "fts0opt.h" +#include "fts0ast.h" +#include + +/** Structure to represent FTS auxiliary table data for insertion */ +struct fts_aux_data_t +{ + /** + CREATE TABLE $FTS_PREFIX_INDEX_[1-6]( + word VARCHAR(FTS_MAX_WORD_LEN), + first_doc_id INT NOT NULL, + last_doc_id UNSIGNED NOT NULL, + doc_count UNSIGNED INT NOT NULL, + ilist VARBINARY NOT NULL, + UNIQUE CLUSTERED INDEX ON (word, first_doc_id)); + */ + const char *word; + ulint word_len; + doc_id_t first_doc_id; + doc_id_t last_doc_id; + uint32_t doc_count; + const byte *ilist; + ulint ilist_len; + + fts_aux_data_t(const char *w, ulint w_len) + : word(w), word_len(w_len) + { + first_doc_id= last_doc_id= doc_count= 0; + ilist= nullptr; + ilist_len= 0; + } + + fts_aux_data_t(const char* w, ulint w_len, doc_id_t first_id, + doc_id_t last_id, uint32_t d_count, const byte* il, + ulint il_len) + : word(w), word_len(w_len), first_doc_id(first_id), + last_doc_id(last_id), doc_count(d_count), ilist(il), + ilist_len(il_len) {} +}; + +/** FTS deletion table types for m_common_tables array indexing */ +enum class FTSDeletionTable : uint8_t +{ + DELETED = 0, + DELETED_CACHE = 1, + BEING_DELETED = 2, + BEING_DELETED_CACHE = 3, + MAX_DELETION_TABLES = 4 +}; + +/** Helper to convert FTSDeletionTable to array index */ +constexpr uint8_t to_index(FTSDeletionTable table_type) noexcept +{ + return static_cast(table_type); +} + +/** Number of deletion tables */ +constexpr uint8_t NUM_DELETION_TABLES = to_index(FTSDeletionTable::MAX_DELETION_TABLES); + +/** Abstraction over QueryExecutor for FTS auxiliary/common tables. +Handles table open/lock and provides typed helpers to insert, +delete and read records in FTS INDEX_1..INDEX_6 +and deletion tables (DELETED, BEING_DELETED, etc.) */ +class FTSQueryExecutor +{ +private: + QueryExecutor m_executor; + const dict_index_t *m_index; + const dict_table_t *const m_table; + /* FTS Auxiliary table pointers */ + dict_table_t *m_aux_tables[6]={nullptr}; + /* FTS deletion table pointers (DELETED, BEING_DELETED, etc.) */ + dict_table_t *m_common_tables[NUM_DELETION_TABLES]={nullptr}; + /* FTS CONFIG table pointer */ + dict_table_t *m_config_table={nullptr}; + + /** Table preparation methods */ + + /** Open FTS INDEX_[1..6] table for the given auxiliary index. + @return DB_SUCCESS or error code */ + dberr_t open_aux_table(uint8_t aux_index) noexcept; + + /** Open a deletion table (DELETED, BEING_DELETED, etc.). + @param table_type deletion table type + @return DB_SUCCESS or error code */ + dberr_t open_deletion_table(FTSDeletionTable table_type) noexcept; + + /** Helper to convert deletion table enum to string name */ + static const char* get_deletion_table_name( + FTSDeletionTable table_type) noexcept; + + /** Table lock operation */ + + /** Acquire a lock on an opened INDEX_[1..6] table. + Retries lock wait once via QueryExecutor. + @return DB_SUCCESS or error code */ + dberr_t lock_aux_tables(uint8_t aux_index, lock_mode mode) noexcept; + + /** Lock all auxiliary tables */ + dberr_t lock_all_aux(lock_mode mode) noexcept; + + /** Acquire a lock on an opened common FTS table. + Retries lock wait once via QueryExecutor. + @return DB_SUCCESS or error code */ + dberr_t lock_common_tables(uint8_t index, lock_mode mode) noexcept; + + /** Acquire lock on opened all common FTS table + @return DB_SUCCESS or error code */ + dberr_t lock_all_common(lock_mode mode) noexcept; + +public: + /** Open all auxiliary tables + @return DB_SUCCESS or error code */ + dberr_t open_all_aux_tables(dict_index_t *fts_index) noexcept; + + /** Open all deletion tables (DELETED, BEING_DELETED, etc.) + @return DB_SUCCESS or error code */ + dberr_t open_all_deletion_tables() noexcept; + + /** Open FTS CONFIG table for configuration operations. + @return DB_SUCCESS or error code */ + dberr_t open_config_table() noexcept; + + /** Set CONFIG table directly (for cases where table is already opened) + @param config_table CONFIG table pointer */ + void set_config_table(dict_table_t *config_table) noexcept + { + m_config_table = config_table; + m_config_table->acquire(); + } + + /** Create executor bound to trx and FTS table/index. + @param trx transaction + @param fts_table FTS table */ + FTSQueryExecutor(trx_t *trx, const dict_table_t *fts_table); + + /** Release any opened table handles and executor resources. */ + ~FTSQueryExecutor(); + + /** High level DML operation on FTS TABLE */ + + /** Insert a row into auxiliary INDEX_[1..6] table. + Expects (word, first_doc_id, trx_id, roll_ptr, last_doc_id, + doc_count, ilist). + @param aux_index auxiliary table index + @param aux_data data to be inserted + @return DB_SUCCESS or error code */ + dberr_t insert_aux_record(uint8_t aux_index, + const fts_aux_data_t *aux_data) noexcept; + + /** Insert a single doc_id into a common table (e.g. DELETED, ...) + @param tbl_name common table name + @param doc_id document id to be inserted + @return DB_SUCCESS or error code */ + dberr_t insert_common_record(const char *tbl_name, doc_id_t doc_id) noexcept; + + /** Insert a key/value into CONFIG table. + @param key key for the config table + @param value value for the key + @return DB_SUCCESS or error code */ + dberr_t insert_config_record(const char *key, const char *value) noexcept; + + /** Delete one word row from INDEX_[1..6] by (word). + @param aux_index auxiliary table index + @param aux_data auxiliary table record + @return DB_SUCCESS or error code */ + dberr_t delete_aux_record(uint8_t aux_index, + const fts_aux_data_t *aux_data) noexcept; + + /** Delete a single doc_id row from a common table by (doc_id). + @param tbl_name common table name + @param doc_id document id to be deleted + @return DB_SUCCESS or error code */ + dberr_t delete_common_record(const char *tbl_name, doc_id_t doc_id) noexcept; + + /** Delete all rows from a common table. + @return DB_SUCCESS or error code */ + dberr_t delete_all_common_records(const char *tbl_name) noexcept; + + /** Delete a key from CONFIG table by (key). + @return DB_SUCCESS or error code */ + dberr_t delete_config_record(const char *key) noexcept; + + /** Upsert a key/value in CONFIG table. + Replaces 'value' if key exists, inserts otherwise. + @return DB_SUCCESS or error code */ + dberr_t update_config_record(const char *key, const char *value) noexcept; + + /** Select-for-update CONFIG row by 'key' + @return DB_SUCCESS or error code */ + dberr_t read_config_with_lock( + const char *key, RecordCallback& callback) noexcept; + + /** Read Auxiliary INDEX_[1..6] table rows at (or) after + 'word' with given cursor mode. Callback is invoked for each + row for comparing it with word and process it if there is a match + @return DB_SUCCESS or error code */ + dberr_t read_aux(uint8_t aux_index, const char *word, + page_cur_mode_t mode, RecordCallback& callback) noexcept; + + /** Read all INDEX_[1..6] rows + Callback is invoked for each row for comparing it with word + and process it if it is matching + @return DB_SUCCESS or error code */ + dberr_t read_aux_all(uint8_t aux_index, RecordCallback& callback) noexcept; + + /** Read all rows from given COMMON table + Callback is invoked for processing the record */ + dberr_t read_all_common(const char *tbl_name, + RecordCallback& callback) noexcept; + mem_heap_t* get_heap() const noexcept + { return m_executor.get_heap(); } + + trx_t* trx() const noexcept + { return m_executor.get_trx(); } + + const dict_index_t* index() const noexcept + { return m_index; } + + void release_lock() { m_executor.commit_mtr(); } + + /** Read records from table using underlying QueryExecutor + @param table table to read + @param tuple search tuple + @param mode cursor mode + @param callback record callback + @return DB_SUCCESS or error code */ + dberr_t read(dict_table_t *table, const dtuple_t *tuple, + page_cur_mode_t mode, RecordCallback& callback) noexcept + { + dberr_t err= m_executor.read(table, tuple, mode, callback); + return (err == DB_SUCCESS_LOCKED_REC) ? DB_SUCCESS : err; + } + + /** Read records by index using underlying QueryExecutor + @param table table to read + @param sec_index secondary index + @param search_tuple search tuple + @param mode cursor mode + @param callback record callback + @return DB_SUCCESS or error code */ + dberr_t read_by_index(dict_table_t *table, dict_index_t *sec_index, + const dtuple_t *search_tuple, page_cur_mode_t mode, + RecordCallback& callback) noexcept + { + dberr_t err= m_executor.read_by_index(table, sec_index, search_tuple, mode, callback); + return (err == DB_SUCCESS_LOCKED_REC) ? DB_SUCCESS : err; + } + + /** Construct FTS auxiliary table name + @param table_name output buffer for table name + @param suffix table suffix (e.g., "CONFIG", "INDEX_1") + @param common_table true for common tables, false for index tables */ + void construct_table_name(char *table_name, const char *suffix, + bool common_table) noexcept; +}; + +/** Callback class for reading common table records +(DELETED, BEING_DELETED, DELETED_CACHE, BEING_DELETED_CACHE) */ +class CommonTableReader : public RecordCallback +{ +private: + std::vector doc_ids; + +public: + CommonTableReader(); + + const std::vector& get_doc_ids() const { return doc_ids; } + void clear() { doc_ids.clear(); } + + /** Fast common table field extraction for known table format. + Structure: (doc_id BIGINT UNSIGNED) - always known schema + @param rec record to extract from + @param index common table index + @param doc_id output doc_id value + @return true if extraction successful */ + static bool extract_common_fields( + const rec_t *rec, const dict_index_t *index, + doc_id_t *doc_id); +}; + +/** Callback class for reading FTS config table records */ +class ConfigReader : public RecordCallback +{ +public: + span value_span; + ConfigReader(); + + /** Extract the config table record. + Structure: (key VARCHAR, db_trx_id, db_roll_ptr, value TEXT) + @param rec record to extract from + @param index config table index + @param key_data output pointer to key data + @param key_len output key length + @param value_data output pointer to value data (optional) + @param value_len output value length (optional) + @return true if extraction successful */ + static bool extract_config_fields( + const rec_t *rec, const dict_index_t *index, + const byte **key_data, ulint *key_len, + const byte **value_data = nullptr, ulint *value_len = nullptr); + + /** Direct config key comparison - compares first field with tuple value. + @param search_tuple search tuple containing target key + @param rec record to compare + @param index config table index + @return comparison action */ + static RecordCompareAction compare_config_key( + const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index); +}; + +/** Type alias for FTS record processor function */ +using FTSRecordProcessor= std::function< + dberr_t(const rec_t*, const dict_index_t*, const rec_offs*, void*)>; + +/** Comparison modes for AuxRecordReader */ +enum class AuxCompareMode +{ + /** >= comparison (range scan from word) */ + GREATER_EQUAL, + /** > comparison (exclude exact match) */ + GREATER, + /** LIKE pattern matching (prefix match) */ + LIKE, + /** = comparison (exact match) */ + EQUAL +}; + +/** Callback class for reading FTS auxiliary index table records */ +class AuxRecordReader : public RecordCallback +{ +private: + void *user_arg; + ulint *total_memory; + AuxCompareMode compare_mode; + +private: + /** FTS-specific record comparison logic */ + RecordCompareAction compare_record( + const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index) noexcept; + +public: + /** Structure to hold extracted auxiliary table fields */ + struct AuxRecordFields + { + const byte* word_data; + ulint word_len; + doc_id_t first_doc_id; + doc_id_t last_doc_id; + ulint doc_count; + byte* ilist_data; + ulint ilist_len; + bool ilist_is_external; + mem_heap_t* ilist_heap; + + AuxRecordFields() : word_data(nullptr), word_len(0), first_doc_id(0), + last_doc_id(0), doc_count(0), ilist_data(nullptr), + ilist_len(0), ilist_is_external(false), ilist_heap(nullptr) {} + + ~AuxRecordFields() { if (ilist_heap) mem_heap_free(ilist_heap); } + + AuxRecordFields(const AuxRecordFields&) = delete; + AuxRecordFields& operator=(const AuxRecordFields&) = delete; + }; + + /** Fast auxiliary table field extraction for known table format. + Structure: (word VARCHAR, first_doc_id BIGINT, ..., last_doc_id BIGINT, doc_count INT, ilist BLOB) + @param rec record to extract from + @param index auxiliary table index + @param fields output structure with extracted fields + @param word_only if true, extract only word field + @return true if extraction successful */ + static bool extract_aux_fields( + const rec_t *rec, const dict_index_t *index, + AuxRecordFields &fields, bool word_only = false); + + /** Default word processor for FTS auxiliary table records */ + dberr_t default_word_processor(const rec_t *rec, const dict_index_t *index, + const rec_offs *offsets, void *user_arg); + + /* Constructor with custom processor */ + template + AuxRecordReader(void* user_data, + ProcessorFunc proc_func, + AuxCompareMode mode= AuxCompareMode::GREATER_EQUAL) + : RecordCallback( + [this, proc_func](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + return proc_func(rec, index, offsets, this->user_arg); + }, + [this](const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index) + -> RecordCompareAction + { + return this->compare_record(search_tuple, rec, index); + } + ), + user_arg(user_data), total_memory(nullptr), + compare_mode(mode) {} + + /* Different constructor with default word processing */ + AuxRecordReader(void *user_data, ulint *memory_counter, + AuxCompareMode mode= AuxCompareMode::GREATER_EQUAL) + : RecordCallback( + [this](const rec_t *rec, const dict_index_t *index, + const rec_offs *offsets) -> dberr_t + { + return this->default_word_processor(rec, index, offsets, + this->user_arg); + }, + [this](const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index) + -> RecordCompareAction + { + return this->compare_record(search_tuple, rec, index); + } + ), + user_arg(user_data), total_memory(memory_counter), + compare_mode(mode) {} + + /** Reset total memory counter */ + void reset_total_memory() { if (total_memory) *total_memory = 0; } +}; diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 9dd48e525194d..3428977632f56 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -40,6 +40,9 @@ Created 2011/09/02 Sunny Bains #include "mysql/plugin_ftparser.h" #include "lex_string.h" +/* Forward declarations */ +class FTSQueryExecutor; + /** "NULL" value of a document id. */ #define FTS_NULL_DOC_ID 0 @@ -611,13 +614,12 @@ fts_create( dict_table_t* table); /*!< out: table with FTS indexes */ -/*********************************************************************//** -Run OPTIMIZE on the given table. -@return DB_SUCCESS if all OK */ +/** Run OPTIMIZE on the given table. +@param table table to be optimized +@param thd thread +@return DB_SUCCESS if all ok */ dberr_t -fts_optimize_table( -/*===============*/ - dict_table_t* table); /*!< in: table to optimiza */ +fts_optimize_table(dict_table_t *table, THD *thd); /**********************************************************************//** Startup the optimize thread and create the work queue. */ @@ -780,14 +782,6 @@ fts_tokenize_document_internal( const char* doc, /*!< in: document to tokenize */ int len); /*!< in: document length */ -/*********************************************************************//** -Fetch COUNT(*) from specified table. -@return the number of rows in the table */ -ulint -fts_get_rows_count( -/*===============*/ - fts_table_t* fts_table); /*!< in: fts table to read */ - /*************************************************************//** Get maximum Doc ID in a table if index "FTS_DOC_ID_INDEX" exists @return max Doc ID or 0 if index "FTS_DOC_ID_INDEX" does not exist */ @@ -804,33 +798,24 @@ fts_get_max_doc_id( CHARSET_INFO *fts_valid_stopword_table(const char *stopword_table_name, const char **row_end= NULL); -/****************************************************************//** -This function loads specified stopword into FTS cache +/** This function loads specified stopword into FTS cache +@param executor FTSQueryExecutor instance +@param table table which has fts index @return true if success */ -bool -fts_load_stopword( -/*==============*/ - const dict_table_t* - table, /*!< in: Table with FTS */ - trx_t* trx, /*!< in: Transaction */ - const char* session_stopword_table, /*!< in: Session stopword table - name */ - bool stopword_is_on, /*!< in: Whether stopword - option is turned on/off */ - bool reload); /*!< in: Whether it is during - reload of FTS table */ +bool fts_load_stopword(FTSQueryExecutor* executor, + const dict_table_t* table); + +/** Read the rows from the fulltext index +@param executor FTSQueryExecutor instance +@param table Fulltext table +@param tbl_name table name +@param doc_ids collecting doc ids +@return DB_SUCCESS or error code */ +dberr_t fts_table_fetch_doc_ids(FTSQueryExecutor *executor, + dict_table_t *table, const char *tbl_name, + fts_doc_ids_t *doc_ids) noexcept; /****************************************************************//** -Read the rows from the FTS index -@return DB_SUCCESS if OK */ -dberr_t -fts_table_fetch_doc_ids( -/*====================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: aux table */ - fts_doc_ids_t* doc_ids); /*!< in: For collecting - doc ids */ -/****************************************************************//** This function brings FTS index in sync when FTS index is first used. There are documents that have not yet sync-ed to auxiliary tables from last server abnormally shutdown, we will need to bring @@ -918,16 +903,56 @@ bool fts_check_aux_table(const char *name, /** Update the last document id. This function could create a new transaction to update the last document id. -@param table table to be updated -@param doc_id last document id -@param trx update trx or null +@param executor query executor +@param table table to be updated +@param doc_id last document id @retval DB_SUCCESS if OK */ dberr_t -fts_update_sync_doc_id(const dict_table_t *table, - doc_id_t doc_id, - trx_t *trx) - MY_ATTRIBUTE((nonnull(1))); +fts_update_sync_doc_id(FTSQueryExecutor *executor, + const dict_table_t *table, + doc_id_t doc_id); /** Sync the table during commit phase @param[in] table table to be synced */ void fts_sync_during_ddl(dict_table_t* table); + +/** Tokenize a document. +@param[in,out] doc document to tokenize +@param[out] result tokenization result +@param[in] parser pluggable parser */ +void fts_tokenize_document( + fts_doc_t* doc, + fts_doc_t* result, + st_mysql_ftparser* parser); + +/** Continue to tokenize a document. +@param[in,out] doc document to tokenize +@param[in] add_pos add this position to all tokens from this tokenization +@param[out] result tokenization result +@param[in] parser pluggable parser */ +void fts_tokenize_document_next( + fts_doc_t* doc, + ulint add_pos, + fts_doc_t* result, + st_mysql_ftparser* parser); + +/** Get a character set based on precise type. +@param prtype precise type +@return the corresponding character set */ +CHARSET_INFO* fts_get_charset(ulint prtype); + +/** Load user defined stopword from designated user table +@param fts fulltext structure +@param stopword_table stopword table +@param stopword_info stopword information +@return whether the operation is successful */ +bool fts_load_user_stopword(FTSQueryExecutor *executor, fts_t *fts, + const char *stopword_table, + fts_stopword_t *stopword_info); + +/****************************************************************//** +This function loads the default InnoDB stopword list */ +void +fts_load_default_stopword( +/*======================*/ + fts_stopword_t* stopword_info); /*!< in: stopword info */ diff --git a/storage/innobase/include/fts0priv.h b/storage/innobase/include/fts0priv.h index 1d3bc323841b9..79860d2c84041 100644 --- a/storage/innobase/include/fts0priv.h +++ b/storage/innobase/include/fts0priv.h @@ -28,10 +28,10 @@ Created 2011/09/02 Sunny Bains #define INNOBASE_FTS0PRIV_H #include "dict0dict.h" -#include "pars0pars.h" #include "que0que.h" #include "que0types.h" #include "fts0types.h" +#include "fts0exec.h" /* The various states of the FTS sub system pertaining to a table with FTS indexes defined on it. */ @@ -106,26 +106,6 @@ component. /** Maximum length of an integer stored in the config table value column. */ #define FTS_MAX_INT_LEN 32 -/******************************************************************//** -Parse an SQL string. %s is replaced with the table's id. -@return query graph */ -que_t* -fts_parse_sql( -/*==========*/ - fts_table_t* fts_table, /*!< in: FTS aux table */ - pars_info_t* info, /*!< in: info struct, or NULL */ - const char* sql) /*!< in: SQL string to evaluate */ - MY_ATTRIBUTE((nonnull(3), malloc, warn_unused_result)); -/******************************************************************//** -Evaluate a parsed SQL statement -@return DB_SUCCESS or error code */ -dberr_t -fts_eval_sql( -/*=========*/ - trx_t* trx, /*!< in: transaction */ - que_t* graph) /*!< in: Parsed statement */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); - /** Construct the name of an internal FTS table for the given table. @param[in] fts_table metadata on fulltext-indexed table @param[out] table_name a name up to MAX_FULL_NAME_LEN @@ -133,77 +113,15 @@ fts_eval_sql( void fts_get_table_name(const fts_table_t* fts_table, char* table_name, bool dict_locked = false) MY_ATTRIBUTE((nonnull)); -/******************************************************************//** -Construct the column specification part of the SQL string for selecting the -indexed FTS columns for the given table. Adds the necessary bound -ids to the given 'info' and returns the SQL string. Examples: - -One indexed column named "text": - "$sel0", - info/ids: sel0 -> "text" - -Two indexed columns named "subject" and "content": - - "$sel0, $sel1", - info/ids: sel0 -> "subject", sel1 -> "content", -@return heap-allocated WHERE string */ -const char* -fts_get_select_columns_str( -/*=======================*/ - dict_index_t* index, /*!< in: FTS index */ - pars_info_t* info, /*!< in/out: parser info */ - mem_heap_t* heap) /*!< in: memory heap */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); - -/** define for fts_doc_fetch_by_doc_id() "option" value, defines whether -we want to get Doc whose ID is equal to or greater or smaller than supplied -ID */ -#define FTS_FETCH_DOC_BY_ID_EQUAL 1 -#define FTS_FETCH_DOC_BY_ID_LARGE 2 -#define FTS_FETCH_DOC_BY_ID_SMALL 3 - -/*************************************************************//** -Fetch document (= a single row's indexed text) with the given -document id. -@return: DB_SUCCESS if fetch is successful, else error */ -dberr_t -fts_doc_fetch_by_doc_id( -/*====================*/ - fts_get_doc_t* get_doc, /*!< in: state */ - doc_id_t doc_id, /*!< in: id of document to fetch */ - dict_index_t* index_to_use, /*!< in: caller supplied FTS index, - or NULL */ - ulint option, /*!< in: search option, if it is - greater than doc_id or equal */ - fts_sql_callback - callback, /*!< in: callback to read - records */ - void* arg) /*!< in: callback arg */ - MY_ATTRIBUTE((nonnull(6))); - -/*******************************************************************//** -Callback function for fetch that stores the text of an FTS document, -converting each column to UTF-16. -@return always FALSE */ -ibool -fts_query_expansion_fetch_doc( -/*==========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ - MY_ATTRIBUTE((nonnull)); -/******************************************************************** -Write out a single word's data as new entry/entries in the INDEX table. -@return DB_SUCCESS if all OK. */ -dberr_t -fts_write_node( -/*===========*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: query graph */ - fts_table_t* fts_table, /*!< in: the FTS aux index */ - fts_string_t* word, /*!< in: word in UTF-8 */ - fts_node_t* node) /*!< in: node columns */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); +/** Write out a single word's data as new entry/entries in the INDEX table. +@param executor FTS Query Executor +@param selected auxiliary index number +@param aux_data auxiliary table data +@return DB_SUCCESS if all OK or error code */ +dberr_t fts_write_node(FTSQueryExecutor *executor, uint8_t selected, + const fts_aux_data_t *aux_data) + MY_ATTRIBUTE((nonnull, warn_unused_result)); /** Check if a fts token is a stopword or less than fts_min_token_size or greater than fts_max_token_size. @@ -252,35 +170,36 @@ fts_word_free( /*==========*/ fts_word_t* word) /*!< in: instance to free.*/ MY_ATTRIBUTE((nonnull)); -/******************************************************************//** -Read the rows from the FTS inde -@return DB_SUCCESS or error code */ -dberr_t -fts_index_fetch_nodes( -/*==================*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: prepared statement */ - fts_table_t* fts_table, /*!< in: FTS aux table */ - const fts_string_t* - word, /*!< in: the word to fetch */ - fts_fetch_t* fetch) /*!< in: fetch callback.*/ - MY_ATTRIBUTE((nonnull)); + +/** Read the rows from FTS index +@param trx transaction +@param index fulltext index +@param word word to fetch +@param user_arg user argument +@param processor custom processor to filter the word from record +@param compare_mode comparison mode for record matching +@return error code or DB_SUCCESS */ +dberr_t fts_index_fetch_nodes(FTSQueryExecutor *executor, dict_index_t *index, + const fts_string_t *word, + void *user_arg, + FTSRecordProcessor processor, + AuxCompareMode compare_mode) + MY_ATTRIBUTE((nonnull)); + #define fts_sql_commit(trx) trx_commit_for_mysql(trx) #define fts_sql_rollback(trx) (trx)->rollback() -/******************************************************************//** -Get value from config table. The caller must ensure that enough -space is allocated for value to hold the column contents + +/** Get value from the config table. The caller must ensure that enough +space is allocated for value to hold the column contents. +@param trx transaction +@param table Indexed fts table +@param name name of the key +@param value value of the key @return DB_SUCCESS or error code */ -dberr_t -fts_config_get_value( -/*=================*/ - trx_t* trx, /* transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ - const char* name, /*!< in: get config value for - this parameter name */ - fts_string_t* value) /*!< out: value read from - config table */ - MY_ATTRIBUTE((nonnull)); +dberr_t fts_config_get_value(FTSQueryExecutor *executor, const dict_table_t *table, + const char *name, fts_string_t *value) + MY_ATTRIBUTE((nonnull)); + /******************************************************************//** Get value specific to an FTS index from the config table. The caller must ensure that enough space is allocated for value to hold the @@ -289,7 +208,7 @@ column contents. dberr_t fts_config_get_index_value( /*=======================*/ - trx_t* trx, /*!< transaction */ + FTSQueryExecutor* executor, /*!< in: query executor */ dict_index_t* index, /*!< in: index */ const char* param, /*!< in: get config value for this parameter name */ @@ -302,8 +221,8 @@ Set the value in the config table for name. dberr_t fts_config_set_value( /*=================*/ - trx_t* trx, /*!< transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ + FTSQueryExecutor* executor, /*!< in: query executor */ + const dict_table_t* table, /*!< in: the indexed FTS table */ const char* name, /*!< in: get config value for this parameter name */ const fts_string_t* @@ -315,8 +234,8 @@ Set an ulint value in the config table. dberr_t fts_config_set_ulint( /*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ + FTSQueryExecutor* executor, /*!< in: query executor */ + const dict_table_t* table, /*!< in: the indexed FTS table */ const char* name, /*!< in: param name */ ulint int_value) /*!< in: value */ MY_ATTRIBUTE((nonnull, warn_unused_result)); @@ -326,7 +245,7 @@ Set the value specific to an FTS index in the config table. dberr_t fts_config_set_index_value( /*=======================*/ - trx_t* trx, /*!< transaction */ + FTSQueryExecutor* executor, /*!< in: query executor */ dict_index_t* index, /*!< in: index */ const char* param, /*!< in: get config value for this parameter name */ @@ -340,8 +259,8 @@ Get an ulint value from the config table. dberr_t fts_config_get_ulint( /*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ + FTSQueryExecutor* executor, /*!< in: query executor */ + const dict_table_t* table, /*!< in: the indexed FTS table */ const char* name, /*!< in: param name */ ulint* int_value) /*!< out: value */ MY_ATTRIBUTE((nonnull)); diff --git a/storage/innobase/include/fts0types.h b/storage/innobase/include/fts0types.h index 17b0f947de277..fc9b5c8c5321c 100644 --- a/storage/innobase/include/fts0types.h +++ b/storage/innobase/include/fts0types.h @@ -313,7 +313,7 @@ fts_get_suffix( @param[in] len string length in bytes @return the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index( const CHARSET_INFO* cs, const byte* str, diff --git a/storage/innobase/include/fts0types.inl b/storage/innobase/include/fts0types.inl index 622d1337bad60..d172ab38ea42c 100644 --- a/storage/innobase/include/fts0types.inl +++ b/storage/innobase/include/fts0types.inl @@ -102,13 +102,13 @@ inline bool fts_is_charset_cjk(const CHARSET_INFO* cs) @param[in] len string length @retval the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index_by_range( const CHARSET_INFO* cs, const byte* str, ulint len) { - ulint selected = 0; + uint8_t selected = 0; ulint value = innobase_strnxfrm(cs, str, len); while (fts_index_selector[selected].value != 0) { @@ -136,7 +136,7 @@ fts_select_index_by_range( @param[in] len string length @retval the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index_by_hash( const CHARSET_INFO* cs, const byte* str, @@ -162,7 +162,7 @@ fts_select_index_by_hash( /* Get collation hash code */ my_ci_hash_sort(&hasher, cs, str, char_len); - return(hasher.m_nr1 % FTS_NUM_AUX_INDEX); + return static_cast(hasher.m_nr1 % FTS_NUM_AUX_INDEX); } /** Select the FTS auxiliary index for the given character. @@ -171,21 +171,17 @@ fts_select_index_by_hash( @param[in] len string length in bytes @retval the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index( const CHARSET_INFO* cs, const byte* str, ulint len) { - ulint selected; - if (fts_is_charset_cjk(cs)) { - selected = fts_select_index_by_hash(cs, str, len); - } else { - selected = fts_select_index_by_range(cs, str, len); + return fts_select_index_by_hash(cs, str, len); } - return(selected); + return fts_select_index_by_range(cs, str, len); } /******************************************************************//** diff --git a/storage/innobase/include/page0cur.h b/storage/innobase/include/page0cur.h index 14a3ce9ba038c..ab871d7ef8511 100644 --- a/storage/innobase/include/page0cur.h +++ b/storage/innobase/include/page0cur.h @@ -244,6 +244,21 @@ bool page_cur_search_with_match_bytes(const dtuple_t &tuple, uint16_t *ilow_bytes) noexcept; +/** Compare a data tuple to a physical record. +@param rec B-tree index record +@param index index B-tree +@param tuple search key +@param match matched fields << 16 | bytes +@param comp nonzero if ROW_FORMAT=REDUNDANT is not being used +@return the comparison result of dtuple and rec +@retval 0 if dtuple is equal to rec +@retval negative if dtuple is less than rec +@retval positive if dtuple is greater than rec */ +int cmp_dtuple_rec_bytes(const rec_t *rec, + const dict_index_t &index, + const dtuple_t &tuple, int *match, ulint comp) + noexcept; + /***********************************************************//** Positions a page cursor on a randomly chosen user record on a page. If there are no user records, sets the cursor on the infimum record. */ diff --git a/storage/innobase/include/row0query.h b/storage/innobase/include/row0query.h new file mode 100644 index 0000000000000..57a45662eeeac --- /dev/null +++ b/storage/innobase/include/row0query.h @@ -0,0 +1,236 @@ +/***************************************************************************** + +Copyright (c) 2025, MariaDB PLC. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/row0query.h +General Query Executor + +Created 2025/10/30 +*******************************************************/ + +#pragma once + +#include "btr0pcur.h" +#include +#include "dict0types.h" +#include "data0types.h" +#include "db0err.h" +#include "lock0types.h" +#include "rem0rec.h" + +/** Comparator action for deciding how to treat a record */ +enum class RecordCompareAction +{ + /** Do not process this record, continue traversal */ + SKIP, + /** Process this record via process_record */ + PROCESS, + /** Stop traversal immediately */ + STOP +}; + +using RecordProcessor= std::function; + +using RecordComparator= std::function< + RecordCompareAction(const dtuple_t*, const rec_t*, + const dict_index_t*)>; + +/** Record processing callback interface using std::function. +Can be used by FTS, stats infrastructure, and other components +that need to process database records with custom logic. */ +class RecordCallback +{ +public: + /** Constructor with processor function and optional comparator + @param[in] processor Function to process each record + @param[in] comparator Optional function to filter records (default: accept all) */ + RecordCallback( + RecordProcessor processor, + RecordComparator comparator= [](const dtuple_t*, const rec_t*, + const dict_index_t*) + { return RecordCompareAction::PROCESS; }) + : process_record(std::move(processor)), + compare_record(std::move(comparator)) {} + + virtual ~RecordCallback() = default; + + /** Called for each matching record */ + RecordProcessor process_record; + + /** Comparison function for custom filtering */ + RecordComparator compare_record; +}; + +/** General-purpose MVCC-aware record traversal and basic +DML executor. Provides a thin abstraction over B-tree cursors for +reading and mutating records with consistent-read (MVCC) handling, +and callback API. +- Open and iterate clustered/secondary indexes with page cursors. +- Build consistent-read versions when needed via transaction +read views. +- Filter and process records using RecordCallback: + - compare_record: decide SKIP/PROCESS/STOP for each record + - process_record: handle visible records; return DB_SUCCESS to continue, DB_SUCCESS_LOCKED_REC to stop +- Basic DML helpers (insert/delete/replace) and table locking. */ +class QueryExecutor +{ +private: + que_thr_t *m_thr; + btr_pcur_t m_pcur; + btr_pcur_t *m_clust_pcur; + mtr_t m_mtr; + mem_heap_t *m_version_heap; + mem_heap_t *m_offsets_heap; + mem_heap_t *m_heap; + + /** Lookup clustered index record from secondary index record + @param sec_index secondary index + @param sec_rec secondary index record + @param sec_offsets secondary index offsets + @param callback callback to process the clustered record + @param match_count counter for processed records + @return DB_SUCCESS to continue, DB_SUCCESS_LOCKED_REC to stop, or error code */ + dberr_t lookup_clustered_record(dict_index_t *sec_index, + const rec_t *sec_rec, + RecordCallback &callback, + ulint &match_count) noexcept; + + /** Process a record with MVCC visibility checking and + version building + @param clust_index clustered index of the record + @param rec record to process + @param callback callback to process the record + @return DB_SUCCESS to continue, DB_SUCCESS_LOCKED_REC to stop, or error code */ + dberr_t process_record_with_mvcc( + dict_index_t *clust_index, const rec_t *rec, + RecordCallback &callback) noexcept; +public: + QueryExecutor(trx_t *trx); + ~QueryExecutor(); + + /** Insert a record in clustered index of the table + @param table table to be inserted + @param tuple tuple to be inserted + @return DB_SUCCESS on success, error code on failure */ + dberr_t insert_record(dict_table_t *table, dtuple_t *tuple) noexcept; + + /** Delete a record from the clustered index of the table + @param table table to be inserted + @param tuple tuple to be inserted + @return DB_SUCCESS on success, error code on failure */ + dberr_t delete_record(dict_table_t *table, dtuple_t *tuple) noexcept; + + /** Delete all records from the clustered index of the table + @param table table be be deleted + @return DB_SUCCESS on success, error code on failure */ + dberr_t delete_all(dict_table_t *table) noexcept; + + /** Acquire and lock a single clustered record for update + Performs a keyed lookup on the clustered index, validates MVCC visibility, + and acquires an X lock on the matching record. + @param[in] table Table containing the record + @param[in] search_tuple Exact key for clustered index lookup + @param[in] callback Optional record callback + @return DB_SUCCESS on successful lock + DB_RECORD_NOT_FOUND if no visible matching record + DB_LOCK_WAIT if waiting was required + error code on failure */ + dberr_t select_for_update(dict_table_t *table, dtuple_t *search_tuple, + RecordCallback *callback= nullptr) noexcept; + + /** Update the currently selected clustered record within an active mtr. + Attempts in-place update; falls back to optimistic/pessimistic update if needed, + including external field storage when required. + select_for_update() has positioned and locked m_pcur on the target row. + @param[in] table target table + @param[in] update update descriptor (fields, new values) + @return DB_SUCCESS on success + DB_OVERFLOW/DB_UNDERFLOW during size-changing paths + error_code on failures */ + dberr_t update_record(dict_table_t *table, const upd_t *update) noexcept; + + + /** Try to update a record by key or insert if not found. + Performs a SELECT ... FOR UPDATE using search_tuple; + if found, updates the row; otherwise inserts a new record. + Note: + On update path, commits or rolls back the active mtr as needed. + On insert path, no active mtr remains upon return + @param[in] table target table + @param[in] search_tuple key identifying the target row + @param[in] update update descriptor (applied when found) + @param[in] insert_tuple tuple to insert when not found + @return DB_SUCCESS on successful update or insert + @retval DB_LOCK_WAIT to be retried, + @return error code on failure */ + dberr_t replace_record(dict_table_t *table, dtuple_t *search_tuple, + const upd_t *update, dtuple_t *insert_tuple) noexcept; + + /** Iterate clustered index records and process via callback. + Handles full table scan and index scan for range/select queries + Calls callback.compare_record() to decide SKIP/PROCESS/STOP for + each matching record. On PROCESS, invokes + callback.process_record() on an MVCC-visible version. + @param table table to read + @param tuple optional search key (range/point). nullptr => full scan + @param mode B-tree search mode (e.g., PAGE_CUR_GE) + @param callback record comparator/processor + @return DB_SUCCESS if at least one record was processed + @retval DB_RECORD_NOT_FOUND if no record matched + @return error code on failure */ + dberr_t read(dict_table_t *table, const dtuple_t *tuple, + page_cur_mode_t mode, + RecordCallback& callback) noexcept; + + /** Read records via a secondary index and process corresponding + clustered rows. Performs a range or point scan on the given secondary index, + filters secondary records with callback.compare_record(), then looks up + the matching clustered record and invokes callback.process_record() + on a MVCC-visible version. + + @param table Table to read + @param sec_index Secondary index used for traversal + @param search_tuple search key or nullptr for full scan + @param mode Cursor search mode + @param callback RecordCallback with comparator+processor + @return DB_SUCCESS on success + DB_RECORD_NOT_FOUND if no matching record was processed + error code on failure */ + __attribute__((nonnull)) + dberr_t read_by_index(dict_table_t *table, dict_index_t *sec_index, + const dtuple_t *search_tuple, + page_cur_mode_t mode, + RecordCallback &callback) noexcept; + + /** Acquire a table lock in the given mode for transaction. + @param table table to lock + @param mode lock mode + @return DB_SUCCESS, DB_LOCK_WAIT or error code */ + dberr_t lock_table(dict_table_t *table, lock_mode mode) noexcept; + + /** Handle a lock wait for the current transaction and thread context. + @param err the lock-related error to handle (e.g., DB_LOCK_WAIT) + @param table_lock true if the wait originated from table lock, else row lock + @return DB_SUCCESS if the wait completed successfully and lock was granted + @retval DB_LOCK_WAIT_TIMEOUT if timed out */ + dberr_t handle_wait(dberr_t err, bool table_lock) noexcept; + mem_heap_t *get_heap() const { return m_heap; } + trx_t *get_trx() const { return m_mtr.trx; } + void commit_mtr() noexcept { m_mtr.commit(); } +}; diff --git a/storage/innobase/include/row0sel.h b/storage/innobase/include/row0sel.h index 35e3cbe66315c..301ce4314541b 100644 --- a/storage/innobase/include/row0sel.h +++ b/storage/innobase/include/row0sel.h @@ -454,3 +454,48 @@ row_sel_field_store_in_mysql_format_func( #endif /* UNIV_DEBUG */ const byte* data, /*!< in: data to store */ ulint len); /*!< in: length of the data */ + +/** Helper class to cache clust_rec and old_vers */ +class Row_sel_get_clust_rec_for_mysql +{ + const rec_t *cached_clust_rec; + rec_t *cached_old_vers; + lsn_t cached_lsn; + page_id_t cached_page_id; + +#ifdef UNIV_DEBUG + void check_eq(const dict_index_t *index, const rec_offs *offsets) const + { + rec_offs vers_offs[REC_OFFS_HEADER_SIZE + MAX_REF_PARTS]; + rec_offs_init(vers_offs); + mem_heap_t *heap= nullptr; + + ut_ad(rec_offs_validate(cached_clust_rec, index, offsets)); + ut_ad(index->first_user_field() <= rec_offs_n_fields(offsets)); + ut_ad(vers_offs == rec_get_offsets(cached_old_vers, index, vers_offs, + index->n_core_fields, + index->db_trx_id(), &heap)); + ut_ad(!heap); + for (auto n= index->db_trx_id(); n--; ) + { + const dict_col_t *col= dict_index_get_nth_col(index, n); + ulint len1, len2; + const byte *b1= rec_get_nth_field(cached_clust_rec, offsets, n, &len1); + const byte *b2= rec_get_nth_field(cached_old_vers, vers_offs, n, &len2); + ut_ad(!cmp_data(col->mtype, col->prtype, false, b1, len1, b2, len2)); + } + } +#endif + +public: + Row_sel_get_clust_rec_for_mysql() : + cached_clust_rec(NULL), cached_old_vers(NULL), cached_lsn(0), + cached_page_id(page_id_t(0,0)) {} + + dberr_t operator()(dtuple_t *clust_ref, btr_pcur_t *clust_pcur, + lock_mode select_lock_type, trx_t *trx, + mem_heap_t *old_vers_heap, dict_index_t *sec_index, + const rec_t *rec, que_thr_t *thr, const rec_t **out_rec, + rec_offs **offsets, mem_heap_t **offset_heap, + dtuple_t **vrow, mtr_t *mtr); +}; diff --git a/storage/innobase/include/ut0new.h b/storage/innobase/include/ut0new.h index bcc129601d11f..a974a093da1af 100644 --- a/storage/innobase/include/ut0new.h +++ b/storage/innobase/include/ut0new.h @@ -854,6 +854,7 @@ constexpr const char* const auto_event_names[] = "fts0ast", "fts0blex", "fts0config", + "fts0exec", "fts0file", "fts0fts", "fts0opt", diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index d1d11b50de8a1..15319bf5b8866 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -78,9 +78,9 @@ static ulint cmp_get_pad_char(const dtype_t &type) noexcept @retval 0 if dtuple is equal to rec @retval negative if dtuple is less than rec @retval positive if dtuple is greater than rec */ -static int cmp_dtuple_rec_bytes(const rec_t *rec, - const dict_index_t &index, - const dtuple_t &tuple, int *match, ulint comp) +int cmp_dtuple_rec_bytes(const rec_t *rec, + const dict_index_t &index, + const dtuple_t &tuple, int *match, ulint comp) noexcept { ut_ad(dtuple_check_typed(&tuple)); diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index b36731a02c21e..0bf8340d09a7c 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -42,6 +42,7 @@ Completed by Sunny Bains and Marko Makela #include "pars0pars.h" #include "ut0sort.h" #include "row0ftsort.h" +#include "fts0exec.h" #include "row0import.h" #include "row0vers.h" #include "handler0alter.h" @@ -3035,10 +3036,27 @@ row_merge_read_clustered_index( new_table->fts->cache->first_doc_id = new_table->fts->cache->next_doc_id; + trx_t *fts_trx = trx_create(); + trx_start_internal(fts_trx); + fts_trx->op_info= "setting last FTS document id"; + FTSQueryExecutor executor( + fts_trx, new_table); err= fts_update_sync_doc_id( + &executor, new_table, - new_table->fts->cache->synced_doc_id, - NULL); + new_table->fts->cache->synced_doc_id); + if (err == DB_SUCCESS) { + trx_commit_for_mysql(fts_trx); + new_table->fts->cache->synced_doc_id++; + } else { + sql_print_error( + "InnoDB: ( %s ) while updating " + "last doc id for table %s", + ut_strerr(err), + new_table->name.m_name); + fts_trx->rollback(); + } + fts_trx->free(); } } diff --git a/storage/innobase/row/row0query.cc b/storage/innobase/row/row0query.cc new file mode 100644 index 0000000000000..eda09cbc329d8 --- /dev/null +++ b/storage/innobase/row/row0query.cc @@ -0,0 +1,667 @@ +/***************************************************************************** + +Copyright (c) 2025, MariaDB PLC. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file row/row0query.cc +General Query Executor + +Created 2025/10/30 +*******************************************************/ + +#include "row0query.h" +#include "pars0pars.h" +#include "dict0dict.h" +#include "row0ins.h" +#include "row0upd.h" +#include "row0row.h" +#include "row0vers.h" +#include "row0sel.h" +#include "mem0mem.h" +#include "que0que.h" +#include "lock0lock.h" +#include "rem0rec.h" +#include "btr0pcur.h" +#include "btr0cur.h" + +/** Extract transaction ID from a clustered index record +@param[in] rec record to extract transaction ID from +@param[in] clust_index clustered index +@return transaction ID */ +static trx_id_t get_record_trx_id(const rec_t* rec, const dict_index_t* clust_index) +{ + if (clust_index->trx_id_offset) + return trx_read_trx_id(rec + clust_index->trx_id_offset); + /* Calculate offset to DB_TRX_ID field by iterating through fields */ + ulint trx_id_field_no= clust_index->db_trx_id(); + ulint trx_id_offset= 0; + + if (dict_table_is_comp(clust_index->table)) + { + const byte* lens= + rec - REC_N_NEW_EXTRA_BYTES - 1 - clust_index->n_core_null_bytes; + for (ulint i= 0; i < trx_id_field_no; i++) + { + const dict_field_t* field= dict_index_get_nth_field(clust_index, i); + ulint field_len; + + if (field->fixed_len) + field_len= field->fixed_len; + else + { + ulint len_byte= *lens--; + if (UNIV_UNLIKELY(len_byte & 0x80) && DATA_BIG_COL(field->col)) + { + len_byte<<= 8; + len_byte|= *lens--; + field_len= (len_byte & 0x3fff); + } + else + field_len= len_byte; + + if (field_len == UNIV_SQL_NULL) + field_len= 0; + } + trx_id_offset+= field_len; + } + return trx_read_trx_id(rec + trx_id_offset); + } + else + { + for (ulint i= 0; i < trx_id_field_no; i++) + { + ulint field_len; + ulint field_offset= rec_get_nth_field_offs_old(rec, i, &field_len); + if (i == 0) + trx_id_offset= field_offset; + if (field_len != UNIV_SQL_NULL) + trx_id_offset= field_offset + field_len; + else + trx_id_offset= field_offset; + } + return trx_read_trx_id(rec + trx_id_offset); + } +} + +QueryExecutor::QueryExecutor(trx_t *trx) : m_mtr(trx) +{ + m_heap= mem_heap_create(256); + m_thr= pars_complete_graph_for_exec(nullptr, trx, m_heap, nullptr); + btr_pcur_init(&m_pcur); + m_clust_pcur= nullptr; + m_version_heap= nullptr; + m_offsets_heap= nullptr; +} + +QueryExecutor::~QueryExecutor() +{ + btr_pcur_close(&m_pcur); + if (m_clust_pcur) + { + btr_pcur_close(m_clust_pcur); + delete m_clust_pcur; + m_clust_pcur= nullptr; + } + if (m_heap) mem_heap_free(m_heap); + if (m_version_heap) mem_heap_free(m_version_heap); +} + +dberr_t QueryExecutor::insert_record(dict_table_t *table, + dtuple_t *tuple) noexcept +{ + dict_index_t* index= dict_table_get_first_index(table); + return row_ins_clust_index_entry(index, tuple, m_thr, 0); +} + +dberr_t QueryExecutor::lock_table(dict_table_t *table, lock_mode mode) noexcept +{ + ut_ad(m_mtr.trx); + trx_start_if_not_started(m_mtr.trx, true); + return ::lock_table(table, nullptr, mode, m_thr); +} + +dberr_t QueryExecutor::handle_wait(dberr_t err, bool table_lock) noexcept +{ + ut_ad(m_mtr.trx); + m_mtr.trx->error_state= err; + if (table_lock) m_thr->lock_state= QUE_THR_LOCK_TABLE; + else m_thr->lock_state= QUE_THR_LOCK_ROW; + if (m_mtr.trx->lock.wait_thr) + { + dberr_t wait_err= lock_wait(m_thr); + if (wait_err == DB_LOCK_WAIT_TIMEOUT) err= wait_err; + if (wait_err == DB_SUCCESS) + { + m_thr->lock_state= QUE_THR_LOCK_NOLOCK; + return DB_SUCCESS; + } + } + return err; +} + +dberr_t QueryExecutor::delete_record(dict_table_t *table, + dtuple_t *tuple) noexcept +{ + dict_index_t *clust_index= dict_table_get_first_index(table); + dberr_t err= DB_SUCCESS; + ulint deleted_count= 0; + +retry: + m_mtr.start(); + m_mtr.set_named_space(table->space); + + m_pcur.btr_cur.page_cur.index= clust_index; + err= btr_pcur_open_on_user_rec(tuple, BTR_MODIFY_LEAF, &m_pcur, &m_mtr); + if (err != DB_SUCCESS) + goto func_exit; + while (btr_pcur_is_on_user_rec(&m_pcur)) + { + rec_t *rec= btr_pcur_get_rec(&m_pcur); + rec_offs *offsets= nullptr; + uint16_t matched_fields= 0; + int cmp= 0; + if (rec_get_deleted_flag(rec, dict_table_is_comp(table))) + goto next_rec; + + offsets= rec_get_offsets(rec, clust_index, offsets, + clust_index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + + cmp= cmp_dtuple_rec_with_match(tuple, rec, clust_index, + offsets, &matched_fields); + if (cmp != 0) + break; + err= lock_clust_rec_read_check_and_lock( + 0, btr_pcur_get_block(&m_pcur), rec, clust_index, offsets, LOCK_X, + LOCK_REC_NOT_GAP, m_thr); + if (err == DB_LOCK_WAIT) + { + m_mtr.commit(); + err= handle_wait(err, false); + if (err != DB_SUCCESS) + return err; + goto retry; + } + else if (err != DB_SUCCESS && err != DB_SUCCESS_LOCKED_REC) + goto func_exit; + + err= btr_cur_del_mark_set_clust_rec(btr_pcur_get_block(&m_pcur), + rec, clust_index, offsets, m_thr, + nullptr, &m_mtr); + if (err != DB_SUCCESS) + goto func_exit; + deleted_count++; +next_rec: + if (!btr_pcur_move_to_next(&m_pcur, &m_mtr)) + break; + } +func_exit: + m_mtr.commit(); + if (err) + return err; + return (deleted_count > 0) ? DB_SUCCESS : DB_RECORD_NOT_FOUND; +} + +dberr_t QueryExecutor::delete_all(dict_table_t *table) noexcept +{ + dict_index_t *clust_index= dict_table_get_first_index(table); + dberr_t err= DB_SUCCESS; +retry: + m_mtr.start(); + m_mtr.set_named_space(table->space); + + err= m_pcur.open_leaf(true, clust_index, BTR_MODIFY_LEAF, &m_mtr); + if (err != DB_SUCCESS || !btr_pcur_move_to_next(&m_pcur, &m_mtr)) + { + m_mtr.commit(); + return err; + } + + while (!btr_pcur_is_after_last_on_page(&m_pcur) && + !btr_pcur_is_after_last_in_tree(&m_pcur)) + { + rec_t* rec= btr_pcur_get_rec(&m_pcur); + rec_offs *offsets= nullptr; + if (rec_get_deleted_flag(rec, dict_table_is_comp(table))) + goto next_rec; + if (rec_get_info_bits( + rec, dict_table_is_comp(table)) & REC_INFO_MIN_REC_FLAG) + goto next_rec; + + offsets= rec_get_offsets(rec, clust_index, nullptr, + clust_index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + err= lock_clust_rec_read_check_and_lock( + 0, btr_pcur_get_block(&m_pcur), rec, clust_index, offsets, LOCK_X, + LOCK_REC_NOT_GAP, m_thr); + + if (err == DB_LOCK_WAIT) + { + m_mtr.commit(); + err= handle_wait(err, false); + if (err != DB_SUCCESS) + return err; + goto retry; + } + else if (err != DB_SUCCESS && err != DB_SUCCESS_LOCKED_REC) + { + m_mtr.commit(); + return err; + } + + err= btr_cur_del_mark_set_clust_rec(btr_pcur_get_block(&m_pcur), + const_cast(rec), clust_index, + offsets, m_thr, nullptr, &m_mtr); + if (err) + break; +next_rec: + if (!btr_pcur_move_to_next(&m_pcur, &m_mtr)) + break; + } + + m_mtr.commit(); + return err; +} + +dberr_t QueryExecutor::select_for_update(dict_table_t *table, + dtuple_t *search_tuple, + RecordCallback *callback) noexcept +{ + ut_ad(m_mtr.trx); + dict_index_t *clust_index= dict_table_get_first_index(table); + dberr_t err= DB_SUCCESS; + rec_t *rec; + rec_offs *offsets; + uint16_t matched_fields= 0; + int cmp= 0; + trx_t *trx= m_mtr.trx; + m_mtr.start(); + m_mtr.set_named_space(table->space); + + if (trx && !trx->read_view.is_open()) + { + trx_start_if_not_started(trx, false); + trx->read_view.open(trx); + } + m_pcur.btr_cur.page_cur.index= clust_index; + err= btr_pcur_open_on_user_rec(search_tuple, BTR_MODIFY_LEAF, + &m_pcur, &m_mtr); + if (err != DB_SUCCESS) + goto err_exit; + + if (!btr_pcur_is_on_user_rec(&m_pcur)) + { + err= DB_RECORD_NOT_FOUND; + goto err_exit; + } + rec= btr_pcur_get_rec(&m_pcur); + offsets= rec_get_offsets(rec, clust_index, nullptr, + clust_index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + + if (trx && trx->read_view.is_open()) + { + trx_id_t rec_trx_id= row_get_rec_trx_id(rec, clust_index, offsets); + if (rec_trx_id && !trx->read_view.changes_visible(rec_trx_id)) + { + err= DB_RECORD_NOT_FOUND; + goto err_exit; + } + } + cmp= cmp_dtuple_rec_with_match(search_tuple, rec, clust_index, + offsets, &matched_fields); + if (cmp != 0) + { + err= DB_RECORD_NOT_FOUND; + goto err_exit; + } + + err= lock_clust_rec_read_check_and_lock( + 0, btr_pcur_get_block(&m_pcur), rec, clust_index, offsets, LOCK_X, + LOCK_REC_NOT_GAP, m_thr); + + if (err == DB_LOCK_WAIT) + { + m_mtr.commit(); + err= handle_wait(err, false); + if (err != DB_SUCCESS) + return err; + return DB_LOCK_WAIT; + } + else if (err != DB_SUCCESS && err != DB_SUCCESS_LOCKED_REC) + { +err_exit: + m_mtr.commit(); + return err; + } + + if (callback) + { + RecordCompareAction action= + callback->compare_record(search_tuple, rec, clust_index); + if (action == RecordCompareAction::PROCESS) + { + dberr_t proc_err= callback->process_record(rec, clust_index, offsets); + if (proc_err != DB_SUCCESS) + { + err= proc_err; + goto err_exit; + } + } + if (action == RecordCompareAction::SKIP) + { + err= DB_RECORD_NOT_FOUND; + goto err_exit; + } + } + return DB_SUCCESS; +} + +dberr_t QueryExecutor::update_record(dict_table_t *table, + const upd_t *update) noexcept +{ + ut_ad(m_mtr.trx); + dict_index_t *clust_index= dict_table_get_first_index(table); + rec_t *rec= btr_pcur_get_rec(&m_pcur); + mtr_x_lock_index(clust_index, &m_mtr); + rec_offs *offsets= rec_get_offsets(rec, clust_index, nullptr, + clust_index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + + dberr_t err= DB_SUCCESS; + ulint cmpl_info= UPD_NODE_NO_ORD_CHANGE | UPD_NODE_NO_SIZE_CHANGE; + for (ulint i = 0; i < update->n_fields; i++) + { + const upd_field_t *upd_field= &update->fields[i]; + ulint field_no= upd_field->field_no; + if (field_no < rec_offs_n_fields(offsets)) + { + ulint old_len= rec_offs_nth_size(offsets, field_no); + ulint new_len= upd_field->new_val.len; + if (new_len != UNIV_SQL_NULL && new_len != old_len) + { + cmpl_info &= ~UPD_NODE_NO_SIZE_CHANGE; + err= DB_OVERFLOW; + break; + } + } + } + + if (cmpl_info & UPD_NODE_NO_SIZE_CHANGE) + err= btr_cur_update_in_place(BTR_NO_LOCKING_FLAG, + btr_pcur_get_btr_cur(&m_pcur), + offsets, const_cast(update), 0, + m_thr, m_mtr.trx->id, &m_mtr); + if (err == DB_OVERFLOW) + { + big_rec_t *big_rec= nullptr; + err= btr_cur_optimistic_update(BTR_NO_LOCKING_FLAG, + btr_pcur_get_btr_cur(&m_pcur), + &offsets, &m_heap, + const_cast(update), + cmpl_info, m_thr, m_mtr.trx->id, &m_mtr); + + if (err == DB_OVERFLOW || err == DB_UNDERFLOW) + { + mem_heap_t* offsets_heap= nullptr; + err= btr_cur_pessimistic_update(BTR_NO_LOCKING_FLAG, + btr_pcur_get_btr_cur(&m_pcur), + &offsets, &offsets_heap, m_heap, + &big_rec, const_cast(update), + cmpl_info, m_thr, m_mtr.trx->id, &m_mtr); + + if (err == DB_SUCCESS && big_rec) + { + err= btr_store_big_rec_extern_fields(&m_pcur, offsets, big_rec, &m_mtr, + BTR_STORE_UPDATE); + dtuple_big_rec_free(big_rec); + } + if (offsets_heap) mem_heap_free(offsets_heap); + } + } + return err; +} + +dberr_t QueryExecutor::replace_record( + dict_table_t *table, dtuple_t *search_tuple, + const upd_t *update, dtuple_t *insert_tuple) noexcept +{ +retry_again: + dberr_t err= select_for_update(table, search_tuple); + if (err == DB_SUCCESS) + { + err= update_record(table, update); + m_mtr.commit(); + return err; + } + else if (err == DB_RECORD_NOT_FOUND) + { + err= insert_record(table, insert_tuple); + return err; + } + else if (err == DB_LOCK_WAIT) + goto retry_again; + return err; +} + +dberr_t QueryExecutor::read(dict_table_t *table, const dtuple_t *tuple, + page_cur_mode_t mode, + RecordCallback& callback) noexcept +{ + ut_ad(table); + dict_index_t *clust_index= dict_table_get_first_index(table); + + m_mtr.start(); + ut_ad(m_mtr.trx); + if (!m_mtr.trx->read_view.is_open()) + { + trx_start_if_not_started(m_mtr.trx, false); + m_mtr.trx->read_view.open(m_mtr.trx); + } + m_pcur.btr_cur.page_cur.index= clust_index; + dberr_t err= DB_SUCCESS; + if (tuple) + { + err= btr_pcur_open_on_user_rec(tuple, BTR_SEARCH_LEAF, &m_pcur, &m_mtr); + if (err != DB_SUCCESS) + { + m_mtr.commit(); + return err; + } + } + else + { + err= m_pcur.open_leaf(true, clust_index, BTR_SEARCH_LEAF, &m_mtr); + if (err != DB_SUCCESS || !btr_pcur_move_to_next(&m_pcur, &m_mtr)) + { + m_mtr.commit(); + return err; + } + } + ulint match_count= 0; + while (btr_pcur_is_on_user_rec(&m_pcur)) + { + const rec_t *rec= btr_pcur_get_rec(&m_pcur); + RecordCompareAction action= callback.compare_record( + tuple, rec, clust_index); + if (action == RecordCompareAction::PROCESS) + { + dberr_t err= process_record_with_mvcc(clust_index, rec, callback); + if (err == DB_SUCCESS_LOCKED_REC) + { + err= DB_SUCCESS; + match_count++; + goto func_exit; + } + if (err != DB_SUCCESS) + { + m_mtr.commit(); + return err; + } + match_count++; + } + else if (action == RecordCompareAction::STOP) + break; + if (!btr_pcur_move_to_next(&m_pcur, &m_mtr)) + break; + } +func_exit: + m_mtr.commit(); + return (match_count > 0 || !tuple) ? DB_SUCCESS : DB_RECORD_NOT_FOUND; +} + +dberr_t QueryExecutor::read_by_index(dict_table_t *table, + dict_index_t *sec_index, + const dtuple_t *search_tuple, + page_cur_mode_t mode, + RecordCallback& callback) noexcept +{ + ut_ad(table); + ut_ad(sec_index); + ut_ad(sec_index->table == table); + ut_ad(!dict_index_is_clust(sec_index)); + + m_mtr.start(); + if (m_mtr.trx && !m_mtr.trx->read_view.is_open()) + { + trx_start_if_not_started(m_mtr.trx, false); + m_mtr.trx->read_view.open(m_mtr.trx); + } + m_pcur.btr_cur.page_cur.index= sec_index; + + ulint match_count= 0; + dberr_t err= btr_pcur_open_on_user_rec( + search_tuple, BTR_SEARCH_LEAF, &m_pcur, &m_mtr); + + if (err != DB_SUCCESS) + goto func_exit; + + while (btr_pcur_is_on_user_rec(&m_pcur)) + { + const rec_t *sec_rec= btr_pcur_get_rec(&m_pcur); + RecordCompareAction action= callback.compare_record(search_tuple, sec_rec, + sec_index); + if (action == RecordCompareAction::PROCESS) + { + /* Lookup clustered record and process it */ + err= lookup_clustered_record( + sec_index, sec_rec, callback, match_count); + if (err != DB_SUCCESS) + goto func_exit; + } + else if (action == RecordCompareAction::STOP) + break; + if (!btr_pcur_move_to_next(&m_pcur, &m_mtr)) break; + } + err= match_count > 0 ? DB_SUCCESS : DB_RECORD_NOT_FOUND; + +func_exit: + m_mtr.commit(); + return err == DB_SUCCESS_LOCKED_REC ? DB_SUCCESS : err; +} + +dberr_t QueryExecutor::lookup_clustered_record(dict_index_t *sec_index, + const rec_t *sec_rec, + RecordCallback& callback, + ulint& match_count) noexcept +{ + ut_ad(sec_index->is_normal_btree()); + ut_ad(!strcmp(sec_index->name(), FTS_DOC_ID_INDEX.str)); + dict_index_t *clust_index= dict_table_get_first_index(sec_index->table); + + /* Build clustered index search tuple from secondary record */ + dtuple_t *clust_ref= row_build_row_ref(ROW_COPY_POINTERS, sec_index, + sec_rec, m_heap); + if (!m_clust_pcur) + { + m_clust_pcur= new btr_pcur_t; + btr_pcur_init(m_clust_pcur); + } + + if (!m_offsets_heap) + m_offsets_heap= mem_heap_create(256); + /* Use Row_sel_get_clust_rec_for_mysql to get clustered record */ + Row_sel_get_clust_rec_for_mysql row_sel_get_clust_rec_for_mysql; + const rec_t *clust_rec= nullptr; + rec_offs *clust_offsets= nullptr; + + ulint savepoint= m_mtr.get_savepoint(); + + dberr_t err= + row_sel_get_clust_rec_for_mysql(clust_ref, m_clust_pcur, LOCK_NONE, + m_mtr.trx, m_version_heap, + sec_index, sec_rec, m_thr, + &clust_rec, &clust_offsets, + &m_offsets_heap, nullptr, &m_mtr); + + if (err == DB_SUCCESS && clust_rec != nullptr) + { + /* Process the clustered record with MVCC handling */ + err= callback.process_record(clust_rec, clust_index, clust_offsets); + if (err == DB_SUCCESS || err == DB_SUCCESS_LOCKED_REC) + match_count++; + } + + /* Clean up offset heap */ + if (m_offsets_heap) + mem_heap_empty(m_offsets_heap); + + m_mtr.rollback_to_savepoint(savepoint, savepoint + 1); + return err; +} + +dberr_t QueryExecutor::process_record_with_mvcc( + dict_index_t *clust_index, const rec_t *rec, + RecordCallback &callback) noexcept +{ + ut_ad(m_mtr.trx); + ut_ad(srv_read_only_mode || m_mtr.trx->read_view.is_open()); + + dberr_t error= DB_SUCCESS; + trx_id_t rec_trx_id= get_record_trx_id(rec, clust_index); + rec_offs* offsets= nullptr; + rec_offs* version_offsets= nullptr; + + rec_t *result_rec= const_cast(rec); + if (rec_trx_id && !m_mtr.trx->read_view.changes_visible(rec_trx_id)) + { + /* Compute clustered index offsets */ + if (!m_version_heap) + m_version_heap= mem_heap_create(128); + offsets= rec_get_offsets(rec, clust_index, nullptr, + clust_index->n_core_fields, + ULINT_UNDEFINED, &m_version_heap); + error= row_vers_build_for_consistent_read( + rec, &m_mtr, clust_index, &offsets, &m_mtr.trx->read_view, + &m_version_heap, m_version_heap, &result_rec, nullptr); + if (error == DB_SUCCESS && result_rec) + version_offsets= rec_get_offsets(result_rec, clust_index, nullptr, + clust_index->n_core_fields, + ULINT_UNDEFINED, &m_version_heap); + } + + if (error != DB_SUCCESS || !result_rec) + goto func_exit; + + if (rec_get_deleted_flag(result_rec, clust_index->table->not_redundant())) + goto func_exit; + + error= callback.process_record(result_rec, clust_index, version_offsets); +func_exit: + if (m_version_heap) mem_heap_empty(m_version_heap); + return error; +} diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index c063c3b34b810..d5c1a89a125cf 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -3260,12 +3260,12 @@ static bool row_sel_store_mysql_rec( DBUG_RETURN(true); } -static void row_sel_reset_old_vers_heap(row_prebuilt_t *prebuilt) +static void row_sel_reset_old_vers_heap(mem_heap_t** old_vers_heap) { - if (prebuilt->old_vers_heap) - mem_heap_empty(prebuilt->old_vers_heap); + if (*old_vers_heap) + mem_heap_empty(*old_vers_heap); else - prebuilt->old_vers_heap= mem_heap_create(200); + *old_vers_heap= mem_heap_create(200); } /*********************************************************************//** @@ -3275,13 +3275,14 @@ static MY_ATTRIBUTE((warn_unused_result)) dberr_t row_sel_build_prev_vers_for_mysql( /*==============================*/ - row_prebuilt_t* prebuilt, /*!< in/out: prebuilt struct */ + trx_t* trx, dict_index_t* clust_index, /*!< in: clustered index */ const rec_t* rec, /*!< in: record in a clustered index */ rec_offs** offsets, /*!< in/out: offsets returned by rec_get_offsets(rec, clust_index) */ mem_heap_t** offset_heap, /*!< in/out: memory heap from which the offsets are allocated */ + mem_heap_t* old_vers_heap, rec_t** old_vers, /*!< out: old version, or NULL if the record does not exist in the view: i.e., it was freshly inserted @@ -3290,57 +3291,14 @@ row_sel_build_prev_vers_for_mysql( column data */ mtr_t* mtr) /*!< in: mtr */ { - row_sel_reset_old_vers_heap(prebuilt); + row_sel_reset_old_vers_heap(&old_vers_heap); return row_vers_build_for_consistent_read( rec, mtr, clust_index, offsets, - &prebuilt->trx->read_view, offset_heap, - prebuilt->old_vers_heap, old_vers, vrow); + &trx->read_view, offset_heap, + old_vers_heap, old_vers, vrow); } -/** Helper class to cache clust_rec and old_vers */ -class Row_sel_get_clust_rec_for_mysql -{ - const rec_t *cached_clust_rec; - rec_t *cached_old_vers; - lsn_t cached_lsn; - page_id_t cached_page_id; - -#ifdef UNIV_DEBUG - void check_eq(const dict_index_t *index, const rec_offs *offsets) const - { - rec_offs vers_offs[REC_OFFS_HEADER_SIZE + MAX_REF_PARTS]; - rec_offs_init(vers_offs); - mem_heap_t *heap= nullptr; - - ut_ad(rec_offs_validate(cached_clust_rec, index, offsets)); - ut_ad(index->first_user_field() <= rec_offs_n_fields(offsets)); - ut_ad(vers_offs == rec_get_offsets(cached_old_vers, index, vers_offs, - index->n_core_fields, - index->db_trx_id(), &heap)); - ut_ad(!heap); - for (auto n= index->db_trx_id(); n--; ) - { - const dict_col_t *col= dict_index_get_nth_col(index, n); - ulint len1, len2; - const byte *b1= rec_get_nth_field(cached_clust_rec, offsets, n, &len1); - const byte *b2= rec_get_nth_field(cached_old_vers, vers_offs, n, &len2); - ut_ad(!cmp_data(col->mtype, col->prtype, false, b1, len1, b2, len2)); - } - } -#endif - -public: - Row_sel_get_clust_rec_for_mysql() : - cached_clust_rec(NULL), cached_old_vers(NULL), cached_lsn(0), - cached_page_id(page_id_t(0,0)) {} - - dberr_t operator()(row_prebuilt_t *prebuilt, dict_index_t *sec_index, - const rec_t *rec, que_thr_t *thr, const rec_t **out_rec, - rec_offs **offsets, mem_heap_t **offset_heap, - dtuple_t **vrow, mtr_t *mtr); -}; - /*********************************************************************//** Retrieves the clustered index record corresponding to a record in a non-clustered index. Does the necessary locking. Used in the MySQL @@ -3349,7 +3307,11 @@ interface. dberr_t Row_sel_get_clust_rec_for_mysql::operator()( /*============================*/ - row_prebuilt_t* prebuilt,/*!< in: prebuilt struct in the handle */ + dtuple_t* clust_ref,/*!< in: clustered index search tuple */ + btr_pcur_t* clust_pcur,/*!< in/out: clustered index cursor */ + lock_mode select_lock_type,/*!< in: lock mode for selection */ + trx_t* trx, /*!< in: transaction */ + mem_heap_t* old_vers_heap,/*!< in/out: memory heap for old versions */ dict_index_t* sec_index,/*!< in: secondary index where rec resides */ const rec_t* rec, /*!< in: record in a non-clustered index; if this is a locking read, then rec is not @@ -3373,36 +3335,34 @@ Row_sel_get_clust_rec_for_mysql::operator()( { dict_index_t* clust_index; rec_t* old_vers; - trx_t* trx; - prebuilt->clust_pcur->old_rec = nullptr; + clust_pcur->old_rec = nullptr; *out_rec = NULL; - trx = thr_get_trx(thr); - row_build_row_ref_in_tuple(prebuilt->clust_ref, rec, + row_build_row_ref_in_tuple(clust_ref, rec, sec_index, *offsets); clust_index = dict_table_get_first_index(sec_index->table); - prebuilt->clust_pcur->btr_cur.page_cur.index = clust_index; + clust_pcur->btr_cur.page_cur.index = clust_index; - dberr_t err = btr_pcur_open_with_no_init(prebuilt->clust_ref, + dberr_t err = btr_pcur_open_with_no_init(clust_ref, PAGE_CUR_LE, BTR_SEARCH_LEAF, - prebuilt->clust_pcur, mtr); + clust_pcur, mtr); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { return err; } - const rec_t* clust_rec = btr_pcur_get_rec(prebuilt->clust_pcur); + const rec_t* clust_rec = btr_pcur_get_rec(clust_pcur); - prebuilt->clust_pcur->trx_if_known = trx; + clust_pcur->trx_if_known = trx; /* Note: only if the search ends up on a non-infimum record is the low_match value the real match to the search tuple */ if (!page_rec_is_user_rec(clust_rec) - || btr_pcur_get_low_match(prebuilt->clust_pcur) + || btr_pcur_get_low_match(clust_pcur) < dict_index_get_n_unique(clust_index)) { - btr_cur_t* btr_cur = btr_pcur_get_btr_cur(prebuilt->pcur); + btr_cur_t* btr_cur = btr_pcur_get_btr_cur(clust_pcur); /* If this is a spatial index scan, and we are reading from a shadow buffer, the record could be already @@ -3413,7 +3373,7 @@ Row_sel_get_clust_rec_for_mysql::operator()( && (!(ulint(rec - btr_cur->rtr_info->matches->block->page.frame) >> srv_page_size_shift) - || rec != btr_pcur_get_rec(prebuilt->pcur))) { + || rec != btr_pcur_get_rec(clust_pcur))) { #ifdef UNIV_DEBUG rtr_info_t* rtr_info = btr_cur->rtr_info; mysql_mutex_lock(&rtr_info->matches->rtr_match_mutex); @@ -3427,23 +3387,23 @@ Row_sel_get_clust_rec_for_mysql::operator()( if (rec_get_deleted_flag(rec, dict_table_is_comp(sec_index->table)) - && prebuilt->select_lock_type == LOCK_NONE) { + && select_lock_type == LOCK_NONE) { clust_rec = NULL; goto func_exit; } - if (rec != btr_pcur_get_rec(prebuilt->pcur)) { + if (rec != btr_pcur_get_rec(clust_pcur)) { clust_rec = NULL; goto func_exit; } /* FIXME: Why is this block not the - same as btr_pcur_get_block(prebuilt->pcur), + same as btr_pcur_get_block(clust_pcur), and is it not unsafe to use RW_NO_LATCH here? */ buf_block_t* block = buf_page_get_gen( - btr_pcur_get_block(prebuilt->pcur)->page.id(), - btr_pcur_get_block(prebuilt->pcur)->zip_size(), + btr_pcur_get_block(clust_pcur)->page.id(), + btr_pcur_get_block(clust_pcur)->zip_size(), RW_NO_LATCH, NULL, BUF_GET, mtr, &err); ut_ad(block); // FIXME: avoid crash mem_heap_t* heap = mem_heap_create(256); @@ -3465,7 +3425,7 @@ Row_sel_get_clust_rec_for_mysql::operator()( #endif /* UNIV_DEBUG */ } else if (!rec_get_deleted_flag(rec, dict_table_is_comp(sec_index->table)) - || prebuilt->select_lock_type != LOCK_NONE) { + || select_lock_type != LOCK_NONE) { /* In a rare case it is possible that no clust rec is found for a delete-marked secondary index record: if row_undo_mod_clust() has already removed @@ -3494,15 +3454,15 @@ Row_sel_get_clust_rec_for_mysql::operator()( clust_index->n_core_fields, ULINT_UNDEFINED, offset_heap); - if (prebuilt->select_lock_type != LOCK_NONE) { + if (select_lock_type != LOCK_NONE) { /* Try to place a lock on the index record; we are searching the clust rec with a unique condition, hence we set a LOCK_REC_NOT_GAP type lock */ err = lock_clust_rec_read_check_and_lock( - 0, btr_pcur_get_block(prebuilt->clust_pcur), + 0, btr_pcur_get_block(clust_pcur), clust_rec, clust_index, *offsets, - prebuilt->select_lock_type, + select_lock_type, LOCK_REC_NOT_GAP, thr); @@ -3536,7 +3496,7 @@ Row_sel_get_clust_rec_for_mysql::operator()( break; case DB_SUCCESS_LOCKED_REC: const buf_page_t& bpage = btr_pcur_get_block( - prebuilt->clust_pcur)->page; + clust_pcur)->page; const lsn_t lsn = mach_read_from_8( bpage.frame + FIL_PAGE_LSN); @@ -3547,9 +3507,10 @@ Row_sel_get_clust_rec_for_mysql::operator()( /* The following call returns 'offsets' associated with 'old_vers' */ err = row_sel_build_prev_vers_for_mysql( - prebuilt, clust_index, - clust_rec, offsets, offset_heap, &old_vers, - vrow, mtr); + trx, clust_index, + clust_rec, offsets, offset_heap, + old_vers_heap, + &old_vers, vrow, mtr); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { return err; @@ -3622,11 +3583,11 @@ Row_sel_get_clust_rec_for_mysql::operator()( func_exit: *out_rec = clust_rec; - if (prebuilt->select_lock_type != LOCK_NONE) { + if (select_lock_type != LOCK_NONE) { /* We may use the cursor in update or in unlock_row(): store its position */ - btr_pcur_store_position(prebuilt->clust_pcur, mtr); + btr_pcur_store_position(clust_pcur, mtr); } return err; @@ -5392,8 +5353,10 @@ row_search_mvcc( /* The following call returns 'offsets' associated with 'old_vers' */ err = row_sel_build_prev_vers_for_mysql( - prebuilt, clust_index, - rec, &offsets, &heap, &old_vers, + prebuilt->trx, clust_index, + rec, &offsets, &heap, + prebuilt->old_vers_heap, + &old_vers, need_vrow ? &vrow : nullptr, &mtr); if (err != DB_SUCCESS) { @@ -5530,8 +5493,12 @@ row_search_mvcc( 'clust_rec'. Note that 'clust_rec' can be an old version built for a consistent read. */ - err = row_sel_get_clust_rec_for_mysql(prebuilt, index, rec, - thr, &clust_rec, + err = row_sel_get_clust_rec_for_mysql(prebuilt->clust_ref, + prebuilt->clust_pcur, + prebuilt->select_lock_type, + prebuilt->trx, + prebuilt->old_vers_heap, + index, rec, thr, &clust_rec, &offsets, &heap, need_vrow ? &vrow : NULL, &mtr); @@ -6360,8 +6327,9 @@ dberr_t row_check_index(row_prebuilt_t *prebuilt, ulint *n_rows) ut_ad(srv_force_recovery < SRV_FORCE_NO_UNDO_LOG_SCAN); rec_t *old_vers; /* The following call returns 'offsets' associated with 'old_vers' */ - err= row_sel_build_prev_vers_for_mysql(prebuilt, index, rec, &offsets, - &heap, &old_vers, nullptr, &mtr); + err= row_sel_build_prev_vers_for_mysql(prebuilt->trx, index, rec, &offsets, + &heap, prebuilt->old_vers_heap, + &old_vers, nullptr, &mtr); if (err != DB_SUCCESS) goto func_exit; @@ -6600,7 +6568,7 @@ dberr_t row_check_index(row_prebuilt_t *prebuilt, ulint *n_rows) clust_index->n_core_fields, ULINT_UNDEFINED, &heap); - row_sel_reset_old_vers_heap(prebuilt); + row_sel_reset_old_vers_heap(&prebuilt->old_vers_heap); /* The following is adapted from row_vers_build_for_consistent_read() because when using check_table_extended_view, we must consider every available version of the clustered index record. */