diff --git a/mysql-test/main/opt_context_replay_basic.result b/mysql-test/main/opt_context_replay_basic.result index f2cca086e4dbc..3ccdc8803791a 100644 --- a/mysql-test/main/opt_context_replay_basic.result +++ b/mysql-test/main/opt_context_replay_basic.result @@ -568,5 +568,106 @@ nv 11 set optimizer_replay_context=''; drop table s1; +# +# MDEV-40383: innodb_gis.point_basic fails on replay +# +CREATE TABLE t1 ( +a INT NOT NULL, +p POINT NOT NULL, +l LINESTRING NOT NULL, +g GEOMETRY NOT NULL, +PRIMARY KEY(p), +SPATIAL KEY `idx2` (p), +SPATIAL KEY `idx3` (l), +SPATIAL KEY `idx4` (g) +); +INSERT INTO t1 VALUES( +1, ST_GeomFromText('POINT(10 10)'), +ST_GeomFromText('LINESTRING(1 1, 5 5, 10 10)'), +ST_GeomFromText('POLYGON((30 30, 40 40, 50 50, 30 50, 30 40, 30 30))')); +INSERT INTO t1 VALUES( +2, ST_GeomFromText('POINT(20 20)'), +ST_GeomFromText('LINESTRING(2 3, 7 8, 9 10, 15 16)'), +ST_GeomFromText('POLYGON((10 30, 30 40, 40 50, 40 30, 30 20, 10 30))')); +set optimizer_record_context=1; +EXPLAIN SELECT a, ST_AsText(p) FROM t1 WHERE a = 2 AND p = ST_GeomFromText('POINT(20 20)'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 const PRIMARY,idx2 PRIMARY 27 const 1 +select context into dumpfile "../../tmp/dump1.sql" +from information_schema.optimizer_context; +set optimizer_record_context=0; +drop table t1; +set optimizer_replay_context='opt_context'; +# Same query as above, must have same explain: +EXPLAIN SELECT a, ST_AsText(p) FROM t1 WHERE a = 2 AND p = ST_GeomFromText('POINT(20 20)'); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 const PRIMARY,idx2 PRIMARY 27 const 1 +set optimizer_replay_context=''; +SELECT a, ST_AsText(p), ST_AsText(l), ST_AsText(g) FROM t1; +a ST_AsText(p) ST_AsText(l) ST_AsText(g) +2 POINT(20 20) LINESTRING(2 3,7 8,9 10,15 16) POLYGON((10 30,30 40,40 50,40 30,30 20,10 30)) +# +# MIN/MAX recording with geometry fields in the table +# +INSERT INTO t1 VALUES( +1, ST_GeomFromText('POINT(10 10)'), +ST_GeomFromText('LINESTRING(1 1, 5 5, 10 10)'), +ST_GeomFromText('POLYGON((30 30, 40 40, 50 50, 30 50, 30 40, 30 30))')); +alter table t1 add index(a); +select a from t1; +a +1 +2 +SELECT MIN(a) FROM t1; +MIN(a) +1 +set optimizer_record_context=1; +EXPLAIN SELECT MIN(a) FROM t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Select tables optimized away +select context into dumpfile "../../tmp/dump1.sql" +from information_schema.optimizer_context; +set optimizer_record_context=0; +drop table t1; +set optimizer_replay_context='opt_context'; +# Same query as above, must have same explain: +EXPLAIN SELECT MIN(a) FROM t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Select tables optimized away +set optimizer_replay_context=''; +SELECT a, ST_AsText(p), ST_AsText(l), ST_AsText(g) FROM t1; +a ST_AsText(p) ST_AsText(l) ST_AsText(g) +1 POINT(10 10) LINESTRING(1 1,5 5,10 10) POLYGON((30 30,40 40,50 50,30 50,30 40,30 30)) +drop table t1; +# +# MIN/MAX recording with a virtual column present. +# +CREATE TABLE t1 ( +a INT NOT NULL, +b INT NOT NULL, +v INT AS (a + 100) VIRTUAL, +KEY(a) +) ENGINE=MyISAM; +INSERT INTO t1 (a,b) VALUES (1,10),(2,20),(3,30),(1,40); +set optimizer_record_context=1; +EXPLAIN SELECT MIN(a) FROM t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Select tables optimized away +select context into dumpfile "../../tmp/dump1.sql" +from information_schema.optimizer_context; +set optimizer_record_context=0; +drop table t1; +set optimizer_replay_context='opt_context'; +# Same query as above, must have same explain: +EXPLAIN SELECT MIN(a) FROM t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Select tables optimized away +set optimizer_replay_context=''; +# MIN(a) row is (1,10); the non-indexed NOT NULL column b must be +# captured (not defaulted to 0), and v must be recomputed as a+100=101: +SELECT a, b, v FROM t1; +a b v +1 10 101 +drop table t1; # End of 13.1 tests drop database db1; diff --git a/mysql-test/main/opt_context_replay_basic.test b/mysql-test/main/opt_context_replay_basic.test index 961609f376e4e..f45f8fd6d517b 100644 --- a/mysql-test/main/opt_context_replay_basic.test +++ b/mysql-test/main/opt_context_replay_basic.test @@ -376,6 +376,119 @@ select lastval(s1) as nv; set optimizer_replay_context=''; --remove_file "$MYSQLTEST_VARDIR/tmp/dump1.sql" drop table s1; + +--echo # +--echo # MDEV-40383: innodb_gis.point_basic fails on replay +--echo # + +CREATE TABLE t1 ( + a INT NOT NULL, + p POINT NOT NULL, + l LINESTRING NOT NULL, + g GEOMETRY NOT NULL, + PRIMARY KEY(p), + SPATIAL KEY `idx2` (p), + SPATIAL KEY `idx3` (l), + SPATIAL KEY `idx4` (g) +); + +INSERT INTO t1 VALUES( +1, ST_GeomFromText('POINT(10 10)'), +ST_GeomFromText('LINESTRING(1 1, 5 5, 10 10)'), +ST_GeomFromText('POLYGON((30 30, 40 40, 50 50, 30 50, 30 40, 30 30))')); + +INSERT INTO t1 VALUES( +2, ST_GeomFromText('POINT(20 20)'), +ST_GeomFromText('LINESTRING(2 3, 7 8, 9 10, 15 16)'), +ST_GeomFromText('POLYGON((10 30, 30 40, 40 50, 40 30, 30 20, 10 30))')); + +set optimizer_record_context=1; +EXPLAIN SELECT a, ST_AsText(p) FROM t1 WHERE a = 2 AND p = ST_GeomFromText('POINT(20 20)'); +select context into dumpfile "../../tmp/dump1.sql" +from information_schema.optimizer_context; +set optimizer_record_context=0; +drop table t1; +--disable_query_log +--disable_result_log +--source "$MYSQLTEST_VARDIR/tmp/dump1.sql" +--enable_query_log +--enable_result_log +set optimizer_replay_context='opt_context'; +--echo # Same query as above, must have same explain: +EXPLAIN SELECT a, ST_AsText(p) FROM t1 WHERE a = 2 AND p = ST_GeomFromText('POINT(20 20)'); + +set optimizer_replay_context=''; +--remove_file "$MYSQLTEST_VARDIR/tmp/dump1.sql" +SELECT a, ST_AsText(p), ST_AsText(l), ST_AsText(g) FROM t1; + +--echo # +--echo # MIN/MAX recording with geometry fields in the table +--echo # +INSERT INTO t1 VALUES( +1, ST_GeomFromText('POINT(10 10)'), +ST_GeomFromText('LINESTRING(1 1, 5 5, 10 10)'), +ST_GeomFromText('POLYGON((30 30, 40 40, 50 50, 30 50, 30 40, 30 30))')); + +alter table t1 add index(a); + +select a from t1; +SELECT MIN(a) FROM t1; + +set optimizer_record_context=1; +EXPLAIN SELECT MIN(a) FROM t1; + +select context into dumpfile "../../tmp/dump1.sql" +from information_schema.optimizer_context; + +set optimizer_record_context=0; +drop table t1; +--disable_query_log +--disable_result_log +--source "$MYSQLTEST_VARDIR/tmp/dump1.sql" +--enable_query_log +--enable_result_log +set optimizer_replay_context='opt_context'; +--echo # Same query as above, must have same explain: +EXPLAIN SELECT MIN(a) FROM t1; + +set optimizer_replay_context=''; +--remove_file "$MYSQLTEST_VARDIR/tmp/dump1.sql" +SELECT a, ST_AsText(p), ST_AsText(l), ST_AsText(g) FROM t1; +drop table t1; + +--echo # +--echo # MIN/MAX recording with a virtual column present. +--echo # +CREATE TABLE t1 ( + a INT NOT NULL, + b INT NOT NULL, + v INT AS (a + 100) VIRTUAL, + KEY(a) +) ENGINE=MyISAM; +INSERT INTO t1 (a,b) VALUES (1,10),(2,20),(3,30),(1,40); + +set optimizer_record_context=1; +EXPLAIN SELECT MIN(a) FROM t1; +select context into dumpfile "../../tmp/dump1.sql" +from information_schema.optimizer_context; +set optimizer_record_context=0; +drop table t1; +--disable_query_log +--disable_result_log +--source "$MYSQLTEST_VARDIR/tmp/dump1.sql" +--enable_query_log +--enable_result_log +set optimizer_replay_context='opt_context'; +--echo # Same query as above, must have same explain: +EXPLAIN SELECT MIN(a) FROM t1; + +set optimizer_replay_context=''; +--remove_file "$MYSQLTEST_VARDIR/tmp/dump1.sql" +--echo # MIN(a) row is (1,10); the non-indexed NOT NULL column b must be +--echo # captured (not defaulted to 0), and v must be recomputed as a+100=101: +SELECT a, b, v FROM t1; +drop table t1; + --echo # End of 13.1 tests drop database db1; diff --git a/sql/filesort.cc b/sql/filesort.cc index 0b8fbfb83feeb..1ed3e38943560 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -3024,6 +3024,7 @@ static uint make_packed_sortkey(Sort_param *param, uchar *to) return length; } + /* @brief Format the row record and store it in the output @@ -3148,12 +3149,26 @@ void format_and_store_row(TABLE *table, const uchar *rec, bool print_names, } field->val_str(&tmp); } - if (require_quote) - output.append('\''); - output.append_for_single_quote_opt_convert(tmp.ptr(), tmp.length(), - field->charset()); - if (require_quote) - output.append('\''); + /* + Emit non-empty values as a hex literal whenever converting field's + charset to the output charset conversion is lossy; otherwise emit the + charset-converted value, quoted only when the type requires it. + */ + if (require_quote && tmp.length() && + !String::is_charset_conversion_lossless(tmp.charset(), + output.charset())) + { + output.append(STRING_WITH_LEN("0x")); + output.append_hex(tmp.ptr(), tmp.length()); + } + else + { + if (require_quote) + output.append('\''); + output.append_for_single_quote_opt_convert(tmp); + if (require_quote) + output.append('\''); + } } } output.append(')'); diff --git a/sql/opt_context_store_replay.cc b/sql/opt_context_store_replay.cc index e69f3dca3c468..86604bc994fb8 100644 --- a/sql/opt_context_store_replay.cc +++ b/sql/opt_context_store_replay.cc @@ -1121,7 +1121,7 @@ void Optimizer_context_recorder::record_records_in_range( String min_key; String max_key; print_key_value(&min_key, key_part, min_range->key, min_range->length); - print_key_value(&max_key, key_part, min_range->key, min_range->length); + print_key_value(&max_key, key_part, max_range->key, max_range->length); if (!(rec_in_range_ctx->min_key= strdup_root(mem_root, &min_key))) return; // OOM @@ -1144,10 +1144,13 @@ void Optimizer_context_recorder::record_table_row(TABLE *tbl, int row_index) StringBuffer<512> output(&my_charset_utf8mb4_bin); /* - The table could have fields that do not have a default value - but are not in the table->read_set. - The record doesn't have values for those. - Use a relaxed sql_mode setting so that REPLACE INTO doesn't fail. + Due to use of prepare_captured_row_read(), we have values for all + table columns. + + However, the row that we're trying to dump might have been inserted into + the table with relaxed settings (no strict mode). + So, use relaxed @@sql_mode setting here also to make the REPLACE statement + is processed (i.e. not fails with an error). */ output.append( STRING_WITH_LEN("SET STATEMENT sql_mode=" @@ -1953,7 +1956,7 @@ bool Optimizer_context_replay::infuse_records_in_range( String max_key; String tbl_name; print_key_value(&min_key, key_part, min_range->key, min_range->length); - print_key_value(&max_key, key_part, min_range->key, min_range->length); + print_key_value(&max_key, key_part, max_range->key, max_range->length); append_base_table_name(tbl, &tbl_name); if (table_context_for_replay *tbl_ctx= @@ -2371,3 +2374,80 @@ void clean_captured_ctx(THD *thd) delete thd->captured_opt_ctx; thd->captured_opt_ctx= nullptr; } + +/* + Point table->read_set at a private bitmap (table->tmp_set) covering every + stored column, so that a subsequent read/record captures the full row. + + Virtual columns are excluded: they cannot be assigned in REPLACE INTO and are + recomputed on read. We copy s->all_set into the per-table tmp_set rather than + aliasing s->all_set directly -- s->all_set is shared across the whole + TABLE_SHARE and must never be mutated. + + Precondition: table->tmp_set must stay free for the caller's use until + read_set is restored. This holds on the const-row and MIN/MAX read paths + precisely because we clear the virtual-column bits: with those bits unset, + TABLE::update_virtual_fields(VCOL_UPDATE_FOR_READ) skips them during the read + and so never reuses tmp_set as its own scratch. A caller on a path that + evaluates virtual columns into tmp_set would corrupt the widened read_set. + + @return the previous read_set, which the caller must restore (via + column_bitmaps_set) once the row has been read and recorded. +*/ +MY_BITMAP *widen_read_set_no_vcols(TABLE *table) +{ + MY_BITMAP *saved_read_set= table->read_set; + /* + We are about to repurpose table->tmp_set as the widened read_set, so it + must not already be in use (e.g. as the current read_set). If this fires, + the caller is on a path that violates the tmp_set precondition documented + above. + */ + DBUG_ASSERT(saved_read_set != &table->tmp_set); + bitmap_copy(&table->tmp_set, &table->s->all_set); + for (Field **pfield= table->field; *pfield; pfield++) + { + /* virtual columns need not be stored. */ + if ((*pfield)->vcol_info) + bitmap_clear_bit(&table->tmp_set, (*pfield)->field_index); + } + table->column_bitmaps_set(&table->tmp_set, table->write_set); + return saved_read_set; +} + + +/* + @brief + Prepare a TABLE to read a row which will be captured into the optimizer + context. + + @detail + The query has set up table->read_set to only include columns of interest. + However, we need to read all non-virtual columns to produce a valid INSERT + statement. This may require disabling 'index-only' read. + + @param table IN Table we're processing + @param state OUT Save the state here. +*/ + +void Optimizer_context_recorder::prepare_captured_row_read(TABLE *table, + Opt_ctx_recorder_state *state) +{ + state->table= table; + state->keyread_state= table->file->ha_end_active_keyread(); + state->saved_read_set= widen_read_set_no_vcols(table); +} + + +/* + @brief + Restore the table state previously saved by prepare_captured_row_read. +*/ + +void Optimizer_context_recorder::finish_captured_row_read(Opt_ctx_recorder_state *state) +{ + if (state->saved_read_set) + state->table->column_bitmaps_set(state->saved_read_set, state->table->write_set); + + state->table->file->ha_restart_keyread(state->keyread_state); +} diff --git a/sql/opt_context_store_replay.h b/sql/opt_context_store_replay.h index 55b587dc2a166..c8d136cf09293 100644 --- a/sql/opt_context_store_replay.h +++ b/sql/opt_context_store_replay.h @@ -39,6 +39,15 @@ class Multi_range_read_const_call_record; void init_optimizer_context_recorder_if_needed(THD *thd, const TABLE_LIST *query_tables); +class Opt_ctx_recorder_state +{ +public: + TABLE *table= nullptr; + + int keyread_state= 0; + MY_BITMAP *saved_read_set= nullptr; +}; + /* Recorder is used to capture the environment during query optimization run. When the optimization is finished, one can save the captured context @@ -51,6 +60,9 @@ class Optimizer_context_recorder ~Optimizer_context_recorder(); + void prepare_captured_row_read(TABLE *table, Opt_ctx_recorder_state *state); + void finish_captured_row_read(Opt_ctx_recorder_state *state); + void record_multi_range_read_info_const(const TABLE *table, uint keynr, Range_print_enumerator *ranges, ha_rows rows, @@ -66,11 +78,6 @@ class Optimizer_context_recorder const KEY_PART_INFO *key_part, uint keynr, const key_range *min_range, const key_range *max_range, ha_rows records); - void record_const_table_row(TABLE *tbl) - { - /* use table->record[1] */ - record_table_row(tbl, 1); - } void record_current_table_row(TABLE *tbl) { /* use table->record[0] */ @@ -140,6 +147,13 @@ int fill_optimizer_context_capture_info(THD *thd, TABLE_LIST *tables, Item *); void clean_captured_ctx(THD *thd); +/* + Widen read_set to all stored (non-virtual) columns via table->tmp_set, so the + next read/record captures the full row; returns the previous read_set for the + caller to restore. See the definition for the tmp_set precondition. +*/ +MY_BITMAP *widen_read_set_no_vcols(TABLE *table); + /*************************************************************************** * Part 3: APIs for loading previously saved Optimizer Context and replaying * it: making the optimizer work as if the environment was like it has been diff --git a/sql/opt_sum.cc b/sql/opt_sum.cc index eca59bbc68ec3..5a35124da867c 100644 --- a/sql/opt_sum.cc +++ b/sql/opt_sum.cc @@ -428,6 +428,12 @@ int opt_sum_query(THD *thd, error= 0; table->file->info_push(INFO_KIND_FORCE_LIMIT_BEGIN, &info_limit); + + /* Prepare to capture the MIN/MAX row for the optimizer context */ + Opt_ctx_recorder_state state; + if (thd->opt_ctx_recorder) + thd->opt_ctx_recorder->prepare_captured_row_read(table, &state); + if (!table->const_table) { if (likely(!(error= table->file->ha_index_init((uint) ref.key, @@ -443,10 +449,12 @@ int opt_sum_query(THD *thd, conds, range_fl, prefix_len)) error= HA_ERR_KEY_NOT_FOUND; - if (!error) + /* Capture the MIN/MAX row for the optimizer context */ + if (Optimizer_context_recorder *rec= thd->opt_ctx_recorder) { - if (Optimizer_context_recorder *rec= thd->opt_ctx_recorder) + if (!error) rec->record_current_table_row(table); + rec->finish_captured_row_read(&state); } if (!table->const_table) { diff --git a/sql/sql_select.cc b/sql/sql_select.cc index a82fb09ab0832..b3993d1bba14a 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -46,6 +46,7 @@ #include "filesort_utils.h" // get_qsort_sort_cost #include "sql_union.h" // mysql_union #include "opt_subselect.h" +#include "opt_context_store_replay.h" #include "sql_derived.h" #include "sql_statistics.h" #include "sql_cte.h" @@ -25066,7 +25067,6 @@ join_read_const_table(THD *thd, JOIN_TAB *tab, POSITION *pos) DBUG_RETURN(0); } - /** Read a constant table when there is at most one matching row, using a table scan. @@ -25084,9 +25084,23 @@ join_read_system(JOIN_TAB *tab) int error; if (table->status & STATUS_GARBAGE) // If first read { - if (unlikely((error= - table->file->ha_read_first_row(table->record[0], - table->s->primary_key)))) + /* Prepare to capture the constant row for the Optimizer Context */ + Opt_ctx_recorder_state state; + if (Optimizer_context_recorder *rec= tab->join->thd->opt_ctx_recorder) + rec->prepare_captured_row_read(table, &state); + + error= table->file->ha_read_first_row(table->record[0], + table->s->primary_key); + + /* Capture the constant row for the Optimizer Context. */ + if (Optimizer_context_recorder *rec= tab->join->thd->opt_ctx_recorder) + { + if (!error) + rec->record_current_table_row(table); + rec->finish_captured_row_read(&state); + } + + if (unlikely(error)) { if (error != HA_ERR_END_OF_FILE) return report_error(table, error); @@ -25095,9 +25109,8 @@ join_read_system(JOIN_TAB *tab) empty_record(table); // Make empty record return -1; } - store_record(table,record[1]); - if (Optimizer_context_recorder *rec= tab->join->thd->opt_ctx_recorder) - rec->record_const_table_row(table); + + store_record(table, record[1]); // cache the row the optimizer used } else if (!table->status) // Only happens with left join restore_record(table,record[1]); // restore old record @@ -25136,11 +25149,25 @@ join_read_const(JOIN_TAB *tab) /* This is probably needed for analyze table */ tab->index= tab->ref.key; } + + /* Prepare to capture the constant row for the Optimizer Context */ + Opt_ctx_recorder_state state; + if (Optimizer_context_recorder *rec= tab->join->thd->opt_ctx_recorder) + rec->prepare_captured_row_read(table, &state); + error= file-> ha_index_read_idx_map(table->record[0],tab->ref.key, (uchar*) tab->ref.key_buff, make_prev_keypart_map(tab->ref.key_parts), HA_READ_KEY_EXACT); + + /* Capture the constant row for the Optimizer Context */ + if (Optimizer_context_recorder *rec= tab->join->thd->opt_ctx_recorder) + { + if (!error) + rec->record_current_table_row(table); + rec->finish_captured_row_read(&state); + } file->ha_end_keyread(); } if (unlikely(error)) @@ -25152,10 +25179,7 @@ join_read_const(JOIN_TAB *tab) return report_error(table, error); return -1; } - store_record(table,record[1]); - - if (Optimizer_context_recorder *rec= tab->join->thd->opt_ctx_recorder) - rec->record_const_table_row(table); + store_record(table, record[1]); // cache the row the optimizer used } else if (!(table->status & ~STATUS_NULL_ROW)) // Only happens with left join { diff --git a/sql/sql_string.cc b/sql/sql_string.cc index 1c465f3ea219a..b6bdb9365bb0c 100644 --- a/sql/sql_string.cc +++ b/sql/sql_string.cc @@ -372,6 +372,26 @@ bool String::needs_conversion_on_storage(size_t arg_length, } +bool String::is_charset_conversion_lossless(const CHARSET_INFO *from_cs, + const CHARSET_INFO *to_cs) +{ + if (to_cs == &my_charset_bin) + return true; // binary swallows any bytes + if (from_cs == &my_charset_bin) + return false; // arbitrary bytes ⊄ text charset + if (my_charset_same(from_cs, to_cs)) + return true; // same repertoire family + if (from_cs->state & MY_CS_PUREASCII) // ASCII-only source... + return my_charset_is_ascii_based(to_cs); // ...into any ASCII-based target + if ((to_cs->state & + MY_CS_UNICODE) && // Unicode target covers every repertoire, + (to_cs->state & + MY_CS_UNICODE_SUPPLEMENT)) // incl. non-BMP (so utf8mb4/utf16/utf32, + return true; // but NOT plain utf8mb3/ucs2) + return false; // unknown → treat as possibly lossy +} + + /* Copy a multi-byte character sets with adding leading zeros. diff --git a/sql/sql_string.h b/sql/sql_string.h index d6a0257ad9bb6..ae88a28dd9854 100644 --- a/sql/sql_string.h +++ b/sql/sql_string.h @@ -945,6 +945,8 @@ class String: public Charset, public Binary_string static bool needs_conversion_on_storage(size_t arg_length, CHARSET_INFO *cs_from, CHARSET_INFO *cs_to); + static bool is_charset_conversion_lossless(const CHARSET_INFO *from_cs, + const CHARSET_INFO *to_cs); bool copy_aligned(const char *s, size_t arg_length, size_t offset, CHARSET_INFO *cs); bool set_or_copy_aligned(const char *s, size_t arg_length, CHARSET_INFO *cs);