[MDEV-31147] json_normalize does not work correctly with MSAN build Created: 2023-04-28  Updated: 2023-11-27  Resolved: 2023-04-28

Status: Closed
Project: MariaDB Server
Component/s: JSON
Affects Version/s: 10.7, 10.8, 10.9, 10.10, 10.11, 11.0, 11.1
Fix Version/s: 11.1.1, 10.11.3, 11.0.2, 10.8.8, 10.9.6, 10.10.4

Type: Bug Priority: Major
Reporter: Rucha Deodhar Assignee: Marko Mäkelä
Resolution: Fixed Votes: 0
Labels: MSAN, overflow


 Description   

11.0 3ef111610b7f8a6a323975cfdf4a4257feb9dcd9

 
CURRENT_TEST: main.func_json
 
mysqltest: At line 2480: query 'INSERT INTO num_table values('15')' failed: <Unknown> (2013): Lost connection to server during query
 
…
 
==44786==WARNING: MemorySanitizer: use-of-uninitialized-value
 
    #0 0x564a01b5fdef in json_normalize_number /mariadb/11/strings/json_normalize.c:155:22
 
    #1 0x564a01b634b8 in json_norm_value_number_init /mariadb/11/strings/json_normalize.c:517:8
 
    #2 0x564a01b634b8 in json_norm_value_init /mariadb/11/strings/json_normalize.c:586:10
 
    #3 0x564a01b605ef in json_norm_build /mariadb/11/strings/json_normalize.c:771:8
 
    #4 0x564a01b605ef in json_normalize /mariadb/11/strings/json_normalize.c:835:8
 
    #5 0x564a000021ab in json_get_normalized_string(st_json_engine_t*, String*, int*) /mariadb/11/sql/json_schema_helper.cc:85:9
 
    #6 0x5649fffe1a78 in Json_schema_const::validate(st_json_engine_t const*, unsigned char const*, unsigned char const*) /mariadb/11/sql/json_schema.cc:447:7
 
    #7 0x5649fffb8bfc in Item_func_json_schema_valid::val_int() /mariadb/11/sql/item_jsonfunc.cc:4757:25
 
    #8 0x5649ffc185b7 in TABLE::verify_constraints(bool) /mariadb/11/sql/table.cc:6447:26
 
    #9 0x5649ffc18053 in TABLE_LIST::view_check_option(THD*, bool) /mariadb/11/sql/table.cc:6420:17
 
    #10 0x5649ff57af73 in mysql_insert(THD*, TABLE_LIST*, List<Item>&, List<List<Item>>&, List<Item>&, List<Item>&, enum_duplicates, bool, select_result*) /mariadb/11/sql/sql_insert.cc:1130:29
 
    #11 0x5649ff6c4e90 in mysql_execute_command(THD*, bool) /mariadb/11/sql/sql_parse.cc:4449:10
 
    #12 0x5649ff69dd01 in mysql_parse(THD*, char*, unsigned int, Parser_state*) /mariadb/11/sql/sql_parse.cc:7760:18
 
    #13 0x5649ff694454 in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool) /mariadb/11/sql/sql_parse.cc:1892:7
 
    #14 0x5649ff69f4ca in do_command(THD*, bool) /mariadb/11/sql/sql_parse.cc:1405:17
 
    #15 0x5649ffd662cc in do_handle_one_connection(CONNECT*, bool) /mariadb/11/sql/sql_connect.cc:1416:11
 
    #16 0x5649ffd658a7 in handle_one_connection /mariadb/11/sql/sql_connect.cc:1318:5
 
    #17 0x564a00625167 in pfs_spawn_thread /mariadb/11/storage/perfschema/pfs.cc:2201:3
 
    #18 0x7f5d1dc10fd3 in start_thread nptl/pthread_create.c:442:8
 
    #19 0x7f5d1dc9081f in clone misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:100
 
 
 
  Memory was marked as uninitialized
 
    #0 0x5649fe71c90d in __msan_allocated_memory (/dev/shm/11/sql/mariadbd+0x10dc90d) (BuildId: 1237e4b564f291c6)
 
    #1 0x564a019118da in my_malloc /mariadb/11/mysys/my_malloc.c:114:7

There are two buffer overflows in the function, on the 2-octet input 15. The following patch fixes it:

diff --git a/strings/json_normalize.c b/strings/json_normalize.c
 
index 0b7f172dae6..2c66c712e81 100644
 
--- a/strings/json_normalize.c
 
+++ b/strings/json_normalize.c
 
@@ -147,13 +147,16 @@ json_normalize_number(DYNAMIC_STRING *out, const char *str, size_t str_len)
 
 
   magnitude = (long)(j - 1);
 
 
-  /* skip the . */
 
-  if (str[i] == '.')
 
-    ++i;
 
+  if (i < str_len)
 
+  {
 
+    /* skip the . */
 
+    if (str[i] == '.')
 
+      ++i;
 
 
-  /* grab rest of digits before the E */
 
-  for (; i < str_len && str[i] != 'e' && str[i] != 'E'; ++i)
 
-    buf[j++] = str[i];
 
+    /* grab rest of digits before the E */
 
+    for (; i < str_len && str[i] != 'e' && str[i] != 'E'; ++i)
 
+      buf[j++] = str[i];
 
+  }
 
 
   /* trim trailing zeros */
 
   for (k = j - 1; k && buf[k] == '0'; --k, --j)
 
@@ -187,7 +190,7 @@ json_normalize_number(DYNAMIC_STRING *out, const char *str, size_t str_len)
 
 
   err|= dynstr_append_mem(out, STRING_WITH_LEN("E"));
 
 
-  if (str[i] == 'e' || str[i] == 'E')
 
+  if (i < str_len && (str[i] == 'e' || str[i] == 'E'))
 
   {
 
     char *endptr = NULL;
 
     /* skip the [eE] */



 Comments   
Comment by Marko Mäkelä [ 2023-04-28 ]

rucha174 provided a test that reproduces this on earlier releases:

diff --git a/mysql-test/main/func_json.test b/mysql-test/main/func_json.test
 
index 9f6c51cbc27..1f61f9abd13 100644
 
--- a/mysql-test/main/func_json.test
 
+++ b/mysql-test/main/func_json.test
 
@@ -1112,3 +1112,16 @@ DROP TABLE t;
 
 --echo #
 
 --echo # End of 10.6 tests
 
 --echo #
 
+
 
+--echo #
 
+--echo # MDEV-31147 json_normalize does not work correctly with MSAN build
 
+--echo #
 
+CREATE TABLE t1 (val JSON);
 
+ALTER TABLE t1 ADD COLUMN normalized_json JSON AS (JSON_NORMALIZE(val));
 
+INSERT INTO t1 (val) VALUES ('15');
 
+SELECT * FROM t1;
 
+DROP TABLE t1;
 
+
 
+--echo #
 
+--echo # End of 10.8 tests
 
+--echo #

10.8 5028b7c7c8beb428d9f36f1da1a1300ec2de9d7b

 
CURRENT_TEST: main.func_json
 
mysqltest: At line 1121: query 'INSERT INTO t1 (val) VALUES ('15')' failed: <Unknown> (2013): Lost connection to server during query
 
…
 
==66762==WARNING: MemorySanitizer: use-of-uninitialized-value
 
    #0 0x55a96268ddaf in json_normalize_number /mariadb/10.8/strings/json_normalize.c:155:22
 
    #1 0x55a962691478 in json_norm_value_number_init /mariadb/10.8/strings/json_normalize.c:517:8
 
    #2 0x55a962691478 in json_norm_value_init /mariadb/10.8/strings/json_normalize.c:586:10
 
    #3 0x55a96268e5af in json_norm_build /mariadb/10.8/strings/json_normalize.c:771:8
 
    #4 0x55a96268e5af in json_normalize /mariadb/10.8/strings/json_normalize.c:835:8
 
    #5 0x55a960ab67a4 in Item_func_json_normalize::val_str(String*) /mariadb/10.8/sql/item_jsonfunc.cc:4183:7
 
    #6 0x55a95f5dc41c in Item::save_str_in_field(Field*, bool) /mariadb/10.8/sql/item.cc:6796:10
 
    #7 0x55a960b7397c in Type_handler_string_result::Item_save_in_field(Item*, Field*, bool) const /mariadb/10.8/sql/sql_type.cc:4332:16
 
    #8 0x55a95f5dd5c4 in Item::save_in_field(Field*, bool) /mariadb/10.8/sql/item.cc:6844:30
 
    #9 0x55a960744268 in TABLE::update_virtual_fields(handler*, enum_vcol_update_mode) /mariadb/10.8/sql/table.cc:8854:24
 
    #10 0x55a95fdf8754 in fill_record(THD*, TABLE*, List<Item>&, List<Item>&, bool, bool) /mariadb/10.8/sql/sql_base.cc:8718:18
 
    #11 0x55a95fdf9516 in fill_record_n_invoke_before_triggers(THD*, TABLE*, List<Item>&, List<Item>&, bool, trg_event_type) /mariadb/10.8/sql/sql_base.cc:8846:11
 
    #12 0x55a96009b823 in mysql_insert(THD*, TABLE_LIST*, List<Item>&, List<List<Item>>&, List<Item>&, List<Item>&, enum_duplicates, bool, select_result*) /mariadb/10.8/sql/sql_insert.cc:1044:13
 
    #13 0x55a9601e53cd in mysql_execute_command(THD*, bool) /mariadb/10.8/sql/sql_parse.cc:4568:10
 
    #14 0x55a9601bd0f1 in mysql_parse(THD*, char*, unsigned int, Parser_state*) /mariadb/10.8/sql/sql_parse.cc:8034:18
 
    #15 0x55a9601b3876 in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool) /mariadb/10.8/sql/sql_parse.cc:1894:7
 
    #16 0x55a9601be809 in do_command(THD*, bool) /mariadb/10.8/sql/sql_parse.cc:1407:17
 
    #17 0x55a96087401c in do_handle_one_connection(CONNECT*, bool) /mariadb/10.8/sql/sql_connect.cc:1416:11
 
    #18 0x55a9608735f7 in handle_one_connection /mariadb/10.8/sql/sql_connect.cc:1318:5
 
    #19 0x55a9610f3f47 in pfs_spawn_thread /mariadb/10.8/storage/perfschema/pfs.cc:2201:3
 
    #20 0x7fb5bbca7fd3 in start_thread nptl/pthread_create.c:442:8
 
    #21 0x7fb5bbd2781f in clone misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:100
 
 
 
  Memory was marked as uninitialized
 
    #0 0x55a95f20cb7d in __msan_allocated_memory (/dev/shm/10.8msan/sql/mariadbd+0x1006b7d) (BuildId: 147dce102f777951)
 
    #1 0x55a96244443a in my_malloc /mariadb/10.8/mysys/my_malloc.c:114:7

The above patch fixes it. The code has not been changed since the time Eric_Herman added JSON_NORMALIZE to MariaDB Server 10.7 (which reached EOL already).

Comment by Eric Herman [ 2023-04-29 ]

Thanks for finding and fixing that!

I notice that the first "if (i < str_len)" is broader than it needs to be, as the for loop which follows also contains that check; thus some indentation could be saved in that case, but it's probably not worth changing.

Generated at Thu Feb 08 10:21:38 UTC 2024 using Jira 8.20.16#820016-sha1:9d11dbea5f4be3d4cc21f03a88dd11d8c8687422.